@openpond/evals 0.11.11 → 0.11.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/CONTRACT.md +20 -0
  2. package/dist/experiments.js +89 -15
  3. package/dist/types/experiments.d.ts +225 -9
  4. package/dist/types/experiments.d.ts.map +1 -1
  5. package/dist/types/learning/repository.d.ts +1 -1
  6. package/dist/types/learning/transport.d.ts +2 -2
  7. package/package.json +1 -1
package/CONTRACT.md CHANGED
@@ -162,3 +162,23 @@ Node subprocess for hosts without Node worker APIs. It strips authored TypeScrip
162
162
  inside its deadline, evaluates the verified source in the same deterministic
163
163
  isolate, and exits before success, failure or cancellation settles. Importing
164
164
  the metrics entrypoint does not require Node's type-stripping API on the host.
165
+
166
+ ## Portable Experiment comparison
167
+
168
+ `verifyExperimentEvidence` verifies the manifest hash, result hash, complete
169
+ case population and exact evaluator bindings. Historical runs without a whole
170
+ execution ceiling report `maximumCostUsd: null`. Dataset-authored graders retain
171
+ their actual version string and implementation hash; reusable Rewards retain
172
+ their numeric release revision. Authored fixture checks use a fixture target.
173
+
174
+ Hosts can pin package, runtime and metric-policy hashes in `execution`. Shared
175
+ comparison requires equal dataset version, population, evaluators and execution
176
+ contract. Harness comparison also requires the same component, evaluation
177
+ definition and environment; candidate source/Harness revisions can differ.
178
+
179
+ `compareExperiments` aligns cases and reports per-feedback eligibility, exclusion
180
+ reasons and paired means/deltas. Only matching completed cases with scored
181
+ numeric or Boolean feedback contribute. Failed, cancelled, absent or ungraded
182
+ cases never become zero-valued measurements. Categorical feedback remains
183
+ inspectable without a numeric delta. This operation reads retained evidence and
184
+ has no execution, grading or billing authority.
package/dist/experiments.js CHANGED
@@ -13,10 +13,15 @@ export const ExperimentTargetSchema = z.discriminatedUnion("kind", [
13
13
  kind: z.literal("harness"),
14
14
  source: ProfileEvaluationRunSourceSchema,
15
15
  }).strict(),
16
+ z.object({ kind: z.literal("fixture"), configurationHash: ReleaseHashSchema }).strict(),
16
17
  ]);
17
18
  export const ExperimentEvaluatorSchema = z.object({
18
- release: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict(),
19
+ // A reusable Reward has a numeric revision; a dataset-authored GraderSpec
20
+ // has its exact version string and implementation hash. Neither is invented.
21
+ release: ImmutableReleaseRefSchema.extend({ revision: z.union([z.number().int().positive(), z.string().trim().min(1).max(200)]) }).strict(),
19
22
  feedbackKey: FeedbackKeySchema,
23
+ /** Binds field mappings and admitted scorer settings without changing the Reward's release hash. */
24
+ configurationHash: ReleaseHashSchema.optional(),
20
25
  output: z.enum(["boolean", "score", "category"]),
21
26
  /** Category labels are meaningful only for category output. */
22
27
  categories: z.array(z.string().trim().min(1).max(120)).max(50),
@@ -41,13 +46,22 @@ export const ExperimentManifestContentSchema = z.object({
41
46
  name: z.string().trim().min(1).max(500),
42
47
  teamId: ReleaseIdSchema,
43
48
  operationId: ReleaseIdSchema,
44
- maximumCostUsd: MoneySchema,
49
+ /** Historical imported runs may have no admitted whole-run ceiling. */
50
+ maximumCostUsd: MoneySchema.nullable(),
45
51
  dataset: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict(),
46
52
  target: ExperimentTargetSchema,
53
+ execution: z.object({ packageHash: ReleaseHashSchema, runtimeTargetHash: ReleaseHashSchema, metricPolicyHash: ReleaseHashSchema }).strict().optional(),
54
+ lineage: z.object({
55
+ definition: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict().nullable(),
56
+ execution: ImmutableReleaseRefSchema,
57
+ scoringPassId: ReleaseIdSchema.nullable(),
58
+ }).strict().optional(),
47
59
  evaluators: z.array(ExperimentEvaluatorSchema).max(100),
48
60
  population: z.array(ExperimentCaseIdentitySchema).min(1).max(10_000),
49
61
  createdAt: ReleaseTimestampSchema,
50
62
  }).strict().superRefine((value, context) => {
63
+ if (value.lineage && value.id !== (value.lineage.scoringPassId ?? value.lineage.execution.id))
64
+ context.addIssue({ code: "custom", path: ["lineage"], message: "Experiment identity must match its execution or independent scoring pass." });
51
65
  if (new Set(value.population.map(experimentCaseKey)).size !== value.population.length)
52
66
  context.addIssue({ code: "custom", path: ["population"], message: "Experiment cases must be unique." });
53
67
  if (new Set(value.evaluators.map((item) => item.feedbackKey)).size !== value.evaluators.length)
@@ -59,6 +73,7 @@ export const ExperimentFeedbackSchema = z.object({
59
73
  evaluator: ExperimentEvaluatorSchema.shape.release,
60
74
  status: z.enum(["scored", "unavailable", "failed", "pending"]),
61
75
  value: z.union([z.boolean(), z.number().finite(), z.string().max(120)]).nullable(),
76
+ passed: z.boolean().nullable().optional(),
62
77
  reasoning: z.string().max(20_000).nullable(),
63
78
  evidenceRefs: z.array(ImmutableArtifactRefSchema).max(1_000),
64
79
  }).strict();
@@ -125,21 +140,27 @@ export function createExperimentResult(input, manifest) {
125
140
  || (evaluator.output === "score" && (feedback.value < 0 || feedback.value > 1)))
126
141
  throw new Error("Experiment feedback value violates its output contract.");
127
142
  }
128
- else if (feedback.value !== null)
129
- throw new Error("Unscored experiment feedback cannot claim a value.");
143
+ else if (feedback.value !== null || feedback.passed !== null && feedback.passed !== undefined)
144
+ throw new Error("Unscored experiment feedback cannot claim a value or pass decision.");
130
145
  }
131
146
  }
132
147
  if (seen.size !== expected.size)
133
148
  throw new Error("Experiment result omits admitted cases.");
134
149
  return ExperimentResultSchema.parse({ ...value, contentHash: contentHash(value) });
135
150
  }
151
+ export function verifyExperimentEvidence(input) {
152
+ const manifest = ExperimentManifestSchema.parse(input.manifest);
153
+ const result = ExperimentResultSchema.parse(input.result);
154
+ const { contentHash: hash, ...content } = result;
155
+ if (contentHash(content) !== hash)
156
+ throw new Error("Experiment result integrity failed.");
157
+ createExperimentResult(content, manifest);
158
+ return { manifest, result };
159
+ }
136
160
  /** A comparison reads retained results; it never executes or regrades a case. */
137
161
  export function compareExperiments(baseline, candidate) {
138
162
  for (const pair of [baseline, candidate]) {
139
- const { contentHash: hash, ...value } = ExperimentResultSchema.parse(pair.result);
140
- if (contentHash(value) !== hash)
141
- throw new Error("Experiment result integrity failed.");
142
- createExperimentResult(value, pair.manifest);
163
+ verifyExperimentEvidence(pair);
143
164
  }
144
165
  const reasons = [];
145
166
  if (baseline.manifest.teamId !== candidate.manifest.teamId)
@@ -150,20 +171,73 @@ export function compareExperiments(baseline, candidate) {
150
171
  reasons.push("different_dataset_version");
151
172
  if (baseline.manifest.target.kind !== candidate.manifest.target.kind)
152
173
  reasons.push("different_target_kind");
153
- if (contentHash(baseline.manifest.evaluators) !== contentHash(candidate.manifest.evaluators))
174
+ if (contentHash(baseline.manifest.execution ?? null) !== contentHash(candidate.manifest.execution ?? null))
175
+ reasons.push("different_execution_contract");
176
+ if (baseline.manifest.target.kind === "harness" && candidate.manifest.target.kind === "harness") {
177
+ const a = baseline.manifest.target.source;
178
+ const b = candidate.manifest.target.source;
179
+ if (a.profileId !== b.profileId || contentHash(a.target) !== contentHash(b.target))
180
+ reasons.push("different_component");
181
+ if (a.environmentHash !== b.environmentHash)
182
+ reasons.push("different_environment");
183
+ if (a.definitionId !== b.definitionId || a.definitionHash !== b.definitionHash)
184
+ reasons.push("different_evaluation_definition");
185
+ }
186
+ const orderedEvaluators = (manifest) => [...manifest.evaluators].sort((a, b) => a.feedbackKey.localeCompare(b.feedbackKey));
187
+ if (contentHash(orderedEvaluators(baseline.manifest)) !== contentHash(orderedEvaluators(candidate.manifest)))
154
188
  reasons.push("different_evaluators");
155
189
  const left = new Map(baseline.result.cases.map((item) => [experimentCaseKey(item.identity), item]));
156
190
  const right = new Map(candidate.result.cases.map((item) => [experimentCaseKey(item.identity), item]));
157
191
  const keys = [...new Set([...left.keys(), ...right.keys()])].sort();
158
192
  if (left.size !== right.size || keys.some((key) => !left.has(key) || !right.has(key)))
159
193
  reasons.push("different_population");
194
+ const comparable = reasons.length === 0;
195
+ const cases = keys.map((key) => {
196
+ const baselineCase = left.get(key) ?? null;
197
+ const candidateCase = right.get(key) ?? null;
198
+ const baselineFeedback = new Map(baselineCase?.feedback.map(item => [item.feedbackKey, item]) ?? []);
199
+ const candidateFeedback = new Map(candidateCase?.feedback.map(item => [item.feedbackKey, item]) ?? []);
200
+ return {
201
+ identity: (baselineCase ?? candidateCase).identity,
202
+ baseline: baselineCase,
203
+ candidate: candidateCase,
204
+ feedback: baseline.manifest.evaluators.map(evaluator => {
205
+ const a = baselineFeedback.get(evaluator.feedbackKey);
206
+ const b = candidateFeedback.get(evaluator.feedbackKey);
207
+ const reason = !comparable ? "incompatible_experiments"
208
+ : !baselineCase || !candidateCase ? "missing_case"
209
+ : baselineCase.status !== "completed" || candidateCase.status !== "completed" || baselineCase.error || candidateCase.error ? "case_not_completed"
210
+ : !a || !b || a.status !== "scored" || b.status !== "scored" ? "feedback_unavailable"
211
+ : evaluator.output === "category" ? "categorical_feedback" : null;
212
+ const numeric = (item) => typeof item?.value === "boolean" ? Number(item.value)
213
+ : typeof item?.value === "number" ? item.value : null;
214
+ const av = reason ? null : numeric(a);
215
+ const bv = reason ? null : numeric(b);
216
+ const pass = (item) => reason ? null : typeof item?.value === "boolean" ? item.value : item?.passed ?? null;
217
+ return { feedbackKey: evaluator.feedbackKey, eligible: reason === null, reason,
218
+ baseline: av, candidate: bv, delta: av === null || bv === null ? null : bv - av,
219
+ baselinePassed: pass(a), candidatePassed: pass(b) };
220
+ }),
221
+ };
222
+ });
223
+ const metrics = baseline.manifest.evaluators.map((evaluator, index) => {
224
+ const eligible = cases.map(item => item.feedback[index]).filter(value => value.eligible);
225
+ const mean = (field) => eligible.length
226
+ ? eligible.reduce((sum, value) => sum + value[field], 0) / eligible.length : null;
227
+ const a = mean("baseline");
228
+ const b = mean("candidate");
229
+ const passRate = (field) => eligible.length && eligible.every(value => value[field] !== null)
230
+ ? eligible.reduce((sum, value) => sum + Number(value[field]), 0) / eligible.length : null;
231
+ const ap = passRate("baselinePassed");
232
+ const bp = passRate("candidatePassed");
233
+ return { feedbackKey: evaluator.feedbackKey, eligibleCount: eligible.length, excludedCount: cases.length - eligible.length,
234
+ baseline: a, candidate: b, delta: a === null || b === null ? null : b - a,
235
+ baselinePassRate: ap, candidatePassRate: bp, passRateDelta: ap === null || bp === null ? null : bp - ap };
236
+ });
160
237
  return {
161
- comparable: reasons.length === 0,
238
+ comparable,
162
239
  reasons,
163
- cases: keys.map((key) => ({
164
- identity: (left.get(key) ?? right.get(key)).identity,
165
- baseline: left.get(key) ?? null,
166
- candidate: right.get(key) ?? null,
167
- })),
240
+ cases,
241
+ metrics,
168
242
  };
169
243
  }
package/dist/types/experiments.d.ts CHANGED
@@ -29,14 +29,18 @@ export declare const ExperimentTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObjec
29
29
  }, z.core.$strict>], "kind">;
30
30
  environmentHash: z.ZodString;
31
31
  }, z.core.$strict>;
32
+ }, z.core.$strict>, z.ZodObject<{
33
+ kind: z.ZodLiteral<"fixture">;
34
+ configurationHash: z.ZodString;
32
35
  }, z.core.$strict>], "kind">;
33
36
  export declare const ExperimentEvaluatorSchema: z.ZodObject<{
34
37
  release: z.ZodObject<{
35
38
  id: z.ZodString;
36
39
  contentHash: z.ZodString;
37
- revision: z.ZodNumber;
40
+ revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
38
41
  }, z.core.$strict>;
39
42
  feedbackKey: z.ZodString;
43
+ configurationHash: z.ZodOptional<z.ZodString>;
40
44
  output: z.ZodEnum<{
41
45
  boolean: "boolean";
42
46
  score: "score";
@@ -57,7 +61,7 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
57
61
  name: z.ZodString;
58
62
  teamId: z.ZodString;
59
63
  operationId: z.ZodString;
60
- maximumCostUsd: z.ZodNumber;
64
+ maximumCostUsd: z.ZodNullable<z.ZodNumber>;
61
65
  dataset: z.ZodObject<{
62
66
  id: z.ZodString;
63
67
  contentHash: z.ZodString;
@@ -93,14 +97,35 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
93
97
  }, z.core.$strict>], "kind">;
94
98
  environmentHash: z.ZodString;
95
99
  }, z.core.$strict>;
100
+ }, z.core.$strict>, z.ZodObject<{
101
+ kind: z.ZodLiteral<"fixture">;
102
+ configurationHash: z.ZodString;
96
103
  }, z.core.$strict>], "kind">;
104
+ execution: z.ZodOptional<z.ZodObject<{
105
+ packageHash: z.ZodString;
106
+ runtimeTargetHash: z.ZodString;
107
+ metricPolicyHash: z.ZodString;
108
+ }, z.core.$strict>>;
109
+ lineage: z.ZodOptional<z.ZodObject<{
110
+ definition: z.ZodNullable<z.ZodObject<{
111
+ id: z.ZodString;
112
+ contentHash: z.ZodString;
113
+ revision: z.ZodNumber;
114
+ }, z.core.$strict>>;
115
+ execution: z.ZodObject<{
116
+ id: z.ZodString;
117
+ contentHash: z.ZodString;
118
+ }, z.core.$strict>;
119
+ scoringPassId: z.ZodNullable<z.ZodString>;
120
+ }, z.core.$strict>>;
97
121
  evaluators: z.ZodArray<z.ZodObject<{
98
122
  release: z.ZodObject<{
99
123
  id: z.ZodString;
100
124
  contentHash: z.ZodString;
101
- revision: z.ZodNumber;
125
+ revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
102
126
  }, z.core.$strict>;
103
127
  feedbackKey: z.ZodString;
128
+ configurationHash: z.ZodOptional<z.ZodString>;
104
129
  output: z.ZodEnum<{
105
130
  boolean: "boolean";
106
131
  score: "score";
@@ -121,7 +146,7 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
121
146
  name: z.ZodString;
122
147
  teamId: z.ZodString;
123
148
  operationId: z.ZodString;
124
- maximumCostUsd: z.ZodNumber;
149
+ maximumCostUsd: z.ZodNullable<z.ZodNumber>;
125
150
  dataset: z.ZodObject<{
126
151
  id: z.ZodString;
127
152
  contentHash: z.ZodString;
@@ -157,14 +182,35 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
157
182
  }, z.core.$strict>], "kind">;
158
183
  environmentHash: z.ZodString;
159
184
  }, z.core.$strict>;
185
+ }, z.core.$strict>, z.ZodObject<{
186
+ kind: z.ZodLiteral<"fixture">;
187
+ configurationHash: z.ZodString;
160
188
  }, z.core.$strict>], "kind">;
189
+ execution: z.ZodOptional<z.ZodObject<{
190
+ packageHash: z.ZodString;
191
+ runtimeTargetHash: z.ZodString;
192
+ metricPolicyHash: z.ZodString;
193
+ }, z.core.$strict>>;
194
+ lineage: z.ZodOptional<z.ZodObject<{
195
+ definition: z.ZodNullable<z.ZodObject<{
196
+ id: z.ZodString;
197
+ contentHash: z.ZodString;
198
+ revision: z.ZodNumber;
199
+ }, z.core.$strict>>;
200
+ execution: z.ZodObject<{
201
+ id: z.ZodString;
202
+ contentHash: z.ZodString;
203
+ }, z.core.$strict>;
204
+ scoringPassId: z.ZodNullable<z.ZodString>;
205
+ }, z.core.$strict>>;
161
206
  evaluators: z.ZodArray<z.ZodObject<{
162
207
  release: z.ZodObject<{
163
208
  id: z.ZodString;
164
209
  contentHash: z.ZodString;
165
- revision: z.ZodNumber;
210
+ revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
166
211
  }, z.core.$strict>;
167
212
  feedbackKey: z.ZodString;
213
+ configurationHash: z.ZodOptional<z.ZodString>;
168
214
  output: z.ZodEnum<{
169
215
  boolean: "boolean";
170
216
  score: "score";
@@ -186,7 +232,7 @@ export declare const ExperimentFeedbackSchema: z.ZodObject<{
186
232
  evaluator: z.ZodObject<{
187
233
  id: z.ZodString;
188
234
  contentHash: z.ZodString;
189
- revision: z.ZodNumber;
235
+ revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
190
236
  }, z.core.$strict>;
191
237
  status: z.ZodEnum<{
192
238
  pending: "pending";
@@ -195,6 +241,7 @@ export declare const ExperimentFeedbackSchema: z.ZodObject<{
195
241
  unavailable: "unavailable";
196
242
  }>;
197
243
  value: z.ZodNullable<z.ZodUnion<readonly [z.ZodBoolean, z.ZodNumber, z.ZodString]>>;
244
+ passed: z.ZodOptional<z.ZodNullable<z.ZodBoolean>>;
198
245
  reasoning: z.ZodNullable<z.ZodString>;
199
246
  evidenceRefs: z.ZodArray<z.ZodObject<{
200
247
  id: z.ZodString;
@@ -232,7 +279,7 @@ export declare const ExperimentCaseResultSchema: z.ZodObject<{
232
279
  evaluator: z.ZodObject<{
233
280
  id: z.ZodString;
234
281
  contentHash: z.ZodString;
235
- revision: z.ZodNumber;
282
+ revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
236
283
  }, z.core.$strict>;
237
284
  status: z.ZodEnum<{
238
285
  pending: "pending";
@@ -241,6 +288,7 @@ export declare const ExperimentCaseResultSchema: z.ZodObject<{
241
288
  unavailable: "unavailable";
242
289
  }>;
243
290
  value: z.ZodNullable<z.ZodUnion<readonly [z.ZodBoolean, z.ZodNumber, z.ZodString]>>;
291
+ passed: z.ZodOptional<z.ZodNullable<z.ZodBoolean>>;
244
292
  reasoning: z.ZodNullable<z.ZodString>;
245
293
  evidenceRefs: z.ZodArray<z.ZodObject<{
246
294
  id: z.ZodString;
@@ -298,7 +346,7 @@ export declare const ExperimentResultContentSchema: z.ZodObject<{
298
346
  evaluator: z.ZodObject<{
299
347
  id: z.ZodString;
300
348
  contentHash: z.ZodString;
301
- revision: z.ZodNumber;
349
+ revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
302
350
  }, z.core.$strict>;
303
351
  status: z.ZodEnum<{
304
352
  pending: "pending";
@@ -307,6 +355,7 @@ export declare const ExperimentResultContentSchema: z.ZodObject<{
307
355
  unavailable: "unavailable";
308
356
  }>;
309
357
  value: z.ZodNullable<z.ZodUnion<readonly [z.ZodBoolean, z.ZodNumber, z.ZodString]>>;
358
+ passed: z.ZodOptional<z.ZodNullable<z.ZodBoolean>>;
310
359
  reasoning: z.ZodNullable<z.ZodString>;
311
360
  evidenceRefs: z.ZodArray<z.ZodObject<{
312
361
  id: z.ZodString;
@@ -366,7 +415,7 @@ export declare const ExperimentResultSchema: z.ZodObject<{
366
415
  evaluator: z.ZodObject<{
367
416
  id: z.ZodString;
368
417
  contentHash: z.ZodString;
369
- revision: z.ZodNumber;
418
+ revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
370
419
  }, z.core.$strict>;
371
420
  status: z.ZodEnum<{
372
421
  pending: "pending";
@@ -375,6 +424,7 @@ export declare const ExperimentResultSchema: z.ZodObject<{
375
424
  unavailable: "unavailable";
376
425
  }>;
377
426
  value: z.ZodNullable<z.ZodUnion<readonly [z.ZodBoolean, z.ZodNumber, z.ZodString]>>;
427
+ passed: z.ZodOptional<z.ZodNullable<z.ZodBoolean>>;
378
428
  reasoning: z.ZodNullable<z.ZodString>;
379
429
  evidenceRefs: z.ZodArray<z.ZodObject<{
380
430
  id: z.ZodString;
@@ -412,9 +462,175 @@ export type ExperimentComparison = {
412
462
  identity: ExperimentCaseIdentity;
413
463
  baseline: ExperimentCaseResult | null;
414
464
  candidate: ExperimentCaseResult | null;
465
+ feedback: ExperimentFeedbackComparison[];
415
466
  }>;
467
+ metrics: ExperimentComparisonMetric[];
416
468
  };
417
469
  export type ExperimentCaseResult = z.infer<typeof ExperimentCaseResultSchema>;
470
+ export type ExperimentFeedbackComparison = {
471
+ feedbackKey: string;
472
+ eligible: boolean;
473
+ reason: "incompatible_experiments" | "missing_case" | "case_not_completed" | "feedback_unavailable" | "categorical_feedback" | null;
474
+ baseline: number | null;
475
+ candidate: number | null;
476
+ delta: number | null;
477
+ baselinePassed: boolean | null;
478
+ candidatePassed: boolean | null;
479
+ };
480
+ export type ExperimentComparisonMetric = {
481
+ feedbackKey: string;
482
+ eligibleCount: number;
483
+ excludedCount: number;
484
+ baseline: number | null;
485
+ candidate: number | null;
486
+ delta: number | null;
487
+ baselinePassRate: number | null;
488
+ candidatePassRate: number | null;
489
+ passRateDelta: number | null;
490
+ };
491
+ export declare function verifyExperimentEvidence(input: {
492
+ manifest: unknown;
493
+ result: unknown;
494
+ }): {
495
+ manifest: {
496
+ schemaVersion: "openpond.experimentManifest.v1";
497
+ id: string;
498
+ name: string;
499
+ teamId: string;
500
+ operationId: string;
501
+ maximumCostUsd: number | null;
502
+ dataset: {
503
+ id: string;
504
+ contentHash: string;
505
+ revision: number;
506
+ };
507
+ target: {
508
+ kind: "model";
509
+ modelId: string;
510
+ configurationHash: string;
511
+ } | {
512
+ kind: "harness";
513
+ source: {
514
+ profileId: string;
515
+ sourceRevision: string;
516
+ harnessRelease: {
517
+ id: string;
518
+ contentHash: string;
519
+ };
520
+ catalogHash: string;
521
+ definitionId: string;
522
+ definitionHash: string;
523
+ target: {
524
+ kind: "profile";
525
+ } | {
526
+ kind: "workflow";
527
+ workflowId: string;
528
+ } | {
529
+ kind: "skill";
530
+ skillPath: string;
531
+ } | {
532
+ kind: "agent_action";
533
+ actionId: string;
534
+ };
535
+ environmentHash: string;
536
+ };
537
+ } | {
538
+ kind: "fixture";
539
+ configurationHash: string;
540
+ };
541
+ evaluators: {
542
+ release: {
543
+ id: string;
544
+ contentHash: string;
545
+ revision: string | number;
546
+ };
547
+ feedbackKey: string;
548
+ output: "boolean" | "score" | "category";
549
+ categories: string[];
550
+ configurationHash?: string | undefined;
551
+ }[];
552
+ population: {
553
+ caseId: string;
554
+ seed: string;
555
+ fixtureId: string | null;
556
+ }[];
557
+ createdAt: string;
558
+ contentHash: string;
559
+ execution?: {
560
+ packageHash: string;
561
+ runtimeTargetHash: string;
562
+ metricPolicyHash: string;
563
+ } | undefined;
564
+ lineage?: {
565
+ definition: {
566
+ id: string;
567
+ contentHash: string;
568
+ revision: number;
569
+ } | null;
570
+ execution: {
571
+ id: string;
572
+ contentHash: string;
573
+ };
574
+ scoringPassId: string | null;
575
+ } | undefined;
576
+ };
577
+ result: {
578
+ schemaVersion: "openpond.experimentResult.v1";
579
+ manifest: {
580
+ id: string;
581
+ contentHash: string;
582
+ };
583
+ status: "failed" | "completed" | "cancelled";
584
+ cases: {
585
+ identity: {
586
+ caseId: string;
587
+ seed: string;
588
+ fixtureId: string | null;
589
+ };
590
+ status: "failed" | "completed" | "cancelled" | "unavailable";
591
+ output: unknown;
592
+ error: {
593
+ code: string;
594
+ message: string;
595
+ } | null;
596
+ feedback: {
597
+ feedbackKey: string;
598
+ evaluator: {
599
+ id: string;
600
+ contentHash: string;
601
+ revision: string | number;
602
+ };
603
+ status: "pending" | "failed" | "scored" | "unavailable";
604
+ value: string | number | boolean | null;
605
+ reasoning: string | null;
606
+ evidenceRefs: {
607
+ id: string;
608
+ contentHash: string;
609
+ mediaType: string | null;
610
+ sizeBytes: number | null;
611
+ }[];
612
+ passed?: boolean | null | undefined;
613
+ }[];
614
+ usage: {
615
+ inputTokens: number | null;
616
+ outputTokens: number | null;
617
+ totalTokens: number | null;
618
+ costUsd: number | null;
619
+ latencyMs: number | null;
620
+ };
621
+ traceRef: {
622
+ id: string;
623
+ contentHash: string;
624
+ mediaType: string | null;
625
+ sizeBytes: number | null;
626
+ } | null;
627
+ startedAt: string | null;
628
+ completedAt: string | null;
629
+ }[];
630
+ completedAt: string;
631
+ contentHash: string;
632
+ };
633
+ };
418
634
  /** A comparison reads retained results; it never executes or regrades a case. */
419
635
  export declare function compareExperiments(baseline: {
420
636
  manifest: ExperimentManifest;
package/dist/types/experiments.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAUjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;kBAWpC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiB1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAC5H,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;kBAO1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;kBAUhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU5B,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AACxH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,EAAE,QAAQ,EAAE,kBAAkB,GAAG,gBAAgB,CAgC3I;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACnI,CAAC;AACF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EACpE,SAAS,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACpE,oBAAoB,CA0BtB"}
1
+ {"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAWjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;kBAepC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA0B1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAC5H,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;kBAQ1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;kBAUhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU5B,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AACxH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,EAAE,QAAQ,EAAE,kBAAkB,GAAG,gBAAgB,CAgC3I;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAC7H,QAAQ,EAAE,4BAA4B,EAAE,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,EAAE,0BAA0B,EAAE,CAAC;CACvC,CAAC;AACF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,4BAA4B,GAAG;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,0BAA0B,GAAG,cAAc,GAAG,oBAAoB,GAAG,sBAAsB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IACpI,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;IAC/B,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;CACjC,CAAC;AACF,MAAM,MAAM,0BAA0B,GAAG;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAOrF;AAED,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EACpE,SAAS,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACpE,oBAAoB,CAsEtB"}
package/dist/types/learning/repository.d.ts CHANGED
@@ -1890,12 +1890,12 @@ export declare const LearningResourceKindSchema: z.ZodEnum<{
1890
1890
  source: "source";
1891
1891
  asset: "asset";
1892
1892
  binding: "binding";
1893
+ definition: "definition";
1893
1894
  decision: "decision";
1894
1895
  grade: "grade";
1895
1896
  dispatch: "dispatch";
1896
1897
  batch: "batch";
1897
1898
  schedule: "schedule";
1898
- definition: "definition";
1899
1899
  draft: "draft";
1900
1900
  iteration: "iteration";
1901
1901
  reward_check: "reward_check";
package/dist/types/learning/transport.d.ts CHANGED
@@ -1594,12 +1594,12 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
1594
1594
  source: "source";
1595
1595
  asset: "asset";
1596
1596
  binding: "binding";
1597
+ definition: "definition";
1597
1598
  decision: "decision";
1598
1599
  grade: "grade";
1599
1600
  dispatch: "dispatch";
1600
1601
  batch: "batch";
1601
1602
  schedule: "schedule";
1602
- definition: "definition";
1603
1603
  draft: "draft";
1604
1604
  iteration: "iteration";
1605
1605
  reward_check: "reward_check";
@@ -1622,12 +1622,12 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
1622
1622
  source: "source";
1623
1623
  asset: "asset";
1624
1624
  binding: "binding";
1625
+ definition: "definition";
1625
1626
  decision: "decision";
1626
1627
  grade: "grade";
1627
1628
  dispatch: "dispatch";
1628
1629
  batch: "batch";
1629
1630
  schedule: "schedule";
1630
- definition: "definition";
1631
1631
  draft: "draft";
1632
1632
  iteration: "iteration";
1633
1633
  reward_check: "reward_check";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openpond/evals",
3
- "version": "0.11.11",
3
+ "version": "0.11.12",
4
4
  "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
5
5
  "license": "MIT",
6
6
  "type": "module",