@openpond/evals 0.11.15 → 0.11.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +9 -0
- package/dist/experiments.js +25 -3
- package/dist/types/experiments.d.ts +22 -0
- package/dist/types/experiments.d.ts.map +1 -1
- package/package.json +1 -1
package/CONTRACT.md
CHANGED
|
@@ -176,6 +176,15 @@ comparison requires equal dataset version, population, evaluators and execution
|
|
|
176
176
|
contract. Harness comparison also requires the same component, evaluation
|
|
177
177
|
definition and environment; candidate source/Harness revisions can differ.
|
|
178
178
|
|
|
179
|
+
Qualified shared executors may declare `execution.compatibility` with the same
|
|
180
|
+
protocol and target kind. For those pairs, comparison matches the package and
|
|
181
|
+
metric policy while retaining each exact `runtimeTargetHash` as placement
|
|
182
|
+
evidence. A missing declaration keeps the original exact-runtime comparison.
|
|
183
|
+
For Profile targets the identical package binds environment semantics, while
|
|
184
|
+
`source.environmentHash` retains the placement-specific preparation evidence.
|
|
185
|
+
The declaration must match the admitted target kind; it does not bypass dataset,
|
|
186
|
+
population, evaluator, component or definition checks.
|
|
187
|
+
|
|
179
188
|
`compareExperiments` aligns cases and reports per-feedback eligibility, exclusion
|
|
180
189
|
reasons and paired means/deltas. Only matching completed cases with scored
|
|
181
190
|
numeric or Boolean feedback contribute. Failed, cancelled, absent or ungraded
|
package/dist/experiments.js
CHANGED
|
@@ -59,7 +59,17 @@ export const ExperimentManifestContentSchema = z.object({
|
|
|
59
59
|
maximumCostUsd: MoneySchema.nullable(),
|
|
60
60
|
dataset: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict(),
|
|
61
61
|
target: ExperimentTargetSchema,
|
|
62
|
-
execution: z.object({
|
|
62
|
+
execution: z.object({
|
|
63
|
+
packageHash: ReleaseHashSchema,
|
|
64
|
+
/** Exact placement/adapter evidence; retained even across compatible locations. */
|
|
65
|
+
runtimeTargetHash: ReleaseHashSchema,
|
|
66
|
+
metricPolicyHash: ReleaseHashSchema,
|
|
67
|
+
/** Only qualified shared executors declare this placement-independent protocol. */
|
|
68
|
+
compatibility: z.object({
|
|
69
|
+
protocol: z.literal("openpond.evaluation-execution.v1"),
|
|
70
|
+
targetKind: z.enum(["model", "agent", "harness", "fixture"]),
|
|
71
|
+
}).strict().optional(),
|
|
72
|
+
}).strict().optional(),
|
|
63
73
|
lineage: z.object({
|
|
64
74
|
definition: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict().nullable(),
|
|
65
75
|
execution: ImmutableReleaseRefSchema,
|
|
@@ -69,6 +79,8 @@ export const ExperimentManifestContentSchema = z.object({
|
|
|
69
79
|
population: z.array(ExperimentCaseIdentitySchema).min(1).max(10_000),
|
|
70
80
|
createdAt: ReleaseTimestampSchema,
|
|
71
81
|
}).strict().superRefine((value, context) => {
|
|
82
|
+
if (value.execution?.compatibility && value.execution.compatibility.targetKind !== value.target.kind)
|
|
83
|
+
context.addIssue({ code: "custom", path: ["execution", "compatibility"], message: "Execution compatibility differs from its admitted target kind." });
|
|
72
84
|
if (value.lineage && value.id !== (value.lineage.scoringPassId ?? value.lineage.execution.id))
|
|
73
85
|
context.addIssue({ code: "custom", path: ["lineage"], message: "Experiment identity must match its execution or independent scoring pass." });
|
|
74
86
|
if (new Set(value.population.map(experimentCaseKey)).size !== value.population.length)
|
|
@@ -180,14 +192,24 @@ export function compareExperiments(baseline, candidate) {
|
|
|
180
192
|
reasons.push("different_dataset_version");
|
|
181
193
|
if (baseline.manifest.target.kind !== candidate.manifest.target.kind)
|
|
182
194
|
reasons.push("different_target_kind");
|
|
183
|
-
|
|
195
|
+
const executionContract = (manifest) => {
|
|
196
|
+
const execution = manifest.execution;
|
|
197
|
+
if (!execution?.compatibility)
|
|
198
|
+
return execution ?? null;
|
|
199
|
+
const { runtimeTargetHash: _placementEvidence, ...semantic } = execution;
|
|
200
|
+
return semantic;
|
|
201
|
+
};
|
|
202
|
+
if (contentHash(executionContract(baseline.manifest)) !== contentHash(executionContract(candidate.manifest)))
|
|
184
203
|
reasons.push("different_execution_contract");
|
|
185
204
|
if (baseline.manifest.target.kind === "harness" && candidate.manifest.target.kind === "harness") {
|
|
186
205
|
const a = baseline.manifest.target.source;
|
|
187
206
|
const b = candidate.manifest.target.source;
|
|
188
207
|
if (a.profileId !== b.profileId || contentHash(a.target) !== contentHash(b.target))
|
|
189
208
|
reasons.push("different_component");
|
|
190
|
-
|
|
209
|
+
const sharedEnvironment = baseline.manifest.execution?.compatibility
|
|
210
|
+
&& candidate.manifest.execution?.compatibility
|
|
211
|
+
&& baseline.manifest.execution.packageHash === candidate.manifest.execution.packageHash;
|
|
212
|
+
if (a.environmentHash !== b.environmentHash && !sharedEnvironment)
|
|
191
213
|
reasons.push("different_environment");
|
|
192
214
|
if (a.definitionId !== b.definitionId || a.definitionHash !== b.definitionHash)
|
|
193
215
|
reasons.push("different_evaluation_definition");
|
|
@@ -149,6 +149,15 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
|
|
|
149
149
|
packageHash: z.ZodString;
|
|
150
150
|
runtimeTargetHash: z.ZodString;
|
|
151
151
|
metricPolicyHash: z.ZodString;
|
|
152
|
+
compatibility: z.ZodOptional<z.ZodObject<{
|
|
153
|
+
protocol: z.ZodLiteral<"openpond.evaluation-execution.v1">;
|
|
154
|
+
targetKind: z.ZodEnum<{
|
|
155
|
+
agent: "agent";
|
|
156
|
+
model: "model";
|
|
157
|
+
fixture: "fixture";
|
|
158
|
+
harness: "harness";
|
|
159
|
+
}>;
|
|
160
|
+
}, z.core.$strict>>;
|
|
152
161
|
}, z.core.$strict>>;
|
|
153
162
|
lineage: z.ZodOptional<z.ZodObject<{
|
|
154
163
|
definition: z.ZodNullable<z.ZodObject<{
|
|
@@ -255,6 +264,15 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
|
|
|
255
264
|
packageHash: z.ZodString;
|
|
256
265
|
runtimeTargetHash: z.ZodString;
|
|
257
266
|
metricPolicyHash: z.ZodString;
|
|
267
|
+
compatibility: z.ZodOptional<z.ZodObject<{
|
|
268
|
+
protocol: z.ZodLiteral<"openpond.evaluation-execution.v1">;
|
|
269
|
+
targetKind: z.ZodEnum<{
|
|
270
|
+
agent: "agent";
|
|
271
|
+
model: "model";
|
|
272
|
+
fixture: "fixture";
|
|
273
|
+
harness: "harness";
|
|
274
|
+
}>;
|
|
275
|
+
}, z.core.$strict>>;
|
|
258
276
|
}, z.core.$strict>>;
|
|
259
277
|
lineage: z.ZodOptional<z.ZodObject<{
|
|
260
278
|
definition: z.ZodNullable<z.ZodObject<{
|
|
@@ -646,6 +664,10 @@ export declare function verifyExperimentEvidence(input: {
|
|
|
646
664
|
packageHash: string;
|
|
647
665
|
runtimeTargetHash: string;
|
|
648
666
|
metricPolicyHash: string;
|
|
667
|
+
compatibility?: {
|
|
668
|
+
protocol: "openpond.evaluation-execution.v1";
|
|
669
|
+
targetKind: "agent" | "model" | "fixture" | "harness";
|
|
670
|
+
} | undefined;
|
|
649
671
|
} | undefined;
|
|
650
672
|
lineage?: {
|
|
651
673
|
definition: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,OAAO,EACL,uCAAuC,EACvC,sCAAsC,EACtC,gCAAgC,EAChC,gCAAgC,EAChC,sCAAsC,EACtC,qCAAqC,EACrC,KAAK,iCAAiC,GACvC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,qCAAqC,EACrC,2CAA2C,EAC3C,oCAAoC,EACpC,oCAAoC,EACpC,oCAAoC,EACpC,KAAK,+BAA+B,EACpC,KAAK,8BAA8B,GACpC,MAAM,wCAAwC,CAAC;AAIhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAiBjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;kBAepC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,eAAO,MAAM,+BAA+B
|
|
1
|
+
{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,OAAO,EACL,uCAAuC,EACvC,sCAAsC,EACtC,gCAAgC,EAChC,gCAAgC,EAChC,sCAAsC,EACtC,qCAAqC,EACrC,KAAK,iCAAiC,GACvC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,qCAAqC,EACrC,2CAA2C,EAC3C,oCAAoC,EACpC,oCAAoC,EACpC,oCAAoC,EACpC,KAAK,+BAA+B,EACpC,KAAK,8BAA8B,GACpC,MAAM,wCAAwC,CAAC;AAIhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAiBjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;kBAepC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAsC1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAC5H,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;kBAQ1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;kBAUhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU5B,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AACxH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,EAAE,QAAQ,EAAE,kBAAkB,GAAG,gBAAgB,CAgC3I;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAC7H,QAAQ,EAAE,4BAA4B,EAAE,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,EAAE,0BAA0B,EAAE,CAAC;CACvC,CAAC;AACF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,4BAA4B,GAAG;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,0BAA0B,GAAG,cAAc,GAAG,oBAAoB,GAAG,sBAAsB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IACpI,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;IAC/B,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;CACjC,CAAC;AACF,MAAM,MAAM,0BAA0B,GAAG;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAOrF;AAED,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EACpE,SAAS,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACpE,oBAAoB,CA+EtB"}
|
package/package.json
CHANGED