@openpond/evals 0.11.8 → 0.11.9

package/RELEASING.md

@@ -20,3 +20,19 @@ Version `0.4.0` removes the retired Harness root barrel and
 `./harness-improvements` / `./harness-workspaces` compatibility subpaths.
 Consumers must import those public contracts from `@openpond/harness`; this is
 an intentional breaking package-boundary correction.
+
+## Version-only CI
+
+Release preparation is shared across Evals, Harness, SDK, and Agent SDK. A stable
+version increase in exactly one package manifest selects package checks on the
+PR. All other manifest fields must be semantically identical. On merge, the
+previous master SHA must have a successful latest `ci.yml` push run and `Checks`
+job; missing, failed, pending, or ambiguous proof selects full CI. Publication
+still waits for the new SHA's `Checks`.
+
+Lockfile changes conservatively use ordinary CI, as do dependency, source, export,
+workflow, and combined package changes. Pure workspace version bumps normally
+leave the lockfile unchanged. The helper reports the selected path before commit.
+A dry run requires clean, current master and prints the commands without changing
+Git or package versions. Existing trusted publishing, registry verification,
+package tags, and retry behavior are unchanged.

package/dist/learning/acceptance-plan.js

@@ -0,0 +1,84 @@
+import { z } from "zod";
+import { assertContentHash, contentHash, ImmutableReleaseRefSchema } from "@openpond/harness";
+const Id = z.string().trim().min(1).max(240);
+const Hash = z.string().regex(/^[a-f0-9]{64}$/);
+export const AcceptanceCheckSchema = z.object({
+    id: Id, name: z.string().trim().min(1).max(200), required: z.boolean(),
+    dataset: ImmutableReleaseRefSchema, evaluator: ImmutableReleaseRefSchema,
+    executionHash: Hash, populationHash: Hash, metric: Id, direction: z.enum(["higher", "lower"]),
+    minimumCoverage: z.number().positive().max(1), threshold: z.number().finite(),
+    maximumRegression: z.number().finite().nonnegative(), maximumSpendUsd: z.number().finite().positive(),
+}).strict();
+export const AcceptancePlanContentSchema = z.object({
+    schemaVersion: z.literal("openpond.acceptancePlan.v1"), id: Id, revision: z.number().int().positive(),
+    checks: z.array(AcceptanceCheckSchema).min(1).max(100),
+    maximumSpendUsd: z.number().finite().positive(),
+}).strict().superRefine((value, ctx) => {
+    if (new Set(value.checks.map(c => c.id)).size !== value.checks.length)
+        ctx.addIssue({ code: "custom", message: "Acceptance check IDs must be unique." });
+    if (!value.checks.some(c => c.required))
+        ctx.addIssue({ code: "custom", message: "An acceptance plan needs at least one required check." });
+    if (value.checks.reduce((total, c) => total + c.maximumSpendUsd, 0) > value.maximumSpendUsd + 1e-9)
+        ctx.addIssue({ code: "custom", message: "Check budgets exceed the total plan budget." });
+});
+export const AcceptancePlanSchema = AcceptancePlanContentSchema.safeExtend({ contentHash: Hash });
+export function createAcceptancePlan(input) {
+    const value = AcceptancePlanContentSchema.parse(input);
+    return { ...value, contentHash: contentHash(value) };
+}
+export const AcceptanceMeasurementSchema = z.object({
+    runId: Id, artifact: ImmutableReleaseRefSchema, checkHash: Hash,
+    populationHash: Hash, scoredPopulationHash: Hash, executionHash: Hash, evaluator: ImmutableReleaseRefSchema,
+    metric: Id, expectedCount: z.number().int().positive(), scoredCount: z.number().int().nonnegative(),
+    score: z.number().finite().nullable(), status: z.enum(["pending", "running", "completed", "failed", "cancelled"]),
+    evidenceHash: Hash,
+}).strict().superRefine((value, ctx) => {
+    if (value.scoredCount > value.expectedCount)
+        ctx.addIssue({ code: "custom", message: "Scored count exceeds declared population." });
+    if ((value.scoredCount === 0) !== (value.score === null))
+        ctx.addIssue({ code: "custom", message: "Scores require a nonempty scored population." });
+});
+/** Evaluates verified retained measurements, never browser-supplied scores.
+ * Hosts must load and verify the referenced receipts and authorize their scope. */
+export function evaluateAcceptancePlan(input) {
+    const plan = AcceptancePlanSchema.parse(input.plan);
+    assertContentHash(plan, "Acceptance plan");
+    const baseline = ImmutableReleaseRefSchema.parse(input.baseline);
+    const candidate = ImmutableReleaseRefSchema.parse(input.candidate);
+    if (new Set(input.measurements.map(m => m.checkId)).size !== input.measurements.length || input.measurements.some(m => !plan.checks.some(c => c.id === m.checkId)))
+        throw new Error("Acceptance evidence has duplicate or unknown checks.");
+    const checks = plan.checks.map(check => {
+        const pair = input.measurements.find(m => m.checkId === check.id);
+        const reasons = [];
+        const expectedHash = contentHash(check);
+        const values = [pair?.baseline ?? null, pair?.candidate ?? null].map((raw, index) => {
+            if (!raw) {
+                reasons.push(index ? "candidate_missing" : "baseline_missing");
+                return null;
+            }
+            const value = AcceptanceMeasurementSchema.parse(raw);
+            if (contentHash(value.artifact) !== contentHash(index ? candidate : baseline))
+                reasons.push("artifact_mismatch");
+            if (value.checkHash !== expectedHash || value.populationHash !== check.populationHash || value.executionHash !== check.executionHash || contentHash(value.evaluator) !== contentHash(check.evaluator) || value.metric !== check.metric)
+                reasons.push("configuration_mismatch");
+            if (value.status !== "completed")
+                reasons.push(`execution_${value.status}`);
+            if (value.score === null || value.scoredCount / value.expectedCount < check.minimumCoverage)
+                reasons.push("coverage_incomplete");
+            return value;
+        });
+        if (values[0] && values[1] && (values[0].expectedCount !== values[1].expectedCount || values[0].scoredCount !== values[1].scoredCount || values[0].scoredPopulationHash !== values[1].scoredPopulationHash))
+            reasons.push("population_count_mismatch");
+        if (reasons.length)
+            return { id: check.id, required: check.required, state: "blocked", reasons: [...new Set(reasons)], delta: null };
+        const before = values[0].score;
+        const after = values[1].score;
+        const delta = (after - before) * (check.direction === "higher" ? 1 : -1);
+        if (check.direction === "higher" ? after < check.threshold : after > check.threshold)
+            reasons.push("threshold_failed");
+        if (delta < -check.maximumRegression)
+            reasons.push("regression_failed");
+        return { id: check.id, required: check.required, state: reasons.length ? "failed" : "passed", reasons, delta };
+    });
+    return { passed: checks.filter(c => c.required).every(c => c.state === "passed"), checks };
+}

package/dist/learning/index.js

@@ -32,3 +32,4 @@ export * from "./schedule-worker.js";
 export * from "./nightly-schedule.js";
 export * from "./policy-inspection.js";
 export * from "./task-queue-inspection.js";
+export * from "./acceptance-plan.js";

package/dist/types/learning/acceptance-plan.d.ts

@@ -0,0 +1,140 @@
+import { z } from "zod";
+import { ImmutableReleaseRefSchema } from "@openpond/harness";
+export declare const AcceptanceCheckSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    required: z.ZodBoolean;
+    dataset: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    evaluator: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    executionHash: z.ZodString;
+    populationHash: z.ZodString;
+    metric: z.ZodString;
+    direction: z.ZodEnum<{
+        higher: "higher";
+        lower: "lower";
+    }>;
+    minimumCoverage: z.ZodNumber;
+    threshold: z.ZodNumber;
+    maximumRegression: z.ZodNumber;
+    maximumSpendUsd: z.ZodNumber;
+}, z.core.$strict>;
+export declare const AcceptancePlanContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.acceptancePlan.v1">;
+    id: z.ZodString;
+    revision: z.ZodNumber;
+    checks: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodString;
+        required: z.ZodBoolean;
+        dataset: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        evaluator: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        executionHash: z.ZodString;
+        populationHash: z.ZodString;
+        metric: z.ZodString;
+        direction: z.ZodEnum<{
+            higher: "higher";
+            lower: "lower";
+        }>;
+        minimumCoverage: z.ZodNumber;
+        threshold: z.ZodNumber;
+        maximumRegression: z.ZodNumber;
+        maximumSpendUsd: z.ZodNumber;
+    }, z.core.$strict>>;
+    maximumSpendUsd: z.ZodNumber;
+}, z.core.$strict>;
+export declare const AcceptancePlanSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.acceptancePlan.v1">;
+    id: z.ZodString;
+    revision: z.ZodNumber;
+    checks: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodString;
+        required: z.ZodBoolean;
+        dataset: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        evaluator: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        executionHash: z.ZodString;
+        populationHash: z.ZodString;
+        metric: z.ZodString;
+        direction: z.ZodEnum<{
+            higher: "higher";
+            lower: "lower";
+        }>;
+        minimumCoverage: z.ZodNumber;
+        threshold: z.ZodNumber;
+        maximumRegression: z.ZodNumber;
+        maximumSpendUsd: z.ZodNumber;
+    }, z.core.$strict>>;
+    maximumSpendUsd: z.ZodNumber;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type AcceptancePlan = z.infer<typeof AcceptancePlanSchema>;
+export declare function createAcceptancePlan(input: z.input<typeof AcceptancePlanContentSchema>): AcceptancePlan;
+export declare const AcceptanceMeasurementSchema: z.ZodObject<{
+    runId: z.ZodString;
+    artifact: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    checkHash: z.ZodString;
+    populationHash: z.ZodString;
+    scoredPopulationHash: z.ZodString;
+    executionHash: z.ZodString;
+    evaluator: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    metric: z.ZodString;
+    expectedCount: z.ZodNumber;
+    scoredCount: z.ZodNumber;
+    score: z.ZodNullable<z.ZodNumber>;
+    status: z.ZodEnum<{
+        pending: "pending";
+        failed: "failed";
+        completed: "completed";
+        cancelled: "cancelled";
+        running: "running";
+    }>;
+    evidenceHash: z.ZodString;
+}, z.core.$strict>;
+export type AcceptanceMeasurement = z.infer<typeof AcceptanceMeasurementSchema>;
+export type AcceptanceVerdict = {
+    passed: boolean;
+    checks: Array<{
+        id: string;
+        required: boolean;
+        state: "passed" | "failed" | "blocked";
+        reasons: string[];
+        delta: number | null;
+    }>;
+};
+/** Evaluates verified retained measurements, never browser-supplied scores.
+ * Hosts must load and verify the referenced receipts and authorize their scope. */
+export declare function evaluateAcceptancePlan(input: {
+    plan: AcceptancePlan;
+    baseline: z.infer<typeof ImmutableReleaseRefSchema>;
+    candidate: z.infer<typeof ImmutableReleaseRefSchema>;
+    measurements: Array<{
+        checkId: string;
+        baseline: AcceptanceMeasurement | null;
+        candidate: AcceptanceMeasurement | null;
+    }>;
+}): AcceptanceVerdict;
+//# sourceMappingURL=acceptance-plan.d.ts.map

package/dist/types/learning/acceptance-plan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"acceptance-plan.d.ts","sourceRoot":"","sources":["../../../../../src/learning/acceptance-plan.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAkC,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAI9F,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;kBAMvB,CAAC;AACZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAQtC,CAAC;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAgE,CAAC;AAClG,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,GAAG,cAAc,CAGvG;AACD,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;kBAStC,CAAC;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,iBAAiB,GAAG;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAA;CAAE,CAAC;AAEvL;mFACmF;AACnF,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;IAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;IAChI,YAAY,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,qBAAqB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,qBAAqB,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CAC3H,GAAG,iBAAiB,CA6BpB"}

package/dist/types/learning/index.d.ts

@@ -32,4 +32,5 @@ export * from "./schedule-worker.js";
 export * from "./nightly-schedule.js";
 export * from "./policy-inspection.js";
 export * from "./task-queue-inspection.js";
+export * from "./acceptance-plan.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/types/learning/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/learning/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,uBAAuB,CAAC;AACtC,cAAc,aAAa,CAAC;AAC5B,cAAc,kBAAkB,CAAC;AACjC,cAAc,mBAAmB,CAAC;AAClC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,sCAAsC,CAAC;AACrD,cAAc,mCAAmC,CAAC;AAClD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,mCAAmC,CAAC;AAClD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAC/E,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/learning/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,uBAAuB,CAAC;AACtC,cAAc,aAAa,CAAC;AAC5B,cAAc,kBAAkB,CAAC;AACjC,cAAc,mBAAmB,CAAC;AAClC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,sCAAsC,CAAC;AACrD,cAAc,mCAAmC,CAAC;AAClD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,mCAAmC,CAAC;AAClD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAC/E,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAE3C,cAAc,sBAAsB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.11.8",
+  "version": "0.11.9",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",
@@ -147,7 +147,7 @@
     "test": "vitest run --root ../.. --config vitest.config.ts --project evals",
     "check:api": "tsx scripts/check-public-package.ts",
     "check:consumer": "tsx scripts/verify-packed-consumer.ts",
-    "check": "pnpm run typecheck && pnpm run test && pnpm run build && pnpm run check:api && pnpm run check:consumer && npm pack --dry-run",
+    "check": "pnpm run build && pnpm run typecheck && pnpm run test && pnpm run check:api && pnpm run check:consumer && npm pack --dry-run",
     "publish:check": "pnpm run build && pnpm run check:api && pnpm run check:consumer && npm pack --dry-run"
   },
   "dependencies": {