@openpond/evals 0.11.9 → 0.11.11

package/LEARNING.md

@@ -359,3 +359,38 @@ unique eligible, unconsumed training attempts across current policies. It
 deduplicates shared sources and excludes held-out family reservations. Paused
 policies retain pending tasks. An invalid policy yields a null count and explicit
 issues rather than a misleading zero. Source credentials cannot read the queue.
+
+## Independent acceptance groups
+
+`createAcceptanceGroupManifest` pins a training Job, exact baseline/candidate
+artifacts, an acceptance plan and bounded concurrency. Each durable check declares
+its role and score unit. Required quality/retention gates remain separate;
+diagnostic checks cannot authorize acceptance. Pin the plan before training
+admission, then create the group when the exact candidate exists. A missing
+required group must block acceptance.
+
+`createAcceptanceGroupSnapshot` verifies plan/manifest/snapshot integrity and
+attempt identity. `assertAcceptanceGroupUpdate` enforces sequential revisions,
+append-only retries, immutable terminal evidence, legal lifecycle transitions,
+concurrency and dispatch budgets including outstanding reservations and prior
+attempt costs. New attempts enter queued; an explicitly reused baseline may enter
+completed. A terminal attempt requires cleanup, settled accounting and no remaining
+reservation. Deliberate reruns create a new immutable group.
+
+`assertAcceptanceBaselineReuse` accepts only fresh, exact, same-team measurements
+matching the artifact, check, qualification reference and execution configuration.
+It rejects recursively reused measurements. The owner must resolve the retained
+source group and current qualification/revocation/access state; hashes alone are
+not authorization or proof of execution.
+
+`acceptanceGroupVerdict` preserves per-check results and blocks acceptance until
+the group is complete, all required attempts have comparable measurements, cleanup
+and accounting are settled, and recorded spend stays within all budgets. Failed
+attempt costs count even if a later retry passes. It never averages unrelated
+metrics into one quality score.
+
+These contracts do not dispatch work or implement a database. Hosts must serialize
+revision changes and budget reservations, resolve authoritative execution/grading
+receipts, isolate holdouts from training, bind the exact candidate at runtime,
+verify cleanup, and enforce the verdict inside the candidate-decision transaction.
+No customer serving is activated by an acceptance group.

package/dist/experiments.js

@@ -0,0 +1,169 @@
+import { z } from "zod";
+import { ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
+import { ProfileEvaluationRunSourceSchema } from "./profile-evaluations.js";
+import { FeedbackKeySchema } from "./rewards.js";
+const MoneySchema = z.number().finite().nonnegative();
+export const ExperimentTargetSchema = z.discriminatedUnion("kind", [
+    z.object({
+        kind: z.literal("model"),
+        modelId: ReleaseIdSchema,
+        configurationHash: ReleaseHashSchema,
+    }).strict(),
+    z.object({
+        kind: z.literal("harness"),
+        source: ProfileEvaluationRunSourceSchema,
+    }).strict(),
+]);
+export const ExperimentEvaluatorSchema = z.object({
+    release: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict(),
+    feedbackKey: FeedbackKeySchema,
+    output: z.enum(["boolean", "score", "category"]),
+    /** Category labels are meaningful only for category output. */
+    categories: z.array(z.string().trim().min(1).max(120)).max(50),
+}).strict().superRefine((value, context) => {
+    if ((value.output === "category") !== (value.categories.length > 0))
+        context.addIssue({ code: "custom", path: ["categories"], message: "Only category feedback declares categories." });
+    if (new Set(value.categories).size !== value.categories.length)
+        context.addIssue({ code: "custom", path: ["categories"], message: "Feedback categories must be unique." });
+});
+export const ExperimentCaseIdentitySchema = z.object({
+    caseId: ReleaseIdSchema,
+    seed: z.string().trim().min(1).max(500),
+    fixtureId: ReleaseIdSchema.nullable(),
+}).strict();
+export function experimentCaseKey(identity) {
+    const value = ExperimentCaseIdentitySchema.parse(identity);
+    return JSON.stringify([value.caseId, value.seed, value.fixtureId]);
+}
+export const ExperimentManifestContentSchema = z.object({
+    schemaVersion: z.literal("openpond.experimentManifest.v1"),
+    id: ReleaseIdSchema,
+    name: z.string().trim().min(1).max(500),
+    teamId: ReleaseIdSchema,
+    operationId: ReleaseIdSchema,
+    maximumCostUsd: MoneySchema,
+    dataset: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict(),
+    target: ExperimentTargetSchema,
+    evaluators: z.array(ExperimentEvaluatorSchema).max(100),
+    population: z.array(ExperimentCaseIdentitySchema).min(1).max(10_000),
+    createdAt: ReleaseTimestampSchema,
+}).strict().superRefine((value, context) => {
+    if (new Set(value.population.map(experimentCaseKey)).size !== value.population.length)
+        context.addIssue({ code: "custom", path: ["population"], message: "Experiment cases must be unique." });
+    if (new Set(value.evaluators.map((item) => item.feedbackKey)).size !== value.evaluators.length)
+        context.addIssue({ code: "custom", path: ["evaluators"], message: "Experiment feedback keys must be unique." });
+});
+export const ExperimentManifestSchema = ExperimentManifestContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
+export const ExperimentFeedbackSchema = z.object({
+    feedbackKey: FeedbackKeySchema,
+    evaluator: ExperimentEvaluatorSchema.shape.release,
+    status: z.enum(["scored", "unavailable", "failed", "pending"]),
+    value: z.union([z.boolean(), z.number().finite(), z.string().max(120)]).nullable(),
+    reasoning: z.string().max(20_000).nullable(),
+    evidenceRefs: z.array(ImmutableArtifactRefSchema).max(1_000),
+}).strict();
+export const ExperimentUsageSchema = z.object({
+    inputTokens: z.number().int().nonnegative().nullable(),
+    outputTokens: z.number().int().nonnegative().nullable(),
+    totalTokens: z.number().int().nonnegative().nullable(),
+    costUsd: MoneySchema.nullable(),
+    latencyMs: z.number().finite().nonnegative().nullable(),
+}).strict().superRefine((usage, context) => {
+    if (usage.inputTokens !== null && usage.outputTokens !== null && usage.totalTokens !== null
+        && usage.totalTokens < usage.inputTokens + usage.outputTokens)
+        context.addIssue({ code: "custom", path: ["totalTokens"], message: "Total tokens cannot be less than input plus output." });
+});
+export const ExperimentCaseResultSchema = z.object({
+    identity: ExperimentCaseIdentitySchema,
+    status: z.enum(["completed", "failed", "cancelled", "unavailable"]),
+    output: z.unknown().nullable(),
+    error: z.object({ code: ReleaseIdSchema, message: z.string().max(2_000) }).strict().nullable(),
+    feedback: z.array(ExperimentFeedbackSchema).max(100),
+    usage: ExperimentUsageSchema,
+    traceRef: ImmutableArtifactRefSchema.nullable(),
+    startedAt: ReleaseTimestampSchema.nullable(),
+    completedAt: ReleaseTimestampSchema.nullable(),
+}).strict();
+export const ExperimentResultContentSchema = z.object({
+    schemaVersion: z.literal("openpond.experimentResult.v1"),
+    manifest: ImmutableReleaseRefSchema,
+    status: z.enum(["completed", "failed", "cancelled"]),
+    cases: z.array(ExperimentCaseResultSchema).min(1).max(10_000),
+    completedAt: ReleaseTimestampSchema,
+}).strict();
+export const ExperimentResultSchema = ExperimentResultContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
+export function createExperimentManifest(input) {
+    const value = ExperimentManifestContentSchema.parse(input);
+    return ExperimentManifestSchema.parse({ ...value, contentHash: contentHash(value) });
+}
+export function createExperimentResult(input, manifest) {
+    ExperimentManifestSchema.parse(manifest);
+    assertContentHash(manifest, "Experiment manifest");
+    const value = ExperimentResultContentSchema.parse(input);
+    if (value.manifest.id !== manifest.id || value.manifest.contentHash !== manifest.contentHash)
+        throw new Error("Experiment result differs from its manifest.");
+    const expected = new Set(manifest.population.map(experimentCaseKey));
+    const seen = new Set();
+    const evaluators = new Map(manifest.evaluators.map((item) => [item.feedbackKey, item]));
+    for (const result of value.cases) {
+        const key = experimentCaseKey(result.identity);
+        if (!expected.has(key) || seen.has(key))
+            throw new Error("Experiment result has an unexpected or repeated case.");
+        seen.add(key);
+        const feedbackKeys = new Set();
+        for (const feedback of result.feedback) {
+            const evaluator = evaluators.get(feedback.feedbackKey);
+            if (!evaluator || feedbackKeys.has(feedback.feedbackKey)
+                || feedback.evaluator.id !== evaluator.release.id
+                || feedback.evaluator.contentHash !== evaluator.release.contentHash
+                || feedback.evaluator.revision !== evaluator.release.revision)
+                throw new Error("Experiment feedback differs from its pinned evaluator.");
+            feedbackKeys.add(feedback.feedbackKey);
+            if (feedback.status === "scored") {
+                if (feedback.value === null || typeof feedback.value !== { boolean: "boolean", score: "number", category: "string" }[evaluator.output]
+                    || (evaluator.output === "category" && !evaluator.categories.includes(feedback.value))
+                    || (evaluator.output === "score" && (feedback.value < 0 || feedback.value > 1)))
+                    throw new Error("Experiment feedback value violates its output contract.");
+            }
+            else if (feedback.value !== null)
+                throw new Error("Unscored experiment feedback cannot claim a value.");
+        }
+    }
+    if (seen.size !== expected.size)
+        throw new Error("Experiment result omits admitted cases.");
+    return ExperimentResultSchema.parse({ ...value, contentHash: contentHash(value) });
+}
+/** A comparison reads retained results; it never executes or regrades a case. */
+export function compareExperiments(baseline, candidate) {
+    for (const pair of [baseline, candidate]) {
+        const { contentHash: hash, ...value } = ExperimentResultSchema.parse(pair.result);
+        if (contentHash(value) !== hash)
+            throw new Error("Experiment result integrity failed.");
+        createExperimentResult(value, pair.manifest);
+    }
+    const reasons = [];
+    if (baseline.manifest.teamId !== candidate.manifest.teamId)
+        reasons.push("different_workspace");
+    if (baseline.manifest.dataset.id !== candidate.manifest.dataset.id
+        || baseline.manifest.dataset.contentHash !== candidate.manifest.dataset.contentHash
+        || baseline.manifest.dataset.revision !== candidate.manifest.dataset.revision)
+        reasons.push("different_dataset_version");
+    if (baseline.manifest.target.kind !== candidate.manifest.target.kind)
+        reasons.push("different_target_kind");
+    if (contentHash(baseline.manifest.evaluators) !== contentHash(candidate.manifest.evaluators))
+        reasons.push("different_evaluators");
+    const left = new Map(baseline.result.cases.map((item) => [experimentCaseKey(item.identity), item]));
+    const right = new Map(candidate.result.cases.map((item) => [experimentCaseKey(item.identity), item]));
+    const keys = [...new Set([...left.keys(), ...right.keys()])].sort();
+    if (left.size !== right.size || keys.some((key) => !left.has(key) || !right.has(key)))
+        reasons.push("different_population");
+    return {
+        comparable: reasons.length === 0,
+        reasons,
+        cases: keys.map((key) => ({
+            identity: (left.get(key) ?? right.get(key)).identity,
+            baseline: left.get(key) ?? null,
+            candidate: right.get(key) ?? null,
+        })),
+    };
+}

package/dist/index.js

@@ -4,6 +4,7 @@ export * from "./artifact-verification.js";
 export * from "./evidence/index.js";
 export * from "./execution-contracts.js";
 export * from "./execution-receipts.js";
+export * from "./experiments.js";
 export * from "./graders.js";
 export * from "./harness.js";
 export * from "./learned-preference.js";

package/dist/learning/acceptance-group.js

@@ -0,0 +1,199 @@
+import { z } from "zod";
+import { assertContentHash, contentHash, ImmutableReleaseRefSchema } from "@openpond/harness";
+import { AcceptanceMeasurementSchema, AcceptancePlanSchema, evaluateAcceptancePlan } from "./acceptance-plan.js";
+const Id = z.string().trim().min(1).max(240);
+const Hash = z.string().regex(/^[a-f0-9]{64}$/);
+const Timestamp = z.iso.datetime();
+const Money = z.number().finite().nonnegative();
+const terminalAttempts = new Set(["completed", "failed", "cancelled", "budget_exhausted"]);
+const terminalGroups = new Set(["completed", "failed", "cancelled", "budget_exhausted"]);
+const groupTransitions = {
+    queued: ["queued", "running", "blocked", "cancelling", "failed", "budget_exhausted"],
+    running: ["running", "blocked", "cancelling", "cleaning", "completed", "failed", "budget_exhausted"],
+    blocked: ["blocked", "running", "cancelling", "failed", "budget_exhausted"],
+    cancelling: ["cancelling", "cleaning", "cancelled", "budget_exhausted"],
+    cleaning: ["cleaning", "completed", "failed", "cancelled", "budget_exhausted"],
+};
+const attemptTransitions = {
+    queued: ["queued", "dispatching", "failed", "cancelled", "budget_exhausted"],
+    dispatching: ["dispatching", "running", "cleaning", "failed", "cancelled", "budget_exhausted"],
+    running: ["running", "grading", "cleaning", "failed", "cancelled", "budget_exhausted"],
+    grading: ["grading", "cleaning", "completed", "failed", "cancelled", "budget_exhausted"],
+    cleaning: ["cleaning", "completed", "failed", "cancelled", "budget_exhausted"],
+};
+/** A group starts only after the exact candidate is available. The host must pin
+ * its acceptance plan before training admission so a missing group blocks review. */
+export const AcceptanceGroupManifestContentSchema = z.object({
+    schemaVersion: z.literal("openpond.acceptanceGroupManifest.v1"),
+    id: Id, teamId: Id, trainingJob: ImmutableReleaseRefSchema,
+    baseline: ImmutableReleaseRefSchema, candidate: ImmutableReleaseRefSchema,
+    plan: AcceptancePlanSchema, maximumParallelAttempts: z.number().int().min(1).max(8),
+    createdAt: Timestamp,
+}).strict().superRefine((manifest, ctx) => {
+    if (manifest.plan.checks.some(check => !check.role || !check.unit))
+        ctx.addIssue({ code: "custom", message: "Durable acceptance checks require explicit roles and score units." });
+});
+export const AcceptanceGroupManifestSchema = AcceptanceGroupManifestContentSchema.safeExtend({ contentHash: Hash });
+export const AcceptanceAttemptSchema = z.object({
+    id: Id, checkId: Id, subject: z.enum(["baseline", "candidate"]),
+    ordinal: z.number().int().positive(), dispatchId: Id,
+    state: z.enum(["queued", "dispatching", "running", "grading", "cleaning", "completed", "failed", "cancelled", "budget_exhausted"]),
+    artifact: ImmutableReleaseRefSchema, checkHash: Hash,
+    measurement: AcceptanceMeasurementSchema.nullable(),
+    executionReceipt: ImmutableReleaseRefSchema.nullable(),
+    reusedFrom: z.object({ group: ImmutableReleaseRefSchema, attemptId: Id }).strict().nullable(),
+    spendUsd: Money, reservedUsd: Money,
+    cleanupComplete: z.boolean(), accountingComplete: z.boolean(),
+    failureCode: Id.nullable(), createdAt: Timestamp, updatedAt: Timestamp,
+}).strict().superRefine((attempt, ctx) => {
+    if (Date.parse(attempt.updatedAt) < Date.parse(attempt.createdAt))
+        ctx.addIssue({ code: "custom", message: "Attempt timestamps are reversed." });
+    if (terminalAttempts.has(attempt.state) && (!attempt.cleanupComplete || !attempt.accountingComplete || attempt.reservedUsd !== 0))
+        ctx.addIssue({ code: "custom", message: "Terminal attempts require cleanup and settled accounting." });
+    if (attempt.state === "completed" && (attempt.measurement?.status !== "completed" || !attempt.executionReceipt))
+        ctx.addIssue({ code: "custom", message: "Completed attempts require retained measurement and execution receipts." });
+    if (attempt.reusedFrom && (attempt.subject !== "baseline" || attempt.state !== "completed" || attempt.spendUsd !== 0))
+        ctx.addIssue({ code: "custom", message: "Only completed baselines may reuse retained evidence without new spend." });
+});
+export const AcceptanceGroupSnapshotContentSchema = z.object({
+    schemaVersion: z.literal("openpond.acceptanceGroupSnapshot.v1"),
+    manifest: AcceptanceGroupManifestSchema,
+    revision: z.number().int().positive(),
+    state: z.enum(["queued", "running", "blocked", "cancelling", "cleaning", "completed", "failed", "cancelled", "budget_exhausted"]),
+    attempts: z.array(AcceptanceAttemptSchema).max(2_000),
+    failureCode: Id.nullable(), updatedAt: Timestamp,
+}).strict();
+export const AcceptanceGroupSnapshotSchema = AcceptanceGroupSnapshotContentSchema.extend({ contentHash: Hash });
+export function createAcceptanceGroupManifest(raw) {
+    const content = AcceptanceGroupManifestContentSchema.parse(raw);
+    assertContentHash(content.plan, "Acceptance plan");
+    return { ...content, contentHash: contentHash(content) };
+}
+export function createAcceptanceGroupSnapshot(raw) {
+    const content = AcceptanceGroupSnapshotContentSchema.parse(raw);
+    return assertAcceptanceGroupSnapshot({ ...content, contentHash: contentHash(content) });
+}
+/** Structural integrity does not replace receipt retrieval, resource access,
+ * qualification/revocation checks, or authoritative ledger verification. */
+export function assertAcceptanceGroupSnapshot(raw) {
+    const snapshot = AcceptanceGroupSnapshotSchema.parse(raw);
+    assertContentHash(snapshot, "Acceptance group snapshot");
+    assertContentHash(snapshot.manifest, "Acceptance group manifest");
+    assertContentHash(snapshot.manifest.plan, "Acceptance plan");
+    const checks = new Map(snapshot.manifest.plan.checks.map(check => [check.id, check]));
+    const identities = new Set(), dispatches = new Set(), positions = new Set();
+    for (const attempt of snapshot.attempts) {
+        const check = checks.get(attempt.checkId);
+        if (!check || attempt.checkHash !== contentHash(check) || contentHash(attempt.artifact) !== contentHash(snapshot.manifest[attempt.subject]))
+            throw new Error("Acceptance attempt does not match its pinned check and artifact.");
+        const position = JSON.stringify([attempt.checkId, attempt.subject, attempt.ordinal]);
+        if (identities.has(attempt.id) || dispatches.has(attempt.dispatchId) || positions.has(position))
+            throw new Error("Acceptance attempts require unique identities, dispatches and positions.");
+        identities.add(attempt.id);
+        dispatches.add(attempt.dispatchId);
+        positions.add(position);
+        if (attempt.measurement && (attempt.measurement.checkHash !== attempt.checkHash || contentHash(attempt.measurement.artifact) !== contentHash(attempt.artifact)))
+            throw new Error("Acceptance measurement does not match its attempt.");
+        if (attempt.reusedFrom && !check.baselineReuse)
+            throw new Error("This check does not permit baseline reuse.");
+    }
+    for (const check of checks.values())
+        for (const subject of ["baseline", "candidate"]) {
+            const attempts = snapshot.attempts.filter(attempt => attempt.checkId === check.id && attempt.subject === subject).sort((a, b) => a.ordinal - b.ordinal);
+            attempts.forEach((attempt, index) => {
+                if (attempt.ordinal !== index + 1 || (index > 0 && (!terminalAttempts.has(attempts[index - 1].state) || attempts[index - 1].state === "completed")))
+                    throw new Error("Retries must follow a settled failed attempt; completed evidence requires a new group to rerun.");
+            });
+        }
+    if (terminalGroups.has(snapshot.state) && snapshot.attempts.some(attempt => !terminalAttempts.has(attempt.state)))
+        throw new Error("Terminal groups cannot retain unsettled attempts.");
+    if (snapshot.attempts.filter(attempt => !terminalAttempts.has(attempt.state) && attempt.state !== "queued").length > snapshot.manifest.maximumParallelAttempts)
+        throw new Error("Acceptance attempts exceed the declared concurrency limit.");
+    if (Date.parse(snapshot.updatedAt) < Date.parse(snapshot.manifest.createdAt) || snapshot.attempts.some(attempt => Date.parse(attempt.createdAt) < Date.parse(snapshot.manifest.createdAt) || Date.parse(attempt.updatedAt) > Date.parse(snapshot.updatedAt)))
+        throw new Error("Acceptance group timestamps do not contain their attempts.");
+    if (snapshot.state === "completed" && snapshot.manifest.plan.checks.some(check => ["baseline", "candidate"].some(subject => !snapshot.attempts.some(attempt => attempt.checkId === check.id && attempt.subject === subject))))
+        throw new Error("Completed groups require an attempt for each declared check and subject.");
+    return snapshot;
+}
+/** Runtime retries append history. A terminal group is immutable; deliberate
+ * reruns create a new manifest and cannot silently replace reviewed evidence. */
+export function assertAcceptanceGroupUpdate(previousRaw, nextRaw) {
+    const previous = assertAcceptanceGroupSnapshot(previousRaw), next = assertAcceptanceGroupSnapshot(nextRaw);
+    if (previous.manifest.contentHash !== next.manifest.contentHash || next.revision !== previous.revision + 1 || terminalGroups.has(previous.state))
+        throw new Error("Acceptance group revision or immutable identity changed.");
+    if (!groupTransitions[previous.state]?.includes(next.state))
+        throw new Error("Invalid acceptance group transition.");
+    if (["cancelling", "cleaning"].includes(previous.state) && next.attempts.length !== previous.attempts.length)
+        throw new Error("Stopping groups cannot dispatch additional attempts.");
+    if (Date.parse(next.updatedAt) < Date.parse(previous.updatedAt))
+        throw new Error("Acceptance group time moved backwards.");
+    const priorIds = new Set(previous.attempts.map(attempt => attempt.id));
+    for (const added of next.attempts.filter(attempt => !priorIds.has(attempt.id))) {
+        if (added.state !== "queued" && !(added.state === "completed" && added.reusedFrom))
+            throw new Error("New attempts must enter the queue or retain an explicitly reused baseline.");
+        if (Date.parse(added.createdAt) < Date.parse(previous.updatedAt))
+            throw new Error("New attempts cannot predate the preceding group revision.");
+    }
+    for (const before of previous.attempts) {
+        const after = next.attempts.find(attempt => attempt.id === before.id);
+        if (!after || after.checkId !== before.checkId || after.subject !== before.subject || after.ordinal !== before.ordinal || after.dispatchId !== before.dispatchId || after.createdAt !== before.createdAt || after.checkHash !== before.checkHash || contentHash(after.reusedFrom) !== contentHash(before.reusedFrom) || contentHash(after.artifact) !== contentHash(before.artifact))
+            throw new Error("Acceptance attempt identity or history changed.");
+        if (after.spendUsd < before.spendUsd || Date.parse(after.updatedAt) < Date.parse(before.updatedAt) || (terminalAttempts.has(before.state) && contentHash(before) !== contentHash(after)))
+            throw new Error("Settled acceptance evidence and spend cannot be rewritten.");
+        if (!terminalAttempts.has(before.state) && !attemptTransitions[before.state]?.includes(after.state))
+            throw new Error("Invalid acceptance attempt transition.");
+    }
+    const admitsWork = next.attempts.some(after => after.state === "dispatching" && previous.attempts.find(before => before.id === after.id)?.state === "queued");
+    if (admitsWork) {
+        const committed = (attempt) => attempt.spendUsd + attempt.reservedUsd;
+        if (next.attempts.reduce((sum, attempt) => sum + committed(attempt), 0) > next.manifest.plan.maximumSpendUsd + 1e-9 ||
+            next.manifest.plan.checks.some(check => next.attempts.filter(attempt => attempt.checkId === check.id).reduce((sum, attempt) => sum + committed(attempt), 0) > check.maximumSpendUsd + 1e-9))
+            throw new Error("Dispatch exceeds the pinned acceptance budget, including outstanding reservations and retry spend.");
+    }
+    return next;
+}
+/** Hosts additionally resolve the qualification reference and check current
+ * revocation/access. This verifier rejects stale or differently configured
+ * cached measurements without requiring the source candidate to have passed. */
+export function assertAcceptanceBaselineReuse(input) {
+    const source = assertAcceptanceGroupSnapshot(input.sourceGroup);
+    const target = AcceptanceGroupManifestSchema.parse(input.targetManifest);
+    assertContentHash(target, "Acceptance group manifest");
+    assertContentHash(target.plan, "Acceptance plan");
+    const check = target.plan.checks.find(value => value.id === input.checkId);
+    const attempt = source.attempts.find(value => value.id === input.attemptId);
+    const now = Date.parse(Timestamp.parse(input.now));
+    if (!check?.baselineReuse || !attempt || !terminalGroups.has(source.state) || source.manifest.id === target.id || source.manifest.teamId !== target.teamId || attempt.state !== "completed" || !attempt.measurement || attempt.reusedFrom || !attempt.cleanupComplete || !attempt.accountingComplete)
+        throw new Error("This retained attempt is not eligible for baseline reuse.");
+    if (attempt.checkId !== check.id || attempt.checkHash !== contentHash(check) || contentHash(attempt.artifact) !== contentHash(target.baseline) || now < Date.parse(attempt.updatedAt) || now - Date.parse(attempt.updatedAt) > check.baselineReuse.maximumAgeSeconds * 1_000)
+        throw new Error("Baseline reuse requires the exact artifact, check, qualification and freshness window.");
+    const measurement = attempt.measurement;
+    if (measurement.status !== "completed" || measurement.score === null || measurement.scoredCount / measurement.expectedCount < check.minimumCoverage || measurement.populationHash !== check.populationHash || measurement.executionHash !== check.executionHash || contentHash(measurement.evaluator) !== contentHash(check.evaluator) || measurement.metric !== check.metric)
+        throw new Error("Baseline reuse requires complete compatible retained measurements.");
+    return attempt;
+}
+export function acceptanceGroupVerdict(raw) {
+    const snapshot = assertAcceptanceGroupSnapshot(raw);
+    const { manifest } = snapshot;
+    const latest = (checkId, subject) => snapshot.attempts
+        .filter(attempt => attempt.checkId === checkId && attempt.subject === subject)
+        .sort((a, b) => b.ordinal - a.ordinal)[0];
+    const measurements = manifest.plan.checks.map(check => ({
+        checkId: check.id,
+        baseline: latest(check.id, "baseline")?.measurement ?? null,
+        candidate: latest(check.id, "candidate")?.measurement ?? null,
+    }));
+    const verdict = evaluateAcceptancePlan({ plan: manifest.plan, baseline: manifest.baseline, candidate: manifest.candidate, measurements });
+    const reasons = [];
+    if (snapshot.state !== "completed")
+        reasons.push("group_not_completed");
+    if (snapshot.attempts.some(attempt => !attempt.cleanupComplete || !attempt.accountingComplete || attempt.reservedUsd !== 0))
+        reasons.push("cleanup_or_accounting_incomplete");
+    const spendUsd = snapshot.attempts.reduce((sum, attempt) => sum + attempt.spendUsd, 0);
+    if (spendUsd > manifest.plan.maximumSpendUsd + 1e-9 || manifest.plan.checks.some(check => snapshot.attempts.filter(attempt => attempt.checkId === check.id).reduce((sum, attempt) => sum + attempt.spendUsd, 0) > check.maximumSpendUsd + 1e-9))
+        reasons.push("acceptance_budget_exceeded");
+    for (const check of manifest.plan.checks.filter(check => check.required))
+        if (["baseline", "candidate"].some(subject => latest(check.id, subject)?.state !== "completed"))
+            reasons.push("required_attempt_incomplete");
+    return { ...verdict, passed: verdict.passed && reasons.length === 0, reasons: [...new Set(reasons)], spendUsd };
+}

package/dist/learning/acceptance-plan.js

@@ -4,11 +4,17 @@ const Id = z.string().trim().min(1).max(240);
 const Hash = z.string().regex(/^[a-f0-9]{64}$/);
 export const AcceptanceCheckSchema = z.object({
     id: Id, name: z.string().trim().min(1).max(200), required: z.boolean(),
+    role: z.enum(["quality", "retention", "diagnostic"]).optional(),
+    unit: z.string().trim().min(1).max(80).optional(),
+    baselineReuse: z.object({ maximumAgeSeconds: z.number().int().positive().max(31_536_000), qualification: ImmutableReleaseRefSchema }).strict().optional(),
     dataset: ImmutableReleaseRefSchema, evaluator: ImmutableReleaseRefSchema,
     executionHash: Hash, populationHash: Hash, metric: Id, direction: z.enum(["higher", "lower"]),
     minimumCoverage: z.number().positive().max(1), threshold: z.number().finite(),
     maximumRegression: z.number().finite().nonnegative(), maximumSpendUsd: z.number().finite().positive(),
-}).strict();
+}).strict().superRefine((check, ctx) => {
+    if (check.role === "diagnostic" && check.required)
+        ctx.addIssue({ code: "custom", message: "Diagnostic checks cannot be required acceptance gates." });
+});
 export const AcceptancePlanContentSchema = z.object({
     schemaVersion: z.literal("openpond.acceptancePlan.v1"), id: Id, revision: z.number().int().positive(),
     checks: z.array(AcceptanceCheckSchema).min(1).max(100),

package/dist/learning/authoring.js

@@ -1,6 +1,6 @@
 import { z } from "zod";
 import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema } from "@openpond/harness";
-import { RewardBindingSourceSchema, RewardReleaseRefSchema } from "../rewards.js";
+import { FeedbackKeySchema, RewardBindingSourceSchema, RewardReleaseRefSchema } from "../rewards.js";
 import { LearningRevisionRefSchema } from "./contracts.js";
 /** Strings intentionally preserve incomplete JSON/code until explicit publication. */
 export const RewardFixtureAuthoringFieldsSchema = z.object({
@@ -14,6 +14,7 @@ export const RewardFixtureAuthoringFieldsSchema = z.object({
 }).strict();
 export const RewardAuthoringFieldsSchema = z.object({
     name: z.string().max(500), description: z.string().max(10_000),
+    feedbackKey: FeedbackKeySchema.optional(),
     kind: z.enum(["custom_verifier", "state", "content", "schema", "artifact", "runtime_event", "model_judge", "learned_model", "human"]),
     fields: z.string(), outputField: z.string(), expectedField: z.string(), expectedValue: z.string(),
     schema: z.string(), reference: z.string(), events: z.string(), code: z.string(), exportName: z.string(), timeout: z.string(),

package/dist/learning/index.js

@@ -33,3 +33,4 @@ export * from "./nightly-schedule.js";
 export * from "./policy-inspection.js";
 export * from "./task-queue-inspection.js";
 export * from "./acceptance-plan.js";
+export * from "./acceptance-group.js";

package/dist/learning/reward-authoring.js

@@ -1,4 +1,4 @@
-import { RewardReleaseSchema, createRewardRelease } from "../rewards.js";
+import { RewardReleaseSchema, createRewardRelease, feedbackKeyForReward } from "../rewards.js";
 import { assertBoundedTaskJson } from "../task-schema.js";
 import { RewardAuthoringFieldsSchema, RewardFixtureAuthoringFieldsSchema } from "./authoring.js";
 import { createLearningTextAsset, verifyLearningTextAsset } from "./assets.js";
@@ -64,7 +64,7 @@ export function compileRewardAuthoring(input) {
     const assets = [...(asset ? [asset] : []), ...(fixtureAsset ? [fixtureAsset] : [])];
     const reward = createRewardRelease({
         schemaVersion: "openpond.rewardRelease.v1", id: input.id, revision: (base?.revision ?? 0) + 1,
-        name: fields.name, description: fields.description, implementation,
+        name: fields.name, description: fields.description, feedbackKey: fields.feedbackKey ?? feedbackKeyForReward({ id: input.id }), implementation,
         rawScore: fields.kind === "learned_model" ? { minimum: authoringNumber(fields.minimum, "Minimum score"), maximum: authoringNumber(fields.maximum, "Maximum score") } : { minimum: 0, maximum: 1 },
         assets: assets.map(value => value.asset), ...(fixtureAsset ? { fixtureSetRef: fixtureAsset.asset } : {}),
         ...(calibrationCheckRef ? { calibrationCheckRef } : {}),
@@ -84,7 +84,7 @@ export function rewardAuthoringFields(reward, sourceAsset, fixtureAsset = null)
     const fixtures = reward?.fixtureSetRef && fixtureAsset ? RewardFixtureAuthoringFieldsSchema.array().max(50).parse(JSON.parse(verifyLearningTextAsset(fixtureAsset, reward.fixtureSetRef))) : [];
     const config = implementation && "config" in implementation ? implementation.config : {};
     return {
-        name: reward?.name ?? "", description: reward?.description ?? "", kind: implementation?.kind ?? "state",
+        name: reward?.name ?? "", description: reward?.description ?? "", ...(reward?.feedbackKey ? { feedbackKey: reward.feedbackKey } : {}), kind: implementation?.kind ?? "state",
         fields: strings(config.fields).join(", ") || "answer", outputField: text(config.outputField) || "text", expectedField: text(config.expectedField) || "text", expectedValue: text(config.expectedValue),
         schema: JSON.stringify(config.jsonSchema ?? { type: "object", properties: { answer: { type: "string" } }, required: ["answer"] }, null, 2),
         reference: text(config.refIncludes), events: strings(config.requiredEvents).join(", "),

package/dist/rewards.js

@@ -5,6 +5,10 @@ import { gradeEvidence, GraderEvidenceSchema } from "./graders.js";
 import { CustomVerifierGraderSpecSchema, DeterministicGraderSpecSchema, GraderSpecSchema, HumanGraderSpecSchema, ModelJudgeGraderSpecSchema } from "./tasksets.js";
 import { assertBoundedTaskJson, validateTaskSchema } from "./task-schema.js";
 const bindingFields = { id: true, version: true, weight: true, hardGate: true, rewardEligible: true, privileged: true };
+export const FeedbackKeySchema = z.string().regex(/^[a-z][a-z0-9_]{0,79}$/);
+export function feedbackKeyForReward(reward) {
+    return reward.feedbackKey ?? `reward_${contentHash(reward.id).slice(0, 24)}`;
+}
 export const RewardImplementationSchema = z.union([
     DeterministicGraderSpecSchema.omit(bindingFields),
     ModelJudgeGraderSpecSchema.omit(bindingFields),
@@ -18,6 +22,7 @@ export const RewardReleaseContentSchema = z.object({
     revision: z.number().int().positive(),
     name: z.string().trim().min(1).max(500),
     description: z.string().max(10_000),
+    feedbackKey: FeedbackKeySchema.optional(),
     implementation: RewardImplementationSchema,
     rawScore: z.object({ minimum: z.number().finite(), maximum: z.number().finite() }).strict(),
     assets: z.array(ImmutableAssetRefSchema).max(1_000),