@openpond/evals 0.12.6 → 0.12.8

package/dist/learning/iteration-dispatch-contracts.js

@@ -89,7 +89,11 @@ export const LearningIterationObservationSchema = z.object({
     evaluation: ImmutableReleaseRefSchema.nullable(),
     candidate: ImmutableReleaseRefSchema.nullable(),
     failure: z.string().max(20_000).nullable(),
+    coverage: z.array(z.object({ evidence: LearningRevisionRefSchema,
+        outcome: z.enum(["trained", "exposed", "unexposed", "unknown"]) }).strict()).max(10_000).optional(),
 }).strict().superRefine((value, context) => {
+    if (value.coverage && (!["succeeded", "failed", "cancelled"].includes(value.status) || !value.cleanupComplete || !value.receipt))
+        context.addIssue({ code: "custom", path: ["coverage"], message: "Coverage requires authoritative terminal cleanup evidence." });
     if (["succeeded", "failed", "cancelled"].includes(value.status) && (!value.cleanupComplete || !value.receipt)) {
         context.addIssue({ code: "custom", path: ["receipt"], message: "Terminal execution requires an authoritative cleanup receipt." });
     }

package/dist/learning/iteration-dispatch-worker.js

@@ -6,6 +6,7 @@ import { LearningIterationAdmissionSchema, LearningIterationDispatchSchema, Lear
 import { requireLearningRelease, requireLearningResource } from "./repository.js";
 import { deferLearningIterationSubmission } from "./iteration-dispatch-deferral.js";
 import { LearningIterationSubmissionDeferralSchema } from "./iteration-dispatch-contracts.js";
+import { settleLearningIterationExposure } from "./iteration-exposure.js";
 const closed = new Set(["accepted", "rejected", "failed", "cancelled", "completed_without_candidate"]);
 export function createLearningIterationWorker(repository, executor, options) {
     const now = options.now ?? (() => new Date().toISOString());
@@ -85,6 +86,9 @@ export function createLearningIterationWorker(repository, executor, options) {
             terminalReceipt: outcome?.receipt ?? null, leaseOwner: null, leaseExpiresAt: null, admission: null,
             nextAttemptAt: terminal ? null : new Date(Date.parse(now()) + 5_000).toISOString(), consecutiveFailures: 0, lastError: null });
         if (terminal) {
+            await settleLearningIterationExposure(tx, { chainId: current.chainId, iterationId: iteration.id,
+                batch: await requireLearningRelease(tx, "batch", current.batch), outcome,
+                unstarted: !outcome && !current.submissionStartedAt });
             const reservation = await requireLearningResource(tx, "reservation", iteration.id);
             const settled = LearningIterationReservationSchema.parse({ ...reservation, revision: reservation.revision + 1,
                 budget: { ...reservation.budget, reservedSpendUsd: 0, settledSpendUsd: updated.spendUsd, settledAt: now() }, updatedAt: now() });

package/dist/learning/iteration-eligibility.js

@@ -46,7 +46,8 @@ export async function inspectIterationEligibility(transaction, policy, options =
                     counts.excluded += 1;
                     continue;
                 }
-                if (await transaction.get("consumption", learningConsumptionId(chainId, evidence))) {
+                const consumption = await transaction.get("consumption", learningConsumptionId(chainId, evidence));
+                if (consumption && consumption.exposure !== "unexposed") {
                     counts.consumed += 1;
                     continue;
                 }

package/dist/learning/iteration-exposure.js

@@ -0,0 +1,30 @@
+import { contentHash } from "@openpond/harness";
+import { sameLearningRef } from "./contracts.js";
+import { LearningDomainError } from "./errors.js";
+import { LearningConsumptionSchema } from "./iteration-reservation-contracts.js";
+/** Called only in fenced terminal settlement. Absence/timeout is never evidence
+ * of zero exposure; unresolved rows keep their original reservation. */
+export async function settleLearningIterationExposure(tx, input) {
+    const coverage = input.outcome?.coverage;
+    if (!input.unstarted && !coverage)
+        return;
+    if (coverage && (coverage.length !== input.batch.examples.length ||
+        new Set(coverage.map(entry => contentHash(entry.evidence))).size !== coverage.length ||
+        coverage.some(entry => !input.batch.examples.some(example => sameLearningRef(example.evidence, entry.evidence)))))
+        throw new LearningDomainError("learning_execution_coverage_mismatch", 409);
+    for (const example of input.batch.examples) {
+        const id = `consumption-${contentHash([input.chainId, example.evidence.id, example.evidence.revision])}`;
+        const current = await tx.get("consumption", id);
+        if (!current || current.iterationId !== input.iterationId || !sameLearningRef(current.evidence, example.evidence))
+            throw new LearningDomainError("learning_execution_consumption_mismatch", 409);
+        const exposure = input.unstarted ? "unexposed" : coverage.find(entry => sameLearningRef(entry.evidence, example.evidence)).outcome;
+        if (exposure === "unknown")
+            continue;
+        // A terminal replay cannot downgrade already proven numerical exposure.
+        if (["trained", "exposed"].includes(current.exposure ?? "") && exposure !== current.exposure)
+            throw new LearningDomainError("learning_execution_coverage_conflict", 409);
+        const next = LearningConsumptionSchema.parse({ ...current, revision: current.revision + 1,
+            exposure, executionReceipt: input.outcome?.receipt ?? null });
+        await tx.put("consumption", next, current.revision, { parentId: input.chainId });
+    }
+}

package/dist/learning/iteration-reservation-cancellation.js

@@ -1,7 +1,8 @@
 import { LearningIterationSchema } from "./contracts.js";
 import { LearningDomainError } from "./errors.js";
 import { LearningChainSchema, LearningIterationReservationSchema } from "./iteration-reservation-contracts.js";
-import { LearningConflictError, requireLearningResource } from "./repository.js";
+import { LearningConflictError, requireLearningRelease, requireLearningResource } from "./repository.js";
+import { settleLearningIterationExposure } from "./iteration-exposure.js";
 /** Only a reservation that has never been dispatched can release spend locally.
  * Submitted executions require authoritative cancellation and settlement. */
 export async function cancelLearningIterationReservation(transaction, input, now) {
@@ -14,6 +15,9 @@ export async function cancelLearningIterationReservation(transaction, input, now
     if (iteration.trainingJob || iteration.evaluationJob || !["ready", "waiting_for_data", "waiting_for_review"].includes(iteration.status)) {
         throw new LearningDomainError("learning_execution_cancellation_required", 409);
     }
+    const dispatch = await transaction.get("dispatch", iteration.dispatchId);
+    if (dispatch?.submissionStartedAt || dispatch?.execution || dispatch?.leaseOwner)
+        throw new LearningDomainError("learning_execution_cancellation_required", 409);
     const updated = LearningIterationSchema.parse({ ...iteration, revision: iteration.revision + 1, status: "cancelled", updatedAt: now });
     const released = LearningIterationReservationSchema.parse({
         ...reservation, revision: reservation.revision + 1,
@@ -25,7 +29,12 @@ export async function cancelLearningIterationReservation(transaction, input, now
     if (chain.activeIterationId === iteration.id) {
         await transaction.put("chain", LearningChainSchema.parse({ ...chain, revision: chain.revision + 1, activeIterationId: null, updatedAt: now }), chain.revision, { parentId: chain.modelProjectId });
     }
-    // Consumption is retained. Retry/replay is an explicit operation, never a
-    // side effect of cancelling that silently makes the same examples new again.
+    // Retain the reservation history; only this transaction's fenced proof that
+    // dispatch never started makes its exact evidence eligible again.
+    if (iteration.batch)
+        await settleLearningIterationExposure(transaction, {
+            chainId: reservation.chainId, iterationId: iteration.id,
+            batch: await requireLearningRelease(transaction, "batch", iteration.batch), outcome: null, unstarted: true,
+        });
     return [{ kind: "iteration", id: updated.id, revision: updated.revision }, { kind: "reservation", id: released.id, revision: released.revision }];
 }

package/dist/learning/iteration-reservation-contracts.js

@@ -24,17 +24,19 @@ export const LearningEligibilityCountsSchema = z.object({
     excluded: z.number().int().nonnegative(),
     consumed: z.number().int().nonnegative(),
 }).strict();
-/** Immutable evidence consumption survives rejection and policy revision changes.
- * A corrected evidence revision is new data; reapproving the same revision isn't. */
+/** Reservation history is retained; authoritative terminal exposure alone may
+ * release proven-unexposed evidence. Ambiguous execution remains reserved. */
 export const LearningConsumptionSchema = z.object({
     schemaVersion: z.literal("openpond.learningConsumption.v1"),
     id: ReleaseIdSchema,
-    revision: z.literal(1),
+    revision: z.number().int().positive(),
     chainId: ReleaseIdSchema,
     iterationId: ReleaseIdSchema,
     evidence: LearningRevisionRefSchema,
     decision: LearningRevisionRefSchema,
     reservedAt: ReleaseTimestampSchema,
+    exposure: z.enum(["reserved", "trained", "exposed", "unexposed"]).optional(),
+    executionReceipt: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict().nullable().optional(),
 }).strict();
 /** Dispatch must reconcile this reservation, never reserve a second batch on retry. */
 export const LearningIterationReservationSchema = z.object({

package/dist/learning/iteration-reservation-service.js

@@ -24,6 +24,8 @@ export async function reserveLearningIteration(transaction, input, actorId, now)
         return [{ kind: "iteration", id, revision: (await requireLearningResource(transaction, "iteration", id)).revision }, { kind: "reservation", id, revision: previous.revision }];
     }
     const current = await requireLearningResource(transaction, "policy", policy.id);
+    if (policy.training.replayBatches.length)
+        throw new LearningDomainError("learning_replay_not_supported", 422);
     if (!sameLearningRef(learningRef(current), input.policy))
         throw new LearningDomainError("learning_policy_revision_stale", 409);
     if (!policy.enabled)
@@ -58,6 +60,7 @@ export async function reserveLearningIteration(transaction, input, actorId, now)
     const parent = await resolveLearningTrainingParent(transaction, policy, acceptedParent);
     let batch = null;
     if (ready) {
+        await transaction.validateTrainingReservation?.({ policy, parent: parent.trainingParentSelection });
         await assertAvailableBudget(transaction, chainId, now, policy.limits.maxIterationSpendUsd, policy.limits.maxDailySpendUsd);
         const sealed = await sealLearningBatch(transaction, {
             action: "seal_batch", operationId: input.operationId, batchId: `batch-${contentHash(id)}`,
@@ -68,11 +71,16 @@ export async function reserveLearningIteration(transaction, input, actorId, now)
         pointers.push(...sealed);
         batch = learningRef(await requireLearningResource(transaction, "batch", sealed[0].id));
         for (const item of eligibility.selected) {
+            const consumptionId = learningConsumptionId(chainId, item.evidence);
+            const prior = await transaction.get("consumption", consumptionId);
+            if (prior && prior.exposure !== "unexposed")
+                throw new LearningDomainError("learning_evidence_already_reserved", 409);
             const consumption = LearningConsumptionSchema.parse({
-                schemaVersion: "openpond.learningConsumption.v1", id: learningConsumptionId(chainId, item.evidence), revision: 1,
+                schemaVersion: "openpond.learningConsumption.v1", id: consumptionId, revision: (prior?.revision ?? 0) + 1,
                 chainId, iterationId: id, evidence: learningRef(item.evidence), decision: learningRef(item.decision), reservedAt: now,
+                exposure: "reserved", executionReceipt: null,
             });
-            await transaction.put("consumption", consumption, 0, { parentId: chainId });
+            await transaction.put("consumption", consumption, prior?.revision ?? 0, { parentId: chainId });
         }
     }
     const iteration = LearningIterationSchema.parse({

package/dist/learning/iteration-training-parent.js

@@ -29,7 +29,9 @@ export async function latestAcceptedLearningParent(tx, chainId, requireComplete
 /** Reservation captures an accepted revision before dispatch or later reviews.
  * Switching the underlying base requires a separately reviewed learning chain. */
 export async function resolveLearningTrainingParent(tx, policy, selection) {
-    if (!selection)
+    // Supervised iterations retrain the reviewed batch from the pinned base.
+    // Accepted SFT candidates remain review evidence, never a GRPO checkpoint handoff.
+    if (!selection || policy.training.method === "sft")
         return { trainingParent: policy.trainingParent, trainingParentSelection: null };
     const parent = await requireLearningResource(tx, "iteration", selection.iterationId, selection.iterationRevision);
     const sourcePolicy = await requireLearningRelease(tx, "policy", parent.policy);

package/dist/learning/reward-authoring.js

@@ -5,6 +5,7 @@ import { createLearningTextAsset, verifyLearningTextAsset } from "./assets.js";
 import { LearningJsonObjectSchema } from "./contracts.js";
 import { LearningDomainError } from "./errors.js";
 import { HumanFormSchema } from "../human-review/contracts.js";
+import { sha256 } from "@openpond/harness";
 export const DEFAULT_REWARD_VERIFIER_SOURCE = `export function verify({ output, expectedOutput }) {
   if (!expectedOutput || !Object.hasOwn(expectedOutput, "answer")) {
     throw new Error("An expected answer is required.");
@@ -19,19 +20,19 @@ export function compileRewardAuthoring(input) {
     const base = input.base ? RewardReleaseSchema.parse(input.base) : null;
     if (base && base.id !== input.id)
         throw new LearningDomainError("reward_authoring_base_mismatch", 422);
-    const fixtureAsset = fields.fixtures?.length ? createLearningTextAsset({ text: JSON.stringify(fields.fixtures), path: "reward-fixtures.json", mediaType: "application/json", visibility: "verifier" }) : null;
+    const fixtureAsset = fields.fixtures?.length ? rewardAsset(JSON.stringify(fields.fixtures), "reward-fixtures.json", "application/json") : null;
     let asset = null;
     let implementation;
     let calibrationCheckRef;
     if (fields.kind === "custom_verifier") {
-        asset = createLearningTextAsset({ text: fields.code, path: "verifier.mjs", mediaType: "application/javascript", visibility: "verifier" });
+        asset = rewardAsset(fields.code, "verifier.mjs", "application/javascript");
         implementation = { kind: fields.kind, verifierRef: asset.asset, exportName: fields.exportName, timeoutMs: authoringNumber(fields.timeout, "Time limit"), networkPolicy: "none",
             ...((fields.verifierRuntime ?? (base?.implementation.kind === "custom_verifier" ? base.implementation.runtime : undefined)) ? { runtime: fields.verifierRuntime ?? (base?.implementation.kind === "custom_verifier" ? base.implementation.runtime : undefined) } : {}) };
     }
     else if (fields.kind === "model_judge" || fields.kind === "human") {
         if (!fields.rubric.trim())
             throw new LearningDomainError("reward_rubric_required", 422, "Write the rubric before checking or publishing this Reward.");
-        asset = createLearningTextAsset({ text: fields.rubric, path: "rubric.md", mediaType: "text/markdown", visibility: "verifier" });
+        asset = rewardAsset(fields.rubric, "rubric.md", "text/markdown");
         if (fields.kind === "human")
             implementation = { kind: fields.kind, rubricRef: asset.asset, reviewerRole: fields.reviewerRole, form: HumanFormSchema.parse(JSON.parse(fields.humanForm || JSON.stringify({ schemaVersion: "openpond.humanForm.v1", mode: "individual", instructions: fields.rubric, criteria: [{ id: "quality", label: "Quality", instructions: fields.rubric, required: true, allowAbstain: true, kind: "score", minimum: 0, maximum: 1, step: 0.1 }] }))) };
         else {
@@ -49,7 +50,7 @@ export function compileRewardAuthoring(input) {
     }
     else if (fields.kind === "learned_model") {
         parseRewardAuthoringObject(fields.inputContract, "Model input contract");
-        asset = createLearningTextAsset({ text: fields.inputContract, path: "input-contract.json", mediaType: "application/json", visibility: "verifier" });
+        asset = rewardAsset(fields.inputContract, "input-contract.json", "application/json");
         implementation = { kind: fields.kind, modelVersion: { id: fields.learnedId, contentHash: fields.learnedHash }, inputContract: asset.asset };
     }
     else if (fields.kind === "state")
@@ -74,6 +75,11 @@ export function compileRewardAuthoring(input) {
     });
     return { reward, assets };
 }
+// Training and evaluation Rewards (including different revisions of one Reward)
+// share a private asset directory. Different bytes must never claim one path.
+function rewardAsset(text, filename, mediaType) {
+    return createLearningTextAsset({ text, path: `reward-assets/${sha256(text)}/${filename}`, mediaType, visibility: "verifier" });
+}
 export function rewardAuthoringFields(reward, sourceAsset, fixtureAsset = null) {
     const implementation = reward?.implementation;
     const ref = implementation && "verifierRef" in implementation ? implementation.verifierRef

package/dist/learning/service.js

@@ -178,6 +178,8 @@ export function createLearningService(repository, options = {}) {
                 break;
             }
             case "policy": {
+                if (input.content.training.replayBatches.length)
+                    throw new LearningDomainError("learning_replay_not_supported", 422, "Replay requires an executor that materializes exact examples and weights.");
                 const definition = await requireLearningRelease(transaction, "definition", input.content.taskDefinition);
                 if (!sameLearningRef(definition.rewardBinding, input.content.rewardBinding))
                     throw new LearningDomainError("learning_policy_binding_mismatch", 422);