@openpond/evals 0.12.5 → 0.12.7

package/dist/learning/admission.js

@@ -88,7 +88,7 @@ export function taskRecordFromEvidence(evidence, definition) {
     return TaskRecordSchema.parse({
         id: evidence.id, clusterKey: evidence.submission.familyKey, split: evidence.submission.split,
         input: evidence.submission.input, expectedOutput: evidence.submission.expected,
-        policyVisibleContext: { instructions: definition.instructions },
+        policyVisibleContext: evidence.submission.policyVisibleContext ?? { instructions: definition.instructions },
         privilegedContextRef: evidence.submission.evaluatorContext === null ? null : `task-evidence:${evidence.contentHash}`,
         artifactRefs: evidence.submission.assets, tags: [definition.category],
         ...(requiredOutputs === undefined ? {} : { requiredOutputs }),

package/dist/learning/contracts.js

@@ -89,6 +89,13 @@ export const LearningReviewOriginSchema = z
     rewardBinding: LearningRevisionRefSchema,
 })
     .strict();
+/** A host must verify this retained import receipt before resolving package
+ * bytes. This portable reference is neither an approval nor an access grant. */
+export const LearningDatasetOriginSchema = z.object({
+    receiptId: ReleaseIdSchema,
+    packageHash: ReleaseHashSchema,
+    taskset: LearningRevisionRefSchema,
+}).strict();
 export const LearningSourceContentSchema = z
     .object({
     schemaVersion: z.literal("openpond.learningSource.v1"),
@@ -102,6 +109,7 @@ export const LearningSourceContentSchema = z
     mapping: TaskSourceMappingSchema.nullable(),
     adapterVersion: z.string().trim().min(1).max(200).nullable(),
     reviewOrigin: LearningReviewOriginSchema.optional(),
+    datasetOrigin: LearningDatasetOriginSchema.optional(),
     /** Host-retained authorization for materializing neutral conversation
      * evidence. The receipt is reauthorized at preparation and paid admission;
      * this portable reference alone never grants access or training consent. */
@@ -115,6 +123,8 @@ export const LearningSourceContentSchema = z
 })
     .strict()
     .superRefine((source, context) => {
+    if (source.datasetOrigin && (source.reviewOrigin || source.capturedOrigin))
+        context.addIssue({ code: "custom", path: ["datasetOrigin"], message: "Choose one retained source origin." });
     if (source.kind !== "direct" && !source.mapping)
         context.addIssue({
             code: "custom",
@@ -149,6 +159,8 @@ export const TaskExampleSubmissionSchema = z
     familyKey: ReleaseIdSchema.nullable(),
     split: TaskSplitSchema,
     input: LearningJsonObjectSchema,
+    /** Per-example policy context remains separate from evaluator-only data. */
+    policyVisibleContext: LearningJsonObjectSchema.optional(),
     observedOutput: LearningJsonObjectSchema.nullable(),
     expected: LearningJsonObjectSchema.nullable(),
     evaluatorContext: LearningJsonObjectSchema.nullable(),

package/dist/learning/iteration-dispatch-contracts.js

@@ -89,7 +89,11 @@ export const LearningIterationObservationSchema = z.object({
     evaluation: ImmutableReleaseRefSchema.nullable(),
     candidate: ImmutableReleaseRefSchema.nullable(),
     failure: z.string().max(20_000).nullable(),
+    coverage: z.array(z.object({ evidence: LearningRevisionRefSchema,
+        outcome: z.enum(["trained", "exposed", "unexposed", "unknown"]) }).strict()).max(10_000).optional(),
 }).strict().superRefine((value, context) => {
+    if (value.coverage && (!["succeeded", "failed", "cancelled"].includes(value.status) || !value.cleanupComplete || !value.receipt))
+        context.addIssue({ code: "custom", path: ["coverage"], message: "Coverage requires authoritative terminal cleanup evidence." });
     if (["succeeded", "failed", "cancelled"].includes(value.status) && (!value.cleanupComplete || !value.receipt)) {
         context.addIssue({ code: "custom", path: ["receipt"], message: "Terminal execution requires an authoritative cleanup receipt." });
     }

package/dist/learning/iteration-dispatch-worker.js

@@ -6,6 +6,7 @@ import { LearningIterationAdmissionSchema, LearningIterationDispatchSchema, Lear
 import { requireLearningRelease, requireLearningResource } from "./repository.js";
 import { deferLearningIterationSubmission } from "./iteration-dispatch-deferral.js";
 import { LearningIterationSubmissionDeferralSchema } from "./iteration-dispatch-contracts.js";
+import { settleLearningIterationExposure } from "./iteration-exposure.js";
 const closed = new Set(["accepted", "rejected", "failed", "cancelled", "completed_without_candidate"]);
 export function createLearningIterationWorker(repository, executor, options) {
     const now = options.now ?? (() => new Date().toISOString());
@@ -85,6 +86,9 @@ export function createLearningIterationWorker(repository, executor, options) {
             terminalReceipt: outcome?.receipt ?? null, leaseOwner: null, leaseExpiresAt: null, admission: null,
             nextAttemptAt: terminal ? null : new Date(Date.parse(now()) + 5_000).toISOString(), consecutiveFailures: 0, lastError: null });
         if (terminal) {
+            await settleLearningIterationExposure(tx, { chainId: current.chainId, iterationId: iteration.id,
+                batch: await requireLearningRelease(tx, "batch", current.batch), outcome,
+                unstarted: !outcome && !current.submissionStartedAt });
             const reservation = await requireLearningResource(tx, "reservation", iteration.id);
             const settled = LearningIterationReservationSchema.parse({ ...reservation, revision: reservation.revision + 1,
                 budget: { ...reservation.budget, reservedSpendUsd: 0, settledSpendUsd: updated.spendUsd, settledAt: now() }, updatedAt: now() });

package/dist/learning/iteration-eligibility.js

@@ -46,7 +46,8 @@ export async function inspectIterationEligibility(transaction, policy, options =
                     counts.excluded += 1;
                     continue;
                 }
-                if (await transaction.get("consumption", learningConsumptionId(chainId, evidence))) {
+                const consumption = await transaction.get("consumption", learningConsumptionId(chainId, evidence));
+                if (consumption && consumption.exposure !== "unexposed") {
                     counts.consumed += 1;
                     continue;
                 }

package/dist/learning/iteration-exposure.js

@@ -0,0 +1,30 @@
+import { contentHash } from "@openpond/harness";
+import { sameLearningRef } from "./contracts.js";
+import { LearningDomainError } from "./errors.js";
+import { LearningConsumptionSchema } from "./iteration-reservation-contracts.js";
+/** Called only in fenced terminal settlement. Absence/timeout is never evidence
+ * of zero exposure; unresolved rows keep their original reservation. */
+export async function settleLearningIterationExposure(tx, input) {
+    const coverage = input.outcome?.coverage;
+    if (!input.unstarted && !coverage)
+        return;
+    if (coverage && (coverage.length !== input.batch.examples.length ||
+        new Set(coverage.map(entry => contentHash(entry.evidence))).size !== coverage.length ||
+        coverage.some(entry => !input.batch.examples.some(example => sameLearningRef(example.evidence, entry.evidence)))))
+        throw new LearningDomainError("learning_execution_coverage_mismatch", 409);
+    for (const example of input.batch.examples) {
+        const id = `consumption-${contentHash([input.chainId, example.evidence.id, example.evidence.revision])}`;
+        const current = await tx.get("consumption", id);
+        if (!current || current.iterationId !== input.iterationId || !sameLearningRef(current.evidence, example.evidence))
+            throw new LearningDomainError("learning_execution_consumption_mismatch", 409);
+        const exposure = input.unstarted ? "unexposed" : coverage.find(entry => sameLearningRef(entry.evidence, example.evidence)).outcome;
+        if (exposure === "unknown")
+            continue;
+        // A terminal replay cannot downgrade already proven numerical exposure.
+        if (["trained", "exposed"].includes(current.exposure ?? "") && exposure !== current.exposure)
+            throw new LearningDomainError("learning_execution_coverage_conflict", 409);
+        const next = LearningConsumptionSchema.parse({ ...current, revision: current.revision + 1,
+            exposure, executionReceipt: input.outcome?.receipt ?? null });
+        await tx.put("consumption", next, current.revision, { parentId: input.chainId });
+    }
+}

package/dist/learning/iteration-reservation-cancellation.js

@@ -1,7 +1,8 @@
 import { LearningIterationSchema } from "./contracts.js";
 import { LearningDomainError } from "./errors.js";
 import { LearningChainSchema, LearningIterationReservationSchema } from "./iteration-reservation-contracts.js";
-import { LearningConflictError, requireLearningResource } from "./repository.js";
+import { LearningConflictError, requireLearningRelease, requireLearningResource } from "./repository.js";
+import { settleLearningIterationExposure } from "./iteration-exposure.js";
 /** Only a reservation that has never been dispatched can release spend locally.
  * Submitted executions require authoritative cancellation and settlement. */
 export async function cancelLearningIterationReservation(transaction, input, now) {
@@ -14,6 +15,9 @@ export async function cancelLearningIterationReservation(transaction, input, now
     if (iteration.trainingJob || iteration.evaluationJob || !["ready", "waiting_for_data", "waiting_for_review"].includes(iteration.status)) {
         throw new LearningDomainError("learning_execution_cancellation_required", 409);
     }
+    const dispatch = await transaction.get("dispatch", iteration.dispatchId);
+    if (dispatch?.submissionStartedAt || dispatch?.execution || dispatch?.leaseOwner)
+        throw new LearningDomainError("learning_execution_cancellation_required", 409);
     const updated = LearningIterationSchema.parse({ ...iteration, revision: iteration.revision + 1, status: "cancelled", updatedAt: now });
     const released = LearningIterationReservationSchema.parse({
         ...reservation, revision: reservation.revision + 1,
@@ -25,7 +29,12 @@ export async function cancelLearningIterationReservation(transaction, input, now
     if (chain.activeIterationId === iteration.id) {
         await transaction.put("chain", LearningChainSchema.parse({ ...chain, revision: chain.revision + 1, activeIterationId: null, updatedAt: now }), chain.revision, { parentId: chain.modelProjectId });
     }
-    // Consumption is retained. Retry/replay is an explicit operation, never a
-    // side effect of cancelling that silently makes the same examples new again.
+    // Retain the reservation history; only this transaction's fenced proof that
+    // dispatch never started makes its exact evidence eligible again.
+    if (iteration.batch)
+        await settleLearningIterationExposure(transaction, {
+            chainId: reservation.chainId, iterationId: iteration.id,
+            batch: await requireLearningRelease(transaction, "batch", iteration.batch), outcome: null, unstarted: true,
+        });
     return [{ kind: "iteration", id: updated.id, revision: updated.revision }, { kind: "reservation", id: released.id, revision: released.revision }];
 }

package/dist/learning/iteration-reservation-contracts.js

@@ -24,17 +24,19 @@ export const LearningEligibilityCountsSchema = z.object({
     excluded: z.number().int().nonnegative(),
     consumed: z.number().int().nonnegative(),
 }).strict();
-/** Immutable evidence consumption survives rejection and policy revision changes.
- * A corrected evidence revision is new data; reapproving the same revision isn't. */
+/** Reservation history is retained; authoritative terminal exposure alone may
+ * release proven-unexposed evidence. Ambiguous execution remains reserved. */
 export const LearningConsumptionSchema = z.object({
     schemaVersion: z.literal("openpond.learningConsumption.v1"),
     id: ReleaseIdSchema,
-    revision: z.literal(1),
+    revision: z.number().int().positive(),
     chainId: ReleaseIdSchema,
     iterationId: ReleaseIdSchema,
     evidence: LearningRevisionRefSchema,
     decision: LearningRevisionRefSchema,
     reservedAt: ReleaseTimestampSchema,
+    exposure: z.enum(["reserved", "trained", "exposed", "unexposed"]).optional(),
+    executionReceipt: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict().nullable().optional(),
 }).strict();
 /** Dispatch must reconcile this reservation, never reserve a second batch on retry. */
 export const LearningIterationReservationSchema = z.object({

package/dist/learning/iteration-reservation-service.js

@@ -24,6 +24,8 @@ export async function reserveLearningIteration(transaction, input, actorId, now)
         return [{ kind: "iteration", id, revision: (await requireLearningResource(transaction, "iteration", id)).revision }, { kind: "reservation", id, revision: previous.revision }];
     }
     const current = await requireLearningResource(transaction, "policy", policy.id);
+    if (policy.training.replayBatches.length)
+        throw new LearningDomainError("learning_replay_not_supported", 422);
     if (!sameLearningRef(learningRef(current), input.policy))
         throw new LearningDomainError("learning_policy_revision_stale", 409);
     if (!policy.enabled)
@@ -58,6 +60,7 @@ export async function reserveLearningIteration(transaction, input, actorId, now)
     const parent = await resolveLearningTrainingParent(transaction, policy, acceptedParent);
     let batch = null;
     if (ready) {
+        await transaction.validateTrainingReservation?.({ policy, parent: parent.trainingParentSelection });
         await assertAvailableBudget(transaction, chainId, now, policy.limits.maxIterationSpendUsd, policy.limits.maxDailySpendUsd);
         const sealed = await sealLearningBatch(transaction, {
             action: "seal_batch", operationId: input.operationId, batchId: `batch-${contentHash(id)}`,
@@ -68,11 +71,16 @@ export async function reserveLearningIteration(transaction, input, actorId, now)
         pointers.push(...sealed);
         batch = learningRef(await requireLearningResource(transaction, "batch", sealed[0].id));
         for (const item of eligibility.selected) {
+            const consumptionId = learningConsumptionId(chainId, item.evidence);
+            const prior = await transaction.get("consumption", consumptionId);
+            if (prior && prior.exposure !== "unexposed")
+                throw new LearningDomainError("learning_evidence_already_reserved", 409);
             const consumption = LearningConsumptionSchema.parse({
-                schemaVersion: "openpond.learningConsumption.v1", id: learningConsumptionId(chainId, item.evidence), revision: 1,
+                schemaVersion: "openpond.learningConsumption.v1", id: consumptionId, revision: (prior?.revision ?? 0) + 1,
                 chainId, iterationId: id, evidence: learningRef(item.evidence), decision: learningRef(item.decision), reservedAt: now,
+                exposure: "reserved", executionReceipt: null,
             });
-            await transaction.put("consumption", consumption, 0, { parentId: chainId });
+            await transaction.put("consumption", consumption, prior?.revision ?? 0, { parentId: chainId });
         }
     }
     const iteration = LearningIterationSchema.parse({

package/dist/learning/iteration-training-parent.js

@@ -29,7 +29,9 @@ export async function latestAcceptedLearningParent(tx, chainId, requireComplete
 /** Reservation captures an accepted revision before dispatch or later reviews.
  * Switching the underlying base requires a separately reviewed learning chain. */
 export async function resolveLearningTrainingParent(tx, policy, selection) {
-    if (!selection)
+    // Supervised iterations retrain the reviewed batch from the pinned base.
+    // Accepted SFT candidates remain review evidence, never a GRPO checkpoint handoff.
+    if (!selection || policy.training.method === "sft")
         return { trainingParent: policy.trainingParent, trainingParentSelection: null };
     const parent = await requireLearningResource(tx, "iteration", selection.iterationId, selection.iterationRevision);
     const sourcePolicy = await requireLearningRelease(tx, "policy", parent.policy);

package/dist/learning/reward-authoring.js

@@ -1,4 +1,4 @@
-import { RewardReleaseSchema, createRewardRelease, feedbackKeyForReward } from "../rewards.js";
+import { RewardReleaseSchema, createRewardRelease } from "../rewards.js";
 import { assertBoundedTaskJson } from "../task-schema.js";
 import { RewardAuthoringFieldsSchema, RewardFixtureAuthoringFieldsSchema } from "./authoring.js";
 import { createLearningTextAsset, verifyLearningTextAsset } from "./assets.js";
@@ -65,7 +65,9 @@ export function compileRewardAuthoring(input) {
     const assets = [...(asset ? [asset] : []), ...(fixtureAsset ? [fixtureAsset] : [])];
     const reward = createRewardRelease({
         schemaVersion: "openpond.rewardRelease.v1", id: input.id, revision: (base?.revision ?? 0) + 1,
-        name: fields.name, description: fields.description, feedbackKey: fields.feedbackKey ?? feedbackKeyForReward({ id: input.id }), implementation,
+        // Derived defaults belong to feedbackKeyForReward at read time. Adding an
+        // unauthored field here changes the immutable calibration snapshot.
+        name: fields.name, description: fields.description, ...(fields.feedbackKey === undefined ? {} : { feedbackKey: fields.feedbackKey }), implementation,
         rawScore: fields.kind === "learned_model" ? { minimum: authoringNumber(fields.minimum, "Minimum score"), maximum: authoringNumber(fields.maximum, "Maximum score") } : { minimum: 0, maximum: 1 },
         assets: assets.map(value => value.asset), ...(fixtureAsset ? { fixtureSetRef: fixtureAsset.asset } : {}),
         ...(calibrationCheckRef ? { calibrationCheckRef } : {}),

package/dist/learning/service.js

@@ -178,6 +178,8 @@ export function createLearningService(repository, options = {}) {
                 break;
             }
             case "policy": {
+                if (input.content.training.replayBatches.length)
+                    throw new LearningDomainError("learning_replay_not_supported", 422, "Replay requires an executor that materializes exact examples and weights.");
                 const definition = await requireLearningRelease(transaction, "definition", input.content.taskDefinition);
                 if (!sameLearningRef(definition.rewardBinding, input.content.rewardBinding))
                     throw new LearningDomainError("learning_policy_binding_mismatch", 422);