@openpond/evals 0.10.9 → 0.11.0

package/dist/learning/reward-calibration.js

@@ -0,0 +1,66 @@
+import { contentHash } from "@openpond/harness";
+import { createRewardRelease } from "../rewards.js";
+import { learningRef, sameLearningRef } from "./contracts.js";
+import { LearningDomainError } from "./errors.js";
+import { requireLearningRelease, requireLearningResource } from "./repository.js";
+import { compileRewardCheck, matchRewardFixture } from "./reward-checks.js";
+/** Qualification is tied to the exact private draft, fixtures and worker-owned
+ * provider receipts. Publishing does not rewrite or re-run those checks. */
+export function qualifyRewardCheck(draft, base, check) {
+    const compiled = compileRewardCheck(draft, base);
+    const implementation = compiled.reward.implementation;
+    if (implementation.kind !== "model_judge" || !implementation.model)
+        throw new LearningDomainError("reward_calibration_model_required", 422);
+    if (!sameLearningRef(check.draft, learningRef(draft)) || !sameLearningRef(check.reward, learningRef(compiled.reward))
+        || check.snapshotHash !== compiled.snapshotHash || contentHash(check.fixtureRefs) !== contentHash(compiled.fixtureRefs))
+        throw new LearningDomainError("reward_calibration_snapshot_mismatch", 422);
+    if (check.status !== "completed" || !check.runtime || check.matchesExpectations !== true || check.failure
+        || check.results.length !== compiled.fixtures.length)
+        throw new LearningDomainError("reward_calibration_check_incomplete", 422);
+    if (!compiled.fixtures.some(fixture => fixture.expected.status === "scored" && fixture.expected.passed === true)
+        || !compiled.fixtures.some(fixture => fixture.expected.status === "scored" && fixture.expected.passed === false)) {
+        throw new LearningDomainError("reward_calibration_coverage_required", 422, "Check both a passing and a failing example before publishing a calibrated judge.");
+    }
+    for (const fixture of compiled.fixtures) {
+        const results = check.results.filter(result => result.fixture.id === fixture.id);
+        const retained = results[0];
+        if (results.length !== 1 || !retained || retained.fixture.contentHash !== contentHash(fixture)
+            || !sameLearningRef(retained.result.reward, check.reward) || !matchRewardFixture(fixture, retained.result).matchesExpectation)
+            throw new LearningDomainError("reward_calibration_fixture_mismatch", 422);
+        if (fixture.expected.status !== "scored")
+            continue;
+        const receipt = retained.result.graderEvidence?.modelJudgeReceipt;
+        const call = check.judgeCalls?.find(call => call.id === `fixture-${contentHash([fixture, compiled.reward.contentHash])}`);
+        if (!receipt || receipt.providerId !== implementation.model.providerId || receipt.modelId !== implementation.model.modelId
+            || (implementation.model.revision !== null && receipt.modelRevision !== implementation.model.revision)
+            || call?.status !== "settled" || !call.response || call.requestHash !== receipt.requestHash
+            || contentHash(call.response) !== receipt.responseHash)
+            throw new LearningDomainError("reward_calibration_provider_receipt_mismatch", 422);
+    }
+    const { contentHash: _hash, ...content } = compiled.reward;
+    return { ...compiled, reward: createRewardRelease({ ...content, implementation: { ...implementation, calibrationStatus: "passed" },
+            calibrationCheckRef: { id: check.id, revision: check.revision, contentHash: contentHash(check) } }) };
+}
+/** Generic publication cannot manufacture a passed flag, including by copying
+ * a check from a different rubric, model, temperature or fixture set. */
+export async function assertRewardCalibration(tx, reward) {
+    if (reward.implementation.kind !== "model_judge" || reward.implementation.calibrationStatus !== "passed") {
+        if (reward.calibrationCheckRef)
+            throw new LearningDomainError("reward_calibration_reference_invalid", 422);
+        return;
+    }
+    const reference = reward.calibrationCheckRef;
+    if (!reference)
+        throw new LearningDomainError("reward_calibration_check_required", 422);
+    const check = await requireLearningResource(tx, "reward_check", reference.id, reference.revision);
+    if (contentHash(check) !== reference.contentHash)
+        throw new LearningDomainError("reward_calibration_check_changed", 422);
+    const draft = await requireLearningRelease(tx, "draft", check.draft);
+    if (draft.targetKind !== "reward")
+        throw new LearningDomainError("reward_calibration_draft_kind_invalid", 422);
+    const base = draft.baseRelease ? await requireLearningRelease(tx, "reward", draft.baseRelease) : null;
+    const qualified = qualifyRewardCheck(draft, base, check).reward;
+    const scoring = (value) => contentHash([value.implementation, value.rawScore, value.assets, value.fixtureSetRef ?? null]);
+    if (reward.id !== qualified.id || scoring(reward) !== scoring(qualified))
+        throw new LearningDomainError("reward_calibration_configuration_changed", 422);
+}

package/dist/learning/reward-check-budget.js

@@ -0,0 +1,27 @@
+import { contentHash } from "@openpond/harness";
+import { requireLearningResource } from "./repository.js";
+import { RewardCheckRunSchema } from "./reward-checks.js";
+/** The same persisted run owns fixture results and budget reservations. A late
+ * provider settlement is retained even after its worker loses the lease. */
+export function createRewardCheckJudgeBudgetStore(input) {
+    const now = input.now ?? (() => new Date().toISOString());
+    return {
+        transaction(intent, update) {
+            return input.repository.transaction(input.scope, async (tx) => {
+                const current = await requireLearningResource(tx, "reward_check", input.run.id);
+                if (current.snapshotHash !== input.run.snapshotHash)
+                    throw new Error("model_judge_budget_snapshot_changed");
+                if (intent === "dispatch" && (current.status !== "running" || !current.leaseOwner || current.leaseOwner !== input.run.leaseOwner
+                    || current.attemptCount !== input.run.attemptCount || !current.leaseExpiresAt || Date.parse(current.leaseExpiresAt) <= Date.parse(now())))
+                    throw new Error("model_judge_budget_owner_inactive");
+                const calls = current.judgeCalls ?? [];
+                const next = update({ maximumSpendUsd: current.maximumSpendUsd, calls });
+                if (contentHash(next.calls) !== contentHash(calls)) {
+                    const updated = RewardCheckRunSchema.parse({ ...current, revision: current.revision + 1, judgeCalls: next.calls, updatedAt: now() });
+                    await tx.put("reward_check", updated, current.revision, { parentId: current.reward.id, status: current.status });
+                }
+                return next.result;
+            });
+        },
+    };
+}

package/dist/learning/reward-check-service.js

@@ -3,7 +3,7 @@ import { currentDraft } from "./authoring-service.js";
 import { learningRef } from "./contracts.js";
 import { compileRewardCheck, RewardCheckRunSchema } from "./reward-checks.js";
 import { LearningConflictError, requireLearningRelease, requireLearningResource } from "./repository.js";
-export async function queueRewardCheck(tx, input, operationId, now) {
+export async function queueRewardCheck(tx, input, operationId, now, requestedBy) {
     const draft = await currentDraft(tx, input.draft);
     if (draft.targetKind !== "reward")
         throw new LearningDomainError("reward_check_draft_kind_invalid", 422);
@@ -13,7 +13,7 @@ export async function queueRewardCheck(tx, input, operationId, now) {
         schemaVersion: "openpond.rewardCheckRun.v1", id: `reward-check-${operationId}`, revision: 1,
         draft: learningRef(draft), reward: learningRef(compiled.reward), snapshotHash: compiled.snapshotHash,
         fixtureRefs: compiled.fixtureRefs, status: "queued", runtime: null, results: [], matchesExpectations: null,
-        timeoutMs: input.timeoutMs, maximumSpendUsd: input.maximumSpendUsd,
+        timeoutMs: input.timeoutMs, maximumSpendUsd: input.maximumSpendUsd, requestedBy,
         leaseOwner: null, leaseExpiresAt: null, attemptCount: 0, failure: null, createdAt: now, updatedAt: now,
     });
     await tx.put("reward_check", check, 0, { parentId: draft.targetId, status: check.status });

package/dist/learning/reward-check-worker.js

@@ -17,7 +17,7 @@ export function createRewardCheckWorker(repository, executor, options) {
             const claimed = RewardCheckRunSchema.parse({ ...current, revision: current.revision + 1,
                 status: current.status === "cancelling" ? "cancelling" : "running", runtime: current.status === "cancelling" ? current.runtime : runtime,
                 leaseOwner: options.workerId, leaseExpiresAt: new Date(Date.parse(now()) + current.timeoutMs + 30_000).toISOString(),
-                attemptCount: current.attemptCount + 1, results: current.status === "cancelling" ? current.results : [], matchesExpectations: null, failure: null, updatedAt: now(),
+                attemptCount: current.attemptCount + 1, results: current.results, matchesExpectations: null, failure: null, updatedAt: now(),
             });
             await tx.put("reward_check", claimed, current.revision, { parentId: claimed.reward.id, status: claimed.status });
             return { acquired: true, run: claimed };
@@ -44,8 +44,16 @@ export function createRewardCheckWorker(repository, executor, options) {
             });
             for (const fixture of compiled.fixtures) {
                 controller.signal.throwIfAborted();
+                const previous = claim.run.results.find(result => result.fixture.id === fixture.id);
+                if (previous) {
+                    if (previous.fixture.contentHash !== contentHash(fixture) || !sameLearningRef(previous.result.reward, claim.run.reward)
+                        || previous.result.graderId !== compiled.reward.id || previous.result.role !== "evaluation")
+                        throw new Error("reward_check_retained_fixture_mismatch");
+                    continue;
+                }
                 const result = BoundRewardResultSchema.parse(await executor.execute({ ...compiled, scope, run: claim.run, fixture, signal: controller.signal }));
-                controller.signal.throwIfAborted();
+                // Retain a settled execution before observing cancellation. Its provider
+                // cost and evidence already exist even if no more fixtures should run.
                 if (result.graderId !== compiled.reward.id || !sameLearningRef(result.reward, claim.run.reward) || result.role !== "evaluation")
                     throw new Error("reward_check_result_identity_mismatch");
                 const retained = await repository.transaction(scope, async (tx) => {

package/dist/learning/reward-checks.js

@@ -5,6 +5,7 @@ import { TaskRecordSchema } from "../tasksets.js";
 import { LearningJsonObjectSchema, LearningRevisionRefSchema, learningRef } from "./contracts.js";
 import { authoringNumber, compileRewardAuthoring, parseRewardAuthoringObject } from "./reward-authoring.js";
 import { LearningDomainError } from "./errors.js";
+import { JudgeCallReservationSchema } from "./judge-budget.js";
 export const RewardFixtureSchema = z.object({
     id: ReleaseIdSchema, name: z.string().trim().min(1).max(500),
     input: LearningJsonObjectSchema, output: LearningJsonObjectSchema,
@@ -32,6 +33,8 @@ export const RewardCheckRunSchema = z.object({
     runtime: RewardCheckRuntimeSchema.nullable(), results: z.array(RewardFixtureCheckResultSchema).max(50),
     matchesExpectations: z.boolean().nullable(), failure: z.string().max(20_000).nullable(),
     timeoutMs: z.number().int().min(100).max(300_000), maximumSpendUsd: z.number().nonnegative().max(1_000),
+    judgeCalls: z.array(JudgeCallReservationSchema).max(1_000).optional(),
+    requestedBy: ReleaseIdSchema.optional(),
     leaseOwner: ReleaseIdSchema.nullable(), leaseExpiresAt: ReleaseTimestampSchema.nullable(), attemptCount: z.number().int().nonnegative(),
     createdAt: ReleaseTimestampSchema, updatedAt: ReleaseTimestampSchema,
 }).strict();
@@ -69,7 +72,7 @@ export async function executeRewardFixture(input) {
     }, [input.reward]);
     const composition = await executeRewardBinding({ binding, rewards: [input.reward], task,
         evidence: { output: fixture.output, artifactRefs: fixture.artifactRefs, runtimeEventRefs: fixture.runtimeEventRefs, infrastructureError: fixture.infrastructureError },
-        signal: input.signal, customVerifier: input.customVerifier, modelJudge: input.modelJudge,
+        signal: input.signal, customVerifier: input.customVerifier, modelJudge: input.modelJudge, purpose: "fixture_calibration",
     });
     return composition.results[0];
 }

package/dist/learning/reward-fixture-executor.js

@@ -4,9 +4,10 @@ import { executeRewardFixture } from "./reward-checks.js";
 export function createIsolatedRewardFixtureExecutor(options) {
     return {
         runtime: options.runtime,
-        execute(input) {
+        async execute(input) {
             return executeRewardFixture({
                 reward: input.reward, fixture: input.fixture, signal: input.signal,
+                modelJudge: input.reward.implementation.kind === "model_judge" ? await options.createModelJudge?.(input) : undefined,
                 customVerifier: async ({ grader, task, evidence }) => {
                     const asset = input.assets.find(asset => asset.id === grader.verifierRef.id);
                     if (!asset)
@@ -24,6 +25,6 @@ export function createIsolatedRewardFixtureExecutor(options) {
             });
         },
         // No remote allocation; executeJavaScript's settlement contract confirms cleanup.
-        async cancel() { return true; },
+        async cancel(input) { return options.createModelJudge ? options.cancelModelJudge?.(input) ?? false : true; },
     };
 }

package/dist/learning/schedule-contracts.js

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema } from "@openpond/harness";
 import { LearningRevisionRefSchema } from "./contracts.js";
+import { NightlyScheduleSchema } from "./nightly-schedule.js";
 /** The durable timer is operational state; every fire retains its own policy. */
 export const LearningScheduleSchema = z.object({
     schemaVersion: z.literal("openpond.learningSchedule.v1"),
@@ -11,6 +12,7 @@ export const LearningScheduleSchema = z.object({
     executionOwner: z.enum(["local", "hosted"]),
     state: z.enum(["scheduled", "disabled", "blocked"]),
     intervalSeconds: z.number().int().min(60).max(31_536_000).nullable(),
+    calendar: NightlyScheduleSchema.nullable().default(null),
     nextRunAt: ReleaseTimestampSchema.nullable(),
     lastFire: LearningRevisionRefSchema.nullable(),
     consecutiveFailures: z.number().int().nonnegative(),
@@ -19,10 +21,12 @@ export const LearningScheduleSchema = z.object({
     createdAt: ReleaseTimestampSchema,
     updatedAt: ReleaseTimestampSchema,
 }).strict().superRefine((value, context) => {
-    if (value.state !== "disabled" && (!value.intervalSeconds || !value.nextRunAt))
+    if (value.state !== "disabled" && ((!value.intervalSeconds && !value.calendar) || !value.nextRunAt))
         context.addIssue({ code: "custom", path: ["nextRunAt"], message: "An active timer must retain its cadence and due time." });
     if (value.state === "disabled" && value.nextRunAt)
         context.addIssue({ code: "custom", path: ["nextRunAt"], message: "A disabled timer cannot have a pending occurrence." });
+    if (value.intervalSeconds && value.calendar)
+        context.addIssue({ code: "custom", path: ["calendar"], message: "A timer uses either an interval or a nightly calendar." });
 });
 /** Immutable occurrence evidence survives timer edits and coalesced downtime. */
 export const LearningScheduleFireSchema = z.object({

package/dist/learning/schedule-service.js

@@ -2,7 +2,8 @@ import { contentHash } from "@openpond/harness";
 import { assertLearningContentHash, learningRef, sameLearningRef } from "./contracts.js";
 import { LearningScheduleSchema } from "./schedule-contracts.js";
 import { LearningDomainError } from "./errors.js";
-import { requireLearningResource } from "./repository.js";
+import { requireLearningRelease, requireLearningResource } from "./repository.js";
+import { nextNightlyOccurrence } from "./nightly-schedule.js";
 export function learningScheduleId(policyId) { return `schedule-${contentHash(policyId)}`; }
 /** Called atomically with policy publication. No external timer or Work is made. */
 export async function synchronizeLearningSchedule(tx, policy, now) {
@@ -14,15 +15,18 @@ export async function synchronizeLearningSchedule(tx, policy, now) {
     const previous = await tx.get("schedule", id);
     if (previous && sameLearningRef(previous.policy, learningRef(policy)))
         return previous;
-    const intervalSeconds = policy.trigger.kind === "schedule" ? policy.trigger.intervalSeconds : null;
-    const enabled = policy.enabled && policy.automation.train && intervalSeconds !== null;
+    const intervalSeconds = policy.trigger.kind === "schedule" ? policy.trigger.intervalSeconds : policy.trigger.kind === "approved_count" ? 60 : null;
+    const calendar = policy.trigger.kind === "nightly" ? { localTime: policy.trigger.localTime, timeZone: policy.trigger.timeZone } : null;
+    const enabled = policy.enabled && policy.automation.train && (intervalSeconds !== null || calendar !== null);
+    const previousPolicy = previous ? await requireLearningRelease(tx, "policy", previous.policy) : null;
     const preserveDue = previous?.nextRunAt && previous.intervalSeconds === intervalSeconds
+        && JSON.stringify(previous.calendar) === JSON.stringify(calendar) && previousPolicy?.trigger.kind === policy.trigger.kind
         && previous.executionOwner === policy.executionOwner;
     const schedule = LearningScheduleSchema.parse({
         schemaVersion: "openpond.learningSchedule.v1", id, revision: (previous?.revision ?? 0) + 1,
         policy: learningRef(policy), modelProjectId: policy.modelProjectId, executionOwner: policy.executionOwner,
-        state: enabled ? "scheduled" : "disabled", intervalSeconds,
-        nextRunAt: enabled ? preserveDue ? previous.nextRunAt : new Date(Date.parse(now) + intervalSeconds * 1_000).toISOString() : null,
+        state: enabled ? "scheduled" : "disabled", intervalSeconds, calendar,
+        nextRunAt: enabled ? preserveDue ? previous.nextRunAt : calendar ? nextNightlyOccurrence(calendar, now) : new Date(Date.parse(now) + intervalSeconds * 1_000).toISOString() : null,
         lastFire: previous?.lastFire ?? null, consecutiveFailures: 0, nextAttemptAt: null, lastError: null,
         createdAt: previous?.createdAt ?? now, updatedAt: now,
     });

package/dist/learning/schedule-worker.js

@@ -4,6 +4,8 @@ import { LearningDomainError } from "./errors.js";
 import { reserveLearningIteration } from "./iteration-reservation-service.js";
 import { LearningScheduleFireSchema, LearningScheduleSchema } from "./schedule-contracts.js";
 import { requireLearningRelease, requireLearningResource } from "./repository.js";
+import { inspectIterationEligibility } from "./iteration-eligibility.js";
+import { coalesceNightlyOccurrences } from "./nightly-schedule.js";
 const skipReasons = new Map([
     ["learning_iteration_active", "active_iteration"], ["learning_iteration_cooldown", "cooldown"],
     ["learning_daily_budget_exhausted", "daily_budget"],
@@ -28,20 +30,33 @@ export function createLearningScheduleWorker(repository, options) {
                 attempt = { schedule, maxRetries: 0 };
                 const policy = await requireLearningRelease(tx, "policy", schedule.policy);
                 attempt.maxRetries = policy.limits.maxRetries;
-                if (policy.executionOwner !== schedule.executionOwner || policy.trigger.kind !== "schedule"
-                    || policy.trigger.intervalSeconds !== schedule.intervalSeconds)
+                const matches = policy.trigger.kind === "nightly" ? schedule.intervalSeconds === null && schedule.calendar?.localTime === policy.trigger.localTime && schedule.calendar.timeZone === policy.trigger.timeZone
+                    : policy.trigger.kind === "schedule" ? schedule.calendar === null && policy.trigger.intervalSeconds === schedule.intervalSeconds
+                        : policy.trigger.kind === "approved_count" && schedule.calendar === null && schedule.intervalSeconds === 60;
+                if (policy.executionOwner !== schedule.executionOwner || !matches)
                     throw new LearningDomainError("learning_schedule_policy_mismatch", 409);
                 const scheduledAt = schedule.nextRunAt;
-                const interval = schedule.intervalSeconds * 1_000;
+                const interval = (schedule.intervalSeconds ?? 86_400) * 1_000;
                 const due = Date.parse(scheduledAt);
                 const coalescedCount = Math.floor((time - due) / interval);
+                const occurrence = schedule.calendar ? coalesceNightlyOccurrences(schedule.calendar, scheduledAt, timestamp) : {
+                    coalescedCount, coalescedThroughAt: new Date(due + coalescedCount * interval).toISOString(), nextRunAt: new Date(due + (coalescedCount + 1) * interval).toISOString(),
+                };
+                // Below-threshold count checks advance their timer without filling the
+                // Model's history with empty iterations or consuming any evidence.
+                if (policy.trigger.kind === "approved_count" && (await inspectIterationEligibility(tx, policy)).counts.eligible < policy.admission.minimumApprovedExamples) {
+                    const updated = LearningScheduleSchema.parse({ ...schedule, revision: schedule.revision + 1, nextRunAt: occurrence.nextRunAt,
+                        consecutiveFailures: 0, nextAttemptAt: null, lastError: null, updatedAt: timestamp });
+                    await tx.put("schedule", updated, schedule.revision, { parentId: schedule.modelProjectId, status: updated.state });
+                    return updated;
+                }
                 const id = `schedule-fire-${contentHash([schedule.id, scheduledAt])}`;
                 let outcome = "skipped";
                 let reason = null;
                 let iteration = null;
                 try {
                     const pointers = await reserveLearningIteration(tx, { action: "reserve_iteration", operationId: id,
-                        policy: schedule.policy, trigger: { kind: "schedule", scheduledAt } }, actorId, timestamp);
+                        policy: schedule.policy, trigger: policy.trigger.kind === "approved_count" ? { kind: "approved_count", checkedAt: scheduledAt } : { kind: "schedule", scheduledAt } }, actorId, timestamp);
                     const pointer = pointers.find(value => value.kind === "iteration");
                     if (!pointer)
                         throw new LearningDomainError("learning_schedule_reservation_missing", 409);
@@ -59,12 +74,12 @@ export function createLearningScheduleWorker(repository, options) {
                 }
                 const fire = LearningScheduleFireSchema.parse(sealLearningContent({
                     schemaVersion: "openpond.learningScheduleFire.v1", id, revision: 1, scheduleId,
-                    policy: schedule.policy, scheduledAt, coalescedThroughAt: new Date(due + coalescedCount * interval).toISOString(),
-                    coalescedCount, outcome, reason, iteration, createdAt: timestamp,
+                    policy: schedule.policy, scheduledAt, coalescedThroughAt: occurrence.coalescedThroughAt,
+                    coalescedCount: occurrence.coalescedCount, outcome, reason, iteration, createdAt: timestamp,
                 }));
                 await tx.put("schedule_fire", fire, 0, { parentId: schedule.id, status: outcome });
                 const updated = LearningScheduleSchema.parse({ ...schedule, revision: schedule.revision + 1,
-                    nextRunAt: new Date(due + (coalescedCount + 1) * interval).toISOString(), lastFire: learningRef(fire),
+                    nextRunAt: occurrence.nextRunAt, lastFire: learningRef(fire),
                     consecutiveFailures: 0, nextAttemptAt: null, lastError: null, updatedAt: timestamp });
                 await tx.put("schedule", updated, schedule.revision, { parentId: schedule.modelProjectId, status: updated.state });
                 return updated;

package/dist/learning/service.js

@@ -4,10 +4,13 @@ import { reserveLearningIteration } from "./iteration-reservation-service.js";
 import { synchronizeLearningSchedule } from "./schedule-service.js";
 import { inspectLearningPolicy } from "./policy-inspection.js";
 import { requireCurrentLearningEvidence as currentEvidence, sealLearningBatch } from "./batch-service.js";
-import { saveAuthoringDraft, archiveAuthoringDraft, finalizeAuthoringDraft } from "./authoring-service.js";
+import { saveAuthoringDraft, archiveAuthoringDraft, finalizeAuthoringDraft, currentDraft } from "./authoring-service.js";
+import { assertRewardCalibration, qualifyRewardCheck } from "./reward-calibration.js";
 import { queueRewardCheck, cancelRewardCheck } from "./reward-check-service.js";
 import { LearningDomainError } from "./errors.js";
 import { contentHash } from "@openpond/harness";
+import { inspectLearningTaskQueue } from "./task-queue-inspection.js";
+import { LearningReadRequestSchema } from "./transport.js";
 import { createRewardBinding, createRewardRelease } from "../rewards.js";
 import { assertBoundedTaskJson, validateTaskValue } from "../task-schema.js";
 import { assertAdmissionDecision, assertGradeIdentity, inspectTaskEvidence } from "./admission.js";
@@ -36,6 +39,29 @@ export function createLearningService(repository, options = {}) {
             }
             let pointers;
             switch (input.action) {
+                case "publish_checked_reward": {
+                    const draft = await currentDraft(transaction, input.draft);
+                    if (draft.targetKind !== "reward")
+                        throw new LearningDomainError("reward_calibration_draft_kind_invalid", 422);
+                    const base = draft.baseRelease ? await requireLearningRelease(transaction, "reward", draft.baseRelease) : null;
+                    const check = await requireLearningResource(transaction, "reward_check", input.checkId, input.checkRevision);
+                    const qualified = qualifyRewardCheck(draft, base, check);
+                    const { contentHash: _rewardHash, ...content } = qualified.reward;
+                    const publication = { action: "publish_resources", operationId: input.operationId,
+                        finalizeDraft: { draft: input.draft, targetKind: "reward", release: learningRef(qualified.reward) },
+                        resources: [
+                            ...qualified.assets.map(({ contentHash: _assetHash, ...content }) => ({ kind: "asset", expectedRevision: 0, content })),
+                            { kind: "reward", expectedRevision: base?.revision ?? 0, content },
+                        ],
+                    };
+                    pointers = [];
+                    for (const resource of publication.resources)
+                        pointers.push(await publish(transaction, { ...resource, action: "publish", operationId: input.operationId }));
+                    const finalized = await finalizeAuthoringDraft(transaction, publication, pointers, now());
+                    if (finalized)
+                        pointers.push(finalized);
+                    break;
+                }
                 case "import_intake":
                     pointers = await importTaskIntake(transaction, input, now(), submit);
                     break;
@@ -50,7 +76,7 @@ export function createLearningService(repository, options = {}) {
                     pointers = await reserveLearningIteration(transaction, input, context.actor.id, now());
                     break;
                 case "queue_reward_check":
-                    pointers = [await queueRewardCheck(transaction, input, operationId, now())];
+                    pointers = [await queueRewardCheck(transaction, input, operationId, now(), context.actor.id)];
                     break;
                 case "cancel_reward_check":
                     pointers = [await cancelRewardCheck(transaction, input, now())];
@@ -83,7 +109,7 @@ export function createLearningService(repository, options = {}) {
                     pointers = [await resolveFeedback(transaction, input, context.actor.id)];
                     break;
                 case "queue_grade":
-                    pointers = [await queueGrade(transaction, input, operationId)];
+                    pointers = [await queueGrade(transaction, input, operationId, context.actor.id)];
                     break;
                 case "cancel_grade":
                     pointers = [await cancelGrade(transaction, input)];
@@ -118,6 +144,7 @@ export function createLearningService(repository, options = {}) {
             }
             case "reward": {
                 resource = createRewardRelease(input.content);
+                await assertRewardCalibration(transaction, resource);
                 const implementation = input.content.implementation;
                 const references = [
                     ...input.content.assets,
@@ -282,7 +309,7 @@ export function createLearningService(repository, options = {}) {
         await transaction.put("feedback", resolved, feedback.revision, { parentId: learningEvidenceId(feedback.submission.sourceId, feedback.submission.exampleId, feedback.submission.attemptId), status: resolved.status });
         return pointer("feedback", resolved);
     }
-    async function queueGrade(transaction, input, operationId) {
+    async function queueGrade(transaction, input, operationId, requestedBy) {
         const evidence = await currentEvidence(transaction, input.evidence);
         const definition = await requireLearningRelease(transaction, "definition", evidence.submission.taskDefinition);
         if (!inspectTaskEvidence(evidence, definition).taskReady)
@@ -296,7 +323,7 @@ export function createLearningService(repository, options = {}) {
             schemaVersion: "openpond.taskGradeRun.v1", id: `grade-${operationId}`, revision: 1,
             evidence: learningRef(evidence), binding: definition.rewardBinding, target: input.target, output,
             status: "queued", composition: null, leaseOwner: null, leaseExpiresAt: null, attemptCount: 0,
-            timeoutMs: input.timeoutMs, maximumSpendUsd: input.maximumSpendUsd, failure: null, createdAt: now(), updatedAt: now(),
+            timeoutMs: input.timeoutMs, maximumSpendUsd: input.maximumSpendUsd, requestedBy, failure: null, createdAt: now(), updatedAt: now(),
         });
         await transaction.put("grade", grade, 0, { parentId: evidence.id, status: grade.status });
         return pointer("grade", grade);
@@ -335,6 +362,13 @@ export function createLearningService(repository, options = {}) {
     }
     return {
         command,
+        async inspectTaskQueue(context, modelProjectId = null) {
+            authorizeRead(context);
+            const input = LearningReadRequestSchema.parse({ action: "inspect_task_queue", scope: context.scope, modelProjectId });
+            if (input.action !== "inspect_task_queue")
+                throw new LearningDomainError("learning_request_invalid", 400);
+            return repository.transaction(context.scope, transaction => inspectLearningTaskQueue(transaction, input.modelProjectId, now()));
+        },
         async inspectPolicy(context, reference) {
             authorizeRead(context);
             const ref = LearningRevisionRefSchema.parse(reference);
@@ -382,7 +416,7 @@ function authorize(context, input) {
         throw new LearningDomainError("learning_scope_required", 400);
     const publications = input.action === "publish" ? [input] : input.action === "publish_resources" ? input.resources : [];
     if (context.actor.role !== "reviewer" && publications.some(value => value.kind === "policy" && value.content.enabled
-        && value.content.automation.train && value.content.trigger.kind === "schedule"))
+        && value.content.automation.train && value.content.trigger.kind !== "manual"))
         throw new LearningDomainError("learning_review_not_authorized", 403);
     if (context.actor.role === "source") {
         const sourceId = input.action === "submit_example" ? input.example.sourceId : input.action === "submit_feedback" ? input.feedback.sourceId : null;

package/dist/learning/task-queue-inspection.js

@@ -0,0 +1,48 @@
+import { ReleaseIdSchema, ReleaseTimestampSchema } from "@openpond/harness";
+import { z } from "zod";
+import { LearningDomainError } from "./errors.js";
+import { inspectIterationEligibility } from "./iteration-eligibility.js";
+import { taskFamilyReservations } from "./admission.js";
+import { requireLearningRelease } from "./repository.js";
+export const LearningTaskQueueInspectionSchema = z.object({
+    modelProjectId: ReleaseIdSchema.nullable(),
+    inspectedAt: ReleaseTimestampSchema,
+    pendingTrainingCount: z.number().int().nonnegative().nullable(),
+    issues: z.array(z.object({ policyId: ReleaseIdSchema, code: z.string(), message: z.string() }).strict()),
+}).strict();
+/** Count exact eligible, unconsumed attempts once across selected policies.
+ * Paused policies still have pending work; review-only and held-out tasks do not. */
+export async function inspectLearningTaskQueue(transaction, modelProjectId, now) {
+    const eligible = new Set();
+    const issues = [];
+    let afterId;
+    do {
+        const page = await transaction.list("policy", { limit: 100, ...(modelProjectId ? { parentId: modelProjectId } : {}), ...(afterId ? { afterId } : {}) });
+        for (const policy of page.items) {
+            try {
+                const inspection = await inspectIterationEligibility(transaction, policy, { collectEligible: true });
+                const definition = await requireLearningRelease(transaction, "definition", policy.taskDefinition);
+                for (const reference of inspection.eligibleEvidence) {
+                    const evidence = await requireLearningRelease(transaction, "evidence", reference);
+                    let heldOut = false;
+                    for (const family of taskFamilyReservations(evidence, definition)) {
+                        const split = await transaction.familySplit(family.namespace, family.kind, family.key);
+                        if (split !== null && split !== "train") {
+                            heldOut = true;
+                            break;
+                        }
+                    }
+                    if (!heldOut)
+                        eligible.add(JSON.stringify([reference.id, reference.revision, reference.contentHash]));
+                }
+            }
+            catch (error) {
+                if (!(error instanceof LearningDomainError))
+                    throw error;
+                issues.push({ policyId: policy.id, code: error.code, message: error.message });
+            }
+        }
+        afterId = page.nextCursor ?? undefined;
+    } while (afterId);
+    return LearningTaskQueueInspectionSchema.parse({ modelProjectId, inspectedAt: now, pendingTrainingCount: issues.length ? null : eligible.size, issues });
+}

package/dist/learning/transport.js

@@ -17,6 +17,7 @@ export function assertLearningRequestJson(raw) {
 /** Scope selection is never authorization; each host resolves its authenticated owner. */
 export const LearningCommandRequestSchema = z.object({ scope: ReleaseIdSchema, command: LearningCommandSchema }).strict();
 export const LearningReadRequestSchema = z.discriminatedUnion("action", [
+    z.object({ action: z.literal("inspect_task_queue"), scope: ReleaseIdSchema, modelProjectId: ReleaseIdSchema.nullable() }).strict(),
     z.object({ action: z.literal("inspect_policy"), scope: ReleaseIdSchema, policy: LearningRevisionRefSchema }).strict(),
     z.object({ action: z.literal("inspect_evidence"), scope: ReleaseIdSchema, evidence: LearningRevisionRefSchema }).strict(),
     z.object({ action: z.literal("get"), scope: ReleaseIdSchema, kind: LearningResourceKindSchema, id: ReleaseIdSchema, revision: z.number().int().positive().optional() }).strict(),

package/dist/rewards.js

@@ -1,7 +1,7 @@
 import { LearningDomainError } from "./learning/errors.js";
 import { z } from "zod";
 import { ImmutableAssetRefSchema, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "@openpond/harness";
-import { gradeEvidence } from "./graders.js";
+import { gradeEvidence, GraderEvidenceSchema } from "./graders.js";
 import { CustomVerifierGraderSpecSchema, DeterministicGraderSpecSchema, GraderSpecSchema, HumanGraderSpecSchema, ModelJudgeGraderSpecSchema } from "./tasksets.js";
 import { assertBoundedTaskJson, validateTaskSchema } from "./task-schema.js";
 const bindingFields = { id: true, version: true, weight: true, hardGate: true, rewardEligible: true, privileged: true };
@@ -22,6 +22,7 @@ export const RewardReleaseContentSchema = z.object({
     rawScore: z.object({ minimum: z.number().finite(), maximum: z.number().finite() }).strict(),
     assets: z.array(ImmutableAssetRefSchema).max(1_000),
     fixtureSetRef: ImmutableAssetRefSchema.optional(),
+    calibrationCheckRef: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).optional(),
 }).strict().superRefine((reward, context) => {
     if (reward.fixtureSetRef && (reward.fixtureSetRef.visibility !== "verifier" || !reward.assets.some(asset => contentHash(asset) === contentHash(reward.fixtureSetRef))))
         context.addIssue({ code: "custom", path: ["fixtureSetRef"], message: "Reward fixtures must be included as a private verifier asset." });
@@ -86,6 +87,7 @@ export const BoundRewardResultSchema = z.object({
     normalizedScore: z.number().min(0).max(1).nullable(),
     passed: z.boolean().nullable(),
     evidenceHashes: z.array(ReleaseHashSchema).max(1_000),
+    graderEvidence: GraderEvidenceSchema.optional(),
     message: z.string().max(20_000).nullable(),
 }).strict();
 export const RewardCompositionContentSchema = z.object({
@@ -136,6 +138,8 @@ export function compileBoundGraders(binding, rewards) {
 }
 export async function executeRewardBinding(input) {
     const resolved = resolveBoundRewards(input.binding, input.rewards);
+    if (input.purpose === "fixture_calibration" && resolved.some(({ source }) => source.role !== "evaluation"))
+        throw new LearningDomainError("reward_calibration_requires_evaluation_binding");
     const results = [];
     for (const { source, reward } of resolved) {
         input.signal?.throwIfAborted();
@@ -154,10 +158,10 @@ export async function executeRewardBinding(input) {
                 }
             }
             else {
-                const [evidence] = await gradeEvidence({ task: input.task, evidence: input.evidence, graders: [compileSource(source, reward)], modelJudge: input.modelJudge, customVerifier: input.customVerifier });
+                const [evidence] = await gradeEvidence({ task: input.task, evidence: input.evidence, graders: [compileSource(source, reward)], modelJudge: input.modelJudge, customVerifier: input.customVerifier, purpose: input.purpose, signal: input.signal });
                 if (!evidence)
                     throw new LearningDomainError("reward_evidence_missing");
-                Object.assign(result, { status: evidence.score === null ? "unavailable" : "scored", rawScore: evidence.score, passed: evidence.score === null ? null : evidence.passed, evidenceHashes: [evidence.contentHash], message: evidence.feedback.join("\n") });
+                Object.assign(result, { status: evidence.score === null ? "unavailable" : "scored", rawScore: evidence.score, passed: evidence.score === null ? null : evidence.passed, evidenceHashes: [evidence.contentHash], graderEvidence: evidence, message: evidence.feedback.join("\n") });
             }
             if (result.status === "scored") {
                 if (result.rawScore === null || !Number.isFinite(result.rawScore) || result.rawScore < reward.rawScore.minimum || result.rawScore > reward.rawScore.maximum)
@@ -186,6 +190,14 @@ export function composeBoundRewards(input) {
         const result = results.find((candidate) => candidate.graderId === source.graderId);
         if (!result || contentHash(result.reward) !== contentHash(source.reward) || result.role !== source.role)
             throw new LearningDomainError("reward_result_binding_mismatch");
+        if (result.graderEvidence) {
+            const { contentHash: evidenceHash, ...body } = result.graderEvidence;
+            if (contentHash(body) !== evidenceHash || !result.evidenceHashes.includes(evidenceHash)
+                || body.graderId !== source.graderId || body.graderVersion !== String(source.reward.revision))
+                throw new LearningDomainError("reward_grader_evidence_mismatch");
+            if (result.status === "scored" && (body.score !== result.rawScore || body.passed !== result.passed))
+                throw new LearningDomainError("reward_grader_evidence_score_mismatch");
+        }
         if ((result.status === "scored") !== (result.rawScore !== null && result.normalizedScore !== null && result.passed !== null))
             throw new LearningDomainError("reward_result_score_state_invalid");
         if (result.status !== "scored" && (result.rawScore !== null || result.normalizedScore !== null || result.passed !== null))

package/dist/rollouts.js

@@ -116,7 +116,7 @@ export const CanonicalRolloutRecordSchema = z.object({
     ...CanonicalRolloutRecordFieldsSchema.shape,
     contentHash: ReleaseHashSchema,
 }).strict().superRefine(validateCanonicalRolloutRecord);
-export const RolloutQualificationSchema = z.object({
+export const RolloutQualificationSchema = /* @__PURE__ */ (() => z.object({
     schemaVersion: z.literal("openpond.rolloutQualification.v1"),
     rolloutCount: z.number().int().nonnegative(),
     scoredCount: z.number().int().nonnegative(),
@@ -128,7 +128,7 @@ export const RolloutQualificationSchema = z.object({
     distinctRewardCount: z.number().int().nonnegative(),
     eligibleForRl: z.boolean(),
     reasons: z.array(z.string().trim().min(1).max(1_000)).max(100),
-}).strict();
+}).strict())();
 export function createCanonicalRolloutRecord(input) {
     const attemptReceipt = AttemptReceiptSchema.parse(input.attemptReceipt);
     if (!verifyAttemptReceipt(attemptReceipt)) {