@openpond/evals 0.10.9 → 0.10.10

package/dist/learning/service.js

@@ -4,7 +4,8 @@ import { reserveLearningIteration } from "./iteration-reservation-service.js";
 import { synchronizeLearningSchedule } from "./schedule-service.js";
 import { inspectLearningPolicy } from "./policy-inspection.js";
 import { requireCurrentLearningEvidence as currentEvidence, sealLearningBatch } from "./batch-service.js";
-import { saveAuthoringDraft, archiveAuthoringDraft, finalizeAuthoringDraft } from "./authoring-service.js";
+import { saveAuthoringDraft, archiveAuthoringDraft, finalizeAuthoringDraft, currentDraft } from "./authoring-service.js";
+import { assertRewardCalibration, qualifyRewardCheck } from "./reward-calibration.js";
 import { queueRewardCheck, cancelRewardCheck } from "./reward-check-service.js";
 import { LearningDomainError } from "./errors.js";
 import { contentHash } from "@openpond/harness";
@@ -36,6 +37,29 @@ export function createLearningService(repository, options = {}) {
             }
             let pointers;
             switch (input.action) {
+                case "publish_checked_reward": {
+                    const draft = await currentDraft(transaction, input.draft);
+                    if (draft.targetKind !== "reward")
+                        throw new LearningDomainError("reward_calibration_draft_kind_invalid", 422);
+                    const base = draft.baseRelease ? await requireLearningRelease(transaction, "reward", draft.baseRelease) : null;
+                    const check = await requireLearningResource(transaction, "reward_check", input.checkId, input.checkRevision);
+                    const qualified = qualifyRewardCheck(draft, base, check);
+                    const { contentHash: _rewardHash, ...content } = qualified.reward;
+                    const publication = { action: "publish_resources", operationId: input.operationId,
+                        finalizeDraft: { draft: input.draft, targetKind: "reward", release: learningRef(qualified.reward) },
+                        resources: [
+                            ...qualified.assets.map(({ contentHash: _assetHash, ...content }) => ({ kind: "asset", expectedRevision: 0, content })),
+                            { kind: "reward", expectedRevision: base?.revision ?? 0, content },
+                        ],
+                    };
+                    pointers = [];
+                    for (const resource of publication.resources)
+                        pointers.push(await publish(transaction, { ...resource, action: "publish", operationId: input.operationId }));
+                    const finalized = await finalizeAuthoringDraft(transaction, publication, pointers, now());
+                    if (finalized)
+                        pointers.push(finalized);
+                    break;
+                }
                 case "import_intake":
                     pointers = await importTaskIntake(transaction, input, now(), submit);
                     break;
@@ -50,7 +74,7 @@ export function createLearningService(repository, options = {}) {
                     pointers = await reserveLearningIteration(transaction, input, context.actor.id, now());
                     break;
                 case "queue_reward_check":
-                    pointers = [await queueRewardCheck(transaction, input, operationId, now())];
+                    pointers = [await queueRewardCheck(transaction, input, operationId, now(), context.actor.id)];
                     break;
                 case "cancel_reward_check":
                     pointers = [await cancelRewardCheck(transaction, input, now())];
@@ -83,7 +107,7 @@ export function createLearningService(repository, options = {}) {
                     pointers = [await resolveFeedback(transaction, input, context.actor.id)];
                     break;
                 case "queue_grade":
-                    pointers = [await queueGrade(transaction, input, operationId)];
+                    pointers = [await queueGrade(transaction, input, operationId, context.actor.id)];
                     break;
                 case "cancel_grade":
                     pointers = [await cancelGrade(transaction, input)];
@@ -118,6 +142,7 @@ export function createLearningService(repository, options = {}) {
             }
             case "reward": {
                 resource = createRewardRelease(input.content);
+                await assertRewardCalibration(transaction, resource);
                 const implementation = input.content.implementation;
                 const references = [
                     ...input.content.assets,
@@ -282,7 +307,7 @@ export function createLearningService(repository, options = {}) {
         await transaction.put("feedback", resolved, feedback.revision, { parentId: learningEvidenceId(feedback.submission.sourceId, feedback.submission.exampleId, feedback.submission.attemptId), status: resolved.status });
         return pointer("feedback", resolved);
     }
-    async function queueGrade(transaction, input, operationId) {
+    async function queueGrade(transaction, input, operationId, requestedBy) {
         const evidence = await currentEvidence(transaction, input.evidence);
         const definition = await requireLearningRelease(transaction, "definition", evidence.submission.taskDefinition);
         if (!inspectTaskEvidence(evidence, definition).taskReady)
@@ -296,7 +321,7 @@ export function createLearningService(repository, options = {}) {
             schemaVersion: "openpond.taskGradeRun.v1", id: `grade-${operationId}`, revision: 1,
             evidence: learningRef(evidence), binding: definition.rewardBinding, target: input.target, output,
             status: "queued", composition: null, leaseOwner: null, leaseExpiresAt: null, attemptCount: 0,
-            timeoutMs: input.timeoutMs, maximumSpendUsd: input.maximumSpendUsd, failure: null, createdAt: now(), updatedAt: now(),
+            timeoutMs: input.timeoutMs, maximumSpendUsd: input.maximumSpendUsd, requestedBy, failure: null, createdAt: now(), updatedAt: now(),
         });
         await transaction.put("grade", grade, 0, { parentId: evidence.id, status: grade.status });
         return pointer("grade", grade);

package/dist/rewards.js

@@ -1,7 +1,7 @@
 import { LearningDomainError } from "./learning/errors.js";
 import { z } from "zod";
 import { ImmutableAssetRefSchema, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "@openpond/harness";
-import { gradeEvidence } from "./graders.js";
+import { gradeEvidence, GraderEvidenceSchema } from "./graders.js";
 import { CustomVerifierGraderSpecSchema, DeterministicGraderSpecSchema, GraderSpecSchema, HumanGraderSpecSchema, ModelJudgeGraderSpecSchema } from "./tasksets.js";
 import { assertBoundedTaskJson, validateTaskSchema } from "./task-schema.js";
 const bindingFields = { id: true, version: true, weight: true, hardGate: true, rewardEligible: true, privileged: true };
@@ -22,6 +22,7 @@ export const RewardReleaseContentSchema = z.object({
     rawScore: z.object({ minimum: z.number().finite(), maximum: z.number().finite() }).strict(),
     assets: z.array(ImmutableAssetRefSchema).max(1_000),
     fixtureSetRef: ImmutableAssetRefSchema.optional(),
+    calibrationCheckRef: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).optional(),
 }).strict().superRefine((reward, context) => {
     if (reward.fixtureSetRef && (reward.fixtureSetRef.visibility !== "verifier" || !reward.assets.some(asset => contentHash(asset) === contentHash(reward.fixtureSetRef))))
         context.addIssue({ code: "custom", path: ["fixtureSetRef"], message: "Reward fixtures must be included as a private verifier asset." });
@@ -86,6 +87,7 @@ export const BoundRewardResultSchema = z.object({
     normalizedScore: z.number().min(0).max(1).nullable(),
     passed: z.boolean().nullable(),
     evidenceHashes: z.array(ReleaseHashSchema).max(1_000),
+    graderEvidence: GraderEvidenceSchema.optional(),
     message: z.string().max(20_000).nullable(),
 }).strict();
 export const RewardCompositionContentSchema = z.object({
@@ -136,6 +138,8 @@ export function compileBoundGraders(binding, rewards) {
 }
 export async function executeRewardBinding(input) {
     const resolved = resolveBoundRewards(input.binding, input.rewards);
+    if (input.purpose === "fixture_calibration" && resolved.some(({ source }) => source.role !== "evaluation"))
+        throw new LearningDomainError("reward_calibration_requires_evaluation_binding");
     const results = [];
     for (const { source, reward } of resolved) {
         input.signal?.throwIfAborted();
@@ -154,10 +158,10 @@ export async function executeRewardBinding(input) {
                 }
             }
             else {
-                const [evidence] = await gradeEvidence({ task: input.task, evidence: input.evidence, graders: [compileSource(source, reward)], modelJudge: input.modelJudge, customVerifier: input.customVerifier });
+                const [evidence] = await gradeEvidence({ task: input.task, evidence: input.evidence, graders: [compileSource(source, reward)], modelJudge: input.modelJudge, customVerifier: input.customVerifier, purpose: input.purpose, signal: input.signal });
                 if (!evidence)
                     throw new LearningDomainError("reward_evidence_missing");
-                Object.assign(result, { status: evidence.score === null ? "unavailable" : "scored", rawScore: evidence.score, passed: evidence.score === null ? null : evidence.passed, evidenceHashes: [evidence.contentHash], message: evidence.feedback.join("\n") });
+                Object.assign(result, { status: evidence.score === null ? "unavailable" : "scored", rawScore: evidence.score, passed: evidence.score === null ? null : evidence.passed, evidenceHashes: [evidence.contentHash], graderEvidence: evidence, message: evidence.feedback.join("\n") });
             }
             if (result.status === "scored") {
                 if (result.rawScore === null || !Number.isFinite(result.rawScore) || result.rawScore < reward.rawScore.minimum || result.rawScore > reward.rawScore.maximum)
@@ -186,6 +190,14 @@ export function composeBoundRewards(input) {
         const result = results.find((candidate) => candidate.graderId === source.graderId);
         if (!result || contentHash(result.reward) !== contentHash(source.reward) || result.role !== source.role)
             throw new LearningDomainError("reward_result_binding_mismatch");
+        if (result.graderEvidence) {
+            const { contentHash: evidenceHash, ...body } = result.graderEvidence;
+            if (contentHash(body) !== evidenceHash || !result.evidenceHashes.includes(evidenceHash)
+                || body.graderId !== source.graderId || body.graderVersion !== String(source.reward.revision))
+                throw new LearningDomainError("reward_grader_evidence_mismatch");
+            if (result.status === "scored" && (body.score !== result.rawScore || body.passed !== result.passed))
+                throw new LearningDomainError("reward_grader_evidence_score_mismatch");
+        }
         if ((result.status === "scored") !== (result.rawScore !== null && result.normalizedScore !== null && result.passed !== null))
             throw new LearningDomainError("reward_result_score_state_invalid");
         if (result.status !== "scored" && (result.rawScore !== null || result.normalizedScore !== null || result.passed !== null))

package/dist/rollouts.js

@@ -116,7 +116,7 @@ export const CanonicalRolloutRecordSchema = z.object({
     ...CanonicalRolloutRecordFieldsSchema.shape,
     contentHash: ReleaseHashSchema,
 }).strict().superRefine(validateCanonicalRolloutRecord);
-export const RolloutQualificationSchema = z.object({
+export const RolloutQualificationSchema = /* @__PURE__ */ (() => z.object({
     schemaVersion: z.literal("openpond.rolloutQualification.v1"),
     rolloutCount: z.number().int().nonnegative(),
     scoredCount: z.number().int().nonnegative(),
@@ -128,7 +128,7 @@ export const RolloutQualificationSchema = z.object({
     distinctRewardCount: z.number().int().nonnegative(),
     eligibleForRl: z.boolean(),
     reasons: z.array(z.string().trim().min(1).max(1_000)).max(100),
-}).strict();
+}).strict())();
 export function createCanonicalRolloutRecord(input) {
     const attemptReceipt = AttemptReceiptSchema.parse(input.attemptReceipt);
     if (!verifyAttemptReceipt(attemptReceipt)) {

package/dist/types/graders.d.ts

@@ -1,6 +1,18 @@
 import { z } from "zod";
 import type { GraderSpec, TaskRecord } from "./tasksets.js";
 export { evaluateDeterministicGrader, portableDeterministicCheck } from "./deterministic-graders.js";
+export declare const ModelJudgeReceiptSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.modelJudgeReceipt.v1">;
+    providerId: z.ZodString;
+    modelId: z.ZodString;
+    modelRevision: z.ZodNullable<z.ZodString>;
+    responseId: z.ZodNullable<z.ZodString>;
+    requestHash: z.ZodString;
+    responseHash: z.ZodString;
+    inputTokens: z.ZodNullable<z.ZodNumber>;
+    outputTokens: z.ZodNullable<z.ZodNumber>;
+    costUsd: z.ZodNullable<z.ZodNumber>;
+}, z.core.$strict>;
 export declare const GraderEvidenceContentSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
     graderId: z.ZodString;
@@ -19,6 +31,18 @@ export declare const GraderEvidenceContentSchema: z.ZodObject<{
     feedback: z.ZodArray<z.ZodString>;
     visibleEvidenceRefs: z.ZodArray<z.ZodString>;
     privilegedEvidenceRefs: z.ZodArray<z.ZodString>;
+    modelJudgeReceipt: z.ZodOptional<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.modelJudgeReceipt.v1">;
+        providerId: z.ZodString;
+        modelId: z.ZodString;
+        modelRevision: z.ZodNullable<z.ZodString>;
+        responseId: z.ZodNullable<z.ZodString>;
+        requestHash: z.ZodString;
+        responseHash: z.ZodString;
+        inputTokens: z.ZodNullable<z.ZodNumber>;
+        outputTokens: z.ZodNullable<z.ZodNumber>;
+        costUsd: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
 }, z.core.$strict>;
 export declare const GraderEvidenceSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
@@ -38,6 +62,18 @@ export declare const GraderEvidenceSchema: z.ZodObject<{
     feedback: z.ZodArray<z.ZodString>;
     visibleEvidenceRefs: z.ZodArray<z.ZodString>;
     privilegedEvidenceRefs: z.ZodArray<z.ZodString>;
+    modelJudgeReceipt: z.ZodOptional<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.modelJudgeReceipt.v1">;
+        providerId: z.ZodString;
+        modelId: z.ZodString;
+        modelRevision: z.ZodNullable<z.ZodString>;
+        responseId: z.ZodNullable<z.ZodString>;
+        requestHash: z.ZodString;
+        responseHash: z.ZodString;
+        inputTokens: z.ZodNullable<z.ZodNumber>;
+        outputTokens: z.ZodNullable<z.ZodNumber>;
+        costUsd: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
     contentHash: z.ZodString;
 }, z.core.$strict>;
 export type AttemptEvidence = {
@@ -52,6 +88,7 @@ export type ModelJudgeRunner = (input: {
     }>;
     task: TaskRecord;
     evidence: AttemptEvidence;
+    signal?: AbortSignal;
 }) => Promise<Omit<GraderEvidence, "schemaVersion" | "graderId" | "graderVersion" | "contentHash">>;
 export type CustomVerifierRunner = (input: {
     grader: Extract<GraderSpec, {
@@ -66,6 +103,8 @@ export declare function gradeEvidence(input: {
     graders: GraderSpec[];
     modelJudge?: ModelJudgeRunner;
     customVerifier?: CustomVerifierRunner;
+    purpose?: "grading" | "fixture_calibration";
+    signal?: AbortSignal;
 }): Promise<GraderEvidence[]>;
 export type GraderEvidence = z.infer<typeof GraderEvidenceSchema>;
 declare const GradeSummarySchema: z.ZodObject<{
@@ -132,6 +171,18 @@ export declare const TaskGradeSchema: z.ZodObject<{
         feedback: z.ZodArray<z.ZodString>;
         visibleEvidenceRefs: z.ZodArray<z.ZodString>;
         privilegedEvidenceRefs: z.ZodArray<z.ZodString>;
+        modelJudgeReceipt: z.ZodOptional<z.ZodObject<{
+            schemaVersion: z.ZodLiteral<"openpond.modelJudgeReceipt.v1">;
+            providerId: z.ZodString;
+            modelId: z.ZodString;
+            modelRevision: z.ZodNullable<z.ZodString>;
+            responseId: z.ZodNullable<z.ZodString>;
+            requestHash: z.ZodString;
+            responseHash: z.ZodString;
+            inputTokens: z.ZodNullable<z.ZodNumber>;
+            outputTokens: z.ZodNullable<z.ZodNumber>;
+            costUsd: z.ZodNullable<z.ZodNumber>;
+        }, z.core.$strict>>;
         contentHash: z.ZodString;
     }, z.core.$strict>>;
     contentHash: z.ZodString;

package/dist/types/graders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAErG,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA8BD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,kBAAkB;;;;;;;;;;;;;;;;;kBAIb,CAAC;AAEZ;6EAC6E;AAC7E,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,GAAG,SAAS,GAAG,QAAQ,GAAG,UAAU,GAAG,gBAAgB,CAAC,CAAC,CAAC;IACtG,UAAU,EAAE,aAAa,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,GAAG,eAAe,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,GAAG,cAAc,CAAC,CAAC,CAAC;IACvI,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAwBrC;AAED,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAIjB,CAAC;AACZ,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAMtG;AAED;+EAC+E;AAC/E,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC,GAAG,SAAS,CAW5I"}
+{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAErG,eAAO,MAAM,uBAAuB;;;;;;;;;;;kBAWzB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAY7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AACrQ,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,OAAO,CAAC,EAAE,SAAS,GAAG,qBAAqB,CAAC;IAC5C,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA0B5B;AA8BD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,kBAAkB;;;;;;;;;;;;;;;;;kBAIb,CAAC;AAEZ;6EAC6E;AAC7E,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,GAAG,SAAS,GAAG,QAAQ,GAAG,UAAU,GAAG,gBAAgB,CAAC,CAAC,CAAC;IACtG,UAAU,EAAE,aAAa,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,GAAG,eAAe,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,GAAG,cAAc,CAAC,CAAC,CAAC;IACvI,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAwBrC;AAED,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAId,CAAC;AACf,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAMtG;AAED;+EAC+E;AAC/E,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC,GAAG,SAAS,CAW5I"}

package/dist/types/learning/admission.d.ts

@@ -337,6 +337,11 @@ export declare const TaskBatchPackageMetadataSchema: z.ZodObject<{
                 host_private: "host_private";
             }>;
         }, z.core.$strict>>;
+        calibrationCheckRef: z.ZodOptional<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodNumber;
+        }, z.core.$strict>>;
         contentHash: z.ZodString;
     }, z.core.$strict>>;
     admissions: z.ZodArray<z.ZodObject<{

package/dist/types/learning/admission.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"admission.d.ts","sourceRoot":"","sources":["../../../../../src/learning/admission.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAsF,KAAK,aAAa,EAAE,KAAK,iBAAiB,EAAE,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AACnL,OAAO,EAAuE,KAAK,UAAU,EAAE,KAAK,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC3I,OAAO,EAA4C,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAwS,KAAK,cAAc,EAAE,KAAK,qBAAqB,EAAE,KAAK,SAAS,EAAE,KAAK,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAE3c,MAAM,MAAM,sBAAsB,GAAG;IACnC,gBAAgB,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,SAAS,EAAE,OAAO,CAAC;IACnB,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IACpC,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B,CAAC;AACF,MAAM,MAAM,eAAe,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;IAAC,KAAK,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAA;CAAE,CAAC;AAElI,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,GAAG,sBAAsB,CAgB9G;AAED,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,qBAAqB,GAAG,qBAAqB,CAWnI;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,qBAAqB,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAmB/M;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAU5E;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,GAAG,UAAU,CAWrG;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;sBAChD,MAAM,EAAE;;EAClD;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,qBAAqB,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,cAAc,CAAC;IAAC,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;CAAE,GAAG,IAAI,CAkB3K;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAGvJ;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,GAAG,eAAe,EAAE,CAM5G;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,cAAc,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,SAAS,EAAE,qBAAqB,EAAE,CAAC;IAAC,WAAW,EAAE,eAAe,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAwBvS;AAED,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAOhC,CAAC;AAEZ,wBAAgB,gBAAgB,CAAC,KAAK,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,cAAc,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAC;IAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,SAAS,EAAE,qBAAqB,EAAE,CAAA;CAAE,GAAG,cAAc,CAuBxN;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAQhH"}
+{"version":3,"file":"admission.d.ts","sourceRoot":"","sources":["../../../../../src/learning/admission.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAsF,KAAK,aAAa,EAAE,KAAK,iBAAiB,EAAE,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AACnL,OAAO,EAAuE,KAAK,UAAU,EAAE,KAAK,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC3I,OAAO,EAA4C,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAwS,KAAK,cAAc,EAAE,KAAK,qBAAqB,EAAE,KAAK,SAAS,EAAE,KAAK,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAE3c,MAAM,MAAM,sBAAsB,GAAG;IACnC,gBAAgB,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,SAAS,EAAE,OAAO,CAAC;IACnB,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IACpC,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B,CAAC;AACF,MAAM,MAAM,eAAe,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;IAAC,KAAK,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAA;CAAE,CAAC;AAElI,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,GAAG,sBAAsB,CAgB9G;AAED,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,qBAAqB,GAAG,qBAAqB,CAWnI;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,qBAAqB,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAmB/M;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAU5E;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,GAAG,UAAU,CAWrG;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;sBAChD,MAAM,EAAE;;EAClD;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,qBAAqB,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,cAAc,CAAC;IAAC,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;CAAE,GAAG,IAAI,CAkB3K;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAGvJ;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,GAAG,eAAe,EAAE,CAM5G;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,cAAc,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,SAAS,EAAE,qBAAqB,EAAE,CAAC;IAAC,WAAW,EAAE,eAAe,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAwBvS;AAED,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAOhC,CAAC;AAEZ,wBAAgB,gBAAgB,CAAC,KAAK,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,cAAc,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAC;IAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,SAAS,EAAE,qBAAqB,EAAE,CAAA;CAAE,GAAG,cAAc,CAuBxN;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAQhH"}

package/dist/types/learning/budgeted-reward-fixtures.d.ts

@@ -0,0 +1,27 @@
+import type { executeJavaScriptVerifier } from "../javascript-verifier.js";
+import { type BoundJudgeRequest, type BoundJudgeResponse } from "./model-judge.js";
+import { type LearningRepository } from "./repository.js";
+import type { RewardCheckRun, RewardCheckRuntime } from "./reward-checks.js";
+import type { RewardFixtureExecutor } from "./reward-check-worker.js";
+export interface BoundJudgeProvider {
+    /** Resolve exact model configuration and a conservative charge ceiling without
+     * performing inference. dispatch settles after the provider request closes. */
+    prepare(request: BoundJudgeRequest, context?: {
+        scope: string;
+        run: Pick<RewardCheckRun, "id" | "requestedBy">;
+    }): Promise<{
+        maximumChargeUsd: number;
+        dispatch(signal?: AbortSignal): Promise<BoundJudgeResponse>;
+    }>;
+    cancel(input: {
+        scope: string;
+        run: Pick<RewardCheckRun, "id" | "judgeCalls">;
+    }): Promise<boolean>;
+}
+export declare function createBudgetedRewardFixtureExecutor(options: {
+    runtime: RewardCheckRuntime;
+    repository: LearningRepository;
+    provider: BoundJudgeProvider;
+    executeJavaScript: typeof executeJavaScriptVerifier;
+}): RewardFixtureExecutor;
+//# sourceMappingURL=budgeted-reward-fixtures.d.ts.map

package/dist/types/learning/budgeted-reward-fixtures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"budgeted-reward-fixtures.d.ts","sourceRoot":"","sources":["../../../../../src/learning/budgeted-reward-fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAG3E,OAAO,EAA+B,KAAK,iBAAiB,EAAE,KAAK,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAGhH,OAAO,EAA2B,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACnF,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEtE,MAAM,WAAW,kBAAkB;IACjC;mFAC+E;IAC/E,OAAO,CAAC,OAAO,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,aAAa,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAA;KAAE,CAAC,CAAC;IACtN,MAAM,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,YAAY,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACpG;AAED,wBAAgB,mCAAmC,CAAC,OAAO,EAAE;IAC3D,OAAO,EAAE,kBAAkB,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAC;IAAC,QAAQ,EAAE,kBAAkB,CAAC;IAC1F,iBAAiB,EAAE,OAAO,yBAAyB,CAAC;CACrD,GAAG,qBAAqB,CAwBxB"}