@openpond/evals 0.7.5 → 0.8.0

package/dist/learning/authoring-service.js

@@ -38,7 +38,7 @@ export async function finalizeAuthoringDraft(tx, input, published, now) {
     await tx.put("draft", completed, draft.revision, { parentId: draft.targetKind, status: completed.status });
     return { kind: "draft", id: completed.id, revision: completed.revision };
 }
-async function currentDraft(tx, ref) {
+export async function currentDraft(tx, ref) {
     const draft = await requireLearningRelease(tx, "draft", ref);
     const current = await tx.get("draft", ref.id);
     if (!current || !sameLearningRef(learningRef(current), ref))

package/dist/learning/authoring.js

@@ -3,6 +3,14 @@ import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema } from "@ope
 import { RewardBindingSourceSchema, RewardReleaseRefSchema } from "../rewards.js";
 import { LearningRevisionRefSchema } from "./contracts.js";
 /** Strings intentionally preserve incomplete JSON/code until explicit publication. */
+export const RewardFixtureAuthoringFieldsSchema = z.object({
+    id: ReleaseIdSchema, name: z.string().max(500),
+    input: z.string(), output: z.string(), expectedOutput: z.string(), evaluatorContext: z.string(),
+    artifactRefs: z.array(z.string().max(500)).max(100), runtimeEventRefs: z.array(z.string().max(500)).max(100),
+    infrastructureError: z.string().max(20_000),
+    expectedStatus: z.enum(["scored", "pending", "unavailable", "failed"]),
+    minimumScore: z.string(), maximumScore: z.string(), expectedPassed: z.enum(["true", "false", "any"]),
+}).strict();
 export const RewardAuthoringFieldsSchema = z.object({
     name: z.string().max(500), description: z.string().max(10_000),
     kind: z.enum(["custom_verifier", "state", "content", "schema", "artifact", "runtime_event", "model_judge", "learned_model", "human"]),
@@ -10,6 +18,7 @@ export const RewardAuthoringFieldsSchema = z.object({
     schema: z.string(), reference: z.string(), events: z.string(), code: z.string(), exportName: z.string(), timeout: z.string(),
     rubric: z.string(), providerId: z.string(), modelId: z.string(), modelRevision: z.string(), temperature: z.string(),
     reviewerRole: z.string(), learnedId: z.string(), learnedHash: z.string(), inputContract: z.string(), minimum: z.string(), maximum: z.string(),
+    fixtures: z.array(RewardFixtureAuthoringFieldsSchema).max(50).optional(),
 }).strict();
 export const TaskFormatAuthoringFieldsSchema = z.object({
     name: z.string().max(500), description: z.string().max(10_000), instructions: z.string().max(20_000),

package/dist/learning/index.js

@@ -8,3 +8,7 @@ export * from "./grade-worker.js";
 export * from "./transport.js";
 export * from "./errors.js";
 export * from "./authoring.js";
+export * from "./reward-authoring.js";
+export * from "./reward-checks.js";
+export * from "./reward-check-worker.js";
+export * from "./reward-fixture-executor.js";

package/dist/learning/operations.js

@@ -34,4 +34,6 @@ export const SealTaskBatchCommandSchema = command.extend({ action: z.literal("se
 export const CancelTaskGradeCommandSchema = command.extend({ action: z.literal("cancel_grade"), gradeId: ReleaseIdSchema, expectedRevision: z.number().int().positive() }).strict();
 export const SaveAuthoringDraftCommandSchema = command.extend({ action: z.literal("save_draft"), expectedRevision: z.number().int().nonnegative(), draft: AuthoringDraftInputSchema }).strict();
 export const ArchiveAuthoringDraftCommandSchema = command.extend({ action: z.literal("archive_draft"), draft: LearningRevisionRefSchema }).strict();
-export const LearningCommandSchema = z.union([SaveAuthoringDraftCommandSchema, ArchiveAuthoringDraftCommandSchema, PublishLearningResourceCommandSchema, PublishLearningResourcesCommandSchema, SubmitTaskExampleCommandSchema, SubmitTaskFeedbackCommandSchema, ApplyTaskCorrectionCommandSchema, ResolveTaskFeedbackCommandSchema, QueueTaskGradeCommandSchema, ReviewTaskEvidenceCommandSchema, SealTaskBatchCommandSchema, CancelTaskGradeCommandSchema]);
+export const QueueRewardCheckCommandSchema = command.extend({ action: z.literal("queue_reward_check"), draft: LearningRevisionRefSchema, timeoutMs: z.number().int().min(100).max(300_000).default(30_000), maximumSpendUsd: z.number().nonnegative().max(1_000).default(0) }).strict();
+export const CancelRewardCheckCommandSchema = command.extend({ action: z.literal("cancel_reward_check"), checkId: ReleaseIdSchema, expectedRevision: z.number().int().positive() }).strict();
+export const LearningCommandSchema = z.union([QueueRewardCheckCommandSchema, CancelRewardCheckCommandSchema, SaveAuthoringDraftCommandSchema, ArchiveAuthoringDraftCommandSchema, PublishLearningResourceCommandSchema, PublishLearningResourcesCommandSchema, SubmitTaskExampleCommandSchema, SubmitTaskFeedbackCommandSchema, ApplyTaskCorrectionCommandSchema, ResolveTaskFeedbackCommandSchema, QueueTaskGradeCommandSchema, ReviewTaskEvidenceCommandSchema, SealTaskBatchCommandSchema, CancelTaskGradeCommandSchema]);

package/dist/learning/repository.js

@@ -6,8 +6,10 @@ import { TasksetReleaseSchema } from "../tasksets.js";
 import { LearningDomainError } from "./errors.js";
 import { AuthoringDraftSchema } from "./authoring.js";
 import { LearningTextAssetSchema } from "./assets.js";
+import { RewardCheckRunSchema } from "./reward-checks.js";
 export const learningResourceSchemas = {
     draft: AuthoringDraftSchema,
+    reward_check: RewardCheckRunSchema,
     asset: LearningTextAssetSchema,
     definition: TaskDefinitionSchema, reward: RewardReleaseSchema, binding: RewardBindingSchema,
     source: LearningSourceSchema, evidence: TaskEvidenceSchema, feedback: TaskFeedbackSchema,

package/dist/learning/reward-authoring.js

@@ -0,0 +1,120 @@
+import { RewardReleaseSchema, createRewardRelease } from "../rewards.js";
+import { assertBoundedTaskJson } from "../task-schema.js";
+import { RewardAuthoringFieldsSchema, RewardFixtureAuthoringFieldsSchema } from "./authoring.js";
+import { createLearningTextAsset, verifyLearningTextAsset } from "./assets.js";
+import { LearningJsonObjectSchema } from "./contracts.js";
+import { LearningDomainError } from "./errors.js";
+export const DEFAULT_REWARD_VERIFIER_SOURCE = `export function verify({ output, expectedOutput }) {
+  if (!expectedOutput || !Object.hasOwn(expectedOutput, "answer")) {
+    throw new Error("An expected answer is required.");
+  }
+  const passed = output.answer === expectedOutput.answer;
+  return { score: passed ? 1 : 0, passed, feedback: passed ? "Answer matched" : "Answer differed" };
+}`;
+/** Both editors and execution owners compile the same draft bytes. No publication occurs here. */
+export function compileRewardAuthoring(input) {
+    assertBoundedTaskJson(input, 262_144);
+    const fields = RewardAuthoringFieldsSchema.parse(input.fields);
+    const base = input.base ? RewardReleaseSchema.parse(input.base) : null;
+    if (base && base.id !== input.id)
+        throw new LearningDomainError("reward_authoring_base_mismatch", 422);
+    const fixtureAsset = fields.fixtures?.length ? createLearningTextAsset({ text: JSON.stringify(fields.fixtures), path: "reward-fixtures.json", mediaType: "application/json", visibility: "verifier" }) : null;
+    let asset = null;
+    let implementation;
+    if (fields.kind === "custom_verifier") {
+        asset = createLearningTextAsset({ text: fields.code, path: "verifier.mjs", mediaType: "application/javascript", visibility: "verifier" });
+        implementation = { kind: fields.kind, verifierRef: asset.asset, exportName: fields.exportName, timeoutMs: authoringNumber(fields.timeout, "Time limit"), networkPolicy: "none" };
+    }
+    else if (fields.kind === "model_judge" || fields.kind === "human") {
+        if (!fields.rubric.trim())
+            throw new LearningDomainError("reward_rubric_required", 422, "Write the rubric before checking or publishing this Reward.");
+        asset = createLearningTextAsset({ text: fields.rubric, path: "rubric.md", mediaType: "text/markdown", visibility: "verifier" });
+        if (fields.kind === "human")
+            implementation = { kind: fields.kind, rubricRef: asset.asset, reviewerRole: fields.reviewerRole };
+        else {
+            const model = { providerId: fields.providerId, modelId: fields.modelId, revision: fields.modelRevision.trim() || null };
+            const temperature = authoringNumber(fields.temperature, "Temperature");
+            const previous = base?.implementation;
+            const unchanged = previous?.kind === "model_judge" && previous.rubricRef.contentHash === asset.asset.contentHash
+                && previous.model?.providerId === model.providerId && previous.model?.modelId === model.modelId
+                && previous.model?.revision === model.revision && (previous.temperature ?? 0) === temperature
+                && base?.fixtureSetRef?.contentHash === fixtureAsset?.asset.contentHash;
+            implementation = { kind: fields.kind, rubricRef: asset.asset, model, temperature, calibrationStatus: unchanged ? previous.calibrationStatus : "pending" };
+        }
+    }
+    else if (fields.kind === "learned_model") {
+        parseRewardAuthoringObject(fields.inputContract, "Model input contract");
+        asset = createLearningTextAsset({ text: fields.inputContract, path: "input-contract.json", mediaType: "application/json", visibility: "verifier" });
+        implementation = { kind: fields.kind, modelVersion: { id: fields.learnedId, contentHash: fields.learnedHash }, inputContract: asset.asset };
+    }
+    else if (fields.kind === "state")
+        implementation = { kind: fields.kind, config: { fields: split(fields.fields) } };
+    else if (fields.kind === "content")
+        implementation = { kind: fields.kind, config: { outputField: fields.outputField, expectedField: fields.expectedField, ...(fields.expectedValue ? { expectedValue: fields.expectedValue } : {}) } };
+    else if (fields.kind === "schema")
+        implementation = { kind: fields.kind, config: { jsonSchema: parseRewardAuthoringObject(fields.schema, "Output schema") } };
+    else if (fields.kind === "artifact")
+        implementation = { kind: fields.kind, config: { refIncludes: fields.reference } };
+    else
+        implementation = { kind: fields.kind, config: { requiredEvents: split(fields.events) } };
+    const assets = [...(asset ? [asset] : []), ...(fixtureAsset ? [fixtureAsset] : [])];
+    const reward = createRewardRelease({
+        schemaVersion: "openpond.rewardRelease.v1", id: input.id, revision: (base?.revision ?? 0) + 1,
+        name: fields.name, description: fields.description, implementation,
+        rawScore: fields.kind === "learned_model" ? { minimum: authoringNumber(fields.minimum, "Minimum score"), maximum: authoringNumber(fields.maximum, "Maximum score") } : { minimum: 0, maximum: 1 },
+        assets: assets.map(value => value.asset), ...(fixtureAsset ? { fixtureSetRef: fixtureAsset.asset } : {}),
+    });
+    return { reward, assets };
+}
+export function rewardAuthoringFields(reward, sourceAsset, fixtureAsset = null) {
+    const implementation = reward?.implementation;
+    const ref = implementation && "verifierRef" in implementation ? implementation.verifierRef
+        : implementation && "rubricRef" in implementation ? implementation.rubricRef
+            : implementation && "inputContract" in implementation ? implementation.inputContract : null;
+    if (ref && !sourceAsset)
+        throw new LearningDomainError("reward_source_required", 422);
+    const source = ref && sourceAsset ? verifyLearningTextAsset(sourceAsset, ref) : "";
+    if (reward?.fixtureSetRef && !fixtureAsset)
+        throw new LearningDomainError("reward_fixtures_required", 422);
+    const fixtures = reward?.fixtureSetRef && fixtureAsset ? RewardFixtureAuthoringFieldsSchema.array().max(50).parse(JSON.parse(verifyLearningTextAsset(fixtureAsset, reward.fixtureSetRef))) : [];
+    const config = implementation && "config" in implementation ? implementation.config : {};
+    return {
+        name: reward?.name ?? "", description: reward?.description ?? "", kind: implementation?.kind ?? "state",
+        fields: strings(config.fields).join(", ") || "answer", outputField: text(config.outputField) || "text", expectedField: text(config.expectedField) || "text", expectedValue: text(config.expectedValue),
+        schema: JSON.stringify(config.jsonSchema ?? { type: "object", properties: { answer: { type: "string" } }, required: ["answer"] }, null, 2),
+        reference: text(config.refIncludes), events: strings(config.requiredEvents).join(", "),
+        code: implementation?.kind === "custom_verifier" ? source : DEFAULT_REWARD_VERIFIER_SOURCE,
+        exportName: implementation?.kind === "custom_verifier" ? implementation.exportName ?? "verify" : "verify",
+        timeout: String(implementation?.kind === "custom_verifier" ? implementation.timeoutMs : 5_000),
+        rubric: implementation?.kind === "model_judge" || implementation?.kind === "human" ? source : "",
+        providerId: implementation?.kind === "model_judge" ? implementation.model?.providerId ?? "" : "",
+        modelId: implementation?.kind === "model_judge" ? implementation.model?.modelId ?? "" : "",
+        modelRevision: implementation?.kind === "model_judge" ? implementation.model?.revision ?? "" : "",
+        temperature: String(implementation?.kind === "model_judge" ? implementation.temperature ?? 0 : 0),
+        reviewerRole: implementation?.kind === "human" ? implementation.reviewerRole : "Subject matter reviewer",
+        learnedId: implementation?.kind === "learned_model" ? implementation.modelVersion.id : "",
+        learnedHash: implementation?.kind === "learned_model" ? implementation.modelVersion.contentHash : "",
+        inputContract: implementation?.kind === "learned_model" ? source : "{}",
+        minimum: String(reward?.rawScore.minimum ?? 0), maximum: String(reward?.rawScore.maximum ?? 1),
+        ...(fixtures.length ? { fixtures } : {}),
+    };
+}
+export function parseRewardAuthoringObject(value, label) {
+    try {
+        const parsed = JSON.parse(value);
+        assertBoundedTaskJson(parsed, 131_072);
+        return LearningJsonObjectSchema.parse(parsed);
+    }
+    catch {
+        throw new LearningDomainError("reward_authoring_json_invalid", 422, `${label} must be a JSON object.`);
+    }
+}
+export function authoringNumber(value, label) {
+    const number = Number(value);
+    if (!value.trim() || !Number.isFinite(number))
+        throw new LearningDomainError("reward_authoring_number_invalid", 422, `${label} must be a finite number.`);
+    return number;
+}
+function text(value) { return typeof value === "string" ? value : ""; }
+function strings(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string") : []; }
+function split(value) { return value.split(",").map(part => part.trim()).filter(Boolean); }

package/dist/learning/reward-check-service.js

@@ -0,0 +1,31 @@
+import { LearningDomainError } from "./errors.js";
+import { currentDraft } from "./authoring-service.js";
+import { learningRef } from "./contracts.js";
+import { compileRewardCheck, RewardCheckRunSchema } from "./reward-checks.js";
+import { LearningConflictError, requireLearningRelease, requireLearningResource } from "./repository.js";
+export async function queueRewardCheck(tx, input, operationId, now) {
+    const draft = await currentDraft(tx, input.draft);
+    if (draft.targetKind !== "reward")
+        throw new LearningDomainError("reward_check_draft_kind_invalid", 422);
+    const base = draft.baseRelease ? await requireLearningRelease(tx, "reward", draft.baseRelease) : null;
+    const compiled = compileRewardCheck(draft, base);
+    const check = RewardCheckRunSchema.parse({
+        schemaVersion: "openpond.rewardCheckRun.v1", id: `reward-check-${operationId}`, revision: 1,
+        draft: learningRef(draft), reward: learningRef(compiled.reward), snapshotHash: compiled.snapshotHash,
+        fixtureRefs: compiled.fixtureRefs, status: "queued", runtime: null, results: [], matchesExpectations: null,
+        timeoutMs: input.timeoutMs, maximumSpendUsd: input.maximumSpendUsd,
+        leaseOwner: null, leaseExpiresAt: null, attemptCount: 0, failure: null, createdAt: now, updatedAt: now,
+    });
+    await tx.put("reward_check", check, 0, { parentId: draft.targetId, status: check.status });
+    return { kind: "reward_check", id: check.id, revision: check.revision };
+}
+export async function cancelRewardCheck(tx, input, now) {
+    const check = await requireLearningResource(tx, "reward_check", input.checkId);
+    if (check.revision !== input.expectedRevision)
+        throw new LearningConflictError("reward_check", check.id, input.expectedRevision, check.revision);
+    if (["completed", "failed", "cancelled"].includes(check.status))
+        return { kind: "reward_check", id: check.id, revision: check.revision };
+    const updated = RewardCheckRunSchema.parse({ ...check, revision: check.revision + 1, status: check.status === "queued" ? "cancelled" : "cancelling", updatedAt: now });
+    await tx.put("reward_check", updated, check.revision, { parentId: check.reward.id, status: updated.status });
+    return { kind: "reward_check", id: updated.id, revision: updated.revision };
+}

package/dist/learning/reward-check-worker.js

@@ -0,0 +1,114 @@
+import { contentHash } from "@openpond/harness";
+import { BoundRewardResultSchema } from "../rewards.js";
+import { sameLearningRef, learningRef } from "./contracts.js";
+import { compileRewardCheck, matchRewardFixture, RewardCheckRunSchema, RewardCheckRuntimeSchema } from "./reward-checks.js";
+import { requireLearningRelease, requireLearningResource } from "./repository.js";
+export function createRewardCheckWorker(repository, executor, options) {
+    const runtime = RewardCheckRuntimeSchema.parse(executor.runtime);
+    const now = options.now ?? (() => new Date().toISOString());
+    const active = new Map();
+    const key = (scope, id) => JSON.stringify([scope, id]);
+    const terminal = (run) => ["completed", "failed", "cancelled"].includes(run.status);
+    async function run(scope, checkId) {
+        const claim = await repository.transaction(scope, async (tx) => {
+            const current = await requireLearningResource(tx, "reward_check", checkId);
+            if (terminal(current) || (current.leaseExpiresAt && Date.parse(current.leaseExpiresAt) > Date.parse(now())))
+                return { acquired: false, run: current };
+            const claimed = RewardCheckRunSchema.parse({ ...current, revision: current.revision + 1,
+                status: current.status === "cancelling" ? "cancelling" : "running", runtime: current.status === "cancelling" ? current.runtime : runtime,
+                leaseOwner: options.workerId, leaseExpiresAt: new Date(Date.parse(now()) + current.timeoutMs + 30_000).toISOString(),
+                attemptCount: current.attemptCount + 1, results: current.status === "cancelling" ? current.results : [], matchesExpectations: null, failure: null, updatedAt: now(),
+            });
+            await tx.put("reward_check", claimed, current.revision, { parentId: claimed.reward.id, status: claimed.status });
+            return { acquired: true, run: claimed };
+        });
+        if (!claim.acquired)
+            return claim.run;
+        if (claim.run.status === "cancelling") {
+            const cleaned = await executor.cancel({ scope, run: claim.run });
+            return finish(scope, claim.run, cleaned ? "cancelled" : "cancelling", null);
+        }
+        const controller = new AbortController();
+        active.set(key(scope, checkId), controller);
+        const timer = setTimeout(() => controller.abort(new Error("reward_check_timeout")), claim.run.timeoutMs);
+        try {
+            const compiled = await repository.transaction(scope, async (tx) => {
+                const draft = await requireLearningRelease(tx, "draft", claim.run.draft);
+                if (draft.targetKind !== "reward")
+                    throw new Error("reward_check_draft_kind_invalid");
+                const base = draft.baseRelease ? await requireLearningRelease(tx, "reward", draft.baseRelease) : null;
+                const compiled = compileRewardCheck(draft, base);
+                if (compiled.snapshotHash !== claim.run.snapshotHash || !sameLearningRef(learningRef(compiled.reward), claim.run.reward) || contentHash(compiled.fixtureRefs) !== contentHash(claim.run.fixtureRefs))
+                    throw new Error("reward_check_snapshot_mismatch");
+                return compiled;
+            });
+            for (const fixture of compiled.fixtures) {
+                controller.signal.throwIfAborted();
+                const result = BoundRewardResultSchema.parse(await executor.execute({ ...compiled, scope, run: claim.run, fixture, signal: controller.signal }));
+                controller.signal.throwIfAborted();
+                if (result.graderId !== compiled.reward.id || !sameLearningRef(result.reward, claim.run.reward) || result.role !== "evaluation")
+                    throw new Error("reward_check_result_identity_mismatch");
+                const retained = await repository.transaction(scope, async (tx) => {
+                    const current = await requireLearningResource(tx, "reward_check", checkId);
+                    if (!owns(current, claim.run) || terminal(current))
+                        return current;
+                    const updated = RewardCheckRunSchema.parse({ ...current, revision: current.revision + 1, results: [...current.results, matchRewardFixture(fixture, result)], updatedAt: now() });
+                    await tx.put("reward_check", updated, current.revision, { parentId: current.reward.id, status: current.status });
+                    return updated;
+                });
+                if (!owns(retained, claim.run) || terminal(retained))
+                    return retained;
+                if (retained.status === "cancelling")
+                    controller.abort(new Error("reward_check_cancel_requested"));
+            }
+            controller.signal.throwIfAborted();
+            return await finish(scope, claim.run, "completed", null);
+        }
+        catch (error) {
+            if (controller.signal.aborted) {
+                const cleaned = await executor.cancel({ scope, run: claim.run });
+                if (!cleaned)
+                    return finish(scope, claim.run, "cancelling", "Execution cleanup is awaiting owner confirmation.");
+                if (controller.signal.reason instanceof Error && controller.signal.reason.message === "reward_check_cancel_requested")
+                    return finish(scope, claim.run, "cancelled", null);
+            }
+            return await finish(scope, claim.run, "failed", error instanceof Error ? error.message : "Reward checks failed.");
+        }
+        finally {
+            clearTimeout(timer);
+            active.delete(key(scope, checkId));
+        }
+    }
+    function owns(current, claim) { return current.leaseOwner === options.workerId && current.attemptCount === claim.attemptCount; }
+    async function finish(scope, claim, status, failure) {
+        return repository.transaction(scope, async (tx) => {
+            const current = await requireLearningResource(tx, "reward_check", claim.id);
+            if (!owns(current, claim) || terminal(current))
+                return current;
+            if (status === "completed" && current.results.length !== current.fixtureRefs.length)
+                throw new Error("reward_check_results_incomplete");
+            const updated = RewardCheckRunSchema.parse({ ...current, revision: current.revision + 1, status, failure,
+                matchesExpectations: status === "completed" ? current.results.every(result => result.matchesExpectation) : null,
+                leaseOwner: null, leaseExpiresAt: status === "cancelling" ? new Date(Date.parse(now()) + 5_000).toISOString() : null, updatedAt: now(),
+            });
+            await tx.put("reward_check", updated, current.revision, { parentId: current.reward.id, status: updated.status });
+            return updated;
+        });
+    }
+    return {
+        run,
+        requestCancellation(scope, id) { active.get(key(scope, id))?.abort(new Error("reward_check_cancel_requested")); },
+        async drain(scope, limit = 20) {
+            const candidates = await repository.transaction(scope, async (tx) => {
+                const rows = [];
+                for (const status of ["queued", "running", "cancelling"])
+                    rows.push(...(await tx.list("reward_check", { status, limit: Math.max(1, Math.min(100, limit)) })).items);
+                return rows;
+            });
+            const completed = [];
+            for (const candidate of candidates)
+                completed.push(await run(scope, candidate.id));
+            return completed;
+        },
+    };
+}

package/dist/learning/reward-checks.js

@@ -0,0 +1,81 @@
+import { z } from "zod";
+import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash } from "@openpond/harness";
+import { BoundRewardResultSchema, createRewardBinding, executeRewardBinding } from "../rewards.js";
+import { TaskRecordSchema } from "../tasksets.js";
+import { LearningJsonObjectSchema, LearningRevisionRefSchema, learningRef } from "./contracts.js";
+import { authoringNumber, compileRewardAuthoring, parseRewardAuthoringObject } from "./reward-authoring.js";
+import { LearningDomainError } from "./errors.js";
+export const RewardFixtureSchema = z.object({
+    id: ReleaseIdSchema, name: z.string().trim().min(1).max(500),
+    input: LearningJsonObjectSchema, output: LearningJsonObjectSchema,
+    expectedOutput: LearningJsonObjectSchema.nullable(), evaluatorContext: LearningJsonObjectSchema.nullable(),
+    artifactRefs: z.array(ReleaseIdSchema).max(100), runtimeEventRefs: z.array(ReleaseIdSchema).max(100),
+    infrastructureError: z.string().max(20_000).nullable(),
+    expected: z.discriminatedUnion("status", [
+        z.object({ status: z.literal("scored"), minimum: z.number().finite(), maximum: z.number().finite(), passed: z.boolean().nullable() }).strict(),
+        z.object({ status: z.enum(["pending", "unavailable", "failed"]) }).strict(),
+    ]),
+}).strict().superRefine((fixture, context) => {
+    if (fixture.expected.status === "scored" && fixture.expected.minimum > fixture.expected.maximum)
+        context.addIssue({ code: "custom", path: ["expected"], message: "Expected score minimum must not exceed maximum." });
+});
+export const RewardCheckRuntimeSchema = z.object({ id: ReleaseIdSchema, packageVersion: z.string().min(1).max(100), engine: z.string().min(1).max(200) }).strict();
+export const RewardFixtureCheckResultSchema = z.object({
+    fixture: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict(),
+    result: BoundRewardResultSchema, matchesExpectation: z.boolean(),
+}).strict();
+export const RewardCheckRunSchema = z.object({
+    schemaVersion: z.literal("openpond.rewardCheckRun.v1"), id: ReleaseIdSchema, revision: z.number().int().positive(),
+    draft: LearningRevisionRefSchema, reward: LearningRevisionRefSchema, snapshotHash: ReleaseHashSchema,
+    fixtureRefs: z.array(z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict()).min(1).max(50),
+    status: z.enum(["queued", "running", "completed", "failed", "cancelling", "cancelled"]),
+    runtime: RewardCheckRuntimeSchema.nullable(), results: z.array(RewardFixtureCheckResultSchema).max(50),
+    matchesExpectations: z.boolean().nullable(), failure: z.string().max(20_000).nullable(),
+    timeoutMs: z.number().int().min(100).max(300_000), maximumSpendUsd: z.number().nonnegative().max(1_000),
+    leaseOwner: ReleaseIdSchema.nullable(), leaseExpiresAt: ReleaseTimestampSchema.nullable(), attemptCount: z.number().int().nonnegative(),
+    createdAt: ReleaseTimestampSchema, updatedAt: ReleaseTimestampSchema,
+}).strict();
+export function compileRewardFixtures(fields) {
+    if (!fields.length || fields.length > 50)
+        throw new LearningDomainError("reward_check_fixtures_required", 422, "Add between one and 50 fixtures before checking this Reward.");
+    if (new Set(fields.map(field => field.id)).size !== fields.length)
+        throw new LearningDomainError("reward_fixture_id_duplicate", 422);
+    return fields.map(field => RewardFixtureSchema.parse({
+        id: field.id, name: field.name,
+        input: parseRewardAuthoringObject(field.input, `${field.name}: input`), output: parseRewardAuthoringObject(field.output, `${field.name}: output`),
+        expectedOutput: field.expectedOutput.trim() ? parseRewardAuthoringObject(field.expectedOutput, `${field.name}: expected answer`) : null,
+        evaluatorContext: field.evaluatorContext.trim() ? parseRewardAuthoringObject(field.evaluatorContext, `${field.name}: evaluator context`) : null,
+        artifactRefs: field.artifactRefs.map(value => value.trim()).filter(Boolean), runtimeEventRefs: field.runtimeEventRefs.map(value => value.trim()).filter(Boolean), infrastructureError: field.infrastructureError || null,
+        expected: field.expectedStatus === "scored" ? { status: field.expectedStatus, minimum: authoringNumber(field.minimumScore, `${field.name}: minimum score`), maximum: authoringNumber(field.maximumScore, `${field.name}: maximum score`), passed: field.expectedPassed === "any" ? null : field.expectedPassed === "true" } : { status: field.expectedStatus },
+    }));
+}
+export function compileRewardCheck(draft, base) {
+    const compiled = compileRewardAuthoring({ id: draft.targetId, fields: draft.fields, base });
+    const fixtures = compileRewardFixtures(draft.fields.fixtures ?? []);
+    const fixtureRefs = fixtures.map(fixture => ({ id: fixture.id, contentHash: contentHash(fixture) }));
+    return { ...compiled, fixtures, fixtureRefs, snapshotHash: contentHash({ reward: compiled.reward, fixtures }) };
+}
+/** Fixture data is authored test data, never an owner-recorded production attempt. */
+export async function executeRewardFixture(input) {
+    const fixture = RewardFixtureSchema.parse(input.fixture);
+    const task = TaskRecordSchema.parse({
+        id: fixture.id, clusterKey: fixture.id, split: "validation", input: fixture.input,
+        expectedOutput: fixture.expectedOutput, privilegedContextRef: null,
+    });
+    const binding = createRewardBinding({
+        schemaVersion: "openpond.rewardBinding.v1", id: `fixture-binding-${contentHash(learningRef(input.reward))}`, revision: 1,
+        sources: [{ graderId: input.reward.id, reward: learningRef(input.reward), role: "evaluation", normalization: input.reward.rawScore.minimum === 0 && input.reward.rawScore.maximum === 1 ? { kind: "identity" } : { kind: "linear", minimum: input.reward.rawScore.minimum, maximum: input.reward.rawScore.maximum, direction: "higher" }, weight: 1, required: true, hardGate: false, privileged: true, fixtureRefs: [] }],
+        aggregation: "weighted_mean", unscorable: "exclude_optional_require_all_required",
+    }, [input.reward]);
+    const composition = await executeRewardBinding({ binding, rewards: [input.reward], task,
+        evidence: { output: fixture.output, artifactRefs: fixture.artifactRefs, runtimeEventRefs: fixture.runtimeEventRefs, infrastructureError: fixture.infrastructureError },
+        signal: input.signal, customVerifier: input.customVerifier, modelJudge: input.modelJudge,
+    });
+    return composition.results[0];
+}
+export function matchRewardFixture(fixture, result) {
+    const expected = fixture.expected;
+    const matchesExpectation = result.status === expected.status && (expected.status !== "scored"
+        || (result.rawScore !== null && result.rawScore >= expected.minimum && result.rawScore <= expected.maximum && (expected.passed === null || expected.passed === result.passed)));
+    return RewardFixtureCheckResultSchema.parse({ fixture: { id: fixture.id, contentHash: contentHash(fixture) }, result, matchesExpectation });
+}

package/dist/learning/reward-fixture-executor.js

@@ -0,0 +1,29 @@
+import { verifyLearningTextAsset } from "./assets.js";
+import { executeRewardFixture } from "./reward-checks.js";
+/** The supplied interpreter must settle only after its worker/process exits. */
+export function createIsolatedRewardFixtureExecutor(options) {
+    return {
+        runtime: options.runtime,
+        execute(input) {
+            return executeRewardFixture({
+                reward: input.reward, fixture: input.fixture, signal: input.signal,
+                customVerifier: async ({ grader, task, evidence }) => {
+                    const asset = input.assets.find(asset => asset.id === grader.verifierRef.id);
+                    if (!asset)
+                        throw new Error("reward_check_source_missing");
+                    const source = verifyLearningTextAsset(asset, grader.verifierRef);
+                    const result = await options.executeJavaScript({
+                        source, exportName: grader.exportName, signal: input.signal,
+                        timeoutMs: Math.min(grader.timeoutMs, input.run.timeoutMs),
+                        value: { task, attempt: evidence, input: task.input, output: evidence.output, expectedOutput: task.expectedOutput,
+                            evaluatorContext: input.fixture.evaluatorContext, infrastructureError: evidence.infrastructureError ?? null },
+                    });
+                    return { score: result.score, passed: result.passed, rewardEligible: grader.rewardEligible, failureClass: null,
+                        feedback: [result.feedback], visibleEvidenceRefs: [], privilegedEvidenceRefs: result.evidenceRefs };
+                },
+            });
+        },
+        // No remote allocation; executeJavaScript's settlement contract confirms cleanup.
+        async cancel() { return true; },
+    };
+}

package/dist/learning/service.js

@@ -1,4 +1,5 @@
 import { saveAuthoringDraft, archiveAuthoringDraft, finalizeAuthoringDraft } from "./authoring-service.js";
+import { queueRewardCheck, cancelRewardCheck } from "./reward-check-service.js";
 import { LearningDomainError } from "./errors.js";
 import { contentHash } from "@openpond/harness";
 import { createRewardBinding, createRewardRelease, resolveBoundRewards } from "../rewards.js";
@@ -28,6 +29,12 @@ export function createLearningService(repository, options = {}) {
             }
             let pointers;
             switch (input.action) {
+                case "queue_reward_check":
+                    pointers = [await queueRewardCheck(transaction, input, operationId, now())];
+                    break;
+                case "cancel_reward_check":
+                    pointers = [await cancelRewardCheck(transaction, input, now())];
+                    break;
                 case "save_draft":
                     pointers = [await saveAuthoringDraft(transaction, input, now())];
                     break;

package/dist/rewards.js

@@ -21,7 +21,10 @@ export const RewardReleaseContentSchema = z.object({
     implementation: RewardImplementationSchema,
     rawScore: z.object({ minimum: z.number().finite(), maximum: z.number().finite() }).strict(),
     assets: z.array(ImmutableAssetRefSchema).max(1_000),
+    fixtureSetRef: ImmutableAssetRefSchema.optional(),
 }).strict().superRefine((reward, context) => {
+    if (reward.fixtureSetRef && (reward.fixtureSetRef.visibility !== "verifier" || !reward.assets.some(asset => contentHash(asset) === contentHash(reward.fixtureSetRef))))
+        context.addIssue({ code: "custom", path: ["fixtureSetRef"], message: "Reward fixtures must be included as a private verifier asset." });
     if (reward.rawScore.minimum >= reward.rawScore.maximum)
         context.addIssue({ code: "custom", path: ["rawScore"], message: "A raw score contract requires minimum < maximum." });
     if (reward.implementation.kind !== "learned_model" && (reward.rawScore.minimum !== 0 || reward.rawScore.maximum !== 1))

package/dist/sql-execution-contract.js

@@ -0,0 +1,52 @@
+import { z } from "zod";
+import { assertBoundedTaskJson } from "./task-schema.js";
+const integer = z.string().regex(/^(0|-?[1-9][0-9]*)$/).max(20).refine(value => {
+    if (value.length > 20 || !/^(0|-?[1-9][0-9]*)$/.test(value))
+        return false;
+    const parsed = BigInt(value);
+    return parsed >= -(1n << 63n) && parsed < (1n << 63n);
+}, "SQL integers must fit signed 64-bit storage.");
+export const SqlValueSchema = z.union([
+    z.null(), z.string().max(262_144),
+    z.number().finite(),
+    z.object({ integer }).strict(),
+    z.object({ blobBase64: z.string().max(349_528).regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/) }).strict(),
+]);
+const identifier = z.string().min(1).max(128).refine(value => !value.includes("\0"));
+export const SqlSnapshotSchema = z.object({
+    tables: z.array(z.object({
+        name: identifier,
+        columns: z.array(z.object({ name: identifier, type: z.enum(["INTEGER", "REAL", "TEXT", "BLOB"]) }).strict()).min(1).max(128),
+        rows: z.array(z.array(SqlValueSchema).max(128)).max(4_096),
+    }).strict()).max(64),
+}).strict().superRefine((snapshot, context) => {
+    const names = new Set();
+    for (const [index, table] of snapshot.tables.entries()) {
+        const name = table.name.toLowerCase();
+        if (names.has(name) || name.startsWith("sqlite_"))
+            context.addIssue({ code: "custom", path: ["tables", index, "name"], message: "Table names must be unique and outside SQLite's reserved namespace." });
+        names.add(name);
+        if (new Set(table.columns.map(column => column.name.toLowerCase())).size !== table.columns.length)
+            context.addIssue({ code: "custom", path: ["tables", index, "columns"], message: "Column names must be unique." });
+        if (table.rows.some(row => row.length !== table.columns.length))
+            context.addIssue({ code: "custom", path: ["tables", index, "rows"], message: "Each row must match the declared columns." });
+    }
+});
+export const SqlExecutionRequestSchema = z.object({
+    schemaVersion: z.literal("openpond.sqlExecution.v1"),
+    snapshot: SqlSnapshotSchema,
+    sql: z.string().min(1).max(65_536).refine(value => !value.includes("\0")),
+    maxRows: z.number().int().min(1).max(4_096),
+    maxResultBytes: z.number().int().min(256).max(262_144),
+}).strict();
+export const SqlExecutionResultSchema = z.discriminatedUnion("status", [
+    z.object({ status: z.literal("completed"), columns: z.array(z.string()).max(128), rows: z.array(z.array(SqlValueSchema).max(128)).max(4_096) }).strict(),
+    z.object({ status: z.literal("rejected"), code: z.enum(["invalid_query", "multiple_statements", "result_too_large", "memory_limit"]) }).strict(),
+]);
+export function assertSqlExecutionRequest(value) {
+    assertBoundedTaskJson(value, 1_048_576);
+    const parsed = SqlExecutionRequestSchema.parse(value);
+    if (new TextEncoder().encode(parsed.sql).byteLength > 65_536)
+        throw new Error("sql_source_too_large");
+    return parsed;
+}