@openpond/evals 0.5.0 → 0.7.0

package/dist/javascript-verifier-worker.js

@@ -0,0 +1,6 @@
+import { parentPort, workerData } from "node:worker_threads";
+import { executeJavaScriptVerifier } from "./javascript-verifier.js";
+// Only this trusted host program runs in Node. Authored source is interpreter data.
+if (!parentPort)
+    throw new Error("Verifier worker requires its execution owner.");
+void executeJavaScriptVerifier(workerData).then((result) => parentPort.postMessage({ ok: true, result }), (error) => parentPort.postMessage({ ok: false, error: error instanceof Error ? error.message : "Verifier execution failed." })).finally(() => parentPort.close());

package/dist/javascript-verifier.js

@@ -0,0 +1,64 @@
+import { newQuickJSWASMModuleFromVariant, Scope } from "quickjs-emscripten-core";
+import variant from "@jitl/quickjs-singlefile-cjs-release-sync";
+import { assertBoundedTaskJson } from "./task-schema.js";
+import { JavaScriptVerifierResultSchema } from "./javascript-verifier-contract.js";
+export { JavaScriptVerifierResultSchema } from "./javascript-verifier-contract.js";
+/**
+ * A fresh WebAssembly interpreter with no host functions, filesystem, network,
+ * timers, or module loader. Hosts should run this on a worker to remain responsive
+ * and terminate that worker when cancellation is requested.
+ */
+export async function executeJavaScriptVerifier(input) {
+    if (!Number.isInteger(input.timeoutMs) || input.timeoutMs < 1 || input.timeoutMs > 300_000)
+        throw new Error("verifier_timeout_invalid");
+    if (new TextEncoder().encode(input.source).byteLength > 524_288)
+        throw new Error("verifier_source_too_large");
+    assertBoundedTaskJson(input.value, 4_194_304);
+    input.signal?.throwIfAborted();
+    const deadline = Date.now() + input.timeoutMs;
+    const module = await newQuickJSWASMModuleFromVariant(variant);
+    input.signal?.throwIfAborted();
+    return Scope.withScope((scope) => {
+        const runtime = scope.manage(module.newRuntime());
+        runtime.setMemoryLimit(33_554_432);
+        runtime.setMaxStackSize(262_144);
+        runtime.setInterruptHandler(() => Date.now() >= deadline || input.signal?.aborted === true);
+        const context = scope.manage(runtime.newContext());
+        const stringify = scope.manage(context.unwrapResult(context.evalCode("JSON.stringify")));
+        const argument = scope.manage(context.unwrapResult(context.evalCode(`JSON.parse(${JSON.stringify(JSON.stringify(input.value))})`)));
+        const exported = scope.manage(context.unwrapResult(context.evalCode(input.source, "verifier.mjs", { type: "module" })));
+        const namespace = scope.manage(settle(context, exported, deadline, input.signal));
+        const fn = scope.manage(context.getProp(namespace, input.exportName ?? "verify"));
+        if (context.typeof(fn) !== "function")
+            throw new Error("verifier_function_export_missing");
+        const returned = scope.manage(context.unwrapResult(context.callFunction(fn, context.undefined, argument)));
+        const result = scope.manage(settle(context, returned, deadline, input.signal));
+        const serialized = scope.manage(context.unwrapResult(context.callFunction(stringify, context.undefined, result)));
+        if (context.typeof(serialized) !== "string")
+            throw new Error("verifier_result_not_json");
+        const json = context.getString(serialized);
+        if (new TextEncoder().encode(json).byteLength > 65_536)
+            throw new Error("verifier_result_too_large");
+        input.signal?.throwIfAborted();
+        if (Date.now() >= deadline)
+            throw new Error("verifier_timeout");
+        const value = JSON.parse(json);
+        assertBoundedTaskJson(value, 65_536);
+        return JavaScriptVerifierResultSchema.parse(value);
+    });
+}
+function settle(context, handle, deadline, signal) {
+    for (;;) {
+        signal?.throwIfAborted();
+        if (Date.now() >= deadline)
+            throw new Error("verifier_timeout");
+        const state = context.getPromiseState(handle);
+        if (state.type === "fulfilled")
+            return state.notAPromise ? state.value.dup() : state.value;
+        if (state.type === "rejected")
+            return context.unwrapResult(state);
+        const jobs = context.runtime.executePendingJobs(1);
+        if (jobs.unwrap() === 0)
+            throw new Error("verifier_unresolved_promise");
+    }
+}

package/dist/learned-preference.js

@@ -0,0 +1,334 @@
+import { z } from "zod";
+import { ImmutableReleaseRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
+import { verifyComparisonAssignment, verifyPreferenceReceipt, } from "./preferences.js";
+const PreferenceDatasetPartitionSchema = z.enum([
+    "reward_train",
+    "reward_validation",
+    "reward_qualification",
+]);
+export const PreferenceEvidenceAuthoritySchema = z.enum([
+    "human",
+    "synthetic_fixture",
+]);
+export const RewardModelQualificationKindSchema = z.enum([
+    "synthetic_smoke",
+    "human_heldout",
+]);
+const PreferenceDatasetGroupSchema = z.object({
+    id: ReleaseIdSchema,
+    assignmentRef: ImmutableReleaseRefSchema,
+    scenarioRef: ImmutableReleaseRefSchema,
+    scenarioSplit: z.enum(["train", "validation"]),
+    preferenceResultRef: ImmutableReleaseRefSchema,
+    receiptRefs: z.array(ImmutableReleaseRefSchema).min(1).max(100),
+    attemptRefs: z.array(ImmutableReleaseRefSchema).min(2).max(4),
+    artifactManifestRefs: z.array(ImmutableReleaseRefSchema).min(2).max(4),
+    orderedBuckets: z.array(z.array(ReleaseIdSchema).min(1).max(4)).max(4),
+    rejectAll: z.boolean(),
+    partition: PreferenceDatasetPartitionSchema,
+    metadata: MetadataSchema,
+}).strict().superRefine((group, context) => {
+    const attemptIds = group.attemptRefs.map((attempt) => attempt.id);
+    if (new Set(attemptIds).size !== attemptIds.length) {
+        context.addIssue({
+            code: "custom",
+            path: ["attemptRefs"],
+            message: "Preference dataset group Attempt refs must be unique.",
+        });
+    }
+    if (group.artifactManifestRefs.length !== group.attemptRefs.length) {
+        context.addIssue({
+            code: "custom",
+            path: ["artifactManifestRefs"],
+            message: "Every preference dataset Attempt requires one Artifact Manifest ref.",
+        });
+    }
+    const bucketIds = group.orderedBuckets.flat();
+    if (group.rejectAll && bucketIds.length > 0) {
+        context.addIssue({
+            code: "custom",
+            path: ["orderedBuckets"],
+            message: "A reject-all preference dataset group cannot contain ordered buckets.",
+        });
+    }
+    if (!group.rejectAll
+        && (bucketIds.length !== attemptIds.length
+            || new Set(bucketIds).size !== bucketIds.length
+            || bucketIds.some((attemptId) => !attemptIds.includes(attemptId)))) {
+        context.addIssue({
+            code: "custom",
+            path: ["orderedBuckets"],
+            message: "Ordered buckets must contain every group Attempt exactly once.",
+        });
+    }
+});
+const PreferenceDatasetPairSchema = z.object({
+    groupId: ReleaseIdSchema,
+    preferredAttemptRef: ImmutableReleaseRefSchema,
+    dispreferredAttemptRef: ImmutableReleaseRefSchema,
+    relation: z.enum(["preferred", "tie"]),
+}).strict().superRefine((pair, context) => {
+    if (pair.preferredAttemptRef.id === pair.dispreferredAttemptRef.id
+        && pair.preferredAttemptRef.contentHash === pair.dispreferredAttemptRef.contentHash) {
+        context.addIssue({
+            code: "custom",
+            message: "A derived preference pair must reference two different Attempts.",
+        });
+    }
+});
+export const PreferenceDatasetReleaseContentSchema = z.object({
+    schemaVersion: z.literal("openpond.preferenceDatasetRelease.v1"),
+    id: ReleaseIdSchema,
+    revision: z.number().int().positive(),
+    tasksetRelease: ImmutableReleaseRefSchema,
+    comparisonRelease: ImmutableReleaseRefSchema,
+    authority: PreferenceEvidenceAuthoritySchema,
+    qualificationEligibility: z.enum(["smoke_only", "human_heldout"]),
+    fixtureRelease: ImmutableReleaseRefSchema.nullable(),
+    groups: z.array(PreferenceDatasetGroupSchema).min(1).max(100_000),
+    derivedPairs: z.array(PreferenceDatasetPairSchema).max(1_000_000),
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+}).strict().superRefine((release, context) => {
+    if (release.authority === "synthetic_fixture"
+        && (release.qualificationEligibility !== "smoke_only"
+            || release.fixtureRelease === null)) {
+        context.addIssue({
+            code: "custom",
+            path: ["qualificationEligibility"],
+            message: "Synthetic preference datasets require a fixture release and are smoke-only.",
+        });
+    }
+    if (release.authority === "human"
+        && (release.qualificationEligibility !== "human_heldout"
+            || release.fixtureRelease !== null)) {
+        context.addIssue({
+            code: "custom",
+            path: ["authority"],
+            message: "Human preference datasets must be human-heldout eligible and cannot bind a fixture release.",
+        });
+    }
+    const groupIds = release.groups.map((group) => group.id);
+    if (new Set(groupIds).size !== groupIds.length) {
+        context.addIssue({
+            code: "custom",
+            path: ["groups"],
+            message: "Preference dataset group IDs must be unique.",
+        });
+    }
+    const groups = new Map(release.groups.map((group) => [group.id, group]));
+    for (const [index, pair] of release.derivedPairs.entries()) {
+        const group = groups.get(pair.groupId);
+        if (!group) {
+            context.addIssue({
+                code: "custom",
+                path: ["derivedPairs", index, "groupId"],
+                message: "Derived preference pairs must reference a group in the same release.",
+            });
+            continue;
+        }
+        const attemptIds = new Set(group.attemptRefs.map((attempt) => attempt.id));
+        if (!attemptIds.has(pair.preferredAttemptRef.id)
+            || !attemptIds.has(pair.dispreferredAttemptRef.id)) {
+            context.addIssue({
+                code: "custom",
+                path: ["derivedPairs", index],
+                message: "Derived preference pairs must reference Attempts in their source group.",
+            });
+        }
+    }
+});
+export const PreferenceDatasetReleaseSchema = PreferenceDatasetReleaseContentSchema
+    .extend({ contentHash: ReleaseHashSchema })
+    .strict();
+const RewardModelQualificationMetricsSchema = z.object({
+    sampleCount: z.number().int().positive(),
+    finiteScoreRate: z.number().min(0).max(1),
+    scoreVariance: z.number().nonnegative(),
+    checkpointReloadPassed: z.boolean(),
+    processorCompatibilityPassed: z.boolean(),
+    invalidAttemptExclusionPassed: z.boolean(),
+    orderedPairAccuracy: z.number().min(0).max(1).nullable(),
+    bucketAccuracy: z.number().min(0).max(1).nullable(),
+    tieAgreement: z.number().min(0).max(1).nullable(),
+}).strict();
+export const RewardModelQualificationReportContentSchema = z.object({
+    schemaVersion: z.literal("openpond.rewardModelQualificationReport.v1"),
+    id: ReleaseIdSchema,
+    kind: RewardModelQualificationKindSchema,
+    rewardModelVersion: ImmutableReleaseRefSchema,
+    preferenceDatasetRelease: ImmutableReleaseRefSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    processorRelease: ImmutableReleaseRefSchema,
+    metrics: RewardModelQualificationMetricsSchema,
+    passed: z.boolean(),
+    productionRewardEligible: z.boolean(),
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+}).strict().superRefine((report, context) => {
+    if (report.kind === "synthetic_smoke" && report.productionRewardEligible) {
+        context.addIssue({
+            code: "custom",
+            path: ["productionRewardEligible"],
+            message: "Synthetic-smoke Reward Model qualification can never grant production reward eligibility.",
+        });
+    }
+    if (report.kind === "human_heldout"
+        && report.productionRewardEligible !== report.passed) {
+        context.addIssue({
+            code: "custom",
+            path: ["productionRewardEligible"],
+            message: "Human-heldout reward eligibility must match the frozen qualification outcome.",
+        });
+    }
+});
+export const RewardModelQualificationReportSchema = RewardModelQualificationReportContentSchema
+    .extend({ contentHash: ReleaseHashSchema })
+    .strict();
+export function createPreferenceDatasetRelease(input) {
+    const content = PreferenceDatasetReleaseContentSchema.parse(input);
+    return PreferenceDatasetReleaseSchema.parse({
+        ...content,
+        contentHash: contentHash(content),
+    });
+}
+export function materializePreferenceDatasetRelease(input) {
+    if (input.groups.length === 0) {
+        throw new Error("Preference dataset materialization requires at least one reviewed group.");
+    }
+    const tasksetRef = releaseRef(input.tasksetRelease);
+    const comparisonRef = releaseRef(input.comparisonRelease);
+    const fixtureRefs = new Map();
+    const groups = input.groups.map(({ assignment, receipt, partition }) => {
+        if (!verifyComparisonAssignment(assignment)) {
+            throw new Error("Preference dataset assignment failed immutable verification.");
+        }
+        if (!verifyPreferenceReceipt(receipt)) {
+            throw new Error("Preference dataset receipt failed immutable verification.");
+        }
+        if (!sameReleaseRef(assignment.lineage.tasksetRelease, tasksetRef)) {
+            throw new Error("Preference dataset assignment belongs to a different Taskset release.");
+        }
+        if (!sameReleaseRef(assignment.comparisonRelease, comparisonRef)) {
+            throw new Error("Preference dataset assignment belongs to a different comparison release.");
+        }
+        if (!sameReleaseRef(receipt.assignmentRef, releaseRef(assignment))) {
+            throw new Error("Preference dataset receipt belongs to a different assignment.");
+        }
+        if (input.authority === "synthetic_fixture") {
+            if (receipt.reviewer.kind !== "fixture") {
+                throw new Error("Synthetic preference datasets accept only fixture receipts.");
+            }
+            fixtureRefs.set(receipt.reviewer.fixtureRelease.contentHash, receipt.reviewer.fixtureRelease);
+        }
+        else if (receipt.reviewer.kind !== "human") {
+            throw new Error("Human preference datasets accept only human receipts.");
+        }
+        const scenario = input.tasksetRelease.tasks.find((task) => task.id === assignment.taskRef.id);
+        if (!scenario || (scenario.split !== "train" && scenario.split !== "validation")) {
+            throw new Error("Preference model datasets require train or validation scenarios.");
+        }
+        if (partition === "reward_train" && scenario.split !== "train") {
+            throw new Error("Reward training groups must use train scenarios.");
+        }
+        if (partition !== "reward_train" && scenario.split !== "validation") {
+            throw new Error("Reward validation and qualification groups must use validation scenarios.");
+        }
+        return {
+            id: `preference-group:${assignment.id}`,
+            assignmentRef: releaseRef(assignment),
+            scenarioRef: assignment.taskRef,
+            scenarioSplit: scenario.split,
+            preferenceResultRef: releaseRef(receipt),
+            receiptRefs: [releaseRef(receipt)],
+            attemptRefs: assignment.candidates.map((candidate) => candidate.attemptRef),
+            artifactManifestRefs: assignment.candidates.map((candidate) => candidate.artifactManifestRef),
+            orderedBuckets: receipt.order,
+            rejectAll: receipt.rejectAll,
+            partition,
+            metadata: { purpose: assignment.purpose, ...assignment.metadata },
+        };
+    });
+    if (input.authority === "synthetic_fixture" && fixtureRefs.size !== 1) {
+        throw new Error("Synthetic preference dataset groups must share one immutable fixture release.");
+    }
+    const fixtureRelease = input.authority === "synthetic_fixture"
+        ? [...fixtureRefs.values()][0]
+        : null;
+    return createPreferenceDatasetRelease({
+        schemaVersion: "openpond.preferenceDatasetRelease.v1",
+        id: input.id,
+        revision: input.revision,
+        tasksetRelease: tasksetRef,
+        comparisonRelease: comparisonRef,
+        authority: input.authority,
+        qualificationEligibility: input.authority === "synthetic_fixture" ? "smoke_only" : "human_heldout",
+        fixtureRelease,
+        groups,
+        derivedPairs: groups.flatMap(derivePairs),
+        createdAt: input.createdAt,
+        metadata: input.metadata ?? {},
+    });
+}
+export function verifyPreferenceDatasetRelease(value) {
+    return verifyHashed(value, PreferenceDatasetReleaseContentSchema, PreferenceDatasetReleaseSchema);
+}
+export function createRewardModelQualificationReport(input) {
+    const content = RewardModelQualificationReportContentSchema.parse(input);
+    return RewardModelQualificationReportSchema.parse({
+        ...content,
+        contentHash: contentHash(content),
+    });
+}
+export function verifyRewardModelQualificationReport(value) {
+    return verifyHashed(value, RewardModelQualificationReportContentSchema, RewardModelQualificationReportSchema);
+}
+function verifyHashed(value, contentSchema, fullSchema) {
+    const parsed = fullSchema.safeParse(value);
+    if (!parsed.success || !parsed.data)
+        return false;
+    const { contentHash: actual, ...content } = parsed.data;
+    try {
+        return contentHash(contentSchema.parse(content)) === actual;
+    }
+    catch {
+        return false;
+    }
+}
+function derivePairs(group) {
+    if (group.rejectAll)
+        return [];
+    const attemptById = new Map(group.attemptRefs.map((attempt) => [attempt.id, attempt]));
+    const pairs = [];
+    for (const [bucketIndex, bucket] of group.orderedBuckets.entries()) {
+        for (let left = 0; left < bucket.length; left += 1) {
+            for (let right = left + 1; right < bucket.length; right += 1) {
+                pairs.push({
+                    groupId: group.id,
+                    preferredAttemptRef: attemptById.get(bucket[left]),
+                    dispreferredAttemptRef: attemptById.get(bucket[right]),
+                    relation: "tie",
+                });
+            }
+        }
+        for (const lowerBucket of group.orderedBuckets.slice(bucketIndex + 1)) {
+            for (const preferredId of bucket) {
+                for (const dispreferredId of lowerBucket) {
+                    pairs.push({
+                        groupId: group.id,
+                        preferredAttemptRef: attemptById.get(preferredId),
+                        dispreferredAttemptRef: attemptById.get(dispreferredId),
+                        relation: "preferred",
+                    });
+                }
+            }
+        }
+    }
+    return pairs;
+}
+function releaseRef(value) {
+    return { id: value.id, contentHash: value.contentHash };
+}
+function sameReleaseRef(left, right) {
+    return left.id === right.id && left.contentHash === right.contentHash;
+}

package/dist/learning/admission.js

@@ -0,0 +1,209 @@
+import { LearningDomainError } from "./errors.js";
+import { z } from "zod";
+import { contentHash } from "@openpond/harness";
+import { compileBoundGraders, resolveBoundRewards, RewardBindingSchema, RewardReleaseSchema } from "../rewards.js";
+import { TaskRecordSchema, TasksetReleaseContentSchema, TasksetReleaseSchema } from "../tasksets.js";
+import { assertBoundedTaskJson, validateTaskValue } from "../task-schema.js";
+import { assertLearningContentHash, LearningJsonObjectSchema, learningRef, LearningRevisionRefSchema, LearningSourceSchema, sameLearningRef, sealLearningContent, TaskAdmissionDecisionSchema, TaskBatchContentSchema, TaskBatchSchema, TaskDefinitionSchema, TaskEvidenceSchema, TaskExampleSubmissionSchema } from "./contracts.js";
+export function inspectTaskEvidence(evidence, definition) {
+    assertLearningContentHash(TaskEvidenceSchema.parse(evidence));
+    assertLearningContentHash(TaskDefinitionSchema.parse(definition));
+    const example = evidence.submission;
+    const issues = [];
+    if (!sameLearningRef(example.taskDefinition, learningRef(definition)))
+        issues.push({ path: "/taskDefinition", code: "definition_mismatch", message: "Evidence must pin this exact task definition." });
+    const input = validateTaskValue(definition.inputSchema, example.input);
+    issues.push(...input.issues.map((issue) => ({ ...issue, path: `/input${issue.path}` })));
+    if (example.expected !== null) {
+        const expected = validateTaskValue(definition.outputSchema, example.expected);
+        issues.push(...expected.issues.map((issue) => ({ ...issue, path: `/expected${issue.path}` })));
+    }
+    const evidenceValidity = issues.length ? "invalid" : "valid";
+    if (!example.familyKey)
+        issues.push({ path: "/familyKey", code: "family_unresolved", message: "Resolve the source family before admitting this task." });
+    return { evidenceValidity, taskReady: issues.length === 0, observedOutputValid: example.observedOutput === null ? null : validateTaskValue(definition.outputSchema, example.observedOutput).valid, issues };
+}
+export function validateSourceSubmission(sourceInput, submissionInput) {
+    const source = LearningSourceSchema.parse(sourceInput);
+    assertLearningContentHash(source);
+    assertBoundedTaskJson(submissionInput);
+    const submission = TaskExampleSubmissionSchema.parse(submissionInput);
+    if (!source.enabled)
+        throw new LearningDomainError("learning_source_disabled", 409);
+    if (source.id !== submission.sourceId || !sameLearningRef(source.taskDefinition, submission.taskDefinition))
+        throw new LearningDomainError("learning_source_definition_mismatch", 422);
+    if (!source.allowedSplits.includes(submission.split))
+        throw new LearningDomainError("learning_source_split_not_allowed", 422);
+    const mappingHash = source.mapping ? contentHash(source.mapping) : null;
+    if (mappingHash !== submission.provenance.mappingHash)
+        throw new LearningDomainError("learning_source_mapping_mismatch", 422);
+    return submission;
+}
+export function mapTaskSourceRecord(input) {
+    assertBoundedTaskJson(input.record);
+    const source = LearningSourceSchema.parse(input.source);
+    assertLearningContentHash(source);
+    if (!source.mapping)
+        throw new LearningDomainError("learning_source_mapping_required", 422);
+    const mapping = source.mapping;
+    const fields = ["exampleId", "attemptId", "occurredAt", "input", "familyKey"];
+    const values = Object.fromEntries(fields.map((field) => [field, readTaskJsonPointer(input.record, mapping[field])]));
+    const missing = fields.filter((field) => values[field] === undefined || values[field] === null).map(String);
+    if (missing.length)
+        return { status: "pending", missing };
+    const example = TaskExampleSubmissionSchema.parse({
+        schemaVersion: "openpond.taskExample.v1", sourceId: source.id, idempotencyKey: input.idempotencyKey,
+        taskDefinition: source.taskDefinition, ...values, split: mapping.split,
+        observedOutput: mapping.observedOutput === null ? null : readTaskJsonPointer(input.record, mapping.observedOutput) ?? null,
+        expected: mapping.expected === null ? null : readTaskJsonPointer(input.record, mapping.expected) ?? null,
+        evaluatorContext: mapping.evaluatorContext === null ? null : readTaskJsonPointer(input.record, mapping.evaluatorContext) ?? null,
+        assets: [], provenance: { sourceRecordRef: null, mappingHash: contentHash(mapping) },
+    });
+    return { status: "mapped", example: validateSourceSubmission(source, example) };
+}
+export function readTaskJsonPointer(value, pointer) {
+    if (pointer === "")
+        return value;
+    if (!pointer.startsWith("/") || /~(?![01])/u.test(pointer))
+        throw new LearningDomainError("task_json_pointer_invalid", 422);
+    let current = value;
+    for (const part of pointer.slice(1).split("/")) {
+        const key = part.replaceAll("~1", "/").replaceAll("~0", "~");
+        if (!current || typeof current !== "object" || !Object.hasOwn(current, key))
+            return undefined;
+        current = current[key];
+    }
+    return current;
+}
+export function taskRecordFromEvidence(evidence, definition) {
+    const inspection = inspectTaskEvidence(evidence, definition);
+    if (!inspection.taskReady)
+        throw new LearningDomainError("task_evidence_not_ready", 422, inspection.issues[0].code);
+    return TaskRecordSchema.parse({
+        id: evidence.id, clusterKey: evidence.submission.familyKey, split: evidence.submission.split,
+        input: evidence.submission.input, expectedOutput: evidence.submission.expected,
+        policyVisibleContext: { instructions: definition.instructions },
+        privilegedContextRef: evidence.submission.evaluatorContext === null ? null : `task-evidence:${evidence.contentHash}`,
+        artifactRefs: evidence.submission.assets, tags: [definition.category],
+    });
+}
+export function taskAttemptEvidence(evidence, output) {
+    return { output, runtimeEventRefs: [], artifactRefs: evidence.submission.assets.map((asset) => asset.id) };
+}
+export function assertAdmissionDecision(input) {
+    const decision = TaskAdmissionDecisionSchema.parse(input.decision);
+    assertLearningContentHash(decision);
+    if (!sameLearningRef(decision.evidence, learningRef(input.evidence)))
+        throw new LearningDomainError("task_admission_evidence_mismatch", 422);
+    const inspection = inspectTaskEvidence(input.evidence, input.definition);
+    if (!inspection.taskReady || decision.evidenceValidity !== "valid" || decision.taskAdmissibility !== "approved")
+        throw new LearningDomainError("task_admission_not_approved", 422);
+    if (input.purpose === "supervised_training") {
+        if (decision.targetApproval !== "approved" || decision.approvedTarget === null)
+            throw new LearningDomainError("supervised_target_not_approved", 422);
+        if (!validateTaskValue(input.definition.outputSchema, decision.approvedTarget).valid)
+            throw new LearningDomainError("supervised_target_schema_invalid", 422);
+        if (!decision.targetGrade)
+            throw new LearningDomainError("supervised_target_grade_required", 422);
+        assertGradeIdentity(decision.targetGrade, input.evidence, input.definition, decision.approvedTarget);
+        const scores = [decision.targetGrade.training, decision.targetGrade.evaluation].filter((score) => score.status !== "not_configured");
+        if (!scores.length || scores.some((score) => score.status !== "scored" || score.passed !== true))
+            throw new LearningDomainError("supervised_target_checks_not_passed", 422);
+    }
+    if (decision.grade) {
+        if (!input.evidence.submission.observedOutput)
+            throw new LearningDomainError("observed_grade_without_output", 422);
+        assertGradeIdentity(decision.grade, input.evidence, input.definition, input.evidence.submission.observedOutput);
+    }
+}
+export function assertGradeIdentity(grade, evidence, definition, output) {
+    assertLearningContentHash(grade);
+    if (!sameLearningRef(grade.binding, definition.rewardBinding) || grade.taskHash !== contentHash(taskRecordFromEvidence(evidence, definition)) || grade.outputHash !== contentHash(taskAttemptEvidence(evidence, output)))
+        throw new LearningDomainError("task_grade_identity_mismatch", 422);
+}
+export function taskFamilyReservations(evidence, definition) {
+    if (!evidence.submission.familyKey)
+        throw new LearningDomainError("task_family_unresolved", 422);
+    return [
+        { namespace: definition.familyNamespace, kind: "family", key: evidence.submission.familyKey, split: evidence.submission.split },
+        { namespace: definition.familyNamespace, kind: "input", key: contentHash(evidence.submission.input), split: evidence.submission.split },
+    ];
+}
+export function sealTaskBatch(input) {
+    const definition = TaskDefinitionSchema.parse(input.definition);
+    assertLearningContentHash(definition);
+    if (!sameLearningRef(definition.rewardBinding, learningRef(input.binding)))
+        throw new LearningDomainError("task_definition_reward_binding_mismatch", 422);
+    resolveBoundRewards(input.binding, input.rewards);
+    if (input.purpose === "reward_training" && !input.binding.sources.some((source) => source.role === "training" && source.weight > 0))
+        throw new LearningDomainError("training_reward_not_configured", 422);
+    if (!input.evidence.length || new Set(input.evidence.map((evidence) => evidence.id)).size !== input.evidence.length)
+        throw new LearningDomainError("task_batch_evidence_set_invalid", 422);
+    const splits = new Map(input.priorSplits.map((reservation) => [contentHash([reservation.namespace, reservation.kind, reservation.key]), reservation.split]));
+    const examples = input.evidence.map((evidence) => {
+        const decision = input.decisions.find((candidate) => sameLearningRef(candidate.evidence, learningRef(evidence)));
+        if (!decision)
+            throw new LearningDomainError("task_admission_missing", 422, evidence.id);
+        assertAdmissionDecision({ decision, evidence, definition, purpose: input.purpose });
+        if ((input.purpose === "evaluation") === (evidence.submission.split === "train"))
+            throw new LearningDomainError("task_batch_split_not_allowed", 422);
+        for (const reservation of taskFamilyReservations(evidence, definition)) {
+            const key = contentHash([reservation.namespace, reservation.kind, reservation.key]);
+            const prior = splits.get(key);
+            if (prior !== undefined && prior !== reservation.split)
+                throw new LearningDomainError("task_family_split_contamination", 409);
+            splits.set(key, reservation.split);
+        }
+        return { evidence: learningRef(evidence), decision: learningRef(decision), familyKey: evidence.submission.familyKey, inputHash: contentHash(evidence.submission.input), split: evidence.submission.split };
+    });
+    const content = TaskBatchContentSchema.parse({ schemaVersion: "openpond.taskBatch.v1", id: input.id, revision: 1, taskDefinition: learningRef(definition), rewardBinding: learningRef(input.binding), purpose: input.purpose, examples, sealedAt: input.now, sealedBy: input.actorId });
+    return TaskBatchSchema.parse(sealLearningContent(content));
+}
+export const TaskBatchPackageMetadataSchema = z.object({
+    schemaVersion: z.literal("openpond.taskBatchPackage.v1"),
+    batch: LearningRevisionRefSchema,
+    definition: TaskDefinitionSchema,
+    binding: RewardBindingSchema,
+    rewards: z.array(RewardReleaseSchema).min(1).max(100),
+    admissions: z.array(z.object({ taskId: z.string(), evidence: LearningRevisionRefSchema, decision: LearningRevisionRefSchema, supervisedTarget: LearningJsonObjectSchema.nullable() }).strict()).min(1).max(10_000),
+}).strict();
+export function compileTaskBatch(input) {
+    assertLearningContentHash(TaskBatchSchema.parse(input.batch));
+    assertLearningContentHash(TaskDefinitionSchema.parse(input.definition));
+    if (!sameLearningRef(input.definition.rewardBinding, learningRef(input.binding)))
+        throw new LearningDomainError("task_definition_reward_binding_mismatch", 422);
+    if (!sameLearningRef(input.batch.taskDefinition, learningRef(input.definition)) || !sameLearningRef(input.batch.rewardBinding, learningRef(input.binding)))
+        throw new LearningDomainError("task_batch_release_mismatch", 422);
+    const tasks = [];
+    const admissions = [];
+    for (const entry of input.batch.examples) {
+        const evidence = input.evidence.find((candidate) => sameLearningRef(learningRef(candidate), entry.evidence));
+        const decision = input.decisions.find((candidate) => sameLearningRef(learningRef(candidate), entry.decision));
+        if (!evidence || !decision)
+            throw new LearningDomainError("task_batch_admission_revision_missing", 422);
+        assertAdmissionDecision({ decision, evidence, definition: input.definition, purpose: input.batch.purpose });
+        if (entry.familyKey !== evidence.submission.familyKey || entry.inputHash !== contentHash(evidence.submission.input) || entry.split !== evidence.submission.split)
+            throw new LearningDomainError("task_batch_evidence_snapshot_mismatch", 422);
+        tasks.push(taskRecordFromEvidence(evidence, input.definition));
+        admissions.push({ taskId: evidence.id, evidence: entry.evidence, decision: entry.decision, supervisedTarget: input.batch.purpose === "supervised_training" ? decision.approvedTarget : null });
+    }
+    const learning = TaskBatchPackageMetadataSchema.parse({ schemaVersion: "openpond.taskBatchPackage.v1", batch: learningRef(input.batch), definition: input.definition, binding: input.binding, rewards: input.rewards, admissions });
+    const content = TasksetReleaseContentSchema.parse({
+        schemaVersion: "openpond.tasksetRelease.v2", id: `task-batch-${input.batch.id}`, revision: 1,
+        ...input.definition.execution, tasks, graders: compileBoundGraders(input.binding, input.rewards),
+        metadata: { learning },
+    });
+    return TasksetReleaseSchema.parse(sealLearningContent(content));
+}
+export function taskBatchPackageMetadata(release) {
+    assertLearningContentHash(TasksetReleaseSchema.parse(release));
+    const metadata = TaskBatchPackageMetadataSchema.parse(release.metadata.learning);
+    assertLearningContentHash(metadata.definition);
+    if (!sameLearningRef(metadata.definition.rewardBinding, learningRef(metadata.binding)))
+        throw new LearningDomainError("task_definition_reward_binding_mismatch", 422);
+    if (contentHash(release.graders) !== contentHash(compileBoundGraders(metadata.binding, metadata.rewards)))
+        throw new LearningDomainError("taskset_reward_snapshot_mismatch", 422);
+    if (metadata.admissions.length !== release.tasks.length || new Set(metadata.admissions.map((entry) => entry.taskId)).size !== release.tasks.length || metadata.admissions.some((entry) => !release.tasks.some((task) => task.id === entry.taskId)))
+        throw new LearningDomainError("taskset_admission_inventory_mismatch", 422);
+    return metadata;
+}

package/dist/learning/assets.js

@@ -0,0 +1,37 @@
+import { z } from "zod";
+import { ImmutableAssetRefSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash, sha256 } from "@openpond/harness";
+import { assertBoundedTaskJson } from "../task-schema.js";
+import { LearningDomainError } from "./errors.js";
+/** Small authored source files. Large binary evidence uses the host asset API. */
+export const LearningTextAssetContentSchema = z.object({
+    schemaVersion: z.literal("openpond.learningTextAsset.v1"),
+    id: ReleaseIdSchema,
+    revision: z.literal(1),
+    asset: ImmutableAssetRefSchema,
+    text: z.string().max(524_288),
+}).strict().superRefine((value, context) => {
+    if (value.asset.id !== value.id)
+        context.addIssue({ code: "custom", path: ["asset", "id"], message: "Asset identity must match the stored resource." });
+    if (value.asset.sizeBytes !== new TextEncoder().encode(value.text).byteLength || value.asset.sizeBytes > 524_288)
+        context.addIssue({ code: "custom", path: ["asset", "sizeBytes"], message: "Authored source must match its byte count and fit within 512 KiB." });
+    if (value.asset.contentHash !== sha256(value.text))
+        context.addIssue({ code: "custom", path: ["asset", "contentHash"], message: "Asset source does not match its SHA-256." });
+});
+export const LearningTextAssetSchema = LearningTextAssetContentSchema.safeExtend({ contentHash: ReleaseHashSchema }).strict();
+export function createLearningTextAsset(input) {
+    assertBoundedTaskJson(input);
+    const { text, ...attributes } = input;
+    const hash = sha256(text);
+    const id = `asset-${hash}-${contentHash(attributes).slice(0, 16)}`;
+    const content = LearningTextAssetContentSchema.parse({
+        schemaVersion: "openpond.learningTextAsset.v1", id, revision: 1,
+        asset: { ...attributes, id, contentHash: hash, sizeBytes: new TextEncoder().encode(text).byteLength }, text,
+    });
+    return LearningTextAssetSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function verifyLearningTextAsset(asset, reference) {
+    const { contentHash: hash, ...content } = LearningTextAssetSchema.parse(asset);
+    if (contentHash(content) !== hash || contentHash(content.asset) !== contentHash(reference))
+        throw new LearningDomainError("learning_asset_reference_mismatch", 409);
+    return content.text;
+}