@openpond/evals 0.5.0 → 0.7.0

package/dist/learning/service.js

@@ -0,0 +1,367 @@
+import { LearningDomainError } from "./errors.js";
+import { contentHash } from "@openpond/harness";
+import { createRewardBinding, createRewardRelease, resolveBoundRewards } from "../rewards.js";
+import { assertBoundedTaskJson, validateTaskValue } from "../task-schema.js";
+import { assertAdmissionDecision, assertGradeIdentity, compileTaskBatch, inspectTaskEvidence, sealTaskBatch, taskFamilyReservations } from "./admission.js";
+import { learningRef, LearningSourceSchema, LearningPolicySchema, sameLearningRef, sealLearningContent, TaskAdmissionDecisionContentSchema, TaskAdmissionDecisionSchema, TaskDefinitionSchema, TaskEvidenceContentSchema, TaskEvidenceSchema, TaskFeedbackSchema, TaskGradeRunSchema } from "./contracts.js";
+import { LearningCommandSchema } from "./operations.js";
+import { LearningConflictError, learningEvidenceId, learningOperationId, requireLearningRelease, requireLearningResource } from "./repository.js";
+import { validateSourceSubmission } from "./admission.js";
+import { LearningTextAssetSchema, verifyLearningTextAsset } from "./assets.js";
+export function createLearningService(repository, options = {}) {
+    const now = options.now ?? (() => new Date().toISOString());
+    async function command(context, raw) {
+        assertBoundedTaskJson(raw, 16_777_216);
+        const input = LearningCommandSchema.parse(raw);
+        authorize(context, input);
+        const producer = input.action === "submit_example" ? input.example : input.action === "submit_feedback" ? input.feedback : null;
+        const operationId = producer ? contentHash([input.action, producer.sourceId, producer.idempotencyKey]) : learningOperationId(context.actor.id, input.operationId);
+        const requestHash = contentHash(producer ? { action: input.action, value: producer } : input);
+        return repository.transaction(context.scope, async (transaction) => {
+            const previous = await transaction.operation(operationId);
+            if (previous) {
+                if (previous.requestHash !== requestHash)
+                    throw new LearningDomainError("learning_idempotency_conflict", 409);
+                return { operationId: input.operationId, resources: await Promise.all(previous.resources.map((pointer) => requireLearningResource(transaction, pointer.kind, pointer.id, pointer.revision))) };
+            }
+            let pointers;
+            switch (input.action) {
+                case "publish":
+                    pointers = [await publish(transaction, input)];
+                    break;
+                case "publish_resources": {
+                    pointers = [];
+                    for (const resource of input.resources)
+                        pointers.push(await publish(transaction, { ...resource, action: "publish", operationId: input.operationId }));
+                    break;
+                }
+                case "submit_example":
+                    pointers = [await submit(transaction, input)];
+                    break;
+                case "submit_feedback":
+                    pointers = [await feedback(transaction, input)];
+                    break;
+                case "apply_correction":
+                    pointers = await correct(transaction, input, context.actor.id);
+                    break;
+                case "resolve_feedback":
+                    pointers = [await resolveFeedback(transaction, input, context.actor.id)];
+                    break;
+                case "queue_grade":
+                    pointers = [await queueGrade(transaction, input, operationId)];
+                    break;
+                case "cancel_grade":
+                    pointers = [await cancelGrade(transaction, input)];
+                    break;
+                case "review":
+                    pointers = [await review(transaction, input, context.actor.id)];
+                    break;
+                case "seal_batch":
+                    pointers = await seal(transaction, input, context.actor.id);
+                    break;
+            }
+            await transaction.saveOperation(operationId, { requestHash, resources: pointers });
+            return { operationId: input.operationId, resources: await Promise.all(pointers.map((pointer) => requireLearningResource(transaction, pointer.kind, pointer.id, pointer.revision))) };
+        });
+    }
+    async function publish(transaction, input) {
+        if (input.content.revision !== input.expectedRevision + 1)
+            throw new LearningDomainError("learning_publication_revision_invalid", 422);
+        let resource;
+        switch (input.kind) {
+            case "asset": {
+                resource = LearningTextAssetSchema.parse(sealLearningContent(input.content));
+                const existing = await transaction.get("asset", resource.id, 1);
+                if (existing?.contentHash === resource.contentHash)
+                    return pointer("asset", existing);
+                break;
+            }
+            case "reward": {
+                resource = createRewardRelease(input.content);
+                const implementation = input.content.implementation;
+                const references = [
+                    ...input.content.assets,
+                    ...("verifierRef" in implementation ? [implementation.verifierRef] : []),
+                    ...("rubricRef" in implementation ? [implementation.rubricRef] : []),
+                    ...("inputContract" in implementation ? [implementation.inputContract] : []),
+                ];
+                for (const reference of references) {
+                    const asset = await requireLearningResource(transaction, "asset", reference.id, 1);
+                    verifyLearningTextAsset(asset, reference);
+                    if (reference.visibility === "policy")
+                        throw new LearningDomainError("reward_asset_visibility_invalid", 422, "Reward source and rubrics must be private to the evaluator.");
+                }
+                break;
+            }
+            case "binding": {
+                if (input.content.recipeRef)
+                    await requireLearningRelease(transaction, "binding", input.content.recipeRef);
+                const rewards = await Promise.all(input.content.sources.map((source) => requireLearningRelease(transaction, "reward", source.reward)));
+                resource = createRewardBinding(input.content, rewards);
+                break;
+            }
+            case "definition": {
+                await requireLearningRelease(transaction, "binding", input.content.rewardBinding);
+                resource = TaskDefinitionSchema.parse(sealLearningContent(input.content));
+                break;
+            }
+            case "source": {
+                await requireLearningRelease(transaction, "definition", input.content.taskDefinition);
+                resource = LearningSourceSchema.parse(sealLearningContent(input.content));
+                break;
+            }
+            case "policy": {
+                const definition = await requireLearningRelease(transaction, "definition", input.content.taskDefinition);
+                if (!sameLearningRef(definition.rewardBinding, input.content.rewardBinding))
+                    throw new LearningDomainError("learning_policy_binding_mismatch", 422);
+                for (const source of input.content.sources) {
+                    const resource = await requireLearningRelease(transaction, "source", source);
+                    if (!sameLearningRef(resource.taskDefinition, input.content.taskDefinition))
+                        throw new LearningDomainError("learning_policy_source_definition_mismatch", 422);
+                }
+                resource = LearningPolicySchema.parse(sealLearningContent(input.content));
+                break;
+            }
+        }
+        await transaction.put(input.kind, resource, input.expectedRevision);
+        return pointer(input.kind, resource);
+    }
+    async function submit(transaction, input) {
+        const source = await requireLearningResource(transaction, "source", input.example.sourceId);
+        const submission = validateSourceSubmission(source, input.example);
+        await requireLearningRelease(transaction, "definition", submission.taskDefinition);
+        const id = learningEvidenceId(submission.sourceId, submission.exampleId, submission.attemptId);
+        const existing = await transaction.get("evidence", id, 1);
+        if (existing) {
+            const { idempotencyKey: _oldKey, ...oldPayload } = existing.submission;
+            const { idempotencyKey: _newKey, ...newPayload } = submission;
+            if (contentHash(oldPayload) !== contentHash(newPayload))
+                throw new LearningDomainError("task_evidence_identity_conflict", 409);
+            return pointer("evidence", existing);
+        }
+        const evidence = TaskEvidenceSchema.parse(sealLearningContent(TaskEvidenceContentSchema.parse({
+            schemaVersion: "openpond.taskEvidence.v1", id, revision: 1, source: learningRef(source), submission,
+            supersedes: null, correctionFeedbackId: null, receivedAt: now(),
+        })));
+        await transaction.put("evidence", evidence, 0, { parentId: source.id });
+        let afterId;
+        do {
+            const page = await transaction.list("feedback", { parentId: evidence.id, status: "pending_example", limit: 100, ...(afterId ? { afterId } : {}) });
+            for (const feedback of page.items) {
+                const updated = TaskFeedbackSchema.parse({ ...feedback, revision: feedback.revision + 1, status: "pending_review", evidence: learningRef(evidence) });
+                await transaction.put("feedback", updated, feedback.revision, { parentId: evidence.id, status: updated.status });
+            }
+            afterId = page.nextCursor ?? undefined;
+        } while (afterId);
+        return pointer("evidence", evidence);
+    }
+    async function feedback(transaction, input) {
+        const source = await requireLearningResource(transaction, "source", input.feedback.sourceId);
+        if (!source.enabled)
+            throw new LearningDomainError("learning_source_disabled", 409);
+        const evidenceId = learningEvidenceId(source.id, input.feedback.exampleId, input.feedback.attemptId);
+        const evidence = await transaction.get("evidence", evidenceId);
+        const record = TaskFeedbackSchema.parse({
+            schemaVersion: "openpond.taskFeedbackRecord.v1", id: `feedback-${contentHash([source.id, input.feedback.idempotencyKey])}`,
+            submission: input.feedback, status: evidence ? "pending_review" : "pending_example",
+            evidence: evidence ? learningRef(evidence) : null, createdAt: now(), revision: 1,
+        });
+        await transaction.put("feedback", record, 0, { parentId: evidenceId, status: record.status });
+        return pointer("feedback", record);
+    }
+    async function correct(transaction, input, actorId) {
+        const feedback = await requireLearningResource(transaction, "feedback", input.feedbackId);
+        if (["applied", "rejected", "superseded"].includes(feedback.status))
+            throw new LearningDomainError("task_feedback_already_resolved", 409);
+        const evidence = await currentEvidence(transaction, input.evidence);
+        const expectedId = learningEvidenceId(feedback.submission.sourceId, feedback.submission.exampleId, feedback.submission.attemptId);
+        if (expectedId !== evidence.id || (feedback.submission.expectedEvidenceHash !== null && feedback.submission.expectedEvidenceHash !== evidence.contentHash))
+            throw new LearningDomainError("task_feedback_evidence_mismatch", 409);
+        const definition = await requireLearningRelease(transaction, "definition", evidence.submission.taskDefinition);
+        const submission = { ...evidence.submission };
+        const { kind, value } = feedback.submission;
+        if (kind === "ground_truth_correction") {
+            if (!validateTaskValue(definition.outputSchema, value).valid)
+                throw new LearningDomainError("task_corrected_expected_schema_invalid", 422);
+            submission.expected = value;
+        }
+        else if (kind === "input_correction") {
+            if (!validateTaskValue(definition.inputSchema, value).valid)
+                throw new LearningDomainError("task_corrected_input_schema_invalid", 422);
+            submission.input = value;
+        }
+        else if (kind === "family_resolution") {
+            if (typeof value.familyKey !== "string" || !value.familyKey.trim())
+                throw new LearningDomainError("task_family_resolution_invalid", 422);
+            submission.familyKey = value.familyKey;
+        }
+        else
+            throw new LearningDomainError("task_feedback_requires_target_review", 422);
+        const { contentHash: _priorHash, ...priorContent } = evidence;
+        const revised = TaskEvidenceSchema.parse(sealLearningContent(TaskEvidenceContentSchema.parse({
+            ...priorContent, revision: evidence.revision + 1, submission, supersedes: learningRef(evidence), correctionFeedbackId: feedback.id, receivedAt: now(),
+        })));
+        const resolved = TaskFeedbackSchema.parse({ ...feedback, revision: feedback.revision + 1, status: "applied", evidence: learningRef(revised), review: { actorId, decision: null, note: "Applied as a corrected evidence revision.", resolvedAt: now() } });
+        await transaction.put("evidence", revised, evidence.revision, { parentId: submission.sourceId });
+        await transaction.put("feedback", resolved, feedback.revision, { parentId: evidence.id, status: resolved.status });
+        return [pointer("evidence", revised), pointer("feedback", resolved)];
+    }
+    async function resolveFeedback(transaction, input, actorId) {
+        const feedback = await requireLearningResource(transaction, "feedback", input.feedbackId);
+        if (feedback.revision !== input.expectedRevision)
+            throw new LearningConflictError("feedback", feedback.id, input.expectedRevision, feedback.revision);
+        if (["applied", "rejected", "superseded"].includes(feedback.status))
+            throw new LearningDomainError("task_feedback_already_resolved", 409);
+        if (input.disposition === "applied" && !input.decision)
+            throw new LearningDomainError("task_feedback_decision_required");
+        if (input.decision) {
+            const decision = await requireLearningRelease(transaction, "decision", input.decision);
+            const current = await requireLearningResource(transaction, "decision", decision.id);
+            if (!sameLearningRef(learningRef(current), input.decision))
+                throw new LearningDomainError("task_admission_revision_stale", 409);
+            const evidence = await currentEvidence(transaction, decision.evidence);
+            if (learningEvidenceId(feedback.submission.sourceId, feedback.submission.exampleId, feedback.submission.attemptId) !== evidence.id || (feedback.submission.expectedEvidenceHash !== null && feedback.submission.expectedEvidenceHash !== evidence.contentHash))
+                throw new LearningDomainError("task_feedback_evidence_mismatch", 409);
+            if (input.disposition === "applied") {
+                if (decision.taskAdmissibility === "pending")
+                    throw new LearningDomainError("task_feedback_decision_pending");
+                if (feedback.submission.kind === "target_correction") {
+                    if (decision.targetApproval !== "approved" || contentHash(decision.approvedTarget) !== contentHash(feedback.submission.value))
+                        throw new LearningDomainError("task_feedback_target_not_approved");
+                }
+                else if (feedback.submission.kind !== "outcome")
+                    throw new LearningDomainError("task_feedback_requires_evidence_correction");
+            }
+        }
+        const resolved = TaskFeedbackSchema.parse({ ...feedback, revision: feedback.revision + 1, status: input.disposition, review: { actorId, decision: input.decision, note: input.note, resolvedAt: now() } });
+        await transaction.put("feedback", resolved, feedback.revision, { parentId: learningEvidenceId(feedback.submission.sourceId, feedback.submission.exampleId, feedback.submission.attemptId), status: resolved.status });
+        return pointer("feedback", resolved);
+    }
+    async function queueGrade(transaction, input, operationId) {
+        const evidence = await currentEvidence(transaction, input.evidence);
+        const definition = await requireLearningRelease(transaction, "definition", evidence.submission.taskDefinition);
+        if (!inspectTaskEvidence(evidence, definition).taskReady)
+            throw new LearningDomainError("task_evidence_not_ready", 422);
+        const output = input.target === "observed" ? evidence.submission.observedOutput : input.proposedTarget;
+        if (output === null)
+            throw new LearningDomainError("task_grade_output_required", 422);
+        if (input.target === "observed" && input.proposedTarget !== null)
+            throw new LearningDomainError("observed_grade_cannot_replace_output", 422);
+        const grade = TaskGradeRunSchema.parse({
+            schemaVersion: "openpond.taskGradeRun.v1", id: `grade-${operationId}`, revision: 1,
+            evidence: learningRef(evidence), binding: definition.rewardBinding, target: input.target, output,
+            status: "queued", composition: null, leaseOwner: null, leaseExpiresAt: null, attemptCount: 0,
+            timeoutMs: input.timeoutMs, maximumSpendUsd: input.maximumSpendUsd, failure: null, createdAt: now(), updatedAt: now(),
+        });
+        await transaction.put("grade", grade, 0, { parentId: evidence.id, status: grade.status });
+        return pointer("grade", grade);
+    }
+    async function cancelGrade(transaction, input) {
+        const grade = await requireLearningResource(transaction, "grade", input.gradeId);
+        if (grade.revision !== input.expectedRevision)
+            throw new LearningConflictError("grade", grade.id, input.expectedRevision, grade.revision);
+        if (["completed", "failed", "cancelled"].includes(grade.status))
+            return pointer("grade", grade);
+        const updated = TaskGradeRunSchema.parse({ ...grade, revision: grade.revision + 1, status: grade.status === "queued" ? "cancelled" : "cancelling", updatedAt: now() });
+        await transaction.put("grade", updated, grade.revision, { parentId: grade.evidence.id, status: updated.status });
+        return pointer("grade", updated);
+    }
+    async function review(transaction, input, actorId) {
+        const evidence = await currentEvidence(transaction, input.evidence);
+        const definition = await requireLearningRelease(transaction, "definition", evidence.submission.taskDefinition);
+        const inspection = inspectTaskEvidence(evidence, definition);
+        const id = `decision-${evidence.id}`;
+        const previous = await transaction.get("decision", id);
+        const observedGrade = await gradeResult(transaction, input.observedGradeId, evidence, "observed");
+        const targetGrade = await gradeResult(transaction, input.targetGradeId, evidence, "proposed_target");
+        const measured = observedGrade?.training.status === "scored" ? observedGrade.training : observedGrade?.evaluation;
+        const decision = TaskAdmissionDecisionSchema.parse(sealLearningContent(TaskAdmissionDecisionContentSchema.parse({
+            schemaVersion: "openpond.taskAdmissionDecision.v1", id, revision: input.expectedRevision + 1,
+            evidence: learningRef(evidence), supersedes: previous ? learningRef(previous) : null,
+            actor: { kind: "human", id: actorId, policy: null }, evidenceValidity: inspection.evidenceValidity,
+            taskAdmissibility: input.disposition, observedQuality: measured?.status === "scored" ? measured.passed ? "passed" : "failed" : observedGrade ? "unavailable" : "unscored",
+            targetApproval: input.targetApproval, approvedTarget: input.approvedTarget, grade: observedGrade, targetGrade,
+            note: input.note, decidedAt: now(),
+        })));
+        if (input.disposition === "approved")
+            assertAdmissionDecision({ decision, evidence, definition, purpose: input.targetApproval === "approved" ? "supervised_training" : "reward_training" });
+        await transaction.put("decision", decision, input.expectedRevision, { parentId: evidence.id, status: decision.taskAdmissibility });
+        return pointer("decision", decision);
+    }
+    async function seal(transaction, input, actorId) {
+        const definition = await requireLearningRelease(transaction, "definition", input.taskDefinition);
+        const binding = await requireLearningRelease(transaction, "binding", definition.rewardBinding);
+        const rewards = await Promise.all(binding.sources.map((source) => requireLearningRelease(transaction, "reward", source.reward)));
+        resolveBoundRewards(binding, rewards);
+        const evidence = await Promise.all(input.evidence.map((ref) => currentEvidence(transaction, ref)));
+        const decisions = await Promise.all(input.decisions.map(async (ref) => {
+            const decision = await requireLearningRelease(transaction, "decision", ref);
+            const current = await requireLearningResource(transaction, "decision", ref.id);
+            if (!sameLearningRef(learningRef(current), ref))
+                throw new LearningDomainError("task_admission_revision_stale", 409);
+            return decision;
+        }));
+        const priorSplits = [];
+        for (const item of evidence) {
+            for (const reservation of taskFamilyReservations(item, definition)) {
+                const split = await transaction.familySplit(reservation.namespace, reservation.kind, reservation.key);
+                if (split !== null)
+                    priorSplits.push({ ...reservation, split: split });
+            }
+        }
+        const batch = sealTaskBatch({ id: input.batchId, definition, binding, rewards, purpose: input.purpose, evidence, decisions, priorSplits, actorId, now: now() });
+        const release = compileTaskBatch({ batch, definition, binding, rewards, evidence, decisions });
+        for (const item of evidence) {
+            for (const reservation of taskFamilyReservations(item, definition))
+                await transaction.reserveFamilySplit(reservation.namespace, reservation.kind, reservation.key, reservation.split);
+        }
+        await transaction.put("batch", batch, 0, { parentId: definition.id });
+        await transaction.put("package", release, 0, { parentId: batch.id });
+        return [pointer("batch", batch), pointer("package", release)];
+    }
+    return {
+        command,
+        async get(context, kind, id, revision) {
+            authorizeRead(context);
+            return repository.transaction(context.scope, (transaction) => requireLearningResource(transaction, kind, id, revision));
+        },
+        async list(context, kind, query = {}) {
+            authorizeRead(context);
+            const limit = Math.max(1, Math.min(100, Math.trunc(query.limit ?? 50)));
+            return repository.transaction(context.scope, (transaction) => transaction.list(kind, { ...query, limit }));
+        },
+    };
+}
+async function currentEvidence(transaction, ref) {
+    const evidence = await requireLearningRelease(transaction, "evidence", ref);
+    const current = await requireLearningResource(transaction, "evidence", ref.id);
+    if (!sameLearningRef(learningRef(current), ref))
+        throw new LearningDomainError("task_evidence_revision_stale", 409);
+    return evidence;
+}
+async function gradeResult(transaction, id, evidence, target) {
+    if (id === null)
+        return null;
+    const grade = await requireLearningResource(transaction, "grade", id);
+    if (grade.status !== "completed" || !grade.composition || grade.target !== target || !sameLearningRef(grade.evidence, learningRef(evidence)))
+        throw new LearningDomainError("task_grade_not_applicable", 422);
+    const definition = await requireLearningRelease(transaction, "definition", evidence.submission.taskDefinition);
+    assertGradeIdentity(grade.composition, evidence, definition, grade.output);
+    return grade.composition;
+}
+function pointer(kind, resource) { return { kind, id: resource.id, revision: resource.revision }; }
+function authorizeRead(context) {
+    if (!context.scope.trim() || !context.actor.id.trim() || context.actor.role === "source")
+        throw new LearningDomainError("learning_read_not_authorized", 403);
+}
+function authorize(context, input) {
+    if (!context.scope.trim() || !context.actor.id.trim())
+        throw new LearningDomainError("learning_scope_required", 400);
+    if (context.actor.role === "source") {
+        const sourceId = input.action === "submit_example" ? input.example.sourceId : input.action === "submit_feedback" ? input.feedback.sourceId : null;
+        if (!sourceId || sourceId !== context.actor.sourceId)
+            throw new LearningDomainError("learning_source_not_authorized", 403);
+    }
+    if (["review", "apply_correction", "resolve_feedback", "seal_batch"].includes(input.action) && context.actor.role !== "reviewer")
+        throw new LearningDomainError("learning_review_not_authorized", 403);
+}

package/dist/learning/transport.js

@@ -0,0 +1,25 @@
+import { z } from "zod";
+import { ReleaseIdSchema } from "@openpond/harness";
+import { LearningCommandSchema } from "./operations.js";
+import { LearningResourceKindSchema, learningResourceSchemas } from "./repository.js";
+import { assertBoundedTaskJson } from "../task-schema.js";
+import { LearningDomainError } from "./errors.js";
+/** Run before recursive schema parsing, including for non-TypeScript producers. */
+export function assertLearningRequestJson(raw) {
+    try {
+        assertBoundedTaskJson(raw, 16_777_216);
+    }
+    catch (error) {
+        throw new LearningDomainError("learning_json_invalid", 400, error instanceof Error ? error.message : "Expected bounded JSON.");
+    }
+}
+/** Scope selection is never authorization; each host resolves its authenticated owner. */
+export const LearningCommandRequestSchema = z.object({ scope: ReleaseIdSchema, command: LearningCommandSchema }).strict();
+export const LearningReadRequestSchema = z.discriminatedUnion("action", [
+    z.object({ action: z.literal("get"), scope: ReleaseIdSchema, kind: LearningResourceKindSchema, id: ReleaseIdSchema, revision: z.number().int().positive().optional() }).strict(),
+    z.object({ action: z.literal("list"), scope: ReleaseIdSchema, kind: LearningResourceKindSchema, parentId: ReleaseIdSchema.optional(), status: z.string().min(1).max(200).optional(), afterId: ReleaseIdSchema.optional(), limit: z.number().int().min(1).max(100).default(50) }).strict(),
+]);
+export const LearningOperationResultSchema = z.object({
+    operationId: ReleaseIdSchema,
+    resources: z.array(z.union(Object.values(learningResourceSchemas))).max(100),
+}).strict();

package/dist/preferences.js

@@ -13,6 +13,11 @@ export const PreferenceReviewerKindSchema = z.enum([
     "human",
     "model",
     "deterministic",
+    "fixture",
+]);
+export const PreferenceRewardScopeSchema = z.enum([
+    "production",
+    "synthetic_smoke",
 ]);
 export const PreferenceRendererSchema = z.enum([
     "markdown",
@@ -111,7 +116,7 @@ const ComparisonCandidateSchema = z.object({
     attemptRef: ImmutableReleaseRefSchema,
     runManifestRef: ImmutableReleaseRefSchema,
     artifactManifestRef: ImmutableReleaseRefSchema,
-    visibleArtifactIds: z.array(ReleaseIdSchema).min(1).max(100_000),
+    visibleArtifactIds: z.array(ReleaseIdSchema).max(100_000),
 }).strict().superRefine((candidate, context) => {
     if (new Set(candidate.visibleArtifactIds).size !== candidate.visibleArtifactIds.length) {
         context.addIssue({
@@ -158,14 +163,24 @@ export const ComparisonAssignmentSchema = ComparisonAssignmentBaseSchema
     }
 });
 const PreferenceOrderSchema = z.array(z.array(ReleaseIdSchema).min(1).max(4)).max(4);
+const PreferenceReviewerSchema = z.discriminatedUnion("kind", [
+    z.object({
+        kind: z.enum(["human", "model", "deterministic"]),
+        releaseRef: ImmutableReleaseRefSchema,
+    }).strict(),
+    z.object({
+        kind: z.literal("fixture"),
+        releaseRef: ImmutableReleaseRefSchema,
+        fixtureRelease: ImmutableReleaseRefSchema,
+        labelSource: z.literal("synthetic"),
+        qualificationEligibility: z.literal("smoke_only"),
+    }).strict(),
+]);
 const PreferenceReceiptBaseSchema = z.object({
     schemaVersion: z.literal("openpond.preferenceReceipt.v1"),
     id: ReleaseIdSchema,
     assignmentRef: ImmutableReleaseRefSchema,
-    reviewer: z.object({
-        kind: PreferenceReviewerKindSchema,
-        releaseRef: ImmutableReleaseRefSchema,
-    }).strict(),
+    reviewer: PreferenceReviewerSchema,
     order: PreferenceOrderSchema,
     rejectAll: z.boolean(),
     criterionScores: z.record(ReleaseIdSchema, z.record(ReleaseIdSchema, z.number().finite().min(0).max(1))),
@@ -244,7 +259,8 @@ export function createComparisonAssignment(input) {
     if (input.candidates.length !== release.candidateCount) {
         throw new Error("Comparison assignment candidate count does not match the immutable comparison release.");
     }
-    const normalized = input.candidates.map((candidate) => validateComparisonCandidate(candidate, taskset));
+    const requiresVisibleArtifact = release.presentation.parts.some((part) => part.source === "artifact");
+    const normalized = input.candidates.map((candidate) => validateComparisonCandidate(candidate, taskset, requiresVisibleArtifact));
     const first = normalized[0];
     for (const candidate of normalized.slice(1)) {
         if (candidate.attempt.taskId !== first.attempt.taskId) {
@@ -331,6 +347,45 @@ export function createPreferenceReceipt(input) {
     validateCriterionScores(content.criterionScores, assignment, release);
     return PreferenceReceiptSchema.parse({ ...content, contentHash: contentHash(content) });
 }
+export function createSyntheticFixturePreferenceReceipt(input) {
+    const candidateIds = input.assignment.candidates.map((candidate) => candidate.attemptRef.id);
+    const ratingIds = Object.keys(input.ratings);
+    if (ratingIds.length !== candidateIds.length
+        || candidateIds.some((candidateId) => !input.ratings[candidateId])
+        || ratingIds.some((candidateId) => !candidateIds.includes(candidateId))) {
+        throw new Error("Synthetic fixture ratings must label every assignment candidate exactly once.");
+    }
+    const orderedRatings = ["love", "like", "reject"];
+    const rejectAll = candidateIds.every((candidateId) => input.ratings[candidateId] === "reject");
+    const order = rejectAll ? [] : orderedRatings
+        .map((rating) => candidateIds.filter((candidateId) => input.ratings[candidateId] === rating))
+        .filter((bucket) => bucket.length > 0);
+    const score = { love: 1, like: 0.5, reject: 0 };
+    return createPreferenceReceipt({
+        id: input.id,
+        assignment: input.assignment,
+        comparisonRelease: input.comparisonRelease,
+        reviewer: {
+            kind: "fixture",
+            releaseRef: input.labelerRelease,
+            fixtureRelease: input.fixtureRelease,
+            labelSource: "synthetic",
+            qualificationEligibility: "smoke_only",
+        },
+        order,
+        rejectAll,
+        criterionScores: Object.fromEntries(candidateIds.map((candidateId) => [
+            candidateId,
+            Object.fromEntries(input.comparisonRelease.criteria.map((criterion) => [
+                criterion.id,
+                score[input.ratings[candidateId]],
+            ])),
+        ])),
+        startedAt: input.startedAt,
+        completedAt: input.completedAt,
+        metadata: input.metadata ?? {},
+    });
+}
 export function verifyPreferenceReceipt(value) {
     return verifyHashed(value, PreferenceReceiptContentSchema, PreferenceReceiptSchema);
 }
@@ -436,12 +491,16 @@ export function createPreferenceRewardComponents(input) {
         ? input.result
         : null;
     const automatedReceipt = modelReceipt?.reviewer.kind === "model";
-    const calibrated = !automatedReceipt || isCalibratedAutomatedReviewer(modelReceipt, release, input.calibrationReport ?? null);
+    const fixtureReceipt = modelReceipt?.reviewer.kind === "fixture";
+    const rewardScope = PreferenceRewardScopeSchema.parse(input.rewardScope ?? "production");
+    const admittedReviewer = fixtureReceipt
+        ? rewardScope === "synthetic_smoke"
+        : !automatedReceipt || isCalibratedAutomatedReviewer(modelReceipt, release, input.calibrationReport ?? null);
     return Object.fromEntries(assignment.candidates.map((candidate) => {
         const candidateEligibility = eligible.get(candidate.attemptRef.id);
         const canScore = candidateEligibility?.eligible ?? true;
         const score = scores[candidate.attemptRef.id];
-        const component = canScore && calibrated
+        const component = canScore && admittedReviewer
             ? RewardComponentReceiptSchema.parse({
                 verifierId: release.rewardProjection.verifierId,
                 verifierVersion: release.rewardProjection.verifierVersion,
@@ -475,7 +534,11 @@ export function createPreferenceRewardComponents(input) {
                 rewardEligible: false,
                 rewardContribution: null,
                 failureOwner: candidateEligibility?.failureOwner ?? "verifier",
-                feedback: [canScore ? "Automated preference reviewer is uncalibrated or inconsistent." : "Candidate is not eligible for comparative preference scoring."],
+                feedback: [canScore
+                        ? fixtureReceipt
+                            ? "Synthetic fixture preference evidence is admitted only to synthetic-smoke reward scope."
+                            : "Automated preference reviewer is uncalibrated or inconsistent."
+                        : "Candidate is not eligible for comparative preference scoring."],
                 visibleEvidenceRefs: [],
                 privilegedEvidenceRefs: [],
                 metadata: {
@@ -487,7 +550,7 @@ export function createPreferenceRewardComponents(input) {
         return [candidate.attemptRef.id, component];
     }));
 }
-function validateComparisonCandidate(input, taskset) {
+function validateComparisonCandidate(input, taskset, requiresVisibleArtifact) {
     const attempt = AttemptReceiptSchema.parse(input.attempt);
     if (!verifyAttemptReceipt(attempt))
         throw new Error("Comparison candidate Attempt Receipt has an invalid content hash.");
@@ -514,7 +577,10 @@ function validateComparisonCandidate(input, taskset) {
     const available = new Set(artifactManifest.entries
         .filter((entry) => entry.status === "collected" && entry.artifact !== null)
         .map((entry) => entry.artifact.id));
-    if (!visibleArtifactIds.length || visibleArtifactIds.some((id) => !available.has(id))) {
+    if (requiresVisibleArtifact && !visibleArtifactIds.length) {
+        throw new Error("Comparison presentation requires each candidate to expose a reviewable artifact.");
+    }
+    if (visibleArtifactIds.some((id) => !available.has(id))) {
         throw new Error("Comparison candidate exposes an artifact that is missing, uncollected, or unreviewable.");
     }
     if (visible.size !== visibleArtifactIds.length)