@prismatic-io/lux 0.0.1

package/lib/optimization/reflective.d.ts

@@ -0,0 +1,27 @@
+import { z } from "zod";
+import { type Optimizer } from "../core/index.js";
+export declare const ReflectiveOptimizerConfigSchema: z.ZodObject<{
+    provider: z.ZodDefault<z.ZodEnum<{
+        "claude-code": "claude-code";
+        codex: "codex";
+    }>>;
+    model: z.ZodOptional<z.ZodString>;
+    command: z.ZodOptional<z.ZodString>;
+    reasoningEffort: z.ZodOptional<z.ZodEnum<{
+        high: "high";
+        low: "low";
+        medium: "medium";
+        minimal: "minimal";
+        none: "none";
+        xhigh: "xhigh";
+    }>>;
+    timeoutMs: z.ZodDefault<z.ZodNumber>;
+    maxComponentChars: z.ZodDefault<z.ZodNumber>;
+    maxAlternativeChars: z.ZodDefault<z.ZodNumber>;
+    maxFeedbackChars: z.ZodDefault<z.ZodNumber>;
+    maxPromptChars: z.ZodDefault<z.ZodNumber>;
+    proposalsPerIteration: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strict>;
+export type ReflectiveOptimizerConfig = z.infer<typeof ReflectiveOptimizerConfigSchema>;
+export declare const createReflectiveOptimizer: (rawConfig?: unknown) => Optimizer;
+//# sourceMappingURL=reflective.d.ts.map

package/lib/optimization/reflective.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reflective.d.ts","sourceRoot":"","sources":["../../src/optimization/reflective.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAML,KAAK,SAAS,EAGf,MAAM,kBAAkB,CAAC;AAE1B,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;kBAcjC,CAAC;AACZ,MAAM,MAAM,yBAAyB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,CAAC;AAkSxF,eAAO,MAAM,yBAAyB,eAAe,OAAO,KAAQ,SACG,CAAC"}

package/lib/optimization/reflective.js

@@ -0,0 +1,257 @@
+import { z } from "zod";
+import { defaultHarnessModel, emptyUsage, ModelCliProviderSchema, ReasoningEffortSchema, runTextModel, } from "../core/index.js";
+export const ReflectiveOptimizerConfigSchema = z
+    .object({
+    provider: ModelCliProviderSchema.default("claude-code"),
+    model: z.string().optional(),
+    command: z.string().optional(),
+    reasoningEffort: ReasoningEffortSchema.optional(),
+    timeoutMs: z.number().int().positive().default(180_000),
+    maxComponentChars: z.number().int().positive().default(80_000),
+    maxAlternativeChars: z.number().int().positive().default(40_000),
+    maxFeedbackChars: z.number().int().positive().default(50_000),
+    /** Hard cap for the complete serialized optimizer prompt. */
+    maxPromptChars: z.number().int().min(5_000).max(500_000).default(120_000),
+    proposalsPerIteration: z.number().int().min(1).max(4).default(1),
+})
+    .strict();
+const ProposalResponseSchema = z
+    .object({
+    proposals: z.array(z
+        .object({
+        rationale: z.string().optional(),
+        updates: z.record(z.string(), z.string()),
+    })
+        .strict()),
+})
+    .strict();
+const parseResponse = (text) => {
+    const direct = (() => {
+        try {
+            return JSON.parse(text);
+        }
+        catch {
+            return undefined;
+        }
+    })();
+    const directResult = ProposalResponseSchema.safeParse(direct);
+    if (directResult.success)
+        return directResult.data;
+    const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(text)?.[1];
+    if (!fenced)
+        return null;
+    try {
+        const parsed = ProposalResponseSchema.safeParse(JSON.parse(fenced));
+        return parsed.success ? parsed.data : null;
+    }
+    catch {
+        return null;
+    }
+};
+const boundedComponents = (components, budget) => {
+    const out = {};
+    const omitted = [];
+    let used = 0;
+    for (const [path, content] of Object.entries(components).sort(([a], [b]) => a.localeCompare(b))) {
+        const serializedChars = JSON.stringify({ [path]: content }).length;
+        if (serializedChars > budget - used) {
+            omitted.push(path);
+            continue;
+        }
+        out[path] = content;
+        used += serializedChars;
+    }
+    return { complete: out, omitted };
+};
+const boundedRecord = (input, budget) => {
+    const complete = {};
+    let used = 0;
+    let omitted = 0;
+    for (const [key, value] of Object.entries(input).sort(([a], [b]) => a.localeCompare(b))) {
+        const size = JSON.stringify({ [key]: value }).length;
+        if (used + size > budget) {
+            omitted++;
+            continue;
+        }
+        complete[key] = value;
+        used += size;
+    }
+    return { complete, omitted };
+};
+const boundedResults = (observations, budget, includeMetrics) => {
+    const results = [];
+    for (const observation of observations) {
+        const share = Math.max(100, Math.floor(budget / Math.max(1, observations.length)));
+        const casePassRates = boundedRecord(observation.casePassRates, Math.floor(share * 0.55));
+        const metrics = boundedRecord(includeMetrics ? observation.metrics : {}, Math.floor(share * 0.3));
+        const next = {
+            profile: observation.profileId.slice(0, 200),
+            split: observation.split,
+            passRate: observation.passRate,
+            casePassRates: casePassRates.complete,
+            omittedCasePassRates: casePassRates.omitted,
+            ...(includeMetrics ? { metrics: metrics.complete, omittedMetrics: metrics.omitted } : {}),
+        };
+        if (JSON.stringify([...results, next]).length > budget)
+            break;
+        results.push(next);
+    }
+    return { results, omitted: observations.length - results.length };
+};
+const boundedFeedback = (observations, budget) => {
+    const failures = observations.flatMap((observation) => observation.failedCases.map((failure) => ({
+        profile: observation.profileId,
+        split: observation.split,
+        caseId: failure.caseId,
+        score: failure.score,
+        feedback: failure.feedback,
+    })));
+    const included = [];
+    for (const failure of failures) {
+        const next = [...included, failure];
+        if (JSON.stringify(next).length > budget)
+            break;
+        included.push(failure);
+    }
+    return { failures: included, omitted: failures.length - included.length };
+};
+const systemPrompt = (kind) => `You improve text components of a
+${kind === "grader" ? "grading system from disagreements with human expert labels" : "coding-agent system from evaluation results"}.
+Propose the smallest changes that fix recurring failures while preserving successful behavior
+and reducing instruction/context size where possible.
+
+Return only JSON:
+{"proposals":[{"rationale":"short explanation","updates":{"relative/path":"complete new file contents"}}]}
+
+Rules:
+- Update only paths present in the supplied components.
+- Each update value is the complete replacement file, never a patch.
+- ${kind === "grader"
+    ? "Align rubric criteria, evidence bounds, thresholds, and weights to the human labels. Do not make a grader merely more lenient or leak annotation text, IDs, or expected labels into it."
+    : "Do not weaken tests, graders, safety rules, or required user-owned decisions."}
+- Prefer deletion, routing, explicit compact contracts, and mechanical checks over added prose.
+- Do not mention the evaluation cases in the proposed component text.
+${kind === "grader" ? "- Preserve stable assertion IDs so labels and historical grades remain comparable." : ""}`;
+const optimizerPrompt = (input, config) => {
+    const cap = config.maxPromptChars;
+    const currentComponents = boundedComponents(input.candidate.components, Math.min(config.maxComponentChars, Math.floor(cap * 0.45)));
+    const resultBudget = Math.floor(cap * 0.12);
+    const currentResults = boundedResults(input.observations, resultBudget, true);
+    const objectives = JSON.stringify(input.campaign.objectives).length <= Math.floor(cap * 0.08)
+        ? input.campaign.objectives
+        : {
+            omitted: true,
+            constraints: input.campaign.objectives.constraints.length,
+            minimize: input.campaign.objectives.minimize.length,
+        };
+    const alternatives = (input.alternatives ?? []).slice(0, 2).map((candidate) => {
+        const observations = input.alternativeObservations?.[candidate.id] ?? [];
+        return {
+            id: candidate.id.slice(0, 200),
+            sourceBytes: candidate.sourceBytes,
+            trainResults: boundedResults(observations, Math.floor((cap * 0.08) / Math.max(1, input.alternatives?.length ?? 1)), false),
+            componentView: boundedComponents(candidate.components, Math.min(Math.floor(config.maxAlternativeChars / Math.max(1, input.alternatives?.length ?? 1)), Math.floor((cap * 0.08) / Math.max(1, input.alternatives?.length ?? 1)))),
+        };
+    });
+    const payload = {
+        campaign: {
+            id: input.campaign.id.slice(0, 200),
+            objectives,
+            remainingCandidates: input.remainingCandidates,
+        },
+        currentCandidate: {
+            id: input.candidate.id.slice(0, 200),
+            sourceBytes: input.candidate.sourceBytes,
+            trainResults: currentResults,
+            components: currentComponents.complete,
+            omittedComponents: currentComponents.omitted,
+        },
+        frontierAlternatives: alternatives,
+        omittedFrontierAlternatives: Math.max(0, (input.alternatives?.length ?? 0) - alternatives.length),
+        failedEvaluations: boundedFeedback(input.observations, Math.min(config.maxFeedbackChars, Math.floor(cap * 0.12))),
+        proposalCount: Math.min(config.proposalsPerIteration, input.remainingCandidates),
+    };
+    let prompt = JSON.stringify(payload, null, 2);
+    if (prompt.length > cap) {
+        // Preserve valid JSON and the component visibility contract while
+        // shedding reproducibly ordered context from least essential sections.
+        payload.frontierAlternatives = [];
+        payload.omittedFrontierAlternatives = input.alternatives?.length ?? 0;
+        payload.failedEvaluations = { failures: [], omitted: input.observations.length };
+        payload.campaign.objectives = {
+            omitted: true,
+            constraints: input.campaign.objectives.constraints.length,
+            minimize: input.campaign.objectives.minimize.length,
+        };
+        payload.currentCandidate.trainResults = {
+            results: [],
+            omitted: input.observations.length,
+        };
+        prompt = JSON.stringify(payload, null, 2);
+    }
+    while (prompt.length > cap) {
+        const paths = Object.keys(currentComponents.complete).sort();
+        const path = paths.at(-1);
+        if (!path)
+            break;
+        delete currentComponents.complete[path];
+        currentComponents.omitted.push(path);
+        prompt = JSON.stringify(payload, null, 2);
+    }
+    if (prompt.length > cap) {
+        throw new Error(`reflective optimizer prompt metadata exceeds maxPromptChars=${cap}`);
+    }
+    return { prompt, visibleComponents: new Set(Object.keys(currentComponents.complete)) };
+};
+const reflectiveOptimizer = (config) => ({
+    propose: async (input) => {
+        const model = config.model ?? defaultHarnessModel(config.provider);
+        const builtPrompt = optimizerPrompt(input, config);
+        let response;
+        try {
+            await input.recordModelAttemptStart?.("proposal");
+            response = await runTextModel({
+                provider: config.provider,
+                model,
+                command: config.command,
+                reasoningEffort: config.reasoningEffort,
+                timeoutMs: config.timeoutMs,
+                system: systemPrompt(input.campaign.evaluation.kind),
+                signal: input.abortSignal,
+                prompt: builtPrompt.prompt,
+            });
+        }
+        catch (error) {
+            // A failed provider call has unknown usage/price. The campaign ledger
+            // must still consume one explicit unpriced call.
+            await input.recordModelCall?.({ usage: emptyUsage() });
+            await input.completeModelAttempts?.();
+            throw error;
+        }
+        await input.recordModelCall?.({
+            usage: response.usage,
+            ...(response.costUsd !== undefined ? { costUsd: response.costUsd } : {}),
+        });
+        await input.completeModelAttempts?.();
+        const parsed = parseResponse(response.text);
+        if (!parsed) {
+            throw new Error(`reflective optimizer returned invalid JSON: ${response.text.slice(0, 300)}`);
+        }
+        const visibleComponents = builtPrompt.visibleComponents;
+        return parsed.proposals
+            .slice(0, Math.min(config.proposalsPerIteration, input.remainingCandidates))
+            .map((proposal) => {
+            const hiddenUpdates = Object.keys(proposal.updates).filter((path) => !visibleComponents.has(path));
+            if (hiddenUpdates.length > 0) {
+                throw new Error(`reflective optimizer attempted to replace components it was not shown in full: ${hiddenUpdates.join(", ")}`);
+            }
+            return {
+                updates: proposal.updates,
+                ...(proposal.rationale ? { rationale: proposal.rationale } : {}),
+                usage: response.usage,
+            };
+        });
+    },
+});
+export const createReflectiveOptimizer = (rawConfig = {}) => reflectiveOptimizer(ReflectiveOptimizerConfigSchema.parse(rawConfig));
+//# sourceMappingURL=reflective.js.map

package/lib/optimization/reflective.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reflective.js","sourceRoot":"","sources":["../../src/optimization/reflective.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAGL,mBAAmB,EACnB,UAAU,EACV,sBAAsB,EAEtB,qBAAqB,EACrB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAE1B,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC;KAC7C,MAAM,CAAC;IACN,QAAQ,EAAE,sBAAsB,CAAC,OAAO,CAAC,aAAa,CAAC;IACvD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,eAAe,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IACjD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;IACvD,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IAC9D,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IAChE,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7D,6DAA6D;IAC7D,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IACzE,qBAAqB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACjE,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,sBAAsB,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,SAAS,EAAE,CAAC,CAAC,KAAK,CAChB,CAAC;SACE,MAAM,CAAC;QACN,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAChC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;KAC1C,CAAC;SACD,MAAM,EAAE,CACZ;CACF,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,aAAa,GAAG,CAAC,IAAY,EAAiD,EAAE;IACpF,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;QACnB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;IACL,MAAM,YAAY,GAAG,sBAAsB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC9D,IAAI,YAAY,CAAC,OAAO;QAAE,OAAO,YAAY,CAAC,IAAI,CAAC;IACnD,MAAM,MAAM,GAAG,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CACxB,UAAkC,EAClC,MAAc,EAC2C,EAAE;IAC3D,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QACnE,IAAI,eAAe,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;QACpB,IAAI,IAAI,eAAe,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AACpC,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CACpB,KAA6B,EAC7B,MAAc,EACyC,EAAE;IACzD,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxF,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,IAAI,GAAG,IAAI,GAAG,MAAM,EAAE,CAAC;YACzB,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACtB,IAAI,IAAI,IAAI,CAAC;IACf,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC/B,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CACrB,YAA4C,EAC5C,MAAc,EACd,cAAuB,EACkB,EAAE;IAC3C,MAAM,OAAO,GAAc,EAAE,CAAC;IAC9B,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,aAAa,CAAC,WAAW,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC;QACzF,MAAM,OAAO,GAAG,aAAa,CAC3B,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EACzC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CACxB,CAAC;QACF,MAAM,IAAI,GAAG;YACX,OAAO,EAAE,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAC5C,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ;YAC9B,aAAa,EAAE,aAAa,CAAC,QAAQ;YACrC,oBAAoB,EAAE,aAAa,CAAC,OAAO;YAC3C,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1F,CAAC;QACF,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM;YAAE,MAAM;QAC9D,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;AACpE,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CACtB,YAA4C,EAC5C,MAAc,EAC4B,EAAE;IAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE,CACpD,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACxC,OAAO,EAAE,WAAW,CAAC,SAAS;QAC9B,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC,CAAC,CACJ,CAAC;IACF,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM;YAAE,MAAM;QAChD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;AAC5E,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,IAAwB,EAAU,EAAE,CAAC;EACzD,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,4DAA4D,CAAC,CAAC,CAAC,6CAA6C;;;;;;;;;;IAWhI,IAAI,KAAK,QAAQ;IACf,CAAC,CAAC,yLAAyL;IAC3L,CAAC,CAAC,+EACN;;;EAGE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,oFAAoF,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AAElH,MAAM,eAAe,GAAG,CACtB,KAA0C,EAC1C,MAAiC,EACmB,EAAE;IACtD,MAAM,GAAG,GAAG,MAAM,CAAC,cAAc,CAAC;IAClC,MAAM,iBAAiB,GAAG,iBAAiB,CACzC,KAAK,CAAC,SAAS,CAAC,UAAU,EAC1B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAC3D,CAAC;IACF,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,cAAc,CAAC,KAAK,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC9E,MAAM,UAAU,GACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC;QACxE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU;QAC3B,CAAC,CAAC;YACE,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM;YACzD,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM;SACpD,CAAC;IACR,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAC5E,MAAM,YAAY,GAAG,KAAK,CAAC,uBAAuB,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QACzE,OAAO;YACL,EAAE,EAAE,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAC9B,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,YAAY,EAAE,cAAc,CAC1B,YAAY,EACZ,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,EACvE,KAAK,CACN;YACD,aAAa,EAAE,iBAAiB,CAC9B,SAAS,CAAC,UAAU,EACpB,IAAI,CAAC,GAAG,CACN,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,EACrF,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,CACxE,CACF;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAA4B;QACvC,QAAQ,EAAE;YACR,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACnC,UAAU;YACV,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;SAC/C;QACD,gBAAgB,EAAE;YAChB,EAAE,EAAE,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACpC,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,WAAW;YACxC,YAAY,EAAE,cAAc;YAC5B,UAAU,EAAE,iBAAiB,CAAC,QAAQ;YACtC,iBAAiB,EAAE,iBAAiB,CAAC,OAAO;SAC7C;QACD,oBAAoB,EAAE,YAAY;QAClC,2BAA2B,EAAE,IAAI,CAAC,GAAG,CACnC,CAAC,EACD,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CACxD;QACD,iBAAiB,EAAE,eAAe,CAChC,KAAK,CAAC,YAAY,EAClB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAC1D;QACD,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,qBAAqB,EAAE,KAAK,CAAC,mBAAmB,CAAC;KACjF,CAAC;IACF,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9C,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACxB,kEAAkE;QAClE,uEAAuE;QACvE,OAAO,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAClC,OAAO,CAAC,2BAA2B,GAAG,KAAK,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC;QACtE,OAAO,CAAC,iBAAiB,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;QAChF,OAAO,CAAC,QAAoC,CAAC,UAAU,GAAG;YACzD,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM;YACzD,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM;SACpD,CAAC;QACD,OAAO,CAAC,gBAA4C,CAAC,YAAY,GAAG;YACnE,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,MAAM;SACnC,CAAC;QACF,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI;YAAE,MAAM;QACjB,OAAO,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,+DAA+D,GAAG,EAAE,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;AACzF,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,MAAiC,EAAa,EAAE,CAAC,CAAC;IAC7E,OAAO,EAAE,KAAK,EAAE,KAAK,EAAgC,EAAE;QACrD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnE,MAAM,WAAW,GAAG,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACnD,IAAI,QAAkD,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,uBAAuB,EAAE,CAAC,UAAU,CAAC,CAAC;YAClD,QAAQ,GAAG,MAAM,YAAY,CAAC;gBAC5B,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,KAAK;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;gBACpD,MAAM,EAAE,KAAK,CAAC,WAAW;gBACzB,MAAM,EAAE,WAAW,CAAC,MAAM;aAC3B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sEAAsE;YACtE,iDAAiD;YACjD,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;YACvD,MAAM,KAAK,CAAC,qBAAqB,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC;YAC5B,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzE,CAAC,CAAC;QACH,MAAM,KAAK,CAAC,qBAAqB,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,+CAA+C,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChG,CAAC;QACD,MAAM,iBAAiB,GAAG,WAAW,CAAC,iBAAiB,CAAC;QACxD,OAAO,MAAM,CAAC,SAAS;aACpB,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,qBAAqB,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;aAC3E,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YAChB,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,CACxD,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CACvC,CAAC;YACF,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,kFAAkF,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7G,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChE,KAAK,EAAE,QAAQ,CAAC,KAAK;aACtB,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,SAAS,GAAY,EAAE,EAAa,EAAE,CAC9E,mBAAmB,CAAC,+BAA+B,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC"}

package/lib/orchestrator/annotation-loader.d.ts

@@ -0,0 +1,65 @@
+import { z } from "zod";
+import { type ExperimentSelector, type GraderAnnotation } from "../core/index.js";
+export declare const LoadedGraderAnnotationsSchema: z.ZodObject<{
+    path: z.ZodString;
+    directory: z.ZodString;
+    dataset: z.ZodObject<{
+        schemaVersion: z.ZodLiteral<1>;
+        annotations: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            runDir: z.ZodString;
+            rubricPath: z.ZodPipe<z.ZodString, z.ZodTransform<string, string>>;
+            tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+            expected: z.ZodObject<{
+                casePassed: z.ZodOptional<z.ZodBoolean>;
+                caseScore: z.ZodOptional<z.ZodNumber>;
+                assertions: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
+                    passed: z.ZodBoolean;
+                    score: z.ZodOptional<z.ZodNumber>;
+                }, z.core.$strict>>>;
+            }, z.core.$strict>;
+            source: z.ZodEnum<{
+                human: "human";
+                synthetic: "synthetic";
+            }>;
+            annotator: z.ZodOptional<z.ZodString>;
+            labeledAt: z.ZodOptional<z.ZodString>;
+            reviewer: z.ZodOptional<z.ZodString>;
+            feedback: z.ZodOptional<z.ZodString>;
+            meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strict>>;
+        meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strict>;
+    annotations: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        runDir: z.ZodString;
+        rubricPath: z.ZodPipe<z.ZodString, z.ZodTransform<string, string>>;
+        tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        expected: z.ZodObject<{
+            casePassed: z.ZodOptional<z.ZodBoolean>;
+            caseScore: z.ZodOptional<z.ZodNumber>;
+            assertions: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
+                passed: z.ZodBoolean;
+                score: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>;
+        source: z.ZodEnum<{
+            human: "human";
+            synthetic: "synthetic";
+        }>;
+        annotator: z.ZodOptional<z.ZodString>;
+        labeledAt: z.ZodOptional<z.ZodString>;
+        reviewer: z.ZodOptional<z.ZodString>;
+        feedback: z.ZodOptional<z.ZodString>;
+        meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strict>>;
+    hash: z.ZodString;
+}, z.core.$strict>;
+export type LoadedGraderAnnotations = z.infer<typeof LoadedGraderAnnotationsSchema>;
+export declare const loadGraderAnnotations: (filePath: string) => Promise<LoadedGraderAnnotations>;
+export declare const selectGraderAnnotations: (annotations: readonly GraderAnnotation[], selector: ExperimentSelector) => GraderAnnotation[];
+export declare const assertDisjointAnnotationSplits: (splits: Array<{
+    name: string;
+    annotations: readonly GraderAnnotation[];
+}>) => void;
+//# sourceMappingURL=annotation-loader.d.ts.map

package/lib/orchestrator/annotation-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"annotation-loader.d.ts","sourceRoot":"","sources":["../../src/orchestrator/annotation-loader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EAItB,MAAM,kBAAkB,CAAC;AAE1B,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAQ/B,CAAC;AACZ,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AAepF,eAAO,MAAM,qBAAqB,aAAoB,MAAM,KAAG,OAAO,CAAC,uBAAuB,CAiB7F,CAAC;AAEF,eAAO,MAAM,uBAAuB,gBACrB,SAAS,gBAAgB,EAAE,YAC9B,kBAAkB,KAC3B,gBAAgB,EAWlB,CAAC;AAEF,eAAO,MAAM,8BAA8B,WACjC,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,SAAS,gBAAgB,EAAE,CAAA;CAAE,CAAC,KACxE,IAsBF,CAAC"}

package/lib/orchestrator/annotation-loader.js

@@ -0,0 +1,73 @@
+import { readFile } from "node:fs/promises";
+import { dirname, extname, resolve, sep } from "node:path";
+import { pathToFileURL } from "node:url";
+import { z } from "zod";
+import { formatZodIssues, GraderAnnotationDatasetSchema, GraderAnnotationSchema, valueHash, } from "../core/index.js";
+export const LoadedGraderAnnotationsSchema = z
+    .object({
+    path: z.string(),
+    directory: z.string(),
+    dataset: GraderAnnotationDatasetSchema,
+    annotations: z.array(GraderAnnotationSchema),
+    hash: z.string(),
+})
+    .strict();
+const loadAuthoredDataset = async (path) => {
+    if (extname(path).toLowerCase() === ".json") {
+        return JSON.parse(await readFile(path, "utf8"));
+    }
+    const mod = (await import(pathToFileURL(path).href));
+    if (mod.default === undefined) {
+        throw new Error(`Lux grader annotations at ${path} have no default export. Use \`export default defineGraderAnnotations({ ... })\`.`);
+    }
+    return mod.default;
+};
+export const loadGraderAnnotations = async (filePath) => {
+    const path = resolve(filePath);
+    const parsed = GraderAnnotationDatasetSchema.safeParse(await loadAuthoredDataset(path));
+    if (!parsed.success) {
+        throw new Error(`Invalid Lux grader annotations at ${path}: ${formatZodIssues(parsed.error)}`);
+    }
+    const directory = dirname(path);
+    return LoadedGraderAnnotationsSchema.parse({
+        path,
+        directory,
+        dataset: parsed.data,
+        annotations: parsed.data.annotations.map((annotation) => ({
+            ...annotation,
+            runDir: resolve(directory, annotation.runDir.split(/[\\/]/).join(sep)),
+        })),
+        hash: valueHash(parsed.data),
+    });
+};
+export const selectGraderAnnotations = (annotations, selector) => {
+    const patterns = selector.patterns.map((pattern) => pattern.toLowerCase());
+    const tags = selector.tags.map((tag) => tag.toLowerCase());
+    return annotations.filter((annotation) => {
+        const haystack = `${annotation.id}\n${annotation.rubricPath}`.toLowerCase();
+        const patternMatch = patterns.length === 0 || patterns.some((pattern) => haystack.includes(pattern));
+        const annotationTags = new Set(annotation.tags.map((tag) => tag.toLowerCase()));
+        const tagMatch = tags.length === 0 || tags.some((tag) => annotationTags.has(tag));
+        return patternMatch && tagMatch;
+    });
+};
+export const assertDisjointAnnotationSplits = (splits) => {
+    const idOwners = new Map();
+    const runOwners = new Map();
+    for (const split of splits) {
+        for (const annotation of split.annotations) {
+            const idOwner = idOwners.get(annotation.id);
+            if (idOwner) {
+                throw new Error(`grader annotation '${annotation.id}' appears in both ${idOwner} and ${split.name} splits`);
+            }
+            idOwners.set(annotation.id, split.name);
+            const runIdentity = resolve(annotation.runDir);
+            const runOwner = runOwners.get(runIdentity);
+            if (runOwner) {
+                throw new Error(`grader evidence run '${annotation.runDir}' appears in both ${runOwner} and ${split.name} splits`);
+            }
+            runOwners.set(runIdentity, split.name);
+        }
+    }
+};
+//# sourceMappingURL=annotation-loader.js.map

package/lib/orchestrator/annotation-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"annotation-loader.js","sourceRoot":"","sources":["../../src/orchestrator/annotation-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAEL,eAAe,EAEf,6BAA6B,EAC7B,sBAAsB,EACtB,SAAS,GACV,MAAM,kBAAkB,CAAC;AAE1B,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC;KAC3C,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,OAAO,EAAE,6BAA6B;IACtC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;CACjB,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,mBAAmB,GAAG,KAAK,EAAE,IAAY,EAAoB,EAAE;IACnE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAA0B,CAAC;IAC9E,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,mFAAmF,CACrH,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC,OAAO,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EAAE,QAAgB,EAAoC,EAAE;IAChG,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,6BAA6B,CAAC,SAAS,CAAC,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;IACxF,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,KAAK,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,OAAO,6BAA6B,CAAC,KAAK,CAAC;QACzC,IAAI;QACJ,SAAS;QACT,OAAO,EAAE,MAAM,CAAC,IAAI;QACpB,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACxD,GAAG,UAAU;YACb,MAAM,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACvE,CAAC,CAAC;QACH,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;KAC7B,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,WAAwC,EACxC,QAA4B,EACR,EAAE;IACtB,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3D,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE;QACvC,MAAM,QAAQ,GAAG,GAAG,UAAU,CAAC,EAAE,KAAK,UAAU,CAAC,UAAU,EAAE,CAAC,WAAW,EAAE,CAAC;QAC5E,MAAM,YAAY,GAChB,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAClF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAClF,OAAO,YAAY,IAAI,QAAQ,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAC5C,MAAyE,EACnE,EAAE;IACR,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC5C,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CACb,sBAAsB,UAAU,CAAC,EAAE,qBAAqB,OAAO,QAAQ,KAAK,CAAC,IAAI,SAAS,CAC3F,CAAC;YACJ,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAC5C,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CACb,wBAAwB,UAAU,CAAC,MAAM,qBAAqB,QAAQ,QAAQ,KAAK,CAAC,IAAI,SAAS,CAClG,CAAC;YACJ,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC,CAAC"}

package/lib/orchestrator/annotation-store.d.ts

@@ -0,0 +1,3 @@
+import { type GraderAnnotation, type GraderAnnotationDataset } from "../core/index.js";
+export declare const upsertGraderAnnotation: (filePath: string, annotation: GraderAnnotation) => Promise<GraderAnnotationDataset>;
+//# sourceMappingURL=annotation-store.d.ts.map

package/lib/orchestrator/annotation-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"annotation-store.d.ts","sourceRoot":"","sources":["../../src/orchestrator/annotation-store.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,uBAAuB,EAE7B,MAAM,kBAAkB,CAAC;AAsE1B,eAAO,MAAM,sBAAsB,aACvB,MAAM,cACJ,gBAAgB,KAC3B,OAAO,CAAC,uBAAuB,CA4BjC,CAAC"}

package/lib/orchestrator/annotation-store.js

@@ -0,0 +1,100 @@
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { hostname } from "node:os";
+import { dirname, resolve } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { GraderAnnotationDatasetSchema, } from "../core/index.js";
+import { claimLockForRecovery, publishOwnedLock, releaseOwnedLock } from "./owned-lock.js";
+const LOCK_WAIT_MS = 5_000;
+const STALE_LOCK_MS = 30_000;
+const processIsAlive = (pid) => {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code !== "ESRCH";
+    }
+};
+const recoverStaleLock = async (lockPath) => {
+    const claim = await claimLockForRecovery(lockPath);
+    if (!claim)
+        return false;
+    let owner;
+    try {
+        try {
+            owner = JSON.parse(await readFile(claim.path, "utf8"));
+        }
+        catch {
+            return false;
+        }
+        const ageMs = Date.now() - Date.parse(owner.acquiredAt);
+        if (owner.hostname !== hostname() ||
+            !Number.isFinite(ageMs) ||
+            ageMs < STALE_LOCK_MS ||
+            processIsAlive(owner.pid)) {
+            return false;
+        }
+        return await claim.removeLock();
+    }
+    finally {
+        await claim.release();
+    }
+};
+const acquireAnnotationLock = async (path) => {
+    const lockPath = `${path}.lock`;
+    const deadline = Date.now() + LOCK_WAIT_MS;
+    while (true) {
+        try {
+            const owner = await publishOwnedLock(lockPath, {
+                pid: process.pid,
+                hostname: hostname(),
+                acquiredAt: new Date().toISOString(),
+            });
+            return async () => {
+                await releaseOwnedLock(lockPath, owner.lockToken);
+            };
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+            if (await recoverStaleLock(lockPath))
+                continue;
+            if (Date.now() >= deadline) {
+                throw new Error(`timed out waiting to update grader annotations: ${path}`);
+            }
+            await delay(10);
+        }
+    }
+};
+export const upsertGraderAnnotation = async (filePath, annotation) => {
+    const path = resolve(filePath);
+    await mkdir(dirname(path), { recursive: true });
+    const release = await acquireAnnotationLock(path);
+    const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
+    try {
+        const dataset = existsSync(path)
+            ? GraderAnnotationDatasetSchema.parse(JSON.parse(await readFile(path, "utf8")))
+            : { schemaVersion: 1, annotations: [] };
+        const existing = dataset.annotations.findIndex((entry) => entry.id === annotation.id);
+        if (existing === -1)
+            dataset.annotations.push(annotation);
+        else {
+            const current = dataset.annotations[existing];
+            if (current?.runDir !== annotation.runDir || current.rubricPath !== annotation.rubricPath) {
+                throw new Error(`annotation id '${annotation.id}' already targets different evidence; choose a unique id`);
+            }
+            dataset.annotations[existing] = annotation;
+        }
+        dataset.annotations.sort((a, b) => a.id.localeCompare(b.id));
+        await writeFile(temporary, `${JSON.stringify(dataset, null, 2)}\n`, { flag: "wx" });
+        await rename(temporary, path);
+        return dataset;
+    }
+    finally {
+        await rm(temporary, { force: true });
+        await release();
+    }
+};
+//# sourceMappingURL=annotation-store.js.map

package/lib/orchestrator/annotation-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"annotation-store.js","sourceRoot":"","sources":["../../src/orchestrator/annotation-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAGL,6BAA6B,GAC9B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAE3F,MAAM,YAAY,GAAG,KAAK,CAAC;AAC3B,MAAM,aAAa,GAAG,MAAM,CAAC;AAQ7B,MAAM,cAAc,GAAG,CAAC,GAAW,EAAW,EAAE;IAC9C,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAQ,KAA+B,CAAC,IAAI,KAAK,OAAO,CAAC;IAC3D,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAgB,EAAoB,EAAE;IACpE,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IACnD,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,IAAI,KAA0B,CAAC;IAC/B,IAAI,CAAC;QACH,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAwB,CAAC;QAChF,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACxD,IACE,KAAK,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAC7B,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YACvB,KAAK,GAAG,aAAa;YACrB,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,EACzB,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC;IAClC,CAAC;YAAS,CAAC;QACT,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,KAAK,EAAE,IAAY,EAAgC,EAAE;IACjF,MAAM,QAAQ,GAAG,GAAG,IAAI,OAAO,CAAC;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAC3C,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE;gBAC7C,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,EAAE;gBACpB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACrC,CAAC,CAAC;YACH,OAAO,KAAK,IAAI,EAAE;gBAChB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YACpD,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAC;YACpE,IAAI,MAAM,gBAAgB,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAC/C,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,mDAAmD,IAAI,EAAE,CAAC,CAAC;YAC7E,CAAC;YACD,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,KAAK,EACzC,QAAgB,EAChB,UAA4B,EACM,EAAE;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,MAAM,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,OAAO,GAA4B,UAAU,CAAC,IAAI,CAAC;YACvD,CAAC,CAAC,6BAA6B,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YAC/E,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,UAAU,CAAC,EAAE,CAAC,CAAC;QACtF,IAAI,QAAQ,KAAK,CAAC,CAAC;YAAE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;aACrD,CAAC;YACJ,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC9C,IAAI,OAAO,EAAE,MAAM,KAAK,UAAU,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1F,MAAM,IAAI,KAAK,CACb,kBAAkB,UAAU,CAAC,EAAE,0DAA0D,CAC1F,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;QAC7C,CAAC;QACD,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7D,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACpF,MAAM,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC9B,OAAO,OAAO,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,OAAO,EAAE,CAAC;IAClB,CAAC;AACH,CAAC,CAAC"}

package/lib/orchestrator/authored-dependencies.d.ts

@@ -0,0 +1,3 @@
+export declare const assertAuthoredImportsContained: (root: string, entryPaths: readonly string[]) => Promise<void>;
+export declare const authoredDependencyGraphHash: (root: string, entryPaths: readonly string[]) => Promise<string>;
+//# sourceMappingURL=authored-dependencies.d.ts.map

package/lib/orchestrator/authored-dependencies.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"authored-dependencies.d.ts","sourceRoot":"","sources":["../../src/orchestrator/authored-dependencies.ts"],"names":[],"mappings":"AA4KA,eAAO,MAAM,8BAA8B,SACnC,MAAM,cACA,SAAS,MAAM,EAAE,KAC5B,OAAO,CAAC,IAAI,CAEd,CAAC;AAEF,eAAO,MAAM,2BAA2B,SAChC,MAAM,cACA,SAAS,MAAM,EAAE,KAC5B,OAAO,CAAC,MAAM,CAUhB,CAAC"}