@prismatic-io/lux 0.0.1

package/lib/orchestrator/grading.d.ts

@@ -0,0 +1,31 @@
+import type { z } from "zod";
+import { type Assertion, type AssertionImpl, type GradingAssertionEntry, type GradingReport, type Registry, type Run, type RunObserver } from "../core/index.js";
+/**
+ * A `.strict()` plugin schema must not choke on framework keys it never
+ * declared, while one that does declare them (rubric's `threshold`,
+ * tool-called's `name`) keeps receiving them. Non-object schemas expose no
+ * shape to consult, so they get the assertion as-is.
+ */
+export declare const assertionConfigForSchema: (a: Assertion, schema: z.ZodTypeAny) => unknown;
+export type AssertionPluginIdentity = {
+    type: string;
+    pluginVersion: string | null;
+};
+export type AssertionEvaluatorIdentity = AssertionPluginIdentity & {
+    evaluate: AssertionImpl["evaluate"];
+};
+export declare const assertionEvaluatorIdentities: (assertions: readonly Assertion[], registry: Registry) => AssertionEvaluatorIdentity[];
+export declare const assertionPluginIdentities: (assertions: readonly Assertion[], registry: Registry) => AssertionPluginIdentity[];
+export declare const graderConfigHash: (input: {
+    luxVersion: string;
+    implementationHash: string;
+    assertions: readonly AssertionEvaluatorIdentity[];
+    validatedAssertionsHash: string;
+    graderContext?: unknown;
+}) => string;
+export declare const validateAssertionPlan: (assertions: readonly Assertion[], registry: Registry) => Promise<unknown[]>;
+export declare const gradeRun: (run: Run, registry: Registry, observer?: RunObserver, abortSignal?: AbortSignal, recordAssertion?: (entry: GradingAssertionEntry, index: number) => void | Promise<void>, modelAttempts?: {
+    start: (kind: "judge") => string | Promise<string>;
+    complete: (attemptIds: readonly string[]) => void | Promise<void>;
+}, validatedConfigs?: readonly unknown[]) => Promise<GradingReport>;
+//# sourceMappingURL=grading.d.ts.map

package/lib/orchestrator/grading.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grading.d.ts","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,EACL,KAAK,SAAS,EACd,KAAK,aAAa,EAGlB,KAAK,qBAAqB,EAC1B,KAAK,aAAa,EAElB,KAAK,QAAQ,EACb,KAAK,GAAG,EACR,KAAK,WAAW,EAEjB,MAAM,kBAAkB,CAAC;AAU1B;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,MAAO,SAAS,UAAU,CAAC,CAAC,UAAU,KAAG,OAM7E,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG,uBAAuB,GAAG;IACjE,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,4BAA4B,eAC3B,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,0BAA0B,EAYzB,CAAC;AAEL,eAAO,MAAM,yBAAyB,eACxB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,uBAAuB,EAIrB,CAAC;AAEN,eAAO,MAAM,gBAAgB,UAAW;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,SAAS,0BAA0B,EAAE,CAAC;IAClD,uBAAuB,EAAE,MAAM,CAAC;IAChC,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,KAAG,MAOA,CAAC;AAEL,eAAO,MAAM,qBAAqB,eACpB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,OAAO,CAAC,OAAO,EAAE,CAuBnB,CAAC;AAWF,eAAO,MAAM,QAAQ,QACd,GAAG,YACE,QAAQ,aACP,WAAW,gBACR,WAAW,oBACP,CAAC,KAAK,EAAE,qBAAqB,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,kBACvE;IACd,KAAK,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,QAAQ,EAAE,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnE,qBACkB,SAAS,OAAO,EAAE,KACpC,OAAO,CAAC,aAAa,CA4HvB,CAAC"}

package/lib/orchestrator/grading.js

@@ -0,0 +1,186 @@
+import { AssertionResultSchema, guardRunObserver, valueHash, } from "../core/index.js";
+import { validatePluginConfigAsync } from "./validate-plugin-config.js";
+const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold"]);
+const shapeOf = (schema) => {
+    const shape = schema.shape;
+    return shape && typeof shape === "object" ? shape : null;
+};
+/**
+ * A `.strict()` plugin schema must not choke on framework keys it never
+ * declared, while one that does declare them (rubric's `threshold`,
+ * tool-called's `name`) keeps receiving them. Non-object schemas expose no
+ * shape to consult, so they get the assertion as-is.
+ */
+export const assertionConfigForSchema = (a, schema) => {
+    const shape = shapeOf(schema);
+    if (!shape)
+        return a;
+    return Object.fromEntries(Object.entries(a).filter(([key]) => !FRAMEWORK_KEYS.has(key) || key in shape));
+};
+export const assertionEvaluatorIdentities = (assertions, registry) => [...new Set(assertions.map((assertion) => assertion.type))].sort().map((type) => {
+    const implementation = registry.getAssertion(type);
+    if (!implementation)
+        throw new Error(`unknown assertion type: ${type}`);
+    return {
+        type,
+        pluginVersion: "pluginVersion" in implementation && typeof implementation.pluginVersion === "string"
+            ? implementation.pluginVersion
+            : null,
+        evaluate: implementation.evaluate,
+    };
+});
+export const assertionPluginIdentities = (assertions, registry) => assertionEvaluatorIdentities(assertions, registry).map(({ type, pluginVersion }) => ({
+    type,
+    pluginVersion,
+}));
+export const graderConfigHash = (input) => valueHash({
+    luxVersion: input.luxVersion,
+    implementationHash: input.implementationHash,
+    assertions: input.assertions,
+    validatedAssertionsHash: input.validatedAssertionsHash,
+    graderContext: input.graderContext ?? null,
+});
+export const validateAssertionPlan = async (assertions, registry) => {
+    const validated = [];
+    // Async refinements may allocate workers or other bounded resources.
+    // Validate in authored order so an unbounded assertion list cannot fan
+    // those resources out before the subject driver even starts.
+    for (const [index, assertion] of assertions.entries()) {
+        const implementation = registry.getAssertion(assertion.type);
+        if (!implementation) {
+            throw new Error(`invalid assertion plan at assertions[${index}]: no assertion implementation registered for type '${assertion.type}'`);
+        }
+        validated.push(await validatePluginConfigAsync(implementation.configSchema, implementation.configSchema
+            ? assertionConfigForSchema(assertion, implementation.configSchema)
+            : assertion, `assertion '${assertion.type}' at assertions[${index}]`));
+    }
+    return validated;
+};
+const describeAssertion = (a) => {
+    if (a.name)
+        return a.name;
+    const fields = Object.entries(a)
+        .filter(([k, v]) => !FRAMEWORK_KEYS.has(k) && typeof v !== "function")
+        .map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`)
+        .join(" ");
+    return fields ? `${a.type} ${fields}` : a.type;
+};
+export const gradeRun = async (run, registry, observer, abortSignal, recordAssertion, modelAttempts, validatedConfigs) => {
+    const guardedObserver = guardRunObserver(observer);
+    const assertions = run.case.assertions ?? [];
+    if (assertions.length === 0) {
+        return { schemaVersion: 1, casePassed: null, caseScore: null, assertions: [] };
+    }
+    const entries = [];
+    let weightedScoreSum = 0;
+    let weightSum = 0;
+    let allPassed = true;
+    let gradingIncomplete = false;
+    for (const [index, a] of assertions.entries()) {
+        // Grading is the last cancellation point — an LLM judge can run for
+        // minutes per assertion. An aborted grade can't claim everything passed.
+        if (abortSignal?.aborted) {
+            gradingIncomplete = true;
+            break;
+        }
+        const impl = registry.getAssertion(a.type);
+        const weight = a.weight ?? 1;
+        // `threshold` is an OPTIONAL extra score floor layered on top of the
+        // assertion's own verdict (`result.passed`). Default 0 means "no floor",
+        // so a graded assertion like `rubric` passes on its own bar (judge
+        // score >= 0.8). A non-zero default here would force every sub-1.0
+        // graded score to fail even when the assertion reports passed.
+        const threshold = a.threshold ?? 0;
+        const id = a.id ?? `${a.type}:${index + 1}`;
+        const name = describeAssertion(a);
+        const progress = {
+            index,
+            total: assertions.length,
+            id,
+            type: a.type,
+            name,
+        };
+        guardedObserver?.assertionStart?.(progress);
+        const startedAt = Date.now();
+        let entry;
+        const assertionAttemptIds = [];
+        try {
+            if (!impl)
+                throw new Error(`no assertion implementation registered for type '${a.type}'`);
+            const validated = validatedConfigs
+                ? validatedConfigs[index]
+                : await validatePluginConfigAsync(impl.configSchema, impl.configSchema ? assertionConfigForSchema(a, impl.configSchema) : a, `assertion '${a.type}'`);
+            if (validatedConfigs && index >= validatedConfigs.length) {
+                throw new Error(`validated assertion plan is missing assertions[${index}]`);
+            }
+            // Pass the validated config so schema defaults (e.g. timeoutMs)
+            // reach evaluate(); the raw assertion object has none applied.
+            const result = AssertionResultSchema.parse(await impl.evaluate(validated, run, {
+                ...(abortSignal ? { abortSignal } : {}),
+                ...(modelAttempts
+                    ? {
+                        recordModelAttemptStart: async (_kind) => {
+                            assertionAttemptIds.push(await modelAttempts.start("judge"));
+                        },
+                    }
+                    : {}),
+            }));
+            const passed = result.errorMessage === undefined && result.passed && result.score >= threshold;
+            entry = {
+                id,
+                type: a.type,
+                name,
+                passed,
+                score: result.score,
+                weight,
+                threshold,
+                durationMs: Date.now() - startedAt,
+                ...(result.comment !== undefined ? { comment: result.comment } : {}),
+                detail: result.detail,
+                ...(result.errorMessage !== undefined ? { errorMessage: result.errorMessage } : {}),
+                ...(result.usage ? { usage: result.usage } : {}),
+            };
+            if (result.errorMessage !== undefined) {
+                gradingIncomplete = true;
+            }
+            else {
+                if (!passed)
+                    allPassed = false;
+                weightedScoreSum += weight * result.score;
+                weightSum += weight;
+            }
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            entry = {
+                id,
+                type: a.type,
+                name,
+                passed: false,
+                score: 0,
+                weight,
+                threshold,
+                durationMs: Date.now() - startedAt,
+                errorMessage: message,
+            };
+            gradingIncomplete = true;
+        }
+        entries.push(entry);
+        guardedObserver?.assertionEnd?.(progress, entry);
+        await recordAssertion?.(entry, index);
+        await modelAttempts?.complete(assertionAttemptIds);
+    }
+    let caseScore = null;
+    if (!gradingIncomplete)
+        caseScore = weightSum > 0 ? weightedScoreSum / weightSum : 0;
+    return {
+        schemaVersion: 1,
+        casePassed: gradingIncomplete ? null : allPassed,
+        caseScore,
+        assertions: entries,
+        ...(gradingIncomplete
+            ? { incomplete: { exitReason: "error", phase: "grading" } }
+            : {}),
+    };
+};
+//# sourceMappingURL=grading.js.map

package/lib/orchestrator/grading.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grading.js","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,qBAAqB,EAGrB,gBAAgB,EAIhB,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAExE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;AAE9E,MAAM,OAAO,GAAG,CAAC,MAAoB,EAAkC,EAAE;IACvE,MAAM,KAAK,GAAI,MAA8B,CAAC,KAAK,CAAC;IACpD,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,IAAI,CAAC;AACxF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAY,EAAE,MAAoB,EAAW,EAAE;IACtF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK,CAAC,CAC9E,CAAC;AACJ,CAAC,CAAC;AAWF,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAC1C,UAAgC,EAChC,QAAkB,EACY,EAAE,CAChC,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;IAC9E,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO;QACL,IAAI;QACJ,aAAa,EACX,eAAe,IAAI,cAAc,IAAI,OAAO,cAAc,CAAC,aAAa,KAAK,QAAQ;YACnF,CAAC,CAAC,cAAc,CAAC,aAAa;YAC9B,CAAC,CAAC,IAAI;QACV,QAAQ,EAAE,cAAc,CAAC,QAAQ;KAClC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,UAAgC,EAChC,QAAkB,EACS,EAAE,CAC7B,4BAA4B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;IACnF,IAAI;IACJ,aAAa;CACd,CAAC,CAAC,CAAC;AAEN,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,KAMhC,EAAU,EAAE,CACX,SAAS,CAAC;IACR,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;IAC5C,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;IACtD,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;CAC3C,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EACxC,UAAgC,EAChC,QAAkB,EACE,EAAE;IACtB,MAAM,SAAS,GAAc,EAAE,CAAC;IAChC,qEAAqE;IACrE,uEAAuE;IACvE,6DAA6D;IAC7D,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACb,wCAAwC,KAAK,uDAAuD,SAAS,CAAC,IAAI,GAAG,CACtH,CAAC;QACJ,CAAC;QACD,SAAS,CAAC,IAAI,CACZ,MAAM,yBAAyB,CAC7B,cAAc,CAAC,YAAY,EAC3B,cAAc,CAAC,YAAY;YACzB,CAAC,CAAC,wBAAwB,CAAC,SAAS,EAAE,cAAc,CAAC,YAAY,CAAC;YAClE,CAAC,CAAC,SAAS,EACb,cAAc,SAAS,CAAC,IAAI,mBAAmB,KAAK,GAAG,CACxD,CACF,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAY,EAAU,EAAE;IACjD,IAAI,CAAC,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;SAC7B,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,UAAU,CAAC;SACrE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,GAAQ,EACR,QAAkB,EAClB,QAAsB,EACtB,WAAyB,EACzB,eAAuF,EACvF,aAGC,EACD,gBAAqC,EACb,EAAE;IAC1B,MAAM,eAAe,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IAC7C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAE9B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,oEAAoE;QACpE,yEAAyE;QACzE,IAAI,WAAW,EAAE,OAAO,EAAE,CAAC;YACzB,iBAAiB,GAAG,IAAI,CAAC;YACzB,MAAM;QACR,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;QAC7B,qEAAqE;QACrE,yEAAyE;QACzE,mEAAmE;QACnE,mEAAmE;QACnE,+DAA+D;QAC/D,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAsB;YAClC,KAAK;YACL,KAAK,EAAE,UAAU,CAAC,MAAM;YACxB,EAAE;YACF,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI;SACL,CAAC;QACF,eAAe,EAAE,cAAc,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,KAA4B,CAAC;QACjC,MAAM,mBAAmB,GAAa,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,gBAAgB;gBAChC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC;gBACzB,CAAC,CAAC,MAAM,yBAAyB,CAC7B,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,EACtE,cAAc,CAAC,CAAC,IAAI,GAAG,CACxB,CAAC;YACN,IAAI,gBAAgB,IAAI,KAAK,IAAI,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBACzD,MAAM,IAAI,KAAK,CAAC,kDAAkD,KAAK,GAAG,CAAC,CAAC;YAC9E,CAAC;YACD,gEAAgE;YAChE,+DAA+D;YAC/D,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAqB,EAAE,GAAG,EAAE;gBAC9C,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,aAAa;oBACf,CAAC,CAAC;wBACE,uBAAuB,EAAE,KAAK,EAAE,KAAc,EAAE,EAAE;4BAChD,mBAAmB,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC/D,CAAC;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CACH,CAAC;YACF,MAAM,MAAM,GACV,MAAM,CAAC,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;YAClF,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,GAAG,CAAC,MAAM,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACjD,CAAC;YACF,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBACtC,iBAAiB,GAAG,IAAI,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM;oBAAE,SAAS,GAAG,KAAK,CAAC;gBAC/B,gBAAgB,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC1C,SAAS,IAAI,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,YAAY,EAAE,OAAO;aACtB,CAAC;YACF,iBAAiB,GAAG,IAAI,CAAC;QAC3B,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,eAAe,EAAE,YAAY,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACjD,MAAM,eAAe,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,aAAa,EAAE,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,SAAS,GAAkB,IAAI,CAAC;IACpC,IAAI,CAAC,iBAAiB;QAAE,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QAChD,SAAS;QACT,UAAU,EAAE,OAAO;QACnB,GAAG,CAAC,iBAAiB;YACnB,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,OAAgB,EAAE,KAAK,EAAE,SAAkB,EAAE,EAAE;YAC7E,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC,CAAC"}

package/lib/orchestrator/index.d.ts

@@ -0,0 +1,22 @@
+export { assertDisjointAnnotationSplits, type LoadedGraderAnnotations, LoadedGraderAnnotationsSchema, loadGraderAnnotations, selectGraderAnnotations, } from "./annotation-loader.js";
+export { upsertGraderAnnotation } from "./annotation-store.js";
+export { applyCandidate, captureSeedCandidate, createCandidate, hashSubjectTree, materializeCandidate, validateCandidate, } from "./candidates.js";
+export { type ComparisonInput, compareRuns, formatComparisonText, type GradingDelta, hasRegression, loadRunBundle, type RunBundle, type RunComparison, } from "./compare.js";
+export { buildRegistryFromConfig, type ExperimentProfile, ExperimentProfileSchema, findConfigFile, type LuxConfig, loadLuxConfig, type ResolvedLuxConfig, } from "./config.js";
+export { type CaseFilter, type DiscoveredCase, type DiscoveryProblem, type DiscoveryResult, discoverCases, selectCases, } from "./discover.js";
+export { type DoctorCampaignReport, DoctorCampaignReportSchema, type DoctorDiagnostic, DoctorDiagnosticSchema, type DoctorOptions, DoctorOptionsSchema, type DoctorReport, DoctorReportSchema, doctorProject, formatDoctorReport, } from "./doctor.js";
+export { type CampaignExecutionOptions, type CampaignExecutionResult, CampaignExecutionResultSchema, deriveExperimentSelection, type ExperimentCampaignPlan, ExperimentCampaignPlanSchema, type ExperimentDecisionCandidate, ExperimentDecisionCandidateSchema, optimizeExperimentCampaign, paretoFrontier, planExperimentCampaign, rankCandidates, runExperimentCampaign, } from "./experiment.js";
+export { type ExperimentCommandMode, ExperimentCommandModeSchema, experimentCampaignMode, type LoadedExperimentCampaign, LoadedExperimentCampaignSchema, loadExperimentCampaign, resolveExperimentCampaignPath, } from "./experiment-loader.js";
+export { createExperimentDecisionReport, type ExperimentDecisionReport, ExperimentDecisionReportSchema, formatExperimentDecisionMarkdown, loadExperimentDecisionReport, loadVerifiedExperimentDecisionSnapshot, type VerifiedExperimentDecisionSnapshot, VerifiedExperimentDecisionSnapshotSchema, verifyExperimentDecisionIntegrity, } from "./experiment-report.js";
+export { type ExperimentBudgetUsage, type ExperimentEvaluation, type ExperimentRun, ExperimentRunSchema, type ExperimentSelection, ExperimentSelectionSchema, formatExperimentRun, formatExperimentRunsTable, isExperimentRun, listExperimentRuns, loadExperimentCandidate, loadExperimentCandidates, loadExperimentRun, migrateExperimentCandidateIntegrity, verifyExperimentRunIntegrity, } from "./experiment-runs.js";
+export { compareGraderAnnotation, evaluateGraderCandidate, type GraderEvaluationEvidence, GraderEvaluationEvidenceSchema, type GraderJudgment, summarizeGraderEvidence, } from "./grader-experiment.js";
+export { gradeRun, validateAssertionPlan } from "./grading.js";
+export { formatRunDetail, formatRunsTable, type ListedRun, listRuns, loadRunDetail, type RunDetail, } from "./list-runs.js";
+export { loadEvalCase } from "./loader.js";
+export { type CaseObserver, Orchestrator, type OrchestratorOptions, type RunRequest, type RunResult, type SuiteCaseResult, type SuiteItem, type SuiteObserver, } from "./orchestrator.js";
+export { GRADING_SNAPSHOT_SCHEMA_VERSION, type GradingSnapshot, GradingSnapshotSchema, loadFrozenRegradableRun, loadRegradableRun, regradeRun, releaseFrozenRegradableRun, } from "./regrade.js";
+export { loadRubricDefinition } from "./rubric-loader.js";
+export { compareSuiteRuns, compareSuiteSummaries, formatSuiteComparisonText, hasSuiteRegression, type SuiteCaseComparison, SuiteCaseComparisonSchema, type SuiteComparison, SuiteComparisonSchema, } from "./suite-compare.js";
+export { formatSuiteRunDetail, formatSuiteRunsTable, listSuiteRuns, loadSuiteRun, type SuiteRun, type SuiteRunItem, SuiteRunSchema, } from "./suite-runs.js";
+export { buildSuiteSummary, type CaseSummary, loadSuiteSummary, type MetricSummary, MetricSummarySchema, type OutcomeSummary, type SuiteSample, SuiteSampleSchema, type SuiteSummary, SuiteSummarySchema, summarizeNumbers, writeSuiteSummary, } from "./suite-summary.js";
+//# sourceMappingURL=index.d.ts.map

package/lib/orchestrator/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/orchestrator/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,8BAA8B,EAC9B,KAAK,uBAAuB,EAC5B,6BAA6B,EAC7B,qBAAqB,EACrB,uBAAuB,GACxB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EACL,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EACf,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,KAAK,eAAe,EACpB,WAAW,EACX,oBAAoB,EACpB,KAAK,YAAY,EACjB,aAAa,EACb,aAAa,EACb,KAAK,SAAS,EACd,KAAK,aAAa,GACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,uBAAuB,EACvB,KAAK,iBAAiB,EACtB,uBAAuB,EACvB,cAAc,EACd,KAAK,SAAS,EACd,aAAa,EACb,KAAK,iBAAiB,GACvB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,KAAK,UAAU,EACf,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,aAAa,EACb,WAAW,GACZ,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,KAAK,oBAAoB,EACzB,0BAA0B,EAC1B,KAAK,gBAAgB,EACrB,sBAAsB,EACtB,KAAK,aAAa,EAClB,mBAAmB,EACnB,KAAK,YAAY,EACjB,kBAAkB,EAClB,aAAa,EACb,kBAAkB,GACnB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,EAC5B,6BAA6B,EAC7B,yBAAyB,EACzB,KAAK,sBAAsB,EAC3B,4BAA4B,EAC5B,KAAK,2BAA2B,EAChC,iCAAiC,EACjC,0BAA0B,EAC1B,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,qBAAqB,GACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,KAAK,qBAAqB,EAC1B,2BAA2B,EAC3B,sBAAsB,EACtB,KAAK,wBAAwB,EAC7B,8BAA8B,EAC9B,sBAAsB,EACtB,6BAA6B,GAC9B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,8BAA8B,EAC9B,KAAK,wBAAwB,EAC7B,8BAA8B,EAC9B,gCAAgC,EAChC,4BAA4B,EAC5B,sCAAsC,EACtC,KAAK,kCAAkC,EACvC,wCAAwC,EACxC,iCAAiC,GAClC,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,aAAa,EAClB,mBAAmB,EACnB,KAAK,mBAAmB,EACxB,yBAAyB,EACzB,mBAAmB,EACnB,yBAAyB,EACzB,eAAe,EACf,kBAAkB,EAClB,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,mCAAmC,EACnC,4BAA4B,GAC7B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,uBAAuB,EACvB,uBAAuB,EACvB,KAAK,wBAAwB,EAC7B,8BAA8B,EAC9B,KAAK,cAAc,EACnB,uBAAuB,GACxB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EACL,eAAe,EACf,eAAe,EACf,KAAK,SAAS,EACd,QAAQ,EACR,aAAa,EACb,KAAK,SAAS,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAI3C,OAAO,EACL,KAAK,YAAY,EACjB,YAAY,EACZ,KAAK,mBAAmB,EACxB,KAAK,UAAU,EACf,KAAK,SAAS,EACd,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,aAAa,GACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,+BAA+B,EAC/B,KAAK,eAAe,EACpB,qBAAqB,EACrB,uBAAuB,EACvB,iBAAiB,EACjB,UAAU,EACV,0BAA0B,GAC3B,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,qBAAqB,EACrB,yBAAyB,EACzB,kBAAkB,EAClB,KAAK,mBAAmB,EACxB,yBAAyB,EACzB,KAAK,eAAe,EACpB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,aAAa,EACb,YAAY,EACZ,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,iBAAiB,EACjB,KAAK,WAAW,EAChB,gBAAgB,EAChB,KAAK,aAAa,EAClB,mBAAmB,EACnB,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,iBAAiB,EACjB,KAAK,YAAY,EACjB,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC"}

package/lib/orchestrator/index.js

@@ -0,0 +1,30 @@
+// Public orchestration surface. Keep this list explicit: several orchestrator
+// modules export helpers solely for focused contract tests, and a wildcard
+// here would silently turn the next such helper into package API.
+export { assertDisjointAnnotationSplits, LoadedGraderAnnotationsSchema, loadGraderAnnotations, selectGraderAnnotations, } from "./annotation-loader.js";
+export { upsertGraderAnnotation } from "./annotation-store.js";
+export { applyCandidate, captureSeedCandidate, createCandidate, hashSubjectTree, materializeCandidate, validateCandidate, } from "./candidates.js";
+export { compareRuns, formatComparisonText, hasRegression, loadRunBundle, } from "./compare.js";
+export { buildRegistryFromConfig, ExperimentProfileSchema, findConfigFile, loadLuxConfig, } from "./config.js";
+export { discoverCases, selectCases, } from "./discover.js";
+export { DoctorCampaignReportSchema, DoctorDiagnosticSchema, DoctorOptionsSchema, DoctorReportSchema, doctorProject, formatDoctorReport, } from "./doctor.js";
+export { CampaignExecutionResultSchema, deriveExperimentSelection, ExperimentCampaignPlanSchema, ExperimentDecisionCandidateSchema, optimizeExperimentCampaign, paretoFrontier, planExperimentCampaign, rankCandidates, runExperimentCampaign, } from "./experiment.js";
+export { ExperimentCommandModeSchema, experimentCampaignMode, LoadedExperimentCampaignSchema, loadExperimentCampaign, resolveExperimentCampaignPath, } from "./experiment-loader.js";
+export { createExperimentDecisionReport, ExperimentDecisionReportSchema, formatExperimentDecisionMarkdown, loadExperimentDecisionReport, loadVerifiedExperimentDecisionSnapshot, VerifiedExperimentDecisionSnapshotSchema, verifyExperimentDecisionIntegrity, } from "./experiment-report.js";
+export { ExperimentRunSchema, ExperimentSelectionSchema, formatExperimentRun, formatExperimentRunsTable, isExperimentRun, listExperimentRuns, loadExperimentCandidate, loadExperimentCandidates, loadExperimentRun, migrateExperimentCandidateIntegrity, verifyExperimentRunIntegrity, } from "./experiment-runs.js";
+export { compareGraderAnnotation, evaluateGraderCandidate, GraderEvaluationEvidenceSchema, summarizeGraderEvidence, } from "./grader-experiment.js";
+export { gradeRun, validateAssertionPlan } from "./grading.js";
+export { formatRunDetail, formatRunsTable, listRuns, loadRunDetail, } from "./list-runs.js";
+export { loadEvalCase } from "./loader.js";
+// The resolution and run-directory helpers exported by orchestrator.ts exist
+// for its own contract tests; the class and observer/request types are the
+// supported programmatic runner.
+export { Orchestrator, } from "./orchestrator.js";
+export { GRADING_SNAPSHOT_SCHEMA_VERSION, GradingSnapshotSchema, loadFrozenRegradableRun, loadRegradableRun, regradeRun, releaseFrozenRegradableRun, } from "./regrade.js";
+export { loadRubricDefinition } from "./rubric-loader.js";
+export { compareSuiteRuns, compareSuiteSummaries, formatSuiteComparisonText, hasSuiteRegression, SuiteCaseComparisonSchema, SuiteComparisonSchema, } from "./suite-compare.js";
+// openSuiteRun is persistence plumbing; callers create suites through
+// Orchestrator and consume them through these read APIs.
+export { formatSuiteRunDetail, formatSuiteRunsTable, listSuiteRuns, loadSuiteRun, SuiteRunSchema, } from "./suite-runs.js";
+export { buildSuiteSummary, loadSuiteSummary, MetricSummarySchema, SuiteSampleSchema, SuiteSummarySchema, summarizeNumbers, writeSuiteSummary, } from "./suite-summary.js";
+//# sourceMappingURL=index.js.map

package/lib/orchestrator/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/orchestrator/index.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2EAA2E;AAC3E,kEAAkE;AAClE,OAAO,EACL,8BAA8B,EAE9B,6BAA6B,EAC7B,qBAAqB,EACrB,uBAAuB,GACxB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EACL,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EACf,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,WAAW,EACX,oBAAoB,EAEpB,aAAa,EACb,aAAa,GAGd,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,uBAAuB,EAEvB,uBAAuB,EACvB,cAAc,EAEd,aAAa,GAEd,MAAM,aAAa,CAAC;AACrB,OAAO,EAKL,aAAa,EACb,WAAW,GACZ,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,0BAA0B,EAE1B,sBAAsB,EAEtB,mBAAmB,EAEnB,kBAAkB,EAClB,aAAa,EACb,kBAAkB,GACnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAGL,6BAA6B,EAC7B,yBAAyB,EAEzB,4BAA4B,EAE5B,iCAAiC,EACjC,0BAA0B,EAC1B,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,qBAAqB,GACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,2BAA2B,EAC3B,sBAAsB,EAEtB,8BAA8B,EAC9B,sBAAsB,EACtB,6BAA6B,GAC9B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,8BAA8B,EAE9B,8BAA8B,EAC9B,gCAAgC,EAChC,4BAA4B,EAC5B,sCAAsC,EAEtC,wCAAwC,EACxC,iCAAiC,GAClC,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAIL,mBAAmB,EAEnB,yBAAyB,EACzB,mBAAmB,EACnB,yBAAyB,EACzB,eAAe,EACf,kBAAkB,EAClB,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,mCAAmC,EACnC,4BAA4B,GAC7B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,uBAAuB,EACvB,uBAAuB,EAEvB,8BAA8B,EAE9B,uBAAuB,GACxB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EACL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,aAAa,GAEd,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,6EAA6E;AAC7E,2EAA2E;AAC3E,iCAAiC;AACjC,OAAO,EAEL,YAAY,GAOb,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,+BAA+B,EAE/B,qBAAqB,EACrB,uBAAuB,EACvB,iBAAiB,EACjB,UAAU,EACV,0BAA0B,GAC3B,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,qBAAqB,EACrB,yBAAyB,EACzB,kBAAkB,EAElB,yBAAyB,EAEzB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,sEAAsE;AACtE,yDAAyD;AACzD,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,aAAa,EACb,YAAY,EAGZ,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,iBAAiB,EAEjB,gBAAgB,EAEhB,mBAAmB,EAGnB,iBAAiB,EAEjB,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC"}

package/lib/orchestrator/lifecycle-hook-execution.d.ts

@@ -0,0 +1,4 @@
+import type { LifecycleHookCleanup, LifecycleHooksPlugin } from "../core/index.js";
+export declare const setupLifecycleHooks: (hooks: readonly LifecycleHooksPlugin[], scope: "suite" | "case", invoke: (hook: LifecycleHooksPlugin) => LifecycleHookCleanup | undefined | Promise<LifecycleHookCleanup | undefined>, abortSignal?: AbortSignal) => Promise<LifecycleHookCleanup>;
+export declare const combineLifecycleFailures: (primary: unknown, cleanup: unknown, message: string) => AggregateError;
+//# sourceMappingURL=lifecycle-hook-execution.d.ts.map

package/lib/orchestrator/lifecycle-hook-execution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lifecycle-hook-execution.d.ts","sourceRoot":"","sources":["../../src/orchestrator/lifecycle-hook-execution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AA8BnF,eAAO,MAAM,mBAAmB,UACvB,SAAS,oBAAoB,EAAE,SAC/B,OAAO,GAAG,MAAM,UACf,CACN,IAAI,EAAE,oBAAoB,KACvB,oBAAoB,GAAG,SAAS,GAAG,OAAO,CAAC,oBAAoB,GAAG,SAAS,CAAC,gBACnE,WAAW,KACxB,OAAO,CAAC,oBAAoB,CA8C9B,CAAC;AAEF,eAAO,MAAM,wBAAwB,YAC1B,OAAO,WACP,OAAO,WACP,MAAM,KACd,cAAiE,CAAC"}

package/lib/orchestrator/lifecycle-hook-execution.js

@@ -0,0 +1,67 @@
+const errorMessage = (error) => error instanceof Error ? error.message : String(error);
+const lifecycleError = (label, phase, error) => new Error(`${label} ${phase} failed: ${errorMessage(error)}`, { cause: error });
+const cleanupLifecycleHooks = async (cleanups) => {
+    const failures = [];
+    for (const { label, cleanup } of [...cleanups].reverse()) {
+        try {
+            await cleanup();
+        }
+        catch (error) {
+            failures.push(lifecycleError(label, "cleanup", error));
+        }
+    }
+    if (failures.length === 1)
+        throw failures[0];
+    if (failures.length > 1) {
+        throw new AggregateError(failures, "multiple lifecycle hook cleanups failed");
+    }
+};
+export const setupLifecycleHooks = async (hooks, scope, invoke, abortSignal) => {
+    const cleanups = [];
+    try {
+        for (const hook of hooks) {
+            abortSignal?.throwIfAborted();
+            const label = `lifecycle hook '${hook.name}' ${scope}`;
+            let cleanup;
+            try {
+                cleanup = await invoke(hook);
+            }
+            catch (error) {
+                throw lifecycleError(label, "setup", error);
+            }
+            if (cleanup !== undefined && typeof cleanup !== "function") {
+                throw new TypeError(`${label} setup returned a non-function cleanup`);
+            }
+            if (cleanup)
+                cleanups.push({ label, cleanup });
+            abortSignal?.throwIfAborted();
+        }
+    }
+    catch (error) {
+        let cleanupError;
+        try {
+            await cleanupLifecycleHooks(cleanups);
+        }
+        catch (cleanupFailure) {
+            cleanupError = cleanupFailure;
+        }
+        const setupError = error instanceof Error &&
+            (/lifecycle hook '.+' (suite|case) setup/.test(error.message) ||
+                /returned a non-function cleanup/.test(error.message))
+            ? error
+            : lifecycleError(`lifecycle hook ${scope}`, "setup", error);
+        if (cleanupError) {
+            throw new AggregateError([setupError, cleanupError], `${scope} lifecycle setup and rollback failed`);
+        }
+        throw setupError;
+    }
+    let cleaned = false;
+    return async () => {
+        if (cleaned)
+            return;
+        cleaned = true;
+        await cleanupLifecycleHooks(cleanups);
+    };
+};
+export const combineLifecycleFailures = (primary, cleanup, message) => new AggregateError([primary, cleanup], message);
+//# sourceMappingURL=lifecycle-hook-execution.js.map

package/lib/orchestrator/lifecycle-hook-execution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lifecycle-hook-execution.js","sourceRoot":"","sources":["../../src/orchestrator/lifecycle-hook-execution.ts"],"names":[],"mappings":"AAOA,MAAM,YAAY,GAAG,CAAC,KAAc,EAAU,EAAE,CAC9C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAEzD,MAAM,cAAc,GAAG,CAAC,KAAa,EAAE,KAA0B,EAAE,KAAc,EAAS,EAAE,CAC1F,IAAI,KAAK,CAAC,GAAG,KAAK,IAAI,KAAK,YAAY,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;AAElF,MAAM,qBAAqB,GAAG,KAAK,EACjC,QAA+C,EAChC,EAAE;IACjB,MAAM,QAAQ,GAAY,EAAE,CAAC;IAC7B,KAAK,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACzD,IAAI,CAAC;YACH,MAAM,OAAO,EAAE,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,cAAc,CAAC,QAAQ,EAAE,yCAAyC,CAAC,CAAC;IAChF,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EACtC,KAAsC,EACtC,KAAuB,EACvB,MAEiF,EACjF,WAAyB,EACM,EAAE;IACjC,MAAM,QAAQ,GAAiC,EAAE,CAAC;IAClD,IAAI,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,WAAW,EAAE,cAAc,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACvD,IAAI,OAAyC,CAAC;YAC9C,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YAC9C,CAAC;YACD,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,KAAK,UAAU,EAAE,CAAC;gBAC3D,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,wCAAwC,CAAC,CAAC;YACxE,CAAC;YACD,IAAI,OAAO;gBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YAC/C,WAAW,EAAE,cAAc,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,YAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,cAAc,EAAE,CAAC;YACxB,YAAY,GAAG,cAAc,CAAC;QAChC,CAAC;QACD,MAAM,UAAU,GACd,KAAK,YAAY,KAAK;YACtB,CAAC,wCAAwC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;gBAC3D,iCAAiC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,cAAc,CAAC,kBAAkB,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAChE,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,IAAI,cAAc,CACtB,CAAC,UAAU,EAAE,YAAY,CAAC,EAC1B,GAAG,KAAK,sCAAsC,CAC/C,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,CAAC;IACnB,CAAC;IAED,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO,KAAK,IAAI,EAAE;QAChB,IAAI,OAAO;YAAE,OAAO;QACpB,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACtC,OAAgB,EAChB,OAAgB,EAChB,OAAe,EACC,EAAE,CAAC,IAAI,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC"}

package/lib/orchestrator/list-runs.d.ts

@@ -0,0 +1,33 @@
+import { type GradingReport, type PersistedCase, type RunEvent, type RunMetadata } from "../core/index.js";
+export type ListedRun = {
+    caseSlug: string;
+    driverName: string;
+    ts: string;
+    runDir: string;
+    exitReason: RunMetadata["exitReason"] | null;
+    driverAgentId: string | null;
+    startedAt: string | null;
+    endedAt: string | null;
+    durationMs: number | null;
+    casePassed: boolean | null;
+    caseScore: number | null;
+    assertionPassed: number;
+    assertionTotal: number;
+};
+export declare const listRuns: (runsRoot: string) => Promise<ListedRun[]>;
+export declare const formatRunsTable: (runs: ListedRun[]) => string;
+export type RunDetail = {
+    runDir: string;
+    case: PersistedCase | null;
+    metadata: RunMetadata | null;
+    grading: GradingReport | null;
+    events: RunEvent[];
+    artifacts: string[];
+};
+export declare const loadRunDetail: (runDir: string, options?: {
+    maxArtifacts?: number;
+    skipArtifactsWalk?: boolean;
+    skipJson?: boolean;
+}) => Promise<RunDetail>;
+export declare const formatRunDetail: (detail: RunDetail) => string;
+//# sourceMappingURL=list-runs.d.ts.map

package/lib/orchestrator/list-runs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"list-runs.d.ts","sourceRoot":"","sources":["../../src/orchestrator/list-runs.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,aAAa,EAGlB,KAAK,aAAa,EAElB,KAAK,QAAQ,EAEb,KAAK,WAAW,EAKjB,MAAM,kBAAkB,CAAC;AAO1B,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,WAAW,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;IAC7C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AA8EF,eAAO,MAAM,QAAQ,aAAoB,MAAM,KAAG,OAAO,CAAC,SAAS,EAAE,CASpE,CAAC;AAEF,eAAO,MAAM,eAAe,SAAU,SAAS,EAAE,KAAG,MAqBnD,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,aAAa,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,WAAW,GAAG,IAAI,CAAC;IAC7B,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB,CAAC;AAiDF,eAAO,MAAM,aAAa,WAChB,MAAM,YACL;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAE,KAClF,OAAO,CAAC,SAAS,CAiBnB,CAAC;AAkHF,eAAO,MAAM,eAAe,WAAY,SAAS,KAAG,MAQnD,CAAC"}