@mcoda/codali 0.1.134 → 0.1.135-codali.4

package/dist/eval/LongWorkflowJourney.js

@@ -0,0 +1,243 @@
+import { createHash } from "node:crypto";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { SessionController } from "../runtime/SessionController.js";
+import { loadOrchestrationConfig } from "../config/OrchestrationConfig.js";
+import { captureWorkspace, changedWorkspacePaths } from "../runtime/WorkspaceSnapshot.js";
+import { runProcess } from "../tools/shell/AsyncProcess.js";
+import { gradeTerminalTask, materializeTerminalTask, terminalClientBuildHash } from "./TerminalCorpusRunner.js";
+import { terminalWorkflowCorpus, terminalWorkflowCorpusHash } from "./TerminalWorkflowCorpus.js";
+import { runEvaluationProcess } from "./EvaluationProcess.js";
+export const LONG_WORKFLOW_VERSION = "long-workflows-v2";
+export const longWorkflowScenarios = [
+    { id: "data-delivery", objective: "Deliver a reliable data-processing toolkit with exact money totals, validated imports, CSV export and additive record migration.", modules: ["invoice-cents", "jsonl-import", "csv-export", "additive-record-migration"] },
+    { id: "service-adapters", objective: "Deliver a service-adapter toolkit with safe headers, bounded async execution, locale fallback and interval reservations.", modules: ["header-normalization", "async-worker-pool", "locale-fallback", "interval-reservation"] },
+    { id: "billing-audit", objective: "Deliver a billing-readiness package: source-backed duplicate-submit and tenant-isolation reports, exact invoice totals and a data-preserving migration.", modules: ["investigate-duplicate-submit", "investigate-tenant-key", "invoice-cents", "additive-record-migration"] },
+    { id: "localized-export", objective: "Deliver localized export adapters with CSV correctness, normalized headers and a source-backed fictional render-service upgrade report.", modules: ["locale-fallback", "csv-export", "header-normalization", "research-pinned-upgrade"] },
+];
+const hash = (value) => createHash("sha256").update(value).digest("hex");
+export const longWorkflowHash = () => hash(JSON.stringify({ version: LONG_WORKFLOW_VERSION, scenarios: longWorkflowScenarios, prompts: longWorkflowScenarios.map(longWorkflowPrompt), steering, workflowHash: terminalWorkflowCorpusHash() }));
+const steering = "Keep every original module contract and protected file. Add DELIVERY.md containing a concise test-and-delivery inventory with the exact path modules/<module-id>/test/task.test.mjs for each of the four modules. Do not claim unavailable checks ran, add features, or remove criteria. This is additional required work, not a replacement objective.";
+const prefixPrompt = (module, prompt) => prompt.replace(/\b(src|test|docs|vendor-docs)\//g, `modules/${module}/$1/`);
+export function longWorkflowPrompt(scenario) {
+    const corpus = terminalWorkflowCorpus();
+    return `${scenario.objective}\nThe repository has four independent module packages under modules/. Each module's package.json, user notes, and any paths not explicitly writable below must remain byte-identical. Root package.json and USER_NOTES.txt must also remain unchanged. No dependencies or unrelated features. Paths used to locate files in these instructions are workspace-relative. Inside a module JSON report, all source-path values (including owner, client, server, callers, evidence and citations) must be relative to that module directory, without its modules/<module-id>/ prefix. API endpoint values remain unchanged.\nRecord canonical requirements R2-R5, one per module contract below, plus the existing original R1 for final integration after all module tasks. Record phases and a dependency-valid plan; R1 is final integration and must depend on all module deliverables. Record a modular architecture decision with rationale and alternatives. Module tasks may be independent; no inter-module imports are needed.\n${scenario.modules.map((id, index) => `R${index + 2}: ${prefixPrompt(id, corpus.find(task => task.id === id).prompt)}`).join("\n\n")}\nInitially you are in planning-only mode: inspect and record the plan, but do not execute or complete it. Execution will resume in a new process with workspace permissions. The overall delivery includes actual tests and all four module outcomes. Preserve that objective across restarts and corrections.`;
+}
+const saveNew = (file, value) => fs.writeFile(file, JSON.stringify(value, null, 2) + "\n", { flag: "wx", mode: 0o600 });
+/** One invocation is one OS process in the parent runner. Stage 2 deliberately
+ * checkpoints after a verified task, never during a pending side effect. */
+export async function runLongWorkflowStage(descriptorPath, stage) {
+    if (![1, 2, 3, 4].includes(stage))
+        throw new Error("Unknown journey stage");
+    const descriptor = JSON.parse(await fs.readFile(descriptorPath, "utf8"));
+    const scenario = longWorkflowScenarios.find(item => item.id === descriptor.scenarioId);
+    if (descriptor.schemaVersion !== 1 || !scenario || descriptor.corpusHash !== longWorkflowHash())
+        throw new Error("Incompatible journey descriptor");
+    const resolved = await loadOrchestrationConfig(descriptor.configPath);
+    if (resolved.hash !== descriptor.configHash || await terminalClientBuildHash() !== descriptor.clientBuildHash)
+        throw new Error("Journey configuration/client changed across processes");
+    const root = path.resolve(descriptor.root);
+    const workspaceRoot = path.join(root, "workspace");
+    const storageRoot = path.join(root, "state");
+    const events = [];
+    let checkpointTriggered = false;
+    let controller;
+    controller = new SessionController({ workspaceRoot, storageRoot, resolved, permissions: stage === 1 ? "plan" : "workspace", effort: "max",
+        onEvent: event => {
+            if (["model_route", "tool_receipt", "turn_failed", "turn_result", "usage"].includes(event.type))
+                events.push(event);
+            if (stage === 2 && !checkpointTriggered && event.type === "tool_receipt"
+                && controller.state?.tasks.some(task => task.status === "verified")) {
+                const receipt = event.data.receipt;
+                if (receipt.tool === "work_verify" && receipt.status === "success") {
+                    checkpointTriggered = true;
+                    controller.cancel();
+                }
+            }
+        } });
+    const startedAt = new Date().toISOString();
+    let failure;
+    let resumePreserved = stage === 1;
+    const abort = () => controller.cancel();
+    process.once("SIGTERM", abort);
+    process.once("SIGINT", abort);
+    try {
+        if (stage === 1)
+            await controller.start(descriptor.objective);
+        else {
+            const previous = JSON.parse(await fs.readFile(path.join(root, `stage-${stage - 1}.handoff.json`), "utf8"));
+            await controller.resume(previous.sessionId);
+            resumePreserved = hash(JSON.stringify(controller.state)) === previous.stateHash;
+            if (!resumePreserved)
+                throw new Error("Canonical state changed across the process boundary");
+        }
+        if (stage === 3) {
+            // Simulates a user edit made while the CLI was stopped. Existing bytes
+            // remain protected; the new line must survive subsequent model work.
+            await fs.appendFile(path.join(workspaceRoot, "USER_NOTES.txt"), "Concurrent user correction: preserve these notes and every original module contract.\n");
+            await controller.steer(steering);
+        }
+        if (stage < 4) {
+            try {
+                await controller.run();
+            }
+            catch (error) {
+                if (!checkpointTriggered || !/Cancelled by user/.test(String(error)))
+                    throw error;
+            }
+        }
+        for (let i = 0; i < (stage === 1 ? 2 : 1); i++)
+            await controller.compact();
+    }
+    catch (error) {
+        failure = error instanceof Error ? error.message : String(error);
+    }
+    finally {
+        process.removeListener("SIGTERM", abort);
+        process.removeListener("SIGINT", abort);
+    }
+    const state = controller.state;
+    const record = { schemaVersion: 1, stage, startedAt, endedAt: new Date().toISOString(), pid: process.pid,
+        sessionId: controller.sessionId, failure, resumePreserved, checkpointTriggered, usage: controller.usageSnapshot(), state,
+        routes: events.filter(event => event.type === "model_route").map(event => event.data),
+        receipts: events.filter(event => event.type === "tool_receipt").map(event => event.data.receipt) };
+    await saveNew(path.join(root, `stage-${stage}.result.json`), record);
+    if (state && controller.sessionId)
+        await saveNew(path.join(root, `stage-${stage}.handoff.json`), { sessionId: controller.sessionId, stage, stateHash: hash(JSON.stringify(state)) });
+    if (failure)
+        throw new Error(failure);
+    if (stage === 2 && !checkpointTriggered)
+        throw new Error("Execution did not reach the required verified-task checkpoint");
+}
+export function gradeLongWorkflowState(objective, state, initial) {
+    const failures = [];
+    if (state.originalObjective !== objective)
+        failures.push("Original objective changed");
+    for (const id of ["R1", "R2", "R3", "R4", "R5", "steering-2"]) {
+        if (!state.requirements.some(r => r.id === id && r.required))
+            failures.push(`Required criterion lost: ${id}`);
+        if (!state.tasks.some(task => task.status === "verified" && task.requirementIds.includes(id)))
+            failures.push(`No verified owner: ${id}`);
+    }
+    if (state.status !== "completed")
+        failures.push("Goal is not completed");
+    if (!state.decisions.length)
+        failures.push("Architecture decision missing");
+    if (!state.phases?.length)
+        failures.push("Phased plan missing");
+    if (initial) {
+        for (const criterion of initial.requirements)
+            if (!state.requirements.some(r => r.id === criterion.id && r.description === criterion.description && r.required === criterion.required))
+                failures.push(`Criterion contract changed: ${criterion.id}`);
+        for (const decision of initial.decisions)
+            if (!state.decisions.some(d => JSON.stringify(d) === JSON.stringify(decision)))
+                failures.push(`Architecture history changed: ${decision.id}`);
+    }
+    const finalOwners = state.tasks.filter(task => task.requirementIds.includes("R1"));
+    const reaches = (id, target, seen = new Set()) => {
+        if (seen.has(id))
+            return false;
+        seen.add(id);
+        const task = state.tasks.find(item => item.id === id);
+        if (!task)
+            return false;
+        const phase = state.phases?.find(item => item.id === task.phaseId);
+        const deps = [...task.dependsOn, ...state.tasks.filter(item => item.phaseId && phase?.dependsOn.includes(item.phaseId)).map(item => item.id)];
+        return deps.some(dep => dep === target || reaches(dep, target, seen));
+    };
+    if (finalOwners.length !== 1 || state.tasks.filter(task => task.requirementIds.some(id => ["R2", "R3", "R4", "R5"].includes(id)))
+        .some(task => !reaches(finalOwners[0]?.id ?? "", task.id)))
+        failures.push("Original delivery integration does not depend on every module owner");
+    if (state.amendments.length !== 1 || state.goalRevision !== 2)
+        failures.push("Steering history was lost or unexpectedly replaced");
+    if (state.checkpoints.filter(c => c.note.startsWith("Context compacted")).length < 5)
+        failures.push("Fewer than five recorded compactions");
+    return failures;
+}
+export async function runLongWorkflowJourney(options) {
+    const scenario = longWorkflowScenarios.find(item => item.id === options.scenarioId);
+    if (!scenario)
+        throw new Error("Unknown long workflow scenario");
+    const root = path.resolve(options.outputRoot);
+    const workspaceRoot = path.join(root, "workspace");
+    await fs.mkdir(root, { mode: 0o700 });
+    await fs.mkdir(workspaceRoot, { mode: 0o700 });
+    await fs.mkdir(path.join(workspaceRoot, "modules"));
+    const tasks = scenario.modules.map(id => terminalWorkflowCorpus().find(task => task.id === id));
+    for (const task of tasks)
+        await materializeTerminalTask(task, path.join(workspaceRoot, "modules", task.id));
+    await fs.writeFile(path.join(workspaceRoot, "package.json"), JSON.stringify({ private: true, type: "module", scripts: { test: "node --test" } }) + "\n");
+    const notes = "User-owned journey notes. Keep every original module and delivery requirement.\n";
+    await fs.writeFile(path.join(workspaceRoot, "USER_NOTES.txt"), notes);
+    await runProcess("git", ["-c", "core.hooksPath=/dev/null", "init", "--quiet"], { workspaceRoot, shellTimeoutMs: 10000 });
+    await runProcess("git", ["-c", "core.hooksPath=/dev/null", "add", "--all"], { workspaceRoot, shellTimeoutMs: 10000 });
+    await runProcess("git", ["-c", "core.hooksPath=/dev/null", "-c", "user.name=Codali evaluation", "-c", "user.email=codali-evaluation@localhost", "-c", "commit.gpgSign=false", "commit", "--quiet", "-m", "Long workflow fixture baseline"], { workspaceRoot, shellTimeoutMs: 10000 });
+    const before = await captureWorkspace(workspaceRoot);
+    const resolved = await loadOrchestrationConfig(options.configPath);
+    const clientBuildHash = await terminalClientBuildHash();
+    const descriptor = { schemaVersion: 1, scenarioId: scenario.id, configPath: path.resolve(options.configPath), configHash: resolved.hash, clientBuildHash, root, objective: longWorkflowPrompt(scenario), corpusHash: longWorkflowHash() };
+    const descriptorPath = path.join(root, "descriptor.json");
+    await saveNew(descriptorPath, descriptor);
+    const startedAt = new Date().toISOString();
+    await saveNew(path.join(root, "manifest.json"), { ...descriptor, startedAt, version: LONG_WORKFLOW_VERSION,
+        limitations: ["Four composed module journeys reuse the workflow corpus; these are retention/recovery probes, not fresh coding hold-outs",
+            "A scripted pause after verified work is a clean checkpoint, not a SIGKILL or an uncertain external side effect",
+            "Multi-hour qualification is reported separately from correctness; idle time is never added to meet duration",
+            "No mixed-media, physical GPU admission, production coexistence or statistical comparison claim"] });
+    const failures = [];
+    const stages = [];
+    for (const stage of [1, 2, 3, 4]) {
+        options.signal?.throwIfAborted();
+        options.onProgress?.({ type: "journey_stage_start", scenarioId: scenario.id, stage, at: new Date().toISOString() });
+        try {
+            await runEvaluationProcess({ file: fileURLToPath(import.meta.url), args: ["--stage", descriptorPath, String(stage)], cwd: root,
+                config: resolved.config, timeoutMs: 4 * 3600000, signal: options.signal });
+        }
+        catch (error) {
+            failures.push(`Stage ${stage}: ${String(error).slice(0, 1000)}`);
+        }
+        try {
+            stages.push(JSON.parse(await fs.readFile(path.join(root, `stage-${stage}.result.json`), "utf8")));
+        }
+        catch {
+            failures.push(`Stage ${stage} did not produce a durable result`);
+        }
+        options.onProgress?.({ type: "journey_stage_end", scenarioId: scenario.id, stage, failures: [...failures], at: new Date().toISOString() });
+        if (failures.length)
+            break;
+    }
+    const grades = [];
+    for (const task of tasks)
+        grades.push({ taskId: task.id, grade: await gradeTerminalTask(task, path.join(workspaceRoot, "modules", task.id)) });
+    for (const result of grades)
+        if (!result.grade.passed)
+            failures.push(`Module ${result.taskId}: ${result.grade.failures.join('; ')}`);
+    const final = stages.at(-1)?.state;
+    if (final)
+        failures.push(...gradeLongWorkflowState(descriptor.objective, final, stages[0]?.state));
+    else
+        failures.push("Final work state missing");
+    const after = await captureWorkspace(workspaceRoot);
+    const changed = changedWorkspacePaths(before, after);
+    const allowed = new Set(["USER_NOTES.txt", "DELIVERY.md", ...tasks.flatMap(task => task.allowedChanges.map(file => `modules/${task.id}/${file}`))]);
+    if (changed.some(file => !allowed.has(file)))
+        failures.push("Changes escaped the journey scope");
+    const expectedNotes = notes + "Concurrent user correction: preserve these notes and every original module contract.\n";
+    if (await fs.readFile(path.join(workspaceRoot, "USER_NOTES.txt"), "utf8").catch(() => "") !== expectedNotes)
+        failures.push("Concurrent user notes were lost or changed");
+    const delivery = await fs.readFile(path.join(workspaceRoot, "DELIVERY.md"), "utf8").catch(() => "");
+    if (!scenario.modules.every(id => delivery.includes(`modules/${id}/test/task.test.mjs`)))
+        failures.push("Additional delivery inventory is incomplete");
+    if (new Set(stages.map(item => item.pid)).size !== 4 || stages.length !== 4 || stages.some(item => !item.resumePreserved))
+        failures.push("Four distinct processes did not retain canonical state");
+    const endedAt = new Date().toISOString();
+    const elapsedMs = Date.parse(endedAt) - Date.parse(startedAt);
+    const result = { schemaVersion: 1, scenarioId: scenario.id, startedAt, endedAt, elapsedMs, passed: failures.length === 0,
+        multiHourDurationReached: elapsedMs >= 2 * 3600000, failures, grades, changedPaths: changed, stages, configHash: resolved.hash, clientBuildHash };
+    await saveNew(path.join(root, "result.json"), result);
+    options.onProgress?.({ type: "journey_result", scenarioId: scenario.id, passed: result.passed, multiHourDurationReached: result.multiHourDurationReached, failures, endedAt });
+}
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) && process.argv[2] === "--stage") {
+    runLongWorkflowStage(process.argv[3], Number(process.argv[4])).catch(error => { process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); process.exitCode = 1; });
+}

package/dist/eval/PlanningCorpus.d.ts

@@ -0,0 +1,45 @@
+import type { WorkState } from "../session/WorkTypes.js";
+export declare const PLANNING_CORPUS_VERSION = "codali-planning-v3";
+export interface PlanningCriterion {
+    id: string;
+    description: string;
+    paths: string[];
+    after: string[];
+    phase: "design" | "build" | "validate";
+}
+export interface PlanningCase {
+    id: string;
+    objective: string;
+    criteria: PlanningCriterion[];
+    constraints: string[];
+    nonGoals: string[];
+    decision: {
+        id: string;
+        question: string;
+        options: Record<string, string>;
+        accepted: string;
+        rationale: string;
+    };
+}
+/** Closed-world planning probes, not a general architecture benchmark. Expected
+ * dependency edges and accepted choices stay outside the model's workspace. */
+export declare function planningCorpus(): PlanningCase[];
+export declare function planningPrompt(item: PlanningCase): string;
+export interface PlanningGrade {
+    passed: boolean;
+    failures: string[];
+    metrics: {
+        lostCriteria: number;
+        inventedCriteria: number;
+        orderingErrors: number;
+        scopeErrors: number;
+        architectureErrors: number;
+        prematureCompletion: boolean;
+    };
+}
+export declare function gradePlanningState(item: PlanningCase, state: WorkState): PlanningGrade;
+/** Oracle construction is used only to mutation-test the grader, never supplied
+ * to a live planner. Rationale grading is structural, not semantic certification. */
+export declare function referencePlanningState(item: PlanningCase): WorkState;
+export declare const planningCorpusHash: () => string;
+//# sourceMappingURL=PlanningCorpus.d.ts.map

package/dist/eval/PlanningCorpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PlanningCorpus.d.ts","sourceRoot":"","sources":["../../src/eval/PlanningCorpus.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEzD,eAAO,MAAM,uBAAuB,uBAAuB,CAAC;AAC5D,MAAM,WAAW,iBAAiB;IAAG,EAAE,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,GAAG,UAAU,CAAC;CAAE;AACjJ,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;CAClH;AAOD;+EAC+E;AAC/E,wBAAgB,cAAc,IAAI,YAAY,EAAE,CA0D/C;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,YAAY,GAAG,MAAM,CAEzD;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,kBAAkB,EAAE,MAAM,CAAC;QAAC,mBAAmB,EAAE,OAAO,CAAA;KAAE,CAAC;CACpK;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,GAAG,aAAa,CAuDtF;AAED;qFACqF;AACrF,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,YAAY,GAAG,SAAS,CAapE;AAED,eAAO,MAAM,kBAAkB,QAAO,MAAkI,CAAC"}

package/dist/eval/PlanningCorpus.js

@@ -0,0 +1,189 @@
+import { createHash } from "node:crypto";
+import { createWorkState, replaceWorkPlan, validateWorkState } from "../session/WorkState.js";
+export const PLANNING_CORPUS_VERSION = "codali-planning-v3";
+const criterion = (id, description, paths, after = [], phase = id === "R2" ? "design" : ["R3", "R4", "R5"].includes(id) ? "build" : "validate") => ({
+    id, description: `${description}${after.length ? ` Prerequisites: ${after.join(", ")} must be complete before this work starts.` : ""}`, paths, after, phase,
+});
+const common = { constraints: ["Preserve existing user edits and public compatibility", "Use existing components; add no runtime dependency"], nonGoals: ["Unrelated cleanup or UI redesign"] };
+/** Closed-world planning probes, not a general architecture benchmark. Expected
+ * dependency edges and accepted choices stay outside the model's workspace. */
+export function planningCorpus() {
+    const cases = [
+        { ...common, id: "additive-migration", objective: "Plan a rolling migration from one display-name column to first/last-name fields without breaking old clients or requiring downtime.",
+            criteria: [
+                criterion("R2", "Document old/new client contracts and an additive migration decision before implementation.", ["docs/contracts.md"]),
+                criterion("R3", "Add nullable fields while retaining the existing column; old clients keep working.", ["db/migration.sql"], ["R2"]),
+                criterion("R4", "Implement compatible dual read/write behavior only after the expanded schema exists.", ["src/profile.ts"], ["R3"]),
+                criterion("R5", "Backfill idempotently after dual writes protect new traffic.", ["src/backfill.ts"], ["R4"]),
+                criterion("R6", "Prove old/new clients and repeated backfill coexist before any cutover.", ["test/migration.test.ts"], ["R5"]),
+                criterion("R7", "Document reversible cutover after compatibility verification; retaining the old column is mandatory.", ["docs/rollout.md"], ["R6"]),
+            ], decision: { id: "D1", question: "Choose the storage transition compatible with rolling old clients.", options: { replace: "Drop the old column and ship the new client together", expand: "Expand storage, use compatible readers/writers, backfill, then switch reads" }, accepted: "expand", rationale: "Old clients remain active throughout the migration." } },
+        { ...common, id: "artifact-recovery", objective: "Plan resumable tenant-scoped media downloads, including disconnected clients and expiring URLs, without duplicate generation.",
+            criteria: [
+                criterion("R2", "Define tenant ownership and stable remote job identity before implementing downloads.", ["docs/jobs.md"]),
+                criterion("R3", "Persist admitted job identity before polling; ambiguous submission must be reconciled before any resubmit.", ["src/jobs.ts"], ["R2"]),
+                criterion("R4", "Refresh expired URLs using the same owned job and artifact identity.", ["src/refresh.ts"], ["R3"]),
+                criterion("R5", "Download bounded bytes to private temporary storage; verify checksum before atomic final rename.", ["src/download.ts"], ["R4"]),
+                criterion("R6", "Test interrupted download, expiry and cross-tenant denial after the recovery path exists.", ["test/recovery.test.ts"], ["R5"]),
+                criterion("R7", "Write a resume runbook using the original job; never promise a render was cancelled without a terminal receipt.", ["docs/resume.md"], ["R6"]),
+            ], decision: { id: "D1", question: "Choose how a resumed client obtains an artifact whose URL expired.", options: { rerender: "Submit a replacement render", refresh: "Resolve ownership and refresh the artifact reference on the original job" }, accepted: "refresh", rationale: "URL expiry does not mean generation failed." } },
+        { ...common, id: "embedding-switch", objective: "Plan switching a repository index to a different embedding dimension while keeping search available and rollback possible.",
+            criteria: [
+                criterion("R2", "Record embedding model revision, dimension and normalization as one space identity.", ["docs/embedding.md"]),
+                criterion("R3", "Create a separate index namespace for the new incompatible space; preserve the active index.", ["src/index.ts"], ["R2"]),
+                criterion("R4", "Backfill the new namespace incrementally and record source freshness.", ["src/backfill.ts"], ["R3"]),
+                criterion("R5", "Verify coverage, dimensional compatibility and query behavior before activating the new index.", ["test/index.test.ts"], ["R4"], "validate"),
+                criterion("R6", "Atomically switch the index pointer only after verification.", ["src/switch.ts"], ["R5"]),
+                criterion("R7", "Test reverting the pointer to the preserved compatible old model/index pair.", ["test/rollback.test.ts"], ["R6"]),
+            ], decision: { id: "D1", question: "Choose how to store embeddings from the new dimension.", options: { mixed: "Write new vectors into the active index and pad old vectors", separate: "Build a separately identified index and atomically switch after validation" }, accepted: "separate", rationale: "Different embedding spaces cannot be mixed by padding dimensions." } },
+        { ...common, id: "lease-recovery", objective: "Plan recovery of distributed jobs after worker crashes without accepting stale results or duplicating external side effects.",
+            criteria: [
+                criterion("R2", "Specify attempt identity, fencing and external-operation reconciliation contracts.", ["docs/leases.md"]),
+                criterion("R3", "Persist each new attempt and its monotonic fence before dispatch.", ["src/claims.ts"], ["R2"]),
+                criterion("R4", "Reject stale completion using the current attempt/fence, including after reassignment.", ["src/results.ts"], ["R3"]),
+                criterion("R5", "Reconcile uncertain external effects before allowing retry; lease age alone cannot prove absence.", ["src/recovery.ts"], ["R4"]),
+                criterion("R6", "Test late results, crashed workers and an operation accepted before connection loss.", ["test/leases.test.ts"], ["R5"]),
+                criterion("R7", "Document recovery of blocked uncertain jobs and audit the original attempt.", ["docs/recovery.md"], ["R6"]),
+            ], decision: { id: "D1", question: "A lease expired during a non-idempotent remote operation. Choose the recovery policy.", options: { retry: "Retry immediately because the lease is old", reconcile: "Fence old results and reconcile the remote operation before retry" }, accepted: "reconcile", rationale: "Lease expiry does not prove that an external action did not occur." } },
+        { ...common, id: "connector-policy", objective: "Plan adding third-party tools without allowing repository instructions or tool metadata to widen user permissions.",
+            criteria: [
+                criterion("R2", "Record user authorization as the authority and tool output as untrusted data.", ["docs/trust.md"]),
+                criterion("R3", "Classify operations centrally; unknown tools require explicit permission.", ["src/policy.ts"], ["R2"]),
+                criterion("R4", "Bind grants to concrete operation/scope and recheck current policy immediately before side effects.", ["src/grants.ts"], ["R3"]),
+                criterion("R5", "Enforce workspace and network boundaries on connector execution.", ["src/executor.ts"], ["R4"]),
+                criterion("R6", "Test forged readOnly metadata, prompt injection and grants invalidated by policy changes.", ["test/policy.test.ts"], ["R5"]),
+                criterion("R7", "Document denial and revocation without exposing credentials.", ["docs/permissions.md"], ["R6"]),
+            ], decision: { id: "D1", question: "An unknown generation tool declares readOnly=true. Choose authorization behavior.", options: { trust: "Allow it because the connector labels it read-only", classify: "Apply central operation classification and require the corresponding authorization" }, accepted: "classify", rationale: "Connector metadata cannot grant authorization." } },
+        { ...common, id: "usage-accounting", objective: "Plan durable token reservations and usage reconciliation across retries, cancellation and reconnects without losing cumulative call counts.",
+            criteria: [
+                criterion("R2", "Specify cumulative call counters, reserved tokens, actual usage and unknown usage as distinct concepts.", ["docs/accounting.md"]),
+                criterion("R3", "Persist a unique reservation before each call; admission failure must leave no partial debit.", ["src/reservations.ts"], ["R2"]),
+                criterion("R4", "Settle exactly once against actual tokens without decrementing the cumulative call count.", ["src/settlement.ts"], ["R3"]),
+                criterion("R5", "Reconcile pending reservations after reconnect; cancellation without usage leaves an honest unknown state.", ["src/reconcile.ts"], ["R4"]),
+                criterion("R6", "Test duplicate settlements, rejected admissions, delayed usage and cancellation.", ["test/accounting.test.ts"], ["R5"]),
+                criterion("R7", "Document actual versus estimated/unknown totals and audit retained invocation IDs.", ["docs/usage.md"], ["R6"]),
+            ], decision: { id: "D1", question: "A completed call has settled its tokens. Choose how its call count changes.", options: { refund: "Decrement cumulative calls to release the call reservation", retain: "Keep the cumulative call count; settle only the token reservation" }, accepted: "retain", rationale: "A completed invocation remains a consumed call." } },
+    ];
+    return structuredClone(cases);
+}
+export function planningPrompt(item) {
+    return `${item.objective}\n\nPlanning only: do not implement, run shell commands, verify implementation or claim completion. Build the durable plan using work_refine/work_plan and record D1 through work_decision. Use three phases named design, build and validate, with phase dependencies in that order. Preserve these criterion IDs and descriptions verbatim (R1 is already the whole request):\n${JSON.stringify(item.criteria.map(({ id, description }) => ({ id, description })))}\n\nRecord these constraints and non-goals verbatim:\n${JSON.stringify({ constraints: item.constraints, nonGoals: item.nonGoals })}\n\nUse one narrowly scoped task for each R2-R7 criterion, with exactly its listed future write paths. Add one final R1 integration task with no write paths, dependent on every criterion task. Use each criterion's listed phase. Derive dependency ordering from the explicit prerequisites in its description; task dependencies supplement phase dependencies. Requirements in the same phase still need their specified ordering.\n${JSON.stringify(Object.fromEntries(item.criteria.map(c => [c.id, { paths: c.paths, phase: c.phase }])))}\n\nDecision ${item.decision.id}: ${item.decision.question}\nOptions: ${JSON.stringify(item.decision.options)}\nSet the decision title to the chosen option ID, give a concrete rationale and list the other option IDs as alternatives. Do not invent additional requirements, APIs, files or work. End with a brief planning summary, leaving the implementation goal open.`;
+}
+export function gradePlanningState(item, state) {
+    const failures = [];
+    const metrics = { lostCriteria: 0, inventedCriteria: 0, orderingErrors: 0, scopeErrors: 0, architectureErrors: 0, prematureCompletion: false };
+    const fail = (message) => { failures.push(message); };
+    try {
+        validateWorkState(state);
+    }
+    catch (error) {
+        fail(`Invalid durable plan: ${String(error)}`);
+    }
+    if (state.originalObjective !== planningPrompt(item) || state.objective !== state.originalObjective)
+        fail("Original planning objective changed");
+    const expectedIds = new Set(["R1", ...item.criteria.map(c => c.id)]);
+    if (!state.requirements.some(r => r.id === "R1" && r.required && r.description === planningPrompt(item)))
+        fail("Whole-request criterion R1 changed");
+    metrics.inventedCriteria = state.requirements.filter(r => !expectedIds.has(r.id)).length;
+    if (metrics.inventedCriteria)
+        fail("Invented criteria");
+    for (const criterion of item.criteria) {
+        const actual = state.requirements.find(r => r.id === criterion.id);
+        if (!actual?.required || actual.description !== criterion.description) {
+            metrics.lostCriteria++;
+            fail(`Lost or rewritten criterion ${criterion.id}`);
+        }
+    }
+    for (const key of ["constraints", "nonGoals"])
+        if (JSON.stringify([...state[key]].sort()) !== JSON.stringify([...item[key]].sort()))
+            fail(`${key} changed`);
+    const active = state.tasks.filter(t => !["cancelled", "superseded"].includes(t.status));
+    metrics.prematureCompletion = state.status === "completed" || state.evidence.length > 0 || active.some(t => ["running", "review_pending", "verified"].includes(t.status));
+    if (metrics.prematureCompletion)
+        fail("Planning falsely claimed execution or verification");
+    if (active.length !== item.criteria.length + 1)
+        fail("Expected one task per criterion and one final integration task");
+    const tasks = new Map(active.map(t => [t.id, t]));
+    const reaches = (from, target, seen = new Set()) => {
+        if (seen.has(from))
+            return false;
+        seen.add(from);
+        const task = tasks.get(from);
+        if (!task)
+            return false;
+        const phase = state.phases?.find(p => p.id === task.phaseId);
+        const dependencies = [...task.dependsOn, ...active.filter(t => t.phaseId && phase?.dependsOn.includes(t.phaseId)).map(t => t.id)];
+        return dependencies.includes(target) || dependencies.some(id => reaches(id, target, seen));
+    };
+    const owner = (id) => active.filter(t => t.requirementIds.includes(id));
+    for (const c of item.criteria) {
+        const assigned = owner(c.id);
+        if (assigned.length !== 1 || assigned[0].requirementIds.length !== 1) {
+            fail(`Ambiguous criterion ownership: ${c.id}`);
+            continue;
+        }
+        const task = assigned[0];
+        if (JSON.stringify([...task.allowedPaths].sort()) !== JSON.stringify([...c.paths].sort())) {
+            metrics.scopeErrors++;
+            fail(`Scope mismatch: ${c.id}`);
+        }
+        if (task.phaseId !== c.phase) {
+            metrics.orderingErrors++;
+            fail(`Wrong phase: ${c.id}`);
+        }
+        for (const predecessor of c.after)
+            if (owner(predecessor).length !== 1 || !reaches(task.id, owner(predecessor)[0].id)) {
+                metrics.orderingErrors++;
+                fail(`Missing prerequisite ${predecessor} before ${c.id}`);
+            }
+    }
+    const final = owner("R1");
+    if (final.length !== 1 || final[0].requirementIds.length !== 1)
+        fail("Final integration must uniquely own R1");
+    else {
+        if (final[0].allowedPaths.length) {
+            metrics.scopeErrors++;
+            fail("Final integration must have no write paths");
+        }
+        if (active.some(t => t !== final[0] && !reaches(final[0].id, t.id))) {
+            metrics.orderingErrors++;
+            fail("Final integration must follow all criterion tasks");
+        }
+        // The prompt fixes criterion phases, but permits the final integration task
+        // to be unphased. Its explicit dependencies still fence all future work.
+    }
+    for (const [id, prior] of [["design", undefined], ["build", "design"], ["validate", "build"]]) {
+        const phase = state.phases?.find(p => p.id === id);
+        if (!phase || prior && !phase.dependsOn.includes(prior)) {
+            metrics.orderingErrors++;
+            fail(`Missing ordered phase ${id}`);
+        }
+    }
+    if (state.phases?.length !== 3)
+        fail("Unexpected phases");
+    const decision = state.decisions.find(d => d.id === item.decision.id && !state.decisions.some(next => next.supersedes === d.id));
+    const rejected = Object.keys(item.decision.options).filter(id => id !== item.decision.accepted);
+    if (!decision || decision.title !== item.decision.accepted || decision.rationale.trim().length < 20 || !rejected.every(id => decision.alternatives.includes(id))) {
+        metrics.architectureErrors++;
+        fail("Architecture choice, rationale or alternatives violate the supplied constraints");
+    }
+    return { passed: failures.length === 0, failures, metrics };
+}
+/** Oracle construction is used only to mutation-test the grader, never supplied
+ * to a live planner. Rationale grading is structural, not semantic certification. */
+export function referencePlanningState(item) {
+    const state = createWorkState(planningPrompt(item));
+    state.requirements.push(...item.criteria.map(c => ({ id: c.id, description: c.description, required: true })));
+    state.constraints = [...item.constraints];
+    state.nonGoals = [...item.nonGoals];
+    const tasks = item.criteria.map(c => ({ id: `task-${c.id}`, title: c.description, requirementIds: [c.id], dependsOn: c.after.map(id => `task-${id}`), allowedPaths: [...c.paths], phaseId: c.phase }));
+    tasks.push({ id: "integrate", title: "Verify complete delivery", requirementIds: ["R1"], dependsOn: tasks.map(t => t.id), allowedPaths: [], phaseId: "validate" });
+    replaceWorkPlan(state, tasks, [
+        { id: "design", title: "Architecture", dependsOn: [], exitRequirementIds: ["R2"] },
+        { id: "build", title: "Implementation", dependsOn: ["design"], exitRequirementIds: item.criteria.filter(c => c.phase === "build").map(c => c.id) },
+        { id: "validate", title: "Validation", dependsOn: ["build"], exitRequirementIds: [...item.criteria.filter(c => c.phase === "validate").map(c => c.id), "R1"] },
+    ]);
+    state.decisions.push({ id: item.decision.id, title: item.decision.accepted, rationale: item.decision.rationale, alternatives: Object.keys(item.decision.options).filter(id => id !== item.decision.accepted), paths: [] });
+    return state;
+}
+export const planningCorpusHash = () => createHash("sha256").update(JSON.stringify({ version: PLANNING_CORPUS_VERSION, cases: planningCorpus() })).digest("hex");

package/dist/eval/PlanningCorpusRunner.d.ts

@@ -0,0 +1,13 @@
+import { type EffortLevel } from "../config/OrchestrationConfig.js";
+export interface PlanningRunOptions {
+    configPath: string;
+    outputRoot: string;
+    caseIds?: string[];
+    effort?: EffortLevel;
+    signal?: AbortSignal;
+    onProgress?: (event: Record<string, unknown>) => void;
+}
+/** Uses the real controller, permission gate and configured route. No evaluator
+ * model can turn a failing deterministic grade into a passing one. */
+export declare function runPlanningCorpus(options: PlanningRunOptions): Promise<void>;
+//# sourceMappingURL=PlanningCorpusRunner.d.ts.map

package/dist/eval/PlanningCorpusRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PlanningCorpusRunner.d.ts","sourceRoot":"","sources":["../../src/eval/PlanningCorpusRunner.ts"],"names":[],"mappings":"AAIA,OAAO,EAA2B,KAAK,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAI7F,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvD;AAED;sEACsE;AACtE,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CA0DlF"}

package/dist/eval/PlanningCorpusRunner.js

@@ -0,0 +1,89 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { SessionController } from "../runtime/SessionController.js";
+import { captureWorkspace, changedWorkspacePaths } from "../runtime/WorkspaceSnapshot.js";
+import { loadOrchestrationConfig } from "../config/OrchestrationConfig.js";
+import { terminalClientBuildHash } from "./TerminalCorpusRunner.js";
+import { gradePlanningState, planningCorpus, planningCorpusHash, planningPrompt, PLANNING_CORPUS_VERSION } from "./PlanningCorpus.js";
+/** Uses the real controller, permission gate and configured route. No evaluator
+ * model can turn a failing deterministic grade into a passing one. */
+export async function runPlanningCorpus(options) {
+    const resolved = await loadOrchestrationConfig(options.configPath);
+    const clientBuildHash = await terminalClientBuildHash();
+    const corpus = planningCorpus();
+    if (options.caseIds?.some(id => !corpus.some(item => item.id === id)))
+        throw new Error("Unknown planning case");
+    const selected = options.caseIds ? corpus.filter(item => options.caseIds.includes(item.id)) : corpus;
+    const root = path.resolve(options.outputRoot);
+    await fs.mkdir(root, { mode: 0o700 });
+    const save = (file, value) => fs.writeFile(path.join(root, file), JSON.stringify(value, null, 2) + "\n", { flag: "wx", mode: 0o600 });
+    await save("manifest.json", { schemaVersion: 1, corpusVersion: PLANNING_CORPUS_VERSION, corpusHash: planningCorpusHash(), clientBuildHash,
+        configHash: resolved.hash, effort: options.effort ?? "high", caseIds: selected.map(item => item.id), startedAt: new Date().toISOString(),
+        limitations: ["Six closed-world planning probes with explicit criterion IDs and constrained architecture options; not general architecture certification",
+            "Checks actual durable controller records, not quality of final prose", "Rationale grading checks presence, not semantic correctness",
+            "Docdex is disabled for these code-free probes; retrieval/media/implementation quality require separate qualification",
+            "Compaction/resume is checked in a new controller instance, not a new OS process", "No helper/review benefit or production performance claim"] });
+    for (const item of selected) {
+        options.signal?.throwIfAborted();
+        if ((await loadOrchestrationConfig(options.configPath)).hash !== resolved.hash || await terminalClientBuildHash() !== clientBuildHash)
+            throw new Error("Evaluation config/build changed; start a separately identified run");
+        const directory = path.join(root, item.id);
+        const workspaceRoot = path.join(directory, "workspace");
+        const storageRoot = path.join(directory, "state");
+        await fs.mkdir(directory, { mode: 0o700 });
+        await fs.mkdir(workspaceRoot, { mode: 0o700 });
+        await fs.writeFile(path.join(workspaceRoot, "USER_NOTES.md"), "Uncommitted user notes. Planning must preserve this file.\n", { mode: 0o600, flag: "wx" });
+        const before = await captureWorkspace(workspaceRoot);
+        const events = [];
+        const controller = new SessionController({ workspaceRoot, storageRoot, resolved, docdex: false, permissions: "plan", effort: options.effort ?? "high",
+            onEvent: event => { if (["model_route", "tool_receipt", "turn_result", "turn_failed"].includes(event.type))
+                events.push(event); } });
+        const startedAt = new Date().toISOString();
+        options.onProgress?.({ type: "planning_start", caseId: item.id, startedAt });
+        let failure;
+        const abort = () => controller.cancel();
+        options.signal?.addEventListener("abort", abort, { once: true });
+        try {
+            await controller.start(planningPrompt(item));
+            options.signal?.throwIfAborted();
+            await controller.run();
+            await controller.compact();
+        }
+        catch (error) {
+            failure = error instanceof Error ? error.message : String(error);
+        }
+        finally {
+            options.signal?.removeEventListener("abort", abort);
+        }
+        if (!controller.state || !controller.sessionId)
+            throw new Error(failure ?? "Planner failed before creating state");
+        const grade = gradePlanningState(item, controller.state);
+        const after = await captureWorkspace(workspaceRoot);
+        const changedPaths = changedWorkspacePaths(before, after);
+        if (changedPaths.length)
+            grade.failures.push("Planning changed workspace files");
+        const sideEffects = events.filter(event => event.type === "tool_receipt").map(event => event.data.receipt)
+            .filter(receipt => ["write_file", "run_shell", "work_verify", "work_complete"].includes(receipt.tool));
+        if (sideEffects.length)
+            grade.failures.push("Planner attempted implementation or completion during planning-only work");
+        const resumed = new SessionController({ workspaceRoot, storageRoot, resolved, docdex: false, permissions: "plan" });
+        let resumePreserved = false;
+        try {
+            await resumed.resume(controller.sessionId);
+            resumePreserved = JSON.stringify(resumed.state) === JSON.stringify(controller.state);
+        }
+        catch (error) {
+            grade.failures.push(`Resume failed: ${String(error)}`);
+        }
+        if (!resumePreserved)
+            grade.failures.push("Durable plan changed or was lost across compaction/resume");
+        grade.passed = grade.failures.length === 0 && !failure;
+        const record = { caseId: item.id, startedAt, endedAt: new Date().toISOString(), passed: grade.passed, failure, grade, changedPaths, resumePreserved,
+            configHash: resolved.hash, clientBuildHash, sessionId: controller.sessionId, usage: controller.usageSnapshot(),
+            routes: events.filter(event => event.type === "model_route").map(event => event.data), sideEffects, state: controller.state };
+        await save(`${item.id}.result.json`, record);
+        await fs.appendFile(path.join(root, "results.jsonl"), JSON.stringify(record) + "\n", { mode: 0o600 });
+        options.onProgress?.({ type: "planning_result", caseId: item.id, passed: record.passed, failures: grade.failures, failure, endedAt: record.endedAt });
+    }
+    await save("completed.json", { endedAt: new Date().toISOString(), cases: selected.length });
+}