@deksden-com/dd-flow-cli 0.9.0-beta.14 → 0.9.0-beta.141

package/dist/services/run-engine-bindings.js

@@ -86,7 +86,7 @@ function isRunEngineBinding(value) {
         && typeof record.run_id === "string"
         && typeof record.project_root === "string"
         && typeof record.bound_at === "string"
-        && ["run_creation", "legacy_recovery"].includes(String(record.source))
+        && ["run_creation", "legacy_recovery", "fork_upgrade"].includes(String(record.source))
         && typeof record.reason === "string"
         && Boolean(engine && typeof engine === "object" && !Array.isArray(engine)
             && typeof engine.package_name === "string"

package/dist/services/run-fork.js

@@ -0,0 +1,147 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { createContext } from "../runtime/context.js";
+import { AppError } from "../shared/errors.js";
+import { canonicalPath } from "../storage/paths.js";
+import { importGuardPath, rearmCompletedSnapshotImport } from "../storage/import-guard.js";
+import { installCurrentEngine, upgradeForkedRunEngine, prepareForkEngine } from "./engines.js";
+import { restoreEvalRunSnapshot, prepareEvalRunSnapshot, prepareEvalRunRestore, prepareSnapshotOperationalDecision, restoredWorkspaceRoot } from "./eval-snapshots.js";
+export function prepareRunFork(input) {
+    const requestId = requireId(input.requestId, "request-id");
+    const snapshot = canonicalPath(input.snapshot, false);
+    const output = canonicalPath(input.output, false);
+    if (snapshot === output || snapshot.startsWith(`${output}${path.sep}`) || output.startsWith(`${snapshot}${path.sep}`)) {
+        throw new AppError("fork_path_conflict", "Fork output must not contain or be contained by its source snapshot", 2, { snapshot, output });
+    }
+    const manifestFile = path.join(snapshot, "snapshot.json");
+    const intentFile = path.join(output, ".fork-intent.json");
+    const receiptFile = path.join(output, "fork.json");
+    const engineVersion = requireText(input.engineVersion, "engine-version");
+    if (fs.existsSync(receiptFile)) {
+        const receipt = readJson(receiptFile);
+        const prior = receipt?.intent;
+        if (!prior || prior.request_id !== requestId || prior.snapshot !== snapshot || prior.engine_version !== engineVersion || prior.integrity_checksum !== (input.integrityChecksum ?? null)
+            || (prior.operational_decision_sha256 ?? null) !== (input.operationalDecisionFile ? fileHash(input.operationalDecisionFile) : null))
+            throw new AppError("fork_request_conflict", "Fork output already belongs to another request", 1, { output });
+        return { receipt, output };
+    }
+    prepareForkEngine(engineVersion, input.integrityChecksum);
+    const manifest = prepareEvalRunSnapshot(snapshot);
+    if (manifest.schema_id !== "dd-flow/eval-run-snapshot@5" || manifest.purpose !== "stage_entry" || typeof manifest.stage_entry !== "string" || typeof manifest.run_id !== "string") {
+        throw new AppError("fork_checkpoint_invalid", "Fork requires a sealed stage-entry snapshot", 2, { snapshot });
+    }
+    const operational = prepareSnapshotOperationalDecision({ snapshot, runId: manifest.run_id, projectRoot: path.join(output, "project"), now: new Date().toISOString(), ...(input.operationalDecisionFile ? { file: input.operationalDecisionFile } : {}) });
+    const intent = {
+        schema_id: "dd-flow/run-fork-intent@1",
+        request_id: requestId,
+        snapshot,
+        snapshot_sha256: fileHash(manifestFile),
+        engine_version: engineVersion,
+        integrity_checksum: input.integrityChecksum ?? null,
+        ...(operational ? { operational_decision_sha256: crypto.createHash("sha256").update(operational.bytes).digest("hex") } : {}),
+        target_stage: manifest.stage_entry
+    };
+    assertForkOutput(output, intentFile, intent);
+    return { manifest, snapshot, output, intent, intentFile, receiptFile, requestId };
+}
+function assertForkOutput(output, intentFile, intent) {
+    if (fs.existsSync(output) && !fs.statSync(output).isDirectory())
+        throw new AppError("fork_output_not_empty", "Fork output must be a directory", 2, { output });
+    if (fs.existsSync(output) && fs.readdirSync(output).length > 0) {
+        if (!fs.existsSync(intentFile) || hash(readJson(intentFile).intent) !== hash(intent))
+            throw new AppError("fork_output_not_empty", "Fork output must be empty", 2, { output });
+        const guardName = path.basename(importGuardPath(path.join(output, "dd-flow-home")));
+        const unknown = fs.readdirSync(output).filter(name => ![".fork-intent.json", "project", "dd-flow-home", guardName].includes(name));
+        if (unknown.length)
+            throw new AppError("fork_retry_inspect_required", "Fork output contains data not owned by the interrupted fork", 1, { output, unknown });
+    }
+}
+export async function forkRunFromSnapshot(context, input, prepared = prepareRunFork(input)) {
+    if ("receipt" in prepared)
+        return prepared.receipt;
+    const { manifest, snapshot, output, intent, intentFile, receiptFile, requestId } = prepared;
+    if (input.operationalDecisionFile && fileHash(input.operationalDecisionFile) !== intent.operational_decision_sha256)
+        throw new AppError("operational_decision_changed", "Accepted fork decision changed after preparation", 2);
+    assertForkOutput(output, intentFile, intent);
+    const projectRoot = path.join(output, "project");
+    const home = path.join(output, "dd-flow-home");
+    const prearmedImportId = rearmCompletedSnapshotImport({ home, projectRoot,
+        workspaceRoot: restoredWorkspaceRoot(manifest, projectRoot, home), sourceManifestSha256: intent.snapshot_sha256 });
+    for (const name of ["project", "dd-flow-home"])
+        fs.rmSync(path.join(output, name), { recursive: true, force: true });
+    fs.mkdirSync(output, { recursive: true, mode: 0o700 });
+    writeJsonAtomic(intentFile, { schema_id: "dd-flow/run-fork-intent@1", intent });
+    fs.mkdirSync(projectRoot, { recursive: true, mode: 0o700 });
+    const targetContext = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_ENGINE_MODE: "1" }, "initialize", prearmedImportId ?? undefined);
+    const restoreInput = { snapshot, projectRoot, ...(input.operationalDecisionFile ? { operationalDecisionFile: input.operationalDecisionFile } : {}) };
+    const restored = restoreEvalRunSnapshot(targetContext, restoreInput, prepareEvalRunRestore(targetContext, restoreInput, manifest), prearmedImportId ?? undefined);
+    const restoredContext = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_ENGINE_MODE: "1" });
+    try {
+        const runHome = String(restored.run_home);
+        const previousBinding = readJson(path.join(runHome, "engine-binding.json"));
+        const lineage = {
+            schema_id: "dd-flow/run-fork-lineage@1",
+            request_id: requestId,
+            source_snapshot: snapshot,
+            source_snapshot_sha256: intent.snapshot_sha256,
+            source_run_id: manifest.run_id,
+            target_stage: manifest.stage_entry,
+            inherited_engine: previousBinding,
+            created_at: restoredContext.now()
+        };
+        writeJsonAtomic(path.join(runHome, "fork-lineage.json"), lineage);
+        // Install the immutable artifact belonging to the currently executing
+        // CLI into the new home. Its exact version/checksum were checked before
+        // any interrupted-copy cleanup; snapshots exclude installed engines.
+        installCurrentEngine(restoredContext, { forRouting: true });
+        const binding = await upgradeForkedRunEngine(restoredContext, {
+            projectRoot,
+            runId: String(restored.run_id),
+            engineVersion: intent.engine_version,
+            ...(intent.integrity_checksum ? { integrityChecksum: intent.integrity_checksum } : {}),
+            reason: `Fork ${requestId} from sealed checkpoint ${intent.snapshot_sha256}.`
+        });
+        const receipt = {
+            ok: true,
+            schema_id: "dd-flow/run-fork@1",
+            status: "ready",
+            intent,
+            fork: {
+                project_root: projectRoot,
+                dd_flow_home: home,
+                run_id: restored.run_id,
+                run_home: runHome,
+                workspace_root: restored.workspace_root,
+                target_stage: manifest.stage_entry,
+                engine: binding.engine,
+                lineage_file: path.join(runHome, "fork-lineage.json")
+            },
+            next_command: `dd-flow stage start ${restored.run_id} --stage ${manifest.stage_entry} --project-root ${JSON.stringify(projectRoot)} --json`
+        };
+        writeJsonAtomic(receiptFile, receipt);
+        return receipt;
+    }
+    finally {
+        restoredContext.db.close?.();
+    }
+}
+function readJson(file) {
+    try {
+        return JSON.parse(fs.readFileSync(file, "utf8"));
+    }
+    catch (error) {
+        throw new AppError("fork_checkpoint_invalid", "Fork metadata is not valid JSON", 2, { file, cause: String(error) });
+    }
+}
+function writeJsonAtomic(file, value) {
+    const temporary = `${file}.tmp-${process.pid}-${crypto.randomUUID()}`;
+    fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
+    fs.renameSync(temporary, file);
+}
+function hash(value) { return crypto.createHash("sha256").update(JSON.stringify(value)).digest("hex"); }
+function fileHash(file) { return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); }
+function requireText(value, name) { const text = value.trim(); if (!text)
+    throw new AppError("usage", `Missing required value: ${name}`, 2); return text; }
+function requireId(value, name) { const text = requireText(value, name); if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(text))
+    throw new AppError("usage", `Invalid ${name}`, 2); return text; }

package/dist/services/run-health.js

@@ -0,0 +1,149 @@
+import { AppError } from "../shared/errors.js";
+import { managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, managedProcessObservation, processTreeIsAlive } from "./managed-processes.js";
+import { reconcileStoredFlowRun } from "./runs.js";
+import { recoveryGuard } from "./run-recovery.js";
+import { spawnSync } from "node:child_process";
+function physicalInventory() {
+    const reply = spawnSync('ps', ['-axo', 'pid=,pgid=,stat='], { encoding: 'utf8', timeout: 5000 });
+    if (reply.status !== 0)
+        return null;
+    return reply.stdout.trim().split('\n').map(line => line.trim().split(/\s+/)).filter(parts => parts.length === 3).map(parts => ({ pid: Number(parts[0]), group: Number(parts[1]), zombie: parts[2].startsWith('Z') }));
+}
+const terminalRun = (status) => ["done", "failed", "cancelled", "discarded"].includes(status);
+const waiting = new Set(["waiting_for_user", "waiting_for_context", "waiting_for_capacity", "waiting_for_children", "waiting_for_merge", "stop_target_reached", "paused", "control_requested"]);
+const table = (context, name) => Boolean(context.db.get("SELECT 1 FROM sqlite_schema WHERE name = ?", [name]));
+const latest = (values) => values.filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value))).sort((a, b) => Date.parse(b) - Date.parse(a))[0] ?? null;
+/** Evidence only: lease expiry and quiet providers never prove an exit. */
+export function assessRunHealth(input) {
+    const phase = input.phase ?? null;
+    let state = "unknown", reason = "owner_identity_missing";
+    if (terminalRun(input.status)) {
+        state = "settled";
+        reason = "saved_terminal_result";
+    }
+    else if (input.childrenAlive && input.owner === "absent") {
+        state = "owner_lost";
+        reason = "native_children_still_active";
+    }
+    else if (phase === 'recovery_required' || phase === 'capture_failed') {
+        state = input.owner === 'absent' ? 'owner_lost' : 'unknown';
+        reason = phase;
+    }
+    else if (phase && waiting.has(phase) || ["waiting_for_user", "blocked"].includes(input.status)) {
+        state = "waiting";
+        reason = phase ?? input.status;
+    }
+    else if (input.owner === "absent") {
+        state = "owner_lost";
+        reason = "physical_owner_retired";
+    }
+    else if (input.owner === "alive") {
+        const age = input.lastProgress ? Date.parse(input.now) - Date.parse(input.lastProgress) : NaN;
+        state = Number.isFinite(age) && age > input.noProgressMs ? "no_progress" : "active";
+        reason = state === "no_progress" ? "progress_overdue" : "physical_owner_observed";
+    }
+    else if (input.status === "paused") {
+        state = "waiting";
+        reason = "saved_pause";
+    }
+    return { schema_id: "dd-flow/run-health@1", state, reason, phase, owner_state: input.owner, last_progress_at: input.lastProgress ?? null, last_heartbeat_at: input.heartbeat ?? null, observed_at: input.now, controller_id: input.controllerId ?? null, generation: input.generation ?? null, no_progress_ms: input.noProgressMs };
+}
+/** No dispatch, cancellation, process signals or provider calls. All changes
+ * are fenced by the retained RUN revision, controller and process binding. */
+export function reconcileFlowRunHealth(context, input = {}) {
+    const noProgressMs = input.noProgressMs ?? 30 * 60_000;
+    if (!Number.isSafeInteger(noProgressMs) || noProgressMs < 1_000)
+        throw new AppError("usage", "no-progress-ms must be an integer of at least 1000", 2);
+    const runs = context.db.all(`SELECT * FROM runs${input.runId ? " WHERE (id = ? OR short_id = ?)" : ""}`, input.runId ? [input.runId, input.runId] : []).filter(run => !input.projectRoot || run.project_root === input.projectRoot);
+    if (input.runId && runs.length !== 1)
+        throw new AppError("not_found", "RUN scope is missing or ambiguous", 1);
+    const processes = physicalInventory();
+    const alive = (record) => {
+        if (!record.pid)
+            return !['stopped', 'failed'].includes(record.state);
+        const physical = processes?.find(item => item.pid === record.pid);
+        if (processes && (!physical || physical.zombie)) {
+            const metadata = JSON.parse(record.metadata_json);
+            // A retired group leader may still own physical children. Merely ending
+            // the controller does not settle those children or remote provider work.
+            return Boolean(metadata.process_group_id && processes.some(item => item.group === metadata.process_group_id && !item.zombie));
+        }
+        return !record.pid_started_at || processTreeIsAlive(record);
+    };
+    return { ok: true, results: runs.map(run => {
+            const index = JSON.parse(run.index_json);
+            const controllers = table(context, "run_controllers") ? context.db.all("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? ORDER BY generation DESC, created_at DESC", [run.project_id, run.id]) : [];
+            const controller = controllers[0];
+            const manifest = controller ? JSON.parse(controller.manifest_json) : null;
+            const resourceContext = manifest?.resource_home ? { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } } : context;
+            let records = [], inventoryComplete = false;
+            try {
+                records = managedProcessStatus(resourceContext).filter(record => record.project_id === run.project_id && record.run_id === run.id && managedProcessRuntimeHome(record, context) === context.ddFlowHome);
+                inventoryComplete = true;
+            }
+            catch { /* Missing registry is uncertainty, never retirement proof. */ }
+            const physical = controller && records.find(record => record.id === controller.process_id && record.lease_token === controller.process_lease_token && record.owner_id === controller.controller_id && record.kind === "run-controller");
+            const owner = physical?.pid && physical.pid_started_at ? processes && !processes.some(item => item.pid === physical.pid && !item.zombie) && !alive(physical) ? 'absent' : managedProcessObservation(physical) : "unknown";
+            const retireController = Boolean(controller && owner === 'absent' && !['completed', 'cancelled', 'superseded', 'stop_target_reached', 'recovery_required', 'capture_failed', 'waiting_for_user', 'waiting_for_context', 'control_requested', 'paused'].includes(controller.status));
+            const phase = retireController ? 'recovery_required' : controller?.status ?? null;
+            const childrenAlive = records.some(record => record.id !== physical?.id && alive(record)) || Boolean(physical && owner === 'absent' && alive(physical));
+            const progressEvent = controller && table(context, "run_controller_events") ? context.db.get("SELECT MAX(created_at) AS at FROM run_controller_events WHERE controller_id = ? AND event_type NOT IN ('health_reconciled','heartbeat')", [controller.controller_id])?.at : null;
+            const workProgress = table(context, "works") ? context.db.get("SELECT MAX(at) AS at FROM (SELECT created_at AS at FROM works WHERE project_id=? AND run_id=? UNION ALL SELECT started_at FROM works WHERE project_id=? AND run_id=? UNION ALL SELECT completed_at FROM works WHERE project_id=? AND run_id=?)", [run.project_id, run.id, run.project_id, run.id, run.project_id, run.id])?.at : null;
+            let hookProgress = null;
+            if (table(context, 'hook_events') && table(context, 'sessions'))
+                try {
+                    hookProgress = context.db.get("SELECT MAX(h.created_at) AS at FROM hook_events h JOIN sessions s ON s.session_id=h.session_id AND s.project_id=h.project_id WHERE s.project_id=? AND s.run_id=? AND h.event_name IN ('PostToolUse','UserPromptSubmit','SubagentStop','Stop','SessionStart')", [run.project_id, run.id])?.at ?? null;
+                }
+                catch { /* Legacy hooks have no certified Run membership. */ }
+            const lastProgress = latest([run.created_at, progressEvent, workProgress, hookProgress, ...index.stage_runs.flatMap(stage => [stage.started_at, stage.completed_at])]);
+            const policy = input.noProgressMs ?? (phase === 'starting' ? 120_000 : phase === 'capturing' ? 300_000 : 30 * 60_000);
+            const health = assessRunHealth({ status: run.status, phase, owner, childrenAlive, lastProgress, heartbeat: physical?.updated_at ?? null, now: context.now(), noProgressMs: policy, controllerId: controller?.controller_id ?? null, generation: controller?.generation ?? null });
+            if (terminalRun(run.status) && (childrenAlive || owner === "alive")) {
+                health.state = "unknown";
+                health.reason = "terminal_result_has_live_runtime";
+            }
+            const sessionState = controller ? JSON.parse(controller.state_json) : null;
+            const nativeSettled = Boolean(sessionState?.sessions.length && sessionState.sessions.every(session => session.stopped === true));
+            const outcomeUnknown = controller && table(context, "run_controller_operations") && context.db.get("SELECT 1 FROM run_controller_operations WHERE controller_id = ? AND generation = ? AND status IN ('accepted','outcome_unknown') LIMIT 1", [controller.controller_id, controller.generation]);
+            let transition;
+            const guard = table(context, 'run_recovery_guards') ? recoveryGuard(context, run.project_id, run.id) : null;
+            const currentGeneration = !guard || !controller || controller.generation === guard.generation;
+            if (!terminalRun(run.status) && currentGeneration) {
+                if (controller && ["recovery_required", "capture_failed"].includes(controller.status))
+                    transition = "paused";
+                else if (controller?.status === "stop_target_reached" && owner !== "alive" && nativeSettled && inventoryComplete && !childrenAlive)
+                    transition = "paused";
+                else if (health.state === "owner_lost" && owner === "absent") {
+                    transition = controller?.last_error_json && nativeSettled && inventoryComplete && !childrenAlive && !outcomeUnknown && !guard ? "failed" : "paused";
+                }
+            }
+            const nextStatus = transition ?? run.status;
+            if (transition === 'failed') {
+                health.state = 'settled';
+                health.reason = 'saved_terminal_result';
+            }
+            const unchanged = index.health && JSON.stringify({ ...index.health, observed_at: null }) === JSON.stringify({ ...health, observed_at: null }) && nextStatus === run.status;
+            let changed = false;
+            if (input.apply && !unchanged) {
+                // Verify the same physical inventory immediately before publishing. A
+                // replacement/renewal invalidates this observation, even with the same PID.
+                const binding = (items) => JSON.stringify(items.map(item => ({ ...managedProcessBinding(item, manifest?.resource_home ?? context.ddFlowHome), state: item.state, updated_at: item.updated_at, lease_expires_at: item.lease_expires_at })));
+                if (inventoryComplete) {
+                    const fresh = managedProcessStatus(resourceContext).filter(record => record.project_id === run.project_id && record.run_id === run.id && managedProcessRuntimeHome(record, context) === context.ddFlowHome);
+                    if (binding(fresh) !== binding(records) || physical && owner === "absent" && managedProcessObservation(physical) !== 'absent')
+                        throw new AppError("run_health_changed", "Runtime ownership changed during reconciliation", 1);
+                }
+                context.db.writeTransaction(() => {
+                    const current = controller && context.db.get("SELECT * FROM run_controllers WHERE controller_id = ?", [controller.controller_id]);
+                    if (controller && JSON.stringify(current) !== JSON.stringify(controller))
+                        throw new AppError("run_health_changed", "Controller changed during reconciliation", 1);
+                    reconcileStoredFlowRun(context, run, health, transition, inventoryComplete && owner !== "alive" && !childrenAlive && nativeSettled);
+                    if (controller && retireController) {
+                        context.db.run("UPDATE run_controllers SET status = 'recovery_required', updated_at = ? WHERE controller_id = ? AND owner_token = ? AND generation = ?", [context.now(), controller.controller_id, controller.owner_token, controller.generation]);
+                    }
+                    changed = true;
+                });
+            }
+            return { project_id: run.project_id, run_id: run.id, previous_status: run.status, status: nextStatus, health, changed, would_change: !unchanged };
+        }) };
+}

package/dist/services/run-observations.js

@@ -0,0 +1,152 @@
+import fs from "node:fs";
+import path from "node:path";
+import { ToolObservations, replayToolJournal } from "../harness-runtime/lib/tool-observations.mjs";
+import { harnessKey } from "./harness-config.js";
+/** Evidence has ancestry, not executable ownership. Never open a source home. */
+export function runObservations(context, projectId, runId) {
+    const home = fs.realpathSync(context.ddFlowHome), sources = new Map(), reasons = [];
+    const sessions = context.db.all("SELECT session_id, provider_session_id, harness, transcript_path FROM sessions WHERE project_id = ? AND run_id = ?", [projectId, runId]);
+    const inside = (root, file) => { const p = path.relative(root, file); return p !== "" && p !== ".." && !p.startsWith(`..${path.sep}`) && !path.isAbsolute(p); };
+    const add = (id, nativeId, journal, origin, provenance, harness) => {
+        if (sources.has(id))
+            return;
+        const candidates = sessions.filter(s => (s.provider_session_id ?? s.session_id) === nativeId && (!harness || s.harness === harness));
+        if (candidates.length > 1) {
+            reasons.push("journal_session_ambiguous");
+            return;
+        }
+        const owner = candidates[0];
+        if (!owner)
+            return;
+        const source = { id, provenance, session: { harness_id: owner.harness, session_id: nativeId }, path: null, status: "unavailable" };
+        sources.set(id, source);
+        if (typeof journal !== "string" || !path.isAbsolute(journal) || !inside(origin, journal)) {
+            source.reason = "journal_outside_owned_home";
+            return;
+        }
+        source.path = path.relative(origin, journal);
+        const file = path.resolve(home, source.path);
+        try {
+            if (!inside(home, fs.realpathSync(file))) {
+                source.reason = "journal_outside_owned_home";
+                return;
+            }
+            if (!fs.statSync(file).isFile()) {
+                source.reason = "journal_not_file";
+                return;
+            }
+            source.status = "available";
+        }
+        catch {
+            source.reason = "journal_missing";
+        }
+    };
+    const controllers = (rows, origin, provenance) => {
+        for (const row of rows) {
+            if (row.project_id !== projectId || row.run_id !== runId)
+                continue;
+            try {
+                for (const session of JSON.parse(row.state_json).sessions ?? [])
+                    add(`${row.controller_id}:${session.id}`, session.id, session.journal, origin, provenance, session.profile?.harness ? harnessKey(session.profile.harness) : undefined);
+            }
+            catch {
+                reasons.push("controller_evidence_invalid");
+            }
+        }
+    };
+    if (context.db.all("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'run_controllers'").length) {
+        controllers(context.db.all("SELECT controller_id, project_id, run_id, state_json FROM run_controllers WHERE project_id = ? AND run_id = ?", [projectId, runId]), home, "current");
+    }
+    const imports = path.join(home, "snapshot-imports");
+    if (fs.existsSync(imports))
+        for (const file of fs.readdirSync(imports).filter(name => name.endsWith(".json")).sort()) {
+            try {
+                const location = path.join(imports, file);
+                if (!inside(home, fs.realpathSync(location))) {
+                    reasons.push("lineage_outside_owned_home");
+                    continue;
+                }
+                const archive = JSON.parse(fs.readFileSync(location, "utf8"));
+                if (archive.schema_id !== "dd-flow/snapshot-runtime-lineage@1" || archive.run_id !== runId)
+                    continue;
+                if (typeof archive.source_runtime !== "string" || !path.isAbsolute(archive.source_runtime)) {
+                    reasons.push("lineage_source_invalid");
+                    continue;
+                }
+                controllers(archive.run_tables?.run_controllers ?? [], archive.source_runtime, "inherited");
+            }
+            catch {
+                reasons.push("lineage_evidence_invalid");
+            }
+        }
+    const reducer = new ToolObservations();
+    for (const reason of reasons)
+        reducer.gaps.add(reason);
+    const expectedCodexIds = sessions.filter(session => session.harness === "codex-desktop").map(session => session.provider_session_id ?? session.session_id);
+    const replay = (source, sessionId) => {
+        const file = path.join(home, source.path);
+        let fd = null;
+        try {
+            const actual = fs.realpathSync(file);
+            if (!inside(home, actual))
+                throw new Error("journal escaped owned home");
+            fd = fs.openSync(actual, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK);
+            if (!fs.fstatSync(fd).isFile())
+                throw new Error("journal is not a regular file");
+            replayToolJournal(file, { harness: source.session.harness_id, rootSessionId: sessionId, text: fs.readFileSync(fd, "utf8"), ...(source.session.harness_id === "codex-desktop" ? { expectedSessionIds: expectedCodexIds } : {}) }, reducer);
+        }
+        catch {
+            source.status = "unavailable";
+            source.reason = "journal_unavailable_during_read";
+        }
+        finally {
+            if (fd !== null)
+                fs.closeSync(fd);
+        }
+    };
+    for (const source of sources.values()) {
+        if (source.status !== "available")
+            continue;
+        replay(source, source.session.session_id);
+    }
+    const covered = () => new Set(reducer.summary().sessions.map(item => JSON.stringify(item.session)));
+    // A managed root journal can contain all of its children's native events.
+    // Select transcript fallback only after replay reveals an actual gap.
+    for (const session of sessions) {
+        const nativeId = session.provider_session_id ?? session.session_id;
+        const sessionSummary = reducer.summary(nativeId);
+        if (!session.transcript_path || covered().has(JSON.stringify({ harness_id: session.harness, session_id: nativeId }))
+            && sessionSummary.completeness === "complete" && sessionSummary.outcome_completeness === "complete")
+            continue;
+        const id = `transcript:${session.harness}:${nativeId}`;
+        add(id, nativeId, session.transcript_path, home, "current", session.harness);
+        const source = sources.get(id);
+        if (source?.status === "available")
+            replay(source, nativeId);
+    }
+    for (const source of sources.values())
+        if (source.status !== "available"
+            && !covered().has(JSON.stringify(source.session)))
+            reducer.gaps.add(source.reason ?? "journal_unavailable");
+    const expected = [...new Map(sessions.map(s => { const session = { harness_id: s.harness, session_id: s.provider_session_id ?? s.session_id }; return [JSON.stringify(session), session]; })).values()];
+    const expectedIds = new Set(expected.map(s => JSON.stringify(s)));
+    for (const [key, event] of reducer.calls)
+        if (!expectedIds.has(JSON.stringify(event.session))) {
+            reducer.calls.delete(key);
+        }
+    for (const [key, event] of reducer.turns)
+        if (!expectedIds.has(JSON.stringify(event.session))) {
+            reducer.turns.delete(key);
+        }
+    for (const [key, event] of reducer.sources)
+        if (!expectedIds.has(JSON.stringify(event.session)))
+            reducer.sources.delete(key);
+    const summary = reducer.summary();
+    const observed = new Set(summary.sessions.map(s => JSON.stringify(s.session)));
+    const missing = expected.filter(s => !observed.has(JSON.stringify(s)));
+    const completeness = missing.length || summary.completeness !== "complete" || summary.outcome_completeness !== "complete" ? "partial" : "complete";
+    return { schema_id: "dd-flow/run-observations@1", home, sources: [...sources.values()], tools: { ...summary,
+            completeness, status: observed.size === 0 ? "unavailable" : completeness === "complete" ? "measured" : "partial",
+            observed_sessions: expected.length - missing.length, expected_sessions: expected.length, unavailable_sessions: missing.map(s => s.session_id), missing_sessions: missing,
+            reasons: [...new Set([...summary.reasons, ...(missing.length ? ["sessions_without_tool_observations"] : [])])] } };
+}

package/dist/services/run-projection.js

@@ -1,8 +1,13 @@
-import fs from "node:fs";
 import path from "node:path";
 import { reconcileSessionCoverageRows } from "../domain/session-coverage.js";
 import { publicSessionIdentity } from "./session-identity.js";
+import { publishRunProjections } from "./runs.js";
 export function refreshRunSessionProjection(context, projectId, runId) {
+    // Review children can finish concurrently in separate processes. Reserve the
+    // writer before reading the RUN index so a sibling cannot invalidate it.
+    context.db.writeTransaction(() => refreshRunSessionProjectionLocked(context, projectId, runId), "run-projection.refresh");
+}
+function refreshRunSessionProjectionLocked(context, projectId, runId) {
     const run = context.db.get("SELECT id, flow_kind, status, project_root, subject_type, subject_id, runtime_path, run_index_path, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
     if (!run)
         return;
@@ -27,7 +32,7 @@ export function refreshRunSessionProjection(context, projectId, runId) {
         missing: coverage.missing_unit_ids,
         diagnostics: coverage.diagnostics
     };
-    const plan = run.subject_type === "protocol" ? planProjection(context, projectId, run.subject_id, run.project_root) : {};
+    const plan = run.subject_type === "protocol" ? planProjection(context, projectId, run.subject_id, run.project_root, index) : {};
     const workers = workerProjection(context, projectId, runId);
     const vnext = run.flow_kind === "vnext_specify" || run.flow_kind === "vnext_protocolize";
     const hadLegacyCoverage = vnext && ("session_coverage" in index || "usage_coverage" in index);
@@ -42,8 +47,8 @@ export function refreshRunSessionProjection(context, projectId, runId) {
         && JSON.stringify(index.works ?? []) === JSON.stringify(works)
         && index.root_work_id === rootWorkId
         && (vnext || JSON.stringify(index.session_coverage) === JSON.stringify(sessionCoverage))
-        && JSON.stringify(index.plan_ref) === JSON.stringify(plan.plan_ref)
-        && JSON.stringify(index.plan_progress) === JSON.stringify(plan.plan_progress)
+        && (!plan.plan_ref || JSON.stringify(index.plan_ref) === JSON.stringify(plan.plan_ref))
+        && (!plan.plan_progress || JSON.stringify(index.plan_progress) === JSON.stringify(plan.plan_progress))
         && (vnext || JSON.stringify(index.workers ?? {}) === JSON.stringify(workers))
         && !hadLegacyCoverage)
         return;
@@ -59,44 +64,23 @@ export function refreshRunSessionProjection(context, projectId, runId) {
     if (!vnext)
         index.workers = workers;
     index.updated_at = context.now();
-    const runtime = readJson(run.runtime_path);
-    const runtimeRevision = typeof runtime?.runtime_revision === "number"
-        ? runtime.runtime_revision + 1
-        : typeof index.runtime_revision === "number"
-            ? index.runtime_revision + 1
-            : 1;
-    index.runtime_revision = runtimeRevision;
-    const authoritative = { ...(runtime ?? {}), ...index, schema_id: String(index.schema_id ?? runtime?.schema_id ?? "dd-flow/flow-run@3") };
-    authoritative.sessions = sessions;
-    authoritative.works = works;
-    authoritative.root_work_id = rootWorkId;
-    if (vnext) {
-        delete authoritative.session_coverage;
-        delete authoritative.usage_coverage;
-        delete authoritative.workers;
-        delete authoritative.current_stage;
-        delete authoritative.attempts;
-    }
-    else
-        authoritative.session_coverage = sessionCoverage;
-    if (plan.plan_ref)
-        authoritative.plan_ref = plan.plan_ref;
-    if (plan.plan_progress)
-        authoritative.plan_progress = plan.plan_progress;
-    if (!vnext)
-        authoritative.workers = workers;
-    authoritative.updated_at = index.updated_at;
-    authoritative.runtime_revision = runtimeRevision;
-    if (run.status !== "discarded" || fs.existsSync(run.runtime_path))
-        writeJson(run.runtime_path, authoritative);
-    context.db.run("UPDATE runs SET index_json = ?, updated_at = ? WHERE project_id = ? AND id = ?", [JSON.stringify(index), index.updated_at, projectId, runId]);
+    index.runtime_revision = typeof index.runtime_revision === "number" ? index.runtime_revision + 1 : 1;
+    const updated = context.db.run("UPDATE runs SET index_json = ?, updated_at = ? WHERE project_id = ? AND id = ? AND index_json = ?", [JSON.stringify(index), index.updated_at, projectId, runId, run.index_json]);
+    if (updated.changes !== 1)
+        throw new Error("RUN projection changed during preparation");
+    context.db.afterCommit(() => publishRunProjections(context, projectId, runId));
 }
-function planProjection(context, projectId, protocolId, projectRoot) {
-    const binding = context.db.get("SELECT plan_path, plan_revision, plan_sha256 FROM plan_bindings WHERE project_id = ? AND protocol_id = ?", [projectId, protocolId]);
+function planProjection(context, projectId, protocolId, projectRoot, index) {
+    const binding = context.db.get("SELECT plan_id, plan_path, plan_revision, plan_sha256 FROM plan_bindings WHERE project_id = ? AND protocol_id = ?", [projectId, protocolId]);
     if (!binding)
         return {};
     const planPath = path.relative(projectRoot, binding.plan_path).split(path.sep).join("/");
-    const planId = readPlanId(binding.plan_path);
+    const priorRef = index.plan_ref && typeof index.plan_ref === "object" && !Array.isArray(index.plan_ref) ? index.plan_ref : {};
+    const planId = binding.plan_id ?? (typeof priorRef.plan_id === "string" ? priorRef.plan_id : null);
+    // A pre-migration binding without a RUN plan_ref needs canonical rebind before
+    // this projection can be updated; never invent a plan identifier.
+    if (!planId)
+        return {};
     const progress = Object.fromEntries(context.db.all("SELECT item_id, status, summary, evidence_json FROM plan_progress WHERE project_id = ? AND protocol_id = ? ORDER BY item_id", [projectId, protocolId]).map((row) => [row.item_id, {
             status: row.status,
             ...(row.summary ? { summary: row.summary } : {}),
@@ -112,18 +96,6 @@ function planProjection(context, projectId, protocolId, projectRoot) {
         plan_progress: progress
     };
 }
-function readPlanId(planPath) {
-    try {
-        const value = JSON.parse(fs.readFileSync(planPath, "utf8"));
-        if (value && typeof value === "object" && !Array.isArray(value) && typeof value.plan_id === "string") {
-            return value.plan_id;
-        }
-    }
-    catch {
-        // The canonical validator owns malformed-plan errors; projection stays bounded.
-    }
-    return "unknown-plan";
-}
 function workerProjection(context, projectId, runId) {
     return Object.fromEntries(context.db.all(`SELECT j.job_id, j.plan_item_id, j.status, j.worker_session_id, s.harness, s.provider_session_id
        FROM flow_jobs j LEFT JOIN sessions s ON s.project_id = j.project_id AND s.session_id = j.worker_session_id
@@ -182,17 +154,3 @@ function parseCoverageUnits(value) {
         return [];
     }
 }
-function readJson(file) {
-    try {
-        return fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : undefined;
-    }
-    catch {
-        return undefined;
-    }
-}
-function writeJson(file, value) {
-    fs.mkdirSync(path.dirname(file), { recursive: true });
-    const temporary = `${file}.tmp-${process.pid}-${Date.now()}`;
-    fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`);
-    fs.renameSync(temporary, file);
-}