@deksden-com/dd-flow-cli 0.9.0-beta.15 → 0.9.0-beta.150

package/dist/services/controller-fanout.js

@@ -0,0 +1,226 @@
+import path from "node:path";
+import { AppError, errorRecord } from "../shared/errors.js";
+import { normalizeNativeChildren, safeNativeFailure } from "../harness-runtime/lib/native-children.mjs";
+import { getVnextFanoutStatus, observeVnextNativeChildren, reconcileVnextFanout, getStageFanoutCapacity, verifiedNativeWorkFailure } from "./vnext-fanout.js";
+import { dispatchVnextPlanReview } from "./vnext-plan-review.js";
+import { resolveExecutionPolicy } from "./execution-policy.js";
+import { lifecycleCommandCompletionInstruction, renderDelegationInstructions, stageLifecycleInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
+import { assertWorkLaunchReady, workStartCommand } from "./work-registry.js";
+import { prepareMemoryNativeReviewCopy } from "./memory-native-review-copy.js";
+import { registerMemoryRepairBaselineFromStage, reserveMemoryRepairWriterCopyFromStage, materializeMemoryRepairWriterCopyFromStage, prepareMemoryRepairDispatchFromStage, registerMemorySecondRepairBaselineFromStage, reserveMemorySecondRepairCopyFromStage, materializeMemorySecondRepairCopyFromStage, prepareMemorySecondRepairDispatchFromStage, registerMemoryRepairCurrentViewFromStage, registerMemorySecondRepairCurrentViewFromStage } from "./memory-review-handler.js";
+import { requireProjectByRoot } from "./projects.js";
+import { stageAttempt, stageWorkIds } from "./stage-work-graph.js";
+import { runtimeCommand } from "./runtime-command.js";
+import { promptDeliveryInstruction } from "./prompt-delivery.js";
+import { currentStageFinishCommand } from "./lifecycle-invocations.js";
+function semanticCompletionPrompt(stage, completed, command) {
+    const task = stage === "plan-review"
+        ? "Classify the reviewer findings, apply every accepted correction in the current PLAN-REVIEW Work, and write the authoritative review decision."
+        : stage === "code-review"
+            ? "Classify the review findings and write or retain the authoritative review decision."
+            : "Integrate the accepted child results and update the Stage's authoritative verification evidence.";
+    return [
+        `All declared ${stage} child Work has settled (${completed} completed).`,
+        task,
+        `Then invoke this exact currently issued Stage finish command:\n\n${command}\n\nRefresh the authoritative verification evidence first; an earlier rejected or successful nonterminal command only reads its retained outcome.`,
+        "If that command returns either outcome = repair_required with a repair Work ID, or error.details.continuation.kind = repair_required, the repair Work is already registered. End this Turn so the controller can dispatch it. Otherwise follow the command's actual result.",
+        "Do not create speculative repair Work or start a successor Stage."
+    ].join(" ");
+}
+/** Stage entry acknowledges the packet before the controller issues child commands. */
+export function controllerStageEntryPrompt(stage, command) {
+    return [
+        `Start the assigned ${stage} Stage now. Your first tool call must be this exact standalone command:`,
+        command,
+        promptDeliveryInstruction,
+        "If the returned packet has orchestration.kind = work_fanout, this Turn is only Stage entry: acknowledge the packet and end your Turn immediately. The packet describes the whole Stage, but its graph/dispatch/finish instructions belong to later controller continuations. Do not list, start, launch or finish Work, create children, change files, run checks, or finish the Stage in this entry Turn. The controller will inspect the graph and send the next assignment with exact issued commands (or dispatch external workers itself).",
+        "Otherwise perform this Stage's semantic work and stop at its boundary or a declared HITL boundary. Do not start a successor Stage.",
+        stageLifecycleInstruction
+    ].join("\n\n");
+}
+export function controllerStageResumePrompt(packet, resumedPromptPath) {
+    return [
+        `The accepted canonical answer for pause ${packet.pause_id} (${packet.stage}, ${packet.work_id}) is retained at this absolute read-only path: ${JSON.stringify(packet.answer_file)} (sha256 ${packet.answer_sha256}). Do not replace it with a placeholder or ask the user to supply it again. A null pause.answer_path before resume does not mean the answer is missing.`,
+        resumedPromptPath
+            ? `This exact answer was already committed. Do not invoke stage resume again. Read the recovered authoritative continuation at ${JSON.stringify(resumedPromptPath)} and perform its current Stage instructions.`
+            : `Only while this exact pause still awaits its answer, your first tool call must be this exact standalone command. It applies the already accepted bound answer file:\n\n${packet.command}`,
+        ...(packet.resume_source_path ? [`If this pause has already resumed, do not invoke stage resume again. Read the complete recovered result.worker_prompt_markdown in ${JSON.stringify(packet.resume_source_path)} instead, and follow that authoritative continuation. The controller restores this committed packet before dispatch. A different pause requires its own accepted answer.`] : []),
+        `Do not alter the answer. ${promptDeliveryInstruction} Follow the returned packet, then stop at this Stage boundary.`,
+        stageLifecycleInstruction
+    ].join("\n\n");
+}
+/** The shared controller owns this former eval-only Work-graph decision. */
+export async function nextControllerFanout(context, input) {
+    // Tree settlement is not a Work result. Keep these children visible to the
+    // association check, especially when a hook rejected work start before CLI.
+    const children = input.children;
+    const observations = { harness_id: input.harness, parent_session_id: input.sessionId, children };
+    if (children.length) {
+        const observed = observeVnextNativeChildren(context, { ...input, observations });
+        const issues = observed.issues;
+        if (issues.length)
+            throw new AppError("fanout_reconciliation_required", "Native children have unresolved lifecycle ownership", 1, { issues });
+    }
+    let fanout = getVnextFanoutStatus(context, input).orchestration;
+    if (!fanout) {
+        if (input.dispatchFence)
+            throw new AppError("fanout_reconciliation_required", "Observed native failure has no current graph; dispatch stays fenced", 1, { stage: input.stage, run_id: input.runId });
+        return { kind: "none" };
+    }
+    if (fanout.state === "compatibility_required")
+        throw new AppError("stage_graph_compatibility_required", "The retained graph cannot be dispatched by this controller; use the RUN's pinned engine", 1, { stage: input.stage, run_id: input.runId, graph: fanout });
+    if (children.length) {
+        const reconciled = await reconcileVnextFanout(context, { ...input, observations });
+        const issues = reconciled.issues.filter(issue => !["child_check_in_progress", "child_capacity_pending"].includes(issue.code));
+        if (issues.length)
+            throw new AppError("fanout_reconciliation_required", "Native child outcome requires reconciliation, not replay", 1, { issues });
+        if (children.some(child => child.status === "running" || child.status === "unknown"))
+            return { kind: "wait", reason: "native_children_unsettled" };
+        fanout = getVnextFanoutStatus(context, input).orchestration;
+    }
+    if (input.dispatchFence && !fanout.works.failed)
+        throw new AppError("fanout_reconciliation_required", "Observed native failure has no proven failed Work; dispatch stays fenced", 1, { stage: input.stage, run_id: input.runId });
+    if (fanout.state === "dispatch_required") {
+        dispatchVnextPlanReview(context, input);
+        fanout = getVnextFanoutStatus(context, input).orchestration;
+        if (fanout.state === "dispatch_required" || fanout.state === "capacity_required")
+            return { kind: "wait", reason: "qualified_capacity_required" };
+    }
+    const prompt = (action, text) => ({ kind: "prompt", action, prompt: action === "dispatch_work" ? text : `${text}\n\n${lifecycleCommandCompletionInstruction}`, fingerprint: JSON.stringify({ stage: input.stage, parent: fanout.parent_work_id, state: fanout.state, graph: fanout.graph_revision }) });
+    if (fanout.state === "coordinator_required")
+        return prompt("materialize_graph", `Continue the current ${input.stage} Stage using its authoritative packet. Materialize only its required Work graph, declaring real dependencies at registration. If the packet's CLI command registers the graph, use its returned Work graph unchanged: do not add or alter dependencies after registration. Accepted predecessor Stages already fence their completed producers; do not add cross-stage edges merely to restate input provenance. Do not launch children, finish the Stage, or start a successor. Return after registering the graph; the runtime will dispatch its ready Work.`);
+    if (fanout.works.failed || fanout.works.cancelled) {
+        const projectId = requireProjectByRoot(context, input.projectRoot).id;
+        const members = stageWorkIds(context, projectId, input.runId, input.stage);
+        const failures = context.db.all("SELECT work_id,status,result FROM works WHERE project_id = ? AND run_id = ? AND parent_work_id = ? AND status IN ('failed','cancelled')", [projectId, input.runId, fanout.parent_work_id]).filter(work => members.has(work.work_id)).map(work => {
+            let receipt = {};
+            try {
+                const value = work.result ? JSON.parse(work.result) : null;
+                if (value && typeof value === "object" && !Array.isArray(value))
+                    receipt = value;
+            }
+            catch { /* Missing legacy metadata is unknown. */ }
+            const providerError = safeNativeFailure(receipt.provider_error);
+            const nativeBinding = verifiedNativeWorkFailure(context, { projectRoot: input.projectRoot, runId: input.runId, stage: input.stage, parentSessionId: input.sessionId, parentWorkId: fanout.parent_work_id, workId: work.work_id, receipt });
+            return { work_id: work.work_id, status: work.status, ...(providerError ? { provider_error: providerError } : {}),
+                ...(nativeBinding ? { native_binding: nativeBinding } : {}),
+                ...Object.fromEntries(["terminal_turn_id", "terminal_observed_at"].filter(key => typeof receipt[key] === "string" && receipt[key].length <= 256).map(key => [key, receipt[key]])) };
+        });
+        throw new AppError("fanout_child_failed", "A child Work failed or was cancelled; no retry authority has been issued", 1, { stage: input.stage, run_id: input.runId, parent_work_id: fanout.parent_work_id, failed: fanout.works.failed, cancelled: fanout.works.cancelled, failures, effect: "unknown" });
+    }
+    if (fanout.works.ready.length) {
+        const choices = fanout.works.ready.map(work => {
+            const payload = context.db.get("SELECT payload_json FROM works WHERE work_id = ?", [work.work_id])?.payload_json;
+            const override = payload ? JSON.parse(payload).execution_override : undefined;
+            return { work, resolved: resolveExecutionPolicy({ ddFlowHome: context.ddFlowHome, policy: input.policy, profiles: input.profiles, stage: input.stage, ...(override ? { work: override } : {}) }) };
+        });
+        const prepareRepair = (work) => {
+            const row = context.db.get("SELECT payload_json,result_schema FROM works WHERE work_id=?", [work.work_id]);
+            const payload = JSON.parse(row?.payload_json ?? "null");
+            if (payload?.kind !== "memory-repair" || row?.result_schema !== "dd-flow/memory-repair-work-result@1")
+                return null;
+            const scope = { projectRoot: input.projectRoot, runId: input.runId, workId: work.work_id };
+            const second = payload.memory_repair?.wave === 2;
+            if (!second && payload.memory_repair?.wave !== 1)
+                throw new AppError("fanout_contract_invalid", "Memory repair wave is invalid", 1);
+            const registerBaseline = second ? registerMemorySecondRepairBaselineFromStage : registerMemoryRepairBaselineFromStage;
+            try {
+                registerBaseline(context, scope);
+            }
+            catch (error) {
+                const missing = error instanceof AppError && (second
+                    ? error.code === "memory_second_repair_source_invalid" && error.details.reason === "registered_second_current_view_required"
+                    : error.code === "memory_review_handler_invalid" && error.details.reason === "current_view_registration_required");
+                if (!missing)
+                    throw error;
+                (second ? registerMemorySecondRepairCurrentViewFromStage : registerMemoryRepairCurrentViewFromStage)(context, scope);
+                registerBaseline(context, scope);
+            }
+            (second ? reserveMemorySecondRepairCopyFromStage : reserveMemoryRepairWriterCopyFromStage)(context, scope);
+            (second ? materializeMemorySecondRepairCopyFromStage : materializeMemoryRepairWriterCopyFromStage)(context, scope);
+            return (second ? prepareMemorySecondRepairDispatchFromStage : prepareMemoryRepairDispatchFromStage)(context, scope);
+        };
+        const external = choices.filter(choice => choice.resolved.delegation.mode === "external");
+        if (external.length) {
+            const maximum = Math.min(...external.map(choice => choice.resolved.delegation.max_parallel ?? 1));
+            const selected = external.slice(0, maximum).map(choice => choice.work);
+            for (const work of selected)
+                prepareRepair(work);
+            return { kind: "external", works: selected };
+        }
+        const capacity = getStageFanoutCapacity(context, { ...input, nativeOnly: true }).available_slots;
+        if (typeof capacity !== "number" || !Number.isInteger(capacity) || capacity < 1)
+            return { kind: "wait", reason: "qualified_capacity_required" };
+        const wave = choices.slice(0, capacity).map(choice => {
+            const work = choice.work;
+            const row = context.db.get("SELECT payload_json,result_schema FROM works WHERE work_id=?", [work.work_id]);
+            const payload = JSON.parse(row?.payload_json ?? "null");
+            if (payload?.kind === "memory-review" && row?.result_schema === "dd-flow/memory-review-work-result@1") {
+                const copy = prepareMemoryNativeReviewCopy(context, work.work_id, true);
+                return { ...work, start_command: `cd '${copy.root.replaceAll("'", "'\\''")}' && ${workStartCommand(context, assertWorkLaunchReady(context, work.work_id))}` };
+            }
+            const packet = prepareRepair(work);
+            const command = workStartCommand(context, assertWorkLaunchReady(context, work.work_id));
+            if (!packet)
+                return { ...work, start_command: command };
+            // Ordinary literal shell grammar, not an invented native spawn CWD field.
+            const quotedRoot = "'" + packet.copy_root.replaceAll("'", "'\\''") + "'";
+            return { ...work, start_command: `cd ${quotedRoot} && ${command}` };
+        });
+        for (const work of wave)
+            if (!work.work_id || !work.start_command)
+                throw new AppError("fanout_contract_invalid", "Ready Work omits its trusted start command", 1);
+        const totalWorks = fanout.works.created + fanout.works.running + fanout.works.completed + fanout.works.failed + fanout.works.cancelled;
+        return prompt("dispatch_work", renderDelegationInstructions({ harness: input.harness, stage: input.stage, capacity, works: wave, totalWorks, remainingWorks: fanout.works.created - wave.length }));
+    }
+    if (fanout.works.running)
+        return { kind: "wait", reason: "work_receipts_pending" };
+    if (fanout.works.created)
+        throw new AppError("fanout_graph_stalled", "Created Work has no ready or running dependency path", 1);
+    if (input.stage === "review") {
+        const projectId = requireProjectByRoot(context, input.projectRoot).id;
+        const run = context.db.get("SELECT flow_kind,run_root,run_dir FROM runs WHERE project_id=? AND id=?", [projectId, input.runId]);
+        if (run?.flow_kind === "mb-init" || run?.flow_kind === "mb-upgrade") {
+            const attempt = stageAttempt(context, projectId, input.runId, "review");
+            const repair = context.db.get(`SELECT CASE event_type WHEN 'memory.second_repair_registration' THEN 2 ELSE 1 END AS wave
+        FROM audit_events WHERE project_id=? AND event_type IN ('memory.repair_registration','memory.second_repair_registration')
+        AND json_extract(payload_json,'$.run_id')=? AND json_extract(payload_json,'$.scope.attempt')=?
+        AND json_extract(payload_json,'$.scope.parent_work_id')=? ORDER BY wave DESC LIMIT 1`, [projectId, input.runId, attempt, fanout.parent_work_id]);
+            if (repair && !context.db.get(`SELECT 1 FROM audit_events WHERE project_id=? AND event_type='memory.corrected_review_registration'
+        AND json_extract(payload_json,'$.run_id')=? AND json_extract(payload_json,'$.scope.attempt')=?
+        AND json_extract(payload_json,'$.scope.parent_work_id')=? AND json_extract(payload_json,'$.wave')=?`, [projectId, input.runId, attempt, fanout.parent_work_id, repair.wave])) {
+                const args = ["--run", input.runId, "--project-root", input.projectRoot, "--wave", String(repair.wave), "--json"];
+                return prompt("semantic_completion", `Correction wave ${repair.wave} Work has settled. Before Stage finish, run ${runtimeCommand(context, ["memory", "corrections", "checks", ...args])}, then ${runtimeCommand(context, ["memory", "review", "selective-register", ...args])}. Read both complete responses, then end this Turn so the controller can dispatch the independent selective reviewers. Do not finish the Stage or treat repair Work completion as independent acceptance.`);
+            }
+            const wave = repair?.wave ?? 0;
+            const runHome = run.run_root ?? run.run_dir;
+            if (!runHome || !path.isAbsolute(runHome))
+                throw new AppError("runtime_missing", "Memory review collection requires its retained absolute RUN home", 1);
+            const collectionArgs = ["--run", input.runId, "--project-root", input.projectRoot, ...(wave ? ["--wave", String(wave)] : []),
+                "--json", "--progress-jsonl", "--response-file", path.join(runHome, `memory-review-collection-wave-${wave}.json`)];
+            const collect = runtimeCommand(context, ["memory", "review", wave ? "selective-collect" : "collect", ...collectionArgs]);
+            return prompt("semantic_completion", [
+                `All declared memory review Work has settled. First run ${collect}. Read the complete canonical collection from its response_file; do not decide acceptance from a truncated preview or reviewer counts.`,
+                "Only if the collection outcome is accepted (projection.outcome = accepted), integrate its accepted results into the original Stage proposal and invoke the exact Stage finish command from the authoritative packet.",
+                wave < 2
+                    ? `If the outcome is needs_repair, use the returned canonical findings and correction_plan_binding to author substantive repair tasks with exact owned write_paths. Register that plan with the packet's issued memory corrections register command for wave ${wave + 1}, then end this Turn so the controller can dispatch it. Do not invoke Stage finish before corrections, checks and independent selective review complete.`
+                    : "If the outcome is needs_repair or blocked, the two correction waves are exhausted. Use the issued pause command for the concrete unresolved blocker; do not mark it accepted.",
+                "For blocked collection, use the issued pause command. Do not calculate finding hashes, merge or drop findings, edit project files in the coordinator, create speculative Work or start a successor Stage."
+            ].join("\n\n"));
+        }
+    }
+    const finish = currentStageFinishCommand(context, input.stage);
+    if (!finish)
+        throw new AppError("controller_finish_authority_missing", "Settled child Work has no current issued Stage finish; reconcile lifecycle authority", 1, { stage: input.stage });
+    return prompt("semantic_completion", semanticCompletionPrompt(input.stage, fanout.works.completed, finish));
+}
+/** Shared pinned runtime normalization; preserve structured errors at the TS boundary. */
+export function controllerNativeChildren(receipt, rootSessionId) {
+    try {
+        return normalizeNativeChildren(receipt, rootSessionId);
+    }
+    catch (error) {
+        const record = errorRecord(error);
+        throw Object.assign(new AppError(record.code, record.message, record.exitCode, record.details), record);
+    }
+}

package/dist/services/controller-memory-task.js

@@ -0,0 +1,45 @@
+import { Worker } from "node:worker_threads";
+import { AppError } from "../shared/errors.js";
+/** Keep synchronous custody/copy work off the controller's lease event loop.
+ * The thread remains inside its registered process; OS-level stop owns both.
+ * Do not resolve until exit, including after cancellation or a typed failure. */
+export function runControllerMemoryTask(context, task, signal) {
+    const aborted = () => new AppError("controller_memory_task_aborted", "Memory operation was cancelled; reconcile its retained effects", 1, { effect: "unknown" });
+    if (signal?.aborted)
+        return Promise.reject(aborted());
+    return new Promise((resolve, reject) => {
+        // Both source tests and installed runtimes use the package's compiled worker.
+        const worker = new Worker(new URL("../../dist/services/controller-memory-worker.js", import.meta.url), {
+            workerData: { task, env: { ...context.env, DD_FLOW_HOME: context.ddFlowHome } }
+        });
+        let reply, failure;
+        const abort = () => worker.postMessage({ abort: true });
+        signal?.addEventListener("abort", abort, { once: true });
+        if (signal?.aborted)
+            abort();
+        worker.on("message", (value) => {
+            const invalid = value && typeof value === "object" && ("error" in value
+                ? !value.error || typeof value.error.code !== "string" || typeof value.error.message !== "string" || !Number.isInteger(value.error.exitCode)
+                    || !value.error.details || typeof value.error.details !== "object" || Array.isArray(value.error.details) || "result" in value
+                : !("result" in value) || !value.result || typeof value.result !== "object" || Array.isArray(value.result));
+            if (reply || !value || typeof value !== "object" || invalid) {
+                failure = new AppError("controller_memory_task_invalid", "Memory worker returned an invalid terminal reply", 1, { effect: "unknown" });
+            }
+            else
+                reply = value;
+        });
+        worker.once("error", error => { failure = error; });
+        worker.once("exit", code => {
+            signal?.removeEventListener("abort", abort);
+            if (failure)
+                return reject(failure);
+            if (reply && "error" in reply)
+                return reject(new AppError(reply.error.code, reply.error.message, reply.error.exitCode, reply.error.details));
+            if (signal?.aborted)
+                return reject(aborted());
+            if (code !== 0 || !reply || !("result" in reply))
+                return reject(new AppError("controller_memory_task_failed", "Memory worker exited without its terminal result", 1, { exit_code: code, effect: "unknown" }));
+            resolve(reply.result);
+        });
+    });
+}

package/dist/services/controller-memory-worker.js

@@ -0,0 +1,49 @@
+import { parentPort, workerData } from "node:worker_threads";
+import { createContext } from "../runtime/context.js";
+import { AppError, errorRecord } from "../shared/errors.js";
+import { nextControllerFanout } from "./controller-fanout.js";
+import { launchExternalWork } from "./external-work-launch.js";
+import { requireProjectByRoot } from "./projects.js";
+import { assertControllerLeaseAllowed } from "./run-controller-process.js";
+import { assertManagedProcessLease } from "./managed-processes.js";
+import { isDiagnosticFlow } from "../domain/diagnostic-flows.js";
+if (!parentPort)
+    throw new AppError("controller_memory_task_invalid", "Memory worker requires its controller thread", 1);
+const port = parentPort, cancellation = new AbortController();
+port.on("message", message => { if (message?.abort === true)
+    cancellation.abort(); });
+const { task, env } = workerData;
+let context;
+let reply;
+try {
+    context = createContext(env, "prepared_existing");
+    const project = requireProjectByRoot(context, task.input.projectRoot);
+    const run = context.db.get("SELECT flow_kind FROM runs WHERE project_id=? AND id=?", [project.id, task.input.runId]);
+    if (!run || !["mb-init", "mb-upgrade"].includes(run.flow_kind) && !isDiagnosticFlow(run.flow_kind))
+        throw new AppError("controller_memory_task_invalid", "Worker requires the selected memory or diagnostic RUN", 1, { effect: "no_effect" });
+    if (task.operation === "external") {
+        const work = context.db.get("SELECT project_id,run_id FROM works WHERE work_id=?", [task.input.workId]);
+        if (!work || work.project_id !== project.id || work.run_id !== task.input.runId)
+            throw new AppError("controller_memory_task_invalid", "External Work belongs to another RUN", 1, { effect: "no_effect" });
+    }
+    const owner = env.DD_FLOW_CONTROLLER_TOKEN ? context.db.get("SELECT * FROM run_controllers WHERE project_id=? AND run_id=? AND owner_token=?", [project.id, task.input.runId, env.DD_FLOW_CONTROLLER_TOKEN]) : undefined;
+    if (!owner)
+        throw new AppError("controller_owner_mismatch", "Memory task lacks its managed controller owner", 1, { effect: "no_effect" });
+    assertControllerLeaseAllowed(context, owner);
+    assertManagedProcessLease(context, { id: owner.process_id, leaseToken: owner.process_lease_token, pid: process.pid });
+    cancellation.signal.throwIfAborted();
+    if (task.operation === "fanout")
+        reply = { result: await nextControllerFanout(context, task.input) };
+    else if (task.operation === "external")
+        reply = { result: await launchExternalWork(context, { ...task.input, signal: cancellation.signal }) };
+    else
+        throw new AppError("controller_memory_task_invalid", "Unknown memory worker operation", 1, { effect: "no_effect" });
+}
+catch (error) {
+    reply = { error: errorRecord(error) };
+}
+finally {
+    context?.db.close?.();
+}
+port.postMessage(reply);
+port.close();