@deksden-com/dd-flow-cli 0.9.0-beta.136 → 0.9.0-beta.138

package/dist/cli/memory-init.js

@@ -0,0 +1,240 @@
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { setTimeout as delay } from "node:timers/promises";
+import { Writable } from "node:stream";
+import { once } from "node:events";
+import { createContext } from "../runtime/context.js";
+import { AppError, errorRecord } from "../shared/errors.js";
+import { hybridDigest } from "../services/hybrid-contracts.js";
+import { readMemoryInitStartupInput } from "../services/memory-init-startup-custody.js";
+import { readMemoryInitRequestInput } from "../services/memory-init-request-input.js";
+import { prepareMemoryInitFreshRequest, prepareMemoryInitSelection, startMemoryInitRun } from "../services/memory-init-start.js";
+import { PreparedMemoryInitLintCapability } from "../services/memory-init-lint-capability.js";
+import { PreparedMemoryInitWorktrunkCapability } from "../services/memory-init-worktrunk-capability.js";
+import { produceMemoryInitNativeCapacity } from "../services/memory-init-native-capacity.js";
+import { openMemoryInitStores, assertMemoryInitStorePrerequisites } from "../services/memory-init-stores.js";
+import { prepareMemoryInitControllerStart } from "../services/memory-init-controller-start.js";
+import { runControllerStatus } from "../services/run-controller.js";
+import { requireRun } from "../services/run-controller-state.js";
+import { requireProjectByRoot } from "../services/projects.js";
+import { routeArgsThroughEngine } from "../services/engines.js";
+import { parseRuntimeBudget } from "../services/runtime-budget.js";
+import { prepareFileBytes } from "./input-preparation.js";
+import { prepareMemoryInitInvocation, assertMemoryInitInvocationReplay } from "./memory-init-invocation.js";
+import { parseOutputOptions } from "./output-options.js";
+import { prepareResponseDestination, writeResponseFile } from "./response-file.js";
+import { writePreparationError } from "./error-output.js";
+import { onboardExecutor } from "../services/executor-onboarding.js";
+import { readOperatorProfiles, readOperatorProfileInput } from "../services/operator-profiles.js";
+import { resolveDdFlowHome } from "../storage/paths.js";
+import { askExecutorChoice } from "./executor-onboarding.js";
+/** Backpressure bounds the observing client's output; closing it never controls the RUN. */
+export async function writeMemoryInitObservation(stdout, text) {
+    const disconnected = () => Boolean(stdout.destroyed);
+    if (disconnected())
+        return false;
+    const waiting = new AbortController();
+    try {
+        // Observe errors even when write returns true: a short pipe write can
+        // report EPIPE asynchronously. Writable emits callback errors as events.
+        const closed = once(stdout, "close", { signal: waiting.signal });
+        const written = new Promise(resolve => {
+            stdout.write(text, error => { if (!error)
+                resolve(); });
+        });
+        await Promise.race([written, closed]);
+        return !disconnected();
+    }
+    catch (error) {
+        if (error.code === "EPIPE")
+            return false;
+        throw error;
+    }
+    finally {
+        waiting.abort();
+    }
+}
+/** Both public spellings enter this owner. The controller, never this observer,
+ * owns Sessions and productive dispatch; disconnect does not request stop. */
+export async function runMemoryInitCli(args, env = process.env, io = process) {
+    let context;
+    let effects = false;
+    try {
+        const output = parseOutputOptions(args), invocation = prepareMemoryInitInvocation(output.args);
+        parseRuntimeBudget(env.DD_FLOW_RUNTIME_BUDGET);
+        if (output.responseFile)
+            prepareResponseDestination(output.responseFile);
+        const reader = createContext(env, "read_existing");
+        let retained;
+        try {
+            retained = readMemoryInitStartupInput(reader, invocation.scope);
+        }
+        finally {
+            reader.db.close?.();
+        }
+        let prepared;
+        if (retained) {
+            if (retained.payload.schema_id !== "dd-flow/memory-init-startup-input@3")
+                throw new AppError("memory_init_original_request_required", "Public init replay requires independently retained original input", 1, { effect: "no_effect" });
+            const raw = readMemoryInitRequestInput(retained.payload.request_input, invocation.scope, retained.payload.objective);
+            // A supplied file is an explicit selector and must still match. Omitted
+            // files never reopen mutable profile inputs during exact replay.
+            if (invocation.profileFile)
+                invocation.execution.rootProfile = readOperatorProfileInput(invocation.profileFile);
+            assertMemoryInitInvocationReplay(invocation, raw.selection, retained.payload.request_input.intake_sha256);
+            if (invocation.nativeConfigHome) {
+                const reference = retained.payload.execution.snapshot.settings.delegation.native_capacity_receipt;
+                const encoded = retained.payload.execution.receipt_blobs?.find(([checksum]) => checksum === reference?.sha256)?.[1];
+                if (!encoded || JSON.parse(Buffer.from(encoded, "base64").toString("utf8")).input.native_config_home !== invocation.nativeConfigHome)
+                    throw new AppError("memory_init_request_conflict", "Native private configuration selection changed", 1, { effect: "no_effect" });
+            }
+            context = openMemoryInitStores(env, { scope: invocation.scope, selectionSha256: hybridDigest(raw.selection) }).context;
+        }
+        else {
+            if (!invocation.intakeFile || !invocation.intakeSha256 || !invocation.gitPolicy)
+                throw new AppError("usage", "Fresh init requires original intake file/hash and explicit Git route/integration branch", 2, { phase: "prepare", effect: "no_effect" });
+            const raw = prepareFileBytes(invocation.intakeFile, "original init intake", 64 * 1024).bytes;
+            if (createHash("sha256").update(raw).digest("hex") !== invocation.intakeSha256)
+                throw new AppError("memory_init_intake_changed", "Original intake bytes differ from --intake-sha256", 2, { phase: "prepare", effect: "no_effect" });
+            let objective;
+            try {
+                objective = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(raw);
+            }
+            catch {
+                throw new AppError("usage", "Original init intake must be UTF-8", 2, { phase: "prepare", effect: "no_effect" });
+            }
+            // Controller birth/lease confirmation uses this OS observation. Known
+            // inability must refuse before candidate, stores or project publication.
+            const identity = spawnSync("ps", ["-o", "lstart=", "-p", String(process.pid)], { env: { ...env, TZ: "UTC", LC_ALL: "C" }, encoding: "utf8", timeout: 1000, shell: false, windowsHide: true });
+            if (identity.error || identity.signal || identity.status !== 0 || !identity.stdout.trim())
+                throw new AppError("memory_init_runtime_unavailable", "Controller physical identity observation is unavailable", 1, { phase: "prepare", effect: "no_effect", next_action: "restore_system_process_observation" });
+            const stdin = io.stdin ?? process.stdin;
+            const nonInteractive = invocation.nonInteractive || output.json || output.progressJsonl
+                || !stdin.isTTY || !io.stderr.isTTY;
+            const selected = await onboardExecutor({
+                ...(typeof invocation.execution.rootProfile === "string" ? { profileId: invocation.execution.rootProfile } : {}),
+                ...(invocation.profileFile ? { profileFile: invocation.profileFile } : {})
+            }, () => readOperatorProfiles(resolveDdFlowHome(env)), { allowSkip: false, nonInteractive }, settings => askExecutorChoice(settings, { stdin, stderr: io.stderr }, { allowSave: false, allowSkip: false }));
+            if (selected.source === "explicit-file" || selected.source === "explicit-profile")
+                invocation.execution.rootProfile = selected.profile;
+            else if (invocation.execution.rootProfile === undefined && selected.source === "explicit-id")
+                invocation.execution.rootProfile = selected.profile.id;
+            let native;
+            if (invocation.execution.delegation?.mode === "native") {
+                if (!invocation.allowLiveProbe)
+                    throw new AppError("usage", "Fresh native init requires --allow-live-probe", 2, { effect: "no_effect" });
+                const admissionReader = createContext(env, "read_existing");
+                let profile;
+                try {
+                    const baseline = prepareMemoryInitSelection(admissionReader, invocation.scope, { execution: {
+                            ...invocation.execution, delegation: { mode: "external", max_parallel: invocation.execution.delegation.max_parallel ?? 1 }
+                        }, gitPolicy: invocation.gitPolicy, objective });
+                    PreparedMemoryInitLintCapability.prepare(baseline.target, env);
+                    PreparedMemoryInitWorktrunkCapability.prepare(baseline.target, env);
+                    baseline.engine.admitMemoryInitFlow(baseline.source, "dd-flow/memory-init-startup-input@3");
+                    assertMemoryInitStorePrerequisites(env, invocation.scope);
+                    const snapshot = baseline.execution.snapshot();
+                    profile = snapshot.profiles.find(entry => entry.id === snapshot.settings.root_profile).profile;
+                    if (snapshot.profiles.some(entry => hybridDigest({ ...entry.profile, id: profile.id }) !== hybridDigest(profile)))
+                        throw new AppError("generic_execution_invalid", "Native delegation cannot change execution profiles", 2, { effect: "no_effect" });
+                }
+                finally {
+                    admissionReader.db.close?.();
+                }
+                // From here a failed reply cannot imply no effect: daemon/child custody
+                // is independent from later startup publication and RUN allocation.
+                effects = true;
+                native = await produceMemoryInitNativeCapacity({ home: resolveDdFlowHome(env), scope: invocation.scope, profile,
+                    maximum: invocation.execution.delegation.max_parallel ?? 1,
+                    ...(invocation.nativeConfigHome !== undefined ? { nativeConfigHome: invocation.nativeConfigHome } : {}),
+                    allowLiveProbe: invocation.allowLiveProbe, env, requestSha256: hybridDigest({ scope: invocation.scope,
+                        execution: invocation.execution, gitPolicy: invocation.gitPolicy, objective, intake: invocation.intakeSha256 }) });
+                invocation.execution.delegation.nativeCapacityReceipt = native.reference;
+            }
+            const owner = prepareMemoryInitFreshRequest(env, invocation.scope, { execution: invocation.execution, gitPolicy: invocation.gitPolicy, objective,
+                ...(native ? { receipts: native.receipts, receiptBytes: native.receiptBytes } : {}) }, raw);
+            context = owner.context;
+            prepared = owner.prepared;
+        }
+        // Beyond this boundary, absence of a success receipt is not no-effect.
+        effects = true;
+        const started = startMemoryInitRun(context, invocation.scope, prepared ? () => prepared : undefined);
+        if (invocation.prepareOnly) {
+            const run = requireRun(context, requireProjectByRoot(context, invocation.scope.projectRoot).id, started.runId);
+            const response = { ok: true, action: "flow_mb_init_prepared", flow_id: "mb-init", ...started,
+                run_id: started.runId, request_id: invocation.scope.requestId, run_root: run.run_root,
+                project_root: run.project_root, workspace_root: run.workspace_root };
+            await writeMemoryInitObservation(io.stdout, `${JSON.stringify(writeResponseFile(response, output), null, output.json || output.progressJsonl ? undefined : 2)}\n`);
+            return 0;
+        }
+        const launch = prepareMemoryInitControllerStart(context, invocation.scope);
+        // Use the existing RUN-bound router: a global CLI must not launch its own
+        // controller code in place of the independently frozen RUN executable.
+        const receiptChunks = [];
+        let receiptSize = 0, oversizedReceipt = false;
+        const capture = new Writable({ write(chunk, _, next) {
+                receiptSize += Buffer.byteLength(chunk);
+                if (receiptSize > 1024 * 1024)
+                    oversizedReceipt = true;
+                else
+                    receiptChunks.push(Buffer.from(chunk));
+                next();
+            } });
+        const routed = await routeArgsThroughEngine(context, ["run", "drive", "launch", "--project-root", launch.input.projectRoot,
+            "--run", launch.input.runId, "--request-id", launch.input.requestId, "--context-file", launch.input.contextFile,
+            "--context-sha256", launch.input.contextSha256, "--json"], { stdout: capture, stderr: io.stderr }, undefined, { ...env, DD_FLOW_ENGINE_MODE: undefined }, invocation.scope.projectRoot);
+        if (!routed?.routed || routed.inProcess || routed.exitCode !== 0 || oversizedReceipt)
+            throw new AppError("memory_init_controller_launch_unknown", "Observe or reconcile the exact RUN-bound controller launch", 1, { effect: "unknown" });
+        let receipt;
+        try {
+            receipt = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(receiptChunks)));
+        }
+        catch {
+            throw new AppError("memory_init_controller_launch_unknown", "Selected engine returned no valid controller receipt", 1, { effect: "unknown" });
+        }
+        const accepted = receipt.controller;
+        if (receipt.ok !== true || !accepted || accepted.run_id !== started.runId || accepted.request_id !== invocation.scope.requestId)
+            throw new AppError("memory_init_controller_launch_unknown", "Selected engine returned a foreign controller receipt", 1, { effect: "unknown" });
+        const emit = (result) => writeMemoryInitObservation(io.stdout, `${JSON.stringify(result, null, output.json || output.progressJsonl ? undefined : 2)}\n`);
+        if (output.progressJsonl && !await emit({ ok: true, action: "flow_accepted", flow_id: "mb-init", ...started, ...receipt }))
+            return 0;
+        let result = receipt;
+        if (!invocation.detach) {
+            let after = 0;
+            let recentEvents = [];
+            for (;;) {
+                // The client has no ownership of the detached controller process.
+                if (io.stdout.destroyed)
+                    return 0;
+                result = runControllerStatus(context, { projectRoot: invocation.scope.projectRoot, runId: started.runId, after });
+                const events = result.events, controller = result.controller;
+                after = result.next_cursor;
+                recentEvents = [...recentEvents, ...events].slice(-100);
+                if (output.progressJsonl && events.length && !await emit({ ok: true, action: "flow_progress", flow_id: "mb-init", ...result }))
+                    return 0;
+                if (!controller)
+                    throw new AppError("memory_init_controller_receipt_missing", "Observe or reconcile the retained controller", 1, { effect: "unknown" });
+                if (["completed", "cancelled", "superseded", "recovery_required", "failed", "waiting_for_user", "waiting_for_context"].includes(controller.status)) {
+                    result = { ...result, events: recentEvents };
+                    break;
+                }
+                await delay(250);
+            }
+        }
+        const response = { ...result, action: "flow_mb_init", flow_id: "mb-init", run_id: started.runId, request_id: invocation.scope.requestId };
+        if (!await emit(writeResponseFile(response, output)))
+            return 0;
+        const status = result.controller?.status;
+        return ["recovery_required", "failed", "cancelled", "superseded"].includes(status ?? "") ? 1 : 0;
+    }
+    catch (error) {
+        const record = errorRecord(error);
+        if (effects)
+            record.details = { ...record.details, effect: record.details.effect === "partial" ? "partial" : "unknown", next_action: "observe_or_reconcile_exact_request" };
+        writePreparationError(io, args, record);
+        return record.exitCode;
+    }
+    finally {
+        context?.db.close?.();
+    }
+}

package/dist/cli/memory-storage-admission.js

@@ -0,0 +1,53 @@
+import fs from "node:fs";
+import path from "node:path";
+import { prepareCommandRoute } from "./command-inputs.js";
+import { parseOutputOptions } from "./output-options.js";
+import { createContext } from "../runtime/context.js";
+import { getResourceDatabase } from "../storage/database.js";
+import { withStoreCleanup } from "../storage/native-cleanup.js";
+import { PreparedMemoryInitRuntimeStorage } from "../services/memory-init-storage.js";
+import { PreparedMemoryInitResourceStorage } from "../services/memory-init-resource-storage.js";
+import { hybridDigest } from "../services/hybrid-contracts.js";
+import { AppError, errorRecord } from "../shared/errors.js";
+import { writePreparationError } from "./error-output.js";
+/** Selected engine's non-migrating operation admission. No AppContext initialize. */
+export function runMemoryStorageAdmissionCli(args, env = process.env, io = process) {
+    try {
+        const output = parseOutputOptions(args), route = prepareCommandRoute(output.args);
+        if (route.key !== "runtime storage admit" || output.responseFile || output.progressJsonl) {
+            throw new AppError("usage", "Storage admission accepts stdout output only", 2, { effect: "no_effect" });
+        }
+        const home = env.DD_FLOW_HOME;
+        if (!home || !path.isAbsolute(home) || path.resolve(home) !== home || fs.realpathSync(home) !== home) {
+            throw new AppError("usage", "Storage admission requires an explicit canonical absolute DD_FLOW_HOME", 2, { effect: "no_effect" });
+        }
+        const file = path.join(home, "db.sqlite"), owner = fs.lstatSync(file);
+        if (!owner.isFile() || owner.nlink !== 1)
+            throw new AppError("run_storage_unprepared", "RUN storage must be a regular single-linked file", 1, { effect: "no_effect" });
+        const resource = PreparedMemoryInitResourceStorage.prepare({ ddFlowHome: home, env });
+        const context = createContext(env, "prepared_existing");
+        const facts = withStoreCleanup(() => {
+            const run = PreparedMemoryInitRuntimeStorage.prepare(context);
+            const resourceWriter = getResourceDatabase(path.dirname(resource.facts().store_path), "prepared_existing");
+            return withStoreCleanup(() => {
+                if (resourceWriter.get("SELECT dd_flow_writer_version() AS version")?.version !== resource.facts().executing_writer_version) {
+                    throw new AppError("store_writer_incompatible", "Resource connection writer differs from admitted storage", 1, { effect: "no_effect" });
+                }
+                resource.revalidate({ ddFlowHome: home, env });
+                const current = fs.lstatSync(file);
+                if (!current.isFile() || current.nlink !== 1 || current.dev !== owner.dev || current.ino !== owner.ino) {
+                    throw new AppError("run_storage_changed", "RUN store changed during admission", 1, { effect: "no_effect" });
+                }
+                return { schema_id: "dd-flow/memory-init-engine-storage@1", operation: "memory-init", route: "verified-no-op", outcome: "no-op",
+                    run: { ...run.facts(), store_dev: owner.dev, store_ino: owner.ino }, resource: resource.facts() };
+            }, () => resourceWriter.close?.());
+        }, () => context.db.close?.());
+        io.stdout.write(`${JSON.stringify({ ok: true, action: "runtime_storage_admit", facts, sha256: hybridDigest(facts) }, null, output.json ? undefined : 2)}\n`);
+        return 0;
+    }
+    catch (error) {
+        const record = errorRecord(error);
+        writePreparationError(io, args, record);
+        return record.exitCode;
+    }
+}

package/dist/cli/native-hook-ingress.js

@@ -8,7 +8,7 @@ export function isNativeHookIngress(args) {
 }
 export async function dispatchNativeHook(args, stdin, env) {
     const started = performance.now();
-    const deadline = env.DD_FLOW_HOOK_DEADLINE_AT === undefined ? undefined : Number(env.DD_FLOW_HOOK_DEADLINE_AT);
+    const deadline = env.DD_FLOW_HOOK_DEADLINE_AT ? Number(env.DD_FLOW_HOOK_DEADLINE_AT) : undefined;
     if (deadline !== undefined && !Number.isSafeInteger(deadline))
         throw new AppError("hook_deadline_invalid", "Native hook deadline must be an absolute millisecond timestamp", 1, { effect: "unknown" });
     const phase = (name) => {

package/dist/cli/response-file.js

@@ -25,6 +25,8 @@ export function prepareResponseDestination(value) {
             parent = next;
             parentStat = inspect(parent);
         }
+        if (parentStat?.isSymbolicLink())
+            parentStat = fs.statSync(parent);
         const targetStat = inspect(file);
         if (!parentStat?.isDirectory() || targetStat && !targetStat.isFile())
             throw new Error();