@deksden-com/dd-flow-cli 0.9.0-beta.124 → 0.9.0-beta.127
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +10 -1
- package/dist/build-info.json +7 -7
- package/dist/cli/argument-values.js +38 -0
- package/dist/cli/command-inputs.js +4 -3
- package/dist/cli/help.js +1 -0
- package/dist/cli/hook-ingress.js +1 -1
- package/dist/cli/run-cli.js +126 -193
- package/dist/cli/runtime-maintenance-diagnostics.js +46 -0
- package/dist/cli/runtime-maintenance-ingress.js +51 -0
- package/dist/cli/runtime-process.js +128 -0
- package/dist/cli.js +8 -5
- package/dist/harness-runtime/bin/dd-agy.mjs +3 -2
- package/dist/harness-runtime/bin/dd-codex.mjs +3 -2
- package/dist/harness-runtime/bin/dd-droid.mjs +3 -2
- package/dist/harness-runtime/bin/dd-grok.mjs +16 -6
- package/dist/harness-runtime/bin/dd-opencode.mjs +3 -2
- package/dist/harness-runtime/bin/dd-zcode.mjs +20 -16
- package/dist/harness-runtime/lib/adapter-timeouts.d.mts +5 -0
- package/dist/harness-runtime/lib/adapter-timeouts.mjs +34 -0
- package/dist/harness-runtime/lib/daemon-observation.d.mts +2 -0
- package/dist/harness-runtime/lib/daemon-observation.mjs +163 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +60 -13
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +151 -72
- package/dist/harness-runtime/lib/dd-agy.mjs +11 -23
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +9 -14
- package/dist/harness-runtime/lib/dd-codex.mjs +252 -80
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +14 -19
- package/dist/harness-runtime/lib/dd-droid.mjs +117 -31
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +56 -30
- package/dist/harness-runtime/lib/dd-grok.mjs +15 -13
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +18 -27
- package/dist/harness-runtime/lib/dd-opencode.mjs +44 -14
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +56 -53
- package/dist/harness-runtime/lib/dd-zcode.mjs +200 -111
- package/dist/harness-runtime/lib/lease-renewal.d.mts +8 -0
- package/dist/harness-runtime/lib/lease-renewal.mjs +23 -0
- package/dist/harness-runtime/lib/lifecycle-command.d.mts +3 -3
- package/dist/harness-runtime/lib/lifecycle-command.mjs +74 -20
- package/dist/harness-runtime/lib/managed-daemon.d.mts +10 -1
- package/dist/harness-runtime/lib/managed-daemon.mjs +228 -40
- package/dist/harness-runtime/lib/native-children.d.mts +13 -0
- package/dist/harness-runtime/lib/native-children.mjs +78 -0
- package/dist/harness-runtime/lib/native-hook-command.d.mts +4 -0
- package/dist/harness-runtime/lib/native-hook-command.mjs +115 -11
- package/dist/harness-runtime/lib/native-streams.mjs +45 -0
- package/dist/harness-runtime/lib/observation-clock.d.mts +18 -0
- package/dist/harness-runtime/lib/observation-clock.mjs +73 -17
- package/dist/harness-runtime/lib/operation-errors.d.mts +14 -0
- package/dist/harness-runtime/lib/operation-errors.mjs +105 -8
- package/dist/harness-runtime/lib/operation-progress.d.mts +7 -0
- package/dist/harness-runtime/lib/operation-progress.mjs +112 -0
- package/dist/harness-runtime/lib/process-json.mjs +136 -38
- package/dist/harness-runtime/lib/tool-observations.d.mts +1 -0
- package/dist/harness-runtime/lib/tool-observations.mjs +5 -2
- package/dist/harness-runtime/lib/zcode-usage.mjs +3 -2
- package/dist/runtime/context.js +8 -4
- package/dist/schemas/run-control-receipt.schema.json +19 -0
- package/dist/services/code-checks.js +79 -9
- package/dist/services/codex-capacity.js +117 -41
- package/dist/services/codex-hook-delivery.js +2 -2
- package/dist/services/compatibility-preflight.js +4 -1
- package/dist/services/controller-fanout.js +9 -76
- package/dist/services/engines.js +70 -20
- package/dist/services/eval-snapshots.js +179 -40
- package/dist/services/external-work-launch.js +13 -8
- package/dist/services/harness-adapter.js +175 -11
- package/dist/services/hooks.js +65 -21
- package/dist/services/lifecycle-invocations.js +404 -32
- package/dist/services/managed-lease-monitor.js +128 -0
- package/dist/services/managed-processes.js +103 -47
- package/dist/services/merge-server.js +58 -21
- package/dist/services/recovery-observation-budget.js +67 -7
- package/dist/services/recovery-snapshot-database.js +7 -4
- package/dist/services/run-control-receipt.js +1 -1
- package/dist/services/run-control-worker.js +177 -118
- package/dist/services/run-control.js +99 -8
- package/dist/services/run-controller-adapter.js +169 -46
- package/dist/services/run-controller-capture.js +14 -2
- package/dist/services/run-controller-process.js +11 -4
- package/dist/services/run-controller-state.js +68 -7
- package/dist/services/run-controller.js +166 -94
- package/dist/services/run-fork.js +8 -3
- package/dist/services/run-recovery-guard.js +75 -0
- package/dist/services/run-recovery-runtime.js +1 -56
- package/dist/services/run-recovery.js +17 -13
- package/dist/services/runs.js +39 -6
- package/dist/services/runtime-budget.js +32 -15
- package/dist/services/runtime-scope-control.js +8 -0
- package/dist/services/runtime-scope-resume.js +30 -8
- package/dist/services/runtime-scope-worker.js +118 -73
- package/dist/services/runtime-service.js +6 -7
- package/dist/services/scope-writer-cleanup.js +3 -2
- package/dist/services/stage-context.js +135 -0
- package/dist/services/stage-lifecycle.js +3 -15
- package/dist/services/stage-pause.js +3 -2
- package/dist/services/usage.js +14 -3
- package/dist/services/vnext-code-review.js +10 -9
- package/dist/services/vnext-code.js +10 -8
- package/dist/services/vnext-contracts.js +10 -1
- package/dist/services/vnext-execution-profile.js +75 -2
- package/dist/services/vnext-fanout.js +5 -0
- package/dist/services/vnext-merge.js +10 -9
- package/dist/services/vnext-plan-review.js +21 -17
- package/dist/services/vnext-plan.js +9 -7
- package/dist/services/vnext-protocolize.js +15 -9
- package/dist/services/vnext-specify.js +6 -6
- package/dist/services/work-registry.js +34 -4
- package/dist/services/workspace-bootstrap.js +14 -3
- package/dist/shared/errors.js +11 -6
- package/dist/storage/agent-output.js +26 -0
- package/dist/storage/database.js +80 -21
- package/package.json +7 -4
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
import { callDaemonObserved } from './daemon-observation.mjs';
|
|
2
|
+
import { admitNativeRequestBuffer, nativeObservationError } from "./native-streams.mjs";
|
|
1
3
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
4
|
import { chmod, lstat, mkdir, open, unlink, realpath } from "node:fs/promises";
|
|
3
5
|
import { spawn } from "node:child_process";
|
|
4
6
|
import net from "node:net";
|
|
5
7
|
import os from "node:os";
|
|
6
8
|
import { DroidError, DroidRuntime, absolute, droidPaths, droidVersion, prepareDroidHome, readJson, removeDroidAuth, processSnapshot } from "./dd-droid.mjs";
|
|
7
|
-
import { durableDaemonDispatch, inspectDaemonOperation, completeDaemonShutdown, commitDaemonShutdown, assertDaemonDispatch, assertDaemonRestartable, trackDaemonConnections, retireDaemonEndpoint, retireFailedDaemon } from "./daemon-operations.mjs";
|
|
8
|
-
import {
|
|
9
|
+
import { durableDaemonDispatch, inspectDaemonOperation, settleDaemonOperation, completeDaemonShutdown, commitDaemonShutdown, assertDaemonDispatch, assertDaemonRestartable, trackDaemonConnections, retireDaemonEndpoint, retireFailedDaemon } from "./daemon-operations.mjs";
|
|
10
|
+
import { ObservationClock, validateObservationDuration } from "./observation-clock.mjs";
|
|
9
11
|
import { writeJsonAtomic } from "./runner-events.mjs";
|
|
10
12
|
import { errorRecord } from "./operation-errors.mjs";
|
|
11
13
|
import { prepareRuntimeOwner, cleanupFailedStart, confirmDaemonProcess, confirmDaemonStopped, finishDaemonProcess, scheduleDaemonProcessHeartbeat, registerDaemonProcess } from "./managed-daemon.mjs";
|
|
@@ -13,25 +15,18 @@ import { prepareRuntimeOwner, cleanupFailedStart, confirmDaemonProcess, confirmD
|
|
|
13
15
|
const REQUEST = "dd-droid/daemon-request@1", RESPONSE = "dd-droid/daemon-response@1";
|
|
14
16
|
async function removeSocket(file) { try { if (!(await lstat(file)).isSocket()) throw new DroidError("unsafe_socket", "Refusing to replace non-socket daemon path"); await unlink(file); } catch (error) { if (error.code !== "ENOENT") throw error; } }
|
|
15
17
|
export async function callDaemon(stateDir, operation, params = {}, timeoutMs = 30_000, operationId = process.env.DD_EVAL_OPERATION_ID ?? randomUUID()) {
|
|
16
|
-
const
|
|
17
|
-
return await
|
|
18
|
-
const client = net.createConnection(paths.socket); let buffer = "", settled = false;
|
|
19
|
-
const finish = (error, value) => { if (settled) return; settled = true; clearTimeout(timer); client.destroy(); error ? reject(error) : resolve(value); };
|
|
20
|
-
const timer = operation === "session.prompt" ? null : observedTimeout(() => finish(new DroidError("daemon_timeout", `${operation} response was not observed`, { operation_id: operationId })), timeoutMs);
|
|
21
|
-
client.setEncoding("utf8"); client.on("connect", () => client.write(`${JSON.stringify(request)}\n`));
|
|
22
|
-
client.on("error", error => finish(new DroidError("daemon_not_running", error.message)));
|
|
23
|
-
client.on("end", () => finish(new DroidError("daemon_connection_closed", "Droid daemon closed the connection before its reply")));
|
|
24
|
-
client.on("close", () => finish(new DroidError("daemon_connection_closed", "Droid daemon connection closed")));
|
|
25
|
-
client.on("data", chunk => { buffer += chunk; const index = buffer.indexOf("\n"); if (index < 0) return; try { const response = JSON.parse(buffer.slice(0, index)); if (response.schema_id !== RESPONSE || response.id !== operationId) throw new DroidError("daemon_protocol", "Droid daemon returned an unrelated response"); response.ok ? finish(null, response.result) : finish(Object.assign(new Error(response.error.message), response.error)); } catch (error) { finish(error); } });
|
|
26
|
-
});
|
|
18
|
+
const request = { schema_id: REQUEST, id: operationId, operation, params };
|
|
19
|
+
return await callDaemonObserved({ stateDir, socket: droidPaths(stateDir).socket, request, responseSchema: RESPONSE, harness: 'droid', timeoutMs });
|
|
27
20
|
}
|
|
28
|
-
function sameConfig(a, b) { return ["cwd", "bin", "model", "reasoning", "mode", "provider", "ddFlowBin", "ddFlowHome", "resourceHome", "projectRoot", "noFlow"].every(key => a?.[key] === b[key]); }
|
|
21
|
+
function sameConfig(a, b) { return ["cwd", "bin", "model", "reasoning", "mode", "provider", "ddFlowBin", "ddFlowHome", "resourceHome", "projectRoot", "noFlow","inactivityMs"].every(key => a?.[key] === b[key]); }
|
|
29
22
|
export async function startDaemon(options) {
|
|
30
23
|
const paths = droidPaths(options.stateDir), noFlow = options.noFlow === true || (!options.ddFlowBin && !options.ddFlowHome);
|
|
31
24
|
const config = { stateDir: paths.dir, home: paths.home, bin: options.bin ?? process.env.DD_DROID_BIN ?? "droid", cwd: absolute(options.cwd, "--cwd"), projectRoot: absolute(options.projectRoot ?? options.cwd, "--project-root"), journal: absolute(options.journal ?? `${paths.dir}/events.jsonl`, "--journal"), entryPath: absolute(options.entryPath, "adapter entry"), authHome: absolute(options.authHome ?? os.homedir(), "auth home"), model: options.model ?? "gpt-5.6-sol", reasoning: options.reasoning ?? "high", mode: options.mode ?? "auto-high", provider: options.provider ?? "openai", noFlow, ddFlowBin: noFlow ? null : absolute(options.ddFlowBin, "--dd-flow-bin"), ddFlowHome: noFlow ? null : absolute(options.ddFlowHome, "--dd-flow-home"), resourceHome: noFlow ? null : process.env.DD_FLOW_RESOURCE_HOME ?? null, inactivityMs: options.timeoutMs ?? 600_000 };
|
|
32
25
|
await prepareRuntimeOwner(config, paths.dir);
|
|
33
26
|
config.cwd = await realpath(config.cwd); config.projectRoot = await realpath(config.projectRoot);
|
|
34
27
|
if (config.mode !== "auto-high") throw new DroidError("droid_mode_unsupported", "The qualified Droid profile requires auto-high");
|
|
28
|
+
config.inactivityMs = options.livenessTimeoutMs ?? options.timeoutMs ?? 600_000;
|
|
29
|
+
validateObservationDuration(config.inactivityMs);
|
|
35
30
|
await mkdir(paths.dir, { recursive: true, mode: 0o700 }); await chmod(paths.dir, 0o700);
|
|
36
31
|
let previous = await readJson(paths.state); assertDaemonRestartable(previous);
|
|
37
32
|
try { const live = await callDaemon(paths.dir, "daemon.status", {}, 1_000); if (!sameConfig(live.config, config)) throw new DroidError("daemon_config_mismatch", "Droid daemon already uses another configuration"); return { ...live, already_running: true }; } catch (error) { if (!["daemon_not_running", "daemon_timeout"].includes(error.code)) throw error; }
|
|
@@ -49,8 +44,8 @@ export async function startDaemon(options) {
|
|
|
49
44
|
const log = await open(paths.log, "a", 0o600); const child = spawn(process.execPath, [config.entryPath, "daemon", "serve", "--state-dir", paths.dir], { cwd: config.cwd, detached: true, stdio: ["ignore", log.fd, log.fd] }); child.unref(); await log.close();
|
|
50
45
|
try {
|
|
51
46
|
await confirmDaemonProcess(config, resource, child);
|
|
52
|
-
const
|
|
53
|
-
while (
|
|
47
|
+
const startupClock = new ObservationClock({ timeoutMs: 30_000 });
|
|
48
|
+
while (!startupClock.sample()) { await new Promise(resolve => setTimeout(resolve, 100)); const saved = await readJson(paths.state); if (saved?.startup_error) throw Object.assign(new Error(saved.startup_error.message), saved.startup_error); try { return await callDaemon(paths.dir, "daemon.status", {}, 1_000); } catch (error) { if (!["daemon_not_running", "daemon_timeout"].includes(error.code)) throw error; } }
|
|
54
49
|
throw new DroidError("daemon_start_failed", "Droid daemon did not become ready");
|
|
55
50
|
} catch (error) { await cleanupFailedStart(config, resource, child, error); throw error; }
|
|
56
51
|
}
|
|
@@ -97,15 +92,15 @@ export async function serveDaemon(stateDir) {
|
|
|
97
92
|
assertDaemonDispatch(paths.dir);
|
|
98
93
|
if (operation === "session.create") { if (params.prompt) throw new DroidError("invalid_create", "create does not execute a prompt"); return await runtime.create(request.id); }
|
|
99
94
|
if (operation === "session.resume") return await runtime.resume(params.sessionId);
|
|
100
|
-
if (["session.prompt", "session.start"].includes(operation)) return await runtime.prompt(params.sessionId, params.prompt, request.id);
|
|
95
|
+
if (["session.prompt", "session.start"].includes(operation)) return await runtime.prompt(params.sessionId, params.prompt, request.id, params.livenessTimeoutMs ?? runtime.config.inactivityMs);
|
|
101
96
|
throw new DroidError("unknown_operation", `Unknown Droid operation: ${operation}`);
|
|
102
97
|
} finally { busy = null; await save({ active_operation: null }); }
|
|
103
98
|
};
|
|
104
99
|
await removeSocket(paths.socket);
|
|
105
|
-
const server = net.createServer(socket => { socket.setEncoding("utf8"); let buffer = "", consumed = false; socket.on("error", () => {}); socket.on("data", chunk => { if (consumed) return; buffer += chunk; const at = buffer.indexOf("\n"); if (at < 0) return; consumed = true;
|
|
100
|
+
const server = net.createServer(socket => { socket.setEncoding("utf8"); let buffer = "", consumed = false; socket.on("error", () => {}); socket.on("data", chunk => { if (consumed) return; buffer += chunk; if (!admitNativeRequestBuffer(buffer, socket, RESPONSE)) { consumed = true; return; } const at = buffer.indexOf("\n"); if (at < 0) return; consumed = true;
|
|
106
101
|
void (async () => { let request; try { request = JSON.parse(buffer.slice(0, at)); if (request.schema_id !== REQUEST || typeof request.id !== "string" || typeof request.operation !== "string") throw new DroidError("daemon_protocol", "Malformed Droid daemon request"); let result;
|
|
107
102
|
if (request.operation === "operation.inspect") {
|
|
108
|
-
result = await inspectDaemonOperation(paths.dir, request.params?.operationId);
|
|
103
|
+
result = request.params?.settle === true ? await settleDaemonOperation(paths.dir, request.params?.operationId) : await inspectDaemonOperation(paths.dir, request.params?.operationId);
|
|
109
104
|
if (!["completed", "failed"].includes(result.state)) {
|
|
110
105
|
const recovered = await runtime.recoverOperation(request.params.operationId);
|
|
111
106
|
if (recovered) {
|
|
@@ -4,22 +4,23 @@ export { processSnapshot } from "./process-snapshot.mjs";
|
|
|
4
4
|
import { captureDispatchGuard, cancelPendingDispatch } from "./dispatch-fence.mjs";
|
|
5
5
|
import { createHash, randomUUID } from "node:crypto";
|
|
6
6
|
import { appendFile, chmod, copyFile, mkdir, readFile, readdir, realpath, stat, unlink, writeFile } from "node:fs/promises";
|
|
7
|
-
import { spawn
|
|
8
|
-
import { promisify } from "node:util";
|
|
7
|
+
import { spawn } from "node:child_process";
|
|
9
8
|
import os from "node:os";
|
|
10
9
|
import path from "node:path";
|
|
11
|
-
import { ObservationClock, observedTimeout } from "./observation-clock.mjs";
|
|
10
|
+
import { ObservationClock, observedTimeout, validateObservationDuration } from "./observation-clock.mjs";
|
|
11
|
+
import { operationProgressReporter } from "./operation-progress.mjs";
|
|
12
12
|
import { writeJsonAtomic } from "./runner-events.mjs";
|
|
13
|
-
import { errorRecord, reportedError } from "./operation-errors.mjs";
|
|
13
|
+
import { errorRecord, reportedError, isObservationLoss } from "./operation-errors.mjs";
|
|
14
14
|
import { hookBudget, invokeNativeHook } from "./native-hook-command.mjs";
|
|
15
15
|
import { checkObservedProfile, observeModel } from "./model-observations.mjs";
|
|
16
|
+
import { readNativeLines, streamNativeDiagnostics, nativeObservationError } from "./native-streams.mjs";
|
|
17
|
+
import { commandText } from "./process-json.mjs";
|
|
16
18
|
import { DroidMetadataReader, readDroidRoutingLog } from "./droid-observation.mjs";
|
|
17
19
|
const metadataReaders = new Map();
|
|
18
20
|
function metadataReader(factory) { if (!metadataReaders.has(factory)) metadataReaders.set(factory, new DroidMetadataReader(factory)); return metadataReaders.get(factory); }
|
|
19
21
|
import { confirmDaemonProcess, finishDaemonProcess, scheduleDaemonProcessHeartbeat, registerDaemonProcess, stopProcessGroup } from "./managed-daemon.mjs";
|
|
20
22
|
import { renderAdapterPolicy, lifecycleRetryInstruction } from "./delegation-instructions.mjs";
|
|
21
23
|
|
|
22
|
-
const run = promisify(execFile);
|
|
23
24
|
export const DROID_CONTRACT = "dd-droid-harness@1";
|
|
24
25
|
export const DROID_PROTOCOL = "1.201.0";
|
|
25
26
|
const LATE_RESPONSE_GRACE_MS = 5_000;
|
|
@@ -41,7 +42,8 @@ export function droidPaths(stateDir) {
|
|
|
41
42
|
export function droidEnvironment(config) {
|
|
42
43
|
const env = { ...process.env, FACTORY_HOME_OVERRIDE: config.home, FACTORY_DROID_AUTO_UPDATE_ENABLED: "false", ...(config.ddFlowHome ? { DD_FLOW_HOME: config.ddFlowHome, DD_FLOW_BIN: config.ddFlowBin } : {}) };
|
|
43
44
|
// This adapter deliberately uses the operator's existing Factory login.
|
|
44
|
-
|
|
45
|
+
// Undefined is a tombstone: downstream env overlays must not resurrect keys.
|
|
46
|
+
for (const key of Object.keys(env)) if (key.endsWith("API_KEY")) env[key] = undefined;
|
|
45
47
|
return env;
|
|
46
48
|
}
|
|
47
49
|
export async function prepareDroidHome(config) {
|
|
@@ -63,7 +65,7 @@ export async function prepareDroidHome(config) {
|
|
|
63
65
|
export async function removeDroidAuth(config) {
|
|
64
66
|
for (const name of ["auth.encrypted", "auth.v2.key", "auth.v2.file", "auth.json"]) await unlink(path.join(config.home, ".factory", name)).catch(error => { if (error.code !== "ENOENT") throw error; });
|
|
65
67
|
}
|
|
66
|
-
export async function droidVersion(bin) { const target = executable(bin, ["--version"]); return
|
|
68
|
+
export async function droidVersion(bin) { const target = executable(bin, ["--version"]); return await commandText(target.command, target.args, { timeoutMs: 10_000, phase: "native-version" }); }
|
|
67
69
|
|
|
68
70
|
/** The qualified OpenAI profile reports uncached input separately from caches;
|
|
69
71
|
* output already includes thinking. Keep native counters alongside normalization. */
|
|
@@ -101,7 +103,7 @@ export class DroidRuntime {
|
|
|
101
103
|
this.config = config; this.state = state; this.paths = droidPaths(config.stateDir); this.persist = persist; this.pending = new Map(); this.lateResponses = new Map(); this.active = null; this.child = null; this.closed = true; this.loaded = false; this.ownedProcesses = state.owned_processes ?? []; this.closing = false; this.workingState = "idle"; this.protocol = null; this.rootId = state.root_session_id ?? null; this.topology = new Map(state.descendants?.map(child => [child.provider_session_id, child]) ?? []); this.draining = Promise.resolve(); this.journalling = Promise.resolve(); this.activity = 0; this.providerProcess = null; this.cancelled = false; this.lastResult = state.last_result ?? null; this.metadata = metadataReader(this.paths.factory); this.fatalError = state.fatal_error ? reportedError(state.fatal_error) : null;
|
|
102
104
|
}
|
|
103
105
|
async journal(kind, payload) { const line = JSON.stringify({ schema_id: "dd-droid/event@1", observed_at: new Date().toISOString(), kind, ...payload }); this.journalling = this.journalling.then(() => appendFile(this.config.journal, `${line}\n`, { mode: 0o600 })); return await this.journalling; }
|
|
104
|
-
async save(patch = {}) { return await this.persist({ root_session_id: this.rootId, descendants: [...this.topology.values()], working_state: this.workingState, last_activity: this.activity, ...patch }); }
|
|
106
|
+
async save(patch = {}) { return await this.persist({ root_session_id: this.rootId, sessions: this.rootId ? [this.rootId] : [], descendants: [...this.topology.values()], working_state: this.workingState, last_activity: this.activity, ...patch }); }
|
|
105
107
|
async start() {
|
|
106
108
|
if (this.child && !this.closed) return;
|
|
107
109
|
this.closed = false; this.loaded = false; this.closing = false; this.protocol = null; this.buffer = "";
|
|
@@ -109,10 +111,14 @@ export class DroidRuntime {
|
|
|
109
111
|
const target = executable(this.config.bin, args);
|
|
110
112
|
this.providerProcess = await registerDaemonProcess(this.config, { kind: "droid-provider", owner: `droid:${this.config.daemonId}`, operation: `droid-provider:${randomUUID()}`, stdout: this.paths.log, stderr: this.paths.log });
|
|
111
113
|
this.child = spawn(target.command, target.args, { cwd: this.config.cwd, env: droidEnvironment(this.config), detached: true, stdio: ["pipe", "pipe", "pipe"] });
|
|
112
|
-
|
|
113
|
-
|
|
114
|
+
const observationFailure = async error => {
|
|
115
|
+
if (error.code === "operation_observation_lost") { this.observationError = error; this.rejectPending(error); return; }
|
|
116
|
+
await this.handleObservationError(error); if (!this.fatalError) this.rejectPending(error);
|
|
117
|
+
};
|
|
118
|
+
this.nativeDrain = readNativeLines(this.child.stdout, line => this.draining = this.event(JSON.parse(line)), observationFailure);
|
|
119
|
+
this.diagnostics = streamNativeDiagnostics(this.child.stderr, chunk => this.journal("stderr", { text: chunk }), observationFailure);
|
|
114
120
|
this.child.on("error", error => this.rejectPending(error));
|
|
115
|
-
this.child.on("close", (code, signal) => { this.closed = true; void this.
|
|
121
|
+
this.child.on("close", (code, signal) => { this.closed = true; void Promise.all([this.nativeDrain, this.diagnostics]).finally(() => { const error = new DroidError("operation_observation_lost", "Droid process exited before a confirmed reply", { code, signal }); this.rejectPending(error); }); });
|
|
116
122
|
await this.captureProcesses(); await confirmDaemonProcess(this.config, this.providerProcess, this.child); scheduleDaemonProcessHeartbeat(this.config, this.providerProcess); await this.save({ provider_pid: this.child.pid, provider_process: this.providerProcess });
|
|
117
123
|
clearInterval(this.processObserver);
|
|
118
124
|
this.processObserver = setInterval(() => { if (!this.processObservation) this.processObservation = this.captureProcesses().catch(error => { this.ownershipError = error; }).finally(() => { this.processObservation = null; }); }, 1_000);
|
|
@@ -145,22 +151,59 @@ export class DroidRuntime {
|
|
|
145
151
|
if (event.method !== "droid.session_notification") return;
|
|
146
152
|
const { sessionId, notification: n } = event.params ?? {}; if (!n) return;
|
|
147
153
|
const fingerprint = createHash("sha256").update(JSON.stringify(event.params)).digest("hex");
|
|
148
|
-
if (fingerprint !== this.lastEvent) { this.activity += 1; this.lastEvent = fingerprint; }
|
|
149
154
|
if (sessionId && this.rootId && sessionId !== this.rootId && !this.topology.has(sessionId)) throw new DroidError("session_identity_mismatch", "Unrelated native Session notification");
|
|
155
|
+
const current = this.active;
|
|
156
|
+
const scoped = current && sessionId && (sessionId === this.rootId || this.topology.get(sessionId)?.operation_id === current.operationId)
|
|
157
|
+
&& (!n.turnId || n.turnId === current.turnId) && (!n.requestId || n.requestId === current.requestId);
|
|
158
|
+
let productive = ["assistant_text_delta", "assistant_thought_delta"].includes(n.type) && Boolean(n.textDelta ?? n.thoughtDelta)
|
|
159
|
+
|| ["agent_turn_completed", "create_message"].includes(n.type);
|
|
160
|
+
if (scoped && ["agent_turn_completed", "create_message"].includes(n.type)) {
|
|
161
|
+
current.lifecycleProgress ??= new Set();
|
|
162
|
+
const key = JSON.stringify([sessionId, n.type, n.requestId ?? null, n.turnId ?? null, n.message?.id ?? null]);
|
|
163
|
+
if (current.lifecycleProgress.has(key)) productive = false;
|
|
164
|
+
else {
|
|
165
|
+
if (current.lifecycleProgress.size >= 1_024) current.lifecycleProgress.delete(current.lifecycleProgress.values().next().value);
|
|
166
|
+
current.lifecycleProgress.add(key);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
if (scoped && ["tool_call_started", "tool_call_update", "tool_call_completed"].includes(n.type)) {
|
|
170
|
+
const toolId = n.toolUseId ?? n.toolCallId ?? n.toolCall?.id;
|
|
171
|
+
current.toolProgress ??= new Set();
|
|
172
|
+
const key = `${sessionId}:${toolId}`, semantic = JSON.stringify({ type: n.type, toolCall: n.toolCall, status: n.status, input: n.input, output: n.output, content: n.content, error: n.error });
|
|
173
|
+
// Keep fixed-size fingerprints, not 1,024 copies of growing tool output.
|
|
174
|
+
const snapshot = createHash("sha256").update(JSON.stringify([key, semantic])).digest("hex");
|
|
175
|
+
if (toolId && !current.toolProgress.has(snapshot)) {
|
|
176
|
+
productive = n.type !== "tool_call_update" || n.status !== undefined || n.output !== undefined || n.content !== undefined || n.toolCall !== undefined || n.error !== undefined;
|
|
177
|
+
if (productive) {
|
|
178
|
+
// ponytail: retain the latest 1,024 snapshots for replay detection,
|
|
179
|
+
// not a lifetime progress quota. Native sequence IDs can extend proof.
|
|
180
|
+
if (current.toolProgress.size >= 1_024) current.toolProgress.delete(current.toolProgress.values().next().value);
|
|
181
|
+
current.toolProgress.add(snapshot);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
if (scoped && productive) {
|
|
186
|
+
this.activity += 1; this.progress = { cursor: this.activity, observed_at: Date.now() };
|
|
187
|
+
const pending = this.pending.get(current.requestId);
|
|
188
|
+
if (pending?.method === "droid.add_user_message") {
|
|
189
|
+
pending.progress = this.progress;
|
|
190
|
+
}
|
|
191
|
+
await current.publish?.("native_progress", { native_request_id: String(current.requestId), session_id: this.rootId, observed_at: new Date(this.progress.observed_at).toISOString() });
|
|
192
|
+
}
|
|
150
193
|
if (n.type === "droid_working_state_changed" && (!sessionId || sessionId === this.rootId)) this.workingState = n.newState;
|
|
151
194
|
if (n.type === "child_session_available") {
|
|
152
195
|
if (!this.rootId || sessionId !== this.rootId || !n.childSessionId || !n.toolUseId) throw new DroidError("droid_child_identity_invalid", "Native child event lacks physical parent or tool identity");
|
|
153
|
-
this.topology.set(n.childSessionId, { provider_session_id: n.childSessionId, parent_provider_session_id: sessionId, tool_use_id: n.toolUseId, status: "running", source: "droid.child_session_available" }); await this.save();
|
|
196
|
+
this.topology.set(n.childSessionId, { provider_session_id: n.childSessionId, parent_provider_session_id: sessionId, tool_use_id: n.toolUseId, operation_id: current?.operationId ?? null, status: "running", source: "droid.child_session_available" }); await this.save();
|
|
154
197
|
}
|
|
155
198
|
if (n.type === "settings_updated" && this.loaded) await this.observeSettings(sessionId ?? this.rootId, n.settings, { kind: "native.settings_updated", event_sha256: fingerprint });
|
|
156
|
-
const current = this.active;
|
|
157
199
|
if (n.type === "create_message" && n.requestId === current?.requestId) {
|
|
158
200
|
if (n.message?.id !== current.turnId) throw new DroidError("droid_turn_identity_mismatch", "Droid did not preserve the requested message ID");
|
|
159
201
|
current.accepted = true; await this.saveBinding(current, { accepted: true });
|
|
160
202
|
}
|
|
161
|
-
if (n.type === "assistant_text_delta" &&
|
|
203
|
+
if (n.type === "assistant_text_delta" && scoped && sessionId === this.rootId) current.text += n.textDelta ?? "";
|
|
162
204
|
if (n.type === "agent_turn_completed" && current && n.turnId === current.turnId && (!sessionId || sessionId === this.rootId)) {
|
|
163
205
|
current.outcome = n; this.lastResult = n; await this.saveBinding(current, { native_outcome: n, assistant_text: current.text }); current.resolve(n);
|
|
206
|
+
if (current.observerReturned && this.active === current) { this.active = null; await this.save({ active_tree: this.workingState !== "idle" }); }
|
|
164
207
|
}
|
|
165
208
|
}
|
|
166
209
|
request(method, params, id = randomUUID(), timeoutMs = 30_000) {
|
|
@@ -171,11 +214,11 @@ export class DroidRuntime {
|
|
|
171
214
|
pending.timedOut = true;
|
|
172
215
|
const timeoutError = new DroidError("rpc_timeout", `${method} has no observed reply`, { request_id: id, timeout_ms: timeoutMs, grace_ms: LATE_RESPONSE_GRACE_MS });
|
|
173
216
|
void this.journal("request_deadline", { method, request_id: id, timeout_ms: timeoutMs, grace_ms: LATE_RESPONSE_GRACE_MS });
|
|
174
|
-
pending.timer = observedTimeout(() => { if (this.pending.get(id) !== pending) return; this.pending.delete(id); reject(timeoutError); }, LATE_RESPONSE_GRACE_MS);
|
|
175
|
-
}, timeoutMs);
|
|
217
|
+
pending.timer = observedTimeout(() => { if (this.pending.get(id) !== pending) return; this.pending.delete(id); reject(timeoutError); }, LATE_RESPONSE_GRACE_MS, { progress: () => pending.progress });
|
|
218
|
+
}, timeoutMs, { progress: () => this.pending.get(id)?.progress });
|
|
176
219
|
this.pending.set(id, { resolve, reject, timer, method });
|
|
177
|
-
try { this.send({ jsonrpc: "2.0", type: "request", factoryApiVersion: "1.0.0", factoryProtocolVersion: DROID_PROTOCOL, id, method, params }, error => { if (error) { clearTimeout(timer); this.pending.delete(id); reject(error); } }); }
|
|
178
|
-
catch (error) { clearTimeout(timer); this.pending.delete(id); reject(error); }
|
|
220
|
+
try { this.send({ jsonrpc: "2.0", type: "request", factoryApiVersion: "1.0.0", factoryProtocolVersion: DROID_PROTOCOL, id, method, params }, error => { if (error) { clearTimeout(timer); this.pending.delete(id); reject(nativeObservationError(error)); } }); }
|
|
221
|
+
catch (error) { clearTimeout(timer); this.pending.delete(id); reject(nativeObservationError(error)); }
|
|
179
222
|
});
|
|
180
223
|
}
|
|
181
224
|
send(value, callback) {
|
|
@@ -186,7 +229,7 @@ export class DroidRuntime {
|
|
|
186
229
|
throw error;
|
|
187
230
|
}
|
|
188
231
|
try { stdin.write(`${JSON.stringify(value)}\n`, callback); }
|
|
189
|
-
catch (
|
|
232
|
+
catch (cause) { const error = nativeObservationError(cause); this.rejectPending(error); throw error; }
|
|
190
233
|
}
|
|
191
234
|
profile(settings) {
|
|
192
235
|
const model = settings?.modelId ?? settings?.model ?? null;
|
|
@@ -244,7 +287,12 @@ export class DroidRuntime {
|
|
|
244
287
|
}
|
|
245
288
|
}
|
|
246
289
|
async handleObservationError(error) {
|
|
247
|
-
if (
|
|
290
|
+
if (error.code === "model_observation_storage_failed") {
|
|
291
|
+
this.observationError = Object.assign(new Error("Droid model observation could not be published", { cause: error }), { code: "operation_observation_lost", details: { native_outcome: "unknown", observation_error: errorRecord(error) } });
|
|
292
|
+
this.rejectPending(this.observationError);
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
if (["profile_integrity_violation", "session_identity_mismatch", "droid_child_identity_invalid", "droid_turn_identity_mismatch", "droid_hook_identity_invalid"].includes(error.code)) {
|
|
248
296
|
if (!this.fatalError) {
|
|
249
297
|
this.fatalError = error;
|
|
250
298
|
try { await this.save({ fatal_error: errorRecord(error) }); await this.journal("integrity_failure", { error: errorRecord(error) }); }
|
|
@@ -276,7 +324,8 @@ export class DroidRuntime {
|
|
|
276
324
|
const source = await readDroidSession(this.paths.factory, id); if (!source || path.resolve(source.header.cwd) !== this.config.cwd) throw new DroidError("session_identity_mismatch", "Resumed Session does not match its controlled workspace");
|
|
277
325
|
this.loaded = true; this.observedProfile = await this.observeSettings(id, result.settings, "native.load_session"); this.workingState = result.workingState ?? (result.isAgentLoopInProgress ? "unknown" : "idle"); this.cancelled = false; await this.save(); return await this.inspect(id);
|
|
278
326
|
}
|
|
279
|
-
async prompt(id, prompt, operationId) {
|
|
327
|
+
async prompt(id, prompt, operationId, livenessTimeoutMs = this.config.inactivityMs ?? 600_000) {
|
|
328
|
+
validateObservationDuration(livenessTimeoutMs);
|
|
280
329
|
const assertDispatch = captureDispatchGuard(this);
|
|
281
330
|
if (this.fatalError) throw this.fatalError;
|
|
282
331
|
this.requireIdentity(id); if (!prompt) throw new DroidError("prompt_required", "prompt is required"); if (this.active) throw new DroidError("operation_busy", "Another Droid operation is active");
|
|
@@ -285,20 +334,31 @@ export class DroidRuntime {
|
|
|
285
334
|
const before = await this.inspect(id); if (!before.settled) throw new DroidError("tree_not_settled", "Prior Droid tree has not settled");
|
|
286
335
|
let resolve, reject; const completed = new Promise((a, b) => { resolve = a; reject = b; }); void completed.catch(() => {});
|
|
287
336
|
const requestId = randomUUID(), turnId = randomUUID();
|
|
288
|
-
const current = { operationId, requestId, turnId, resolve, reject, text: "", saved: { operation: "session.prompt", provider_session_id: id, request_id: requestId, turn_id: turnId, accepted: false } };
|
|
337
|
+
const current = { operationId, requestId, turnId, resolve, reject, text: "", publish: operationProgressReporter(), saved: { operation: "session.prompt", provider_session_id: id, request_id: requestId, turn_id: turnId, accepted: false } };
|
|
289
338
|
this.active = current; this.cancelled = false; await this.saveBinding(current); await this.save({ active_tree: true });
|
|
290
|
-
const clock = new ObservationClock({ timeoutMs:
|
|
339
|
+
const clock = new ObservationClock({ timeoutMs: livenessTimeoutMs, onGap: gap => { void this.journal("observation_gap", { gap }).catch(() => {}); } });
|
|
291
340
|
let checkingTerminal = false;
|
|
292
341
|
const timer = setInterval(() => {
|
|
293
|
-
if (!clock.sample(this.
|
|
342
|
+
if (checkingTerminal || !clock.sample(this.progress)) return;
|
|
294
343
|
checkingTerminal = true;
|
|
344
|
+
const cursor = this.progress?.cursor;
|
|
295
345
|
void this.recoverOperation(operationId).then(() => {
|
|
296
|
-
if (!current.outcome) reject(new DroidError("
|
|
297
|
-
}).catch(error => reject(new DroidError("operation_observation_lost", "Cannot reconcile native outcome before declaring inactivity", { error: errorRecord(error) })));
|
|
298
|
-
}, 1_000);
|
|
346
|
+
if (this.active === current && !current.outcome && this.progress?.cursor === cursor && clock.sample(this.progress)) reject(new DroidError("operation_observation_lost", "Droid native outcome remains unknown after its silence window", { operation_id: operationId, native_outcome: "unknown" }));
|
|
347
|
+
}).catch(error => reject(new DroidError("operation_observation_lost", "Cannot reconcile native outcome before declaring inactivity", { error: errorRecord(error) }))).finally(() => { checkingTerminal = false; });
|
|
348
|
+
}, Math.max(10, Math.min(1_000, livenessTimeoutMs / 4)));
|
|
349
|
+
let primary;
|
|
299
350
|
try {
|
|
300
351
|
assertDispatch();
|
|
352
|
+
current.dispatched = true;
|
|
301
353
|
await this.request("droid.add_user_message", { text: prompt, messageId: turnId }, requestId);
|
|
354
|
+
try { await this.journal("native_dispatch", { operation_id: operationId, provider_session_id: id, native_request_id: requestId }); }
|
|
355
|
+
catch (cause) {
|
|
356
|
+
const error = nativeObservationError(cause);
|
|
357
|
+
if (!current.outcome || error.code !== "operation_observation_lost") throw error;
|
|
358
|
+
throw Object.assign(new Error("Droid completed natively but its dispatch journal could not be saved", { cause }), {
|
|
359
|
+
code: "native_outcome_observation_failed", details: { native_outcome: { provider_session_id: id, turn_id: turnId, result: current.outcome }, settled: false, observation_error: errorRecord(error) }
|
|
360
|
+
});
|
|
361
|
+
}
|
|
302
362
|
const outcome = await completed; await this.draining;
|
|
303
363
|
try {
|
|
304
364
|
const receipt = await this.inspect(id, { ignoreActive: true });
|
|
@@ -310,8 +370,22 @@ export class DroidRuntime {
|
|
|
310
370
|
return { provider_session_id: id, adapter_session_id: id, result: { ...outcome, status: status(outcome.reason) }, assistant_text: current.text, turn_id: turnId, settled: false, settlement_observation_error: errorRecord(error) };
|
|
311
371
|
}
|
|
312
372
|
} catch (error) {
|
|
313
|
-
|
|
314
|
-
|
|
373
|
+
if (current.outcome && isObservationLoss(error)) error = Object.assign(new Error("Droid completed natively but its dispatch acknowledgement was lost", { cause: error }), {
|
|
374
|
+
code: "native_outcome_observation_failed", details: { native_outcome: { provider_session_id: id, turn_id: turnId, result: current.outcome }, settled: false, observation_error: errorRecord(error) }
|
|
375
|
+
});
|
|
376
|
+
primary = error;
|
|
377
|
+
try { await this.journal("prompt_error", { operation_id: operationId, error: errorRecord(error) }); }
|
|
378
|
+
catch (publication) { error.cleanup_error ??= errorRecord(publication); }
|
|
379
|
+
throw error;
|
|
380
|
+
} finally {
|
|
381
|
+
clearInterval(timer); current.observerReturned = true;
|
|
382
|
+
if (this.active === current && (current.outcome || !current.dispatched)) this.active = null;
|
|
383
|
+
try { await this.save({ active_tree: Boolean(this.active) || this.workingState !== "idle" }); }
|
|
384
|
+
catch (publication) {
|
|
385
|
+
if (primary) primary.cleanup_error ??= errorRecord(publication);
|
|
386
|
+
else throw Object.assign(new Error("Droid outcome bookkeeping could not be published", { cause: publication }), { code: current.outcome ? "native_outcome_observation_failed" : "operation_observation_lost", details: { native_outcome: current.outcome ?? "unknown", observation_error: errorRecord(publication) } });
|
|
387
|
+
}
|
|
388
|
+
}
|
|
315
389
|
}
|
|
316
390
|
async recoverOperation(operationId) {
|
|
317
391
|
const binding = await readJson(path.join(this.paths.bindings, `${createHash("sha256").update(operationId).digest("hex")}.json`));
|
|
@@ -332,6 +406,11 @@ export class DroidRuntime {
|
|
|
332
406
|
current.outcome = outcome; this.lastResult = outcome;
|
|
333
407
|
await this.saveBinding(current, { native_outcome: outcome });
|
|
334
408
|
current.resolve(outcome);
|
|
409
|
+
if (current.observerReturned) {
|
|
410
|
+
if (this.active === current) this.active = null;
|
|
411
|
+
await this.save({ active_tree: this.workingState !== "idle" });
|
|
412
|
+
return { ...receipt, result: { ...outcome, status: status(outcome.reason) }, assistant_text: binding.assistant_text ?? "", turn_id: binding.turn_id, recovered_from: "native.agent_turn_outcome" };
|
|
413
|
+
}
|
|
335
414
|
// Let the original dispatch persist its one terminal reply. Settlement
|
|
336
415
|
// is reconciled separately; do not race a second writer or replay it.
|
|
337
416
|
return null;
|
|
@@ -420,10 +499,17 @@ export class DroidRuntime {
|
|
|
420
499
|
try {
|
|
421
500
|
return await this.flow("event", "handle", { schema_id: "dd-flow/droid-tool-event@1", phase: event === "PreToolUse" ? "before" : "after", event_id: eventId, daemon_id: this.config.daemonId, session: { provider_session_id: id, parent_provider_session_id: parent, directory: this.config.cwd }, tool: payload.tool_name, input: payload.tool_input, profile, transcript_path: native.transcript_path, outcome: { status: payload.tool_response?.is_error ? "error" : "completed" } }, budget.remaining("admission", 15_000));
|
|
422
501
|
} catch (error) {
|
|
502
|
+
const details = error.details;
|
|
503
|
+
if (event === "PreToolUse" && error.code === "lifecycle_shell_syntax_invalid" && details?.phase === "prepare" && details.effect === "no_effect" && details.recoverable === true && details.disposition === "pre_cli_shell_refusal" && (details.refusal_event_key || details.event_key)) {
|
|
504
|
+
const refusal = errorRecord(error);
|
|
505
|
+
await this.journal("hook_rejected", { error: refusal, event_id: eventId, session_id: id });
|
|
506
|
+
await this.save();
|
|
507
|
+
return { continue: false, stopReason: error.message };
|
|
508
|
+
}
|
|
423
509
|
// Identity is proven above; only this owned admission can fail its turn.
|
|
424
510
|
this.fatalError ??= Object.assign(error, { details: { ...error.details, provider_session_id: id, event_id: eventId, effect: "unknown" } });
|
|
425
511
|
this.rejectPending(this.fatalError);
|
|
426
|
-
try { await this.save({ fatal_error: errorRecord(this.fatalError) }); await this.journal("hook_rejected", { error: errorRecord(this.fatalError) }); }
|
|
512
|
+
try { await budget.run(async () => { await this.save({ fatal_error: errorRecord(this.fatalError) }); await this.journal("hook_rejected", { error: errorRecord(this.fatalError) }); }, "failure_persistence"); }
|
|
427
513
|
catch (storage) { this.fatalError.details.receipt_error = errorRecord(storage); }
|
|
428
514
|
throw this.fatalError;
|
|
429
515
|
}
|
|
@@ -1,6 +1,12 @@
|
|
|
1
|
+
import { callDaemonObserved } from './daemon-observation.mjs';
|
|
2
|
+
import { canonicalNativeOperation } from './adapter-timeouts.mjs';
|
|
3
|
+
import { admitNativeRequestBuffer } from "./native-streams.mjs";
|
|
1
4
|
import { captureDispatchGuard, cancelPendingDispatch } from "./dispatch-fence.mjs";
|
|
5
|
+
import { errorRecord, isProvenShellCorrection } from "./operation-errors.mjs";
|
|
2
6
|
import { ToolObservations, acpToolObservation } from "./tool-observations.mjs";
|
|
3
|
-
import {
|
|
7
|
+
import { acceptNativeHookRequest, hookBudget, readNativeHookFailures } from "./native-hook-command.mjs";
|
|
8
|
+
import { parseLifecycleCommand, continuationCommand } from "./lifecycle-command.mjs";
|
|
9
|
+
import { ObservationClock } from "./observation-clock.mjs";
|
|
4
10
|
import { createHash, randomUUID } from "node:crypto";
|
|
5
11
|
import { chmod, copyFile, cp, lstat, mkdir, open, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
|
|
6
12
|
import { spawn } from "node:child_process";
|
|
@@ -20,7 +26,7 @@ const RESPONSE_SCHEMA = "dd-grok/daemon-response@1";
|
|
|
20
26
|
const STATE_SCHEMA = "dd-grok/daemon-state@1";
|
|
21
27
|
// Normalize raw socket callers too, before the shared layer reserves capacity.
|
|
22
28
|
export function durableDaemonDispatch(root, request, action, observe) {
|
|
23
|
-
return dispatchOperation(root,
|
|
29
|
+
return dispatchOperation(root, { ...request, operation: canonicalNativeOperation("grok", request.operation) }, action, observe);
|
|
24
30
|
}
|
|
25
31
|
class DaemonError extends Error { constructor(code, message, retryable = false, details) { super(message); this.code = code; this.retryable = retryable; this.details = details; } }
|
|
26
32
|
function absolute(value, label) { if (!value || !path.isAbsolute(value)) throw new DaemonError("invalid_path", `${label} must be an absolute path`); return path.resolve(value); }
|
|
@@ -28,11 +34,11 @@ function locations(stateDir) { const dir = absolute(stateDir, "--state-dir"); co
|
|
|
28
34
|
async function readState(dir) { try { return JSON.parse(await readFile(locations(dir).state, "utf8")); } catch (error) { if (error.code === "ENOENT") return null; throw error; } }
|
|
29
35
|
async function writeState(file, value) { const temporary = `${file}.${process.pid}.tmp`; await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); await rename(temporary, file); }
|
|
30
36
|
async function removeSocket(socket, dir) { if (socket !== locations(dir).socket) throw new DaemonError("unsafe_socket", "daemon socket does not match its state directory"); try { const info = await lstat(socket); if (!info.isSocket()) throw new DaemonError("unsafe_socket", "refusing to remove a non-socket daemon path"); await unlink(socket); } catch (error) { if (error.code !== "ENOENT") throw error; } }
|
|
31
|
-
|
|
37
|
+
const errorPayload = errorRecord;
|
|
32
38
|
function shellQuote(value) { return `'${String(value).replace(/'/g, "'\\''")}'`; }
|
|
33
39
|
|
|
34
40
|
function executable(command, args = []) { const resolved = command ?? process.env.DD_GROK_BIN ?? "grok"; return /\.[cm]?js$/.test(resolved) ? { command: process.execPath, args: [resolved, ...args] } : { command: resolved, args }; }
|
|
35
|
-
export function isolatedEnv(config) { const env = { ...process.env, HOME: config.grokHome, GROK_HOME: config.grokHome, ...(config.ddFlowHome ? { DD_FLOW_HOME: config.ddFlowHome } : {}), ...(config.resourceHome ? { DD_FLOW_RESOURCE_HOME: config.resourceHome } : {}), ...(config.authPath ? { GROK_AUTH_PATH: config.authPath } : {}) };
|
|
41
|
+
export function isolatedEnv(config) { const env = { ...process.env, HOME: config.grokHome, GROK_HOME: config.grokHome, ...(config.ddFlowHome ? { DD_FLOW_HOME: config.ddFlowHome } : {}), ...(config.resourceHome ? { DD_FLOW_RESOURCE_HOME: config.resourceHome } : {}), ...(config.authPath ? { GROK_AUTH_PATH: config.authPath } : {}) }; env.DD_FLOW_DAEMON_ID = undefined; if (config.ddFlowHome && config.daemonId) env.DD_FLOW_DAEMON_ID = config.daemonId; return env; }
|
|
36
42
|
function within(candidate, root) { const resolved = path.resolve(candidate); const parent = path.resolve(root); return resolved === parent || resolved.startsWith(`${parent}${path.sep}`); }
|
|
37
43
|
|
|
38
44
|
async function writeIsolatedConfig(paths) {
|
|
@@ -42,11 +48,7 @@ async function writeIsolatedConfig(paths) {
|
|
|
42
48
|
|
|
43
49
|
async function inspectIsolatedConfig(config) {
|
|
44
50
|
const target = executable(config.bin, ["inspect", "--json"]);
|
|
45
|
-
const raw = await
|
|
46
|
-
const child = spawn(target.command, target.args, { cwd: config.cwd, env: isolatedEnv(config), stdio: ["ignore", "pipe", "pipe"] }); let stdout = ""; let stderr = "";
|
|
47
|
-
child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); child.stderr.setEncoding("utf8").on("data", (chunk) => (stderr += chunk));
|
|
48
|
-
child.on("error", reject); child.on("close", (code) => code === 0 ? resolve(stdout) : reject(new DaemonError("grok_inspect_failed", stderr.trim() || `${target.command} exited ${code}`)));
|
|
49
|
-
});
|
|
51
|
+
const raw = await commandText(target.command, target.args, { cwd: config.cwd, env: isolatedEnv(config), phase: "native-inspect" });
|
|
50
52
|
let report; try { report = JSON.parse(raw); } catch (error) { throw new DaemonError("grok_inspect_invalid", `grok inspect did not return JSON: ${error.message}`); }
|
|
51
53
|
const foreign = [];
|
|
52
54
|
const checkSource = (kind, value, roots) => { const candidate = value?.source?.path ?? value?.path ?? value?.filePath; if (candidate && !roots.some((root) => within(candidate, root))) foreign.push({ kind, path: candidate }); };
|
|
@@ -147,15 +149,9 @@ async function materializeSessionArchive(paths, archiveDir) {
|
|
|
147
149
|
}
|
|
148
150
|
|
|
149
151
|
export async function callDaemon(stateDir, operation, params = {}, timeoutMs = 30_000) {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
return await new Promise((resolve, reject) => {
|
|
154
|
-
const client = net.createConnection(socket); let buffer = ""; let settled = false;
|
|
155
|
-
const timer = operation === "session.prompt" ? null : observedTimeout(() => finish(new DaemonError("daemon_timeout", `${operation} timed out`, true)), timeoutMs);
|
|
156
|
-
const finish = (error, value) => { if (settled) return; settled = true; clearTimeout(timer); client.destroy(); error ? reject(error) : resolve(value); };
|
|
157
|
-
client.setEncoding("utf8"); client.on("connect", () => client.write(`${JSON.stringify(request)}\n`)); client.on("data", (chunk) => { buffer += chunk; const newline = buffer.indexOf("\n"); if (newline < 0) return; try { const response = JSON.parse(buffer.slice(0, newline)); if (response.schema_id !== RESPONSE_SCHEMA || response.id !== request.id) return finish(new DaemonError("daemon_protocol_mismatch", "invalid daemon response")); if (!response.ok) return finish(new DaemonError(response.error?.code ?? "operation_failed", response.error?.message ?? "daemon operation failed", response.error?.retryable, response.error?.details)); finish(null, response.result); } catch (error) { finish(new DaemonError("daemon_protocol_mismatch", error.message)); } }); client.on("error", (error) => finish(new DaemonError("daemon_not_running", error.message, true))); client.on("end", () => finish(new DaemonError("daemon_connection_closed", `${operation} connection closed before a response`, true))); client.on("close", () => finish(new DaemonError("daemon_connection_closed", `${operation} connection closed before a response`, true)));
|
|
158
|
-
});
|
|
152
|
+
operation = canonicalNativeOperation('grok', operation);
|
|
153
|
+
const request = { schema_id: REQUEST_SCHEMA, id: process.env.DD_EVAL_OPERATION_ID ?? randomUUID(), operation, params };
|
|
154
|
+
return await callDaemonObserved({ stateDir, socket: locations(stateDir).socket, request, responseSchema: RESPONSE_SCHEMA, harness: 'grok', timeoutMs });
|
|
159
155
|
}
|
|
160
156
|
|
|
161
157
|
function sameConfig(state, config) { return ["cwd", "journal", "bin", "authSource", "model", "reasoning", "mode", "ddFlowBin", "ddFlowHome", "projectRoot", "sessionArchive", "resourceHome", "noFlow"].every((key) => (state.config?.[key] ?? null) === (config[key] ?? null)); }
|
|
@@ -178,7 +174,7 @@ export async function startDaemon(options) {
|
|
|
178
174
|
const daemonId = randomUUID();
|
|
179
175
|
const state = { descendants: previous?.descendants ?? [], schema_id: STATE_SCHEMA, daemon_id: daemonId, pid: null, socket: paths.socket, started_at: new Date().toISOString(), shutdown_state: "starting", active_tree: false, recovery_status: previous?.shutdown_state === "running" ? "recovered_idle" : "clean_start", auth_status: copiedAuth ? "copied" : "absent", ...(importedSession ? { imported_session: importedSession } : {}), versions: versions.versions, config_isolation: configIsolation, config: { ...config, daemonId }, sessions: importedSession ? [{ provider_session_id: importedSession.provider_session_id, adapter_session_id: importedSession.provider_session_id, parent_provider_session_id: null, root_provider_session_id: importedSession.provider_session_id, cwd: importedSession.source_cwd }] : (previous?.sessions ?? []), resource_process: resourceProcess };
|
|
180
176
|
await writeState(paths.state, state); const log = await open(paths.log, "a", 0o600); const child = spawn(process.execPath, [absolute(options.entryPath, "dd-grok entry"), "daemon", "serve", "--state-dir", paths.dir], { cwd: config.cwd, env: isolatedEnv(state.config), detached: true, stdio: ["ignore", log.fd, log.fd] }); child.unref(); await log.close();
|
|
181
|
-
try { await confirmDaemonProcess(config, resourceProcess, child); const
|
|
177
|
+
try { await confirmDaemonProcess(config, resourceProcess, child); const startupClock = new ObservationClock({ timeoutMs: 120_000 }); while (!startupClock.sample()) { await new Promise((resolve) => setTimeout(resolve, 100)); const failure = grokStartupFailure(await readState(paths.dir)); if (failure) throw failure; try { const status = await callDaemon(paths.dir, "daemon.ready", {}, 5000); if (status.ready) return status; } catch (error) { if (error.code !== "daemon_not_running" && error.code !== "daemon_timeout") throw error; } } throw new DaemonError("daemon_start_failed", "daemon did not become ready", true); }
|
|
182
178
|
catch (error) { await cleanupFailedDaemonStart(config, resourceProcess, child, error, () => readState(paths.dir), daemonId); throw error; }
|
|
183
179
|
}
|
|
184
180
|
export async function stopDaemon(options) { const timeoutMs = options.timeoutMs ?? 30_000; return await confirmDaemonStopped(() => callDaemon(options.stateDir, "daemon.stop", { cancelTree: options.cancelTree === true }, timeoutMs), locations(options.stateDir).socket, timeoutMs, { readState: () => readState(options.stateDir) }); }
|
|
@@ -236,7 +232,7 @@ export class Runtime {
|
|
|
236
232
|
cwd,
|
|
237
233
|
native_root_receipt: prior?.native_root_receipt ?? null
|
|
238
234
|
});
|
|
239
|
-
this.bridge.registerModelSession?.(id, id, this.state.config, parent);
|
|
235
|
+
this.bridge.registerModelSession?.(id, id, this.state.config, parent, true, newAttempt);
|
|
240
236
|
this.descendants.set(id, {
|
|
241
237
|
provider_session_id: id,
|
|
242
238
|
parent_provider_session_id: parent,
|
|
@@ -340,10 +336,16 @@ export class Runtime {
|
|
|
340
336
|
return running;
|
|
341
337
|
}
|
|
342
338
|
async requireSettled() { const running = await this.refreshTree(); if (running.length) throw new DaemonError("tree_not_settled", "daemon still owns a running Session tree", false, { sessions: running }); }
|
|
343
|
-
async productive(name, task, sessionId) {
|
|
339
|
+
async productive(name, task, sessionId, operationId = null) {
|
|
344
340
|
assertDaemonDispatch(this.paths?.dir);
|
|
345
341
|
if (this.active) throw new DaemonError("operation_busy", `${this.active} is already running`, true);
|
|
346
342
|
this.active = name;
|
|
343
|
+
const current = this.nativeHookOperation = { operationId, sessionId, generation: (this.hookGeneration = (this.hookGeneration ?? 0) + 1), requests: new Map() };
|
|
344
|
+
let rejectHook;
|
|
345
|
+
const hookFailed = new Promise((_, reject) => { rejectHook = reject; });
|
|
346
|
+
void hookFailed.catch(() => {});
|
|
347
|
+
current.reject = error => { if (this.nativeHookOperation !== current || current.error) return; current.error = error; this.bridge.lifecycleFailure ??= error; this.bridge.notificationError = this.bridge.lifecycleFailure; this.bridge.rejectAll(this.bridge.lifecycleFailure); rejectHook(this.bridge.lifecycleFailure); };
|
|
348
|
+
const timer = setInterval(() => { void this.observeHookFailures(current).catch(error => current.reject(error)); }, 100);
|
|
347
349
|
// Only a new native turn invalidates its own root proof. Creation, fork
|
|
348
350
|
// and local archive export must not erase an unrelated Session's receipt.
|
|
349
351
|
if (name === "session.prompt" && this.sessions.has(sessionId)) this.sessions.get(sessionId).native_root_receipt = null;
|
|
@@ -351,7 +353,7 @@ export class Runtime {
|
|
|
351
353
|
await this.persist({ active_tree: true, active_operation: name });
|
|
352
354
|
assertDaemonDispatch(this.paths?.dir);
|
|
353
355
|
const before = new Map(this.descendants);
|
|
354
|
-
const result = await task();
|
|
356
|
+
const result = await Promise.race([task(), hookFailed]);
|
|
355
357
|
this.track(result, before);
|
|
356
358
|
const session = this.sessions.get(result.provider_session_id);
|
|
357
359
|
if (session && (["session.create", "session.fork"].includes(name) || ["end_turn", "cancelled"].includes(result.turn?.stopReason))) session.native_root_receipt = { kind: name === "session.prompt" ? result.turn.stopReason : "created", daemon_id: this.state.daemon_id };
|
|
@@ -369,18 +371,41 @@ export class Runtime {
|
|
|
369
371
|
const session = this.sessions.get(sessionId);
|
|
370
372
|
if (session) session.native_root_receipt = { kind: "failed", daemon_id: this.state.daemon_id, request_id: response.request_id };
|
|
371
373
|
}
|
|
372
|
-
await this.persist({ active_tree: true, active_operation: null });
|
|
374
|
+
try { await hookBudget(1_000, null).run(() => this.persist({ active_tree: true, active_operation: null }), "failure_persistence"); }
|
|
375
|
+
catch (storage) { error.cleanup_error = errorRecord(storage); }
|
|
376
|
+
throw error;
|
|
373
377
|
}
|
|
374
|
-
finally { this.active = null; }
|
|
378
|
+
finally { clearInterval(timer); current.requests.clear(); if (this.nativeHookOperation === current) this.nativeHookOperation = null; this.active = null; }
|
|
379
|
+
}
|
|
380
|
+
async observeHookFailures(current = this.nativeHookOperation) {
|
|
381
|
+
if (!current || this.nativeHookOperation !== current || current.reading) return;
|
|
382
|
+
current.reading = true;
|
|
383
|
+
try { await readNativeHookFailures(this.paths.dir, current.requests, current, error => {
|
|
384
|
+
if (!isProvenShellCorrection(error)) current.reject(error);
|
|
385
|
+
}); } finally { current.reading = false; }
|
|
386
|
+
}
|
|
387
|
+
async acceptHook(identity, sessionId, payload) {
|
|
388
|
+
const input = payload?.tool_input ?? payload?.toolInput;
|
|
389
|
+
const command = input?.command ?? input?.cmd;
|
|
390
|
+
if ((payload?.hook_event_name ?? payload?.hookEventName) !== "PreToolUse" || typeof command !== "string" || (!["lifecycle", "auxiliary"].includes(parseLifecycleCommand(command).participation) && !continuationCommand(command))) return identity;
|
|
391
|
+
const current = this.nativeHookOperation, tool = payload.tool_use_id ?? payload.toolUseId ?? payload.tool_call_id;
|
|
392
|
+
const root = current?.sessionId ?? this.loadedSessionId;
|
|
393
|
+
const pending = [...(this.bridge.pending?.values() ?? [])].some(request => request.method === "session/prompt" && request.sessionId === root);
|
|
394
|
+
if (!current || !pending || identity.rootProviderSessionId !== root || !tool) throw new DaemonError("native_hook_unproven", "Native hook has no exact active Session operation");
|
|
395
|
+
const binding = await hookBudget(1_000, null).run(() => acceptNativeHookRequest(this.paths.dir, { daemon_id: this.state.daemon_id, operation_id: current.operationId, turn_generation: current.generation, provider_session_id: sessionId, root_provider_session_id: root, tool_call_id: tool, event: "PreToolUse", command_sha256: createHash("sha256").update(command).digest("hex"), native_cwd: payload.cwd ?? null }, current), "hook_binding");
|
|
396
|
+
if (this.nativeHookOperation !== current) throw new DaemonError("native_hook_unproven", "Native hook operation changed during acceptance");
|
|
397
|
+
current.requests.set(binding.request_id, { current, binding });
|
|
398
|
+
return { ...identity, nativeHookBinding: binding };
|
|
375
399
|
}
|
|
376
|
-
async dispatch(operation, params) {
|
|
400
|
+
async dispatch(operation, params, operationId = null) {
|
|
377
401
|
const assertDispatch = captureDispatchGuard(this);
|
|
378
402
|
if (["session.create", "session.prompt", "session.fork", "session.archive"].includes(operation)) assertDaemonDispatch(this.paths.dir);
|
|
379
403
|
if (operation === "daemon.ready") return { ready: this.ready, daemon_id: this.state.daemon_id, pid: process.pid, socket: this.paths.socket, versions: this.state.versions, config_isolation: this.state.config_isolation, shutdown_state: this.state.shutdown_state, recovery_status: this.state.recovery_status, auth_status: this.state.auth_status, active_tree: this.state.active_tree, active_operation: this.active, sessions: [...this.sessions.values()], config: this.state.config };
|
|
380
404
|
if (operation === "daemon.status") { let observationError; if (!this.active && !this.state.shutdown?.phases?.tree) try { await this.refreshTree(); } catch (error) { observationError = errorPayload(error); } return { ...await this.dispatch("daemon.ready", {}), observation_error: observationError ?? this.state.observation_error ?? null }; }
|
|
381
|
-
if (operation === "hook.resolve") { const sessionId = String(params.sessionId ?? ""); if (!sessionId) throw new DaemonError("hook_identity_missing", "hook has no sessionId"); let session = this.sessions.get(sessionId); if (!session) { const roots = [...this.sessions.values()].filter((item) => !item.parent_provider_session_id); if (roots.length !== 1) throw new DaemonError("hook_identity_unknown", "unknown hook Session has no unique root"); const observed = await inspectSessionWithBridge(this.bridge, this.options({ sessionId, liveSession: true })); this.track(observed); session = this.sessions.get(sessionId); if (!session || session.root_provider_session_id !== roots[0].provider_session_id) throw new DaemonError("hook_identity_unknown", "hook Session is not a member of the retained native tree"); await this.persist(); } return { daemonId: this.state.daemon_id, rootProviderSessionId: session.root_provider_session_id, parentProviderSessionId: session.parent_provider_session_id ?? null, observedProfile: this.bridge.modelProfiles?.get(sessionId) ?? {} }; }
|
|
382
|
-
if (operation === "
|
|
383
|
-
if (operation === "session.
|
|
405
|
+
if (operation === "hook.resolve") { const sessionId = String(params.sessionId ?? ""); if (!sessionId) throw new DaemonError("hook_identity_missing", "hook has no sessionId"); let session = this.sessions.get(sessionId); if (!session) { const roots = [...this.sessions.values()].filter((item) => !item.parent_provider_session_id); if (roots.length !== 1) throw new DaemonError("hook_identity_unknown", "unknown hook Session has no unique root"); const observed = await inspectSessionWithBridge(this.bridge, this.options({ sessionId, liveSession: true })); this.track(observed); session = this.sessions.get(sessionId); if (!session || session.root_provider_session_id !== roots[0].provider_session_id) throw new DaemonError("hook_identity_unknown", "hook Session is not a member of the retained native tree"); await this.persist(); } return await this.acceptHook({ daemonId: this.state.daemon_id, rootProviderSessionId: session.root_provider_session_id, parentProviderSessionId: session.parent_provider_session_id ?? null, observedProfile: this.bridge.modelProfiles?.get(sessionId) ?? {} }, sessionId, params.payload); }
|
|
406
|
+
if (operation === "hook.failed") { await this.observeHookFailures(); return { observed: true }; }
|
|
407
|
+
if (operation === "session.create") return await this.productive(operation, async () => { const result = await createSessionWithBridge(this.bridge, { ...this.options(params), onSessionCreated: async (session) => { this.track(session); this.loadedSessionId = session.provider_session_id; await this.persist(); } }, this.initialized); this.loadedSessionId = result.provider_session_id; return result; }, null, operationId);
|
|
408
|
+
if (operation === "session.prompt") { await this.requireSettled(); return await this.productive(operation, async () => { const result = await promptSessionWithBridge(this.bridge, { ...this.options(params), assertDispatch, operationId }); this.loadedSessionId = params.sessionId; return result; }, params.sessionId, operationId); }
|
|
384
409
|
if (operation === "session.fork") { await this.requireSettled(); return await this.productive(operation, async () => { const result = await forkSessionWithBridge(this.bridge, this.options(params)); this.loadedSessionId = params.sessionId; return result; }); }
|
|
385
410
|
if (operation === "session.inspect" || operation === "session.resume") { const before = new Map(this.descendants); const result = await inspectSessionWithBridge(this.bridge, this.options({ ...params, liveSession: true })); this.track(result, before); await this.persist(); return { ...result, settled: !this.active && !this.treeRunning(result), descendants: this.topology(result.provider_session_id) }; }
|
|
386
411
|
if (operation === "session.cancel") { if (!this.sessions.has(params.sessionId)) throw new DaemonError("session_identity_mismatch", "cancellation Session is not owned by this daemon"); cancelPendingDispatch(this); const result = await cancelSessionWithBridge(this.bridge, this.options({ ...params, liveSession: true })); await this.persist({ active_tree: Boolean(this.active) || !result.settled }); return result; }
|
|
@@ -441,13 +466,13 @@ export async function serveDaemon(stateDir) {
|
|
|
441
466
|
connection.setEncoding("utf8"); let buffer = "", consumed = false;
|
|
442
467
|
connection.on("data", chunk => {
|
|
443
468
|
if (consumed) return;
|
|
444
|
-
buffer += chunk; const at = buffer.indexOf("\n"); if (at < 0) return; consumed = true;
|
|
469
|
+
buffer += chunk; if (!admitNativeRequestBuffer(buffer, connection, RESPONSE_SCHEMA)) { consumed = true; return; } const at = buffer.indexOf("\n"); if (at < 0) return; consumed = true;
|
|
445
470
|
void (async () => {
|
|
446
471
|
let request;
|
|
447
472
|
try {
|
|
448
473
|
request = JSON.parse(buffer.slice(0, at));
|
|
449
474
|
if (request.schema_id !== REQUEST_SCHEMA || !request.id || !request.operation) throw new DaemonError("daemon_protocol_mismatch", "invalid daemon request");
|
|
450
|
-
const result = await durableDaemonDispatch(paths.dir, request, () => runtime.dispatch(request.operation, request.params ?? {}), () => runtime.dispatch("session.inspect", request.params ?? {}));
|
|
475
|
+
const result = await durableDaemonDispatch(paths.dir, request, () => runtime.dispatch(request.operation, request.params ?? {}, request.id), () => runtime.dispatch("session.inspect", request.params ?? {}));
|
|
451
476
|
if (result._shutdown) retireDaemonEndpoint(server);
|
|
452
477
|
connection.end(`${JSON.stringify({ schema_id: RESPONSE_SCHEMA, id: request.id, ok: true, result: { ...result, _shutdown: undefined } })}\n`);
|
|
453
478
|
} catch (error) {
|
|
@@ -461,3 +486,4 @@ export async function serveDaemon(stateDir) {
|
|
|
461
486
|
await new Promise((resolve, reject) => { server.once("error", reject); server.listen(paths.socket, resolve); }); await chmod(paths.socket, 0o600); await runtime.persist({ shutdown_state: "running", active_tree: false }); runtime.ready = true; return await new Promise(() => {});
|
|
462
487
|
}
|
|
463
488
|
import { authorizeRetainedDaemonResume } from "./driver-recovery.mjs";
|
|
489
|
+
import { commandText } from "./process-json.mjs";
|