@deksden-com/dd-flow-cli 0.9.0-beta.124 → 0.9.0-beta.127
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +10 -1
- package/dist/build-info.json +7 -7
- package/dist/cli/argument-values.js +38 -0
- package/dist/cli/command-inputs.js +4 -3
- package/dist/cli/help.js +1 -0
- package/dist/cli/hook-ingress.js +1 -1
- package/dist/cli/run-cli.js +126 -193
- package/dist/cli/runtime-maintenance-diagnostics.js +46 -0
- package/dist/cli/runtime-maintenance-ingress.js +51 -0
- package/dist/cli/runtime-process.js +128 -0
- package/dist/cli.js +8 -5
- package/dist/harness-runtime/bin/dd-agy.mjs +3 -2
- package/dist/harness-runtime/bin/dd-codex.mjs +3 -2
- package/dist/harness-runtime/bin/dd-droid.mjs +3 -2
- package/dist/harness-runtime/bin/dd-grok.mjs +16 -6
- package/dist/harness-runtime/bin/dd-opencode.mjs +3 -2
- package/dist/harness-runtime/bin/dd-zcode.mjs +20 -16
- package/dist/harness-runtime/lib/adapter-timeouts.d.mts +5 -0
- package/dist/harness-runtime/lib/adapter-timeouts.mjs +34 -0
- package/dist/harness-runtime/lib/daemon-observation.d.mts +2 -0
- package/dist/harness-runtime/lib/daemon-observation.mjs +163 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +60 -13
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +151 -72
- package/dist/harness-runtime/lib/dd-agy.mjs +11 -23
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +9 -14
- package/dist/harness-runtime/lib/dd-codex.mjs +252 -80
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +14 -19
- package/dist/harness-runtime/lib/dd-droid.mjs +117 -31
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +56 -30
- package/dist/harness-runtime/lib/dd-grok.mjs +15 -13
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +18 -27
- package/dist/harness-runtime/lib/dd-opencode.mjs +44 -14
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +56 -53
- package/dist/harness-runtime/lib/dd-zcode.mjs +200 -111
- package/dist/harness-runtime/lib/lease-renewal.d.mts +8 -0
- package/dist/harness-runtime/lib/lease-renewal.mjs +23 -0
- package/dist/harness-runtime/lib/lifecycle-command.d.mts +3 -3
- package/dist/harness-runtime/lib/lifecycle-command.mjs +74 -20
- package/dist/harness-runtime/lib/managed-daemon.d.mts +10 -1
- package/dist/harness-runtime/lib/managed-daemon.mjs +228 -40
- package/dist/harness-runtime/lib/native-children.d.mts +13 -0
- package/dist/harness-runtime/lib/native-children.mjs +78 -0
- package/dist/harness-runtime/lib/native-hook-command.d.mts +4 -0
- package/dist/harness-runtime/lib/native-hook-command.mjs +115 -11
- package/dist/harness-runtime/lib/native-streams.mjs +45 -0
- package/dist/harness-runtime/lib/observation-clock.d.mts +18 -0
- package/dist/harness-runtime/lib/observation-clock.mjs +73 -17
- package/dist/harness-runtime/lib/operation-errors.d.mts +14 -0
- package/dist/harness-runtime/lib/operation-errors.mjs +105 -8
- package/dist/harness-runtime/lib/operation-progress.d.mts +7 -0
- package/dist/harness-runtime/lib/operation-progress.mjs +112 -0
- package/dist/harness-runtime/lib/process-json.mjs +136 -38
- package/dist/harness-runtime/lib/tool-observations.d.mts +1 -0
- package/dist/harness-runtime/lib/tool-observations.mjs +5 -2
- package/dist/harness-runtime/lib/zcode-usage.mjs +3 -2
- package/dist/runtime/context.js +8 -4
- package/dist/schemas/run-control-receipt.schema.json +19 -0
- package/dist/services/code-checks.js +79 -9
- package/dist/services/codex-capacity.js +117 -41
- package/dist/services/codex-hook-delivery.js +2 -2
- package/dist/services/compatibility-preflight.js +4 -1
- package/dist/services/controller-fanout.js +9 -76
- package/dist/services/engines.js +70 -20
- package/dist/services/eval-snapshots.js +179 -40
- package/dist/services/external-work-launch.js +13 -8
- package/dist/services/harness-adapter.js +175 -11
- package/dist/services/hooks.js +65 -21
- package/dist/services/lifecycle-invocations.js +404 -32
- package/dist/services/managed-lease-monitor.js +128 -0
- package/dist/services/managed-processes.js +103 -47
- package/dist/services/merge-server.js +58 -21
- package/dist/services/recovery-observation-budget.js +67 -7
- package/dist/services/recovery-snapshot-database.js +7 -4
- package/dist/services/run-control-receipt.js +1 -1
- package/dist/services/run-control-worker.js +177 -118
- package/dist/services/run-control.js +99 -8
- package/dist/services/run-controller-adapter.js +169 -46
- package/dist/services/run-controller-capture.js +14 -2
- package/dist/services/run-controller-process.js +11 -4
- package/dist/services/run-controller-state.js +68 -7
- package/dist/services/run-controller.js +166 -94
- package/dist/services/run-fork.js +8 -3
- package/dist/services/run-recovery-guard.js +75 -0
- package/dist/services/run-recovery-runtime.js +1 -56
- package/dist/services/run-recovery.js +17 -13
- package/dist/services/runs.js +39 -6
- package/dist/services/runtime-budget.js +32 -15
- package/dist/services/runtime-scope-control.js +8 -0
- package/dist/services/runtime-scope-resume.js +30 -8
- package/dist/services/runtime-scope-worker.js +118 -73
- package/dist/services/runtime-service.js +6 -7
- package/dist/services/scope-writer-cleanup.js +3 -2
- package/dist/services/stage-context.js +135 -0
- package/dist/services/stage-lifecycle.js +3 -15
- package/dist/services/stage-pause.js +3 -2
- package/dist/services/usage.js +14 -3
- package/dist/services/vnext-code-review.js +10 -9
- package/dist/services/vnext-code.js +10 -8
- package/dist/services/vnext-contracts.js +10 -1
- package/dist/services/vnext-execution-profile.js +75 -2
- package/dist/services/vnext-fanout.js +5 -0
- package/dist/services/vnext-merge.js +10 -9
- package/dist/services/vnext-plan-review.js +21 -17
- package/dist/services/vnext-plan.js +9 -7
- package/dist/services/vnext-protocolize.js +15 -9
- package/dist/services/vnext-specify.js +6 -6
- package/dist/services/work-registry.js +34 -4
- package/dist/services/workspace-bootstrap.js +14 -3
- package/dist/shared/errors.js +11 -6
- package/dist/storage/agent-output.js +26 -0
- package/dist/storage/database.js +80 -21
- package/package.json +7 -4
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/** Shared policy only; transports and owner-specific fences stay at callsites. */
|
|
2
|
+
export const RENEWAL_POLICY = Object.freeze({ budgetMs: 30_000, attemptMs: 5_000 });
|
|
3
|
+
export function createRenewalState() { return { uncertainSince: null, attempt: 0, exhausted: false }; }
|
|
4
|
+
export function renewalRemaining(state, now = performance.now()) {
|
|
5
|
+
const remaining = state.exhausted ? 0 : state.uncertainSince === null ? RENEWAL_POLICY.budgetMs : Math.max(0, RENEWAL_POLICY.budgetMs - (now - state.uncertainSince));
|
|
6
|
+
if (remaining === 0) state.exhausted = true;
|
|
7
|
+
return remaining;
|
|
8
|
+
}
|
|
9
|
+
export function renewalFailure(state, now = performance.now()) { state.uncertainSince ??= now; state.attempt++; return renewalRemaining(state, now); }
|
|
10
|
+
export function renewalDelay(state) { return Math.min(2_000, 250 * 2 ** Math.min(3, Math.max(0, state.attempt - 1))); }
|
|
11
|
+
export function renewalConfirmed(state, now = performance.now()) { if (!renewalRemaining(state, now)) return false; state.uncertainSince = null; state.attempt = 0; return true; }
|
|
12
|
+
export function maintenanceRetryable(error) {
|
|
13
|
+
let retryable = false;
|
|
14
|
+
// Inspect only bounded typed causes from our maintenance transport, never text.
|
|
15
|
+
for (let depth = 0; error && depth < 8; depth++, error = error.cause ?? error.details?.cause) {
|
|
16
|
+
if (error.details?.cleanup_unconfirmed === true) return false;
|
|
17
|
+
if (["process_lease_lost", "controller_lease_lost", "process_maintenance_receipt_invalid", "runtime_owner_mismatch"].includes(error.code)) return false;
|
|
18
|
+
if (error.code === "process_ownership_unknown" && error.details?.reason === "physical_identity_unavailable" && ["process_confirmation", "observer_registration"].includes(error.details.phase) && error.details.effect === "no_effect" && error.details.native_dispatch_started === false) retryable = true;
|
|
19
|
+
if (["process_heartbeat_timeout", "process_maintenance_timeout", "hook_deadline_exceeded", "native_hook_timeout", "SQLITE_BUSY", "SQLITE_LOCKED"].includes(error.code)) retryable = true;
|
|
20
|
+
if (error.code === "ERR_SQLITE_ERROR" && [5, 6].includes((error.details?.sqlite_extended_code ?? error.sqlite_extended_code ?? error.errcode) & 255)) retryable = true;
|
|
21
|
+
}
|
|
22
|
+
return retryable;
|
|
23
|
+
}
|
|
@@ -5,13 +5,13 @@ export interface LifecycleInvocation {
|
|
|
5
5
|
operation: LifecycleOperation; executable: string; launcher?: string; argv: string[]; env: Record<string, string>; args: ParsedCommandArgs; command: string;
|
|
6
6
|
rewriteSafe: boolean; wrapped: boolean; heredoc: boolean; sourceRange?: { start: number; end: number }; contexts?: string[]; expectedCwd?: string;
|
|
7
7
|
}
|
|
8
|
-
export type LifecycleAnalysis =
|
|
8
|
+
export type LifecycleAnalysis = ({ participation: "none" | "lifecycle" | "auxiliary" | "unproven"; auxiliary?: { operation: string; args: ParsedCommandArgs; expectedCwd?: string } } & (
|
|
9
9
|
| { kind: "none"; unresolved?: true; reason?: string }
|
|
10
10
|
| { kind: "standalone"; invocation: LifecycleInvocation; supported?: boolean; invocations?: LifecycleInvocation[] }
|
|
11
|
-
| { kind: "compound"; invocation: LifecycleInvocation; reason: string; supported?: boolean; invocations?: LifecycleInvocation[] };
|
|
11
|
+
| { kind: "compound"; invocation: LifecycleInvocation; reason: string; supported?: boolean; invocations?: LifecycleInvocation[] }));
|
|
12
12
|
export function lifecycleCommandFromPayload(payload: { command?: unknown; tool_input?: unknown }): string | undefined;
|
|
13
13
|
export function parseLifecycleCommand(command: string): LifecycleAnalysis;
|
|
14
|
-
export function continuationCommand(command: string): { operation: string; args: ParsedCommandArgs } | null;
|
|
14
|
+
export function continuationCommand(command: string): { operation: string; args: ParsedCommandArgs; expectedCwd?: string } | null;
|
|
15
15
|
export function unwrapShellCommand(command: string): string;
|
|
16
16
|
export function parseCommandArgs(argv: string[]): ParsedCommandArgs;
|
|
17
17
|
export function commandOption(invocation: LifecycleInvocation, name: string): string | undefined;
|
|
@@ -1,13 +1,21 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
|
-
import
|
|
3
|
-
import Bash from "tree-sitter-bash";
|
|
2
|
+
import { createRequire } from "node:module";
|
|
4
3
|
import { parse, quote } from "shell-quote";
|
|
5
4
|
import { lifecycleOperationForArgv } from "./lifecycle-contract.mjs";
|
|
6
5
|
|
|
7
6
|
// One grammar owns structure. shell-quote decodes identified command words;
|
|
8
7
|
// neither parser grants authority or executes expansions.
|
|
9
|
-
const
|
|
10
|
-
parser
|
|
8
|
+
const require = createRequire(import.meta.url);
|
|
9
|
+
let parser;
|
|
10
|
+
function shellParser() {
|
|
11
|
+
if (!parser) {
|
|
12
|
+
const Parser = require("tree-sitter"), Bash = require("tree-sitter-bash");
|
|
13
|
+
const initialized = new Parser();
|
|
14
|
+
initialized.setLanguage(Bash);
|
|
15
|
+
parser = initialized;
|
|
16
|
+
}
|
|
17
|
+
return parser;
|
|
18
|
+
}
|
|
11
19
|
const shells = new Set(["bash", "sh", "zsh"]);
|
|
12
20
|
const repeatedContexts = new Set(["for_statement", "while_statement", "function_definition", "c_style_for_statement"]);
|
|
13
21
|
const dynamicTypes = ["command_substitution", "process_substitution", "arithmetic_expansion", "brace_expression", "extglob_pattern"];
|
|
@@ -97,15 +105,48 @@ function invocationFromWords(values, node, wrapped, env) {
|
|
|
97
105
|
return { operation, executable, ...(launcher ? { launcher } : {}), argv, env, args: parseCommandArgs(argv.slice(operation === "recovery_accept" ? 3 : 2)),
|
|
98
106
|
command: quote(values), rewriteSafe: !wrapped, wrapped, heredoc, contexts, ...(!wrapped ? { sourceRange: { start: node.startIndex, end: node.endIndex } } : {}) };
|
|
99
107
|
}
|
|
100
|
-
export function parseLifecycleCommand(command) {
|
|
108
|
+
export function parseLifecycleCommand(command) {
|
|
109
|
+
const result = analyze(command, false, 0);
|
|
110
|
+
return { ...result, participation: result.kind !== "none" ? "lifecycle" : result.participation ?? (result.auxiliary ? "auxiliary" : result.unresolved ? "unproven" : "none") };
|
|
111
|
+
}
|
|
112
|
+
function auxiliaryFromWords(values) {
|
|
113
|
+
const env = {}, index = launcherIndex(values, env);
|
|
114
|
+
let start = index;
|
|
115
|
+
if (path.basename(values[start] ?? "") === "node" && (path.basename(values[start + 1] ?? "") === "cli.js" || values[start + 1] === "$DD_FLOW_BIN")) start++;
|
|
116
|
+
if (!(path.basename(values[start] ?? "") === "dd-flow" || values[start] === "$DD_FLOW_BIN" || start !== index && path.basename(values[start] ?? "") === "cli.js")) return null;
|
|
117
|
+
const args = parseCommandArgs(values.slice(start + 1)), route = args.positional.slice(0, 3).join(" ");
|
|
118
|
+
const operation = route === "work repair add" ? "work_repair_add" : args.positional[0] === "stage" && args.positional[1] === "block" ? "stage_block" : null;
|
|
119
|
+
return operation ? { operation, args } : null;
|
|
120
|
+
}
|
|
121
|
+
// Recognition is not admission: recovered commands never provide argv authority.
|
|
122
|
+
function recognizedParticipation(root, depth) {
|
|
123
|
+
for (const node of root.descendantsOfType("command")) {
|
|
124
|
+
const name = node.childForFieldName("name");
|
|
125
|
+
if (!name || name.type === "ERROR" || dynamicWord(name)) continue;
|
|
126
|
+
const values = words(node.text);
|
|
127
|
+
if (!values) continue;
|
|
128
|
+
if (invocationFromWords(values, node, true, {})) return "lifecycle";
|
|
129
|
+
if (auxiliaryFromWords(values)) return "auxiliary";
|
|
130
|
+
const index = launcherIndex(values, {});
|
|
131
|
+
if (values[index] === "eval" && depth < 8) {
|
|
132
|
+
const payload = node.namedChildren[index + 1];
|
|
133
|
+
if (payload && !dynamicWord(payload)) {
|
|
134
|
+
const nested = analyze(values.slice(index + 1).join(" "), true, depth + 1);
|
|
135
|
+
if (nested.kind !== "none" || nested.participation === "lifecycle") return "lifecycle";
|
|
136
|
+
if (nested.auxiliary || nested.participation === "auxiliary") return "auxiliary";
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return "unproven";
|
|
141
|
+
}
|
|
101
142
|
function analyze(command, wrapped, depth) {
|
|
102
143
|
if (depth > 8 || command.length > 1_000_000) return { kind: "none", unresolved: true, reason: "shell_input_limit" };
|
|
103
|
-
const tree =
|
|
144
|
+
const tree = shellParser().parse(command);
|
|
104
145
|
if (!tree) return { kind: "none", unresolved: true, reason: "shell_parse_error" };
|
|
105
146
|
try {
|
|
106
147
|
const root = tree.rootNode;
|
|
107
|
-
if (root.hasError) return { kind: "none", unresolved: true, reason: "shell_parse_error" };
|
|
108
|
-
const nodes = root.descendantsOfType("command"), found = [];
|
|
148
|
+
if (root.hasError) return { kind: "none", unresolved: true, participation: recognizedParticipation(root, depth), reason: "shell_parse_error" };
|
|
149
|
+
const nodes = root.descendantsOfType("command"), found = [], auxiliaries = [];
|
|
109
150
|
let unsupported;
|
|
110
151
|
for (const node of nodes) {
|
|
111
152
|
const name = node.childForFieldName("name"), values = words(node.text);
|
|
@@ -122,6 +163,13 @@ function analyze(command, wrapped, depth) {
|
|
|
122
163
|
}
|
|
123
164
|
}
|
|
124
165
|
const direct = invocationFromWords(values, node, wrapped, inherited);
|
|
166
|
+
const auxiliary = auxiliaryFromWords(values);
|
|
167
|
+
if (auxiliary) {
|
|
168
|
+
const route = executionCwd(node, nodes);
|
|
169
|
+
if (dynamicWord(node) || backgroundContext(node) || ancestors(node).some(type => repeatedContexts.has(type) || ["command_substitution", "process_substitution"].includes(type)) || route.unresolved) unsupported ??= "unsupported_auxiliary_execution";
|
|
170
|
+
auxiliaries.push({ ...auxiliary, ...(route.cwd ? { expectedCwd: route.cwd } : {}) });
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
125
173
|
if (direct) {
|
|
126
174
|
if (backgroundContext(node)) unsupported ??= "background_lifecycle_execution";
|
|
127
175
|
if (node.namedChildren.some(dynamicWord)) unsupported ??= "unresolved_lifecycle_word";
|
|
@@ -140,6 +188,7 @@ function analyze(command, wrapped, depth) {
|
|
|
140
188
|
if (values[nestedIndex] === "--") nestedIndex++;
|
|
141
189
|
if (wrapper === "timeout" && /^\d+(?:\.\d+)?[smhd]?$/.test(values[nestedIndex] ?? "")) nestedIndex++;
|
|
142
190
|
const nested = analyze(quote(values.slice(nestedIndex)), true, depth + 1);
|
|
191
|
+
if (nested.auxiliary || ["lifecycle", "auxiliary"].includes(nested.participation)) return { kind: "none", participation: nested.auxiliary ? "auxiliary" : nested.participation, unresolved: true, reason: "opaque_execution_wrapper" };
|
|
143
192
|
if (nested.kind !== "none") { found.push(...(nested.invocations ?? [nested.invocation])); unsupported ??= "opaque_execution_wrapper"; continue; }
|
|
144
193
|
if (nested.unresolved) unsupported ??= nested.reason;
|
|
145
194
|
}
|
|
@@ -148,6 +197,12 @@ function analyze(command, wrapped, depth) {
|
|
|
148
197
|
if (!payloadNode || dynamicWord(payloadNode)) { unsupported ??= "dynamic_shell_payload"; continue; }
|
|
149
198
|
if (node.namedChildren.some(child => child !== payloadNode && dynamicWord(child))) unsupported ??= "unresolved_shell_environment";
|
|
150
199
|
const nested = analyze(values[shellIndex + 2], true, depth + 1);
|
|
200
|
+
if (nested.auxiliary) {
|
|
201
|
+
const route = executionCwd(node, nodes);
|
|
202
|
+
if (route.unresolved || backgroundContext(node) || ancestors(node).some(type => repeatedContexts.has(type) || ["command_substitution", "process_substitution"].includes(type))) unsupported ??= "unsupported_auxiliary_execution";
|
|
203
|
+
auxiliaries.push({ ...nested.auxiliary, ...(route.cwd && !nested.auxiliary.expectedCwd ? { expectedCwd: route.cwd } : {}) });
|
|
204
|
+
}
|
|
205
|
+
if (nested.participation && ["lifecycle", "auxiliary"].includes(nested.participation) && nested.unresolved) return { kind: "none", unresolved: true, participation: nested.participation, reason: nested.reason };
|
|
151
206
|
if (nested.kind !== "none") {
|
|
152
207
|
const route = executionCwd(node, nodes);
|
|
153
208
|
if (route.unresolved) unsupported ??= "unresolved_execution_cwd";
|
|
@@ -162,10 +217,18 @@ function analyze(command, wrapped, depth) {
|
|
|
162
217
|
if (nested.supported === false) unsupported ??= nested.reason ?? "unsupported_wrapper";
|
|
163
218
|
}
|
|
164
219
|
else if (nested.unresolved) unsupported ??= nested.reason;
|
|
165
|
-
} else if (values[shellIndex] === "eval")
|
|
220
|
+
} else if (values[shellIndex] === "eval") {
|
|
221
|
+
const participation = recognizedParticipation(root, depth);
|
|
222
|
+
if (participation !== "unproven") return { kind: "none", unresolved: true, participation, reason: "dynamic_eval" };
|
|
223
|
+
unsupported ??= "dynamic_eval";
|
|
224
|
+
}
|
|
166
225
|
else if (values[shellIndex] === "coproc") unsupported ??= "background_lifecycle_execution";
|
|
167
226
|
else if (dynamicWord(name)) unsupported ??= "dynamic_executable";
|
|
168
227
|
}
|
|
228
|
+
if (auxiliaries.length) {
|
|
229
|
+
if (found.length || auxiliaries.length !== 1) unsupported ??= "multiple_lifecycle_invocations";
|
|
230
|
+
if (!found.length) return { kind: "none", participation: "auxiliary", ...(unsupported ? { unresolved: true, reason: unsupported } : { auxiliary: auxiliaries[0] }) };
|
|
231
|
+
}
|
|
169
232
|
if (!found.length) return unsupported ? { kind: "none", unresolved: true, reason: unsupported } : { kind: "none" };
|
|
170
233
|
if (found.length > 1) unsupported ??= "multiple_lifecycle_invocations";
|
|
171
234
|
const invocation = found[0];
|
|
@@ -180,20 +243,11 @@ export function lifecycleCommandFromPayload(payload) {
|
|
|
180
243
|
return typeof command === "string" ? unwrapShellCommand(command) : undefined;
|
|
181
244
|
}
|
|
182
245
|
export function continuationCommand(command) {
|
|
183
|
-
|
|
184
|
-
const start = values.findIndex(word => ["stage", "work"].includes(word)); if (start < 1) return null;
|
|
185
|
-
const launcher = values.slice(0, start);
|
|
186
|
-
while (launcher[0] && /^[A-Za-z_][A-Za-z0-9_]*=/.test(launcher[0])) launcher.shift();
|
|
187
|
-
const direct = launcher.length === 1 && (path.basename(launcher[0]) === "dd-flow" || launcher[0] === "$DD_FLOW_BIN");
|
|
188
|
-
const node = launcher.length === 2 && path.basename(launcher[0]) === "node" && (path.basename(launcher[1]) === "cli.js" || launcher[1] === "$DD_FLOW_BIN");
|
|
189
|
-
if (!direct && !node) return null;
|
|
190
|
-
const args = parseCommandArgs(values.slice(start)), route = args.positional.slice(0, 3).join(" ");
|
|
191
|
-
const operation = route === "work repair add" ? "work_repair_add" : args.positional[0] === "stage" && args.positional[1] === "block" ? "stage_block" : null;
|
|
192
|
-
return operation ? { operation, args } : null;
|
|
246
|
+
return parseLifecycleCommand(command).auxiliary ?? null;
|
|
193
247
|
}
|
|
194
248
|
export function unwrapShellCommand(command) {
|
|
195
249
|
const values = words(command); if (!values || values.length !== 3 || !shells.has(path.basename(values[0])) || !/^-(?:l?c|cl)$/.test(values[1])) return command;
|
|
196
|
-
const tree =
|
|
250
|
+
const tree = shellParser().parse(command);
|
|
197
251
|
try { const node = tree?.rootNode.namedChildren[0]?.namedChildren[2]; return node && !dynamicWord(node) ? values[2] : command; }
|
|
198
252
|
finally { /* Native Tree lifetime is managed by the Node binding. */ }
|
|
199
253
|
}
|
|
@@ -1,2 +1,11 @@
|
|
|
1
1
|
import type { ChildProcess } from "node:child_process";
|
|
2
|
-
|
|
2
|
+
type RuntimeConfig = Record<string, unknown>;
|
|
3
|
+
type ProcessBinding = { id: string; lease_token: string; kind?: string; lease_expires_at?: string; registration_sha256?: string };
|
|
4
|
+
export function runtimeProcess(config: RuntimeConfig, action: string, options?: Record<string, unknown>, transport?: { timeoutMs?: number; signal?: AbortSignal }): Promise<Record<string, unknown> | null>;
|
|
5
|
+
export function registerDaemonProcess(config: RuntimeConfig, input: { kind: string; [key: string]: unknown }): Promise<ProcessBinding | null>;
|
|
6
|
+
export function confirmDaemonProcess(config: RuntimeConfig, record: ProcessBinding | null | undefined, child: Pick<ChildProcess, "pid"> | null | undefined): Promise<void>;
|
|
7
|
+
export function assertDaemonOwnership(stateDir: string, dispatch?: Record<string, unknown>, assertCurrent?: () => void): Promise<void>;
|
|
8
|
+
export function heartbeatDaemonProcess(config: RuntimeConfig, record: ProcessBinding | null | undefined): Promise<void>;
|
|
9
|
+
export function finishDaemonProcess(config: RuntimeConfig, record: ProcessBinding | null | undefined, state?: string, reason?: string): Promise<void>;
|
|
10
|
+
export function completeBudgetOperation(stateDir: string, request: { id: string; operation: string; params?: { sessionId?: string } }, result: unknown, observe?: () => Promise<Record<string, unknown>>): Promise<Record<string, unknown>>;
|
|
11
|
+
export function stopProcessGroup(child: ChildProcess | null | undefined, graceMs?: number, options?: { detached?: boolean; settlementMs?: number }): Promise<void>;
|
|
@@ -5,8 +5,13 @@ import { commandJson } from "./process-json.mjs";
|
|
|
5
5
|
import { errorRecord } from "./operation-errors.mjs";
|
|
6
6
|
import { writeJsonAtomic } from "./runner-events.mjs";
|
|
7
7
|
import { runBoundedCommand } from "./native-hook-command.mjs";
|
|
8
|
+
import { createRenewalState, maintenanceRetryable, renewalConfirmed, renewalDelay, renewalFailure, renewalRemaining, RENEWAL_POLICY } from "./lease-renewal.mjs";
|
|
8
9
|
|
|
9
10
|
const leases = new Map();
|
|
11
|
+
// A rejected Promise is not physical exit. Do not start another writer for
|
|
12
|
+
// this exact binding after an unconfirmed maintenance subtree cleanup.
|
|
13
|
+
const maintenancePoison = new Map();
|
|
14
|
+
const maintenanceFlights = new Map();
|
|
10
15
|
|
|
11
16
|
/** Freeze resource ownership independently of whether Subject hooks are enabled. */
|
|
12
17
|
export function bindRuntimeOwner(config, env = process.env) {
|
|
@@ -38,6 +43,7 @@ export function bindRuntimeOwner(config, env = process.env) {
|
|
|
38
43
|
|
|
39
44
|
/** Reject rebinding before provider setup, including the already-running path. */
|
|
40
45
|
export async function prepareRuntimeOwner(config, stateDir, env = process.env) {
|
|
46
|
+
config.stateDir = path.resolve(stateDir);
|
|
41
47
|
const owner = bindRuntimeOwner(config, env);
|
|
42
48
|
if (owner && path.resolve(owner.state_dir) !== path.resolve(stateDir)) {
|
|
43
49
|
throw Object.assign(new Error("runtime owner belongs to another state directory"), { code: "runtime_owner_mismatch" });
|
|
@@ -52,27 +58,44 @@ export async function prepareRuntimeOwner(config, stateDir, env = process.env) {
|
|
|
52
58
|
return owner;
|
|
53
59
|
}
|
|
54
60
|
|
|
55
|
-
export async function assertDaemonOwnership(stateDir, dispatch = {}, assertCurrent = () => {}) {
|
|
56
|
-
|
|
61
|
+
export async function assertDaemonOwnership(stateDir, dispatch = {}, assertCurrent = () => {}, progress = async () => {}, waiting = async () => {}) {
|
|
62
|
+
const gate = async () => {
|
|
63
|
+
assertCurrent();
|
|
64
|
+
for (const lease of leases.values()) {
|
|
65
|
+
if (lease.stateDir !== path.resolve(stateDir)) continue;
|
|
66
|
+
await reconfirmLease(lease, assertCurrent);
|
|
67
|
+
assertCurrent();
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
await gate();
|
|
71
|
+
await progress('ownership_confirmed');
|
|
57
72
|
const state = JSON.parse(await readFile(path.join(stateDir, "daemon.json"), "utf8"));
|
|
58
73
|
const owner = bindRuntimeOwner({ runtime_owner: state.config?.runtime_owner ?? null }, {});
|
|
59
74
|
if (!owner) return;
|
|
60
75
|
const record = state.resource_process;
|
|
61
76
|
if (owner.state_dir !== stateDir || !record?.id || !record.lease_token) throw Object.assign(new Error("owned daemon has no confirmed resource binding"), { code: "process_ownership_unknown" });
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if (Date.now() >= lease.expiresAt) throw Object.assign(new Error("managed process renewal is overdue"), { code: "process_ownership_unknown", details: { process_id: lease.record.id, cause: lease.error ? errorRecord(lease.error) : null } });
|
|
65
|
-
// A failed renewal is not a lost lease. Reconfirm before productive work;
|
|
66
|
-
// never dispatch on the old lease merely because it has time remaining.
|
|
67
|
-
if (lease.error && !lease.lost) await heartbeatDaemonProcess(lease.config, lease.record);
|
|
68
|
-
if (lease.error) throw lease.error;
|
|
69
|
-
}
|
|
77
|
+
scheduleDaemonProcessHeartbeat(state.config, record);
|
|
78
|
+
const admissionRenewal = createRenewalState();
|
|
70
79
|
for (;;) {
|
|
80
|
+
await gate();
|
|
81
|
+
admissionRenewal.uncertainSince ??= performance.now();
|
|
82
|
+
let result;
|
|
83
|
+
try { result = await runtimeProcess(state.config, "admission", { id: record.id, "lease-token": record.lease_token, owner: owner.owner_id, "state-dir": stateDir, "daemon-id": state.daemon_id ?? state.config?.daemonId, ...dispatch }, { timeoutMs: renewalRemaining(admissionRenewal) }); }
|
|
84
|
+
catch (error) {
|
|
85
|
+
if (!maintenanceRetryable(error) || !renewalFailure(admissionRenewal)) throw error;
|
|
86
|
+
await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(admissionRenewal), renewalRemaining(admissionRenewal))));
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
71
89
|
assertCurrent();
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
90
|
+
if (result?.ok !== true || result.process_id !== record.id || result.admitted !== true && result.reason !== "provider_capacity") throw Object.assign(new Error("productive runtime admission was not confirmed"), { code: "process_ownership_unknown" });
|
|
91
|
+
requireTimelyMaintenance(admissionRenewal, "admission", result, record);
|
|
92
|
+
// This bound ACK ends admission uncertainty. The post-ACK lease checks
|
|
93
|
+
// have their own episodes; successful renewal must not exhaust an already
|
|
94
|
+
// confirmed admission while the provider/daemon still retain ownership.
|
|
95
|
+
if (!renewalConfirmed(admissionRenewal)) requireTimelyMaintenance(admissionRenewal, "admission", result, record);
|
|
96
|
+
await gate();
|
|
97
|
+
await waiting({ ...result, lease_expires_at: result.lease_expires_at ?? record.lease_expires_at });
|
|
98
|
+
if (result.admitted === true) { await progress('admission_confirmed'); return; }
|
|
76
99
|
const predecessor = result.blocking_turns?.find(turn => turn?.process_id === record.id
|
|
77
100
|
&& turn?.session_id === dispatch["native-session-id"]
|
|
78
101
|
&& turn?.operation_id !== dispatch["operation-id"]);
|
|
@@ -153,20 +176,92 @@ async function budgetState(stateDir) {
|
|
|
153
176
|
|
|
154
177
|
async function releaseBudget(state, options) {
|
|
155
178
|
const record = state.resource_process, owner = state.config.runtime_owner;
|
|
156
|
-
const receipt = await runtimeProcess(state.config, "release-turn", { id: record.id, "lease-token": record.lease_token, owner: owner.owner_id, "daemon-id": state.daemon_id ?? state.config?.daemonId, ...options });
|
|
179
|
+
const receipt = await retryBookkeeping(timeoutMs => runtimeProcess(state.config, "release-turn", { id: record.id, "lease-token": record.lease_token, owner: owner.owner_id, "daemon-id": state.daemon_id ?? state.config?.daemonId, ...options }, { timeoutMs }));
|
|
157
180
|
if (receipt?.ok !== true) throw Object.assign(new Error("runtime budget release was not confirmed"), { code: "runtime_budget_release_unconfirmed" });
|
|
158
181
|
}
|
|
159
182
|
|
|
160
|
-
export async function runtimeProcess(config, action, options = {}) {
|
|
183
|
+
export async function runtimeProcess(config, action, options = {}, { timeoutMs = RENEWAL_POLICY.budgetMs, signal } = {}) {
|
|
184
|
+
const resourceHome = config.runtime_owner?.resource_home ?? config.resourceHome;
|
|
185
|
+
if (action === "status" || !options.id || !resourceHome) return executeRuntimeProcess(config, action, options, { timeoutMs, signal });
|
|
186
|
+
const key = JSON.stringify([path.resolve(resourceHome), options.id, options["lease-token"] ?? null]);
|
|
187
|
+
const validBudget = Number.isFinite(timeoutMs) && timeoutMs > 0;
|
|
188
|
+
const deadline = performance.now() + (validBudget ? timeoutMs : 0);
|
|
189
|
+
const prior = maintenanceFlights.get(key);
|
|
190
|
+
let started = false;
|
|
191
|
+
let settle;
|
|
192
|
+
const flight = new Promise(resolve => { settle = resolve; });
|
|
193
|
+
maintenanceFlights.set(key, flight);
|
|
194
|
+
try {
|
|
195
|
+
if (!validBudget) throw Object.assign(new Error("Managed maintenance budget is invalid or exhausted before dispatch"), { code: "process_maintenance_timeout", details: { effect: "no_effect", native_dispatch_started: false } });
|
|
196
|
+
if (prior) {
|
|
197
|
+
let timer;
|
|
198
|
+
try {
|
|
199
|
+
await Promise.race([prior, new Promise((_, reject) => {
|
|
200
|
+
const expire = () => {
|
|
201
|
+
const remaining = deadline - performance.now();
|
|
202
|
+
// A timer wakeup is not proof that its monotonic deadline passed.
|
|
203
|
+
if (remaining > 0) { timer = setTimeout(expire, Math.min(2_147_483_647, Math.ceil(remaining))); return; }
|
|
204
|
+
reject(Object.assign(new Error("managed maintenance queue deadline expired"), { code: "process_maintenance_timeout", details: { effect: "no_effect" } }));
|
|
205
|
+
};
|
|
206
|
+
expire();
|
|
207
|
+
})]);
|
|
208
|
+
} finally { clearTimeout(timer); }
|
|
209
|
+
}
|
|
210
|
+
const remaining = deadline - performance.now();
|
|
211
|
+
if (remaining <= 0) throw Object.assign(new Error("managed maintenance queue deadline expired"), { code: "process_maintenance_timeout", details: { effect: "no_effect" } });
|
|
212
|
+
started = true;
|
|
213
|
+
return await executeRuntimeProcess(config, action, options, { timeoutMs: remaining, signal });
|
|
214
|
+
} catch (error) {
|
|
215
|
+
const lease = leases.get(key);
|
|
216
|
+
const text = value => typeof value === "string" ? value.slice(0, 128) : undefined;
|
|
217
|
+
const request = { phase: started ? "request" : "queue", action, process_id: text(options.id), operation_id: text(options["operation-id"] ?? config.runtime_owner?.operation_id), role: text(lease?.record.kind ?? config.runtime_owner?.role), attempt_kind: prior ? "join" : "fresh", attempt: (lease?.renewal.attempt ?? 0) + 1, remaining_ms: Math.max(0, Math.round(deadline - performance.now())), native_dispatch_started: false, maintenance_effect: started ? "unknown" : "no_effect" };
|
|
218
|
+
// Parent queue/import failures precede any child trace. Keep only bounded,
|
|
219
|
+
// credential-free context and the existing child phase allowlist.
|
|
220
|
+
const childPhases = Array.isArray(error.details?.hook_diagnostics) ? error.details.hook_diagnostics : [];
|
|
221
|
+
error.details = { ...error.details, hook_diagnostics: [request, ...childPhases.slice(-15)] };
|
|
222
|
+
throw error;
|
|
223
|
+
} finally {
|
|
224
|
+
// Even a waiter whose deadline expires must retain its predecessor's
|
|
225
|
+
// physical fence until that predecessor settles.
|
|
226
|
+
const finish = () => {
|
|
227
|
+
settle();
|
|
228
|
+
if (maintenanceFlights.get(key) === flight) maintenanceFlights.delete(key);
|
|
229
|
+
};
|
|
230
|
+
if (prior) void prior.then(finish); else finish();
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function executeRuntimeProcess(config, action, options = {}, { timeoutMs, signal }) {
|
|
161
235
|
const owner = config.runtime_owner;
|
|
162
236
|
const resourceHome = owner?.resource_home ?? config.resourceHome;
|
|
163
237
|
const ddFlowHome = owner?.dd_flow_home ?? config.ddFlowHome;
|
|
164
238
|
const ddFlowBin = owner?.dd_flow_bin ?? config.ddFlowBin;
|
|
165
239
|
if (!resourceHome || !ddFlowHome || !ddFlowBin) return null;
|
|
240
|
+
const poisonKey = token => JSON.stringify([path.resolve(resourceHome), options.id, token ?? null]);
|
|
241
|
+
if (action !== "status" && options.id) {
|
|
242
|
+
const poison = maintenancePoison.get(poisonKey(options["lease-token"])) ?? maintenancePoison.get(poisonKey(null));
|
|
243
|
+
if (poison) throw poison;
|
|
244
|
+
}
|
|
166
245
|
const args = ["runtime", "process", action];
|
|
167
246
|
for (const [key, value] of Object.entries(options)) if (value !== null && value !== undefined) args.push(`--${key}`, String(value));
|
|
168
|
-
if (
|
|
169
|
-
|
|
247
|
+
if (["register", "confirm", "heartbeat", "admission", "check-admission", "finish", "release-turn", "status"].includes(action)) {
|
|
248
|
+
const maintenanceStarted = performance.now();
|
|
249
|
+
process.stderr.write(`${JSON.stringify({ kind: "process_maintenance_phase", phase: "request", observed_at: new Date().toISOString(), operation_id: options['operation-id'] ?? config.runtime_owner?.operation_id ?? null, action, process_id: options.id ?? null, remaining_ms: Math.round(timeoutMs), native_dispatch_started: false })}\n`);
|
|
250
|
+
let output;
|
|
251
|
+
try { output = await runBoundedCommand({ bin: ddFlowBin, args: [...args, "--json"], cwd: config.cwd,
|
|
252
|
+
env: { ...(config.env ?? {}), DD_FLOW_HOME: ddFlowHome, DD_FLOW_RESOURCE_HOME: resourceHome }, timeoutMs,
|
|
253
|
+
timeoutCode: action === "heartbeat" ? "process_heartbeat_timeout" : "process_maintenance_timeout", inheritDeadline: false, cleanupWithinBudget: true,
|
|
254
|
+
maintenanceProgress: { action, processId: options.id ?? null, inactivityMs: RENEWAL_POLICY.attemptMs }, signal }); }
|
|
255
|
+
catch (error) {
|
|
256
|
+
if (action !== "status" && options.id && error.details?.cleanup_unconfirmed) maintenancePoison.set(poisonKey(options["lease-token"]), error);
|
|
257
|
+
throw error;
|
|
258
|
+
}
|
|
259
|
+
let result;
|
|
260
|
+
try { result = JSON.parse(output); } catch { throw Object.assign(new Error("Invalid managed process maintenance response"), { code: "process_maintenance_receipt_invalid", details: { action } }); }
|
|
261
|
+
if (result?.ok === false && result.error?.code) throw Object.assign(new Error(result.error.message ?? "Managed process maintenance failed"), { code: result.error.code, details: result.error.details });
|
|
262
|
+
process.stderr.write(`${JSON.stringify({ kind: "process_maintenance_phase", phase: "ack", observed_at: new Date().toISOString(), duration_ms: Math.round(performance.now() - maintenanceStarted), operation_id: options['operation-id'] ?? config.runtime_owner?.operation_id ?? null, action, process_id: options.id ?? null, lease_expires_at: result.lease_expires_at ?? result.process?.lease_expires_at ?? null, native_dispatch_started: false })}\n`);
|
|
263
|
+
return result;
|
|
264
|
+
}
|
|
170
265
|
return await commandJson(ddFlowBin, args, {
|
|
171
266
|
cwd: config.cwd,
|
|
172
267
|
env: { ...(config.env ?? {}), DD_FLOW_HOME: ddFlowHome, DD_FLOW_RESOURCE_HOME: resourceHome }
|
|
@@ -177,8 +272,38 @@ export async function registerDaemonProcess(config, input) {
|
|
|
177
272
|
const owner = bindRuntimeOwner(config);
|
|
178
273
|
const harness = ({ "codex-daemon": "codex-desktop", "codex-provider": "codex-desktop", "zcode-daemon": "zcode-acp", "zcode-provider": "zcode-acp", "grok-daemon": "grok-acp", "grok-provider": "grok-acp", "agy-daemon": "antigravity-cli", "opencode-daemon": "opencode-server", "droid-daemon": "droid-cli" })[input.kind];
|
|
179
274
|
const scope = owner ? { owner: owner.owner_id, "project-id": owner.project_id, run: owner.run_id, work: owner.work_id, role: owner.role, "state-dir": owner.state_dir, "adapter-executable": owner.adapter_executable, ...(owner.budget ? { "budget-json": JSON.stringify(owner.budget), harness } : {}), ...(owner.project_id && owner.run_id || owner.budget ? { "dispatch-barrier": "run-generation@1" } : {}), operation: `${owner.operation_id}:${input.operation ?? input.kind}` } : {};
|
|
180
|
-
const
|
|
181
|
-
|
|
275
|
+
const options = { ...input, ...scope, id: `PROC-${input.kind}-${randomUUID()}`, "owner-pid": process.pid, "lease-ms": 300_000 };
|
|
276
|
+
const renewal = createRenewalState();
|
|
277
|
+
renewal.uncertainSince = performance.now();
|
|
278
|
+
let result;
|
|
279
|
+
for (;;) {
|
|
280
|
+
try { result = await runtimeProcess(config, "register", options, { timeoutMs: renewalRemaining(renewal) }); break; }
|
|
281
|
+
catch (error) {
|
|
282
|
+
if (!maintenanceRetryable(error) || !renewalFailure(renewal)) throw error;
|
|
283
|
+
// The child has exited before a timeout is retryable. Reconcile the exact
|
|
284
|
+
// startup identity, not an arbitrary PID or another operation's owner.
|
|
285
|
+
let status;
|
|
286
|
+
try { status = await retryBookkeeping(timeoutMs => runtimeProcess(config, "status", {}, { timeoutMs }), renewal); }
|
|
287
|
+
catch (reconciliationError) {
|
|
288
|
+
// Retain the failed write's phases without changing the reconciliation
|
|
289
|
+
// failure's classification or authorizing another startup attempt.
|
|
290
|
+
reconciliationError.details = { ...reconciliationError.details, registration_error: errorRecord(error) };
|
|
291
|
+
throw reconciliationError;
|
|
292
|
+
}
|
|
293
|
+
const retained = status?.processes?.find(record => record.id === options.id);
|
|
294
|
+
if (retained) {
|
|
295
|
+
let metadata; try { metadata = JSON.parse(retained.metadata_json); } catch { /* Invalid binding below. */ }
|
|
296
|
+
if (retained.kind !== options.kind || retained.owner_id !== options.owner || retained.operation_id !== options.operation || retained.state !== "starting" || metadata?.owner_pid !== process.pid) throw Object.assign(new Error("Startup reconciliation changed managed process binding"), { code: "process_maintenance_receipt_invalid" });
|
|
297
|
+
result = { process: retained }; break;
|
|
298
|
+
}
|
|
299
|
+
if (!Array.isArray(status?.processes)) throw Object.assign(new Error("Startup reconciliation has no registry proof"), { code: "process_maintenance_receipt_invalid" });
|
|
300
|
+
await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(renewal), renewalRemaining(renewal))));
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
if (!result) return null;
|
|
304
|
+
if (result.process?.id !== options.id || !result.process.lease_token) throw Object.assign(new Error("Managed startup receipt belongs to another registration"), { code: "process_maintenance_receipt_invalid" });
|
|
305
|
+
requireTimelyMaintenance(renewal, "register", result, result.process);
|
|
306
|
+
return { id: result.process.id, lease_token: result.process.lease_token, kind: input.kind, lease_expires_at: result.process.lease_expires_at };
|
|
182
307
|
}
|
|
183
308
|
|
|
184
309
|
export async function confirmDaemonProcess(config, record, child) {
|
|
@@ -187,7 +312,11 @@ export async function confirmDaemonProcess(config, record, child) {
|
|
|
187
312
|
await finishDaemonProcess(config, record, "failed", "daemon_spawn_missing_pid");
|
|
188
313
|
return;
|
|
189
314
|
}
|
|
190
|
-
|
|
315
|
+
const renewal = createRenewalState();
|
|
316
|
+
const result = await retryBookkeeping(timeoutMs => runtimeProcess(config, "confirm", { id: record.id, "lease-token": record.lease_token, pid: child.pid, "process-group-id": child.pid, "owner-pid": record.kind?.endsWith("-daemon") ? child.pid : process.pid, "lease-ms": 300_000 }, { timeoutMs }), renewal);
|
|
317
|
+
if (!result?.process || result.process.id !== record.id || result.process.lease_token !== record.lease_token) throw Object.assign(new Error("Managed process confirmation binding is invalid"), { code: "process_maintenance_receipt_invalid" });
|
|
318
|
+
retainReceipt(record, result.process);
|
|
319
|
+
requireTimelyMaintenance(renewal, "confirm", result, record);
|
|
191
320
|
}
|
|
192
321
|
|
|
193
322
|
/** Provider bridges use a different process group from their daemon. Publish
|
|
@@ -217,29 +346,40 @@ export async function closeManagedBridge(config, state, bridge) {
|
|
|
217
346
|
|
|
218
347
|
export function scheduleDaemonProcessHeartbeat(config, record) {
|
|
219
348
|
if (!record?.id || !record.lease_token) return;
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
349
|
+
const key = leaseKey(config, record);
|
|
350
|
+
if (leases.has(key)) return;
|
|
351
|
+
const lease = { pending: null, error: null, timer: null, lost: false, closing: false, expiresAt: Date.parse(record.lease_expires_at ?? ""), renewal: createRenewalState(), stateDir: path.resolve(config.runtime_owner?.state_dir ?? config.stateDir ?? config.cwd), config, record };
|
|
352
|
+
leases.set(key, lease);
|
|
353
|
+
// Retry inside the same uncertainty episode while native work is active.
|
|
354
|
+
// Waiting for the next 60s tick would exhaust the 30s budget after one BUSY.
|
|
355
|
+
lease.timer = setInterval(() => reconfirmLease(lease, () => {}).catch(() => {}), 60_000);
|
|
224
356
|
lease.timer.unref();
|
|
225
357
|
}
|
|
226
358
|
|
|
227
359
|
export async function heartbeatDaemonProcess(config, record) {
|
|
228
360
|
if (!record?.id || !record.lease_token) return;
|
|
229
361
|
scheduleDaemonProcessHeartbeat(config, record);
|
|
230
|
-
const lease = leases.get(record
|
|
362
|
+
const lease = leases.get(leaseKey(config, record));
|
|
363
|
+
if (lease.closing) throw Object.assign(new Error("Managed process is closing"), { code: "process_closing" });
|
|
231
364
|
if (lease.lost) throw lease.error;
|
|
365
|
+
if (lease.error && !maintenanceRetryable(lease.error)) throw lease.error;
|
|
366
|
+
if (!renewalRemaining(lease.renewal)) throw lease.error;
|
|
232
367
|
if (lease.pending) return await lease.pending;
|
|
233
368
|
lease.pending = (async () => {
|
|
234
369
|
try {
|
|
235
|
-
|
|
236
|
-
|
|
370
|
+
lease.renewal.uncertainSince ??= performance.now();
|
|
371
|
+
const result = await runtimeProcess(config, "heartbeat", { id: record.id, "lease-token": record.lease_token, "lease-ms": 300_000 }, { timeoutMs: renewalRemaining(lease.renewal) });
|
|
372
|
+
if (result?.ok === false) {
|
|
237
373
|
lease.lost = true; clearInterval(lease.timer);
|
|
238
374
|
throw Object.assign(new Error("managed process lease is no longer owned; new productive operations are blocked"), { code: "process_lease_lost", details: { process_id: record.id } });
|
|
239
375
|
}
|
|
240
|
-
|
|
241
|
-
lease.
|
|
376
|
+
if (result?.ok !== true || result.process_id !== record.id) throw Object.assign(new Error("Managed renewal response belongs to another process"), { code: "process_maintenance_receipt_invalid", details: { process_id: record.id } });
|
|
377
|
+
if (!lease.closing) {
|
|
378
|
+
if (!renewalConfirmed(lease.renewal)) requireTimelyMaintenance(lease.renewal, "heartbeat", result, record);
|
|
379
|
+
retainReceipt(record, result); lease.error = null; lease.expiresAt = Date.parse(record.lease_expires_at);
|
|
380
|
+
}
|
|
242
381
|
} catch (cause) {
|
|
382
|
+
if (!lease.closing) renewalFailure(lease.renewal);
|
|
243
383
|
lease.error = cause.code === "process_lease_lost" ? cause : Object.assign(new Error("cannot confirm managed process ownership"), { code: "process_ownership_unknown", cause, details: { process_id: record.id, cause: errorRecord(cause) } });
|
|
244
384
|
throw lease.error;
|
|
245
385
|
}
|
|
@@ -249,13 +389,59 @@ export async function heartbeatDaemonProcess(config, record) {
|
|
|
249
389
|
|
|
250
390
|
export async function finishDaemonProcess(config, process, state = "stopped", reason = "daemon_stopped") {
|
|
251
391
|
if (process?.id && process.lease_token) {
|
|
252
|
-
const
|
|
253
|
-
const
|
|
392
|
+
const settlement = createRenewalState(); settlement.uncertainSince = performance.now();
|
|
393
|
+
const key = leaseKey(config, process), lease = leases.get(key);
|
|
394
|
+
if (lease) { lease.closing = true; clearInterval(lease.timer); await lease.pending?.catch(() => {}); if (lease.error?.cause?.details?.cleanup_unconfirmed) throw lease.error; }
|
|
395
|
+
const result = await retryBookkeeping(timeoutMs => runtimeProcess(config, "finish", { id: process.id, "lease-token": process.lease_token, state, reason }, { timeoutMs }), settlement);
|
|
254
396
|
if (result?.ok !== true) throw Object.assign(new Error("managed process finish was not accepted"), { code: "process_lease_lost", details: { process_id: process.id } });
|
|
255
|
-
leases.delete(
|
|
397
|
+
leases.delete(key);
|
|
256
398
|
}
|
|
257
399
|
}
|
|
258
400
|
|
|
401
|
+
function leaseKey(config, record) { return JSON.stringify([path.resolve(config.runtime_owner?.resource_home ?? config.resourceHome), record.id, record.lease_token]); }
|
|
402
|
+
function retainReceipt(record, receipt) {
|
|
403
|
+
const expiresAt = Date.parse(receipt.lease_expires_at ?? "");
|
|
404
|
+
if (!Number.isFinite(expiresAt) || !/^[a-f0-9]{64}$/.test(receipt.registration_sha256 ?? "") || record.registration_sha256 && record.registration_sha256 !== receipt.registration_sha256) throw Object.assign(new Error("Managed lease receipt has no authoritative expiry/binding"), { code: "process_maintenance_receipt_invalid", details: { process_id: record.id } });
|
|
405
|
+
record.lease_expires_at = receipt.lease_expires_at;
|
|
406
|
+
record.registration_sha256 = receipt.registration_sha256;
|
|
407
|
+
}
|
|
408
|
+
async function reconfirmLease(lease, assertCurrent) {
|
|
409
|
+
for (;;) {
|
|
410
|
+
assertCurrent();
|
|
411
|
+
if (lease.closing || lease.lost) throw lease.error ?? Object.assign(new Error("Managed process is closing"), { code: "process_closing" });
|
|
412
|
+
try { await heartbeatDaemonProcess(lease.config, lease.record); }
|
|
413
|
+
catch (error) {
|
|
414
|
+
if (!maintenanceRetryable(error) || !renewalRemaining(lease.renewal)) throw error;
|
|
415
|
+
await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(lease.renewal), renewalRemaining(lease.renewal))));
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
assertCurrent();
|
|
419
|
+
if (lease.closing || lease.error || !Number.isFinite(lease.expiresAt) || Date.now() >= lease.expiresAt) throw lease.error ?? Object.assign(new Error("Committed lease receipt expired before admission"), { code: "process_ownership_unknown" });
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
async function retryBookkeeping(action, renewal = createRenewalState()) {
|
|
424
|
+
renewal.uncertainSince ??= performance.now();
|
|
425
|
+
for (;;) {
|
|
426
|
+
try { return await action(renewalRemaining(renewal)); }
|
|
427
|
+
catch (error) { if (!maintenanceRetryable(error) || !renewalFailure(renewal)) throw error; await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(renewal), renewalRemaining(renewal)))); }
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// A committed ACK is evidence, but cannot revive an expired authority episode.
|
|
432
|
+
// Cleanup-only finish/release receipts remain facts, not productive grants.
|
|
433
|
+
function requireTimelyMaintenance(renewal, action, receipt, record) {
|
|
434
|
+
if (renewalRemaining(renewal)) return;
|
|
435
|
+
const safeText = value => typeof value === "string" ? value.slice(0, 128) : undefined;
|
|
436
|
+
const hash = receipt?.process?.registration_sha256 ?? receipt?.registration_sha256 ?? record?.registration_sha256;
|
|
437
|
+
throw Object.assign(new Error("Managed ownership confirmation arrived after its uncertainty budget"), {
|
|
438
|
+
code: "process_ownership_unknown", details: { action, process_id: safeText(record?.id ?? receipt?.process_id),
|
|
439
|
+
registration_sha256: typeof hash === "string" && /^[a-f0-9]{64}$/.test(hash) ? hash : undefined,
|
|
440
|
+
lease_expires_at: safeText(receipt?.process?.lease_expires_at ?? receipt?.lease_expires_at ?? record?.lease_expires_at),
|
|
441
|
+
effect: action === "admission" && receipt?.admitted !== true ? "no_effect" : "committed", native_dispatch_started: false }
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
|
|
259
445
|
/** Preserve the startup error and retain the lease/resources if cleanup cannot
|
|
260
446
|
* prove termination. A failed cleanup is not a terminal managed process. */
|
|
261
447
|
export async function cleanupFailedStart(config, record, child, original) {
|
|
@@ -287,7 +473,7 @@ export async function cleanupFailedDaemonStart(config, daemonRecord, daemonChild
|
|
|
287
473
|
}
|
|
288
474
|
|
|
289
475
|
const processStops = new WeakMap();
|
|
290
|
-
export async function closeBridgeProcess(bridge, drain) {
|
|
476
|
+
export async function closeBridgeProcess(bridge, drain, stopOptions = {}) {
|
|
291
477
|
if (bridge.closePending) return await bridge.closePending;
|
|
292
478
|
const pending = (async () => {
|
|
293
479
|
let primary;
|
|
@@ -301,7 +487,7 @@ export async function closeBridgeProcess(bridge, drain) {
|
|
|
301
487
|
const timer = setTimeout(done, 2_000); child.once("exit", done);
|
|
302
488
|
});
|
|
303
489
|
}
|
|
304
|
-
await stopProcessGroup(child, 2_000);
|
|
490
|
+
await stopProcessGroup(child, 2_000, stopOptions);
|
|
305
491
|
bridge.physicallyClosed = true;
|
|
306
492
|
}
|
|
307
493
|
} catch (error) { primary = error; }
|
|
@@ -316,14 +502,14 @@ export async function closeBridgeProcess(bridge, drain) {
|
|
|
316
502
|
finally { bridge.closePending = null; }
|
|
317
503
|
}
|
|
318
504
|
|
|
319
|
-
export async function stopProcessGroup(child, graceMs = 1_000, { detached = true } = {}) {
|
|
505
|
+
export async function stopProcessGroup(child, graceMs = 1_000, { detached = true, settlementMs = 10_000 } = {}) {
|
|
320
506
|
if (!child?.pid) return;
|
|
321
507
|
if (processStops.has(child)) return await processStops.get(child);
|
|
322
|
-
const pending = stopOwnedChild(child, graceMs, detached);
|
|
508
|
+
const pending = stopOwnedChild(child, graceMs, detached, settlementMs);
|
|
323
509
|
processStops.set(child, pending);
|
|
324
510
|
try { return await pending; } finally { processStops.delete(child); }
|
|
325
511
|
}
|
|
326
|
-
async function stopOwnedChild(child, graceMs, detached) {
|
|
512
|
+
async function stopOwnedChild(child, graceMs, detached, settlementMs) {
|
|
327
513
|
const target = process.platform === "win32" || !detached ? child.pid : -child.pid;
|
|
328
514
|
const details = signal => ({ pid: child.pid, pgid: target < 0 ? child.pid : null, signal, platform: process.platform, exit_code: child.exitCode ?? null, signal_code: child.signalCode ?? null, observed_at: new Date().toISOString() });
|
|
329
515
|
const alive = () => { try { process.kill(target, 0); return true; } catch (error) { if (error.code === "ESRCH") return false; if (error.code === "EPERM") return true; throw error; } };
|
|
@@ -351,8 +537,10 @@ async function stopOwnedChild(child, graceMs, detached) {
|
|
|
351
537
|
if (!alive()) return;
|
|
352
538
|
const leaderExited = () => child.exitCode != null || child.signalCode != null;
|
|
353
539
|
const observeLeaderless = async () => {
|
|
354
|
-
|
|
355
|
-
|
|
540
|
+
// Signal grace is not a drain deadline. Helpers can retire naturally after
|
|
541
|
+
// their leader exits; observing them does not authorize another signal.
|
|
542
|
+
if (await gone(Math.max(1_000, graceMs, settlementMs))) return;
|
|
543
|
+
throw Object.assign(new Error("process group remains after its leader exited; ownership cannot be proven from this child handle"), { code: "process_group_ownership_unknown", details: { ...details(null), settlement_ms: Math.max(1_000, graceMs, settlementMs) } });
|
|
356
544
|
};
|
|
357
545
|
if (leaderExited()) return await observeLeaderless();
|
|
358
546
|
if (!await signal("SIGTERM")) return;
|