@deksden-com/dd-flow-cli 0.9.0-beta.7 → 0.9.0-beta.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +396 -0
- package/README.md +65 -0
- package/dist/build-info.json +10 -10
- package/dist/cli/help.js +81 -9
- package/dist/cli/run-cli.js +442 -46
- package/dist/harness-runtime/bin/dd-agy.mjs +37 -0
- package/dist/harness-runtime/bin/dd-codex.mjs +25 -0
- package/dist/harness-runtime/bin/dd-droid.mjs +33 -0
- package/dist/harness-runtime/bin/dd-grok.mjs +30 -0
- package/dist/harness-runtime/bin/dd-opencode.mjs +21 -0
- package/dist/harness-runtime/bin/dd-zcode.mjs +83 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +215 -0
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +423 -0
- package/dist/harness-runtime/lib/dd-agy.mjs +59 -0
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +141 -0
- package/dist/harness-runtime/lib/dd-codex.mjs +438 -0
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +125 -0
- package/dist/harness-runtime/lib/dd-droid.mjs +470 -0
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +274 -0
- package/dist/harness-runtime/lib/dd-grok.mjs +174 -0
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +198 -0
- package/dist/harness-runtime/lib/dd-opencode.mjs +98 -0
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +644 -0
- package/dist/harness-runtime/lib/dd-zcode.mjs +910 -0
- package/dist/harness-runtime/lib/dispatch-fence.mjs +14 -0
- package/dist/harness-runtime/lib/driver-recovery.mjs +129 -0
- package/dist/harness-runtime/lib/droid-observation.mjs +66 -0
- package/dist/harness-runtime/lib/managed-daemon.mjs +260 -0
- package/dist/harness-runtime/lib/model-observations.mjs +77 -0
- package/dist/harness-runtime/lib/native-hook-command.mjs +34 -0
- package/dist/harness-runtime/lib/observation-clock.mjs +36 -0
- package/dist/harness-runtime/lib/operation-context.mjs +5 -0
- package/dist/harness-runtime/lib/operation-errors.mjs +19 -0
- package/dist/harness-runtime/lib/process-json.mjs +70 -0
- package/dist/harness-runtime/lib/process-snapshot.mjs +10 -0
- package/dist/harness-runtime/lib/runner-events.mjs +200 -0
- package/dist/harness-runtime/lib/runner-lock.mjs +42 -0
- package/dist/harness-runtime/lib/session-settlement.mjs +30 -0
- package/dist/harness-runtime/lib/tool-observations.d.mts +17 -0
- package/dist/harness-runtime/lib/tool-observations.mjs +153 -0
- package/dist/runtime/context.js +3 -1
- package/dist/schemas/agent-profile.schema.json +1 -1
- package/dist/schemas/code-review-result.schema.json +2 -2
- package/dist/schemas/code-work-batch.schema.json +4 -3
- package/dist/schemas/code-work-result.schema.json +4 -4
- package/dist/schemas/harness-config.schema.json +23 -0
- package/dist/schemas/plan-review-result.schema.json +2 -2
- package/dist/schemas/run-control-receipt.schema.json +99 -0
- package/dist/schemas/run-control-request.schema.json +36 -0
- package/dist/schemas/vnext-protocol-plan.schema.json +2 -2
- package/dist/services/cleanup.js +8 -5
- package/dist/services/cli-operation-classifier.js +11 -1
- package/dist/services/code-checks.js +543 -96
- package/dist/services/controller-fanout.js +120 -0
- package/dist/services/dashboard.js +9 -0
- package/dist/services/engines.js +50 -19
- package/dist/services/eval-snapshots.js +639 -60
- package/dist/services/execution-policy.js +177 -0
- package/dist/services/external-work-launch.js +110 -0
- package/dist/services/harness-adapter.js +147 -24
- package/dist/services/harness-config.js +68 -0
- package/dist/services/hooks.js +413 -168
- package/dist/services/lanes.js +1 -1
- package/dist/services/lifecycle-command.js +74 -6
- package/dist/services/lifecycle-invocations.js +637 -0
- package/dist/services/managed-daemon-binding.js +38 -0
- package/dist/services/managed-processes.js +138 -22
- package/dist/services/merge-queue.js +6 -6
- package/dist/services/merge-server.js +75 -29
- package/dist/services/migrations.js +1 -1
- package/dist/services/native-daemon-history.js +49 -0
- package/dist/services/native-session-control.js +39 -0
- package/dist/services/projects.js +7 -3
- package/dist/services/prompts.js +4 -2
- package/dist/services/protocols.js +11 -2
- package/dist/services/recovery-observation-budget.js +61 -0
- package/dist/services/recovery-snapshot-database.js +107 -0
- package/dist/services/run-control-receipt.js +56 -0
- package/dist/services/run-control-worker.js +363 -0
- package/dist/services/run-control.js +876 -0
- package/dist/services/run-controller-adapter.js +211 -0
- package/dist/services/run-controller-capture.js +134 -0
- package/dist/services/run-controller-process.js +189 -0
- package/dist/services/run-controller-recovery.js +222 -0
- package/dist/services/run-controller-state.js +31 -0
- package/dist/services/run-controller.js +733 -0
- package/dist/services/run-engine-bindings.js +20 -62
- package/dist/services/run-fork.js +124 -0
- package/dist/services/run-observations.js +112 -0
- package/dist/services/run-recovery-runtime.js +69 -0
- package/dist/services/run-recovery.js +327 -0
- package/dist/services/runs.js +280 -40
- package/dist/services/runtime-budget.js +362 -0
- package/dist/services/runtime-scope-capture.js +50 -0
- package/dist/services/runtime-scope-control.js +450 -0
- package/dist/services/runtime-scope-resume.js +543 -0
- package/dist/services/runtime-scope-stop.js +99 -0
- package/dist/services/runtime-scope-worker.js +210 -0
- package/dist/services/runtime-service.js +96 -0
- package/dist/services/schema-validation.js +12 -12
- package/dist/services/stage-context.js +2 -2
- package/dist/services/stage-lifecycle.js +32 -10
- package/dist/services/stage-pause.js +25 -13
- package/dist/services/usage.js +37 -93
- package/dist/services/vnext-code-review.js +149 -89
- package/dist/services/vnext-code.js +223 -215
- package/dist/services/vnext-execution-profile.js +5 -3
- package/dist/services/vnext-fanout.js +135 -21
- package/dist/services/vnext-merge.js +258 -89
- package/dist/services/vnext-plan-review.js +75 -64
- package/dist/services/vnext-plan.js +77 -29
- package/dist/services/vnext-protocolize.js +21 -14
- package/dist/services/vnext-specify.js +26 -16
- package/dist/services/work-registry.js +592 -154
- package/dist/services/workspace-bootstrap.js +76 -0
- package/dist/shared/errors.js +12 -0
- package/dist/storage/database.js +341 -31
- package/dist/storage/writer-contract.js +77 -0
- package/dist/storage/writer-migration.js +102 -0
- package/package.json +20 -13
- package/tools/repair-paused-run-status.mjs +59 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { AppError } from "../shared/errors.js";
|
|
5
|
+
import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
|
|
6
|
+
import { runtimeHarness } from "./execution-policy.js";
|
|
7
|
+
import { appendEvent, digest, digestBytes, errorRecord } from "./run-controller-state.js";
|
|
8
|
+
/** A paused RUN admits only its controller's exact accepted answer Turn. */
|
|
9
|
+
export function controllerHitlAdmission(context, record, input) {
|
|
10
|
+
if (input.operation !== "session.prompt" || !input.operationId || !input.promptSha256)
|
|
11
|
+
return false;
|
|
12
|
+
if (!context.db.get("SELECT 1 FROM sqlite_schema WHERE name = 'run_controller_operations'"))
|
|
13
|
+
return false;
|
|
14
|
+
const row = context.db.get("SELECT * FROM run_controllers WHERE controller_id = ? AND project_id = ? AND run_id = ? AND status = 'waiting_for_user'", [record.owner_id, record.project_id, record.run_id]);
|
|
15
|
+
if (!row || row.generation !== JSON.parse(record.metadata_json).run_generation)
|
|
16
|
+
return false;
|
|
17
|
+
const suffix = `:prompt:${row.generation}`;
|
|
18
|
+
if (!input.operationId.endsWith(suffix))
|
|
19
|
+
return false;
|
|
20
|
+
const state = JSON.parse(row.state_json);
|
|
21
|
+
const session = state.current_session === null ? undefined : state.sessions[state.current_session];
|
|
22
|
+
if (!session || session.stopped || session.id !== input.nativeSessionId || session.state_dir !== input.stateDir || session.owner.owner_id !== record.owner_id)
|
|
23
|
+
return false;
|
|
24
|
+
const operation = (id) => context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'dispatching'", [id, row.controller_id, row.generation]);
|
|
25
|
+
const answer = context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation <= ? AND status = 'accepted'", [input.operationId.slice(0, -suffix.length), row.controller_id, row.generation]);
|
|
26
|
+
const prompt = operation(input.operationId);
|
|
27
|
+
if (!answer || !prompt)
|
|
28
|
+
return false;
|
|
29
|
+
const accepted = JSON.parse(answer.input_json);
|
|
30
|
+
const dispatch = JSON.parse(prompt.input_json);
|
|
31
|
+
const run = context.db.get("SELECT index_json FROM runs WHERE project_id = ? AND id = ?", [record.project_id, record.run_id]);
|
|
32
|
+
const stage = run && JSON.parse(run.index_json).stage_runs.find(item => item.stage === accepted.stage);
|
|
33
|
+
if (accepted.action !== "user_answer" || state.stage !== accepted.stage || stage?.status !== "paused" || !stage.pause || stage.pause.answer_path || stage.pause.id !== accepted.pause_id || stage.pause.work_id !== accepted.work_id)
|
|
34
|
+
return false;
|
|
35
|
+
if (answer.input_hash !== digest({ pause_id: accepted.pause_id, answer_sha256: accepted.sha256 }) || prompt.input_hash !== digest(dispatch))
|
|
36
|
+
return false;
|
|
37
|
+
const value = (option) => dispatch.args.includes(option) ? dispatch.args[dispatch.args.indexOf(option) + 1] : undefined;
|
|
38
|
+
if (dispatch.owner !== record.owner_id || dispatch.state_dir !== input.stateDir || dispatch.args[0] !== "session" || dispatch.args[1] !== "prompt" || value("--session-id") !== input.nativeSessionId || dispatch.prompt_sha256 !== input.promptSha256 || !value("--prompt-file"))
|
|
39
|
+
return false;
|
|
40
|
+
return digestBytes(fs.readFileSync(accepted.file)) === accepted.sha256 && digestBytes(fs.readFileSync(value("--prompt-file"))) === input.promptSha256;
|
|
41
|
+
}
|
|
42
|
+
export function sessionArguments(context, manifest, session) {
|
|
43
|
+
const profile = session.profile;
|
|
44
|
+
return ["--dd-flow-bin", manifest.flow_executable, "--dd-flow-home", context.ddFlowHome, "--project-root", manifest.request.project_root, "--state-dir", session.state_dir, "--journal", session.journal, "--cwd", session.cwd, "--provider", profile.provider, "--model", profile.model, "--reasoning", profile.reasoning, "--mode", profile.mode, "--permission", profile.permission, ...manifest.adapters[runtimeHarness(profile)].runtime_args, "--json"];
|
|
45
|
+
}
|
|
46
|
+
export function controllerEnvironment(context, manifest, session) {
|
|
47
|
+
const env = { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined };
|
|
48
|
+
for (const key of ["DD_FLOW_CONTROLLER_TOKEN", "DD_FLOW_CONTROL_TOKEN", "DD_FLOW_DAEMON_ID", "DD_FLOW_RUNTIME_OWNER", "DD_FLOW_WORK_ID", "DD_FLOW_SESSION_ID", "CODEX_THREAD_ID"])
|
|
49
|
+
delete env[key];
|
|
50
|
+
if (session?.codex_home)
|
|
51
|
+
env.CODEX_HOME = session.codex_home;
|
|
52
|
+
return env;
|
|
53
|
+
}
|
|
54
|
+
function settlement(value) {
|
|
55
|
+
const state = value.settlement;
|
|
56
|
+
return state && typeof state === "object" && !Array.isArray(state) ? state : null;
|
|
57
|
+
}
|
|
58
|
+
function settlementInspectArgs(args) {
|
|
59
|
+
const values = ["session", "inspect"];
|
|
60
|
+
for (let index = 2; index < args.length; index += 1) {
|
|
61
|
+
const value = args[index];
|
|
62
|
+
if (["--prompt", "--prompt-file", "--answers-file", "--target-json"].includes(value)) {
|
|
63
|
+
index += 1;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
values.push(value);
|
|
67
|
+
}
|
|
68
|
+
return values;
|
|
69
|
+
}
|
|
70
|
+
function settlementRetryArgs(args, operationId) {
|
|
71
|
+
const stateDir = args[args.indexOf("--state-dir") + 1];
|
|
72
|
+
if (!stateDir)
|
|
73
|
+
throw new AppError("settlement_recovery_required", "Harness settlement has no state directory", 1, { operation_id: operationId });
|
|
74
|
+
return ["daemon", "operation", "settle", "--state-dir", stateDir, "--operation-id", operationId, "--json"];
|
|
75
|
+
}
|
|
76
|
+
function operationInspectArgs(args, operationId) {
|
|
77
|
+
const stateDir = args[args.indexOf("--state-dir") + 1];
|
|
78
|
+
if (!stateDir)
|
|
79
|
+
throw new AppError("controller_operation_binding_unknown", "Harness operation has no state directory", 1, { operation_id: operationId });
|
|
80
|
+
return ["daemon", "operation", "--state-dir", stateDir, "--operation-id", operationId, "--json"];
|
|
81
|
+
}
|
|
82
|
+
function expectedMethod(args) {
|
|
83
|
+
return `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`;
|
|
84
|
+
}
|
|
85
|
+
function validateReceipt(receipt, session, args) {
|
|
86
|
+
// `daemon start --session-id` names the retained state being reopened; the
|
|
87
|
+
// start receipt itself is a daemon lifecycle result and need not echo a
|
|
88
|
+
// provider Session identity. Only session operations promise that binding.
|
|
89
|
+
const target = args[0] === "session" && args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
|
|
90
|
+
if (target && adapterSessionId(receipt) !== target)
|
|
91
|
+
throw new AppError("session_identity_mismatch", "Managed adapter reply belongs to another or unidentified Session", 1);
|
|
92
|
+
if (receipt.harness !== undefined && receipt.harness !== runtimeHarness(session.profile))
|
|
93
|
+
throw new AppError("profile_integrity_violation", "Managed adapter returned another harness identity", 1);
|
|
94
|
+
const fork = receipt.fork_receipt;
|
|
95
|
+
const forkUnproven = (args[0] === "session" && args[1] === "fork" && receipt.fork_continuity_error !== undefined)
|
|
96
|
+
|| (fork && typeof fork === "object" && !Array.isArray(fork) && fork.history_prefix_proven === false);
|
|
97
|
+
if (forkUnproven) {
|
|
98
|
+
throw new AppError("fork_continuity_unproven", "Native fork exists but its source-history continuity is not proven", 1);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
async function recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal) {
|
|
102
|
+
// Creation has no stable client Session id yet. Its durable daemon journal
|
|
103
|
+
// is reconciled by run-control after the owner is fenced; probing it from the
|
|
104
|
+
// failing transport path would be a second adapter invocation and can itself
|
|
105
|
+
// be mistaken for replay.
|
|
106
|
+
if (args[0] === "session" && args[1] === "create")
|
|
107
|
+
return null;
|
|
108
|
+
// A detached adapter without a native daemon has no durable operation
|
|
109
|
+
// journal to inspect. Avoid a second adapter invocation in that case; the
|
|
110
|
+
// original failure remains outcome-unknown and the controller will fence it.
|
|
111
|
+
if (!fs.existsSync(path.join(session.state_dir, "daemon.json")))
|
|
112
|
+
return null;
|
|
113
|
+
let observed;
|
|
114
|
+
try {
|
|
115
|
+
observed = await runHarnessAdapter({ executable, args: operationInspectArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:observe`, runtimeOwner: session.owner, timeoutMs: 30_000, ...(signal ? { signal } : {}) });
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
|
|
121
|
+
const result = observed.result;
|
|
122
|
+
if (observed.operation_id !== operationId || observed.operation !== expectedMethod(args) || observed.state !== "completed"
|
|
123
|
+
|| !result || typeof result !== "object" || Array.isArray(result)
|
|
124
|
+
|| target && (observed.session_id !== target || adapterSessionId(result) !== target))
|
|
125
|
+
return null;
|
|
126
|
+
return { ...result, ...(settlement(observed) ? { settlement: settlement(observed) } : {}) };
|
|
127
|
+
}
|
|
128
|
+
async function awaitSettlement(context, row, manifest, session, executable, args, operationId, initial, assertOwner, signal) {
|
|
129
|
+
let current = initial, delay = 250;
|
|
130
|
+
const deadline = Date.now() + 120_000;
|
|
131
|
+
while (["pending", "blocked"].includes(String(settlement(current)?.state))) {
|
|
132
|
+
if (signal?.aborted)
|
|
133
|
+
throw new AppError("harness_settlement_interrupted", "Harness settlement was interrupted", 1, { operation_id: operationId });
|
|
134
|
+
if (Date.now() >= deadline)
|
|
135
|
+
throw new AppError("settlement_recovery_required", "Completed provider operation requires settlement recovery", 1, { operation_id: operationId, settlement: settlement(current) });
|
|
136
|
+
await new Promise(resolve => setTimeout(resolve, Math.min(delay, Math.max(0, deadline - Date.now()))));
|
|
137
|
+
delay = Math.min(delay * 2, 8_000);
|
|
138
|
+
assertOwner?.();
|
|
139
|
+
if (Date.now() >= deadline)
|
|
140
|
+
continue;
|
|
141
|
+
try {
|
|
142
|
+
// This only retries durable bookkeeping against the same operation
|
|
143
|
+
// receipt. It cannot replay the provider request.
|
|
144
|
+
current = await runHarnessAdapter({ executable, args: settlementRetryArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement`, runtimeOwner: session.owner, timeoutMs: Math.min(30_000, Math.max(1, deadline - Date.now())), ...(signal ? { signal } : {}) });
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
|
|
148
|
+
}
|
|
149
|
+
if (!(["pending", "blocked"].includes(String(settlement(current)?.state))))
|
|
150
|
+
continue;
|
|
151
|
+
// A receipt without final tree proof needs a fresh native inspection. Do
|
|
152
|
+
// it only after the receipt-only retry, and only when this is a session.
|
|
153
|
+
if (!args.includes("--session-id") || !["tree_unsettled", "settlement_evidence_required"].includes(String(settlement(current)?.reason)) || Date.now() >= deadline)
|
|
154
|
+
continue;
|
|
155
|
+
try {
|
|
156
|
+
current = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-inspect`, runtimeOwner: session.owner, timeoutMs: Math.min(30_000, Math.max(1, deadline - Date.now())), ...(signal ? { signal } : {}) });
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return current;
|
|
163
|
+
}
|
|
164
|
+
export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal) {
|
|
165
|
+
assertOwner?.();
|
|
166
|
+
const operationId = retainedOperationId ?? `${row.controller_id}:${crypto.randomUUID()}`;
|
|
167
|
+
const executable = manifest.adapters[runtimeHarness(session.profile)].executable;
|
|
168
|
+
const promptFile = args.indexOf("--prompt-file"), inlinePrompt = args.indexOf("--prompt");
|
|
169
|
+
const promptSha256 = promptFile >= 0 ? digestBytes(fs.readFileSync(args[promptFile + 1])) : inlinePrompt >= 0 ? digestBytes(args[inlinePrompt + 1] ?? "") : undefined;
|
|
170
|
+
const input = { executable, args, owner: session.owner.owner_id, state_dir: session.state_dir, ...(args[0] === "session" && args[1] === "prompt" && promptSha256 ? { prompt_sha256: promptSha256 } : {}) };
|
|
171
|
+
const now = context.now();
|
|
172
|
+
context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'dispatching', ?, ?)", [operationId, row.controller_id, row.generation, digest(input), JSON.stringify(input), now, now]);
|
|
173
|
+
appendEvent(context, row.controller_id, "operation_requested", { operation_id: operationId, operation: args.slice(0, 2), session_id: session.id || null });
|
|
174
|
+
try {
|
|
175
|
+
assertOwner?.();
|
|
176
|
+
let receipt = await runHarnessAdapter({ executable, args, env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs: args[0] === "session" && args[1] === "prompt" ? null : 120_000, ...(signal ? { signal } : {}) });
|
|
177
|
+
if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
|
|
178
|
+
const recovered = await awaitSettlement(context, row, manifest, session, executable, args, operationId, receipt, assertOwner, signal);
|
|
179
|
+
receipt = { ...receipt, settlement: settlement(recovered) };
|
|
180
|
+
}
|
|
181
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
|
|
182
|
+
validateReceipt(receipt, session, args);
|
|
183
|
+
context.db.run("UPDATE run_controller_operations SET status = 'completed', updated_at = ? WHERE operation_id = ?", [context.now(), operationId]);
|
|
184
|
+
assertOwner?.();
|
|
185
|
+
appendEvent(context, row.controller_id, "operation_completed", { operation_id: operationId, operation: args.slice(0, 2), session_id: adapterSessionId(receipt) });
|
|
186
|
+
return receipt;
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
const retained = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
|
|
190
|
+
if (retained) {
|
|
191
|
+
let receipt = retained;
|
|
192
|
+
if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
|
|
193
|
+
const recovered = await awaitSettlement(context, row, manifest, session, executable, args, operationId, receipt, assertOwner, signal);
|
|
194
|
+
receipt = { ...receipt, settlement: settlement(recovered) };
|
|
195
|
+
}
|
|
196
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
|
|
197
|
+
validateReceipt(receipt, session, args);
|
|
198
|
+
context.db.run("UPDATE run_controller_operations SET status = 'completed', error_json = NULL, updated_at = ? WHERE operation_id = ?", [context.now(), operationId]);
|
|
199
|
+
assertOwner?.();
|
|
200
|
+
appendEvent(context, row.controller_id, "operation_recovered", { operation_id: operationId, operation: args.slice(0, 2), session_id: adapterSessionId(receipt) });
|
|
201
|
+
return receipt;
|
|
202
|
+
}
|
|
203
|
+
try {
|
|
204
|
+
context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
|
|
205
|
+
}
|
|
206
|
+
catch (persistence) {
|
|
207
|
+
process.stderr.write(`${JSON.stringify({ operation_id: operationId, primary: errorRecord(error), persistence_error: errorRecord(persistence) })}\n`);
|
|
208
|
+
}
|
|
209
|
+
throw error;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
4
|
+
import { AppError } from "../shared/errors.js";
|
|
5
|
+
import { createEvalRunSnapshot, verifyControllerBoundarySnapshot } from "./eval-snapshots.js";
|
|
6
|
+
import { observeControllerBoundary } from "./run-control.js";
|
|
7
|
+
import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
|
|
8
|
+
import { appendEvent, digest, errorRecord, requireController, requireRun } from "./run-controller-state.js";
|
|
9
|
+
import { getFlowRunStatus } from "./runs.js";
|
|
10
|
+
import { recoveryObservationBudget, withRecoveryObservation } from "./recovery-observation-budget.js";
|
|
11
|
+
/** Capture is performed by the productive owner, before it releases the next
|
|
12
|
+
* Stage. An observer never races the controller to copy a completed boundary. */
|
|
13
|
+
export async function captureControllerBoundary(context, row, manifest, stage, nextStage, assertPhysicalOwner) {
|
|
14
|
+
const root = manifest.request.capture_root;
|
|
15
|
+
if (!root)
|
|
16
|
+
return;
|
|
17
|
+
const identity = (value) => ({ run_id: row.run_id, stage: value.stage, attempt: value.attempt ?? null, started_at: value.started_at ?? null, completed_at: value.completed_at ?? null });
|
|
18
|
+
const boundaryKey = digest(identity(stage));
|
|
19
|
+
const operationId = `${row.controller_id}:boundary:${boundaryKey}`;
|
|
20
|
+
const output = path.join(root, `${stage.stage}-${boundaryKey}`);
|
|
21
|
+
const scope = { projectRoot: manifest.request.project_root, runId: row.run_id };
|
|
22
|
+
const expected = { controllerId: row.controller_id, operationId, boundaryKey, runId: row.run_id, stageEntry: nextStage };
|
|
23
|
+
const prior = context.db.get("SELECT status,receipt_json FROM run_controller_operations WHERE operation_id = ?", [operationId]);
|
|
24
|
+
if (prior?.status === "completed") {
|
|
25
|
+
try {
|
|
26
|
+
const receipt = JSON.parse(prior.receipt_json ?? "{}");
|
|
27
|
+
if (typeof receipt.manifest_sha256 !== "string")
|
|
28
|
+
throw new AppError("controller_boundary_snapshot_invalid", "Completed capture has no immutable manifest receipt", 1);
|
|
29
|
+
verifyControllerBoundarySnapshot(output, { ...expected, manifestSha256: receipt.manifest_sha256 });
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
assertPhysicalOwner();
|
|
34
|
+
context.db.run("UPDATE run_controllers SET status = 'capture_failed', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
context.db.beginWriteTransaction();
|
|
39
|
+
try {
|
|
40
|
+
assertPhysicalOwner();
|
|
41
|
+
assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
|
|
42
|
+
const now = context.now();
|
|
43
|
+
context.db.run("UPDATE run_controllers SET status = 'capturing', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [now, row.controller_id, row.owner_token]);
|
|
44
|
+
context.db.run("INSERT INTO run_controller_operations (operation_id,controller_id,generation,input_hash,input_json,status,created_at,updated_at) VALUES (?,?,?,?,?,'capturing',?,?) ON CONFLICT(operation_id) DO UPDATE SET generation = excluded.generation, status = 'capturing', error_json = NULL, updated_at = excluded.updated_at", [operationId, row.controller_id, row.generation, boundaryKey, JSON.stringify({ action: "boundary_capture", ...identity(stage), stage_entry: nextStage, output }), now, now]);
|
|
45
|
+
appendEvent(context, row.controller_id, "boundary_capture_requested", { operation_id: operationId, boundary_key: boundaryKey, stage: stage.stage, stage_entry: nextStage });
|
|
46
|
+
context.db.exec("COMMIT");
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
context.db.exec("ROLLBACK");
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
const assertCurrent = () => {
|
|
53
|
+
assertPhysicalOwner();
|
|
54
|
+
const guard = recoveryGuard(context, row.project_id, row.run_id);
|
|
55
|
+
if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
|
|
56
|
+
throw new AppError("run_recovery_guarded", "Recovery or operator control superseded boundary capture", 1);
|
|
57
|
+
if (requireController(context, row.controller_id).status !== "capturing" || context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ?", [operationId])?.status !== "capturing")
|
|
58
|
+
throw new AppError("controller_capture_owner_lost", "Controller no longer owns boundary capture", 1);
|
|
59
|
+
const index = JSON.parse(requireRun(context, row.project_id, row.run_id).index_json);
|
|
60
|
+
const current = index.stage_runs.find(item => item.stage === stage.stage);
|
|
61
|
+
if (!current || !["done", "skipped"].includes(current.status) || digest(identity(current)) !== boundaryKey)
|
|
62
|
+
throw new AppError("controller_boundary_changed", "Stage identity changed after its capture fence", 1);
|
|
63
|
+
const continuation = getFlowRunStatus(context, scope).continuation;
|
|
64
|
+
if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal")
|
|
65
|
+
throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
|
|
66
|
+
};
|
|
67
|
+
let priorObservation = null;
|
|
68
|
+
try {
|
|
69
|
+
priorObservation = prior?.receipt_json ? JSON.parse(prior.receipt_json) : null;
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
priorObservation = { recovery_observation: { remaining_ms: 0 } };
|
|
73
|
+
}
|
|
74
|
+
const observationBudget = recoveryObservationBudget(priorObservation);
|
|
75
|
+
try {
|
|
76
|
+
let receipt;
|
|
77
|
+
if (fs.existsSync(output)) {
|
|
78
|
+
assertCurrent();
|
|
79
|
+
receipt = verifyControllerBoundarySnapshot(output, expected);
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
let lastPending = "";
|
|
83
|
+
for (;;) {
|
|
84
|
+
if (observationBudget.exhausted()) {
|
|
85
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), context.now(), operationId]);
|
|
86
|
+
throw new AppError("recovery_observation_budget_exhausted", "Boundary capture has no stable settlement within the active observation budget", 1, { recovery_observation: observationBudget.state() });
|
|
87
|
+
}
|
|
88
|
+
assertCurrent();
|
|
89
|
+
let verify = () => { throw new AppError("controller_boundary_unsettled", "Boundary has no stable writer observation", 1); };
|
|
90
|
+
const observation = await withRecoveryObservation(observationBudget, () => observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => { verify = value; } }));
|
|
91
|
+
if (observation.settled === true) {
|
|
92
|
+
try {
|
|
93
|
+
createEvalRunSnapshot(context, { ...scope, output, ...(nextStage ? { stageEntry: nextStage } : { candidate: true }), boundary: { controllerId: row.controller_id, operationId, boundaryKey, observation, verify } });
|
|
94
|
+
receipt = verifyControllerBoundarySnapshot(output, expected);
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
if (!(error instanceof AppError) || !["snapshot_source_changed", "owned_inventory_changed"].includes(error.code))
|
|
99
|
+
throw error;
|
|
100
|
+
observation.settled = false;
|
|
101
|
+
observation.pending_reasons = [error.code];
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
observationBudget.exhausted();
|
|
105
|
+
const published = { ...observation, recovery_observation: observationBudget.state() };
|
|
106
|
+
const pending = digest(observation.pending_reasons);
|
|
107
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(published), context.now(), operationId]);
|
|
108
|
+
if (pending !== lastPending) {
|
|
109
|
+
appendEvent(context, row.controller_id, "boundary_capture_pending", { operation_id: operationId, pending_reasons: observation.pending_reasons });
|
|
110
|
+
lastPending = pending;
|
|
111
|
+
}
|
|
112
|
+
await delay(observationBudget.nextDelay());
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
context.db.beginWriteTransaction();
|
|
116
|
+
try {
|
|
117
|
+
assertCurrent();
|
|
118
|
+
context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
|
|
119
|
+
appendEvent(context, row.controller_id, "boundary_captured", { ...identity(stage), boundary_key: boundaryKey, operation_id: operationId, ...receipt });
|
|
120
|
+
context.db.run("UPDATE run_controllers SET status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
|
|
121
|
+
context.db.exec("COMMIT");
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
context.db.exec("ROLLBACK");
|
|
125
|
+
throw error;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
observationBudget.exhausted();
|
|
130
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)) WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), operationId]);
|
|
131
|
+
context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { AppError } from "../shared/errors.js";
|
|
7
|
+
import { getResourceDatabase } from "../storage/database.js";
|
|
8
|
+
import { finishManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess } from "./managed-processes.js";
|
|
9
|
+
import { setFlowRunRecoveryPaused } from "./runs.js";
|
|
10
|
+
import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
|
|
11
|
+
import { requireRun, requireController, controllerReceipt, appendEvent, digest, errorRecord } from "./run-controller-state.js";
|
|
12
|
+
const launchId = (row) => `${row.controller_id}:owner-launch:${digest(row.owner_token)}`;
|
|
13
|
+
const processOperation = (context, row) => `controller:${digest({ database: context.db.path, project: row.project_id, run: row.run_id })}`;
|
|
14
|
+
const birth = () => ({ pid: process.pid, pid_started_at: processStartedAt(process.pid) });
|
|
15
|
+
const dead = (value) => Boolean(value && Number.isSafeInteger(value.pid) && value.pid > 0 && typeof value.pid_started_at === "string" && value.pid_started_at && !processIsAlive(value));
|
|
16
|
+
/** Called inside the RUN intent transaction, before the client can disappear. */
|
|
17
|
+
export function recordControllerLauncher(context, row, retired) {
|
|
18
|
+
const request = { action: "controller_owner_launch", owner_id: launchId(row) };
|
|
19
|
+
const receipt = { ...birth(), registration: "controller_self@1", ...(retired ? { retired_process: { id: retired.id, lease_token: retired.lease_token, pid: retired.pid, pid_started_at: retired.pid_started_at } } : {}) };
|
|
20
|
+
const now = context.now();
|
|
21
|
+
context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'completed', ?, ?, ?)", [launchId(row), row.controller_id, row.generation, digest(request), JSON.stringify(request), JSON.stringify(receipt), now, now]);
|
|
22
|
+
}
|
|
23
|
+
function launchRecord(context, row) {
|
|
24
|
+
const record = context.db.get("SELECT receipt_json FROM run_controller_operations WHERE operation_id = ?", [launchId(row)]);
|
|
25
|
+
if (!record)
|
|
26
|
+
return null;
|
|
27
|
+
const value = JSON.parse(record.receipt_json);
|
|
28
|
+
return value?.registration === "controller_self@1" ? { ...record, value } : null;
|
|
29
|
+
}
|
|
30
|
+
function assertLaunchAllowed(context, row) {
|
|
31
|
+
const run = requireRun(context, row.project_id, row.run_id);
|
|
32
|
+
if (!["running", "paused", "waiting_for_user"].includes(run.status))
|
|
33
|
+
throw new AppError("invalid_run_state", "A terminal RUN cannot start a physical controller", 1);
|
|
34
|
+
const state = JSON.parse(row.state_json);
|
|
35
|
+
if (!state.recovery) {
|
|
36
|
+
assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const guard = recoveryGuard(context, row.project_id, row.run_id);
|
|
40
|
+
if (guard?.generation !== row.generation || guard.recovery_id !== state.recovery.recovery_id || !["sealed", "resuming"].includes(guard.status)
|
|
41
|
+
|| !context.db.get("SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = ? AND status = 'resuming'", [state.recovery.control_id, state.recovery.recovery_id]))
|
|
42
|
+
throw new AppError("run_control_superseded", "Controller launch no longer owns the accepted recovery", 1);
|
|
43
|
+
}
|
|
44
|
+
function retire(context, row, identity) {
|
|
45
|
+
const record = managedProcessStatus(context).find(item => item.id === identity.id);
|
|
46
|
+
if (!record || record.kind !== "run-controller" || record.owner_id !== row.controller_id || record.project_id !== row.project_id || record.run_id !== row.run_id
|
|
47
|
+
|| managedProcessRuntimeHome(record, context) !== context.ddFlowHome || record.lease_token !== identity.lease_token || record.pid !== identity.pid || record.pid_started_at !== identity.pid_started_at || !dead(identity) || processTreeIsAlive(record)
|
|
48
|
+
|| !finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: "controller_launch_owner_retired" }))
|
|
49
|
+
throw new AppError("controller_owner_unsettled", "Prior controller identity is not proven dead and retired", 1);
|
|
50
|
+
}
|
|
51
|
+
/** Retry only before productive admission, with exact prior birth and journal CAS. */
|
|
52
|
+
function prepareSpawn(context, row) {
|
|
53
|
+
const launch = launchRecord(context, row);
|
|
54
|
+
if (!launch || launch.value.dispatch_ready || !["starting", "running", "recovery_required"].includes(row.status))
|
|
55
|
+
return null;
|
|
56
|
+
const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: JSON.parse(row.manifest_json).resource_home } };
|
|
57
|
+
const ours = launch.value.pid === process.pid && launch.value.pid_started_at === processStartedAt(process.pid);
|
|
58
|
+
let token = row.owner_token;
|
|
59
|
+
if (!ours) {
|
|
60
|
+
if (row.owner_pid) {
|
|
61
|
+
if (launch.value.owner?.pid !== row.owner_pid || !dead(launch.value.owner))
|
|
62
|
+
return null;
|
|
63
|
+
const records = managedProcessStatus(resourceContext).filter(item => item.operation_id === processOperation(context, row));
|
|
64
|
+
const owned = records.filter(item => { const metadata = JSON.parse(item.metadata_json); return metadata.owner_pid === row.owner_pid && metadata.owner_pid_started_at === launch.value.owner.pid_started_at; });
|
|
65
|
+
if (owned.length > 1 || row.process_id && owned[0]?.id !== row.process_id)
|
|
66
|
+
return null;
|
|
67
|
+
const record = owned[0];
|
|
68
|
+
if (records.some(item => item.id !== record?.id && (item.state !== "stopped" || !item.pid_started_at || processTreeIsAlive(item))))
|
|
69
|
+
return null;
|
|
70
|
+
if (record) {
|
|
71
|
+
if (record.kind !== "run-controller" || record.owner_id !== row.controller_id || record.project_id !== row.project_id || record.run_id !== row.run_id || managedProcessRuntimeHome(record, context) !== context.ddFlowHome || row.process_lease_token && record.lease_token !== row.process_lease_token || record.pid !== null && record.pid !== row.owner_pid || processTreeIsAlive(record))
|
|
72
|
+
return null;
|
|
73
|
+
if (record.pid === null) {
|
|
74
|
+
const db = getResourceDatabase(JSON.parse(row.manifest_json).resource_home);
|
|
75
|
+
if (db.run("UPDATE managed_processes SET pid = ?, pid_started_at = ? WHERE id = ? AND lease_token = ? AND pid IS NULL", [row.owner_pid, launch.value.owner.pid_started_at, record.id, record.lease_token]).changes !== 1)
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
retire(resourceContext, row, { id: record.id, lease_token: record.lease_token, ...launch.value.owner });
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
if (row.process_id || row.process_lease_token || !dead(launch.value))
|
|
83
|
+
return null;
|
|
84
|
+
if (launch.value.child && !dead(launch.value.child))
|
|
85
|
+
return null;
|
|
86
|
+
if (managedProcessStatus(resourceContext).some(item => item.operation_id === processOperation(context, row) && item.id !== launch.value.retired_process?.id && (item.state !== "stopped" || !item.pid_started_at || processTreeIsAlive(item))))
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
token = crypto.randomUUID();
|
|
90
|
+
}
|
|
91
|
+
else if (row.owner_pid || launch.value.spawn_started)
|
|
92
|
+
return null;
|
|
93
|
+
if (launch.value.retired_process)
|
|
94
|
+
retire(resourceContext, row, launch.value.retired_process);
|
|
95
|
+
context.db.beginWriteTransaction();
|
|
96
|
+
try {
|
|
97
|
+
assertLaunchAllowed(context, row);
|
|
98
|
+
const changed = context.db.run(`UPDATE run_controllers SET owner_token = ?, owner_pid = NULL, process_id = NULL, process_lease_token = NULL, status = 'starting', last_error_json = NULL, updated_at = ?
|
|
99
|
+
WHERE controller_id = ? AND owner_token = ? AND owner_pid IS ? AND process_id IS ? AND status = ? AND state_json = ?
|
|
100
|
+
AND (SELECT receipt_json FROM run_controller_operations WHERE operation_id = ?) = ?`, [token, context.now(), row.controller_id, row.owner_token, row.owner_pid, row.process_id, row.status, row.state_json, launchId(row), launch.receipt_json]);
|
|
101
|
+
if (changed.changes !== 1) {
|
|
102
|
+
context.db.exec("ROLLBACK");
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
const current = requireController(context, row.controller_id);
|
|
106
|
+
if (!ours) {
|
|
107
|
+
recordControllerLauncher(context, current);
|
|
108
|
+
appendEvent(context, row.controller_id, "launch_owner_reconciled", { prior_launch: launchId(row), generation: row.generation, prior_process_id: row.process_id });
|
|
109
|
+
}
|
|
110
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.spawn_started', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(current)]);
|
|
111
|
+
context.db.exec("COMMIT");
|
|
112
|
+
return current;
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
context.db.exec("ROLLBACK");
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/** The physical child records its own birth before resource registration. */
|
|
120
|
+
export function claimControllerProcess(context, row) {
|
|
121
|
+
const launch = launchRecord(context, row);
|
|
122
|
+
if (!launch && (!row.process_id || !row.process_lease_token))
|
|
123
|
+
throw new AppError("controller_owner_mismatch", "Managed registration is not authorized", 1);
|
|
124
|
+
context.db.beginWriteTransaction();
|
|
125
|
+
try {
|
|
126
|
+
const claimed = context.db.run("UPDATE run_controllers SET owner_pid = ?, status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [process.pid, context.now(), row.controller_id, row.owner_token]);
|
|
127
|
+
if (claimed.changes !== 1)
|
|
128
|
+
throw new AppError("controller_already_owned", "Managed owner has already claimed this launch", 1);
|
|
129
|
+
if (launch)
|
|
130
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.owner', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(birth()), context.now(), launchId(row)]);
|
|
131
|
+
context.db.exec("COMMIT");
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
context.db.exec("ROLLBACK");
|
|
135
|
+
throw error;
|
|
136
|
+
}
|
|
137
|
+
if (!row.process_id) {
|
|
138
|
+
const manifest = JSON.parse(row.manifest_json);
|
|
139
|
+
const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
|
|
140
|
+
const log = path.join(row.state_dir, "controller.log");
|
|
141
|
+
const record = registerManagedProcess(resourceContext, { kind: "run-controller", ownerId: row.controller_id, projectId: row.project_id, runId: row.run_id, operationId: processOperation(context, row), uniqueActiveOperation: true, ownerPid: process.pid, stdoutPath: log, stderrPath: log, metadata: { controller_id: row.controller_id, state_dir: row.state_dir, role: "controller", budget: manifest.budget } });
|
|
142
|
+
row.process_id = record.id;
|
|
143
|
+
row.process_lease_token = record.lease_token;
|
|
144
|
+
if (context.db.run("UPDATE run_controllers SET process_id = ?, process_lease_token = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND owner_pid = ? AND status = 'running'", [record.id, record.lease_token, context.now(), row.controller_id, row.owner_token, process.pid]).changes !== 1)
|
|
145
|
+
throw new AppError("controller_owner_mismatch", "Managed owner changed during registration", 1);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
export function markControllerDispatchReady(context, row) {
|
|
149
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.dispatch_ready', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(row)]);
|
|
150
|
+
}
|
|
151
|
+
export async function spawnController(context, row) {
|
|
152
|
+
const admitted = prepareSpawn(context, row);
|
|
153
|
+
if (!admitted)
|
|
154
|
+
return controllerReceipt(requireController(context, row.controller_id));
|
|
155
|
+
row = admitted;
|
|
156
|
+
const manifest = JSON.parse(row.manifest_json);
|
|
157
|
+
const run = requireRun(context, row.project_id, row.run_id);
|
|
158
|
+
const controllerId = row.controller_id, root = row.state_dir, token = row.owner_token;
|
|
159
|
+
const entry = fileURLToPath(new URL("../cli.js", import.meta.url));
|
|
160
|
+
try {
|
|
161
|
+
if (!fs.existsSync(entry))
|
|
162
|
+
throw new AppError("harness_runtime_unbuilt", "Build the CLI before launching its detached controller", 1);
|
|
163
|
+
const output = fs.openSync(path.join(root, "controller.log"), "a", 0o600);
|
|
164
|
+
try {
|
|
165
|
+
const child = spawn(process.execPath, [entry, "run", "drive", "serve", "--run", run.id, "--project-root", run.project_root, "--controller-id", controllerId, "--json"], { cwd: run.workspace_root, detached: true, stdio: ["ignore", output, output], env: { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_CONTROLLER_TOKEN: token } });
|
|
166
|
+
await new Promise((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); });
|
|
167
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.child', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify({ pid: child.pid, pid_started_at: child.pid ? processStartedAt(child.pid) : null }), context.now(), launchId(row)]);
|
|
168
|
+
child.unref();
|
|
169
|
+
}
|
|
170
|
+
finally {
|
|
171
|
+
fs.closeSync(output);
|
|
172
|
+
}
|
|
173
|
+
return controllerReceipt(requireController(context, controllerId));
|
|
174
|
+
}
|
|
175
|
+
catch (error) {
|
|
176
|
+
context.db.exec("SAVEPOINT controller_spawn_status");
|
|
177
|
+
try {
|
|
178
|
+
const changed = context.db.run("UPDATE run_controllers SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [JSON.stringify(errorRecord(error)), context.now(), controllerId, token]);
|
|
179
|
+
if (changed.changes && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation)
|
|
180
|
+
setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
|
|
181
|
+
context.db.exec("RELEASE controller_spawn_status");
|
|
182
|
+
}
|
|
183
|
+
catch (failure) {
|
|
184
|
+
context.db.exec("ROLLBACK TO controller_spawn_status; RELEASE controller_spawn_status");
|
|
185
|
+
throw failure;
|
|
186
|
+
}
|
|
187
|
+
throw error;
|
|
188
|
+
}
|
|
189
|
+
}
|