@deksden-com/dd-flow-cli 0.9.0-beta.64 → 0.9.0-beta.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/cli/help.js +2 -0
- package/dist/cli/run-cli.js +15 -0
- package/dist/harness-runtime/bin/dd-agy.mjs +4 -2
- package/dist/harness-runtime/bin/dd-codex.mjs +4 -2
- package/dist/harness-runtime/bin/dd-droid.mjs +4 -2
- package/dist/harness-runtime/bin/dd-grok.mjs +4 -2
- package/dist/harness-runtime/bin/dd-opencode.mjs +4 -2
- package/dist/harness-runtime/bin/dd-zcode.mjs +15 -7
- package/dist/harness-runtime/lib/daemon-operations.mjs +108 -10
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +5 -4
- package/dist/harness-runtime/lib/dd-codex.mjs +38 -4
- package/dist/harness-runtime/lib/dd-droid.mjs +55 -20
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +3 -2
- package/dist/harness-runtime/lib/dd-grok.mjs +14 -8
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +43 -8
- package/dist/harness-runtime/lib/dd-opencode.mjs +5 -4
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +182 -16
- package/dist/harness-runtime/lib/dd-zcode.mjs +162 -63
- package/dist/harness-runtime/lib/driver-recovery.mjs +2 -2
- package/dist/harness-runtime/lib/managed-daemon.mjs +27 -5
- package/dist/harness-runtime/lib/operation-errors.mjs +1 -1
- package/dist/harness-runtime/lib/tool-observations.d.mts +17 -0
- package/dist/harness-runtime/lib/tool-observations.mjs +153 -0
- package/dist/services/code-checks.js +120 -19
- package/dist/services/controller-fanout.js +11 -0
- package/dist/services/engines.js +40 -9
- package/dist/services/eval-snapshots.js +30 -47
- package/dist/services/harness-adapter.js +12 -6
- package/dist/services/lifecycle-invocations.js +12 -8
- package/dist/services/native-session-control.js +6 -0
- package/dist/services/projects.js +6 -2
- package/dist/services/recovery-observation-budget.js +9 -1
- package/dist/services/run-control-worker.js +19 -0
- package/dist/services/run-control.js +87 -13
- package/dist/services/run-controller-adapter.js +134 -7
- package/dist/services/run-controller.js +11 -3
- package/dist/services/run-engine-bindings.js +1 -1
- package/dist/services/run-fork.js +124 -0
- package/dist/services/run-observations.js +112 -0
- package/dist/services/runs.js +1 -11
- package/dist/services/schema-validation.js +1 -1
- package/dist/services/stage-lifecycle.js +14 -2
- package/dist/services/usage.js +20 -87
- package/dist/services/vnext-code-review.js +35 -27
- package/dist/services/vnext-code.js +62 -26
- package/dist/services/vnext-merge.js +8 -8
- package/dist/services/vnext-plan-review.js +16 -16
- package/dist/services/vnext-plan.js +51 -11
- package/dist/services/work-registry.js +48 -27
- package/dist/shared/errors.js +12 -0
- package/package.json +1 -1
|
@@ -38,7 +38,9 @@ export function prepareHarnessFlowExecutable(context, stateDir) {
|
|
|
38
38
|
}
|
|
39
39
|
/** Runs one harness command with the lifecycle guarantees shared by agent services. */
|
|
40
40
|
export function runHarnessAdapter(input) {
|
|
41
|
-
|
|
41
|
+
// `null` is deliberate: a productive Turn has no adapter-wall-clock limit.
|
|
42
|
+
// Preserve it instead of treating it like an omitted short control request.
|
|
43
|
+
const timeoutMs = recoveryRpcTimeout(input.timeoutMs === undefined ? 120_000 : input.timeoutMs);
|
|
42
44
|
if (!input.operationId.trim())
|
|
43
45
|
throw new AppError("operation_id_required", "Harness calls require a scoped operation id", 2);
|
|
44
46
|
// The current adapter transport uses this name; its identity is runtime-owned,
|
|
@@ -89,7 +91,7 @@ export function runHarnessAdapter(input) {
|
|
|
89
91
|
}
|
|
90
92
|
};
|
|
91
93
|
const heartbeat = input.progress && input.progressMessage ? setInterval(() => deliver(() => input.progress?.(`${input.progressMessage}; waiting for adapter completion`)), 15_000) : undefined;
|
|
92
|
-
const timer = setTimeout(() => { timedOut = true; stopTransport(); }, timeoutMs);
|
|
94
|
+
const timer = timeoutMs === null ? undefined : setTimeout(() => { timedOut = true; stopTransport(); }, timeoutMs);
|
|
93
95
|
const abort = () => { aborted = true; stopTransport(); };
|
|
94
96
|
if (input.signal?.aborted)
|
|
95
97
|
abort();
|
|
@@ -111,7 +113,8 @@ export function runHarnessAdapter(input) {
|
|
|
111
113
|
if (settled)
|
|
112
114
|
return;
|
|
113
115
|
settled = true;
|
|
114
|
-
|
|
116
|
+
if (timer)
|
|
117
|
+
clearTimeout(timer);
|
|
115
118
|
input.signal?.removeEventListener("abort", abort);
|
|
116
119
|
if (heartbeat)
|
|
117
120
|
clearInterval(heartbeat);
|
|
@@ -138,7 +141,7 @@ export function runHarnessAdapter(input) {
|
|
|
138
141
|
if (!reply || typeof reply !== "object" || Array.isArray(reply))
|
|
139
142
|
throw new Error("not object");
|
|
140
143
|
const value = reply;
|
|
141
|
-
if (value.ok === false
|
|
144
|
+
if (value.ok === false)
|
|
142
145
|
return reject(new AppError("harness_adapter_failed", "Harness adapter reported a runtime failure", 1, { cause: adapterCause(stdout) }));
|
|
143
146
|
resolve(reply);
|
|
144
147
|
}
|
|
@@ -153,8 +156,11 @@ function adapterCause(...outputs) {
|
|
|
153
156
|
for (const line of [output.trim(), ...output.trim().split("\n").reverse()]) {
|
|
154
157
|
try {
|
|
155
158
|
const value = JSON.parse(line);
|
|
156
|
-
|
|
157
|
-
|
|
159
|
+
const nested = value.error && typeof value.error === "object" && !Array.isArray(value.error) ? value.error : null;
|
|
160
|
+
const code = typeof nested?.code === "string" ? nested.code : typeof value.code === "string" ? value.code : null;
|
|
161
|
+
const message = typeof nested?.message === "string" ? nested.message : typeof value.error === "string" ? value.error : code;
|
|
162
|
+
if (code)
|
|
163
|
+
return { code, message, ...(nested?.details ?? value.details ? { details: nested?.details ?? value.details } : {}), ...((nested?.retryable ?? value.retryable) === true ? { retryable: true } : {}) };
|
|
158
164
|
}
|
|
159
165
|
catch { /* Adapter diagnostics need not be JSON. */ }
|
|
160
166
|
}
|
|
@@ -3,7 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import { quote } from "shell-quote";
|
|
5
5
|
import { setTimeout as delay } from "node:timers/promises";
|
|
6
|
-
import { AppError } from "../shared/errors.js";
|
|
6
|
+
import { AppError, errorRecord } from "../shared/errors.js";
|
|
7
7
|
import { commandOption, parseLifecycleCommand, shellSuffixOffset } from "./lifecycle-command.js";
|
|
8
8
|
import { requireProjectByRoot } from "./projects.js";
|
|
9
9
|
import { assertRunMutationAllowed, assertStageSettlementAllowed, assertWorkSettlementAllowed, recoveryGuard } from "./run-recovery.js";
|
|
@@ -16,7 +16,9 @@ import { runtimeHarness } from "./execution-policy.js";
|
|
|
16
16
|
// Conclusive result/check rejections, never transport, ownership, registry or
|
|
17
17
|
// unknown-effect failures. These validators reject before accepting the result.
|
|
18
18
|
export function isRetryableLifecycleRejection(code, details = {}) {
|
|
19
|
-
|
|
19
|
+
if (details.recoverable === false || details.effect === "unknown" || details.effect === "committed")
|
|
20
|
+
return false;
|
|
21
|
+
return details.recoverable === true || ["validation", "schema_validation", "work_checks_failed", "code_gate_failed", "code_review_gate_failed", "merge_gate_failed",
|
|
20
22
|
"review_repair_incomplete", "plan_review_incomplete", "semantic_result_invalid",
|
|
21
23
|
"review_evidence_invalid", "evidence_obligation_unknown", "invalid_evidence_ref",
|
|
22
24
|
"evidence_ref_missing", "evidence_line_out_of_range", "document_update_missing",
|
|
@@ -82,7 +84,7 @@ export function assertLifecycleOutcomes(context, scope) {
|
|
|
82
84
|
if (outcome.disposition === "retrying" && outcome.deadline <= Date.now())
|
|
83
85
|
throw new AppError("lifecycle_outcome_unknown", "Storage retry has no terminal receipt; reconcile its committed evidence", 1, { lifecycle_outcome: outcome, effect: "unknown" });
|
|
84
86
|
if (outcome.disposition === "fatal")
|
|
85
|
-
throw new AppError(outcome.error
|
|
87
|
+
throw new AppError(outcome.error?.code ?? "lifecycle_outcome_unknown", outcome.error?.message || "Mandatory lifecycle failed (historical receipt has no message)", 1, { ...outcome.error?.details, lifecycle_outcome: outcome });
|
|
86
88
|
}
|
|
87
89
|
}
|
|
88
90
|
if (!context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'lifecycle_invocations'"))
|
|
@@ -94,7 +96,7 @@ export function assertLifecycleOutcomes(context, scope) {
|
|
|
94
96
|
ORDER BY a.rowid`, [scope.projectRoot, scope.runId, scope.generation]);
|
|
95
97
|
for (const row of rows) {
|
|
96
98
|
const error = (row.outcome_json ? JSON.parse(row.outcome_json) : null)?.error;
|
|
97
|
-
if (typeof error?.code === "string" &&
|
|
99
|
+
if (typeof error?.code === "string" && !isRetryableLifecycleRejection(error.code, error.details))
|
|
98
100
|
throw new AppError(error.code, typeof error.message === "string" ? error.message : "Mandatory lifecycle failed", 1, { ...(error.details ?? {}), invocation_id: row.id });
|
|
99
101
|
}
|
|
100
102
|
}
|
|
@@ -174,8 +176,9 @@ export function observeLifecycleCommand(context, command) {
|
|
|
174
176
|
delete context.observeLifecycleHook;
|
|
175
177
|
if (!anchor)
|
|
176
178
|
return;
|
|
177
|
-
const
|
|
178
|
-
const
|
|
179
|
+
const error = outcome.error ? errorRecord(outcome.error) : undefined;
|
|
180
|
+
const retryable = error && isRetryableLifecycleRejection(error.code, error.details);
|
|
181
|
+
const record = { ...anchor, ...(error ? { error } : outcome), effect: error ? error.details.effect ?? (retryable ? "no_effect" : "unknown") : "committed", disposition: error ? retryable ? "correctable" : "fatal" : "completed" };
|
|
179
182
|
try {
|
|
180
183
|
context.db.writeTransaction(() => {
|
|
181
184
|
const retained = context.db.get("SELECT outcome_json FROM hook_events WHERE id = ?", [anchor.hook_event_id]);
|
|
@@ -280,7 +283,7 @@ export function managedLifecycleCommand(context, command) {
|
|
|
280
283
|
const retained = context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'lifecycle_invocations'")
|
|
281
284
|
? context.db.get("SELECT * FROM lifecycle_invocations WHERE scope_json = ? AND fingerprint = ? ORDER BY rowid DESC LIMIT 1", [JSON.stringify(scope), fingerprint(parsed)]) : undefined;
|
|
282
285
|
if (!retained)
|
|
283
|
-
throw new AppError("invocation_command_unprepared", "
|
|
286
|
+
throw new AppError("invocation_command_unprepared", "The controller has not issued this lifecycle command yet. End this Turn and await its next assignment with the exact command; do not generate or reuse an invocation ID.", 1);
|
|
284
287
|
return renderInvocationCommand(retained, command);
|
|
285
288
|
}
|
|
286
289
|
storage(context);
|
|
@@ -352,7 +355,7 @@ export function settleLifecycleNoEffectRejection(context, id, command, error, id
|
|
|
352
355
|
function settleUnadmittedInvocation(context, prior, error) {
|
|
353
356
|
if (!["issued", "expired"].includes(prior.status))
|
|
354
357
|
throw new AppError("invocation_rejection_conflict", "Attempt already admitted native execution", 1);
|
|
355
|
-
const details = { ...error.details, retry_command: successorLifecycleInvocationCommand(context, prior),
|
|
358
|
+
const details = { ...error.details, effect: "no_effect", recoverable: true, retry_command: successorLifecycleInvocationCommand(context, prior),
|
|
356
359
|
retry_instruction: "Use retry_command as one standalone lifecycle command. The rejected command did not change lifecycle state." };
|
|
357
360
|
if (context.db.run("UPDATE lifecycle_invocations SET status = 'settled', outcome_json = ?, updated_at = ? WHERE id = ? AND status = ?", [JSON.stringify({ error: { ...error, details } }), context.now(), prior.id, prior.status]).changes !== 1) {
|
|
358
361
|
throw new AppError("invocation_settlement_conflict", "Lifecycle rejection could not be committed", 1, { invocation_id: prior.id });
|
|
@@ -375,6 +378,7 @@ export function settleLifecycleRejection(context, id, error) {
|
|
|
375
378
|
if (isRetryableLifecycleRejection(error.code, error.details) || storageRetry) {
|
|
376
379
|
const retry = retryLifecycleInvocationCommand(context, id);
|
|
377
380
|
details.retry_command = retry;
|
|
381
|
+
details.recoverable = true;
|
|
378
382
|
if (storageRetry)
|
|
379
383
|
details.storage_retry = 1;
|
|
380
384
|
details.retry_instruction = "Correct only the evidenced defect, then use retry_command with the required result input. Reissuing the old command only reads its retained outcome.";
|
|
@@ -7,6 +7,12 @@ export async function cancelControlledNativeSession(input) {
|
|
|
7
7
|
env: input.env, operationId: input.observe ? `${input.operationId}:observe` : input.operationId, timeoutMs: 30_000 });
|
|
8
8
|
if (input.observe && (reply.operation_id !== input.operationId || reply.operation !== "session.cancel" || reply.session_id !== input.sessionId))
|
|
9
9
|
throw new AppError("controller_operation_mismatch", "Cancellation observation belongs to another operation or Session", 1);
|
|
10
|
+
if (input.observe && reply.state === "failed") {
|
|
11
|
+
if (!reply.error || typeof reply.error !== "object" || Array.isArray(reply.error))
|
|
12
|
+
throw new AppError("controller_operation_mismatch", "Cancellation failure has no terminal error", 1);
|
|
13
|
+
input.assertCurrent();
|
|
14
|
+
return { status: "failed", receipt: reply };
|
|
15
|
+
}
|
|
10
16
|
const result = input.observe ? reply.state === "completed" ? reply.result : null : reply;
|
|
11
17
|
// A completed cancel command is an immutable ACK, not proof that the
|
|
12
18
|
// provider tree has stopped. Callers must use a separate fresh inspection.
|
|
@@ -232,9 +232,13 @@ export function findProjectByRoot(context, root) {
|
|
|
232
232
|
return context.db.get("SELECT * FROM projects WHERE root = ?", [root]);
|
|
233
233
|
}
|
|
234
234
|
export function requireProjectByRoot(context, root) {
|
|
235
|
-
|
|
235
|
+
// Callers may receive a path through a native adapter or shell. Resolve
|
|
236
|
+
// aliases such as macOS `/var` -> `/private/var` before comparing with the
|
|
237
|
+
// canonical root persisted in the project registry.
|
|
238
|
+
const canonical = normalizeStoredRoot(root);
|
|
239
|
+
const project = findProjectByRoot(context, canonical);
|
|
236
240
|
if (!project) {
|
|
237
|
-
throw new AppError("not_found", `Project is not registered: ${
|
|
241
|
+
throw new AppError("not_found", `Project is not registered: ${canonical}`, 1);
|
|
238
242
|
}
|
|
239
243
|
return project;
|
|
240
244
|
}
|
|
@@ -3,7 +3,15 @@ import { AppError } from "../shared/errors.js";
|
|
|
3
3
|
export const recoveryObservationBudgetMs = 120_000;
|
|
4
4
|
const activeObservation = new AsyncLocalStorage();
|
|
5
5
|
export const withRecoveryObservation = (budget, action) => activeObservation.run(budget, action);
|
|
6
|
-
export const recoveryRpcTimeout = (maximum) =>
|
|
6
|
+
export const recoveryRpcTimeout = (maximum) => {
|
|
7
|
+
const observation = activeObservation.getStore();
|
|
8
|
+
// An unbounded productive transport wait is still bounded while recovery is
|
|
9
|
+
// actively reconciling. Outside recovery, its owner policy decides when to
|
|
10
|
+
// stop rather than a hidden adapter timer.
|
|
11
|
+
if (maximum === null)
|
|
12
|
+
return observation ? observation.timeout() : null;
|
|
13
|
+
return observation?.timeout(maximum ?? 30_000) ?? maximum ?? 30_000;
|
|
14
|
+
};
|
|
7
15
|
/** Call only while holding the existing owner-claim transaction. A replay of
|
|
8
16
|
* an explicit request never replenishes time, even after another request. */
|
|
9
17
|
export function renewRecoveryObservation(snapshot, requestId) {
|
|
@@ -92,12 +92,31 @@ async function launchControlWorker(context, input) {
|
|
|
92
92
|
finally {
|
|
93
93
|
fs.closeSync(output);
|
|
94
94
|
}
|
|
95
|
+
// The detached worker is the durable owner of the stop. Do not let the
|
|
96
|
+
// caller tear down its RUN/home before that owner has either registered
|
|
97
|
+
// its managed-process lease or recorded a startup failure.
|
|
98
|
+
await waitForControlWorkerAdmission(context, input.controlId, token);
|
|
95
99
|
}
|
|
96
100
|
catch (error) {
|
|
97
101
|
context.db.run("UPDATE run_control_workers SET status = 'failed', error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [JSON.stringify(failure(error)), context.now(), input.controlId, token]);
|
|
98
102
|
// The durable intent remains accepted even if its worker could not start.
|
|
99
103
|
}
|
|
100
104
|
}
|
|
105
|
+
async function waitForControlWorkerAdmission(context, controlId, ownerToken, timeoutMs = 5_000) {
|
|
106
|
+
const deadline = Date.now() + timeoutMs;
|
|
107
|
+
for (;;) {
|
|
108
|
+
const worker = context.db.get("SELECT status, process_id, owner_token FROM run_control_workers WHERE control_id = ?", [controlId]);
|
|
109
|
+
if (!worker || worker.owner_token !== ownerToken)
|
|
110
|
+
return;
|
|
111
|
+
// A running worker without a resource lease is still in the vulnerable
|
|
112
|
+
// registration gap; keep the parent alive until that gap closes.
|
|
113
|
+
if (worker.process_id || ["failed", "completed", "superseded"].includes(worker.status))
|
|
114
|
+
return;
|
|
115
|
+
if (Date.now() >= deadline)
|
|
116
|
+
throw new AppError("run_control_worker_start_timeout", "Control worker did not publish its managed-process lease", 1, { control_id: controlId, timeout_ms: timeoutMs });
|
|
117
|
+
await delay(25);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
101
120
|
const launchOperationId = (controlId, token) => `${controlId}:worker-launch:${crypto.createHash("sha256").update(token).digest("hex")}`;
|
|
102
121
|
function recordControlLauncher(context, controlId, token) {
|
|
103
122
|
const now = context.now();
|
|
@@ -298,7 +298,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
298
298
|
const nativeOperations = retainedDaemonOperations(directory);
|
|
299
299
|
nativeVersions.set(directory, nativeInventoryVersion(directory, nativeOperations));
|
|
300
300
|
for (const operation of nativeOperations)
|
|
301
|
-
if (!["completed", "failed"].includes(String(operation.state)))
|
|
301
|
+
if (!["completed", "failed", "interrupted"].includes(String(operation.state)))
|
|
302
302
|
unresolved.push({ state_dir: directory, operation_id: operation.operation_id, status: "pending", reason: "native_operation_unsettled" });
|
|
303
303
|
if (state.shutdown_state !== "clean" && !records.some(record => record.kind.endsWith("-daemon") && record.id === state.resource_process?.id && record.pid === state.pid && record.pid_started_at && record.lease_token === state.resource_process.lease_token && JSON.parse(record.metadata_json).dispatch_barrier === "run-generation@1"))
|
|
304
304
|
unresolved.push({ state_dir: directory, status: "pending", reason: "native_dispatch_barrier_unverified" });
|
|
@@ -321,8 +321,13 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
321
321
|
if (records.some(record => !record.pid || !record.pid_started_at || processTreeIsAlive(record)))
|
|
322
322
|
throw new AppError("shutdown_process_unsettled", "Closed daemon still has an unconfirmed or live owned process", 1);
|
|
323
323
|
const receipts = retainedDaemonOperations(directory);
|
|
324
|
-
|
|
325
|
-
|
|
324
|
+
// A whole-RUN stop is also the recovery fence for a create whose reply
|
|
325
|
+
// was lost after native allocation. Keep that one outstanding operation
|
|
326
|
+
// observable until terminalization below; unrelated unfinished work is
|
|
327
|
+
// still a shutdown proof failure.
|
|
328
|
+
const unfinished = receipts.filter(receipt => !["completed", "failed", "interrupted"].includes(String(receipt.state)));
|
|
329
|
+
if (unfinished.some(receipt => receipt.daemon_id !== daemonId || String(receipt.operation) !== "session.create" || receipt.session_id !== null))
|
|
330
|
+
throw new AppError("shutdown_operation_unsettled", "Closed daemon retains an unfinished non-create operation", 1);
|
|
326
331
|
const stopped = receipts.find(receipt => receipt.daemon_id === daemonId && receipt.operation === "daemon.stop" && receipt.state === "completed" && receipt.result?.stopped === true && receipt.result?.clean === true);
|
|
327
332
|
if (!stopped)
|
|
328
333
|
throw new AppError("shutdown_receipt_required", "Closed daemon has no clean native shutdown receipt for its current identity", 1);
|
|
@@ -366,6 +371,12 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
366
371
|
const retained = JSON.parse(controllers.find(row => row.controller_id === operation.controller_id).state_json).sessions.find(session => session.state_dir === saved.state_dir && (!target || session.id === target));
|
|
367
372
|
if (!retained)
|
|
368
373
|
throw new AppError("controller_operation_binding_unknown", "Unfinished operation has no retained controller Session", 1);
|
|
374
|
+
// A create without a native daemon state file has no provider-side
|
|
375
|
+
// receipt to inspect. Do not invoke the adapter again merely to discover
|
|
376
|
+
// that fact; retain the original outcome-unknown operation for the
|
|
377
|
+
// explicit controller fence/recovery path.
|
|
378
|
+
if (method === "session.create" && !fs.existsSync(path.join(saved.state_dir, "daemon.json")))
|
|
379
|
+
return { operation_id: operation.operation_id, status: "pending", reason: "create_observation_unavailable" };
|
|
369
380
|
const harness = runtimeHarness(retained.profile);
|
|
370
381
|
const env = { ...context.env, DD_FLOW_HOME: context.ddFlowHome };
|
|
371
382
|
delete env.DD_FLOW_CONTROLLER_TOKEN;
|
|
@@ -374,7 +385,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
374
385
|
const receipt = await runHarnessAdapter({ executable: saved.executable, args: ["daemon", "operation", "--operation-id", operation.operation_id, "--state-dir", saved.state_dir, "--json"], env, operationId: `${observationId}:observe:${operation.operation_id}`, timeoutMs: 30_000 });
|
|
375
386
|
if (receipt.operation_id !== operation.operation_id || receipt.operation !== method || target && receipt.session_id !== target)
|
|
376
387
|
throw new AppError("controller_operation_mismatch", "Operation observation belongs to another retained dispatch", 1);
|
|
377
|
-
if (!["completed", "failed"].includes(String(receipt.state)))
|
|
388
|
+
if (!["completed", "failed", "interrupted"].includes(String(receipt.state)))
|
|
378
389
|
return { operation_id: operation.operation_id, status: "pending", reason: "controller_operation_unsettled", receipt };
|
|
379
390
|
if (receipt.state === "completed" && (!receipt.result || typeof receipt.result !== "object" || Array.isArray(receipt.result) || target && method !== "daemon.start" && adapterSessionId(receipt.result) !== target || method === "session.create" && !adapterSessionId(receipt.result)))
|
|
380
391
|
throw new AppError("controller_operation_mismatch", "Terminal operation has no valid result for its retained Session", 1);
|
|
@@ -453,7 +464,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
453
464
|
if (!session.id)
|
|
454
465
|
return closed && !session.create_operation_id && Array.isArray(closed.session_ids) && closed.session_ids.length === 0
|
|
455
466
|
? { state_dir: session.state_dir, status: "completed", receipt: closed }
|
|
456
|
-
: { state_dir: session.state_dir, status: "pending", reason: "provider_identity_unknown" };
|
|
467
|
+
: { state_dir: session.state_dir, create_operation_id: session.create_operation_id ?? null, status: "pending", reason: "provider_identity_unknown" };
|
|
457
468
|
bindings = readBindings();
|
|
458
469
|
if (bindings.some(binding => foreign(binding) && underRoot(binding, session.id, harness)))
|
|
459
470
|
return { session_id: session.id, state_dir: session.state_dir, status: "pending", reason: "session_scope_conflict" };
|
|
@@ -486,8 +497,11 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
486
497
|
}
|
|
487
498
|
else {
|
|
488
499
|
const { status, receipt } = await cancelControlledNativeSession({ executable, sessionId: session.id, stateDir: session.state_dir, env, operationId, observe: !inserted.changes, assertCurrent });
|
|
489
|
-
|
|
500
|
+
const terminalFailure = status === "failed" ? receipt.error && typeof receipt.error === "object" && !Array.isArray(receipt.error) ? receipt.error : { code: "native_cancel_failed" } : null;
|
|
501
|
+
context.db.run("UPDATE run_control_operations SET status = ?, receipt_json = ?, error_json = ?, updated_at = ? WHERE operation_id = ?", [status, JSON.stringify(receipt), terminalFailure ? JSON.stringify(terminalFailure) : null, context.now(), operationId]);
|
|
490
502
|
cancellation = receipt;
|
|
503
|
+
if (terminalFailure)
|
|
504
|
+
cancellationError = { code: typeof terminalFailure.code === "string" ? terminalFailure.code : "native_cancel_failed", receipt };
|
|
491
505
|
}
|
|
492
506
|
}
|
|
493
507
|
catch (error) {
|
|
@@ -551,9 +565,14 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
551
565
|
}
|
|
552
566
|
}));
|
|
553
567
|
const [nodes, processNodes, operations] = await Promise.all([sessionNodes, processResults, operationNodes]);
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
568
|
+
// Every whole-RUN stop retires its owned daemon boundary once cancellation
|
|
569
|
+
// has been admitted. `force` controls escalation inside that lifecycle; it
|
|
570
|
+
// must not decide whether the lifecycle exists at all.
|
|
571
|
+
const closeNodes = interrupt && control?.action === "stop" ? await Promise.all([...daemonInventories].map(async ([directory, { state, records }]) => {
|
|
572
|
+
if (closedDaemons.has(directory)) {
|
|
573
|
+
const receipt = closedDaemons.get(directory);
|
|
574
|
+
return { state_dir: directory, status: "completed", operation_id: receipt.operation_id, receipt };
|
|
575
|
+
}
|
|
557
576
|
bindings = readBindings();
|
|
558
577
|
const ids = state.sessions.map(item => typeof item === "string" ? item : adapterSessionId(item));
|
|
559
578
|
const harness = directoryHarnesses.get(directory) ?? null;
|
|
@@ -598,6 +617,11 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
598
617
|
const receipt = await runHarnessAdapter({ executable, args: inserted.changes ? ["daemon", "stop", "--cancel-tree", "--state-dir", directory, "--json"] : ["daemon", "operation", "--operation-id", operationId, "--state-dir", directory, "--json"], env, operationId: inserted.changes ? operationId : `${operationId}:observe`, timeoutMs: 30_000 });
|
|
599
618
|
if (!inserted.changes && (receipt.operation_id !== operationId || receipt.operation !== "daemon.stop" || receipt.daemon_id !== daemonId))
|
|
600
619
|
throw new AppError("controller_operation_mismatch", "Daemon close observation belongs to another operation or incarnation", 1);
|
|
620
|
+
if (receipt.state === "failed") {
|
|
621
|
+
const failure = receipt.error && typeof receipt.error === "object" && !Array.isArray(receipt.error) ? receipt.error : { code: "daemon_close_failed" };
|
|
622
|
+
context.db.run("UPDATE run_control_operations SET status = 'failed', receipt_json = ?, error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), JSON.stringify(failure), context.now(), operationId]);
|
|
623
|
+
return { operation_id: operationId, state_dir: directory, status: "pending", reason: typeof failure.code === "string" ? failure.code : "daemon_close_failed", receipt };
|
|
624
|
+
}
|
|
601
625
|
const observed = inserted.changes ? receipt : receipt.state === "completed" ? receipt.result : null;
|
|
602
626
|
if (observed)
|
|
603
627
|
assertDaemonCloseReceipt(observed);
|
|
@@ -611,6 +635,47 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
611
635
|
return { operation_id: operationId, state_dir: directory, status: "pending", reason: failure.code };
|
|
612
636
|
}
|
|
613
637
|
})) : [];
|
|
638
|
+
// Once the same control owns a clean whole-tree stop, an older productive
|
|
639
|
+
// request cannot gain a provider reply through this daemon. Seal that fact in
|
|
640
|
+
// the shared native journal before the next observation pass. This prevents a
|
|
641
|
+
// closed daemon from becoming a permanent prerequisite for its own recovery.
|
|
642
|
+
const terminalizedOperations = [];
|
|
643
|
+
if (interrupt && control?.action === "stop")
|
|
644
|
+
for (const node of closeNodes) {
|
|
645
|
+
if (node.status !== "completed")
|
|
646
|
+
continue;
|
|
647
|
+
const directory = node.state_dir;
|
|
648
|
+
const inventory = daemonInventories.get(directory);
|
|
649
|
+
const daemonId = inventory?.state.daemon_id ?? inventory?.state.config?.daemonId;
|
|
650
|
+
const executable = directoryAdapters.get(directory);
|
|
651
|
+
if (!inventory)
|
|
652
|
+
continue;
|
|
653
|
+
if (!daemonId || !executable || !node.operation_id) {
|
|
654
|
+
terminalizedOperations.push({ state_dir: directory, status: "pending", reason: "operation_terminalization_binding_unknown" });
|
|
655
|
+
continue;
|
|
656
|
+
}
|
|
657
|
+
const pending = retainedDaemonOperations(directory).filter(operation => operation.daemon_id === daemonId && ["running", "awaiting_provider"].includes(String(operation.state)) && ["session.create", "session.prompt", "session.start", "session.fork", "session.resume"].includes(String(operation.operation)) && typeof operation.operation_id === "string");
|
|
658
|
+
terminalizedOperations.push(...await Promise.all(pending.map(async (operation) => {
|
|
659
|
+
try {
|
|
660
|
+
const operationId = operation.operation_id, sessionId = typeof operation.session_id === "string" ? operation.session_id : null;
|
|
661
|
+
assertCurrent();
|
|
662
|
+
const env = { ...context.env, DD_FLOW_HOME: context.ddFlowHome };
|
|
663
|
+
delete env.DD_FLOW_CONTROL_TOKEN;
|
|
664
|
+
delete env.DD_FLOW_CONTROLLER_TOKEN;
|
|
665
|
+
const interruptArgs = ["daemon", "operation", "interrupt", "--operation-id", operationId, "--control-id", control.control_id, "--stop-operation-id", String(node.operation_id), "--daemon-id", String(daemonId)];
|
|
666
|
+
if (sessionId)
|
|
667
|
+
interruptArgs.push("--session-id", sessionId);
|
|
668
|
+
interruptArgs.push("--state-dir", directory, "--json");
|
|
669
|
+
const receipt = await runHarnessAdapter({ executable, args: interruptArgs, env, operationId: `${observationId}:interrupt-operation:${operationId}`, timeoutMs: 30_000 });
|
|
670
|
+
if (receipt.operation_id !== operationId || receipt.state !== "interrupted" || receipt.daemon_id !== daemonId || receipt.session_id !== sessionId)
|
|
671
|
+
throw new AppError("controller_operation_mismatch", "Interrupted operation receipt belongs to another native request", 1);
|
|
672
|
+
return { operation_id: operationId, state_dir: directory, status: "completed", receipt };
|
|
673
|
+
}
|
|
674
|
+
catch (error) {
|
|
675
|
+
return { operation_id: operation.operation_id, state_dir: directory, status: "pending", reason: error instanceof AppError ? error.code : "operation_terminalization_failed" };
|
|
676
|
+
}
|
|
677
|
+
})));
|
|
678
|
+
}
|
|
614
679
|
let verifyInventory = assertCurrent;
|
|
615
680
|
if (!interrupt) {
|
|
616
681
|
const importVersions = new Map();
|
|
@@ -681,7 +746,16 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
681
746
|
}
|
|
682
747
|
}
|
|
683
748
|
assertCurrent();
|
|
684
|
-
|
|
749
|
+
// A clean daemon close plus a sealed interruption terminalizes an in-flight
|
|
750
|
+
// create even though its controller Session row never received an id. Do
|
|
751
|
+
// not leave the pre-terminalization `provider_identity_unknown` shadow node
|
|
752
|
+
// in the final result.
|
|
753
|
+
const terminalized = new Set(terminalizedOperations.filter(node => node.status === "completed").map(node => node.operation_id));
|
|
754
|
+
const reconciledNodes = nodes.map(node => {
|
|
755
|
+
const createId = "create_operation_id" in node && typeof node.create_operation_id === "string" ? node.create_operation_id : null;
|
|
756
|
+
return createId && terminalized.has(createId) ? { ...node, status: "completed", reason: undefined, receipt: terminalizedOperations.find(item => item.operation_id === createId)?.receipt } : node;
|
|
757
|
+
});
|
|
758
|
+
const allNodes = [...reconciledNodes, ...processNodes, ...operations, ...closeNodes, ...terminalizedOperations, ...unresolved];
|
|
685
759
|
const settled = !interrupt && allNodes.every(node => node.status === "completed");
|
|
686
760
|
if (settled)
|
|
687
761
|
boundary?.retainVerification(verifyInventory);
|
|
@@ -706,11 +780,11 @@ export function retainedDaemonOperations(directory) {
|
|
|
706
780
|
const requested = JSON.parse(fs.readFileSync(path.join(folder, "requested.json"), "utf8"));
|
|
707
781
|
if (requested.schema_id !== "dd-flow/daemon-operation@1" || typeof requested.operation_id !== "string" || crypto.createHash("sha256").update(requested.operation_id).digest("hex") !== entry.name || typeof requested.operation !== "string")
|
|
708
782
|
throw new AppError("daemon_operation_inventory_invalid", "Native operation request has a conflicting identity", 1);
|
|
709
|
-
const outcomePath = ["result.json", "observation-lost.json"].map(file => path.join(folder, file)).find(file => fs.existsSync(file));
|
|
783
|
+
const outcomePath = ["result.json", "interrupted.json", "observation-lost.json"].map(file => path.join(folder, file)).find(file => fs.existsSync(file));
|
|
710
784
|
const outcome = outcomePath ? JSON.parse(fs.readFileSync(outcomePath, "utf8")) : { state: "running" };
|
|
711
|
-
if (!outcome || typeof outcome !== "object" || Array.isArray(outcome) || !["completed", "failed", "running", "awaiting_provider"].includes(String(outcome.state)) || ["operation_id", "operation", "daemon_id", "session_id"].some(key => key in outcome && outcome[key] !== requested[key]))
|
|
785
|
+
if (!outcome || typeof outcome !== "object" || Array.isArray(outcome) || !["completed", "failed", "interrupted", "running", "awaiting_provider"].includes(String(outcome.state)) || ["operation_id", "operation", "daemon_id", "session_id"].some(key => key in outcome && outcome[key] !== requested[key]))
|
|
712
786
|
throw new AppError("daemon_operation_inventory_invalid", "Native operation result has an invalid identity or state", 1);
|
|
713
|
-
if (outcome.state === "completed" && (!outcome.result || typeof outcome.result !== "object" || Array.isArray(outcome.result)) || outcome.state
|
|
787
|
+
if (outcome.state === "completed" && (!outcome.result || typeof outcome.result !== "object" || Array.isArray(outcome.result)) || ["failed", "interrupted"].includes(String(outcome.state)) && (!outcome.error || typeof outcome.error !== "object" || Array.isArray(outcome.error)))
|
|
714
788
|
throw new AppError("daemon_operation_inventory_invalid", "Native operation omits its terminal evidence", 1);
|
|
715
789
|
return { ...requested, ...outcome };
|
|
716
790
|
});
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
3
4
|
import { AppError } from "../shared/errors.js";
|
|
4
5
|
import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
|
|
5
6
|
import { runtimeHarness } from "./execution-policy.js";
|
|
@@ -50,29 +51,155 @@ export function controllerEnvironment(context, manifest, session) {
|
|
|
50
51
|
env.CODEX_HOME = session.codex_home;
|
|
51
52
|
return env;
|
|
52
53
|
}
|
|
54
|
+
function settlement(value) {
|
|
55
|
+
const state = value.settlement;
|
|
56
|
+
return state && typeof state === "object" && !Array.isArray(state) ? state : null;
|
|
57
|
+
}
|
|
58
|
+
function settlementInspectArgs(args) {
|
|
59
|
+
const values = ["session", "inspect"];
|
|
60
|
+
for (let index = 2; index < args.length; index += 1) {
|
|
61
|
+
const value = args[index];
|
|
62
|
+
if (["--prompt", "--prompt-file", "--answers-file", "--target-json"].includes(value)) {
|
|
63
|
+
index += 1;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
values.push(value);
|
|
67
|
+
}
|
|
68
|
+
return values;
|
|
69
|
+
}
|
|
70
|
+
function settlementRetryArgs(args, operationId) {
|
|
71
|
+
const stateDir = args[args.indexOf("--state-dir") + 1];
|
|
72
|
+
if (!stateDir)
|
|
73
|
+
throw new AppError("settlement_recovery_required", "Harness settlement has no state directory", 1, { operation_id: operationId });
|
|
74
|
+
return ["daemon", "operation", "settle", "--state-dir", stateDir, "--operation-id", operationId, "--json"];
|
|
75
|
+
}
|
|
76
|
+
function operationInspectArgs(args, operationId) {
|
|
77
|
+
const stateDir = args[args.indexOf("--state-dir") + 1];
|
|
78
|
+
if (!stateDir)
|
|
79
|
+
throw new AppError("controller_operation_binding_unknown", "Harness operation has no state directory", 1, { operation_id: operationId });
|
|
80
|
+
return ["daemon", "operation", "--state-dir", stateDir, "--operation-id", operationId, "--json"];
|
|
81
|
+
}
|
|
82
|
+
function expectedMethod(args) {
|
|
83
|
+
return `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`;
|
|
84
|
+
}
|
|
85
|
+
function validateReceipt(receipt, session, args) {
|
|
86
|
+
// `daemon start --session-id` names the retained state being reopened; the
|
|
87
|
+
// start receipt itself is a daemon lifecycle result and need not echo a
|
|
88
|
+
// provider Session identity. Only session operations promise that binding.
|
|
89
|
+
const target = args[0] === "session" && args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
|
|
90
|
+
if (target && adapterSessionId(receipt) !== target)
|
|
91
|
+
throw new AppError("session_identity_mismatch", "Managed adapter reply belongs to another or unidentified Session", 1);
|
|
92
|
+
if (receipt.harness !== undefined && receipt.harness !== runtimeHarness(session.profile))
|
|
93
|
+
throw new AppError("profile_integrity_violation", "Managed adapter returned another harness identity", 1);
|
|
94
|
+
const fork = receipt.fork_receipt;
|
|
95
|
+
const forkUnproven = (args[0] === "session" && args[1] === "fork" && receipt.fork_continuity_error !== undefined)
|
|
96
|
+
|| (fork && typeof fork === "object" && !Array.isArray(fork) && fork.history_prefix_proven === false);
|
|
97
|
+
if (forkUnproven) {
|
|
98
|
+
throw new AppError("fork_continuity_unproven", "Native fork exists but its source-history continuity is not proven", 1);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
async function recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal) {
|
|
102
|
+
// Creation has no stable client Session id yet. Its durable daemon journal
|
|
103
|
+
// is reconciled by run-control after the owner is fenced; probing it from the
|
|
104
|
+
// failing transport path would be a second adapter invocation and can itself
|
|
105
|
+
// be mistaken for replay.
|
|
106
|
+
if (args[0] === "session" && args[1] === "create")
|
|
107
|
+
return null;
|
|
108
|
+
// A detached adapter without a native daemon has no durable operation
|
|
109
|
+
// journal to inspect. Avoid a second adapter invocation in that case; the
|
|
110
|
+
// original failure remains outcome-unknown and the controller will fence it.
|
|
111
|
+
if (!fs.existsSync(path.join(session.state_dir, "daemon.json")))
|
|
112
|
+
return null;
|
|
113
|
+
let observed;
|
|
114
|
+
try {
|
|
115
|
+
observed = await runHarnessAdapter({ executable, args: operationInspectArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:observe`, runtimeOwner: session.owner, timeoutMs: 30_000, ...(signal ? { signal } : {}) });
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
|
|
121
|
+
const result = observed.result;
|
|
122
|
+
if (observed.operation_id !== operationId || observed.operation !== expectedMethod(args) || observed.state !== "completed"
|
|
123
|
+
|| !result || typeof result !== "object" || Array.isArray(result)
|
|
124
|
+
|| target && (observed.session_id !== target || adapterSessionId(result) !== target))
|
|
125
|
+
return null;
|
|
126
|
+
return { ...result, ...(settlement(observed) ? { settlement: settlement(observed) } : {}) };
|
|
127
|
+
}
|
|
128
|
+
async function awaitSettlement(context, row, manifest, session, executable, args, operationId, initial, assertOwner, signal) {
|
|
129
|
+
let current = initial, delay = 250;
|
|
130
|
+
const deadline = Date.now() + 120_000;
|
|
131
|
+
while (["pending", "blocked"].includes(String(settlement(current)?.state))) {
|
|
132
|
+
if (signal?.aborted)
|
|
133
|
+
throw new AppError("harness_settlement_interrupted", "Harness settlement was interrupted", 1, { operation_id: operationId });
|
|
134
|
+
if (Date.now() >= deadline)
|
|
135
|
+
throw new AppError("settlement_recovery_required", "Completed provider operation requires settlement recovery", 1, { operation_id: operationId, settlement: settlement(current) });
|
|
136
|
+
await new Promise(resolve => setTimeout(resolve, Math.min(delay, Math.max(0, deadline - Date.now()))));
|
|
137
|
+
delay = Math.min(delay * 2, 8_000);
|
|
138
|
+
assertOwner?.();
|
|
139
|
+
if (Date.now() >= deadline)
|
|
140
|
+
continue;
|
|
141
|
+
try {
|
|
142
|
+
// This only retries durable bookkeeping against the same operation
|
|
143
|
+
// receipt. It cannot replay the provider request.
|
|
144
|
+
current = await runHarnessAdapter({ executable, args: settlementRetryArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement`, runtimeOwner: session.owner, timeoutMs: Math.min(30_000, Math.max(1, deadline - Date.now())), ...(signal ? { signal } : {}) });
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
|
|
148
|
+
}
|
|
149
|
+
if (!(["pending", "blocked"].includes(String(settlement(current)?.state))))
|
|
150
|
+
continue;
|
|
151
|
+
// A receipt without final tree proof needs a fresh native inspection. Do
|
|
152
|
+
// it only after the receipt-only retry, and only when this is a session.
|
|
153
|
+
if (!args.includes("--session-id") || !["tree_unsettled", "settlement_evidence_required"].includes(String(settlement(current)?.reason)) || Date.now() >= deadline)
|
|
154
|
+
continue;
|
|
155
|
+
try {
|
|
156
|
+
current = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-inspect`, runtimeOwner: session.owner, timeoutMs: Math.min(30_000, Math.max(1, deadline - Date.now())), ...(signal ? { signal } : {}) });
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return current;
|
|
163
|
+
}
|
|
53
164
|
export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal) {
|
|
54
165
|
assertOwner?.();
|
|
55
166
|
const operationId = retainedOperationId ?? `${row.controller_id}:${crypto.randomUUID()}`;
|
|
56
167
|
const executable = manifest.adapters[runtimeHarness(session.profile)].executable;
|
|
57
|
-
const
|
|
168
|
+
const promptFile = args.indexOf("--prompt-file"), inlinePrompt = args.indexOf("--prompt");
|
|
169
|
+
const promptSha256 = promptFile >= 0 ? digestBytes(fs.readFileSync(args[promptFile + 1])) : inlinePrompt >= 0 ? digestBytes(args[inlinePrompt + 1] ?? "") : undefined;
|
|
170
|
+
const input = { executable, args, owner: session.owner.owner_id, state_dir: session.state_dir, ...(args[0] === "session" && args[1] === "prompt" && promptSha256 ? { prompt_sha256: promptSha256 } : {}) };
|
|
58
171
|
const now = context.now();
|
|
59
172
|
context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'dispatching', ?, ?)", [operationId, row.controller_id, row.generation, digest(input), JSON.stringify(input), now, now]);
|
|
60
173
|
appendEvent(context, row.controller_id, "operation_requested", { operation_id: operationId, operation: args.slice(0, 2), session_id: session.id || null });
|
|
61
174
|
try {
|
|
62
175
|
assertOwner?.();
|
|
63
|
-
|
|
176
|
+
let receipt = await runHarnessAdapter({ executable, args, env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs: args[0] === "session" && args[1] === "prompt" ? null : 120_000, ...(signal ? { signal } : {}) });
|
|
177
|
+
if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
|
|
178
|
+
const recovered = await awaitSettlement(context, row, manifest, session, executable, args, operationId, receipt, assertOwner, signal);
|
|
179
|
+
receipt = { ...receipt, settlement: settlement(recovered) };
|
|
180
|
+
}
|
|
64
181
|
context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
|
|
65
|
-
|
|
66
|
-
if (args[0] === "session" && args.includes("--session-id") && adapterSessionId(receipt) !== target)
|
|
67
|
-
throw new AppError("session_identity_mismatch", "Managed adapter reply belongs to another or unidentified Session", 1);
|
|
68
|
-
if (receipt.harness !== undefined && receipt.harness !== runtimeHarness(session.profile))
|
|
69
|
-
throw new AppError("profile_integrity_violation", "Managed adapter returned another harness identity", 1);
|
|
182
|
+
validateReceipt(receipt, session, args);
|
|
70
183
|
context.db.run("UPDATE run_controller_operations SET status = 'completed', updated_at = ? WHERE operation_id = ?", [context.now(), operationId]);
|
|
71
184
|
assertOwner?.();
|
|
72
185
|
appendEvent(context, row.controller_id, "operation_completed", { operation_id: operationId, operation: args.slice(0, 2), session_id: adapterSessionId(receipt) });
|
|
73
186
|
return receipt;
|
|
74
187
|
}
|
|
75
188
|
catch (error) {
|
|
189
|
+
const retained = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
|
|
190
|
+
if (retained) {
|
|
191
|
+
let receipt = retained;
|
|
192
|
+
if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
|
|
193
|
+
const recovered = await awaitSettlement(context, row, manifest, session, executable, args, operationId, receipt, assertOwner, signal);
|
|
194
|
+
receipt = { ...receipt, settlement: settlement(recovered) };
|
|
195
|
+
}
|
|
196
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
|
|
197
|
+
validateReceipt(receipt, session, args);
|
|
198
|
+
context.db.run("UPDATE run_controller_operations SET status = 'completed', error_json = NULL, updated_at = ? WHERE operation_id = ?", [context.now(), operationId]);
|
|
199
|
+
assertOwner?.();
|
|
200
|
+
appendEvent(context, row.controller_id, "operation_recovered", { operation_id: operationId, operation: args.slice(0, 2), session_id: adapterSessionId(receipt) });
|
|
201
|
+
return receipt;
|
|
202
|
+
}
|
|
76
203
|
try {
|
|
77
204
|
context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
|
|
78
205
|
}
|
|
@@ -20,8 +20,9 @@ import { confirmManagedProcess, heartbeatManagedProcess, finishManagedProcess, p
|
|
|
20
20
|
import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
21
21
|
import { initCodexHome } from "./hooks.js";
|
|
22
22
|
import { launchExternalWork } from "./external-work-launch.js";
|
|
23
|
-
import { controllerNativeChildren, nextControllerFanout } from "./controller-fanout.js";
|
|
23
|
+
import { controllerNativeChildren, controllerStageEntryPrompt, nextControllerFanout } from "./controller-fanout.js";
|
|
24
24
|
import { claimQueuedMergeRequest } from "./merge-server.js";
|
|
25
|
+
import { prepareVnextCodeWorkspace } from "./vnext-code.js";
|
|
25
26
|
import { requireRun, requireController, controllerReceipt, appendEvent, saveState, digest, digestBytes, errorRecord } from "./run-controller-state.js";
|
|
26
27
|
import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady } from "./run-controller-process.js";
|
|
27
28
|
import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
|
|
@@ -405,7 +406,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
405
406
|
throw fatal; checkOutcomes(); assertOwnership(); };
|
|
406
407
|
const policy = manifest.execution_profile.settings.execution;
|
|
407
408
|
const profiles = manifest.execution_profile.agent_profiles;
|
|
408
|
-
let executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined } };
|
|
409
|
+
let executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined, DD_FLOW_CODE_READINESS_OWNER: row.controller_id } };
|
|
409
410
|
const bindInvocationScope = (session) => {
|
|
410
411
|
executionContext = managedInvocationContext(executionContext, { harness: runtimeHarness(session.profile), stateDir: session.state_dir, sessionId: session.id,
|
|
411
412
|
projectRoot: manifest.request.project_root, runId: row.run_id, generation: row.generation });
|
|
@@ -631,10 +632,17 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
631
632
|
state.last_continuation = null;
|
|
632
633
|
saveState(context, row, state);
|
|
633
634
|
setStatus("running");
|
|
635
|
+
if (next.stage === "code" || next.stage === "code-review") {
|
|
636
|
+
// A CODE_REVIEW fork has source files but intentionally no installed
|
|
637
|
+
// dependencies. Its repair Work executes checks just like CODE Work.
|
|
638
|
+
const readiness = await prepareVnextCodeWorkspace(executionContext, { projectRoot: run.project_root, runId: run.id, ownerId: row.controller_id, stage: next.stage });
|
|
639
|
+
assertOwner();
|
|
640
|
+
appendEvent(context, row.controller_id, "code_workspace_prepared", { stage: next.stage, readiness_status: readiness.status ?? null });
|
|
641
|
+
}
|
|
634
642
|
const responseFile = path.join(row.state_dir, `${next.stage}-${next.attempt ?? 1}.stage-start-response.json`);
|
|
635
643
|
const command = managedLifecycleCommand(executionContext, `${flowCommand(executionContext)} stage start ${run.id} --stage ${next.stage} --project-root ${JSON.stringify(run.project_root)} --context-file ${JSON.stringify(stageContext.file)} --context-sha256 ${stageContext.sha256} --require-session-binding --response-file ${JSON.stringify(responseFile)} --json --progress-jsonl`);
|
|
636
644
|
appendEvent(context, row.controller_id, "stage_entered", { stage: next.stage, session_id: session.id, context_sha256: stageContext.sha256 });
|
|
637
|
-
await prompt(session,
|
|
645
|
+
await prompt(session, controllerStageEntryPrompt(next.stage, command, responseFile));
|
|
638
646
|
continue;
|
|
639
647
|
}
|
|
640
648
|
if (state.current_session === null)
|