@deksden-com/dd-flow-cli 0.9.0-beta.119 → 0.9.0-beta.121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/build-info.json +3 -3
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +3 -2
- package/dist/harness-runtime/lib/dd-codex.d.mts +1 -0
- package/dist/harness-runtime/lib/dd-codex.mjs +100 -23
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +12 -2
- package/dist/harness-runtime/lib/dd-opencode.mjs +2 -2
- package/dist/harness-runtime/lib/dd-zcode.mjs +5 -3
- package/dist/services/code-checks.js +40 -0
- package/dist/services/codex-capacity.js +71 -0
- package/dist/services/controller-fanout.js +13 -1
- package/dist/services/external-work-launch.js +37 -3
- package/dist/services/managed-processes.js +42 -6
- package/dist/services/merge-server.js +34 -4
- package/dist/services/run-control.js +7 -1
- package/dist/services/run-controller-adapter.js +20 -2
- package/dist/services/run-controller-recovery.js +47 -3
- package/dist/services/run-controller.js +105 -10
- package/dist/services/run-recovery-runtime.js +43 -2
- package/dist/services/runtime-scope-stop.js +23 -2
- package/dist/services/vnext-code.js +7 -1
- package/dist/services/vnext-plan.js +29 -14
- package/package.json +1 -1
|
@@ -61,7 +61,12 @@ export function prepareManagedProcessRegistration(context, input) {
|
|
|
61
61
|
throw new AppError("validation", "Unique managed operation requires operationId", 2);
|
|
62
62
|
let metadataJson;
|
|
63
63
|
try {
|
|
64
|
-
|
|
64
|
+
const metadata = { ...input.metadata, dd_flow_home: context.ddFlowHome, ...(input.ownerPid ? { owner_pid: input.ownerPid, owner_pid_started_at: processStartedAt(input.ownerPid) } : {}) };
|
|
65
|
+
delete metadata.process_birth;
|
|
66
|
+
delete metadata.owner_birth;
|
|
67
|
+
if (input.ownerPid)
|
|
68
|
+
metadata.owner_birth = stableProcessBirth(input.ownerPid);
|
|
69
|
+
metadataJson = JSON.stringify(metadata);
|
|
65
70
|
}
|
|
66
71
|
catch {
|
|
67
72
|
throw new AppError("validation", "metadata must be JSON-serializable", 2);
|
|
@@ -86,7 +91,8 @@ export function registerManagedProcess(context, rawInput) {
|
|
|
86
91
|
if (!input.uniqueActiveOperation)
|
|
87
92
|
return insertManagedProcess(context, input, metadataJson);
|
|
88
93
|
const db = registry(context);
|
|
89
|
-
|
|
94
|
+
const observerProcessBirth = observer ? stableProcessBirth(input.pid) : null;
|
|
95
|
+
if (observer && !processIsAlive({ pid: input.pid, pid_started_at: observerBirth, metadata_json: JSON.stringify({ process_birth: observerProcessBirth }) }))
|
|
90
96
|
throw new AppError("process_ownership_unknown", "EVAL observer changed before registration", 1);
|
|
91
97
|
return db.writeTransaction(() => {
|
|
92
98
|
const existing = db.get("SELECT * FROM managed_processes WHERE operation_id = ? AND state IN ('starting','running','stopping','orphaned')", [input.operationId]);
|
|
@@ -97,6 +103,7 @@ export function registerManagedProcess(context, rawInput) {
|
|
|
97
103
|
const metadata = JSON.parse(metadataJson);
|
|
98
104
|
if (input.processGroupId)
|
|
99
105
|
metadata.process_group_id = input.processGroupId;
|
|
106
|
+
metadata.process_birth = observerProcessBirth;
|
|
100
107
|
const now = context.now();
|
|
101
108
|
db.run("UPDATE managed_processes SET pid = ?, pid_started_at = ?, state = 'running', lease_expires_at = ?, updated_at = ?, metadata_json = ? WHERE id = ? AND lease_token = ? AND state = 'starting'", [input.pid, observerBirth, leaseExpiry(now, input.leaseMs), now, JSON.stringify(metadata), result.id, result.lease_token]);
|
|
102
109
|
result = requireProcess(db, result.id);
|
|
@@ -137,9 +144,11 @@ export function confirmManagedProcess(context, input) {
|
|
|
137
144
|
const metadata = parseMetadata(current.metadata_json);
|
|
138
145
|
if (input.processGroupId)
|
|
139
146
|
metadata.process_group_id = input.processGroupId;
|
|
147
|
+
metadata.process_birth = stableProcessBirth(input.pid);
|
|
140
148
|
if (input.ownerPid) {
|
|
141
149
|
metadata.owner_pid = input.ownerPid;
|
|
142
150
|
metadata.owner_pid_started_at = processStartedAt(input.ownerPid);
|
|
151
|
+
metadata.owner_birth = stableProcessBirth(input.ownerPid);
|
|
143
152
|
}
|
|
144
153
|
const result = db.run(`UPDATE managed_processes
|
|
145
154
|
SET pid = ?, pid_started_at = ?, state = 'running', lease_expires_at = ?, updated_at = ?, metadata_json = ?
|
|
@@ -216,6 +225,11 @@ export function processIsAlive(record) {
|
|
|
216
225
|
catch (error) {
|
|
217
226
|
return error.code !== "ESRCH";
|
|
218
227
|
}
|
|
228
|
+
const metadata = record.metadata_json ? parseMetadata(record.metadata_json) : {};
|
|
229
|
+
if (Object.hasOwn(metadata, "process_birth")) {
|
|
230
|
+
const birth = parseBirth(metadata.process_birth), observed = stableProcessBirth(record.pid);
|
|
231
|
+
return !birth || !observed || observed === birth;
|
|
232
|
+
}
|
|
219
233
|
const started = processStartedAt(record.pid);
|
|
220
234
|
return !record.pid_started_at || !started || record.pid_started_at === started;
|
|
221
235
|
}
|
|
@@ -223,6 +237,17 @@ export function processTreeIsAlive(record) {
|
|
|
223
237
|
if (processIsAlive(record))
|
|
224
238
|
return true;
|
|
225
239
|
const group = parseMetadata(record.metadata_json).process_group_id;
|
|
240
|
+
const birth = parseBirth(parseMetadata(record.metadata_json).process_birth);
|
|
241
|
+
// POSIX cannot reuse a leader PID while its old process group survives.
|
|
242
|
+
if (process.platform !== "win32" && group === record.pid && record.pid && birth) {
|
|
243
|
+
try {
|
|
244
|
+
process.kill(record.pid, 0);
|
|
245
|
+
}
|
|
246
|
+
catch { /* A missing leader can still have living children. */ }
|
|
247
|
+
const observed = stableProcessBirth(record.pid);
|
|
248
|
+
if (observed && observed !== birth)
|
|
249
|
+
return false;
|
|
250
|
+
}
|
|
226
251
|
if (process.platform !== "win32" && typeof group === "number") {
|
|
227
252
|
try {
|
|
228
253
|
process.kill(-group, 0);
|
|
@@ -238,7 +263,7 @@ export function managedOwnerIsAlive(record) {
|
|
|
238
263
|
const metadata = parseMetadata(record.metadata_json);
|
|
239
264
|
if (typeof metadata.owner_pid !== "number")
|
|
240
265
|
return null;
|
|
241
|
-
return processIsAlive({ pid: metadata.owner_pid, pid_started_at: typeof metadata.owner_pid_started_at === "string" ? metadata.owner_pid_started_at : null });
|
|
266
|
+
return processIsAlive({ pid: metadata.owner_pid, pid_started_at: typeof metadata.owner_pid_started_at === "string" ? metadata.owner_pid_started_at : null, metadata_json: JSON.stringify({ process_birth: metadata.owner_birth }) });
|
|
242
267
|
}
|
|
243
268
|
/**
|
|
244
269
|
* Expiry proves that observation stopped, not that the owner died. A live PID
|
|
@@ -413,16 +438,27 @@ function leaseExpiry(now, leaseMs = defaultLeaseMs) {
|
|
|
413
438
|
return new Date(Date.parse(now) + leaseMs).toISOString();
|
|
414
439
|
}
|
|
415
440
|
export function processStartedAt(pid) { const result = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8", timeout: 1_000 }); const value = result.status === 0 ? result.stdout.trim() : ""; return value || null; }
|
|
441
|
+
function stableProcessBirth(pid) {
|
|
442
|
+
const result = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8", timeout: 1_000, env: { ...process.env, TZ: "UTC", LC_ALL: "C" } });
|
|
443
|
+
const value = result.status === 0 ? result.stdout.trim() : "";
|
|
444
|
+
return value ? `ps-lstart-utc-c@1:${value}` : null;
|
|
445
|
+
}
|
|
446
|
+
function parseBirth(value) { return typeof value === "string" && value.startsWith("ps-lstart-utc-c@1:") && value.length > "ps-lstart-utc-c@1:".length ? value : null; }
|
|
416
447
|
function releasePortClaims(db, claims) { for (const claim of claims)
|
|
417
448
|
db.run("DELETE FROM managed_resources WHERE resource_kind = 'port' AND resource_key = ? AND lease_token = ?", [claim.key, claim.token]); }
|
|
418
449
|
function terminateOwnedProcess(record, signal) {
|
|
419
450
|
// Never signal a group merely because its former leader PID was once ours.
|
|
420
451
|
// PID reuse or a departed leader makes group ownership unprovable.
|
|
421
|
-
if (!record.pid || !record.pid_started_at
|
|
452
|
+
if (!record.pid || !record.pid_started_at)
|
|
422
453
|
return;
|
|
423
|
-
const
|
|
454
|
+
const metadata = parseMetadata(record.metadata_json);
|
|
455
|
+
const birth = parseBirth(metadata.process_birth);
|
|
456
|
+
if (Object.hasOwn(metadata, "process_birth") ? !birth || stableProcessBirth(record.pid) !== birth : processStartedAt(record.pid) !== record.pid_started_at)
|
|
457
|
+
return;
|
|
458
|
+
const group = metadata.process_group_id;
|
|
459
|
+
// A non-leader PID does not establish ownership of its whole process group.
|
|
424
460
|
try {
|
|
425
|
-
process.kill(process.platform === "win32" ||
|
|
461
|
+
process.kill(process.platform === "win32" || group !== record.pid ? record.pid : -group, signal);
|
|
426
462
|
}
|
|
427
463
|
catch { /* inspect again below */ }
|
|
428
464
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { managedInvocationContext, managedLifecycleCommand } from "./lifecycle-invocations.js";
|
|
2
2
|
import { stageLifecycleInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
|
|
3
|
+
import { continueCodexCapacity } from "./codex-capacity.js";
|
|
4
|
+
import { errorRecord } from "./run-controller-state.js";
|
|
3
5
|
import crypto from "node:crypto";
|
|
4
6
|
import fs from "node:fs";
|
|
5
7
|
import path from "node:path";
|
|
@@ -110,7 +112,7 @@ async function dispatch(context, input) {
|
|
|
110
112
|
return; const now = context.now(); context.db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE merge_request_id = ? AND status = 'dispatching' AND dispatch_owner = ? AND dispatch_lease_token = ?", [new Date(Date.now() + 120_000).toISOString(), now, input.request.merge_request_id, input.serverId, token]); }, 30_000);
|
|
111
113
|
leaseHeartbeat.unref();
|
|
112
114
|
try {
|
|
113
|
-
const adapterCall = (args, timeoutMs, progressMessage) => { assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation); return runHarnessAdapter({ executable, args, env: runtimeContext.env, runtimeOwner, operationId:
|
|
115
|
+
const adapterCall = (args, timeoutMs, progressMessage, id = `${input.request.merge_request_id}:${token}:${args[0]}.${args[1]}`) => { assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation); return runHarnessAdapter({ executable, args, env: runtimeContext.env, runtimeOwner, operationId: id, ...(timeoutMs ? { timeoutMs } : {}), ...(input.progress ? { progress: input.progress } : {}), ...(progressMessage ? { progressMessage } : {}), onEvidence: (value) => event(input.journal, { type: "adapter_call", ...value }) }); };
|
|
114
116
|
await adapterCall(["daemon", "start", ...common]);
|
|
115
117
|
const created = await adapterCall(["session", "create", ...common], runtimeHarness(input.profile) === "zcode-acp" ? 180_000 : undefined);
|
|
116
118
|
const sessionId = adapterSessionId(created);
|
|
@@ -121,7 +123,35 @@ async function dispatch(context, input) {
|
|
|
121
123
|
runtimeContext = managedInvocationContext(runtimeContext, { harness: runtimeHarness(input.profile), stateDir, sessionId, projectRoot: run.project_root, runId: input.request.run_id, generation });
|
|
122
124
|
const stageCommand = managedLifecycleCommand(runtimeContext, `${flowCommand(runtimeContext)} stage start ${input.request.run_id} --stage merge --project-root ${JSON.stringify(run.project_root)} --json --progress-jsonl`);
|
|
123
125
|
fs.writeFileSync(promptFile, `Start the assigned MERGE stage now. Your first tool call must be this exact standalone command:\n\n${stageCommand}\n\nTrust and follow the complete stage packet it returns. Continue through merge apply and stage finish. Stop only when the stage reaches a terminal result or explicitly requests user/operator action.\n\n${stageLifecycleInstruction}\n`);
|
|
124
|
-
|
|
126
|
+
let prompted;
|
|
127
|
+
try {
|
|
128
|
+
prompted = await adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], 45 * 60 * 1000, `MERGE ${input.request.merge_request_id} Session ${sessionId} is still running`);
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
if (runtimeHarness(input.profile) !== "codex-desktop")
|
|
132
|
+
throw error;
|
|
133
|
+
const resumed = await continueCodexCapacity({ error, sessionId, operationId: String(retained.prompt_operation_id),
|
|
134
|
+
inspectOperation: id => adapterCall(["daemon", "operation", "--state-dir", stateDir, "--operation-id", id, "--json"], 30_000, undefined, `${id}:capacity-observe`),
|
|
135
|
+
inspectSession: () => adapterCall(["session", "inspect", "--session-id", sessionId, ...common], 30_000, undefined, `${input.request.merge_request_id}:${token}:capacity-inspect`),
|
|
136
|
+
assertCurrent: () => {
|
|
137
|
+
assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation);
|
|
138
|
+
const current = context.db.get("SELECT status, dispatch_owner, dispatch_lease_token, dispatch_lease_expires_at, target_workspace FROM merge_requests WHERE merge_request_id = ?", [input.request.merge_request_id]);
|
|
139
|
+
if (current?.status !== "dispatching" || current.dispatch_owner !== input.serverId || current.dispatch_lease_token !== token
|
|
140
|
+
|| current.dispatch_lease_expires_at <= context.now() || current.target_workspace !== input.request.target_workspace)
|
|
141
|
+
throw new AppError("capacity_continuation_blocked", "MERGE dispatch owner or target changed during capacity wait", 1, { request_id: input.request.merge_request_id });
|
|
142
|
+
if (context.db.get("SELECT status FROM works WHERE work_id = ?", [input.request.executor_work_id])?.status !== "created")
|
|
143
|
+
throw new AppError("capacity_continuation_blocked", "MERGE Work already changed state", 1);
|
|
144
|
+
},
|
|
145
|
+
retainIntent: intent => { retained.capacity_intent = intent; retain(); },
|
|
146
|
+
prompt: (id, text, ordinal) => {
|
|
147
|
+
const file = path.join(stateDir, `capacity-${ordinal}.md`);
|
|
148
|
+
fs.writeFileSync(file, `${text}\n`, { flag: "wx", mode: 0o600 });
|
|
149
|
+
return adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", file, ...common], 45 * 60 * 1000, `MERGE ${input.request.merge_request_id} Session ${sessionId} capacity continuation ${ordinal}/2`, id);
|
|
150
|
+
} });
|
|
151
|
+
if (!resumed)
|
|
152
|
+
throw error;
|
|
153
|
+
prompted = resumed;
|
|
154
|
+
}
|
|
125
155
|
Object.assign(retained, { response: prompted });
|
|
126
156
|
retain();
|
|
127
157
|
assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation);
|
|
@@ -137,9 +167,9 @@ async function dispatch(context, input) {
|
|
|
137
167
|
}
|
|
138
168
|
const request = context.db.get("SELECT status FROM merge_requests WHERE merge_request_id = ?", [input.request.merge_request_id]);
|
|
139
169
|
if (request?.status === "dispatching")
|
|
140
|
-
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_launch_failed", error:
|
|
170
|
+
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_launch_failed", error: errorRecord(error) }), context.now(), input.request.merge_request_id]);
|
|
141
171
|
else if (request?.status === "active")
|
|
142
|
-
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_lost_after_stage_start", error:
|
|
172
|
+
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_lost_after_stage_start", error: errorRecord(error) }), context.now(), input.request.merge_request_id]);
|
|
143
173
|
throw error;
|
|
144
174
|
}
|
|
145
175
|
finally {
|
|
@@ -11,7 +11,7 @@ import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
|
|
|
11
11
|
import { runtimeHarness } from "./execution-policy.js";
|
|
12
12
|
import { harnessKey } from "./harness-config.js";
|
|
13
13
|
import { finishManagedProcess, managedLeaseReleased, managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive, resourceHome, stopManagedProcess } from "./managed-processes.js";
|
|
14
|
-
import { reconcileUnfinishedChecks } from "./code-checks.js";
|
|
14
|
+
import { reconcileUnfinishedChecks, terminalCheckSettled } from "./code-checks.js";
|
|
15
15
|
import { controlStatusReceipt } from "./run-control-receipt.js";
|
|
16
16
|
import { assertNativeCancellationReceipt, cancelControlledNativeSession, inspectControlledNativeSession } from "./native-session-control.js";
|
|
17
17
|
/** Durable projection only: status never probes a provider or acquires an owner. */
|
|
@@ -545,6 +545,12 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
545
545
|
return { process_id: record.id, kind: record.kind, status: "pending", reason: "managed_lease_unsettled" };
|
|
546
546
|
if (!record.pid || !record.pid_started_at)
|
|
547
547
|
return { process_id: record.id, kind: record.kind, status: "pending", reason: "managed_process_identity_unknown" };
|
|
548
|
+
// Code checks mark their process terminal only after proving its tree is
|
|
549
|
+
// empty, then publish the exact check receipt. A later PGID reuse cannot
|
|
550
|
+
// turn that already-settled check back into an owned live process.
|
|
551
|
+
const check = record.kind === "check" && record.project_id === project.id && record.run_id === input.runId ? terminalCheckSettled(context, record) : null;
|
|
552
|
+
if (check)
|
|
553
|
+
return { process_id: record.id, kind: record.kind, status: "completed", receipt: { source: "terminal_check_tree_settled", ...managedProcessBinding(record, home), check_id: record.check_id, finished_at: check.finished_at } };
|
|
548
554
|
if (!interrupt && processTreeIsAlive(record))
|
|
549
555
|
return { process_id: record.id, kind: record.kind, status: "pending", reason: "managed_process_running" };
|
|
550
556
|
if (boundary)
|
|
@@ -15,14 +15,32 @@ export function controllerHitlAdmission(context, record, input) {
|
|
|
15
15
|
if (!row || row.generation !== JSON.parse(record.metadata_json).run_generation)
|
|
16
16
|
return false;
|
|
17
17
|
const suffix = `:prompt:${row.generation}`;
|
|
18
|
-
|
|
18
|
+
const successor = /^(.*):capacity:([12]):session\.prompt$/.exec(input.operationId);
|
|
19
|
+
const baseId = successor ? successor[1] : input.operationId;
|
|
20
|
+
if (!baseId.endsWith(suffix))
|
|
19
21
|
return false;
|
|
22
|
+
if (successor) {
|
|
23
|
+
const ordinal = Number(successor[2]);
|
|
24
|
+
const predecessorId = ordinal === 1 ? baseId : `${baseId}:capacity:1:session.prompt`;
|
|
25
|
+
const predecessor = context.db.get("SELECT status, receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [predecessorId, row.controller_id, row.generation]);
|
|
26
|
+
if (predecessor?.status !== "failed" || !predecessor.receipt_json)
|
|
27
|
+
return false;
|
|
28
|
+
let retained;
|
|
29
|
+
try {
|
|
30
|
+
retained = JSON.parse(predecessor.receipt_json);
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
if (retained.capacity_intent?.predecessor !== predecessorId || retained.capacity_intent.operation_id !== input.operationId || retained.capacity_intent.ordinal !== ordinal)
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
20
38
|
const state = JSON.parse(row.state_json);
|
|
21
39
|
const session = state.current_session === null ? undefined : state.sessions[state.current_session];
|
|
22
40
|
if (!session || session.stopped || session.id !== input.nativeSessionId || session.state_dir !== input.stateDir || session.owner.owner_id !== record.owner_id)
|
|
23
41
|
return false;
|
|
24
42
|
const operation = (id) => context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'dispatching'", [id, row.controller_id, row.generation]);
|
|
25
|
-
const answer = context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation <= ? AND status = 'accepted'", [
|
|
43
|
+
const answer = context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation <= ? AND status = 'accepted'", [baseId.slice(0, -suffix.length), row.controller_id, row.generation]);
|
|
26
44
|
const prompt = operation(input.operationId);
|
|
27
45
|
if (!answer || !prompt)
|
|
28
46
|
return false;
|
|
@@ -8,11 +8,13 @@ import { requireProjectByRoot } from "./projects.js";
|
|
|
8
8
|
import { loadExternalStageContext } from "./stage-context.js";
|
|
9
9
|
import { verifyRecoverySnapshot } from "./eval-snapshots.js";
|
|
10
10
|
import { recoveryGuard, resumeRunRecovery } from "./run-recovery.js";
|
|
11
|
-
import { prepareRunRecoveryDispatch } from "./run-recovery-runtime.js";
|
|
11
|
+
import { prepareRunRecoveryDispatch, retainRecoveryCapacityIntent } from "./run-recovery-runtime.js";
|
|
12
12
|
import { runtimeHarness } from "./execution-policy.js";
|
|
13
13
|
import { managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive, processStartedAt } from "./managed-processes.js";
|
|
14
14
|
import { requireRun, requireController, controllerReceipt, appendEvent, saveState, digest, errorRecord } from "./run-controller-state.js";
|
|
15
|
-
import { controllerAdapter, sessionArguments } from "./run-controller-adapter.js";
|
|
15
|
+
import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
|
|
16
|
+
import { runHarnessAdapter } from "./harness-adapter.js";
|
|
17
|
+
import { continueCodexCapacity } from "./codex-capacity.js";
|
|
16
18
|
import { spawnController, recordControllerLauncher } from "./run-controller-process.js";
|
|
17
19
|
import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
18
20
|
import { managedInvocationContext } from "./lifecycle-invocations.js";
|
|
@@ -194,7 +196,49 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
194
196
|
fs.writeFileSync(file, dispatch.prompt, { flag: "wx", mode: 0o600 });
|
|
195
197
|
state.turns += 1;
|
|
196
198
|
saveState(context, row, state);
|
|
197
|
-
const
|
|
199
|
+
const sendAck = (operationId, prompt) => {
|
|
200
|
+
let promptFile = file;
|
|
201
|
+
if (operationId !== dispatch.operation_id) {
|
|
202
|
+
promptFile = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
|
|
203
|
+
fs.writeFileSync(promptFile, prompt, { flag: "wx", mode: 0o600 });
|
|
204
|
+
state.turns += 1;
|
|
205
|
+
saveState(context, row, state);
|
|
206
|
+
}
|
|
207
|
+
return call(["session", "prompt", "--session-id", session.id, "--prompt-file", promptFile, ...sessionArguments(context, manifest, session)], operationId);
|
|
208
|
+
};
|
|
209
|
+
let receipt;
|
|
210
|
+
try {
|
|
211
|
+
receipt = await sendAck(dispatch.operation_id, dispatch.prompt);
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
if (runtimeHarness(session.profile) !== "codex-desktop")
|
|
215
|
+
throw error;
|
|
216
|
+
const continuationText = `The preceding native recovery ACK Turn failed before any observed tool or file effect. This is the same retained Session and recovery generation. First inspect the current RUN recovery state; only if the exact ACK still awaits acceptance, execute the exact standalone acceptance command below once. Never deliver it again after acceptance. Then stop at the original Stage or Work-graph boundary.\n\n${dispatch.prompt}`;
|
|
217
|
+
const inspect = (args, operationId) => runHarnessAdapter({ executable: manifest.adapters["codex-desktop"].executable,
|
|
218
|
+
args, env: controllerEnvironment(executionContext, manifest, session), runtimeOwner: session.owner, operationId, timeoutMs: 30_000 });
|
|
219
|
+
const resumed = await continueCodexCapacity({ error, sessionId: session.id, operationId: dispatch.operation_id, allowSettledDescendants: true, continuationText,
|
|
220
|
+
inspectOperation: id => inspect(["daemon", "operation", "--state-dir", session.state_dir, "--operation-id", id, "--json"], `${id}:capacity-observe`),
|
|
221
|
+
inspectSession: () => inspect(["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], `${dispatch.operation_id}:capacity-inspect`),
|
|
222
|
+
assertCurrent: () => {
|
|
223
|
+
assertRecoveryOwner();
|
|
224
|
+
const guard = recoveryGuard(context, row.project_id, row.run_id);
|
|
225
|
+
const binding = context.db.get("SELECT status, daemon_id FROM run_recovery_bindings WHERE recovery_id = ?", [recovery.recovery_id]);
|
|
226
|
+
const ack = context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [recovery.operation_id, row.controller_id, row.generation]);
|
|
227
|
+
if (guard?.status !== "resuming" || binding?.status !== "prepared" || binding.daemon_id !== daemonId || ack?.status !== "accepted")
|
|
228
|
+
throw new AppError("recovery_capacity_owner_changed", "Recovery ACK was accepted or superseded during capacity wait", 1);
|
|
229
|
+
const current = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
|
|
230
|
+
const metadata = JSON.parse(current.process.metadata_json);
|
|
231
|
+
if (current.process.id !== dispatch.process_id || current.state.daemon_id !== daemonId || current.process.state !== "running"
|
|
232
|
+
|| current.process.lease_expires_at <= context.now() || metadata.run_generation !== row.generation || !processIsAlive(current.process))
|
|
233
|
+
throw new AppError("recovery_capacity_owner_changed", "Replacement native owner changed during capacity wait", 1);
|
|
234
|
+
},
|
|
235
|
+
retainIntent: intent => retainRecoveryCapacityIntent(executionContext, { projectId: row.project_id, runId: row.run_id, recoveryId: recovery.recovery_id,
|
|
236
|
+
controllerId: row.controller_id, generation: row.generation, daemonId, intent, prompt: continuationText }),
|
|
237
|
+
prompt: (operationId, prompt) => sendAck(operationId, prompt) });
|
|
238
|
+
if (!resumed)
|
|
239
|
+
throw error;
|
|
240
|
+
receipt = resumed;
|
|
241
|
+
}
|
|
198
242
|
assertRecoveryOwner();
|
|
199
243
|
if (recoveryGuard(context, row.project_id, row.run_id)?.status !== "recovered")
|
|
200
244
|
throw new AppError("trusted_recovery_acceptance_required", "Recovery Turn ended without its native acknowledgement", 1);
|
|
@@ -30,9 +30,12 @@ import { requireRun, requireController, controllerReceipt, appendEvent, saveStat
|
|
|
30
30
|
import { assertNoCorrelatedRejection } from "./continuation-outcome.js";
|
|
31
31
|
import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady } from "./run-controller-process.js";
|
|
32
32
|
import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
|
|
33
|
+
import { continueCodexCapacity } from "./codex-capacity.js";
|
|
33
34
|
import { resumeControllerSession, failControllerRecovery } from "./run-controller-recovery.js";
|
|
34
35
|
import { captureControllerBoundary } from "./run-controller-capture.js";
|
|
35
36
|
import { requestRunControl } from "./run-control-worker.js";
|
|
37
|
+
import { storageSessionId } from "./session-identity.js";
|
|
38
|
+
import { terminalCodexOverload } from "../harness-runtime/lib/dd-codex.mjs";
|
|
36
39
|
export { resumeRunController } from "./run-controller-recovery.js";
|
|
37
40
|
export function prepareRunControllerLaunch(context, input) {
|
|
38
41
|
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !/^[a-f0-9]{64}$/.test(input.contextSha256))
|
|
@@ -526,16 +529,61 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
526
529
|
throw new AppError("settlement_reconciliation_required", "The managed Session still has an unsettled productive predecessor", 1, { session_id: session.id, settlement: current });
|
|
527
530
|
}
|
|
528
531
|
};
|
|
529
|
-
const prompt = async (session, text, operationId) => {
|
|
532
|
+
const prompt = async (session, text, operationId, kind) => {
|
|
533
|
+
const hitl = kind === "hitl";
|
|
530
534
|
await reconcilePromptPredecessor(session);
|
|
531
|
-
const
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
+
const firstOperationId = operationId ?? `${row.controller_id}:prompt:${crypto.randomUUID()}`;
|
|
536
|
+
const send = (value, id) => {
|
|
537
|
+
const file = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
|
|
538
|
+
fs.writeFileSync(file, `${value}\n`, { flag: "wx", mode: 0o600 });
|
|
539
|
+
state.turns += 1;
|
|
540
|
+
saveState(context, row, state);
|
|
541
|
+
return controllerAdapter(executionContext, row, manifest, session, ["session", "prompt", "--session-id", session.id, "--prompt-file", file, ...sessionArguments(context, manifest, session)], assertOwner, id, cancellation.signal);
|
|
542
|
+
};
|
|
535
543
|
let receipt;
|
|
536
|
-
const adapter = controllerAdapter(executionContext, row, manifest, session, ["session", "prompt", "--session-id", session.id, "--prompt-file", file, ...sessionArguments(context, manifest, session)], assertOwner, operationId, cancellation.signal);
|
|
537
544
|
try {
|
|
538
|
-
|
|
545
|
+
try {
|
|
546
|
+
receipt = await send(text, firstOperationId);
|
|
547
|
+
}
|
|
548
|
+
catch (error) {
|
|
549
|
+
if (runtimeHarness(session.profile) !== "codex-desktop")
|
|
550
|
+
throw error;
|
|
551
|
+
const resumeCommand = hitl ? text.split("\n\n")[1] : undefined;
|
|
552
|
+
if (hitl && !resumeCommand?.includes(" stage resume "))
|
|
553
|
+
throw error;
|
|
554
|
+
const inspect = (args, id) => runHarnessAdapter({ executable: manifest.adapters["codex-desktop"].executable,
|
|
555
|
+
args, env: controllerEnvironment(context, manifest, session), runtimeOwner: session.owner, operationId: id, timeoutMs: 30_000, signal: cancellation.signal });
|
|
556
|
+
const resumed = await continueCodexCapacity({ error, sessionId: session.id, operationId: firstOperationId,
|
|
557
|
+
inspectOperation: id => inspect(["daemon", "operation", "--state-dir", session.state_dir, "--operation-id", id, "--json"], `${id}:capacity-observe`),
|
|
558
|
+
inspectSession: () => inspect(["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], `${firstOperationId}:capacity-inspect`),
|
|
559
|
+
assertCurrent: () => {
|
|
560
|
+
assertOwner();
|
|
561
|
+
const view = getFlowRunStatus(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id });
|
|
562
|
+
const next = view.continuation;
|
|
563
|
+
if (hitl) {
|
|
564
|
+
const answerId = firstOperationId.slice(0, -`:prompt:${row.generation}`.length);
|
|
565
|
+
const answer = context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND status = 'accepted'", [answerId, row.controller_id]);
|
|
566
|
+
const accepted = answer && JSON.parse(answer.input_json);
|
|
567
|
+
const pause = view.index.stage_runs.find(item => item.stage === accepted?.stage)?.pause;
|
|
568
|
+
if (next.kind !== "paused" || next.stage !== state.stage || !accepted || pause?.id !== accepted.pause_id || pause.work_id !== accepted.work_id || pause.answer_path)
|
|
569
|
+
throw new AppError("capacity_continuation_blocked", "Accepted HITL pause changed during capacity wait", 1, { continuation: next });
|
|
570
|
+
}
|
|
571
|
+
else if (next.stage !== state.stage || ["terminal", "paused", "blocked", "wait"].includes(next.kind)) {
|
|
572
|
+
throw new AppError("capacity_continuation_blocked", "RUN changed while native capacity was unavailable", 1, { continuation: next });
|
|
573
|
+
}
|
|
574
|
+
},
|
|
575
|
+
retainIntent: intent => context.db.writeTransaction(() => {
|
|
576
|
+
const changed = context.db.run("UPDATE run_controller_operations SET status = 'failed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'outcome_unknown'", [JSON.stringify({ capacity_intent: intent }), context.now(), intent.predecessor, row.controller_id, row.generation]);
|
|
577
|
+
if (changed.changes !== 1)
|
|
578
|
+
throw new AppError("capacity_continuation_blocked", "Failed native operation changed before continuation", 1);
|
|
579
|
+
appendEvent(context, row.controller_id, "capacity_continuation", intent);
|
|
580
|
+
}),
|
|
581
|
+
prompt: (id, value) => send(value, id),
|
|
582
|
+
...(resumeCommand ? { continuationText: `The preceding Turn failed before any tool call. Continue the same accepted HITL answer in this Session. If the pause is still current, run this exact previously issued standalone command once without changing the answer:\n\n${resumeCommand}\n\nDo not accept or deliver the answer again if the pause has changed.` } : {}) });
|
|
583
|
+
if (!resumed)
|
|
584
|
+
throw error;
|
|
585
|
+
receipt = resumed;
|
|
586
|
+
}
|
|
539
587
|
checkOutcomes();
|
|
540
588
|
if (fatal)
|
|
541
589
|
throw fatal;
|
|
@@ -545,7 +593,6 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
545
593
|
throw error;
|
|
546
594
|
await stopAfterFailure(error);
|
|
547
595
|
cancellation.abort();
|
|
548
|
-
await adapter.catch(() => { });
|
|
549
596
|
throw fatal ?? error;
|
|
550
597
|
}
|
|
551
598
|
state.last_receipt = receipt;
|
|
@@ -613,7 +660,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
613
660
|
// The accepted answer is logical intent; only its generation-bound
|
|
614
661
|
// adapter operation represents a physical dispatch that stop must drain.
|
|
615
662
|
const command = managedLifecycleCommand(executionContext, `${flowCommand(executionContext)} stage resume ${run.id} --stage ${next.stage} --work ${input.work_id} --project-root ${quote([run.project_root])} --answer-file ${quote([input.file])} --json`, { replaceUnobservedAnswer: true });
|
|
616
|
-
const receipt = await prompt(state.sessions[state.current_session], controllerStageResumePrompt(pause.id, command), `${answer.operation_id}:prompt:${row.generation}
|
|
663
|
+
const receipt = await prompt(state.sessions[state.current_session], controllerStageResumePrompt(pause.id, command), `${answer.operation_id}:prompt:${row.generation}`, "hitl");
|
|
617
664
|
const afterAnswer = getFlowRunStatus(executionContext, { projectRoot: run.project_root, runId: run.id });
|
|
618
665
|
if (afterAnswer.index.stage_runs.some(stage => stage.status === "paused" && stage.pause?.id === pause.id)) {
|
|
619
666
|
throw new AppError("controller_answer_not_applied", "The answer Turn completed without resuming the assigned HITL pause", 1, { pause_id: pause.id, operation_id: answer.operation_id, stage: next.stage, effect: "no_progress" });
|
|
@@ -712,7 +759,55 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
712
759
|
throw new AppError("controller_attach_required", "An unfinished Stage has no managed coordinator", 1);
|
|
713
760
|
const session = state.sessions[state.current_session];
|
|
714
761
|
const baseline = new Set(state.native_baseline ?? []);
|
|
715
|
-
|
|
762
|
+
let children = controllerNativeChildren(state.last_receipt ?? {}, session.id).filter(child => !baseline.has(child.session_id));
|
|
763
|
+
if (runtimeHarness(session.profile) === "codex-desktop" && children.some(child => child.status === "failed")) {
|
|
764
|
+
const inspected = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
765
|
+
state.last_receipt = inspected;
|
|
766
|
+
saveState(context, row, state);
|
|
767
|
+
children = controllerNativeChildren(inspected, session.id).filter(child => !baseline.has(child.session_id));
|
|
768
|
+
const failed = children.filter(child => child.status === "failed");
|
|
769
|
+
const child = failed.length === 1 && children.every(item => item === failed[0] || item.status === "completed") ? failed[0] : undefined;
|
|
770
|
+
const turnId = child?.terminal_turn_id;
|
|
771
|
+
const native = child && turnId ? { code: "turn_interrupted", details: { provider_session_id: child.session_id, turn_id: turnId, native_turn_id: turnId,
|
|
772
|
+
terminal_status: "failed", provider_error: child.provider_error } } : null;
|
|
773
|
+
const prior = child && state.child_capacity?.[child.session_id];
|
|
774
|
+
const links = child ? context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: "codex-desktop", session_id: child.session_id })]) : [];
|
|
775
|
+
if (child && turnId && native && terminalCodexOverload(native, child.session_id, turnId)
|
|
776
|
+
&& child.native_turn_items?.observed === true && child.native_turn_items.possible_effects === false
|
|
777
|
+
&& inspected.settled === true && child.parent_session_id === session.id && links.length <= 1
|
|
778
|
+
&& (!links.length || links[0].status === "running") && (prior?.attempts ?? 0) < 2) {
|
|
779
|
+
if (prior?.turn_id === turnId)
|
|
780
|
+
throw new AppError("capacity_child_followup_unconfirmed", "Parent continuation did not start a new native child Turn", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
781
|
+
const attempt = (prior?.attempts ?? 0) + 1;
|
|
782
|
+
state.child_capacity ??= {};
|
|
783
|
+
state.child_capacity[child.session_id] = { turn_id: turnId, attempts: attempt };
|
|
784
|
+
saveState(context, row, state);
|
|
785
|
+
const operationId = `${row.controller_id}:child:${digest([child.session_id, turnId]).slice(0, 16)}:followup:${attempt}`;
|
|
786
|
+
appendEvent(context, row.controller_id, "capacity_child_followup", { child_session_id: child.session_id, turn_id: turnId, attempt, operation_id: operationId });
|
|
787
|
+
await delay(attempt === 1 ? 5_000 : 15_000);
|
|
788
|
+
assertOwner();
|
|
789
|
+
const current = getFlowRunStatus(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id });
|
|
790
|
+
if (current.continuation.stage !== state.stage || ["terminal", "paused", "blocked", "wait"].includes(current.continuation.kind))
|
|
791
|
+
throw new AppError("capacity_child_followup_blocked", "RUN changed during native child capacity wait", 1, { child_session_id: child.session_id });
|
|
792
|
+
const fresh = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
793
|
+
const still = controllerNativeChildren(fresh, session.id).find(item => item.session_id === child.session_id);
|
|
794
|
+
const currentLinks = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: "codex-desktop", session_id: child.session_id })]);
|
|
795
|
+
if (fresh.settled !== true || !still || still.parent_session_id !== session.id || still.status !== "failed" || still.terminal_turn_id !== turnId
|
|
796
|
+
|| still.native_turn_items?.observed !== true || still.native_turn_items.possible_effects !== false || currentLinks.length > 1
|
|
797
|
+
|| currentLinks.some(link => link.status !== "running"))
|
|
798
|
+
throw new AppError("capacity_child_followup_blocked", "Native child or Work changed during capacity wait", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
799
|
+
assertOwner();
|
|
800
|
+
const text = `The existing native child ${child.session_id} has a confirmed effect-free failed Turn. Use the native followup_task tool exactly once for that same child ID with this short message: "Continue your assigned Work from its current durable state. Do not repeat an observed lifecycle command. Finish only the original Work." Do not spawn another child, claim another Work, or perform the child's Work in this parent Turn. End this Turn after the followup_task response.`;
|
|
801
|
+
await prompt(session, text, operationId, "child");
|
|
802
|
+
const after = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
803
|
+
state.last_receipt = after;
|
|
804
|
+
saveState(context, row, state);
|
|
805
|
+
const observed = controllerNativeChildren(after, session.id).find(item => item.session_id === child.session_id);
|
|
806
|
+
if (!observed || observed.status === "failed" && observed.terminal_turn_id === turnId)
|
|
807
|
+
throw new AppError("capacity_child_followup_unconfirmed", "Native child did not advance to a new Turn after parent followup", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
808
|
+
continue;
|
|
809
|
+
}
|
|
810
|
+
}
|
|
716
811
|
const fanout = await nextControllerFanout(executionContext, { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles });
|
|
717
812
|
assertOwner();
|
|
718
813
|
if (fanout.kind === "none") {
|
|
@@ -56,6 +56,42 @@ export function prepareRunRecoveryDispatch(context, input) {
|
|
|
56
56
|
return dispatch;
|
|
57
57
|
});
|
|
58
58
|
}
|
|
59
|
+
/** Publish only an exact successor to a failed, effect-free recovery ACK Turn. */
|
|
60
|
+
export function retainRecoveryCapacityIntent(context, input) {
|
|
61
|
+
const { intent } = input;
|
|
62
|
+
if (![1, 2].includes(intent.ordinal) || !intent.turn_id || !intent.not_before)
|
|
63
|
+
throw new AppError("recovery_capacity_intent_invalid", "Recovery capacity successor is invalid", 1);
|
|
64
|
+
context.db.writeTransaction(() => {
|
|
65
|
+
const guard = recoveryGuard(context, input.projectId, input.runId);
|
|
66
|
+
const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared'", [input.recoveryId, input.daemonId]);
|
|
67
|
+
if (!guard || guard.recovery_id !== input.recoveryId || guard.generation !== input.generation || guard.status !== "resuming" || !binding?.runtime_dispatch_json)
|
|
68
|
+
throw new AppError("recovery_capacity_owner_changed", "Recovery ACK no longer owns the current generation", 1);
|
|
69
|
+
const controller = context.db.get("SELECT generation, status FROM run_controllers WHERE controller_id = ? AND project_id = ? AND run_id = ?", [input.controllerId, input.projectId, input.runId]);
|
|
70
|
+
const control = context.db.get("SELECT status FROM run_controls WHERE recovery_id = ?", [input.recoveryId]);
|
|
71
|
+
if (controller?.generation !== input.generation || ["completed", "cancelled", "superseded"].includes(controller.status) || control?.status !== "resuming")
|
|
72
|
+
throw new AppError("recovery_capacity_owner_changed", "Recovery controller or control changed before successor publication", 1);
|
|
73
|
+
const dispatch = JSON.parse(binding.runtime_dispatch_json);
|
|
74
|
+
const currentProcess = getResourceDatabase(resourceHome(context)).get("SELECT * FROM managed_processes WHERE id = ?", [dispatch.process_id]);
|
|
75
|
+
if (!currentProcess || currentProcess.state !== "running" || currentProcess.lease_expires_at <= context.now() || currentProcess.owner_id !== dispatch.owner_id
|
|
76
|
+
|| currentProcess.project_id !== input.projectId || currentProcess.run_id !== input.runId)
|
|
77
|
+
throw new AppError("recovery_capacity_owner_changed", "Replacement daemon lease changed before successor publication", 1);
|
|
78
|
+
const prior = dispatch.capacity_intents ?? [];
|
|
79
|
+
const predecessor = intent.ordinal === 1 ? dispatch.operation_id : prior[0]?.operation_id;
|
|
80
|
+
if (prior.length !== intent.ordinal - 1 || predecessor !== intent.predecessor || intent.operation_id !== `${dispatch.operation_id}:capacity:${intent.ordinal}:session.prompt`)
|
|
81
|
+
throw new AppError("recovery_capacity_intent_conflict", "Recovery capacity successor does not extend the pinned ACK", 1);
|
|
82
|
+
const failed = context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [intent.predecessor, input.controllerId, input.generation]);
|
|
83
|
+
if (failed?.status !== "outcome_unknown")
|
|
84
|
+
throw new AppError("recovery_capacity_intent_conflict", "Recovery predecessor is not the failed native operation", 1);
|
|
85
|
+
const next = { ...intent, prompt_sha256: crypto.createHash("sha256").update(input.prompt).digest("hex") };
|
|
86
|
+
const updated = { ...dispatch, capacity_intents: [...prior, next] };
|
|
87
|
+
const changed = context.db.run("UPDATE run_recovery_bindings SET runtime_dispatch_json = ? WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared' AND runtime_dispatch_json = ?", [JSON.stringify(updated), input.recoveryId, input.daemonId, binding.runtime_dispatch_json]);
|
|
88
|
+
if (changed.changes !== 1)
|
|
89
|
+
throw new AppError("recovery_capacity_intent_conflict", "Recovery dispatch changed before successor publication", 1);
|
|
90
|
+
const marked = context.db.run("UPDATE run_controller_operations SET status = 'failed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'outcome_unknown'", [JSON.stringify({ capacity_intent: next }), context.now(), intent.predecessor, input.controllerId, input.generation]);
|
|
91
|
+
if (marked.changes !== 1)
|
|
92
|
+
throw new AppError("recovery_capacity_intent_conflict", "Recovery predecessor changed before successor publication", 1);
|
|
93
|
+
});
|
|
94
|
+
}
|
|
59
95
|
/** The caller has already checked PID/start/lease and runtime-home identity. */
|
|
60
96
|
export function recoveryRuntimeAdmission(context, record, input) {
|
|
61
97
|
if (!record.project_id || !record.run_id || !input.daemonId)
|
|
@@ -73,8 +109,13 @@ export function recoveryRuntimeAdmission(context, record, input) {
|
|
|
73
109
|
if (metadata.run_generation !== guard.generation)
|
|
74
110
|
return false;
|
|
75
111
|
if (guard.status === "recovered" && binding.status === "accepted")
|
|
76
|
-
return
|
|
112
|
+
return !input.operationId?.startsWith(`${dispatch.operation_id}:capacity:`);
|
|
77
113
|
if (guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.resume")
|
|
78
114
|
return typeof dispatch.resume_operation_id === "string" && input.operationId === dispatch.resume_operation_id && input.nativeSessionId === binding.provider_session_id;
|
|
79
|
-
|
|
115
|
+
const active = dispatch.capacity_intents?.at(-1);
|
|
116
|
+
if (active && active.not_before > context.now())
|
|
117
|
+
return false;
|
|
118
|
+
return guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.prompt"
|
|
119
|
+
&& input.operationId === (active?.operation_id ?? dispatch.operation_id) && input.nativeSessionId === binding.provider_session_id
|
|
120
|
+
&& input.promptSha256 === (active?.prompt_sha256 ?? dispatch.prompt_sha256);
|
|
80
121
|
}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { createContext } from "../runtime/context.js";
|
|
4
5
|
import { AppError } from "../shared/errors.js";
|
|
5
6
|
import { runHarnessAdapter } from "./harness-adapter.js";
|
|
6
|
-
import { managedLeaseReleased, managedProcessBinding, managedProcessStatus, processIsAlive, processTreeIsAlive, resourceHome, stopManagedProcess } from "./managed-processes.js";
|
|
7
|
+
import { managedLeaseReleased, managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive, resourceHome, stopManagedProcess } from "./managed-processes.js";
|
|
8
|
+
import { terminalCheckSettled } from "./code-checks.js";
|
|
7
9
|
import { fenceRuntimeBudget, runtimeBudgetStatus, scopeLocalProcessRole } from "./runtime-budget.js";
|
|
8
10
|
import { retainedDaemonOperations } from "./run-control.js";
|
|
9
11
|
import { historicalDaemonSettlement } from "./native-daemon-history.js";
|
|
@@ -80,10 +82,29 @@ export async function stopRuntimeScope(context, scopeId, requestId) {
|
|
|
80
82
|
const metadata = JSON.parse(record.metadata_json);
|
|
81
83
|
// RUN control owns active controllers/checks. Adopt only its already-finished
|
|
82
84
|
// physical registrations here; death alone is not a completed RUN operation.
|
|
85
|
+
const terminalCheck = record.kind === "check" && metadata.role === "check" && record.check_id && record.project_id && record.run_id
|
|
86
|
+
? (() => {
|
|
87
|
+
const home = managedProcessRuntimeHome(record, context);
|
|
88
|
+
if (!home)
|
|
89
|
+
return null;
|
|
90
|
+
let reader;
|
|
91
|
+
try {
|
|
92
|
+
reader = createContext({ ...context.env, DD_FLOW_HOME: home }, "read_existing");
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
return terminalCheckSettled(reader, record);
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
reader.db.close?.();
|
|
102
|
+
}
|
|
103
|
+
})() : null;
|
|
83
104
|
if ((record.kind === "run-controller" && metadata.role === "controller" && record.operation_id
|
|
84
105
|
|| record.kind === "check" && metadata.role === "check" && record.check_id)
|
|
85
106
|
&& record.project_id && record.run_id && ["stopped", "failed"].includes(record.state)
|
|
86
|
-
&& record.pid && record.pid_started_at && !processTreeIsAlive(record)) {
|
|
107
|
+
&& record.pid && record.pid_started_at && (terminalCheck || !processTreeIsAlive(record))) {
|
|
87
108
|
const operationId = record.operation_id ?? record.check_id;
|
|
88
109
|
nodes.push({ process_id: record.id, state_dir: metadata.state_dir ?? "", settled: true, operation_id: operationId, owner_operation_id: operationId });
|
|
89
110
|
continue;
|
|
@@ -582,7 +582,13 @@ export function verificationProjection(works, receipts, options = {}) {
|
|
|
582
582
|
if (item.obligation_ref)
|
|
583
583
|
rawEvidence.set(item.obligation_ref, unique([...(rawEvidence.get(item.obligation_ref) ?? []), ...(item.refs ?? [])]));
|
|
584
584
|
}
|
|
585
|
-
const
|
|
585
|
+
const byCriterionGate = new Map();
|
|
586
|
+
for (const entry of packets.flatMap((value) => value.acceptance)) {
|
|
587
|
+
const key = JSON.stringify([entry.criterion_id, entry.gate]);
|
|
588
|
+
const previous = byCriterionGate.get(key);
|
|
589
|
+
byCriterionGate.set(key, { criterion_id: entry.criterion_id, gate: entry.gate, check_refs: unique([...(previous?.check_refs ?? []), ...entry.check_refs]) });
|
|
590
|
+
}
|
|
591
|
+
const acceptance = [...byCriterionGate.values()];
|
|
586
592
|
const historicalReceiptRefs = new Set((options.historicalReceipts ?? receipts).flatMap((receipt) => receiptReferences(receipt, options)));
|
|
587
593
|
const acceptanceEvidence = new Map(acceptance.map((item) => {
|
|
588
594
|
const reported = rawEvidence.get(item.criterion_id) ?? [];
|