@deksden-com/dd-flow-cli 0.9.0-beta.103 → 0.9.0-beta.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/build-info.json +3 -3
- package/dist/cli/run-cli.js +32 -11
- package/dist/harness-runtime/bin/dd-agy.mjs +1 -1
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +2 -2
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +4 -3
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +2 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +8 -1
- package/dist/harness-runtime/lib/managed-daemon.mjs +11 -6
- package/dist/protocol/local-files.js +18 -9
- package/dist/runtime/context.js +2 -2
- package/dist/services/cleanup.js +81 -32
- package/dist/services/code-checks.js +18 -19
- package/dist/services/continuation-outcome.js +1 -1
- package/dist/services/controller-fanout.js +3 -3
- package/dist/services/eval-snapshots.js +69 -41
- package/dist/services/execution-policy.js +12 -1
- package/dist/services/hooks.js +150 -42
- package/dist/services/lanes.js +23 -18
- package/dist/services/lifecycle-command.js +11 -2
- package/dist/services/lifecycle-invocations.js +291 -54
- package/dist/services/managed-processes.js +35 -23
- package/dist/services/merge-queue.js +71 -67
- package/dist/services/merge-server.js +3 -11
- package/dist/services/migrations.js +28 -12
- package/dist/services/plan-runtime.js +8 -4
- package/dist/services/projects.js +3 -11
- package/dist/services/protocols.js +98 -68
- package/dist/services/recovery-snapshot-database.js +43 -8
- package/dist/services/run-control-worker.js +15 -32
- package/dist/services/run-control.js +3 -17
- package/dist/services/run-controller-capture.js +58 -27
- package/dist/services/run-controller-process.js +21 -30
- package/dist/services/run-controller-recovery.js +36 -37
- package/dist/services/run-controller.js +81 -86
- package/dist/services/run-fork.js +10 -6
- package/dist/services/run-projection.js +19 -64
- package/dist/services/run-recovery-runtime.js +24 -14
- package/dist/services/run-recovery.js +56 -54
- package/dist/services/runs.js +218 -160
- package/dist/services/runtime-budget.js +19 -45
- package/dist/services/runtime-scope-capture.js +8 -6
- package/dist/services/runtime-scope-control.js +49 -39
- package/dist/services/runtime-scope-resume.js +134 -122
- package/dist/services/runtime-scope-worker.js +48 -49
- package/dist/services/scope-writer-cleanup.js +26 -0
- package/dist/services/stage-context.js +6 -1
- package/dist/services/stage-lifecycle.js +1 -1
- package/dist/services/stage-pause.js +13 -29
- package/dist/services/stage-report-renderer.js +116 -6
- package/dist/services/usage.js +2 -9
- package/dist/services/vnext-code-review.js +10 -7
- package/dist/services/vnext-code.js +9 -17
- package/dist/services/vnext-merge.js +64 -21
- package/dist/services/vnext-plan-review.js +43 -30
- package/dist/services/vnext-plan.js +28 -24
- package/dist/services/vnext-protocolize.js +183 -20
- package/dist/services/vnext-specify.js +27 -19
- package/dist/services/work-registry.js +229 -150
- package/dist/storage/database.js +286 -56
- package/dist/storage/foreign-key-transaction.js +39 -0
- package/dist/storage/immutable-input.js +45 -0
- package/dist/storage/import-guard.js +102 -0
- package/dist/storage/native-cleanup.js +52 -0
- package/dist/storage/projection-lock.js +73 -0
- package/dist/storage/writer-contract.js +3 -3
- package/dist/storage/writer-migration.js +24 -41
- package/package.json +1 -1
- package/tools/repair-paused-run-status.mjs +48 -23
|
@@ -14,9 +14,9 @@ const processOperation = (context, row) => `controller:${digest({ database: cont
|
|
|
14
14
|
const birth = () => ({ pid: process.pid, pid_started_at: processStartedAt(process.pid) });
|
|
15
15
|
const dead = (value) => Boolean(value && Number.isSafeInteger(value.pid) && value.pid > 0 && typeof value.pid_started_at === "string" && value.pid_started_at && !processIsAlive(value));
|
|
16
16
|
/** Called inside the RUN intent transaction, before the client can disappear. */
|
|
17
|
-
export function recordControllerLauncher(context, row, retired) {
|
|
17
|
+
export function recordControllerLauncher(context, row, retired, identity = birth()) {
|
|
18
18
|
const request = { action: "controller_owner_launch", owner_id: launchId(row) };
|
|
19
|
-
const receipt = { ...
|
|
19
|
+
const receipt = { ...identity, registration: "controller_self@1", ...(retired ? { retired_process: { id: retired.id, lease_token: retired.lease_token, pid: retired.pid, pid_started_at: retired.pid_started_at } } : {}) };
|
|
20
20
|
const now = context.now();
|
|
21
21
|
context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'completed', ?, ?, ?)", [launchId(row), row.controller_id, row.generation, digest(request), JSON.stringify(request), JSON.stringify(receipt), now, now]);
|
|
22
22
|
}
|
|
@@ -92,48 +92,37 @@ function prepareSpawn(context, row) {
|
|
|
92
92
|
return null;
|
|
93
93
|
if (launch.value.retired_process)
|
|
94
94
|
retire(resourceContext, row, launch.value.retired_process);
|
|
95
|
-
|
|
96
|
-
|
|
95
|
+
const launcher = birth();
|
|
96
|
+
return context.db.writeTransaction(() => {
|
|
97
97
|
assertLaunchAllowed(context, row);
|
|
98
98
|
const changed = context.db.run(`UPDATE run_controllers SET owner_token = ?, owner_pid = NULL, process_id = NULL, process_lease_token = NULL, status = 'starting', last_error_json = NULL, updated_at = ?
|
|
99
99
|
WHERE controller_id = ? AND owner_token = ? AND owner_pid IS ? AND process_id IS ? AND status = ? AND state_json = ?
|
|
100
100
|
AND (SELECT receipt_json FROM run_controller_operations WHERE operation_id = ?) = ?`, [token, context.now(), row.controller_id, row.owner_token, row.owner_pid, row.process_id, row.status, row.state_json, launchId(row), launch.receipt_json]);
|
|
101
|
-
if (changed.changes !== 1)
|
|
102
|
-
context.db.exec("ROLLBACK");
|
|
101
|
+
if (changed.changes !== 1)
|
|
103
102
|
return null;
|
|
104
|
-
}
|
|
105
103
|
const current = requireController(context, row.controller_id);
|
|
106
104
|
if (!ours) {
|
|
107
|
-
recordControllerLauncher(context, current);
|
|
105
|
+
recordControllerLauncher(context, current, undefined, launcher);
|
|
108
106
|
appendEvent(context, row.controller_id, "launch_owner_reconciled", { prior_launch: launchId(row), generation: row.generation, prior_process_id: row.process_id });
|
|
109
107
|
}
|
|
110
108
|
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.spawn_started', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(current)]);
|
|
111
|
-
context.db.exec("COMMIT");
|
|
112
109
|
return current;
|
|
113
|
-
}
|
|
114
|
-
catch (error) {
|
|
115
|
-
context.db.exec("ROLLBACK");
|
|
116
|
-
throw error;
|
|
117
|
-
}
|
|
110
|
+
});
|
|
118
111
|
}
|
|
119
112
|
/** The physical child records its own birth before resource registration. */
|
|
120
113
|
export function claimControllerProcess(context, row) {
|
|
121
114
|
const launch = launchRecord(context, row);
|
|
122
115
|
if (!launch && (!row.process_id || !row.process_lease_token))
|
|
123
116
|
throw new AppError("controller_owner_mismatch", "Managed registration is not authorized", 1);
|
|
124
|
-
|
|
125
|
-
|
|
117
|
+
const owner = birth();
|
|
118
|
+
context.db.writeTransaction(() => {
|
|
126
119
|
const claimed = context.db.run("UPDATE run_controllers SET owner_pid = ?, status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [process.pid, context.now(), row.controller_id, row.owner_token]);
|
|
127
120
|
if (claimed.changes !== 1)
|
|
128
121
|
throw new AppError("controller_already_owned", "Managed owner has already claimed this launch", 1);
|
|
129
122
|
if (launch)
|
|
130
|
-
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.owner', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
catch (error) {
|
|
134
|
-
context.db.exec("ROLLBACK");
|
|
135
|
-
throw error;
|
|
136
|
-
}
|
|
123
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.owner', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(owner), context.now(), launchId(row)]);
|
|
124
|
+
});
|
|
125
|
+
row.owner_pid = process.pid;
|
|
137
126
|
if (!row.process_id) {
|
|
138
127
|
const manifest = JSON.parse(row.manifest_json);
|
|
139
128
|
const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
|
|
@@ -173,16 +162,18 @@ export async function spawnController(context, row) {
|
|
|
173
162
|
return controllerReceipt(requireController(context, controllerId));
|
|
174
163
|
}
|
|
175
164
|
catch (error) {
|
|
176
|
-
context.db.exec("SAVEPOINT controller_spawn_status");
|
|
177
165
|
try {
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
166
|
+
context.db.writeTransaction(() => {
|
|
167
|
+
const changed = context.db.run("UPDATE run_controllers SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [JSON.stringify(errorRecord(error)), context.now(), controllerId, token]);
|
|
168
|
+
if (changed.changes && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation)
|
|
169
|
+
setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
|
|
170
|
+
});
|
|
182
171
|
}
|
|
183
172
|
catch (failure) {
|
|
184
|
-
|
|
185
|
-
|
|
173
|
+
const secondary = { status_persistence_error: errorRecord(failure) };
|
|
174
|
+
if (error instanceof AppError)
|
|
175
|
+
throw new AppError(error.code, error.message, error.exitCode, { ...error.details, ...secondary });
|
|
176
|
+
throw new AppError("controller_spawn_failed", error instanceof Error ? error.message : String(error), 1, { cause: errorRecord(error), ...secondary });
|
|
186
177
|
}
|
|
187
178
|
throw error;
|
|
188
179
|
}
|
|
@@ -10,7 +10,7 @@ import { verifyRecoverySnapshot } from "./eval-snapshots.js";
|
|
|
10
10
|
import { recoveryGuard, resumeRunRecovery } from "./run-recovery.js";
|
|
11
11
|
import { prepareRunRecoveryDispatch } from "./run-recovery-runtime.js";
|
|
12
12
|
import { runtimeHarness } from "./execution-policy.js";
|
|
13
|
-
import { managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive } from "./managed-processes.js";
|
|
13
|
+
import { managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive, processStartedAt } from "./managed-processes.js";
|
|
14
14
|
import { requireRun, requireController, controllerReceipt, appendEvent, saveState, digest, errorRecord } from "./run-controller-state.js";
|
|
15
15
|
import { controllerAdapter, sessionArguments } from "./run-controller-adapter.js";
|
|
16
16
|
import { spawnController, recordControllerLauncher } from "./run-controller-process.js";
|
|
@@ -18,6 +18,7 @@ import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-da
|
|
|
18
18
|
import { managedInvocationContext } from "./lifecycle-invocations.js";
|
|
19
19
|
import { runControlStatus } from "./run-control.js";
|
|
20
20
|
import { runtimeBudgetStatus, waitForRuntimeScopeRecovery } from "./runtime-budget.js";
|
|
21
|
+
import { prepareControllerPublication } from "./run-controller-capture.js";
|
|
21
22
|
/** CLI observation is bounded; the physical controller owns the recovery work. */
|
|
22
23
|
export async function resumeRunController(context, input) {
|
|
23
24
|
if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
|
|
@@ -35,6 +36,11 @@ export async function resumeRunController(context, input) {
|
|
|
35
36
|
}
|
|
36
37
|
/** Retain the logical controller and its receipts; replace only its dead owner. */
|
|
37
38
|
export function recordControllerResume(context, input) {
|
|
39
|
+
return prepareControllerResume(context, input)();
|
|
40
|
+
}
|
|
41
|
+
/** Verify immutable recovery evidence before a caller reserves the registry
|
|
42
|
+
* writer. The returned operation rechecks SQL ownership under the RUN CAS. */
|
|
43
|
+
export function prepareControllerResume(context, input) {
|
|
38
44
|
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !input.controlId.trim())
|
|
39
45
|
throw new AppError("usage", "Resume requires request-id and the exact source control", 2);
|
|
40
46
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
@@ -53,7 +59,7 @@ export function recordControllerResume(context, input) {
|
|
|
53
59
|
};
|
|
54
60
|
const prior = findPrior();
|
|
55
61
|
if (prior)
|
|
56
|
-
return prior;
|
|
62
|
+
return () => findPrior() ?? prior;
|
|
57
63
|
try {
|
|
58
64
|
const row = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND status NOT IN ('completed','cancelled','superseded')", [project.id, run.id]);
|
|
59
65
|
if (!row)
|
|
@@ -78,18 +84,20 @@ export function recordControllerResume(context, input) {
|
|
|
78
84
|
if (!previousOwner || previousOwner.lease_token !== row.process_lease_token || previousOwner.kind !== "run-controller" || previousOwner.owner_id !== row.controller_id || previousOwner.project_id !== project.id || previousOwner.run_id !== run.id || managedProcessRuntimeHome(previousOwner, context) !== context.ddFlowHome || !previousOwner.pid || !previousOwner.pid_started_at || processTreeIsAlive(previousOwner))
|
|
79
85
|
throw new AppError("controller_owner_unsettled", "Resume requires proof that the previous physical controller has exited", 1);
|
|
80
86
|
const token = crypto.randomUUID(), now = context.now();
|
|
81
|
-
|
|
82
|
-
|
|
87
|
+
const launcher = { pid: process.pid, pid_started_at: processStartedAt(process.pid) };
|
|
88
|
+
return () => context.db.writeTransaction(() => {
|
|
83
89
|
const duplicate = context.db.get("SELECT input_hash, status FROM run_controller_operations WHERE operation_id = ?", [operationId]);
|
|
84
90
|
if (duplicate) {
|
|
85
91
|
if (duplicate.input_hash !== hash)
|
|
86
92
|
throw new AppError("runtime_request_conflict", "Resume request id has different inputs", 1);
|
|
87
|
-
context.db.exec("COMMIT");
|
|
88
93
|
return { controller: requireController(context, row.controller_id), operationId };
|
|
89
94
|
}
|
|
90
95
|
const current = recoveryGuard(context, project.id, run.id);
|
|
91
96
|
if (current?.recovery_id !== guard.recovery_id || current.generation !== guard.generation || current.status !== "sealed" || current.capture_path !== guard.capture_path)
|
|
92
97
|
throw new AppError("run_control_superseded", "Control changed before resume could be accepted", 1);
|
|
98
|
+
const currentOwner = managedProcessStatus(resourceContext).find(record => record.id === previousOwner.id);
|
|
99
|
+
if (!currentOwner || digest(managedProcessBinding(currentOwner, manifest.resource_home)) !== digest(managedProcessBinding(previousOwner, manifest.resource_home)))
|
|
100
|
+
throw new AppError("controller_owner_unsettled", "Retired controller binding changed while recovery was prepared", 1);
|
|
93
101
|
if (!["running", "paused", "waiting_for_user"].includes(requireRun(context, project.id, run.id).status))
|
|
94
102
|
throw new AppError("invalid_run_state", "RUN became terminal before resume could be accepted", 1);
|
|
95
103
|
const changed = context.db.run("UPDATE run_controls SET status = 'resuming', updated_at = ? WHERE control_id = ? AND recovery_id = ? AND status = 'settled'", [now, input.controlId, guard.recovery_id]);
|
|
@@ -100,22 +108,17 @@ export function recordControllerResume(context, input) {
|
|
|
100
108
|
if (claimed.changes !== 1)
|
|
101
109
|
throw new AppError("controller_owner_mismatch", "Controller changed before resume could claim its next physical owner", 1);
|
|
102
110
|
context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'accepted', ?, ?)", [operationId, row.controller_id, guard.generation, hash, JSON.stringify(request), now, now]);
|
|
103
|
-
recordControllerLauncher(context, requireController(context, row.controller_id), previousOwner);
|
|
111
|
+
recordControllerLauncher(context, requireController(context, row.controller_id), previousOwner, launcher);
|
|
104
112
|
appendEvent(context, row.controller_id, "resume_requested", { operation_id: operationId, control_id: input.controlId, recovery_id: guard.recovery_id, generation: guard.generation, retired_process_id: previousOwner.id, retained_session_id: state.sessions[state.current_session].id });
|
|
105
|
-
context.
|
|
106
|
-
}
|
|
107
|
-
catch (error) {
|
|
108
|
-
context.db.exec("ROLLBACK");
|
|
109
|
-
throw error;
|
|
110
|
-
}
|
|
111
|
-
return { controller: requireController(context, row.controller_id), operationId, retiredProcess: previousOwner };
|
|
113
|
+
return { controller: requireController(context, row.controller_id), operationId, retiredProcess: previousOwner };
|
|
114
|
+
});
|
|
112
115
|
}
|
|
113
116
|
catch (error) {
|
|
114
117
|
// A concurrent identical client may commit between our first lookup and
|
|
115
118
|
// validation. Its durable request wins even if our preflight saw the fence.
|
|
116
119
|
const concurrent = findPrior();
|
|
117
120
|
if (concurrent)
|
|
118
|
-
return concurrent;
|
|
121
|
+
return () => concurrent;
|
|
119
122
|
throw error;
|
|
120
123
|
}
|
|
121
124
|
}
|
|
@@ -126,8 +129,9 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
126
129
|
throw new AppError("recovery_owner_missing", "Recovery lost its retained coordinator Session", 1);
|
|
127
130
|
let executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
|
|
128
131
|
const scopeControl = manifest.budget ? runtimeBudgetStatus(executionContext, manifest.budget.scope_id).control : null;
|
|
129
|
-
const assertRecoveryOwner = () => {
|
|
130
|
-
|
|
132
|
+
const assertRecoveryOwner = (physical = true) => {
|
|
133
|
+
if (physical)
|
|
134
|
+
assertPhysicalOwner();
|
|
131
135
|
const guard = recoveryGuard(context, row.project_id, row.run_id);
|
|
132
136
|
if (guard?.generation !== row.generation || guard.recovery_id !== recovery.recovery_id)
|
|
133
137
|
throw new AppError("run_recovery_generation_stale", "A newer control superseded this recovery owner", 1);
|
|
@@ -168,22 +172,21 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
168
172
|
throw new AppError("recovery_session_unsettled", "Retained native Session is not ready for same-Session recovery", 1);
|
|
169
173
|
assertRecoveryOwner();
|
|
170
174
|
const prepared = { prepared: true, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId, process_id: dispatch.process_id, resume_operation_id: dispatch.resume_operation_id };
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
175
|
+
const binding = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
|
|
176
|
+
if (binding.process.id !== dispatch.process_id || binding.state.daemon_id !== daemonId || binding.process.state !== "running" || !processIsAlive(binding.process))
|
|
177
|
+
throw new AppError("recovery_runtime_mismatch", "Prepared recovery lost its physical Session owner", 1);
|
|
178
|
+
const assertPreparedOwner = prepareControllerPublication(executionContext, row, assertPhysicalOwner);
|
|
179
|
+
context.db.writeTransaction(() => {
|
|
180
|
+
assertPreparedOwner();
|
|
181
|
+
assertRecoveryOwner(false);
|
|
182
|
+
const currentNative = managedProcessStatus(executionContext).find(item => item.id === binding.process.id);
|
|
183
|
+
if (!currentNative || currentNative.state !== "running" || currentNative.lease_expires_at <= context.now() || digest(managedProcessBinding(currentNative, manifest.resource_home)) !== digest(managedProcessBinding(binding.process, manifest.resource_home)))
|
|
184
|
+
throw new AppError("recovery_runtime_mismatch", "Prepared recovery lost its current native lease", 1);
|
|
177
185
|
const recorded = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'accepted'", [JSON.stringify(prepared), context.now(), recovery.operation_id, row.controller_id, row.generation]);
|
|
178
186
|
if (recorded.changes !== 1)
|
|
179
187
|
throw new AppError("run_recovery_generation_stale", "Recovery preparation no longer owns its operation", 1);
|
|
180
188
|
appendEvent(context, row.controller_id, "recovery_prepared", prepared);
|
|
181
|
-
|
|
182
|
-
}
|
|
183
|
-
catch (error) {
|
|
184
|
-
context.db.exec("ROLLBACK");
|
|
185
|
-
throw error;
|
|
186
|
-
}
|
|
189
|
+
});
|
|
187
190
|
if (scopeControl && manifest.budget) {
|
|
188
191
|
await waitForRuntimeScopeRecovery(executionContext, { scopeId: manifest.budget.scope_id, generation: scopeControl.generation, controllerId: row.controller_id, operationId: recovery.operation_id }, assertRecoveryOwner);
|
|
189
192
|
}
|
|
@@ -195,9 +198,10 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
195
198
|
assertRecoveryOwner();
|
|
196
199
|
if (recoveryGuard(context, row.project_id, row.run_id)?.status !== "recovered")
|
|
197
200
|
throw new AppError("trusted_recovery_acceptance_required", "Recovery Turn ended without its native acknowledgement", 1);
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
+
const assertAcceptedOwner = prepareControllerPublication(executionContext, row, assertPhysicalOwner);
|
|
202
|
+
context.db.writeTransaction(() => {
|
|
203
|
+
assertAcceptedOwner();
|
|
204
|
+
assertRecoveryOwner(false);
|
|
201
205
|
context.db.run("UPDATE run_controls SET status = 'resumed', updated_at = ? WHERE control_id = ? AND status = 'resuming'", [context.now(), recovery.control_id]);
|
|
202
206
|
context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'accepted'", [JSON.stringify({ resumed: true, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId }), context.now(), recovery.operation_id]);
|
|
203
207
|
state.last_receipt = receipt;
|
|
@@ -205,12 +209,7 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
205
209
|
delete state.recovery;
|
|
206
210
|
saveState(context, row, state);
|
|
207
211
|
appendEvent(context, row.controller_id, "recovery_resumed", { control_id: recovery.control_id, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId, generation: row.generation });
|
|
208
|
-
|
|
209
|
-
}
|
|
210
|
-
catch (error) {
|
|
211
|
-
context.db.exec("ROLLBACK");
|
|
212
|
-
throw error;
|
|
213
|
-
}
|
|
212
|
+
});
|
|
214
213
|
}
|
|
215
214
|
export function failControllerRecovery(context, row, state, error) {
|
|
216
215
|
if (state.recovery) {
|
|
@@ -7,6 +7,7 @@ import { fileURLToPath } from "node:url";
|
|
|
7
7
|
import { quote } from "shell-quote";
|
|
8
8
|
import { ensureRunControllerStorage, getDatabase } from "../storage/database.js";
|
|
9
9
|
import { canonicalPath, resolveProjectRoot } from "../storage/paths.js";
|
|
10
|
+
import { publishImmutableInput } from "../storage/immutable-input.js";
|
|
10
11
|
import { AppError } from "../shared/errors.js";
|
|
11
12
|
import { prepareFileBytes, prepareJsonText } from "../cli/input-preparation.js";
|
|
12
13
|
import { parseRuntimeBudget } from "./runtime-budget.js";
|
|
@@ -15,10 +16,10 @@ import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
|
|
|
15
16
|
import { getFlowRunStatus, setFlowRunRecoveryPaused } from "./runs.js";
|
|
16
17
|
import { loadExternalStageContext } from "./stage-context.js";
|
|
17
18
|
import { flowCommand } from "./stage-pause.js";
|
|
18
|
-
import { resolveStageTransition, sameSessionProfile, runtimeHarness } from "./execution-policy.js";
|
|
19
|
+
import { resolveStageTransition, sameSessionProfile, stageExecutionCwd, runtimeHarness } from "./execution-policy.js";
|
|
19
20
|
import { resolveHarnessCommand, harnessRuntimeArguments } from "./harness-config.js";
|
|
20
21
|
import { adapterSessionId, prepareHarnessFlowExecutable, runHarnessAdapter } from "./harness-adapter.js";
|
|
21
|
-
import { confirmManagedProcess, heartbeatManagedProcess, finishManagedProcess, processTreeIsAlive, resourceHome } from "./managed-processes.js";
|
|
22
|
+
import { assertManagedProcessLease, confirmManagedProcess, heartbeatManagedProcess, finishManagedProcess, processStartedAt, processTreeIsAlive, resourceHome } from "./managed-processes.js";
|
|
22
23
|
import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
23
24
|
import { initCodexHome } from "./hooks.js";
|
|
24
25
|
import { launchExternalWork } from "./external-work-launch.js";
|
|
@@ -98,13 +99,12 @@ export async function launchRunController(context, input, prepared = prepareRunC
|
|
|
98
99
|
const token = crypto.randomUUID();
|
|
99
100
|
const now = context.now();
|
|
100
101
|
const state = { stage: null, sessions: [], current_session: null, turns: 0, last_continuation: null };
|
|
101
|
-
|
|
102
|
-
|
|
102
|
+
const launcher = { pid: process.pid, pid_started_at: processStartedAt(process.pid) };
|
|
103
|
+
const duplicate = context.db.writeTransaction(() => {
|
|
103
104
|
const duplicate = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND request_id = ?", [project.id, run.id, input.requestId]);
|
|
104
105
|
if (duplicate) {
|
|
105
106
|
if (duplicate.request_hash !== requestHash)
|
|
106
107
|
throw new AppError("runtime_request_conflict", "Managed launch request id has different inputs", 1);
|
|
107
|
-
context.db.exec("COMMIT");
|
|
108
108
|
return { ...controllerReceipt(duplicate), reused: true };
|
|
109
109
|
}
|
|
110
110
|
assertRunMutationAllowed(context, project.id, run.id, generation);
|
|
@@ -116,14 +116,12 @@ export async function launchRunController(context, input, prepared = prepareRunC
|
|
|
116
116
|
if (context.db.get("SELECT 1 FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.status = 'running' LIMIT 1", [project.id, run.id]))
|
|
117
117
|
throw new AppError("controller_attach_required", "A WorkSession acquired the RUN before managed launch", 1);
|
|
118
118
|
context.db.run("INSERT INTO run_controllers (controller_id, project_id, run_id, request_id, request_hash, generation, status, owner_token, manifest_json, state_json, state_dir, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 'starting', ?, ?, ?, ?, ?, ?)", [controllerId, project.id, run.id, input.requestId, requestHash, generation, token, JSON.stringify(manifest), JSON.stringify(state), root, now, now]);
|
|
119
|
-
recordControllerLauncher(context, requireController(context, controllerId));
|
|
119
|
+
recordControllerLauncher(context, requireController(context, controllerId), undefined, launcher);
|
|
120
120
|
appendEvent(context, controllerId, "launch_requested", { request_id: input.requestId, generation, request_hash: requestHash });
|
|
121
|
-
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
throw error;
|
|
126
|
-
}
|
|
121
|
+
return null;
|
|
122
|
+
});
|
|
123
|
+
if (duplicate)
|
|
124
|
+
return duplicate;
|
|
127
125
|
return await spawnController(context, requireController(context, controllerId));
|
|
128
126
|
}
|
|
129
127
|
/**
|
|
@@ -136,24 +134,34 @@ export async function launchRunController(context, input, prepared = prepareRunC
|
|
|
136
134
|
export function controllerLeaseMonitor(heartbeat) {
|
|
137
135
|
let lost = false;
|
|
138
136
|
let unconfirmed = null;
|
|
137
|
+
let confirmedUntil = 0;
|
|
139
138
|
const lostError = () => new AppError("controller_lease_lost", "Managed owner lease could not be confirmed", 1);
|
|
139
|
+
const uncertainError = () => new AppError("controller_lease_unconfirmed", "Managed owner lease requires renewal outside the RUN writer", 1);
|
|
140
140
|
return {
|
|
141
141
|
tick() {
|
|
142
142
|
try {
|
|
143
143
|
if (!heartbeat())
|
|
144
144
|
lost = true;
|
|
145
|
-
else
|
|
145
|
+
else {
|
|
146
146
|
unconfirmed = null;
|
|
147
|
+
confirmedUntil = Date.now() + 14 * 60_000;
|
|
148
|
+
}
|
|
147
149
|
}
|
|
148
150
|
catch (error) {
|
|
149
151
|
unconfirmed = error;
|
|
150
152
|
}
|
|
151
153
|
},
|
|
152
|
-
isConfirmed() { return !lost && unconfirmed === null; },
|
|
154
|
+
isConfirmed() { return !lost && unconfirmed === null && Date.now() < confirmedUntil; },
|
|
155
|
+
assertNoRenewal() {
|
|
156
|
+
if (lost)
|
|
157
|
+
throw lostError();
|
|
158
|
+
if (unconfirmed !== null || Date.now() >= confirmedUntil)
|
|
159
|
+
throw uncertainError();
|
|
160
|
+
},
|
|
153
161
|
assertConfirmed() {
|
|
154
162
|
if (lost)
|
|
155
163
|
throw lostError();
|
|
156
|
-
if (unconfirmed === null)
|
|
164
|
+
if (unconfirmed === null && Date.now() < confirmedUntil)
|
|
157
165
|
return;
|
|
158
166
|
try {
|
|
159
167
|
if (!heartbeat()) {
|
|
@@ -161,6 +169,7 @@ export function controllerLeaseMonitor(heartbeat) {
|
|
|
161
169
|
throw lostError();
|
|
162
170
|
}
|
|
163
171
|
unconfirmed = null;
|
|
172
|
+
confirmedUntil = Date.now() + 14 * 60_000;
|
|
164
173
|
}
|
|
165
174
|
catch (error) {
|
|
166
175
|
if (error instanceof AppError)
|
|
@@ -194,6 +203,7 @@ export async function serveRunController(context, input) {
|
|
|
194
203
|
throw new AppError("controller_owner_mismatch", "Managed owner has no resource lease", 1);
|
|
195
204
|
confirmManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
|
|
196
205
|
const lease = controllerLeaseMonitor(() => heartbeatManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token }));
|
|
206
|
+
lease.assertConfirmed();
|
|
197
207
|
const heartbeat = setInterval(() => {
|
|
198
208
|
try {
|
|
199
209
|
lease.tick();
|
|
@@ -209,7 +219,8 @@ export async function serveRunController(context, input) {
|
|
|
209
219
|
const manifest = JSON.parse(row.manifest_json);
|
|
210
220
|
const state = JSON.parse(row.state_json);
|
|
211
221
|
const assertPhysicalOwner = () => {
|
|
212
|
-
lease.
|
|
222
|
+
lease.assertNoRenewal();
|
|
223
|
+
assertManagedProcessLease(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid });
|
|
213
224
|
const current = requireController(context, row.controller_id);
|
|
214
225
|
if (current.owner_pid !== process.pid || current.owner_token !== row.owner_token || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
|
|
215
226
|
throw new AppError("controller_owner_mismatch", "Controller no longer owns productive dispatch", 1);
|
|
@@ -220,21 +231,15 @@ export async function serveRunController(context, input) {
|
|
|
220
231
|
process.stderr.write(`${JSON.stringify({ controller_id: row.controller_id, phase, primary: errorRecord(primaryFailure), secondary: errorRecord(error) })}\n`);
|
|
221
232
|
};
|
|
222
233
|
try {
|
|
223
|
-
context.db.
|
|
224
|
-
try {
|
|
234
|
+
context.db.writeTransaction(() => {
|
|
225
235
|
markControllerDispatchReady(context, row);
|
|
226
236
|
if (!state.recovery && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation && requireRun(context, row.project_id, row.run_id).status === "paused")
|
|
227
237
|
setFlowRunRecoveryPaused(context, row.project_id, row.run_id, false);
|
|
228
|
-
|
|
229
|
-
}
|
|
230
|
-
catch (error) {
|
|
231
|
-
context.db.exec("ROLLBACK TO controller_ready_status; RELEASE controller_ready_status");
|
|
232
|
-
throw error;
|
|
233
|
-
}
|
|
238
|
+
});
|
|
234
239
|
appendEvent(context, row.controller_id, "owner_started", { pid: process.pid, generation: row.generation });
|
|
235
240
|
if (state.recovery)
|
|
236
241
|
await resumeControllerSession(context, row, manifest, state, assertPhysicalOwner);
|
|
237
|
-
await executeController(context, row, manifest, state, assertOwner, assertPhysicalOwner);
|
|
242
|
+
await executeController(context, row, manifest, state, assertOwner, assertPhysicalOwner, () => lease.assertConfirmed());
|
|
238
243
|
return controllerReceipt(requireController(context, row.controller_id));
|
|
239
244
|
}
|
|
240
245
|
catch (error) {
|
|
@@ -258,25 +263,19 @@ export async function serveRunController(context, input) {
|
|
|
258
263
|
return controllerReceipt(requireController(context, row.controller_id));
|
|
259
264
|
}
|
|
260
265
|
}
|
|
261
|
-
context.db.exec("SAVEPOINT controller_failure_status");
|
|
262
266
|
try {
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
267
|
+
context.db.writeTransaction(() => {
|
|
268
|
+
// A failed capture must not reopen admission when this process exits.
|
|
269
|
+
const changed = context.db.run("UPDATE run_controllers SET status = CASE WHEN status IN ('capturing','capture_failed') THEN 'capture_failed' ELSE 'recovery_required' END, last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status NOT IN ('completed','cancelled','superseded')", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
|
|
270
|
+
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status IN ('dispatching','active')", [JSON.stringify(errorRecord(error)), context.now(), row.project_id, row.run_id, row.controller_id]);
|
|
271
|
+
if (changed.changes && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation) {
|
|
272
|
+
setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
|
|
273
|
+
}
|
|
274
|
+
appendEvent(context, row.controller_id, "recovery_required", errorRecord(error));
|
|
275
|
+
});
|
|
271
276
|
}
|
|
272
277
|
catch (failure) {
|
|
273
278
|
retainSecondary("persist_controller_failure", failure);
|
|
274
|
-
try {
|
|
275
|
-
context.db.exec("ROLLBACK TO controller_failure_status; RELEASE controller_failure_status");
|
|
276
|
-
}
|
|
277
|
-
catch (rollback) {
|
|
278
|
-
retainSecondary("rollback_controller_failure", rollback);
|
|
279
|
-
}
|
|
280
279
|
}
|
|
281
280
|
}
|
|
282
281
|
catch (secondary) {
|
|
@@ -321,7 +320,9 @@ export function prepareRunControllerAnswer(context, input) {
|
|
|
321
320
|
const operationId = `${row.controller_id}:answer:${input.requestId}`;
|
|
322
321
|
const answerHash = digestBytes(bytes);
|
|
323
322
|
const hash = digest({ pause_id: input.pauseId, answer_sha256: answerHash });
|
|
324
|
-
prepareControllerAnswerAcceptance(context, input, project.id, row, operationId, hash);
|
|
323
|
+
const previous = prepareControllerAnswerAcceptance(context, input, project.id, row, operationId, hash);
|
|
324
|
+
if (previous.prior)
|
|
325
|
+
return { kind: "replay", operationId, status: previous.prior.status };
|
|
325
326
|
return { kind: "create", project, row, bytes, operationId, answerHash, hash };
|
|
326
327
|
}
|
|
327
328
|
function prepareControllerAnswerAcceptance(context, input, projectId, row, operationId, hash) {
|
|
@@ -350,37 +351,26 @@ export function answerRunController(context, input, prepared = prepareRunControl
|
|
|
350
351
|
if (prepared.kind === "replay")
|
|
351
352
|
return { ok: true, operation_id: prepared.operationId, status: prepared.status, reused: true };
|
|
352
353
|
const { project, row, bytes, operationId, answerHash, hash } = prepared;
|
|
353
|
-
context.
|
|
354
|
-
|
|
354
|
+
const previous = prepareControllerAnswerAcceptance(context, input, project.id, row, operationId, hash);
|
|
355
|
+
if (previous.prior)
|
|
356
|
+
return { ok: true, operation_id: operationId, status: previous.prior.status, reused: true };
|
|
357
|
+
const file = path.join(row.state_dir, `answer-${digest(operationId)}-${answerHash}.md`);
|
|
358
|
+
publishImmutableInput(file, bytes, "runtime_request_conflict");
|
|
359
|
+
return context.db.writeTransaction(() => {
|
|
355
360
|
// Recheck only mutable acceptance state under the write transaction; bytes are already prepared.
|
|
356
361
|
const acceptance = prepareControllerAnswerAcceptance(context, input, project.id, row, operationId, hash);
|
|
357
362
|
if (acceptance.prior) {
|
|
358
|
-
context.db.exec("COMMIT");
|
|
359
363
|
return { ok: true, operation_id: operationId, status: acceptance.prior.status, reused: true };
|
|
360
364
|
}
|
|
361
365
|
const { run, stage, workId } = acceptance;
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
fs.writeFileSync(file, bytes, { flag: "wx", mode: 0o600 });
|
|
365
|
-
}
|
|
366
|
-
catch (error) {
|
|
367
|
-
if (error.code !== "EEXIST")
|
|
368
|
-
throw error;
|
|
369
|
-
// A crash before COMMIT may leave the immutable answer file behind.
|
|
370
|
-
if (!fs.readFileSync(file).equals(bytes))
|
|
371
|
-
throw new AppError("runtime_request_conflict", "Persisted answer bytes differ from this retry", 1);
|
|
372
|
-
}
|
|
366
|
+
if (requireController(context, row.controller_id).state_dir !== row.state_dir)
|
|
367
|
+
throw new AppError("controller_owner_mismatch", "Controller input directory changed before acceptance", 1);
|
|
373
368
|
const now = context.now();
|
|
374
369
|
const generation = recoveryGuard(context, project.id, run.id)?.generation ?? row.generation;
|
|
375
370
|
context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'accepted', ?, ?)", [operationId, row.controller_id, generation, hash, JSON.stringify({ action: "user_answer", pause_id: input.pauseId, stage: stage.stage, work_id: workId, file, sha256: answerHash }), now, now]);
|
|
376
371
|
appendEvent(context, row.controller_id, "answer_accepted", { operation_id: operationId, pause_id: input.pauseId, answer_sha256: answerHash });
|
|
377
|
-
context.db.exec("COMMIT");
|
|
378
372
|
return { ok: true, operation_id: operationId, status: "accepted" };
|
|
379
|
-
}
|
|
380
|
-
catch (error) {
|
|
381
|
-
context.db.exec("ROLLBACK");
|
|
382
|
-
throw error;
|
|
383
|
-
}
|
|
373
|
+
});
|
|
384
374
|
}
|
|
385
375
|
export function prepareRunControllerContext(context, input) {
|
|
386
376
|
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !/^[a-z][a-z-]*$/.test(input.stage) || !/^[a-f0-9]{64}$/.test(input.contextSha256))
|
|
@@ -405,8 +395,19 @@ export function supplyRunControllerContext(context, input, prepared = prepareRun
|
|
|
405
395
|
if (prepared.kind === "replay")
|
|
406
396
|
return { ok: true, operation_id: prepared.operationId, status: prepared.status, reused: true };
|
|
407
397
|
const { project, operationId, hash, bytes } = prepared;
|
|
408
|
-
context.
|
|
409
|
-
|
|
398
|
+
const preparedOwner = requireController(context, input.controllerId);
|
|
399
|
+
if (preparedOwner.project_id !== project.id || preparedOwner.run_id !== input.runId)
|
|
400
|
+
throw new AppError("controller_owner_mismatch", "Context submission belongs to another RUN", 1);
|
|
401
|
+
const existing = context.db.get("SELECT input_hash, status FROM run_controller_operations WHERE operation_id = ?", [operationId]);
|
|
402
|
+
if (existing) {
|
|
403
|
+
if (existing.input_hash !== hash)
|
|
404
|
+
throw new AppError("runtime_request_conflict", "Context request id has different inputs", 1);
|
|
405
|
+
return { ok: true, operation_id: operationId, status: existing.status, reused: true };
|
|
406
|
+
}
|
|
407
|
+
availableControllerContext(context, input, preparedOwner);
|
|
408
|
+
const file = path.join(preparedOwner.state_dir, `${input.stage}-${input.contextSha256}.context.json`);
|
|
409
|
+
publishImmutableInput(file, bytes, "stage_context_replacement");
|
|
410
|
+
return context.db.writeTransaction(() => {
|
|
410
411
|
const row = requireController(context, input.controllerId);
|
|
411
412
|
if (row.project_id !== project.id || row.run_id !== input.runId)
|
|
412
413
|
throw new AppError("controller_owner_mismatch", "Context submission belongs to another RUN", 1);
|
|
@@ -414,32 +415,18 @@ export function supplyRunControllerContext(context, input, prepared = prepareRun
|
|
|
414
415
|
if (prior) {
|
|
415
416
|
if (prior.input_hash !== hash)
|
|
416
417
|
throw new AppError("runtime_request_conflict", "Context request id has different inputs", 1);
|
|
417
|
-
context.db.exec("COMMIT");
|
|
418
418
|
return { ok: true, operation_id: operationId, status: prior.status, reused: true };
|
|
419
419
|
}
|
|
420
420
|
const manifest = availableControllerContext(context, input, row);
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
fs.writeFileSync(file, bytes, { flag: "wx", mode: 0o600 });
|
|
424
|
-
}
|
|
425
|
-
catch (error) {
|
|
426
|
-
if (error.code !== "EEXIST")
|
|
427
|
-
throw error;
|
|
428
|
-
if (!fs.readFileSync(file).equals(bytes))
|
|
429
|
-
throw new AppError("stage_context_replacement", "Unpublished context bytes differ from this request", 1);
|
|
430
|
-
}
|
|
421
|
+
if (row.state_dir !== preparedOwner.state_dir)
|
|
422
|
+
throw new AppError("controller_owner_mismatch", "Controller input directory changed before acceptance", 1);
|
|
431
423
|
manifest.contexts[input.stage] = { file, sha256: input.contextSha256 };
|
|
432
424
|
const now = context.now();
|
|
433
425
|
context.db.run("UPDATE run_controllers SET manifest_json = ?, updated_at = ? WHERE controller_id = ?", [JSON.stringify(manifest), now, row.controller_id]);
|
|
434
426
|
context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'accepted', ?, ?)", [operationId, row.controller_id, row.generation, hash, JSON.stringify({ action: "stage_context", stage: input.stage, file, sha256: input.contextSha256 }), now, now]);
|
|
435
427
|
appendEvent(context, row.controller_id, "context_accepted", { operation_id: operationId, stage: input.stage, context_sha256: input.contextSha256 });
|
|
436
|
-
context.db.exec("COMMIT");
|
|
437
428
|
return { ok: true, operation_id: operationId, status: "accepted" };
|
|
438
|
-
}
|
|
439
|
-
catch (error) {
|
|
440
|
-
context.db.exec("ROLLBACK");
|
|
441
|
-
throw error;
|
|
442
|
-
}
|
|
429
|
+
});
|
|
443
430
|
}
|
|
444
431
|
function availableControllerContext(context, input, row) {
|
|
445
432
|
assertRunMutationAllowed(context, row.project_id, input.runId, row.generation);
|
|
@@ -450,7 +437,7 @@ function availableControllerContext(context, input, row) {
|
|
|
450
437
|
throw new AppError("stage_context_replacement", "Controller already has an immutable context for this stage", 1);
|
|
451
438
|
return manifest;
|
|
452
439
|
}
|
|
453
|
-
async function executeController(context, row, manifest, state, assertOwnership, assertPhysicalOwner) {
|
|
440
|
+
async function executeController(context, row, manifest, state, assertOwnership, assertPhysicalOwner, renewLease) {
|
|
454
441
|
assertOwnership();
|
|
455
442
|
const observer = { ...context, db: getDatabase(context.ddFlowHome, "read_existing") };
|
|
456
443
|
const checkOutcomes = () => assertLifecycleOutcomes(observer, { projectRoot: manifest.request.project_root, runId: row.run_id, generation: row.generation });
|
|
@@ -578,6 +565,9 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
578
565
|
monitor.unref();
|
|
579
566
|
try {
|
|
580
567
|
for (;;) {
|
|
568
|
+
// Renewal touches the resource registry; never do it from a callback that
|
|
569
|
+
// may already own the RUN writer.
|
|
570
|
+
renewLease();
|
|
581
571
|
assertOwner();
|
|
582
572
|
if (state.current_session !== null && !state.sessions[state.current_session].stopped)
|
|
583
573
|
bindInvocationScope(state.sessions[state.current_session]);
|
|
@@ -650,7 +640,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
650
640
|
loadExternalStageContext({ file: stageContext.file, expectedSha256: stageContext.sha256, stage: next.stage });
|
|
651
641
|
const continuation = JSON.stringify(["start_stage", next.stage, next.attempt ?? null]);
|
|
652
642
|
retainControllerContinuation(state, continuation, { code: "stage_entry_nonprogressing", message: "The coordinator returned without starting the assigned Stage", details: { stage: next.stage, attempt: next.attempt ?? null, last_receipt: state.last_receipt ?? null } });
|
|
653
|
-
const transition = resolveStageTransition({ ddFlowHome: context.ddFlowHome, policy, profiles, ...(state.stage ? { fromStage: state.stage } : {}), stage: next.stage, stageSessionMode: manifest.execution_profile.settings.stage_session_mode, mergeMode: manifest.execution_profile.settings.merge_mode });
|
|
643
|
+
const transition = resolveStageTransition({ ddFlowHome: context.ddFlowHome, policy, profiles, ...(state.stage ? { fromStage: state.stage } : {}), stage: next.stage, projectRoot: run.project_root, workspaceRoot: run.workspace_root, stageSessionMode: manifest.execution_profile.settings.stage_session_mode, mergeMode: manifest.execution_profile.settings.merge_mode });
|
|
654
644
|
if (next.stage === "merge" && manifest.execution_profile.settings.merge_mode === "server" && !state.merge_claim) {
|
|
655
645
|
const request = context.db.get("SELECT merge_request_id, status FROM merge_requests WHERE project_id = ? AND run_id = ? ORDER BY created_at DESC, merge_request_id DESC LIMIT 1", [row.project_id, row.run_id]);
|
|
656
646
|
if (!request || request.status !== "queued")
|
|
@@ -665,8 +655,12 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
665
655
|
saveState(context, row, state);
|
|
666
656
|
appendEvent(context, row.controller_id, "merge_claimed", { request_id: claim.request.merge_request_id });
|
|
667
657
|
}
|
|
658
|
+
const stageCwd = stageExecutionCwd({ stage: next.stage, projectRoot: run.project_root, workspaceRoot: run.workspace_root, ...(state.merge_claim ? { mergeTargetWorkspace: state.merge_claim.target_workspace } : {}) });
|
|
668
659
|
let session = state.current_session === null ? null : state.sessions[state.current_session];
|
|
669
|
-
if (
|
|
660
|
+
if (session && next.stage === "merge" && manifest.execution_profile.settings.merge_mode === "same_session" && path.resolve(session.cwd) !== stageCwd) {
|
|
661
|
+
throw new AppError("merge_session_workspace_conflict", "Same-session MERGE cannot change the native Session working directory", 1, { session_cwd: session.cwd, target_cwd: stageCwd });
|
|
662
|
+
}
|
|
663
|
+
if (!session || transition.session_mode === "new_session" || !sameSessionProfile(session.profile, transition.coordinator) || path.resolve(session.cwd) !== stageCwd) {
|
|
670
664
|
if (session)
|
|
671
665
|
await stopSession(session);
|
|
672
666
|
assertOwner();
|
|
@@ -677,7 +671,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
677
671
|
owner.role = "merge";
|
|
678
672
|
owner.work_id = state.merge_claim.work_id;
|
|
679
673
|
}
|
|
680
|
-
session = { id: "", profile: transition.coordinator, state_dir: directory, cwd:
|
|
674
|
+
session = { id: "", profile: transition.coordinator, state_dir: directory, cwd: stageCwd, journal: path.join(directory, "adapter.events.jsonl"), owner };
|
|
681
675
|
if (runtimeHarness(session.profile) === "codex-desktop") {
|
|
682
676
|
session.codex_home = path.join(directory, "codex-home");
|
|
683
677
|
initCodexHome(context, { projectRoot: run.project_root, profile: `controller-${digest(directory).slice(0, 16)}`, targetHome: session.codex_home, ...(context.env.CODEX_HOME ? { sourceHome: context.env.CODEX_HOME } : {}) });
|
|
@@ -722,9 +716,10 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
722
716
|
const fanout = await nextControllerFanout(executionContext, { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles });
|
|
723
717
|
assertOwner();
|
|
724
718
|
if (fanout.kind === "none") {
|
|
725
|
-
|
|
719
|
+
assertNoCorrelatedRejection(context, { projectId: row.project_id, runId: row.run_id, generation: row.generation, stage: next.stage, turn: state.turns });
|
|
720
|
+
const cause = currentLifecycleFailure(executionContext, { runId: run.id, generation: row.generation, stage: next.stage, turn: state.turns, projectId: row.project_id, projectRoot: run.project_root });
|
|
726
721
|
if (cause)
|
|
727
|
-
throw new AppError(cause.code, cause.message, 1, { ...cause.details, invocation_id: cause.invocation_id, controller_wrapper: "incomplete_subject_turn", stage: next.stage });
|
|
722
|
+
throw new AppError(cause.code, cause.message, 1, { ...cause.details, ...(cause.invocation_id ? { invocation_id: cause.invocation_id } : {}), ...(cause.hook_event_id ? { hook_event_id: cause.hook_event_id } : {}), controller_wrapper: "incomplete_subject_turn", stage: next.stage });
|
|
728
723
|
throw new AppError("incomplete_subject_turn", "Managed Turn ended without a completed Stage or declared continuation", 1, { stage: next.stage });
|
|
729
724
|
}
|
|
730
725
|
if (fanout.kind === "wait") {
|