@deksden-com/dd-flow-cli 0.9.0-beta.38 → 0.9.0-beta.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -1
- package/dist/build-info.json +5 -5
- package/dist/cli/help.js +42 -1
- package/dist/cli/run-cli.js +145 -17
- package/dist/harness-runtime/bin/dd-agy.mjs +3 -2
- package/dist/harness-runtime/bin/dd-codex.mjs +4 -3
- package/dist/harness-runtime/bin/dd-droid.mjs +3 -2
- package/dist/harness-runtime/bin/dd-grok.mjs +4 -3
- package/dist/harness-runtime/bin/dd-opencode.mjs +4 -3
- package/dist/harness-runtime/bin/dd-zcode.mjs +4 -3
- package/dist/harness-runtime/lib/daemon-operations.mjs +48 -9
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +49 -12
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +57 -16
- package/dist/harness-runtime/lib/dd-codex.mjs +10 -2
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +3 -2
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +12 -6
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +4 -3
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +8 -5
- package/dist/harness-runtime/lib/managed-daemon.mjs +125 -6
- package/dist/harness-runtime/lib/process-json.mjs +1 -1
- package/dist/harness-runtime/lib/session-settlement.mjs +4 -2
- package/dist/schemas/run-control-receipt.schema.json +99 -0
- package/dist/schemas/run-control-request.schema.json +36 -0
- package/dist/services/cleanup.js +7 -4
- package/dist/services/cli-operation-classifier.js +3 -1
- package/dist/services/code-checks.js +33 -7
- package/dist/services/controller-fanout.js +105 -0
- package/dist/services/engines.js +2 -0
- package/dist/services/eval-snapshots.js +347 -36
- package/dist/services/execution-policy.js +122 -6
- package/dist/services/external-work-launch.js +33 -12
- package/dist/services/harness-adapter.js +49 -2
- package/dist/services/managed-daemon-binding.js +38 -0
- package/dist/services/managed-processes.js +54 -7
- package/dist/services/merge-server.js +53 -30
- package/dist/services/native-daemon-history.js +49 -0
- package/dist/services/native-session-control.js +31 -0
- package/dist/services/protocols.js +9 -0
- package/dist/services/recovery-snapshot-database.js +103 -0
- package/dist/services/run-control-receipt.js +56 -0
- package/dist/services/run-control-worker.js +297 -0
- package/dist/services/run-control.js +752 -0
- package/dist/services/run-controller-adapter.js +44 -0
- package/dist/services/run-controller-capture.js +117 -0
- package/dist/services/run-controller-process.js +178 -0
- package/dist/services/run-controller-recovery.js +219 -0
- package/dist/services/run-controller-state.js +31 -0
- package/dist/services/run-controller.js +538 -0
- package/dist/services/run-recovery-runtime.js +69 -0
- package/dist/services/run-recovery.js +33 -9
- package/dist/services/runs.js +65 -13
- package/dist/services/runtime-budget.js +362 -0
- package/dist/services/runtime-scope-capture.js +50 -0
- package/dist/services/runtime-scope-control.js +431 -0
- package/dist/services/runtime-scope-resume.js +543 -0
- package/dist/services/runtime-scope-stop.js +95 -0
- package/dist/services/runtime-scope-worker.js +165 -0
- package/dist/services/runtime-service.js +4 -0
- package/dist/services/vnext-code-review.js +1 -1
- package/dist/services/vnext-code.js +1 -1
- package/dist/services/vnext-execution-profile.js +2 -35
- package/dist/services/vnext-fanout.js +23 -7
- package/dist/services/vnext-plan-review.js +12 -9
- package/dist/services/vnext-plan.js +1 -1
- package/dist/services/vnext-protocolize.js +10 -4
- package/dist/services/vnext-specify.js +12 -5
- package/dist/services/work-registry.js +26 -8
- package/dist/services/workspace-bootstrap.js +10 -1
- package/dist/storage/database.js +160 -13
- package/dist/storage/writer-contract.js +76 -0
- package/dist/storage/writer-migration.js +102 -0
- package/package.json +1 -1
|
@@ -2,11 +2,12 @@ import crypto from "node:crypto";
|
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { AppError } from "../shared/errors.js";
|
|
5
|
+
import { runtimeBudgetForRun } from "./runtime-budget.js";
|
|
5
6
|
import { flowCommand } from "./stage-pause.js";
|
|
6
|
-
import {
|
|
7
|
-
import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
|
|
7
|
+
import { adapterSessionId, prepareHarnessFlowExecutable, runHarnessAdapter } from "./harness-adapter.js";
|
|
8
8
|
import { harnessRuntimeArguments, resolveHarnessCommand } from "./harness-config.js";
|
|
9
9
|
import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
|
|
10
|
+
import { loadAgentProfile, resolveExecutionPolicy } from "./execution-policy.js";
|
|
10
11
|
/** A deterministic dispatcher; only the launched harness Session performs agent work. */
|
|
11
12
|
export async function serveMergeRequests(context, input) {
|
|
12
13
|
const running = context.db.get("SELECT server_id, pid FROM merge_servers WHERE status IN ('running','stopping') ORDER BY started_at LIMIT 1");
|
|
@@ -59,49 +60,80 @@ export async function serveMergeRequests(context, input) {
|
|
|
59
60
|
export function mergeServerStatus(context) { return { ok: true, servers: context.db.all("SELECT server_id, profile_id, status, pid, started_at, heartbeat_at, stopped_at, journal_path FROM merge_servers ORDER BY started_at DESC") }; }
|
|
60
61
|
export function stopMergeServer(context, input) { const changed = context.db.run("UPDATE merge_servers SET status = 'stopping', heartbeat_at = ? WHERE server_id = ? AND status = 'running'", [context.now(), input.serverId]); if (changed.changes !== 1)
|
|
61
62
|
throw new AppError("not_found", "Running merge-server was not found", 1, { server_id: input.serverId }); return { ok: true, server_id: input.serverId, status: "stopping" }; }
|
|
62
|
-
|
|
63
|
-
|
|
63
|
+
/** Both dispatchers enter the same project FIFO before creating any provider Session. */
|
|
64
|
+
export function claimQueuedMergeRequest(context, input) {
|
|
64
65
|
context.db.exec("BEGIN IMMEDIATE");
|
|
65
|
-
const token = crypto.randomUUID();
|
|
66
|
-
const leaseUntil = new Date(Date.now() + 120_000).toISOString();
|
|
67
66
|
try {
|
|
68
|
-
|
|
69
|
-
|
|
67
|
+
const request = context.db.get("SELECT merge_request_id, project_id, run_id, executor_work_id, target_workspace, execution_route, status, created_at FROM merge_requests WHERE merge_request_id = ?", [input.requestId]);
|
|
68
|
+
if (!request)
|
|
69
|
+
throw new AppError("not_found", "MERGE request was not found", 1);
|
|
70
|
+
const generation = assertRunMutationAllowed(context, request.project_id, request.run_id, input.expectedGeneration);
|
|
71
|
+
const earlier = context.db.get("SELECT 1 FROM merge_requests WHERE project_id = ? AND status NOT IN ('completed','failed','cancelled','superseded') AND (created_at < ? OR (created_at = ? AND merge_request_id < ?)) LIMIT 1", [request.project_id, request.created_at, request.created_at, request.merge_request_id]);
|
|
72
|
+
if (request.status !== "queued" || request.execution_route !== "server" || earlier) {
|
|
73
|
+
context.db.exec("COMMIT");
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
const token = crypto.randomUUID();
|
|
77
|
+
context.db.run("UPDATE merge_requests SET status = 'dispatching', dispatch_owner = ?, dispatch_lease_token = ?, dispatch_lease_expires_at = ?, updated_at = ? WHERE merge_request_id = ? AND status = 'queued'", [input.ownerId, token, new Date(Date.now() + 120_000).toISOString(), context.now(), request.merge_request_id]);
|
|
70
78
|
context.db.exec("COMMIT");
|
|
71
|
-
|
|
72
|
-
return;
|
|
79
|
+
return { request, token, generation };
|
|
73
80
|
}
|
|
74
81
|
catch (error) {
|
|
75
82
|
context.db.exec("ROLLBACK");
|
|
76
83
|
throw error;
|
|
77
84
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
85
|
+
}
|
|
86
|
+
async function dispatch(context, input) {
|
|
87
|
+
const run = context.db.get("SELECT index_json, project_root FROM runs WHERE project_id = ? AND id = ?", [input.request.project_id, input.request.run_id]);
|
|
88
|
+
if (!run)
|
|
89
|
+
throw new AppError("not_found", "MERGE RUN was not found", 1, { run_id: input.request.run_id });
|
|
90
|
+
const snapshot = JSON.parse(run.index_json).execution_profile;
|
|
91
|
+
if (snapshot?.settings?.execution) {
|
|
92
|
+
if (!snapshot.agent_profiles)
|
|
93
|
+
throw new AppError("execution_profile_not_frozen", "MERGE dispatch requires frozen RUN agent profiles", 1, { run_id: input.request.run_id });
|
|
94
|
+
const resolved = resolveExecutionPolicy({ ddFlowHome: context.ddFlowHome, policy: snapshot.settings.execution, profiles: snapshot.agent_profiles, stage: "merge" });
|
|
95
|
+
input = { ...input, profile: resolved.coordinator };
|
|
96
|
+
}
|
|
97
|
+
const claim = claimQueuedMergeRequest(context, { requestId: input.request.merge_request_id, ownerId: input.serverId });
|
|
98
|
+
if (!claim)
|
|
99
|
+
return;
|
|
100
|
+
const { generation, token } = claim;
|
|
81
101
|
const stateDir = path.join(input.root, input.request.merge_request_id);
|
|
82
102
|
fs.mkdirSync(stateDir, { recursive: true });
|
|
83
103
|
const promptFile = path.join(stateDir, "launch.md");
|
|
84
104
|
const adapterJournal = path.join(stateDir, "adapter.events.jsonl");
|
|
85
|
-
const
|
|
105
|
+
const flowExecutable = prepareHarnessFlowExecutable(context, stateDir);
|
|
106
|
+
const runtimeContext = { ...context, env: { ...context.env, DD_FLOW_BIN: flowExecutable } };
|
|
107
|
+
const stageCommand = `${flowCommand(runtimeContext)} stage start ${input.request.run_id} --stage merge --project-root ${JSON.stringify(run.project_root)} --json --progress-jsonl`;
|
|
86
108
|
fs.writeFileSync(promptFile, `Start the assigned MERGE stage now. Your first tool call must be this exact standalone command:\n\n${stageCommand}\n\nTrust and follow the complete stage packet it returns. Continue through merge apply and stage finish. Stop only when the stage reaches a terminal result or explicitly requests user/operator action.\n`);
|
|
109
|
+
const runtimeOwner = { schema_id: "dd-flow/runtime-owner@1", owner_id: `run:${input.request.run_id}:merge:${input.request.merge_request_id}`, role: "merge", project_id: input.request.project_id, run_id: input.request.run_id, work_id: input.request.executor_work_id, operation_id: `${input.request.merge_request_id}:${token}`, state_dir: stateDir, dd_flow_home: context.ddFlowHome, dd_flow_bin: flowExecutable, resource_home: context.env.DD_FLOW_RESOURCE_HOME ?? context.ddFlowHome, budget: runtimeBudgetForRun(context, input.request.project_id, input.request.run_id) };
|
|
87
110
|
event(input.journal, { type: "launch_intent", server_id: input.serverId, request_id: input.request.merge_request_id, token, profile: input.profile, prompt_file: promptFile, at: context.now() });
|
|
88
111
|
input.progress?.(`launching ${input.request.merge_request_id} with ${input.profile.id}`);
|
|
89
112
|
const executable = resolveHarnessCommand(context.ddFlowHome, input.profile.harness, "adapter");
|
|
90
|
-
const common = ["--state-dir", stateDir, "--journal", adapterJournal, "--cwd", input.request.target_workspace, "--provider", input.profile.provider, "--model", input.profile.model, "--reasoning", input.profile.reasoning, "--mode", input.profile.mode, "--permission", input.profile.permission, "--project-root",
|
|
113
|
+
const common = ["--dd-flow-bin", flowExecutable, "--state-dir", stateDir, "--journal", adapterJournal, "--cwd", input.request.target_workspace, "--provider", input.profile.provider, "--model", input.profile.model, "--reasoning", input.profile.reasoning, "--mode", input.profile.mode, "--permission", input.profile.permission, "--project-root", run.project_root, "--dd-flow-home", context.ddFlowHome, ...harnessRuntimeArguments(context.ddFlowHome, input.profile.harness), "--json"];
|
|
114
|
+
const retained = { profile: input.profile, executable, owner: runtimeOwner, session_id: null, state_dir: stateDir, create_operation_id: `${input.request.merge_request_id}:${token}:session.create`, prompt_operation_id: `${input.request.merge_request_id}:${token}:session.prompt` };
|
|
115
|
+
const retain = () => { context.db.run("UPDATE merge_requests SET adapter_receipt_json = ?, updated_at = ? WHERE merge_request_id = ? AND dispatch_owner = ? AND dispatch_lease_token = ?", [JSON.stringify(retained), context.now(), input.request.merge_request_id, input.serverId, token]); };
|
|
116
|
+
retain();
|
|
117
|
+
const leaseHeartbeat = setInterval(() => { if (guarded(context, input.request) || (recoveryGuard(context, input.request.project_id, input.request.run_id)?.generation ?? 0) !== generation)
|
|
118
|
+
return; const now = context.now(); context.db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE merge_request_id = ? AND status = 'dispatching' AND dispatch_owner = ? AND dispatch_lease_token = ?", [new Date(Date.now() + 120_000).toISOString(), now, input.request.merge_request_id, input.serverId, token]); }, 30_000);
|
|
119
|
+
leaseHeartbeat.unref();
|
|
91
120
|
try {
|
|
92
|
-
const adapterCall = (args, timeoutMs, progressMessage) => { assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation); return runHarnessAdapter({ executable, args, env:
|
|
121
|
+
const adapterCall = (args, timeoutMs, progressMessage) => { assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation); return runHarnessAdapter({ executable, args, env: runtimeContext.env, runtimeOwner, operationId: `${input.request.merge_request_id}:${token}:${args[0]}.${args[1]}`, ...(timeoutMs ? { timeoutMs } : {}), ...(input.progress ? { progress: input.progress } : {}), ...(progressMessage ? { progressMessage } : {}), onEvidence: (value) => event(input.journal, { type: "adapter_call", ...value }) }); };
|
|
93
122
|
await adapterCall(["daemon", "start", ...common]);
|
|
94
123
|
const created = await adapterCall(["session", "create", ...common]);
|
|
95
124
|
const sessionId = adapterSessionId(created);
|
|
96
125
|
if (!sessionId)
|
|
97
126
|
throw new AppError("merge_adapter_invalid", "Harness adapter did not return a Session id", 1, { harness: input.profile.harness, response: created });
|
|
127
|
+
Object.assign(retained, { session_id: sessionId, create: created });
|
|
128
|
+
retain();
|
|
98
129
|
const prompted = await adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], 45 * 60 * 1000, `MERGE ${input.request.merge_request_id} Session ${sessionId} is still running`);
|
|
130
|
+
Object.assign(retained, { response: prompted });
|
|
131
|
+
retain();
|
|
99
132
|
assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation);
|
|
100
133
|
const request = context.db.get("SELECT status FROM merge_requests WHERE merge_request_id = ?", [input.request.merge_request_id]);
|
|
101
|
-
context.db.run("UPDATE merge_requests SET adapter_receipt_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ profile: input.profile, executable, session_id: sessionId, response: prompted }), context.now(), input.request.merge_request_id]);
|
|
102
134
|
event(input.journal, { type: "adapter_settled", request_id: input.request.merge_request_id, session_id: sessionId, request_status: request?.status ?? null, at: context.now() });
|
|
103
135
|
if (request?.status === "dispatching")
|
|
104
|
-
context.db.run("UPDATE merge_requests SET status = '
|
|
136
|
+
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_did_not_start_stage", response: prompted }), context.now(), input.request.merge_request_id]);
|
|
105
137
|
}
|
|
106
138
|
catch (error) {
|
|
107
139
|
if (error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale"].includes(error.code)) {
|
|
@@ -110,7 +142,7 @@ async function dispatch(context, input) {
|
|
|
110
142
|
}
|
|
111
143
|
const request = context.db.get("SELECT status FROM merge_requests WHERE merge_request_id = ?", [input.request.merge_request_id]);
|
|
112
144
|
if (request?.status === "dispatching")
|
|
113
|
-
context.db.run("UPDATE merge_requests SET status = '
|
|
145
|
+
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_launch_failed", error: String(error) }), context.now(), input.request.merge_request_id]);
|
|
114
146
|
else if (request?.status === "active")
|
|
115
147
|
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_lost_after_stage_start", error: String(error) }), context.now(), input.request.merge_request_id]);
|
|
116
148
|
throw error;
|
|
@@ -125,18 +157,9 @@ function reconcileExpiredDispatches(context) { const expired = context.db.all("S
|
|
|
125
157
|
if (guarded(context, request))
|
|
126
158
|
continue;
|
|
127
159
|
const work = context.db.get("SELECT status FROM works WHERE work_id = ?", [request.executor_work_id]);
|
|
128
|
-
|
|
129
|
-
context.db.run("UPDATE merge_requests SET status = 'queued', dispatch_owner = NULL, dispatch_lease_token = NULL, dispatch_lease_expires_at = NULL, last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "expired_dispatch_requeued" }), context.now(), request.merge_request_id]);
|
|
130
|
-
else
|
|
131
|
-
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "expired_dispatch_with_started_work", work_status: work?.status ?? null }), context.now(), request.merge_request_id]);
|
|
160
|
+
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "expired_dispatch_outcome_unknown", work_status: work?.status ?? null }), context.now(), request.merge_request_id]);
|
|
132
161
|
} }
|
|
133
|
-
function loadProfile(context, id) {
|
|
134
|
-
value = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
135
|
-
}
|
|
136
|
-
catch (error) {
|
|
137
|
-
throw new AppError("agent_profile_missing", "Merge-server agent profile is unavailable", 1, { id, file, cause: String(error) });
|
|
138
|
-
} validateSchema({ schemaName: "agent-profile", file }); if (value.id !== id)
|
|
139
|
-
throw new AppError("agent_profile_invalid", "Agent profile id does not match its requested filename", 2, { file, expected: id, actual: value.id ?? null }); return value; }
|
|
162
|
+
function loadProfile(context, id) { return loadAgentProfile(context.ddFlowHome, id); }
|
|
140
163
|
function event(file, value) { fs.appendFileSync(file, `${JSON.stringify(value)}\n`); }
|
|
141
164
|
function guarded(context, request) { const guard = recoveryGuard(context, request.project_id, request.run_id); return Boolean(guard && ["draining", "sealed", "resuming"].includes(guard.status)); }
|
|
142
165
|
function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { AppError } from "../shared/errors.js";
|
|
5
|
+
import { managedProcessBinding, managedProcessStatus, processTreeIsAlive, resourceHome } from "./managed-processes.js";
|
|
6
|
+
import { retainedDaemonOperations } from "./run-control.js";
|
|
7
|
+
import { managedProviderBelongsToDaemon } from "./managed-daemon-binding.js";
|
|
8
|
+
/** A replaced daemon is a settled resource, never another Session to resume. */
|
|
9
|
+
export function historicalDaemonSettlement(context, scopeId, record) {
|
|
10
|
+
const metadata = JSON.parse(record.metadata_json);
|
|
11
|
+
if (!record.kind.endsWith("-daemon") || !metadata.state_dir || !path.isAbsolute(metadata.state_dir))
|
|
12
|
+
throw new AppError("runtime_scope_owner_unknown", "Native history requires an absolute daemon directory", 1);
|
|
13
|
+
const current = JSON.parse(fs.readFileSync(path.join(metadata.state_dir, "daemon.json"), "utf8"));
|
|
14
|
+
if (current.resource_process?.id === record.id)
|
|
15
|
+
return null;
|
|
16
|
+
if (record.state !== "stopped" || record.termination_reason !== "daemon_stopped" || !record.pid || !record.pid_started_at || processTreeIsAlive(record))
|
|
17
|
+
throw new AppError("runtime_scope_owner_unknown", "Previous daemon has no physical completion proof", 1);
|
|
18
|
+
const directory = path.join(metadata.state_dir, "daemon-history");
|
|
19
|
+
const matches = fs.readdirSync(directory).filter(name => /^[a-zA-Z0-9-]+\.json$/.test(name)).map(name => {
|
|
20
|
+
const file = path.join(directory, name);
|
|
21
|
+
if (!fs.lstatSync(file).isFile())
|
|
22
|
+
throw new AppError("runtime_scope_owner_unknown", "Daemon history must contain regular receipts", 1);
|
|
23
|
+
const bytes = fs.readFileSync(file, "utf8");
|
|
24
|
+
return { file, bytes, state: JSON.parse(bytes) };
|
|
25
|
+
}).filter(candidate => candidate.state.resource_process?.id === record.id);
|
|
26
|
+
if (matches.length !== 1)
|
|
27
|
+
throw new AppError("runtime_scope_owner_unknown", "Previous daemon has no unique retained receipt", 1);
|
|
28
|
+
const { file, bytes, state } = matches[0], owner = state.config?.runtime_owner;
|
|
29
|
+
if (!state.daemon_id || path.basename(file) !== `${state.daemon_id}.json` || !owner || owner.budget?.scope_id !== scopeId || owner.owner_id !== record.owner_id || owner.role !== metadata.role || owner.state_dir !== metadata.state_dir || owner.adapter_executable !== metadata.adapter_executable || owner.dd_flow_home !== metadata.dd_flow_home || owner.resource_home !== resourceHome(context) || (owner.project_id ?? null) !== record.project_id || (owner.run_id ?? null) !== record.run_id || (owner.work_id ?? null) !== record.work_id || !record.operation_id?.startsWith(`${owner.operation_id}:`) || state.resource_process?.lease_token !== record.lease_token || state.pid !== record.pid)
|
|
30
|
+
throw new AppError("runtime_scope_owner_unknown", "Archived daemon does not match its registration", 1);
|
|
31
|
+
const closed = state.shutdown_state === "clean" && state.active_tree === false
|
|
32
|
+
? retainedDaemonOperations(metadata.state_dir).find(operation => operation.daemon_id === state.daemon_id && operation.operation === "daemon.stop" && operation.state === "completed" && operation.result?.stopped === true && operation.result?.clean === true) : undefined;
|
|
33
|
+
const fresh = managedProcessStatus(context).find(item => item.id === record.id);
|
|
34
|
+
if (!closed || !fresh || fresh.lease_token !== record.lease_token || fresh.pid !== record.pid || fresh.pid_started_at !== record.pid_started_at || fresh.metadata_json !== record.metadata_json || fresh.state !== "stopped" || fresh.termination_reason !== "daemon_stopped" || processTreeIsAlive(fresh) || fs.readFileSync(file, "utf8") !== bytes)
|
|
35
|
+
throw new AppError("runtime_scope_outcome_unknown", "Previous daemon lacks stable clean shutdown evidence", 1);
|
|
36
|
+
const providers = managedProcessStatus(context).filter(item => managedProviderBelongsToDaemon(item, fresh, state));
|
|
37
|
+
if (state.provider_process && (providers.length !== 1 || providers[0].state !== "stopped" || providers[0].termination_reason !== "daemon_stopped" || !providers[0].operation_id?.startsWith(`${owner.operation_id}:`) || processTreeIsAlive(providers[0])))
|
|
38
|
+
throw new AppError("runtime_scope_outcome_unknown", "Archived provider lacks its exact physical completion proof", 1);
|
|
39
|
+
return { historical: true, process_id: record.id, role: metadata.role, state_dir: metadata.state_dir, daemon_id: state.daemon_id,
|
|
40
|
+
processes: providers.map(item => ({ process_id: item.id, native_settled: true, binding: managedProcessBinding(item, resourceHome(context)) })),
|
|
41
|
+
settled: true, native_settled: true, operation_id: String(closed.operation_id), owner_operation_id: owner.operation_id,
|
|
42
|
+
inventory_version: crypto.createHash("sha256").update(JSON.stringify([bytes, closed])).digest("hex") };
|
|
43
|
+
}
|
|
44
|
+
export function assertHistoricalDaemonSettlement(context, scopeId, proof) {
|
|
45
|
+
const record = managedProcessStatus(context).find(item => item.id === proof.process_id);
|
|
46
|
+
const fresh = record && historicalDaemonSettlement(context, scopeId, record);
|
|
47
|
+
if (!fresh || proof.historical !== true || proof.native_settled !== true || fresh.inventory_version !== proof.inventory_version)
|
|
48
|
+
throw new AppError("runtime_scope_inventory_changed", "Archived daemon settlement changed after observation", 1);
|
|
49
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { AppError } from "../shared/errors.js";
|
|
2
|
+
import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
|
|
3
|
+
export async function cancelControlledNativeSession(input) {
|
|
4
|
+
input.assertCurrent();
|
|
5
|
+
const reply = await runHarnessAdapter({ executable: input.executable,
|
|
6
|
+
args: input.observe ? ["daemon", "operation", "--operation-id", input.operationId, "--state-dir", input.stateDir, "--json"] : ["session", "cancel", "--session-id", input.sessionId, "--state-dir", input.stateDir, "--json"],
|
|
7
|
+
env: input.env, operationId: input.observe ? `${input.operationId}:observe` : input.operationId, timeoutMs: 30_000 });
|
|
8
|
+
if (input.observe && (reply.operation_id !== input.operationId || reply.operation !== "session.cancel" || reply.session_id !== input.sessionId))
|
|
9
|
+
throw new AppError("controller_operation_mismatch", "Cancellation observation belongs to another operation or Session", 1);
|
|
10
|
+
const result = input.observe ? reply.state === "completed" ? reply.result : null : reply;
|
|
11
|
+
if (!input.observe || reply.state === "completed")
|
|
12
|
+
assertNativeCancellationReceipt(result, input.sessionId);
|
|
13
|
+
input.assertCurrent();
|
|
14
|
+
return { status: result ? "completed" : "outcome_unknown", receipt: (result ?? reply) };
|
|
15
|
+
}
|
|
16
|
+
export function assertNativeCancellationReceipt(value, sessionId) {
|
|
17
|
+
const identity = adapterSessionId(value);
|
|
18
|
+
if (!value || typeof value !== "object" || Array.isArray(value) || identity && identity !== sessionId)
|
|
19
|
+
throw new AppError("controller_operation_mismatch", "Cancellation receipt has no valid result for its target Session", 1);
|
|
20
|
+
}
|
|
21
|
+
/** Native observation is shared by RUN and non-Work EVAL owners. */
|
|
22
|
+
export async function inspectControlledNativeSession(input) {
|
|
23
|
+
input.assertCurrent();
|
|
24
|
+
const receipt = await runHarnessAdapter({ executable: input.executable, args: ["session", "inspect", "--session-id", input.sessionId, "--state-dir", input.stateDir, "--json"], env: input.env, operationId: input.operationId, timeoutMs: 30_000 });
|
|
25
|
+
if (adapterSessionId(receipt) !== input.sessionId)
|
|
26
|
+
throw new AppError("controller_operation_mismatch", "Session observation belongs to another target", 1);
|
|
27
|
+
if (input.harness && receipt.harness !== undefined && receipt.harness !== input.harness)
|
|
28
|
+
throw new AppError("controller_operation_mismatch", "Session observation belongs to another harness", 1);
|
|
29
|
+
input.assertCurrent();
|
|
30
|
+
return receipt;
|
|
31
|
+
}
|
|
@@ -733,9 +733,18 @@ function releaseRelatedMergeLocks(context, protocol, reason) {
|
|
|
733
733
|
return { ok: true, released: released.length, locks: released };
|
|
734
734
|
}
|
|
735
735
|
export function readProtocolRuntimeState(context, protocol) {
|
|
736
|
+
const expected = runtimeStateJsonPath(context.ddFlowHome, protocol.project_id, protocol.id);
|
|
737
|
+
if (protocol.state_path !== expected)
|
|
738
|
+
throw new AppError("protocol_state_migration_required", "Protocol runtime path requires explicit cleanup scan/apply before use", 1, { protocol_id: protocol.id, path: protocol.state_path, expected_path: expected });
|
|
739
|
+
return repairProtocolRuntimeState(context, protocol);
|
|
740
|
+
}
|
|
741
|
+
/** Explicit cleanup migration; productive readers accept only the current path. */
|
|
742
|
+
export function repairProtocolRuntimeState(context, protocol) {
|
|
736
743
|
const diagnostics = [];
|
|
737
744
|
normalizeProtocolPlanPath(context, protocol, diagnostics);
|
|
738
745
|
try {
|
|
746
|
+
if (protocol.state_path !== runtimeStateJsonPath(context.ddFlowHome, protocol.project_id, protocol.id))
|
|
747
|
+
throw new AppError("protocol_state_migration_required", "Retired state files are not runtime authority", 1);
|
|
739
748
|
return { state: readStateFile(protocol.state_path, protocol.id), diagnostics };
|
|
740
749
|
}
|
|
741
750
|
catch {
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { AppError } from "../shared/errors.js";
|
|
5
|
+
import { registerStoreWriter } from "../storage/writer-contract.js";
|
|
6
|
+
import { managedProcessRuntimeHome } from "./managed-processes.js";
|
|
7
|
+
/** Export one RUN's relational evidence, never mutate the live authority. */
|
|
8
|
+
export function captureRecoveryDatabase(source, output, projectId, runId, protocolIds = []) {
|
|
9
|
+
source.exec(`VACUUM INTO '${output.replace(/'/g, "''")}'`);
|
|
10
|
+
const db = new DatabaseSync(output);
|
|
11
|
+
let complete = false;
|
|
12
|
+
try {
|
|
13
|
+
registerStoreWriter(db);
|
|
14
|
+
db.exec("PRAGMA foreign_keys = OFF; PRAGMA secure_delete = ON; BEGIN IMMEDIATE");
|
|
15
|
+
db.exec("CREATE TEMP TABLE capture_scope (project_id TEXT, run_id TEXT)");
|
|
16
|
+
db.prepare("INSERT INTO capture_scope VALUES (?, ?)").run(projectId, runId);
|
|
17
|
+
db.exec("CREATE TEMP TABLE capture_protocols (id TEXT)");
|
|
18
|
+
for (const id of protocolIds)
|
|
19
|
+
db.prepare("INSERT INTO capture_protocols VALUES (?)").run(id);
|
|
20
|
+
if (!db.prepare("SELECT 1 FROM runs WHERE (project_id, id) IN (SELECT * FROM capture_scope)").get())
|
|
21
|
+
throw new AppError("snapshot_run_missing", "Recovery export requires the selected RUN", 1);
|
|
22
|
+
const ownRun = "(project_id, run_id) IN (SELECT * FROM capture_scope)";
|
|
23
|
+
const ownProject = "project_id IN (SELECT project_id FROM capture_scope)";
|
|
24
|
+
const works = `SELECT work_id FROM works WHERE ${ownRun}`;
|
|
25
|
+
if (db.prepare(`SELECT 1 FROM sessions WHERE ${ownProject} AND run_id IS NOT NULL AND run_id NOT IN (SELECT run_id FROM capture_scope) AND session_id IN (SELECT session_id FROM work_sessions WHERE work_id IN (${works})) LIMIT 1`).get())
|
|
26
|
+
throw new AppError("snapshot_scope_conflict", "Selected Work refers to a Session currently bound to another RUN", 1);
|
|
27
|
+
const sessions = `SELECT session_id FROM sessions WHERE ${ownProject} AND (run_id IN (SELECT run_id FROM capture_scope) OR session_id IN (SELECT session_id FROM work_sessions WHERE work_id IN (${works})))`;
|
|
28
|
+
const nativeSessions = `SELECT harness, provider_session_id FROM sessions WHERE ${ownProject} AND session_id IN (${sessions})`;
|
|
29
|
+
const protocols = `SELECT id FROM capture_protocols UNION SELECT subject_id FROM runs WHERE (project_id, id) IN (SELECT * FROM capture_scope) UNION SELECT value FROM merge_requests, json_each(merge_requests.protocol_ids_json) WHERE ${ownRun} UNION SELECT protocol_id FROM flow_jobs WHERE ${ownRun}`;
|
|
30
|
+
const rules = {
|
|
31
|
+
dd_flow_writer_contract: "1",
|
|
32
|
+
dd_flow_writer_migrations: "0",
|
|
33
|
+
projects: "id IN (SELECT project_id FROM capture_scope)",
|
|
34
|
+
runs: "(project_id, id) IN (SELECT * FROM capture_scope)",
|
|
35
|
+
sessions: `${ownProject} AND session_id IN (${sessions})`,
|
|
36
|
+
work_sessions: `work_id IN (${works})`, work_launches: `work_id IN (${works})`,
|
|
37
|
+
work_recovery_segments: `work_id IN (${works})`,
|
|
38
|
+
run_recovery_bindings: `recovery_id IN (SELECT recovery_id FROM run_recovery_guards WHERE ${ownRun})`,
|
|
39
|
+
hook_events: `${ownProject} AND (session_id IN (${sessions}) OR (harness, provider_session_id) IN (${nativeSessions}))`,
|
|
40
|
+
harness_usage_snapshots: `${ownProject} AND (harness, provider_session_id) IN (${nativeSessions})`,
|
|
41
|
+
protocols: `${ownProject} AND id IN (${protocols})`,
|
|
42
|
+
project_config: ownProject,
|
|
43
|
+
// Host installations, caches, queue owners and leases are not portable RUN authority.
|
|
44
|
+
runtime_config: "0", hook_installations: "0", codex_home_profiles: "0", codex_session_bindings: "0",
|
|
45
|
+
merge_servers: "0", merge_sessions: "0", merge_queue: "0", lanes: "0", lane_locks: "0", lane_waiters: "0",
|
|
46
|
+
sqlite_sequence: "0"
|
|
47
|
+
};
|
|
48
|
+
for (const table of ["works", "check_receipts", "merge_requests", "flow_run_flag_mutations", "usage", "flow_session_segments", "flow_jobs", "run_recovery_guards", "run_controllers", "run_controls"])
|
|
49
|
+
rules[table] = ownRun;
|
|
50
|
+
for (const table of ["plan_bindings", "plan_progress", "worktree_records", "audit_events"])
|
|
51
|
+
rules[table] = `${ownProject} AND protocol_id IN (${protocols})`;
|
|
52
|
+
for (const table of ["run_controller_events", "run_controller_operations"])
|
|
53
|
+
rules[table] = `controller_id IN (SELECT controller_id FROM run_controllers WHERE ${ownRun})`;
|
|
54
|
+
for (const table of ["run_control_admissions", "run_control_stage_admissions", "run_control_operations", "run_control_workers"])
|
|
55
|
+
rules[table] = `control_id IN (SELECT control_id FROM run_controls WHERE ${ownRun})`;
|
|
56
|
+
const tables = db.prepare("SELECT name FROM sqlite_schema WHERE type = 'table'").all();
|
|
57
|
+
const quote = (name) => `"${name.replaceAll('"', '""')}"`;
|
|
58
|
+
// Evaluate every relationship before deleting parents, including historical bindings.
|
|
59
|
+
for (const { name } of tables) {
|
|
60
|
+
const predicate = rules[name];
|
|
61
|
+
if (predicate === undefined && db.prepare(`SELECT 1 FROM ${quote(name)} LIMIT 1`).get())
|
|
62
|
+
throw new AppError("snapshot_scope_unknown", "Recovery export has no scope rule for a populated table", 1, { table: name });
|
|
63
|
+
db.exec(`CREATE TEMP TABLE ${quote(`keep_${name}`)} AS SELECT rowid AS retained_id FROM ${quote(name)} WHERE ${predicate ?? "0"}`);
|
|
64
|
+
}
|
|
65
|
+
for (const { name } of tables)
|
|
66
|
+
db.exec(`DELETE FROM ${quote(name)} WHERE rowid NOT IN (SELECT retained_id FROM ${quote(`keep_${name}`)})`);
|
|
67
|
+
const issues = db.prepare("PRAGMA foreign_key_check").all();
|
|
68
|
+
if (issues.length)
|
|
69
|
+
throw new AppError("snapshot_scope_inconsistent", "Scoped recovery export has unresolved relational references", 1, { issues });
|
|
70
|
+
db.exec("COMMIT; VACUUM");
|
|
71
|
+
complete = true;
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
db.close();
|
|
75
|
+
if (!complete)
|
|
76
|
+
fs.rmSync(output, { force: true });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/** Physical leases are evidence, not authority to resurrect on another host. */
|
|
80
|
+
export function readRecoveryResourceEvidence(resourceHome, projectId, runId, authority) {
|
|
81
|
+
const file = path.join(resourceHome, "runtime.sqlite");
|
|
82
|
+
const scope = { schema_id: "dd-flow/recovery-resource-evidence@1", source_home: resourceHome, dd_flow_home: authority.ddFlowHome, project_id: projectId, run_id: runId };
|
|
83
|
+
if (!fs.existsSync(file))
|
|
84
|
+
return { ...scope, database_present: false, processes: [], resources: [] };
|
|
85
|
+
const db = new DatabaseSync(file, { readOnly: true });
|
|
86
|
+
try {
|
|
87
|
+
db.exec("PRAGMA busy_timeout = 4000; BEGIN");
|
|
88
|
+
const candidates = db.prepare("SELECT * FROM managed_processes WHERE project_id = ? AND run_id = ? ORDER BY id").all(projectId, runId).map(record => ({ record, home: managedProcessRuntimeHome(record, authority) }));
|
|
89
|
+
if (candidates.some(item => item.home === null))
|
|
90
|
+
throw new AppError("snapshot_resource_scope_unproven", "A retained process has no proven runtime-home identity", 1, { resource_home: resourceHome });
|
|
91
|
+
const processes = candidates.filter(item => item.home === authority.ddFlowHome).map(item => item.record);
|
|
92
|
+
const ids = new Set(processes.map(record => record.id)), owners = new Set(processes.map(record => record.owner_id));
|
|
93
|
+
const resources = db.prepare("SELECT r.* FROM managed_resources r JOIN managed_processes p ON p.id = r.process_id WHERE p.project_id = ? AND p.run_id = ? ORDER BY r.resource_kind, r.resource_key").all(projectId, runId).filter(row => typeof row.process_id === "string" && ids.has(row.process_id));
|
|
94
|
+
const unbound = db.prepare("SELECT owner_id FROM managed_resources WHERE process_id IS NULL AND owner_id IN (SELECT owner_id FROM managed_processes WHERE project_id = ? AND run_id = ?)").all(projectId, runId).some(row => typeof row.owner_id === "string" && owners.has(row.owner_id));
|
|
95
|
+
if (unbound)
|
|
96
|
+
throw new AppError("snapshot_resource_scope_unproven", "A retained resource has no process binding proving its RUN scope", 1, { resource_home: resourceHome });
|
|
97
|
+
db.exec("COMMIT");
|
|
98
|
+
return { ...scope, database_present: true, processes, resources };
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
db.close();
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
const object = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
3
|
+
const records = (value) => Array.isArray(value) ? value.map(object) : [];
|
|
4
|
+
const text = (value) => typeof value === "string" && value.length ? value : null;
|
|
5
|
+
/** Pure projection of durable evidence. Never infer quiescence from an ACK,
|
|
6
|
+
* a terminal RUN label, or a settlement captured before a subsequent resume. */
|
|
7
|
+
export function controlStatusReceipt(observation, request = {}) {
|
|
8
|
+
const control = object(observation.control), worker = object(observation.worker), resume = object(observation.resume_operation);
|
|
9
|
+
const present = Boolean(control.control_id), current = control.current === true;
|
|
10
|
+
const activeResume = Boolean(resume.operation_id);
|
|
11
|
+
const historical = !current || activeResume || ["resuming", "resumed"].includes(String(control.status));
|
|
12
|
+
const settlement = object(control.settlement);
|
|
13
|
+
const snapshot = historical ? {} : control.admission === "sealed" ? settlement : object(worker.snapshot);
|
|
14
|
+
const nodes = records(snapshot.nodes).map((node, index) => {
|
|
15
|
+
const identity = Object.fromEntries(["operation_id", "controller_id", "process_id", "check_id", "merge_request_id", "work_id", "work_session_id", "session_id", "harness", "state_dir", "resource_home"].filter(key => node[key] !== undefined).map(key => [key, node[key]]));
|
|
16
|
+
const settled = node.status === "completed";
|
|
17
|
+
const role = ["subject", "worker", "merge", "judge", "probe", "check", "process"].includes(String(node.role)) ? node.role : node.check_id ? "check" : node.merge_request_id ? "merge" : node.process_id ? "process" : "unknown";
|
|
18
|
+
return { id: `node:${crypto.createHash("sha256").update(JSON.stringify(Object.keys(identity).length ? identity : { index, reason: node.reason })).digest("hex")}`, role, status: settled ? "completed" : node.status === "outcome_unknown" ? "unknown" : "pending", acknowledged: settled || object(node.receipt).cancelled === true, settled, reason: settled ? null : text(node.reason) ?? "node_settlement_required", details: node };
|
|
19
|
+
});
|
|
20
|
+
const settled = present && current && !historical && control.status === "settled" && control.admission === "sealed" && Boolean(control.capture_path) && settlement.settled === true && Array.isArray(settlement.nodes) && nodes.every(node => node.settled);
|
|
21
|
+
let state;
|
|
22
|
+
if (!present)
|
|
23
|
+
state = request.action && ["done", "failed", "cancelled"].includes(String(observation.run_status)) ? "terminal_noop" : "not_requested";
|
|
24
|
+
else if (!current)
|
|
25
|
+
state = "superseded";
|
|
26
|
+
else if (resume.status === "failed" || worker.status === "failed")
|
|
27
|
+
state = "recovery_required";
|
|
28
|
+
else if (control.status === "resumed" && control.admission === "recovered" && resume.status === "completed")
|
|
29
|
+
state = "running";
|
|
30
|
+
else if (activeResume || control.status === "resuming")
|
|
31
|
+
state = "resuming";
|
|
32
|
+
else if (settled)
|
|
33
|
+
state = control.requested_mode === "pause" ? "paused" : "stopped";
|
|
34
|
+
else
|
|
35
|
+
state = control.requested_mode === "pause" ? "pause_requested" : "stop_requested";
|
|
36
|
+
const reasons = settled ? [] : [...new Set([
|
|
37
|
+
...nodes.filter(node => !node.settled).map(node => node.reason),
|
|
38
|
+
...(Array.isArray(snapshot.pending_reasons) ? snapshot.pending_reasons.filter((reason) => typeof reason === "string" && Boolean(reason)) : []),
|
|
39
|
+
...(!present ? [state === "terminal_noop" ? "terminal_settlement_not_observed" : "control_not_requested"]
|
|
40
|
+
: !current ? ["control_superseded"]
|
|
41
|
+
: state === "running" ? ["run_active"]
|
|
42
|
+
: state === "resuming" ? ["recovery_acceptance_pending"]
|
|
43
|
+
: state === "recovery_required" ? [text(object(resume.error).code) ?? text(object(worker.error).code) ?? "control_recovery_required"]
|
|
44
|
+
: control.admission === "sealed" ? ["recovery_capture_required"] : ["settlement_observation_required"])
|
|
45
|
+
])];
|
|
46
|
+
return {
|
|
47
|
+
...observation, schema_id: "dd-flow/run-control-receipt@1", ok: true,
|
|
48
|
+
request_id: request.requestId ?? text(resume.request_id) ?? text(control.request_id), control_id: text(control.control_id),
|
|
49
|
+
generation: Number.isSafeInteger(control.generation) ? control.generation : 0,
|
|
50
|
+
requested_mode: request.action ?? (activeResume ? "resume" : text(control.requested_mode)),
|
|
51
|
+
applied_mode: settled ? control.requested_mode : state === "running" ? "resume" : null,
|
|
52
|
+
state, settled, nodes, unsettled_reasons: reasons,
|
|
53
|
+
resumability: settled || state === "recovery_required" ? "recovery_required" : ["running", "not_requested", "terminal_noop", "superseded"].includes(state) ? "not_applicable" : "pending",
|
|
54
|
+
recovery_id: text(control.recovery_id)
|
|
55
|
+
};
|
|
56
|
+
}
|