@deksden-com/dd-flow-cli 0.9.0-beta.103 → 0.9.0-beta.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/build-info.json +3 -3
- package/dist/cli/run-cli.js +32 -11
- package/dist/harness-runtime/bin/dd-agy.mjs +1 -1
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +2 -2
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +4 -3
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +2 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +8 -1
- package/dist/harness-runtime/lib/managed-daemon.mjs +11 -6
- package/dist/protocol/local-files.js +18 -9
- package/dist/runtime/context.js +2 -2
- package/dist/services/cleanup.js +81 -32
- package/dist/services/code-checks.js +18 -19
- package/dist/services/continuation-outcome.js +1 -1
- package/dist/services/controller-fanout.js +3 -3
- package/dist/services/eval-snapshots.js +69 -41
- package/dist/services/execution-policy.js +12 -1
- package/dist/services/hooks.js +150 -42
- package/dist/services/lanes.js +23 -18
- package/dist/services/lifecycle-command.js +11 -2
- package/dist/services/lifecycle-invocations.js +291 -54
- package/dist/services/managed-processes.js +35 -23
- package/dist/services/merge-queue.js +71 -67
- package/dist/services/merge-server.js +3 -11
- package/dist/services/migrations.js +28 -12
- package/dist/services/plan-runtime.js +8 -4
- package/dist/services/projects.js +3 -11
- package/dist/services/protocols.js +98 -68
- package/dist/services/recovery-snapshot-database.js +43 -8
- package/dist/services/run-control-worker.js +15 -32
- package/dist/services/run-control.js +3 -17
- package/dist/services/run-controller-capture.js +58 -27
- package/dist/services/run-controller-process.js +21 -30
- package/dist/services/run-controller-recovery.js +36 -37
- package/dist/services/run-controller.js +81 -86
- package/dist/services/run-fork.js +10 -6
- package/dist/services/run-projection.js +19 -64
- package/dist/services/run-recovery-runtime.js +24 -14
- package/dist/services/run-recovery.js +56 -54
- package/dist/services/runs.js +218 -160
- package/dist/services/runtime-budget.js +19 -45
- package/dist/services/runtime-scope-capture.js +8 -6
- package/dist/services/runtime-scope-control.js +49 -39
- package/dist/services/runtime-scope-resume.js +134 -122
- package/dist/services/runtime-scope-worker.js +48 -49
- package/dist/services/scope-writer-cleanup.js +26 -0
- package/dist/services/stage-context.js +6 -1
- package/dist/services/stage-lifecycle.js +1 -1
- package/dist/services/stage-pause.js +13 -29
- package/dist/services/stage-report-renderer.js +116 -6
- package/dist/services/usage.js +2 -9
- package/dist/services/vnext-code-review.js +10 -7
- package/dist/services/vnext-code.js +9 -17
- package/dist/services/vnext-merge.js +64 -21
- package/dist/services/vnext-plan-review.js +43 -30
- package/dist/services/vnext-plan.js +28 -24
- package/dist/services/vnext-protocolize.js +183 -20
- package/dist/services/vnext-specify.js +27 -19
- package/dist/services/work-registry.js +229 -150
- package/dist/storage/database.js +286 -56
- package/dist/storage/foreign-key-transaction.js +39 -0
- package/dist/storage/immutable-input.js +45 -0
- package/dist/storage/import-guard.js +102 -0
- package/dist/storage/native-cleanup.js +52 -0
- package/dist/storage/projection-lock.js +73 -0
- package/dist/storage/writer-contract.js +3 -3
- package/dist/storage/writer-migration.js +24 -41
- package/package.json +1 -1
- package/tools/repair-paused-run-status.mjs +48 -23
|
@@ -4,8 +4,9 @@ import path from "node:path";
|
|
|
4
4
|
import { createContext } from "../runtime/context.js";
|
|
5
5
|
import { AppError } from "../shared/errors.js";
|
|
6
6
|
import { canonicalPath } from "../storage/paths.js";
|
|
7
|
+
import { importGuardPath, rearmCompletedSnapshotImport } from "../storage/import-guard.js";
|
|
7
8
|
import { installCurrentEngine, upgradeForkedRunEngine, prepareForkEngine } from "./engines.js";
|
|
8
|
-
import { restoreEvalRunSnapshot, prepareEvalRunSnapshot, prepareEvalRunRestore } from "./eval-snapshots.js";
|
|
9
|
+
import { restoreEvalRunSnapshot, prepareEvalRunSnapshot, prepareEvalRunRestore, restoredWorkspaceRoot } from "./eval-snapshots.js";
|
|
9
10
|
export function prepareRunFork(input) {
|
|
10
11
|
const requestId = requireId(input.requestId, "request-id");
|
|
11
12
|
const snapshot = canonicalPath(input.snapshot, false);
|
|
@@ -47,7 +48,8 @@ function assertForkOutput(output, intentFile, intent) {
|
|
|
47
48
|
if (fs.existsSync(output) && fs.readdirSync(output).length > 0) {
|
|
48
49
|
if (!fs.existsSync(intentFile) || hash(readJson(intentFile).intent) !== hash(intent))
|
|
49
50
|
throw new AppError("fork_output_not_empty", "Fork output must be empty", 2, { output });
|
|
50
|
-
const
|
|
51
|
+
const guardName = path.basename(importGuardPath(path.join(output, "dd-flow-home")));
|
|
52
|
+
const unknown = fs.readdirSync(output).filter(name => ![".fork-intent.json", "project", "dd-flow-home", guardName].includes(name));
|
|
51
53
|
if (unknown.length)
|
|
52
54
|
throw new AppError("fork_retry_inspect_required", "Fork output contains data not owned by the interrupted fork", 1, { output, unknown });
|
|
53
55
|
}
|
|
@@ -57,16 +59,18 @@ export function forkRunFromSnapshot(context, input, prepared = prepareRunFork(in
|
|
|
57
59
|
return prepared.receipt;
|
|
58
60
|
const { manifest, snapshot, output, intent, intentFile, receiptFile, requestId } = prepared;
|
|
59
61
|
assertForkOutput(output, intentFile, intent);
|
|
62
|
+
const projectRoot = path.join(output, "project");
|
|
63
|
+
const home = path.join(output, "dd-flow-home");
|
|
64
|
+
const prearmedImportId = rearmCompletedSnapshotImport({ home, projectRoot,
|
|
65
|
+
workspaceRoot: restoredWorkspaceRoot(manifest, projectRoot, home), sourceManifestSha256: intent.snapshot_sha256 });
|
|
60
66
|
for (const name of ["project", "dd-flow-home"])
|
|
61
67
|
fs.rmSync(path.join(output, name), { recursive: true, force: true });
|
|
62
68
|
fs.mkdirSync(output, { recursive: true, mode: 0o700 });
|
|
63
69
|
writeJsonAtomic(intentFile, { schema_id: "dd-flow/run-fork-intent@1", intent });
|
|
64
|
-
const projectRoot = path.join(output, "project");
|
|
65
|
-
const home = path.join(output, "dd-flow-home");
|
|
66
70
|
fs.mkdirSync(projectRoot, { recursive: true, mode: 0o700 });
|
|
67
|
-
const targetContext = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_ENGINE_MODE: "1" });
|
|
71
|
+
const targetContext = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_ENGINE_MODE: "1" }, "initialize", prearmedImportId ?? undefined);
|
|
68
72
|
const restoreInput = { snapshot, projectRoot };
|
|
69
|
-
const restored = restoreEvalRunSnapshot(targetContext, restoreInput, prepareEvalRunRestore(targetContext, restoreInput, manifest));
|
|
73
|
+
const restored = restoreEvalRunSnapshot(targetContext, restoreInput, prepareEvalRunRestore(targetContext, restoreInput, manifest), prearmedImportId ?? undefined);
|
|
70
74
|
const restoredContext = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_ENGINE_MODE: "1" });
|
|
71
75
|
try {
|
|
72
76
|
const runHome = String(restored.run_home);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import fs from "node:fs";
|
|
2
1
|
import path from "node:path";
|
|
3
2
|
import { reconcileSessionCoverageRows } from "../domain/session-coverage.js";
|
|
4
3
|
import { publicSessionIdentity } from "./session-identity.js";
|
|
4
|
+
import { publishRunProjections } from "./runs.js";
|
|
5
5
|
export function refreshRunSessionProjection(context, projectId, runId) {
|
|
6
6
|
const run = context.db.get("SELECT id, flow_kind, status, project_root, subject_type, subject_id, runtime_path, run_index_path, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
|
|
7
7
|
if (!run)
|
|
@@ -27,7 +27,7 @@ export function refreshRunSessionProjection(context, projectId, runId) {
|
|
|
27
27
|
missing: coverage.missing_unit_ids,
|
|
28
28
|
diagnostics: coverage.diagnostics
|
|
29
29
|
};
|
|
30
|
-
const plan = run.subject_type === "protocol" ? planProjection(context, projectId, run.subject_id, run.project_root) : {};
|
|
30
|
+
const plan = run.subject_type === "protocol" ? planProjection(context, projectId, run.subject_id, run.project_root, index) : {};
|
|
31
31
|
const workers = workerProjection(context, projectId, runId);
|
|
32
32
|
const vnext = run.flow_kind === "vnext_specify" || run.flow_kind === "vnext_protocolize";
|
|
33
33
|
const hadLegacyCoverage = vnext && ("session_coverage" in index || "usage_coverage" in index);
|
|
@@ -42,8 +42,8 @@ export function refreshRunSessionProjection(context, projectId, runId) {
|
|
|
42
42
|
&& JSON.stringify(index.works ?? []) === JSON.stringify(works)
|
|
43
43
|
&& index.root_work_id === rootWorkId
|
|
44
44
|
&& (vnext || JSON.stringify(index.session_coverage) === JSON.stringify(sessionCoverage))
|
|
45
|
-
&& JSON.stringify(index.plan_ref) === JSON.stringify(plan.plan_ref)
|
|
46
|
-
&& JSON.stringify(index.plan_progress) === JSON.stringify(plan.plan_progress)
|
|
45
|
+
&& (!plan.plan_ref || JSON.stringify(index.plan_ref) === JSON.stringify(plan.plan_ref))
|
|
46
|
+
&& (!plan.plan_progress || JSON.stringify(index.plan_progress) === JSON.stringify(plan.plan_progress))
|
|
47
47
|
&& (vnext || JSON.stringify(index.workers ?? {}) === JSON.stringify(workers))
|
|
48
48
|
&& !hadLegacyCoverage)
|
|
49
49
|
return;
|
|
@@ -59,44 +59,25 @@ export function refreshRunSessionProjection(context, projectId, runId) {
|
|
|
59
59
|
if (!vnext)
|
|
60
60
|
index.workers = workers;
|
|
61
61
|
index.updated_at = context.now();
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
?
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
const authoritative = { ...(runtime ?? {}), ...index, schema_id: String(index.schema_id ?? runtime?.schema_id ?? "dd-flow/flow-run@3") };
|
|
70
|
-
authoritative.sessions = sessions;
|
|
71
|
-
authoritative.works = works;
|
|
72
|
-
authoritative.root_work_id = rootWorkId;
|
|
73
|
-
if (vnext) {
|
|
74
|
-
delete authoritative.session_coverage;
|
|
75
|
-
delete authoritative.usage_coverage;
|
|
76
|
-
delete authoritative.workers;
|
|
77
|
-
delete authoritative.current_stage;
|
|
78
|
-
delete authoritative.attempts;
|
|
79
|
-
}
|
|
80
|
-
else
|
|
81
|
-
authoritative.session_coverage = sessionCoverage;
|
|
82
|
-
if (plan.plan_ref)
|
|
83
|
-
authoritative.plan_ref = plan.plan_ref;
|
|
84
|
-
if (plan.plan_progress)
|
|
85
|
-
authoritative.plan_progress = plan.plan_progress;
|
|
86
|
-
if (!vnext)
|
|
87
|
-
authoritative.workers = workers;
|
|
88
|
-
authoritative.updated_at = index.updated_at;
|
|
89
|
-
authoritative.runtime_revision = runtimeRevision;
|
|
90
|
-
if (run.status !== "discarded" || fs.existsSync(run.runtime_path))
|
|
91
|
-
writeJson(run.runtime_path, authoritative);
|
|
92
|
-
context.db.run("UPDATE runs SET index_json = ?, updated_at = ? WHERE project_id = ? AND id = ?", [JSON.stringify(index), index.updated_at, projectId, runId]);
|
|
62
|
+
index.runtime_revision = typeof index.runtime_revision === "number" ? index.runtime_revision + 1 : 1;
|
|
63
|
+
context.db.writeTransaction(() => {
|
|
64
|
+
const updated = context.db.run("UPDATE runs SET index_json = ?, updated_at = ? WHERE project_id = ? AND id = ? AND index_json = ?", [JSON.stringify(index), index.updated_at, projectId, runId, run.index_json]);
|
|
65
|
+
if (updated.changes !== 1)
|
|
66
|
+
throw new Error("RUN projection changed during preparation");
|
|
67
|
+
context.db.afterCommit(() => publishRunProjections(context, projectId, runId));
|
|
68
|
+
}, "run-projection.refresh");
|
|
93
69
|
}
|
|
94
|
-
function planProjection(context, projectId, protocolId, projectRoot) {
|
|
95
|
-
const binding = context.db.get("SELECT plan_path, plan_revision, plan_sha256 FROM plan_bindings WHERE project_id = ? AND protocol_id = ?", [projectId, protocolId]);
|
|
70
|
+
function planProjection(context, projectId, protocolId, projectRoot, index) {
|
|
71
|
+
const binding = context.db.get("SELECT plan_id, plan_path, plan_revision, plan_sha256 FROM plan_bindings WHERE project_id = ? AND protocol_id = ?", [projectId, protocolId]);
|
|
96
72
|
if (!binding)
|
|
97
73
|
return {};
|
|
98
74
|
const planPath = path.relative(projectRoot, binding.plan_path).split(path.sep).join("/");
|
|
99
|
-
const
|
|
75
|
+
const priorRef = index.plan_ref && typeof index.plan_ref === "object" && !Array.isArray(index.plan_ref) ? index.plan_ref : {};
|
|
76
|
+
const planId = binding.plan_id ?? (typeof priorRef.plan_id === "string" ? priorRef.plan_id : null);
|
|
77
|
+
// A pre-migration binding without a RUN plan_ref needs canonical rebind before
|
|
78
|
+
// this projection can be updated; never invent a plan identifier.
|
|
79
|
+
if (!planId)
|
|
80
|
+
return {};
|
|
100
81
|
const progress = Object.fromEntries(context.db.all("SELECT item_id, status, summary, evidence_json FROM plan_progress WHERE project_id = ? AND protocol_id = ? ORDER BY item_id", [projectId, protocolId]).map((row) => [row.item_id, {
|
|
101
82
|
status: row.status,
|
|
102
83
|
...(row.summary ? { summary: row.summary } : {}),
|
|
@@ -112,18 +93,6 @@ function planProjection(context, projectId, protocolId, projectRoot) {
|
|
|
112
93
|
plan_progress: progress
|
|
113
94
|
};
|
|
114
95
|
}
|
|
115
|
-
function readPlanId(planPath) {
|
|
116
|
-
try {
|
|
117
|
-
const value = JSON.parse(fs.readFileSync(planPath, "utf8"));
|
|
118
|
-
if (value && typeof value === "object" && !Array.isArray(value) && typeof value.plan_id === "string") {
|
|
119
|
-
return value.plan_id;
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
catch {
|
|
123
|
-
// The canonical validator owns malformed-plan errors; projection stays bounded.
|
|
124
|
-
}
|
|
125
|
-
return "unknown-plan";
|
|
126
|
-
}
|
|
127
96
|
function workerProjection(context, projectId, runId) {
|
|
128
97
|
return Object.fromEntries(context.db.all(`SELECT j.job_id, j.plan_item_id, j.status, j.worker_session_id, s.harness, s.provider_session_id
|
|
129
98
|
FROM flow_jobs j LEFT JOIN sessions s ON s.project_id = j.project_id AND s.session_id = j.worker_session_id
|
|
@@ -182,17 +151,3 @@ function parseCoverageUnits(value) {
|
|
|
182
151
|
return [];
|
|
183
152
|
}
|
|
184
153
|
}
|
|
185
|
-
function readJson(file) {
|
|
186
|
-
try {
|
|
187
|
-
return fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : undefined;
|
|
188
|
-
}
|
|
189
|
-
catch {
|
|
190
|
-
return undefined;
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
function writeJson(file, value) {
|
|
194
|
-
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
195
|
-
const temporary = `${file}.tmp-${process.pid}-${Date.now()}`;
|
|
196
|
-
fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`);
|
|
197
|
-
fs.renameSync(temporary, file);
|
|
198
|
-
}
|
|
@@ -4,7 +4,8 @@ import { AppError } from "../shared/errors.js";
|
|
|
4
4
|
import { resolveProjectRoot } from "../storage/paths.js";
|
|
5
5
|
import { requireProjectByRoot } from "./projects.js";
|
|
6
6
|
import { acceptanceCommand, recoveryGuard, recoveryPacketReceipt, requireRecoveryRun } from "./run-recovery.js";
|
|
7
|
-
import { processIsAlive } from "./managed-processes.js";
|
|
7
|
+
import { managedProcessBinding, processIsAlive, resourceHome } from "./managed-processes.js";
|
|
8
|
+
import { getResourceDatabase } from "../storage/database.js";
|
|
8
9
|
import { requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
9
10
|
/** Pin retained Session loading and its first acknowledgement Turn, never a general fence bypass. */
|
|
10
11
|
export function prepareRunRecoveryDispatch(context, input) {
|
|
@@ -12,8 +13,7 @@ export function prepareRunRecoveryDispatch(context, input) {
|
|
|
12
13
|
throw new AppError("usage", "Recovery dispatch requires an operation id and absolute state directory", 2);
|
|
13
14
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
14
15
|
const run = requireRecoveryRun(context, project.id, input.runId);
|
|
15
|
-
context.db.
|
|
16
|
-
try {
|
|
16
|
+
const prepared = context.db.readSnapshot(() => {
|
|
17
17
|
const guard = recoveryGuard(context, project.id, run.id);
|
|
18
18
|
const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ?", [input.recoveryId]);
|
|
19
19
|
if (!guard || guard.recovery_id !== input.recoveryId || !binding || !["resuming", "recovered"].includes(guard.status))
|
|
@@ -23,8 +23,7 @@ export function prepareRunRecoveryDispatch(context, input) {
|
|
|
23
23
|
const prior = JSON.parse(binding.runtime_dispatch_json);
|
|
24
24
|
if (prior.state_dir !== input.stateDir || prior.operation_id !== input.operationId)
|
|
25
25
|
throw new AppError("recovery_binding_mismatch", "Recovery dispatch is already pinned to another operation", 1);
|
|
26
|
-
|
|
27
|
-
return prior;
|
|
26
|
+
return { dispatch: prior, guard, binding };
|
|
28
27
|
}
|
|
29
28
|
if (guard.status !== "resuming" || binding.status !== "prepared")
|
|
30
29
|
throw new AppError("recovery_state_changed", "Recovery acknowledgement is no longer awaiting dispatch", 1);
|
|
@@ -34,16 +33,27 @@ export function prepareRunRecoveryDispatch(context, input) {
|
|
|
34
33
|
const metadata = JSON.parse(record.metadata_json);
|
|
35
34
|
if (metadata.state_dir !== input.stateDir || metadata.run_generation !== guard.generation || metadata.dispatch_barrier !== "run-generation@1" || !["subject", "merge"].includes(metadata.role ?? ""))
|
|
36
35
|
throw new AppError("recovery_runtime_mismatch", "Replacement daemon lacks this recovery generation's native dispatch barrier", 1);
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
catch (error) {
|
|
44
|
-
context.db.exec("ROLLBACK");
|
|
45
|
-
throw error;
|
|
36
|
+
return { guard, binding, record };
|
|
37
|
+
});
|
|
38
|
+
let dispatch = prepared.dispatch;
|
|
39
|
+
if (!dispatch) {
|
|
40
|
+
const prompt = `Resume this retained RUN in its existing native Session. Your first tool call must be this exact standalone command:\n\n${acceptanceCommand(context, run.id, prepared.guard.recovery_id, run.project_root)}\n\nFollow the verified recovery packet returned by that command. Preserve completed Work and existing HITL; do not start a successor Stage yourself. Stop at the Stage or declared Work-graph boundary.\n`;
|
|
41
|
+
dispatch = { process_id: prepared.record.id, owner_id: prepared.record.owner_id, state_dir: input.stateDir, daemon_id: prepared.binding.daemon_id, operation_id: input.operationId, resume_operation_id: `${input.operationId}:load`, prompt_sha256: crypto.createHash("sha256").update(prompt).digest("hex"), prompt };
|
|
46
42
|
}
|
|
43
|
+
const registry = prepared.record ? getResourceDatabase(resourceHome(context)) : undefined;
|
|
44
|
+
return context.db.writeTransaction(() => {
|
|
45
|
+
const guard = recoveryGuard(context, project.id, run.id);
|
|
46
|
+
const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ?", [input.recoveryId]);
|
|
47
|
+
if (JSON.stringify(guard) !== JSON.stringify(prepared.guard) || JSON.stringify(binding) !== JSON.stringify(prepared.binding))
|
|
48
|
+
throw new AppError("recovery_state_changed", "Recovery generation changed while dispatch was prepared", 1);
|
|
49
|
+
if (prepared.record) {
|
|
50
|
+
const current = registry.get("SELECT * FROM managed_processes WHERE id = ?", [prepared.record.id]);
|
|
51
|
+
if (!current || current.state !== "running" || current.lease_expires_at <= context.now() || managedProcessBinding(current, resourceHome(context)).registration_sha256 !== managedProcessBinding(prepared.record, resourceHome(context)).registration_sha256)
|
|
52
|
+
throw new AppError("recovery_runtime_mismatch", "Recovery replacement daemon lease changed before dispatch publication", 1);
|
|
53
|
+
context.db.run("UPDATE run_recovery_bindings SET runtime_dispatch_json = ? WHERE recovery_id = ? AND status = 'prepared' AND runtime_dispatch_json IS NULL", [JSON.stringify(dispatch), prepared.binding.recovery_id]);
|
|
54
|
+
}
|
|
55
|
+
return dispatch;
|
|
56
|
+
});
|
|
47
57
|
}
|
|
48
58
|
/** The caller has already checked PID/start/lease and runtime-home identity. */
|
|
49
59
|
export function recoveryRuntimeAdmission(context, record, input) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { managedLifecycleCommand, recoverLifecycleCommands } from "./lifecycle-invocations.js";
|
|
1
|
+
import { hasLifecyclePacketAssignments, managedLifecycleCommand, recoverLifecycleCommands } from "./lifecycle-invocations.js";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
@@ -91,15 +91,18 @@ export function assertWorkSettlementAllowed(context, projectId, runId, workId) {
|
|
|
91
91
|
assertRunMutationAllowed(context, projectId, runId);
|
|
92
92
|
}
|
|
93
93
|
/** Completion rights are pinned to the Stage attempt present at the pause fence. */
|
|
94
|
-
export function assertStageSettlementAllowed(context, projectId, runId, stage) {
|
|
94
|
+
export function assertStageSettlementAllowed(context, projectId, runId, stage, expectedGeneration) {
|
|
95
95
|
const guard = recoveryGuard(context, projectId, runId);
|
|
96
|
+
const generation = guard?.generation ?? 0;
|
|
97
|
+
if (expectedGeneration !== undefined && generation !== expectedGeneration)
|
|
98
|
+
throw new AppError("run_recovery_generation_stale", "RUN execution generation changed while the operation was in flight", 1, { run_id: runId, expected_generation: expectedGeneration, generation });
|
|
96
99
|
if (guard?.status === "draining" && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_control_stage_admissions'")) {
|
|
97
100
|
const admitted = context.db.get(`SELECT a.work_session_id FROM run_control_stage_admissions a JOIN run_controls c ON c.control_id = a.control_id JOIN run_control_admissions w ON w.control_id = a.control_id AND w.work_session_id = a.work_session_id JOIN work_sessions ws ON ws.id = w.work_session_id JOIN runs r ON r.project_id = c.project_id AND r.id = c.run_id, json_each(r.index_json, '$.stage_runs') live
|
|
98
101
|
WHERE c.project_id = ? AND c.run_id = ? AND c.recovery_id = ? AND c.action = 'pause' AND c.status = 'requested' AND a.stage = ? AND json_extract(live.value, '$.stage') = a.stage AND COALESCE(json_extract(live.value, '$.attempt'), 'try-001') = a.attempt AND ws.work_id = w.work_id AND ws.session_id = w.session_id AND NOT EXISTS (SELECT 1 FROM work_sessions newer WHERE newer.work_id = w.work_id AND newer.status = 'running' AND newer.id <> ws.id) LIMIT 1`, [projectId, runId, guard.recovery_id, stage]);
|
|
99
102
|
if (admitted)
|
|
100
|
-
return;
|
|
103
|
+
return generation;
|
|
101
104
|
}
|
|
102
|
-
assertRunMutationAllowed(context, projectId, runId);
|
|
105
|
+
return assertRunMutationAllowed(context, projectId, runId, expectedGeneration);
|
|
103
106
|
}
|
|
104
107
|
/** A recovered RUN never grants the retired process a new execution segment. */
|
|
105
108
|
export function assertRecoveryHookCurrent(context, projectId, runId, identity) {
|
|
@@ -117,29 +120,25 @@ export function beginRunRecovery(context, input, interruption = prepareRecoveryP
|
|
|
117
120
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
118
121
|
const run = requireRecoveryRun(context, project.id, input.runId);
|
|
119
122
|
const recoveryId = `RCV-${crypto.randomUUID()}`, now = context.now();
|
|
120
|
-
|
|
121
|
-
context.db.beginWriteTransaction();
|
|
122
|
-
try {
|
|
123
|
+
const accepted = context.db.writeTransaction(() => {
|
|
123
124
|
const prior = recoveryGuard(context, project.id, run.id);
|
|
124
125
|
const supersedes = prior?.status === "resuming" && interruption.recovery_parent_id === prior.recovery_id;
|
|
125
126
|
if (prior && ["draining", "sealed", "resuming"].includes(prior.status) && !supersedes) {
|
|
126
|
-
context.db.exec("COMMIT");
|
|
127
127
|
return { ok: true, recovery: { ...prior, interruption: JSON.parse(prior.interruption_json) }, reused: true };
|
|
128
128
|
}
|
|
129
|
-
generation = (prior?.generation ?? 0) + 1;
|
|
129
|
+
const generation = (prior?.generation ?? 0) + 1;
|
|
130
130
|
if (supersedes) {
|
|
131
131
|
context.db.run("UPDATE run_recovery_guards SET status = 'abandoned', updated_at = ? WHERE recovery_id = ?", [now, prior.recovery_id]);
|
|
132
132
|
context.db.run("UPDATE run_recovery_bindings SET status = 'abandoned' WHERE recovery_id = ? AND status = 'prepared'", [prior.recovery_id]);
|
|
133
133
|
}
|
|
134
134
|
context.db.run("INSERT INTO run_recovery_guards (recovery_id, project_id, run_id, generation, status, interruption_json, settlement_json, capture_path, created_at, updated_at, sealed_at, resumed_at) VALUES (?, ?, ?, ?, 'draining', ?, NULL, NULL, ?, ?, NULL, NULL)", [recoveryId, project.id, run.id, generation, JSON.stringify(interruption), now, now]);
|
|
135
135
|
setFlowRunRecoveryPaused(context, project.id, run.id, true);
|
|
136
|
-
context.
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
appendFlowRunTimelineEvent(context, project.id, run.id, { type: "run_recovery_draining", recovery_id: recoveryId, generation, interruption });
|
|
136
|
+
appendFlowRunTimelineEvent(context, project.id, run.id, { type: "run_recovery_draining", recovery_id: recoveryId, generation, interruption });
|
|
137
|
+
return { ok: true, recovery: { recovery_id: recoveryId, run_id: run.id, generation, status: "draining", interruption }, reused: false };
|
|
138
|
+
});
|
|
139
|
+
if (accepted.reused)
|
|
140
|
+
return accepted;
|
|
141
|
+
const generation = accepted.recovery.generation;
|
|
143
142
|
return { ok: true, recovery: { recovery_id: recoveryId, run_id: run.id, generation, status: "draining", interruption } };
|
|
144
143
|
}
|
|
145
144
|
export function sealRunRecovery(context, input, settlement = prepareRecoveryPayload(input.settlement, "settlement")) {
|
|
@@ -154,18 +153,12 @@ export function sealRunRecovery(context, input, settlement = prepareRecoveryPayl
|
|
|
154
153
|
throw new AppError("recovery_invalid_state", "Recovery is not draining", 1, { recovery_id: guard.recovery_id, status: guard.status });
|
|
155
154
|
materializePendingWorkResults(context, project.id, run.id);
|
|
156
155
|
const now = context.now();
|
|
157
|
-
context.db.
|
|
158
|
-
try {
|
|
156
|
+
context.db.writeTransaction(() => {
|
|
159
157
|
const updated = context.db.run("UPDATE run_recovery_guards SET status = 'sealed', settlement_json = ?, sealed_at = ?, updated_at = ? WHERE recovery_id = ? AND status = 'draining' AND generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = ? AND run_id = ?)", [JSON.stringify(settlement), now, now, guard.recovery_id, project.id, run.id]);
|
|
160
158
|
if (updated.changes !== 1)
|
|
161
159
|
throw new AppError("recovery_state_changed", "Recovery state changed before it could be sealed", 1, { recovery_id: guard.recovery_id });
|
|
162
160
|
setFlowRunRecoveryPaused(context, project.id, run.id, true);
|
|
163
|
-
|
|
164
|
-
}
|
|
165
|
-
catch (error) {
|
|
166
|
-
context.db.exec("ROLLBACK TO seal_run_status; RELEASE seal_run_status");
|
|
167
|
-
throw error;
|
|
168
|
-
}
|
|
161
|
+
});
|
|
169
162
|
appendFlowRunTimelineEvent(context, project.id, run.id, { type: "run_recovery_sealed", recovery_id: guard.recovery_id, generation: guard.generation, settlement });
|
|
170
163
|
return { ok: true, recovery: { recovery_id: guard.recovery_id, run_id: run.id, generation: guard.generation, status: "sealed", settlement } };
|
|
171
164
|
}
|
|
@@ -222,10 +215,18 @@ export function resumeRunRecovery(context, input, prepared = prepareRunRecoveryR
|
|
|
222
215
|
const acceptCommand = acceptanceCommand(context, run.id, guard.recovery_id, run.project_root);
|
|
223
216
|
if (prepared.reused)
|
|
224
217
|
return { ok: true, recovery_id: guard.recovery_id, generation: guard.generation, resumed: guard.status === "recovered", awaiting_acceptance: guard.status === "resuming", accept_command: acceptCommand, reused: true };
|
|
225
|
-
const
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
218
|
+
const authority = () => JSON.stringify([
|
|
219
|
+
recoveryGuard(context, project.id, run.id), requireRecoveryRun(context, project.id, run.id),
|
|
220
|
+
context.db.all("SELECT * FROM works WHERE project_id = ? AND run_id = ? ORDER BY work_id", [project.id, run.id]),
|
|
221
|
+
context.db.all("SELECT ws.* FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? ORDER BY ws.id", [project.id, run.id]),
|
|
222
|
+
context.db.all("SELECT DISTINCT s.* FROM sessions s JOIN work_sessions ws ON ws.session_id = s.session_id JOIN works w ON w.work_id = ws.work_id AND w.project_id = s.project_id WHERE w.project_id = ? AND w.run_id = ? ORDER BY s.session_id", [project.id, run.id]),
|
|
223
|
+
context.db.all("SELECT DISTINCT d.work_id, d.status, d.result FROM works w, json_each(w.depends_on_json) dependency JOIN works d ON d.work_id = dependency.value WHERE w.project_id = ? AND w.run_id = ? ORDER BY d.work_id", [project.id, run.id]),
|
|
224
|
+
context.db.all("SELECT s.* FROM work_recovery_segments s JOIN works w ON w.work_id = s.work_id WHERE w.project_id = ? AND w.run_id = ? ORDER BY s.recovery_id, s.work_id", [project.id, run.id]),
|
|
225
|
+
context.db.all("SELECT id FROM hook_events WHERE project_id = ? AND daemon_id = ? ORDER BY id", [project.id, input.daemonId]),
|
|
226
|
+
context.db.all("SELECT b.recovery_id FROM run_recovery_bindings b JOIN run_recovery_guards g ON g.recovery_id = b.recovery_id WHERE g.project_id = ? AND b.daemon_id = ? ORDER BY b.recovery_id", [project.id, input.daemonId])
|
|
227
|
+
]);
|
|
228
|
+
const publication = context.db.readSnapshot(() => {
|
|
229
|
+
const signature = authority();
|
|
229
230
|
const current = recoveryGuard(context, project.id, run.id);
|
|
230
231
|
if (!current || current.recovery_id !== guard.recovery_id || current.generation !== guard.generation || current.status !== "sealed" || current.capture_path !== guard.capture_path)
|
|
231
232
|
throw new AppError("recovery_state_changed", "Recovery state changed before its resume transaction", 1, { recovery_id: guard.recovery_id });
|
|
@@ -249,23 +250,26 @@ export function resumeRunRecovery(context, input, prepared = prepareRunRecoveryR
|
|
|
249
250
|
if (context.db.get("SELECT id FROM hook_events WHERE project_id = ? AND daemon_id = ? LIMIT 1", [project.id, input.daemonId]) || context.db.get("SELECT b.recovery_id FROM run_recovery_bindings b JOIN run_recovery_guards g ON g.recovery_id = b.recovery_id WHERE g.project_id = ? AND b.daemon_id = ? LIMIT 1", [project.id, input.daemonId]))
|
|
250
251
|
throw new AppError("recovery_daemon_stale", "Replacement daemon already has historical lifecycle authority in this project", 1);
|
|
251
252
|
const packet = rootRecoveryPacket(context, run, guard, owner);
|
|
253
|
+
const running = context.db.all("SELECT work_id, started_at, status FROM works WHERE project_id = ? AND run_id = ? AND work_id <> ? AND parent_work_id IS NOT NULL AND (status = 'running' OR (status = 'created' AND EXISTS (SELECT 1 FROM work_recovery_segments s WHERE s.work_id = works.work_id AND s.status = 'prepared'))) ORDER BY work_id", [project.id, run.id, owner.work_id]);
|
|
254
|
+
const evidence = new Map(running.map(work => [work.work_id, recoveryEvidencePath(context, run, guard, work.work_id)]));
|
|
255
|
+
return { signature, owner, packet, running, evidence };
|
|
256
|
+
});
|
|
257
|
+
const { owner, packet, running } = publication;
|
|
258
|
+
context.db.writeTransaction(() => {
|
|
259
|
+
if (authority() !== publication.signature)
|
|
260
|
+
throw new AppError("recovery_state_changed", "Recovery inputs changed while its packet was prepared", 1, { recovery_id: guard.recovery_id });
|
|
261
|
+
const now = context.now();
|
|
252
262
|
context.db.run("INSERT INTO run_recovery_bindings (recovery_id, work_id, prior_work_session_id, session_id, harness, provider_session_id, daemon_id, status, created_at, packet_json, packet_sha256) VALUES (?, ?, ?, ?, ?, ?, ?, 'prepared', ?, ?, ?)", [guard.recovery_id, owner.work_id, owner.id, owner.session_id, input.harness, input.nativeSessionId, input.daemonId, now, packet, crypto.createHash("sha256").update(packet).digest("hex")]);
|
|
253
|
-
running = context.db.all("SELECT work_id, started_at, status FROM works WHERE project_id = ? AND run_id = ? AND work_id <> ? AND parent_work_id IS NOT NULL AND (status = 'running' OR (status = 'created' AND EXISTS (SELECT 1 FROM work_recovery_segments s WHERE s.work_id = works.work_id AND s.status = 'prepared'))) ORDER BY work_id", [project.id, run.id, owner.work_id]);
|
|
254
263
|
for (const work of running)
|
|
255
264
|
context.db.run("UPDATE work_recovery_segments SET status = 'superseded', updated_at = ? WHERE work_id = ? AND status = 'prepared'", [now, work.work_id]);
|
|
256
265
|
for (const work of running) {
|
|
257
|
-
const evidence =
|
|
266
|
+
const evidence = publication.evidence.get(work.work_id);
|
|
258
267
|
context.db.run("INSERT INTO work_recovery_segments (recovery_id, work_id, generation, prior_started_at, evidence_path, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'prepared', ?, ?)", [guard.recovery_id, work.work_id, guard.generation, work.started_at, evidence, now, now]);
|
|
259
268
|
context.db.run("UPDATE work_sessions SET status = 'interrupted', completed_at = ?, updated_at = ? WHERE work_id = ? AND status = 'running'", [now, now, work.work_id]);
|
|
260
269
|
context.db.run("UPDATE works SET status = 'created', result = NULL, completed_at = NULL, updated_at = ? WHERE work_id = ? AND status = 'running'", [now, work.work_id]);
|
|
261
270
|
}
|
|
262
271
|
context.db.run("UPDATE run_recovery_guards SET status = 'resuming', updated_at = ? WHERE recovery_id = ? AND status = 'sealed'", [now, guard.recovery_id]);
|
|
263
|
-
|
|
264
|
-
}
|
|
265
|
-
catch (error) {
|
|
266
|
-
context.db.exec("ROLLBACK");
|
|
267
|
-
throw error;
|
|
268
|
-
}
|
|
272
|
+
});
|
|
269
273
|
appendFlowRunTimelineEvent(context, project.id, run.id, { type: "run_recovery_prepared", recovery_id: guard.recovery_id, generation: guard.generation, daemon_id: input.daemonId, interrupted_works: running.map((work) => work.work_id) });
|
|
270
274
|
return { ok: true, recovery_id: guard.recovery_id, generation: guard.generation, resumed: false, awaiting_acceptance: true, accept_command: acceptCommand, interrupted_works: running.map((work) => work.work_id) };
|
|
271
275
|
}
|
|
@@ -276,28 +280,28 @@ export function acceptRunRecovery(context, input) {
|
|
|
276
280
|
const eventKey = input.hookEventId ?? findRecentMatchingHookEvent(context, { projectId: project.id, matchKey, errorCode: "trusted_recovery_acceptance_required", operation: "run recovery accept" }).eventKey;
|
|
277
281
|
const identity = hookSessionIdentity(context, project.id, eventKey);
|
|
278
282
|
const now = context.now();
|
|
279
|
-
|
|
280
|
-
context.db.beginWriteTransaction();
|
|
281
|
-
try {
|
|
283
|
+
const result = context.db.writeTransaction(() => {
|
|
282
284
|
const guard = recoveryGuard(context, project.id, run.id);
|
|
283
285
|
const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ?", [input.recoveryId]);
|
|
284
286
|
const hook = context.db.get("SELECT status, match_key FROM hook_events WHERE id = ?", [identity.hookEventId]);
|
|
285
287
|
if (!guard || guard.recovery_id !== input.recoveryId || !binding || binding.session_id !== identity.sessionId || binding.harness !== identity.harness || binding.provider_session_id !== identity.providerSessionId || binding.daemon_id !== identity.daemonId || !hook?.match_key || !matchKey.includes(hook.match_key))
|
|
286
288
|
throw new AppError("recovery_binding_mismatch", "Recovery acknowledgement does not belong to the authorized physical Session and generation", 1);
|
|
287
|
-
packetReceipt = recoveryPacketReceipt(binding);
|
|
288
|
-
|
|
289
|
-
|
|
289
|
+
const packetReceipt = recoveryPacketReceipt(binding);
|
|
290
|
+
const retained = packetReceipt.packet;
|
|
291
|
+
if (binding.harness === "zcode-acp" || (binding.harness === "codex-desktop" && hasLifecyclePacketAssignments(context, retained.original_prompt.text))) {
|
|
290
292
|
const scope = { projectRoot: run.project_root, daemonId: binding.daemon_id, rootSessionId: binding.provider_session_id, runId: run.id, generation: guard.generation };
|
|
293
|
+
const assignments = new Set();
|
|
291
294
|
packetReceipt.lifecycle_recovery = {
|
|
292
|
-
commands: retained.continuation === "stage_boundary" ? [] : recoverLifecycleCommands(context, retained.original_prompt.text, scope),
|
|
293
|
-
instruction: "original_prompt is immutable historical evidence. Do not execute its old
|
|
295
|
+
commands: retained.continuation === "stage_boundary" ? [] : recoverLifecycleCommands({ ...context, captureLifecycleAssignment: id => { assignments.add(id); } }, retained.original_prompt.text, scope),
|
|
296
|
+
instruction: "original_prompt is immutable historical evidence. Do not execute its old command authority. Use only replacement commands below; reconcile unknown outcomes before continuing. At a completed stage boundary return control to the runner.",
|
|
294
297
|
work_list_command: `${flowCommand({ ...context, env: { ...context.env, DD_FLOW_INVOCATION_SCOPE: JSON.stringify(scope) } })} work ls --run ${run.id} --ready --project-root ${JSON.stringify(run.project_root)} --json`
|
|
295
298
|
};
|
|
299
|
+
if (context.captureLifecycleAssignment)
|
|
300
|
+
context.db.afterCommit(() => { for (const id of assignments)
|
|
301
|
+
context.captureLifecycleAssignment(id); });
|
|
296
302
|
}
|
|
297
|
-
if (guard.status === "recovered" && binding.status === "accepted" && binding.hook_event_key === eventKey && hook.status === "claimed")
|
|
298
|
-
context.db.exec("COMMIT");
|
|
303
|
+
if (guard.status === "recovered" && binding.status === "accepted" && binding.hook_event_key === eventKey && hook.status === "claimed")
|
|
299
304
|
return { ok: true, accepted: true, recovery_id: guard.recovery_id, generation: guard.generation, reused: true, ...packetReceipt };
|
|
300
|
-
}
|
|
301
305
|
if (guard.status !== "resuming" || binding.status !== "prepared" || hook.status !== "observed")
|
|
302
306
|
throw new AppError("trusted_recovery_acceptance_required", "Recovery requires a fresh native acknowledgement before work can continue", 1);
|
|
303
307
|
const packet = JSON.parse(binding.packet_json);
|
|
@@ -313,14 +317,12 @@ export function acceptRunRecovery(context, input) {
|
|
|
313
317
|
context.db.run("UPDATE run_recovery_bindings SET status = 'accepted', hook_event_key = ?, accepted_at = ? WHERE recovery_id = ?", [eventKey, now, binding.recovery_id]);
|
|
314
318
|
context.db.run("UPDATE run_recovery_guards SET status = 'recovered', resumed_at = ?, updated_at = ? WHERE recovery_id = ?", [now, now, guard.recovery_id]);
|
|
315
319
|
setFlowRunRecoveryPaused(context, project.id, run.id, false);
|
|
316
|
-
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
throw error;
|
|
321
|
-
}
|
|
320
|
+
return { ok: true, accepted: true, recovery_id: input.recoveryId, ...packetReceipt };
|
|
321
|
+
});
|
|
322
|
+
if ("reused" in result && result.reused)
|
|
323
|
+
return result;
|
|
322
324
|
appendFlowRunTimelineEvent(context, project.id, run.id, { type: "run_recovery_accepted", recovery_id: input.recoveryId, daemon_id: identity.daemonId, hook_event_key: eventKey });
|
|
323
|
-
return
|
|
325
|
+
return result;
|
|
324
326
|
}
|
|
325
327
|
function recoveryEvidencePath(context, run, guard, workId) {
|
|
326
328
|
// The verified capture is already immutable evidence. Referencing it avoids
|