@deksden-com/dd-flow-cli 0.9.0-beta.103 → 0.9.0-beta.105

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/build-info.json +3 -3
  3. package/dist/cli/run-cli.js +32 -11
  4. package/dist/harness-runtime/bin/dd-agy.mjs +1 -1
  5. package/dist/harness-runtime/lib/dd-agy-daemon.mjs +2 -2
  6. package/dist/harness-runtime/lib/dd-grok-daemon.mjs +4 -3
  7. package/dist/harness-runtime/lib/delegation-instructions.d.mts +2 -0
  8. package/dist/harness-runtime/lib/delegation-instructions.mjs +8 -1
  9. package/dist/harness-runtime/lib/managed-daemon.mjs +11 -6
  10. package/dist/protocol/local-files.js +18 -9
  11. package/dist/runtime/context.js +2 -2
  12. package/dist/services/cleanup.js +81 -32
  13. package/dist/services/code-checks.js +18 -19
  14. package/dist/services/continuation-outcome.js +1 -1
  15. package/dist/services/controller-fanout.js +3 -3
  16. package/dist/services/eval-snapshots.js +69 -41
  17. package/dist/services/execution-policy.js +12 -1
  18. package/dist/services/hooks.js +150 -42
  19. package/dist/services/lanes.js +23 -18
  20. package/dist/services/lifecycle-command.js +11 -2
  21. package/dist/services/lifecycle-invocations.js +291 -54
  22. package/dist/services/managed-processes.js +35 -23
  23. package/dist/services/merge-queue.js +71 -67
  24. package/dist/services/merge-server.js +3 -11
  25. package/dist/services/migrations.js +28 -12
  26. package/dist/services/plan-runtime.js +8 -4
  27. package/dist/services/projects.js +3 -11
  28. package/dist/services/protocols.js +98 -68
  29. package/dist/services/recovery-snapshot-database.js +43 -8
  30. package/dist/services/run-control-worker.js +15 -32
  31. package/dist/services/run-control.js +3 -17
  32. package/dist/services/run-controller-capture.js +58 -27
  33. package/dist/services/run-controller-process.js +21 -30
  34. package/dist/services/run-controller-recovery.js +36 -37
  35. package/dist/services/run-controller.js +81 -86
  36. package/dist/services/run-fork.js +10 -6
  37. package/dist/services/run-projection.js +19 -64
  38. package/dist/services/run-recovery-runtime.js +24 -14
  39. package/dist/services/run-recovery.js +56 -54
  40. package/dist/services/runs.js +218 -160
  41. package/dist/services/runtime-budget.js +19 -45
  42. package/dist/services/runtime-scope-capture.js +8 -6
  43. package/dist/services/runtime-scope-control.js +49 -39
  44. package/dist/services/runtime-scope-resume.js +134 -122
  45. package/dist/services/runtime-scope-worker.js +48 -49
  46. package/dist/services/scope-writer-cleanup.js +26 -0
  47. package/dist/services/stage-context.js +6 -1
  48. package/dist/services/stage-lifecycle.js +1 -1
  49. package/dist/services/stage-pause.js +13 -29
  50. package/dist/services/stage-report-renderer.js +116 -6
  51. package/dist/services/usage.js +2 -9
  52. package/dist/services/vnext-code-review.js +10 -7
  53. package/dist/services/vnext-code.js +9 -17
  54. package/dist/services/vnext-merge.js +64 -21
  55. package/dist/services/vnext-plan-review.js +43 -30
  56. package/dist/services/vnext-plan.js +28 -24
  57. package/dist/services/vnext-protocolize.js +183 -20
  58. package/dist/services/vnext-specify.js +27 -19
  59. package/dist/services/work-registry.js +229 -150
  60. package/dist/storage/database.js +286 -56
  61. package/dist/storage/foreign-key-transaction.js +39 -0
  62. package/dist/storage/immutable-input.js +45 -0
  63. package/dist/storage/import-guard.js +102 -0
  64. package/dist/storage/native-cleanup.js +52 -0
  65. package/dist/storage/projection-lock.js +73 -0
  66. package/dist/storage/writer-contract.js +3 -3
  67. package/dist/storage/writer-migration.js +24 -41
  68. package/package.json +1 -1
  69. package/tools/repair-paused-run-status.mjs +48 -23
@@ -1,5 +1,6 @@
1
1
  import fs from "node:fs";
2
2
  import path from "node:path";
3
+ import crypto from "node:crypto";
3
4
  import { spawnSync } from "node:child_process";
4
5
  import { parse as parseYaml } from "yaml";
5
6
  import { canTransition } from "../domain/contracts.js";
@@ -9,8 +10,9 @@ import { AppError } from "../shared/errors.js";
9
10
  import { prepareJsonFile } from "../cli/input-preparation.js";
10
11
  import { ensureDir, planJsonPath, resolveProjectRoot, resolveProtocolPlanPath, runtimeProtocolDir, runtimeStateJsonPath } from "../storage/paths.js";
11
12
  import { appendAudit, getAuditEvents } from "./audit.js";
13
+ import { withProjectionLock } from "../storage/projection-lock.js";
12
14
  import { requireProjectByRoot } from "./projects.js";
13
- import { ensureRuntimeProtocolFiles, readStateFile, writeState } from "../protocol/local-files.js";
15
+ import { prepareRuntimeProtocolFiles, publishRuntimeProtocolMetadata, readStateFile, writeState } from "../protocol/local-files.js";
14
16
  import { planSummary, planWithProgress, readCanonicalPlan } from "./plan-runtime.js";
15
17
  import { activeCodexSessionBindingsForProject, activeFlowSessionBindingsForProject, codexHomeProfilesForProject, codexHookEventsForProject, hookStatusForProject } from "./hooks.js";
16
18
  import { buildProtocolFlowGuidance } from "./flow-guidance.js";
@@ -30,7 +32,7 @@ export function prepareProtocolRegistration(context, input) {
30
32
  export function registerProtocol(context, input) {
31
33
  const { projectRoot, project, workspacePath, protocolId, runtimeDir } = prepareProtocolRegistration(context, input);
32
34
  const now = context.now();
33
- const state = ensureRuntimeProtocolFiles({
35
+ const initialized = prepareRuntimeProtocolFiles({
34
36
  runtimeDir,
35
37
  protocolId,
36
38
  projectRoot,
@@ -38,25 +40,27 @@ export function registerProtocol(context, input) {
38
40
  workspacePath,
39
41
  now
40
42
  });
41
- const existing = findProtocol(context, protocolId, project.id);
42
- const params = [
43
- protocolId,
44
- input.handshakeId,
45
- project.id,
46
- projectRoot,
47
- state.status,
48
- state.stage,
49
- state.next_action,
50
- JSON.stringify(state.route),
51
- JSON.stringify(state.workspace),
52
- JSON.stringify(state.blockers),
53
- JSON.stringify(state.active_def),
54
- runtimeStateJsonPath(context.ddFlowHome, project.id, protocolId),
55
- planJsonPath(workspacePath ?? projectRoot, protocolId),
56
- existing?.created_at ?? now,
57
- now
58
- ];
59
- context.db.run(`INSERT INTO protocols
43
+ return context.db.writeTransaction(() => {
44
+ const existing = findProtocol(context, protocolId, project.id);
45
+ const state = existing ? latestProtocolProjection(context, project.id, protocolId) ?? initialized : initialized;
46
+ const params = [
47
+ protocolId,
48
+ input.handshakeId,
49
+ project.id,
50
+ projectRoot,
51
+ state.status,
52
+ state.stage,
53
+ state.next_action,
54
+ JSON.stringify(state.route),
55
+ JSON.stringify(state.workspace),
56
+ JSON.stringify(state.blockers),
57
+ JSON.stringify(state.active_def),
58
+ runtimeStateJsonPath(context.ddFlowHome, project.id, protocolId),
59
+ planJsonPath(workspacePath ?? projectRoot, protocolId),
60
+ existing?.created_at ?? now,
61
+ now
62
+ ];
63
+ context.db.run(`INSERT INTO protocols
60
64
  (id, handshake_id, project_id, project_root, status, stage, next_action,
61
65
  route_json, workspace_json, blockers_json, active_def_json, state_path, plan_path,
62
66
  created_at, updated_at)
@@ -75,19 +79,23 @@ export function registerProtocol(context, input) {
75
79
  state_path = excluded.state_path,
76
80
  plan_path = excluded.plan_path,
77
81
  updated_at = excluded.updated_at`, params);
78
- appendAudit(context, {
79
- protocolId,
80
- projectId: project.id,
81
- eventType: existing ? "protocol.registration_refreshed" : "protocol.registered",
82
- payload: {
83
- protocol_id: protocolId,
84
- handshake_id: input.handshakeId,
85
- project_root: projectRoot,
86
- workspace_path: workspacePath ?? null,
87
- runtime_dir: runtimeDir
88
- }
89
- });
90
- return { ok: true, protocol: protocolStatusPayload(requireProtocol(context, protocolId, project.id)), state };
82
+ appendAudit(context, { protocolId, projectId: project.id, eventType: "protocol.state_projection", payload: { state } });
83
+ context.db.afterCommit(() => publishProtocolProjection(context, project.id, protocolId));
84
+ context.db.afterCommit(() => publishRuntimeProtocolMetadata({ runtimeDir, protocolId, handshakeId: input.handshakeId, projectRoot }));
85
+ appendAudit(context, {
86
+ protocolId,
87
+ projectId: project.id,
88
+ eventType: existing ? "protocol.registration_refreshed" : "protocol.registered",
89
+ payload: {
90
+ protocol_id: protocolId,
91
+ handshake_id: input.handshakeId,
92
+ project_root: projectRoot,
93
+ workspace_path: workspacePath ?? null,
94
+ runtime_dir: runtimeDir
95
+ }
96
+ });
97
+ return { ok: true, protocol: protocolStatusPayload(requireProtocol(context, protocolId, project.id)), state };
98
+ }, "protocol.register");
91
99
  }
92
100
  export function getProtocolStatus(context, input) {
93
101
  const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
@@ -438,8 +446,7 @@ export function syncProtocolFromRun(context, input) {
438
446
  next_action: nextActionForSyncedTarget(targetStage),
439
447
  updated_at: now
440
448
  };
441
- context.db.beginWriteTransaction();
442
- try {
449
+ context.db.writeTransaction(() => {
443
450
  persistProtocolState(context, protocol, nextState);
444
451
  if (targetStage === "ready_for_merge") {
445
452
  context.db.run(`INSERT INTO merge_queue
@@ -471,12 +478,7 @@ export function syncProtocolFromRun(context, input) {
471
478
  stage_chain: stageChain(index)
472
479
  }
473
480
  });
474
- context.db.exec("COMMIT");
475
- }
476
- catch (error) {
477
- context.db.exec("ROLLBACK");
478
- throw error;
479
- }
481
+ }, "protocols.syncFromRun");
480
482
  return {
481
483
  ok: true,
482
484
  protocol_id: protocol.id,
@@ -508,8 +510,7 @@ export function cancelProtocol(context, input) {
508
510
  const now = context.now();
509
511
  const worktree = context.db.get("SELECT worktree_path, status FROM worktree_records WHERE project_id = ? AND protocol_id = ?", [protocol.project_id, protocol.id]);
510
512
  const worktreeRemoval = input.worktree === "remove" ? removeProtocolWorktreeBeforeCancel(protocol, state, worktree, input.force) : undefined;
511
- context.db.beginWriteTransaction();
512
- try {
513
+ return context.db.writeTransaction(() => {
513
514
  const nextState = {
514
515
  ...state,
515
516
  stage: "cancelled",
@@ -570,7 +571,6 @@ export function cancelProtocol(context, input) {
570
571
  worktree_outcome: worktreeOutcome
571
572
  }
572
573
  });
573
- context.db.exec("COMMIT");
574
574
  return {
575
575
  ok: true,
576
576
  protocol_id: protocol.id,
@@ -582,11 +582,7 @@ export function cancelProtocol(context, input) {
582
582
  worktree_outcome: worktreeOutcome,
583
583
  worktree: context.db.get("SELECT * FROM worktree_records WHERE project_id = ? AND protocol_id = ?", [protocol.project_id, protocol.id]) ?? null
584
584
  };
585
- }
586
- catch (error) {
587
- context.db.exec("ROLLBACK");
588
- throw error;
589
- }
585
+ }, "protocols.cancel");
590
586
  }
591
587
  export function requireProtocol(context, protocolId, projectId) {
592
588
  const matches = projectId
@@ -761,6 +757,9 @@ export function readProtocolRuntimeState(context, protocol) {
761
757
  const expected = runtimeStateJsonPath(context.ddFlowHome, protocol.project_id, protocol.id);
762
758
  if (protocol.state_path !== expected)
763
759
  throw new AppError("protocol_state_migration_required", "Protocol runtime path requires explicit cleanup scan/apply before use", 1, { protocol_id: protocol.id, path: protocol.state_path, expected_path: expected });
760
+ const accepted = latestProtocolProjection(context, protocol.project_id, protocol.id);
761
+ if (accepted)
762
+ return { state: accepted, diagnostics: [] };
764
763
  return resolveProtocolRuntimeState(context, { ...protocol }, false);
765
764
  }
766
765
  /** Explicit cleanup migration; productive readers accept only the current path. */
@@ -772,6 +771,19 @@ export function repairProtocolRuntimeState(context, protocol) {
772
771
  Object.assign(protocol, stored);
773
772
  return resolveProtocolRuntimeState(context, protocol, true);
774
773
  }
774
+ /** Read-only repair preparation; caller CASes paths, then persists this state. */
775
+ export function prepareProtocolRuntimeRepair(context, protocol) {
776
+ const stored = context.db.get("SELECT * FROM protocols WHERE project_id = ? AND id = ?", [protocol.project_id, protocol.id]);
777
+ if (!stored)
778
+ throw new AppError("not_found", "Protocol is not registered", 1, { protocol_id: protocol.id });
779
+ const state = latestProtocolProjection(context, stored.project_id, stored.id) ?? resolveProtocolRuntimeState(context, { ...stored }, false).state;
780
+ return {
781
+ state,
782
+ statePath: runtimeStateJsonPath(context.ddFlowHome, stored.project_id, stored.id),
783
+ planPath: planJsonPath(stored.project_root, stored.id),
784
+ previousStatePath: stored.state_path
785
+ };
786
+ }
775
787
  function resolveProtocolRuntimeState(context, protocol, repair) {
776
788
  const diagnostics = [];
777
789
  normalizeProtocolPlanPath(context, protocol, diagnostics, repair);
@@ -890,31 +902,49 @@ export function protocolRunDiagnostics(context, protocol, state) {
890
902
  return { latest_run: latestWithStages, diagnostics };
891
903
  }
892
904
  export function persistProtocolState(context, protocol, state) {
893
- ensureDir(path.dirname(protocol.state_path));
894
- writeState(protocol.state_path, state);
895
- context.db.run(`UPDATE protocols SET
905
+ context.db.writeTransaction(() => {
906
+ context.db.run(`UPDATE protocols SET
896
907
  status = ?, stage = ?, next_action = ?, route_json = ?, workspace_json = ?,
897
908
  blockers_json = ?, active_def_json = ?, updated_at = ?
898
909
  WHERE project_id = ? AND id = ?`, [
899
- state.status,
900
- state.stage,
901
- state.next_action,
902
- JSON.stringify(state.route),
903
- JSON.stringify(state.workspace),
904
- JSON.stringify(state.blockers),
905
- JSON.stringify(state.active_def),
906
- state.updated_at,
907
- protocol.project_id,
908
- protocol.id
909
- ]);
910
- if (state.status === "closed" || state.stage === "closed") {
911
- const now = context.now();
912
- context.db.run(`UPDATE sessions
910
+ state.status,
911
+ state.stage,
912
+ state.next_action,
913
+ JSON.stringify(state.route),
914
+ JSON.stringify(state.workspace),
915
+ JSON.stringify(state.blockers),
916
+ JSON.stringify(state.active_def),
917
+ state.updated_at,
918
+ protocol.project_id,
919
+ protocol.id
920
+ ]);
921
+ appendAudit(context, { protocolId: protocol.id, projectId: protocol.project_id, eventType: "protocol.state_projection", payload: { state } });
922
+ if (state.status === "closed" || state.stage === "closed") {
923
+ const now = context.now();
924
+ context.db.run(`UPDATE sessions
913
925
  SET status = 'stopped', stop_reason = 'protocol closed', updated_at = ?, stopped_at = ?
914
926
  WHERE project_id = ?
915
927
  AND protocol_id = ?
916
928
  AND status IN ('pending', 'active', 'waiting_user', 'blocked', 'stopping')`, [now, now, protocol.project_id, protocol.id]);
917
- }
929
+ }
930
+ context.db.afterCommit(() => publishProtocolProjection(context, protocol.project_id, protocol.id));
931
+ }, "protocols.persistProtocolState");
932
+ }
933
+ function latestProtocolProjection(context, projectId, protocolId) {
934
+ const row = context.db.get("SELECT payload_json FROM audit_events WHERE project_id = ? AND protocol_id = ? AND event_type = 'protocol.state_projection' ORDER BY id DESC LIMIT 1", [projectId, protocolId]);
935
+ return row ? JSON.parse(row.payload_json).state : null;
936
+ }
937
+ export function publishProtocolProjection(context, projectId, protocolId) {
938
+ const protocol = requireProtocol(context, protocolId, projectId);
939
+ ensureDir(path.dirname(protocol.state_path));
940
+ withProjectionLock(`${protocol.state_path}.publish-lock`, () => {
941
+ const accepted = latestProtocolProjection(context, projectId, protocolId);
942
+ if (!accepted)
943
+ throw new AppError("runtime_state_missing", "Protocol has no accepted SQL state", 1, { protocol_id: protocolId });
944
+ const temporary = `${protocol.state_path}.tmp-${process.pid}-${crypto.randomUUID()}`;
945
+ fs.writeFileSync(temporary, `${JSON.stringify(accepted, null, 2)}\n`);
946
+ fs.renameSync(temporary, protocol.state_path);
947
+ });
918
948
  }
919
949
  function findProtocol(context, protocolId, projectId) {
920
950
  if (projectId) {
@@ -4,14 +4,16 @@ import path from "node:path";
4
4
  import { AppError } from "../shared/errors.js";
5
5
  import { registerStoreWriter } from "../storage/writer-contract.js";
6
6
  import { managedProcessRuntimeHome } from "./managed-processes.js";
7
+ import { cleanupScopeWriters } from "./scope-writer-cleanup.js";
7
8
  /** Export one RUN's relational evidence, never mutate the live authority. */
8
9
  export function captureRecoveryDatabase(source, output, projectId, runId, protocolIds = []) {
9
10
  source.exec(`VACUUM INTO '${output.replace(/'/g, "''")}'`);
10
11
  const db = new DatabaseSync(output);
11
- let complete = false;
12
+ let complete = false, held = false, failed = false, primary;
12
13
  try {
13
14
  registerStoreWriter(db);
14
15
  db.exec("PRAGMA foreign_keys = OFF; PRAGMA secure_delete = ON; BEGIN IMMEDIATE");
16
+ held = true;
15
17
  db.exec("CREATE TEMP TABLE capture_scope (project_id TEXT, run_id TEXT)");
16
18
  db.prepare("INSERT INTO capture_scope VALUES (?, ?)").run(projectId, runId);
17
19
  db.exec("CREATE TEMP TABLE capture_protocols (id TEXT)");
@@ -40,6 +42,7 @@ export function captureRecoveryDatabase(source, output, projectId, runId, protoc
40
42
  work_recovery_segments: `work_id IN (${works})`,
41
43
  run_recovery_bindings: `recovery_id IN (SELECT recovery_id FROM run_recovery_guards WHERE ${ownRun})`,
42
44
  lifecycle_invocations: invocations,
45
+ lifecycle_packet_assignments: `invocation_id IN (SELECT id FROM lifecycle_invocations WHERE ${invocations})`,
43
46
  hook_events: `${ownProject} AND (session_id IN (${sessions}) OR (harness, provider_session_id) IN (${nativeSessions})${invocationReceipts})`,
44
47
  harness_usage_snapshots: `${ownProject} AND (harness, provider_session_id) IN (${nativeSessions})`,
45
48
  protocols: `${ownProject} AND id IN (${protocols})`,
@@ -71,13 +74,31 @@ export function captureRecoveryDatabase(source, output, projectId, runId, protoc
71
74
  const issues = db.prepare("PRAGMA foreign_key_check").all();
72
75
  if (issues.length)
73
76
  throw new AppError("snapshot_scope_inconsistent", "Scoped recovery export has unresolved relational references", 1, { issues });
74
- db.exec("COMMIT; VACUUM");
77
+ db.exec("COMMIT");
78
+ held = false;
79
+ db.exec("VACUUM");
75
80
  complete = true;
76
81
  }
77
- finally {
78
- db.close();
79
- if (!complete)
82
+ catch (error) {
83
+ primary = error;
84
+ failed = true;
85
+ }
86
+ const cleanup = cleanupScopeWriters([{ name: output, db, rollback: held, close: "always" }]);
87
+ if (!complete) {
88
+ try {
80
89
  fs.rmSync(output, { force: true });
90
+ }
91
+ catch (error) {
92
+ cleanup.push({ target: output, phase: "remove_incomplete_output", message: String(error) });
93
+ }
94
+ }
95
+ if (failed || cleanup.length) {
96
+ if (primary instanceof AppError) {
97
+ if (cleanup.length)
98
+ primary.details.snapshot_cleanup_errors = cleanup;
99
+ throw primary;
100
+ }
101
+ throw new AppError("snapshot_database_failed", primary instanceof Error ? primary.message : "Snapshot database cleanup failed", 1, { ...(failed ? { cause: String(primary) } : {}), snapshot_cleanup_errors: cleanup });
81
102
  }
82
103
  }
83
104
  /** Physical leases are evidence, not authority to resurrect on another host. */
@@ -87,8 +108,10 @@ export function readRecoveryResourceEvidence(resourceHome, projectId, runId, aut
87
108
  if (!fs.existsSync(file))
88
109
  return { ...scope, database_present: false, processes: [], resources: [] };
89
110
  const db = new DatabaseSync(file, { readOnly: true });
111
+ let held = false, failed = false, primary, result;
90
112
  try {
91
113
  db.exec("PRAGMA busy_timeout = 4000; BEGIN");
114
+ held = true;
92
115
  const candidates = db.prepare("SELECT * FROM managed_processes WHERE project_id = ? AND run_id = ? ORDER BY id").all(projectId, runId).map(record => ({ record, home: managedProcessRuntimeHome(record, authority) }));
93
116
  if (candidates.some(item => item.home === null))
94
117
  throw new AppError("snapshot_resource_scope_unproven", "A retained process has no proven runtime-home identity", 1, { resource_home: resourceHome });
@@ -99,9 +122,21 @@ export function readRecoveryResourceEvidence(resourceHome, projectId, runId, aut
99
122
  if (unbound)
100
123
  throw new AppError("snapshot_resource_scope_unproven", "A retained resource has no process binding proving its RUN scope", 1, { resource_home: resourceHome });
101
124
  db.exec("COMMIT");
102
- return { ...scope, database_present: true, processes, resources };
125
+ held = false;
126
+ result = { ...scope, database_present: true, processes, resources };
103
127
  }
104
- finally {
105
- db.close();
128
+ catch (error) {
129
+ primary = error;
130
+ failed = true;
131
+ }
132
+ const cleanup = cleanupScopeWriters([{ name: file, db, rollback: held, close: "always" }]);
133
+ if (failed || cleanup.length) {
134
+ if (primary instanceof AppError) {
135
+ if (cleanup.length)
136
+ primary.details.snapshot_cleanup_errors = cleanup;
137
+ throw primary;
138
+ }
139
+ throw new AppError("snapshot_resource_evidence_failed", primary instanceof Error ? primary.message : "Resource evidence cleanup failed", 1, { ...(failed ? { cause: String(primary) } : {}), snapshot_cleanup_errors: cleanup });
106
140
  }
141
+ return result;
107
142
  }
@@ -56,21 +56,16 @@ async function launchControlWorker(context, input) {
56
56
  const home = resourceHome(context);
57
57
  // Reclaim only a proven dead launcher or admitted worker. Lease expiry
58
58
  // never proves death or authorizes a second physical owner.
59
+ const launcherBirth = processStartedAt(process.pid);
59
60
  let claimed = false;
60
- context.db.beginWriteTransaction();
61
- try {
61
+ context.db.writeTransaction(() => {
62
62
  claimed = context.db.run("INSERT OR IGNORE INTO run_control_workers (control_id, owner_token, status, resource_home, state_dir, created_at, updated_at) VALUES (?, ?, 'starting', ?, ?, ?, ?)", [input.controlId, token, home, directory, now, now]).changes === 1;
63
63
  if (claimed) {
64
- recordControlLauncher(context, input.controlId, token);
64
+ recordControlLauncher(context, input.controlId, token, launcherBirth);
65
65
  if (input.reconcileId)
66
66
  context.db.run("UPDATE run_control_workers SET snapshot_json = ? WHERE control_id = ?", [JSON.stringify(renewRecoveryObservation({}, input.reconcileId)), input.controlId]);
67
67
  }
68
- context.db.exec("COMMIT");
69
- }
70
- catch (error) {
71
- context.db.exec("ROLLBACK");
72
- throw error;
73
- }
68
+ });
74
69
  if (!claimed && !reclaimControlWorker(context, input, project.id, token, home))
75
70
  return;
76
71
  try {
@@ -123,9 +118,9 @@ export async function waitForControlWorkerAdmission(context, controlId, ownerTok
123
118
  }
124
119
  }
125
120
  const launchOperationId = (controlId, token) => `${controlId}:worker-launch:${crypto.createHash("sha256").update(token).digest("hex")}`;
126
- function recordControlLauncher(context, controlId, token) {
121
+ function recordControlLauncher(context, controlId, token, launcherBirth) {
127
122
  const now = context.now();
128
- context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [launchOperationId(controlId, token), controlId, JSON.stringify({ kind: "control_worker_launch_claim" }), JSON.stringify({ pid: process.pid, pid_started_at: processStartedAt(process.pid), registration: "worker_self@1" }), now, now]);
123
+ context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [launchOperationId(controlId, token), controlId, JSON.stringify({ kind: "control_worker_launch_claim" }), JSON.stringify({ pid: process.pid, pid_started_at: launcherBirth, registration: "worker_self@1" }), now, now]);
129
124
  }
130
125
  function reclaimControlWorker(context, input, projectId, token, home) {
131
126
  const previous = context.db.get("SELECT * FROM run_control_workers WHERE control_id = ?", [input.controlId]);
@@ -187,32 +182,25 @@ function reclaimControlWorker(context, input, projectId, token, home) {
187
182
  // and a crash before the RUN CAS leaves the same dead owner reconcilable.
188
183
  if (record && !finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: "control_owner_crash_reconciled" }))
189
184
  return false;
190
- context.db.beginWriteTransaction();
191
- try {
185
+ const launcherBirth = processStartedAt(process.pid);
186
+ return context.db.writeTransaction(() => {
192
187
  const changed = context.db.run(`UPDATE run_control_workers SET owner_token = ?, owner_pid = NULL, status = 'starting', process_id = NULL, process_lease_token = NULL, updated_at = ?
193
188
  WHERE control_id = ? AND owner_token = ? AND process_id IS ? AND owner_pid IS ? AND status = ?
194
189
  AND (SELECT receipt_json FROM run_control_operations WHERE operation_id = ?) IS ?
195
190
  AND EXISTS (SELECT 1 FROM run_controls c JOIN run_recovery_guards g ON g.recovery_id = c.recovery_id
196
191
  WHERE c.control_id = run_control_workers.control_id AND c.status = 'requested' AND g.status IN ('draining','sealed')
197
192
  AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [token, context.now(), input.controlId, previous.owner_token, previous.process_id, previous.owner_pid, previous.status, operationId, launch?.receipt_json ?? null]);
198
- if (changed.changes !== 1) {
199
- context.db.exec("ROLLBACK");
193
+ if (changed.changes !== 1)
200
194
  return false;
201
- }
202
195
  if (input.reconcileId) {
203
196
  const snapshot = renewRecoveryObservation(JSON.parse(previous.snapshot_json ?? "{}"), input.reconcileId);
204
197
  context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = NULL WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), input.controlId, token]);
205
198
  }
206
199
  const now = context.now();
207
200
  context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, previous.owner_token)}:reconciled`, input.controlId, JSON.stringify({ kind: "control_worker_reconciliation", process_id: record?.id ?? null, resource_home: home }), JSON.stringify({ process_id: record?.id ?? null, pid: record?.pid ?? null, pid_started_at: record?.pid_started_at ?? null, prior_status: previous.status, snapshot: JSON.parse(previous.snapshot_json ?? "null"), error: JSON.parse(previous.error_json ?? "null") }), now, now]);
208
- recordControlLauncher(context, input.controlId, token);
209
- context.db.exec("COMMIT");
201
+ recordControlLauncher(context, input.controlId, token, launcherBirth);
210
202
  return true;
211
- }
212
- catch (error) {
213
- context.db.exec("ROLLBACK");
214
- throw error;
215
- }
203
+ });
216
204
  }
217
205
  export async function serveRunControl(context, input) {
218
206
  const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
@@ -226,19 +214,14 @@ export async function serveRunControl(context, input) {
226
214
  throw new AppError("run_control_owner_mismatch", "Control registration is not authorized", 1);
227
215
  // Physical identity is committed with the sole RUN claim, before the worker
228
216
  // registers itself in the separate resource store or performs any dispatch.
229
- context.db.beginWriteTransaction();
230
- try {
217
+ const workerBirth = processStartedAt(process.pid);
218
+ context.db.writeTransaction(() => {
231
219
  const claimed = context.db.run("UPDATE run_control_workers SET status = 'running', owner_pid = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [process.pid, context.now(), input.controlId, worker.owner_token]);
232
220
  if (claimed.changes !== 1)
233
221
  throw new AppError("run_control_already_owned", "A control owner has already claimed this request", 1);
234
222
  const now = context.now();
235
- context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, worker.owner_token)}:worker-claim`, input.controlId, JSON.stringify({ kind: "control_worker_self_claim" }), JSON.stringify({ pid: process.pid, pid_started_at: processStartedAt(process.pid) }), now, now]);
236
- context.db.exec("COMMIT");
237
- }
238
- catch (error) {
239
- context.db.exec("ROLLBACK");
240
- throw error;
241
- }
223
+ context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, worker.owner_token)}:worker-claim`, input.controlId, JSON.stringify({ kind: "control_worker_self_claim" }), JSON.stringify({ pid: process.pid, pid_started_at: workerBirth }), now, now]);
224
+ });
242
225
  const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: worker.resource_home } };
243
226
  let leaseLost = false;
244
227
  const heartbeat = setInterval(() => {
@@ -19,13 +19,7 @@ export function runControlStatus(context, input) {
19
19
  const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
20
20
  // One read snapshot prevents a concurrent stop/resume from mixing generations
21
21
  // across the request, arbiter and worker projections. Savepoints also nest.
22
- context.db.exec("SAVEPOINT run_control_status");
23
- try {
24
- return readControlStatus(context, input, project);
25
- }
26
- finally {
27
- context.db.exec("RELEASE run_control_status");
28
- }
22
+ return context.db.readSnapshot(() => readControlStatus(context, input, project));
29
23
  }
30
24
  function readControlStatus(context, input, project) {
31
25
  const run = context.db.get("SELECT status FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
@@ -822,20 +816,17 @@ export function recordRunControlIntent(context, input) {
822
816
  const request = { project_root: project.root, run_id: input.runId, action: input.action, ...(input.action === "pause" ? { grace_ms: input.graceMs ?? 30_000, escalate: input.escalate ?? null } : { force: input.force ?? false }) };
823
817
  const requestHash = crypto.createHash("sha256").update(JSON.stringify(request)).digest("hex");
824
818
  ensureRunControlStorage(context.db);
825
- context.db.beginWriteTransaction();
826
- try {
819
+ return context.db.writeTransaction(() => {
827
820
  const existing = context.db.get("SELECT * FROM run_controls WHERE project_id = ? AND run_id = ? AND request_id = ?", [project.id, input.runId, input.requestId]);
828
821
  if (existing) {
829
822
  if (existing.request_hash !== requestHash)
830
823
  throw new AppError("runtime_request_conflict", "Control request id has different inputs", 1);
831
- context.db.exec("COMMIT");
832
824
  return { ok: true, control_id: existing.control_id, recovery_id: existing.recovery_id, action: existing.action, status: existing.status, reused: true };
833
825
  }
834
826
  const run = context.db.get("SELECT status FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
835
827
  if (!run)
836
828
  throw new AppError("not_found", "RUN is not registered", 1);
837
829
  if (["done", "failed", "cancelled"].includes(run.status)) {
838
- context.db.exec("COMMIT");
839
830
  return { ok: true, terminal: true, status: run.status, control_id: null };
840
831
  }
841
832
  const prior = recoveryGuard(context, project.id, input.runId);
@@ -866,11 +857,6 @@ export function recordRunControlIntent(context, input) {
866
857
  WHERE r.project_id = ? AND r.id = ? AND json_extract(stage.value, '$.status') = 'running'
867
858
  AND EXISTS (SELECT 1 FROM run_control_admissions a JOIN sessions s ON s.session_id = a.session_id AND s.project_id = r.project_id WHERE a.control_id = ? AND s.current_stage = json_extract(stage.value, '$.stage'))`, [controlId, controlId, project.id, input.runId, controlId]);
868
859
  setFlowRunRecoveryPaused(context, project.id, input.runId, true);
869
- context.db.exec("COMMIT");
870
860
  return { ok: true, control_id: controlId, recovery_id: recoveryId, generation, action: input.action, status: "requested", settled: false };
871
- }
872
- catch (error) {
873
- context.db.exec("ROLLBACK");
874
- throw error;
875
- }
861
+ });
876
862
  }
@@ -8,6 +8,25 @@ import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
8
8
  import { appendEvent, digest, errorRecord, requireController, requireRun } from "./run-controller-state.js";
9
9
  import { getFlowRunStatus } from "./runs.js";
10
10
  import { recoveryObservationBudget, withRecoveryObservation } from "./recovery-observation-budget.js";
11
+ import { getResourceDatabase } from "../storage/database.js";
12
+ import { managedProcessBinding, resourceHome } from "./managed-processes.js";
13
+ /** Physical probes precede reservation; the accepted publisher still checks
14
+ * exact owner identity and fresh lease expiry after waiting for the writer. */
15
+ export function prepareControllerPublication(context, row, assertPhysicalOwner) {
16
+ assertPhysicalOwner();
17
+ const registry = getResourceDatabase(resourceHome(context));
18
+ const owner = registry.get("SELECT * FROM managed_processes WHERE id = ?", [row.process_id]);
19
+ if (!owner || owner.lease_token !== row.process_lease_token || owner.pid !== row.owner_pid || owner.owner_id !== row.controller_id)
20
+ throw new AppError("controller_owner_mismatch", "Controller publication lacks its physical owner", 1, { process_id: row.process_id, controller_id: row.controller_id, owner_found: Boolean(owner), lease_matches: owner?.lease_token === row.process_lease_token, pid_matches: owner?.pid === row.owner_pid, owner_matches: owner?.owner_id === row.controller_id, resource_home: resourceHome(context) });
21
+ const binding = managedProcessBinding(owner, resourceHome(context));
22
+ return () => {
23
+ const current = requireController(context, row.controller_id), process = registry.get("SELECT * FROM managed_processes WHERE id = ?", [owner.id]);
24
+ if (current.owner_token !== row.owner_token || current.owner_pid !== row.owner_pid || current.process_id !== row.process_id || current.process_lease_token !== row.process_lease_token || current.generation !== row.generation
25
+ || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status)
26
+ || !process || process.state !== "running" || process.lease_expires_at <= context.now() || digest(managedProcessBinding(process, resourceHome(context))) !== digest(binding))
27
+ throw new AppError("controller_owner_mismatch", "Controller publication lost its current physical lease", 1);
28
+ };
29
+ }
11
30
  /** Capture is performed by the productive owner, before it releases the next
12
31
  * Stage. An observer never races the controller to copy a completed boundary. */
13
32
  export async function captureControllerBoundary(context, row, manifest, stage, nextStage, assertPhysicalOwner) {
@@ -30,27 +49,29 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
30
49
  return;
31
50
  }
32
51
  catch (error) {
33
- assertPhysicalOwner();
34
- context.db.run("UPDATE run_controllers SET status = 'capture_failed', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
52
+ try {
53
+ assertPhysicalOwner();
54
+ context.db.run("UPDATE run_controllers SET status = 'capture_failed', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
55
+ }
56
+ catch (recordError) {
57
+ if (error instanceof AppError)
58
+ error.details.capture_error_recording = errorRecord(recordError);
59
+ }
35
60
  throw error;
36
61
  }
37
62
  }
38
- context.db.beginWriteTransaction();
39
- try {
40
- assertPhysicalOwner();
63
+ const assertStartOwner = prepareControllerPublication(context, row, assertPhysicalOwner);
64
+ context.db.writeTransaction(() => {
65
+ assertStartOwner();
41
66
  assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
42
67
  const now = context.now();
43
68
  context.db.run("UPDATE run_controllers SET status = 'capturing', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [now, row.controller_id, row.owner_token]);
44
69
  context.db.run("INSERT INTO run_controller_operations (operation_id,controller_id,generation,input_hash,input_json,status,created_at,updated_at) VALUES (?,?,?,?,?,'capturing',?,?) ON CONFLICT(operation_id) DO UPDATE SET generation = excluded.generation, status = 'capturing', error_json = NULL, updated_at = excluded.updated_at", [operationId, row.controller_id, row.generation, boundaryKey, JSON.stringify({ action: "boundary_capture", ...identity(stage), stage_entry: nextStage, output }), now, now]);
45
70
  appendEvent(context, row.controller_id, "boundary_capture_requested", { operation_id: operationId, boundary_key: boundaryKey, stage: stage.stage, stage_entry: nextStage });
46
- context.db.exec("COMMIT");
47
- }
48
- catch (error) {
49
- context.db.exec("ROLLBACK");
50
- throw error;
51
- }
52
- const assertCurrent = () => {
53
- assertPhysicalOwner();
71
+ });
72
+ const assertCurrent = (physical = true) => {
73
+ if (physical)
74
+ assertPhysicalOwner();
54
75
  const guard = recoveryGuard(context, row.project_id, row.run_id);
55
76
  if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
56
77
  throw new AppError("run_recovery_guarded", "Recovery or operator control superseded boundary capture", 1);
@@ -60,9 +81,11 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
60
81
  const current = index.stage_runs.find(item => item.stage === stage.stage);
61
82
  if (!current || !["done", "skipped"].includes(current.status) || digest(identity(current)) !== boundaryKey)
62
83
  throw new AppError("controller_boundary_changed", "Stage identity changed after its capture fence", 1);
63
- const continuation = getFlowRunStatus(context, scope).continuation;
64
- if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal")
65
- throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
84
+ if (physical) {
85
+ const continuation = getFlowRunStatus(context, scope).continuation;
86
+ if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal")
87
+ throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
88
+ }
66
89
  };
67
90
  let priorObservation = null;
68
91
  try {
@@ -114,23 +137,31 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
114
137
  await delay(observationBudget.nextDelay());
115
138
  }
116
139
  }
117
- context.db.beginWriteTransaction();
118
- try {
119
- assertCurrent();
140
+ assertCurrent();
141
+ const assertFinishOwner = prepareControllerPublication(context, row, assertPhysicalOwner);
142
+ const preparedIndex = requireRun(context, row.project_id, row.run_id).index_json;
143
+ context.db.writeTransaction(() => {
144
+ assertFinishOwner();
145
+ assertCurrent(false);
146
+ if (requireRun(context, row.project_id, row.run_id).index_json !== preparedIndex)
147
+ throw new AppError("controller_boundary_changed", "RUN changed before capture publication", 1);
120
148
  context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
121
149
  appendEvent(context, row.controller_id, "boundary_captured", { ...identity(stage), boundary_key: boundaryKey, operation_id: operationId, ...receipt });
122
150
  context.db.run("UPDATE run_controllers SET status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
123
- context.db.exec("COMMIT");
124
- }
125
- catch (error) {
126
- context.db.exec("ROLLBACK");
127
- throw error;
128
- }
151
+ });
129
152
  }
130
153
  catch (error) {
131
154
  observationBudget.exhausted();
132
- context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)) WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), operationId]);
133
- context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
155
+ try {
156
+ context.db.writeTransaction(() => {
157
+ context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)) WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), operationId]);
158
+ context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
159
+ });
160
+ }
161
+ catch (recordError) {
162
+ if (error instanceof AppError)
163
+ error.details.capture_error_recording = errorRecord(recordError);
164
+ }
134
165
  throw error;
135
166
  }
136
167
  }