@deksden-com/dd-flow-cli 0.9.0-beta.103 → 0.9.0-beta.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/build-info.json +3 -3
- package/dist/cli/run-cli.js +32 -11
- package/dist/harness-runtime/bin/dd-agy.mjs +1 -1
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +2 -2
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +4 -3
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +2 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +8 -1
- package/dist/harness-runtime/lib/managed-daemon.mjs +11 -6
- package/dist/protocol/local-files.js +18 -9
- package/dist/runtime/context.js +2 -2
- package/dist/services/cleanup.js +81 -32
- package/dist/services/code-checks.js +18 -19
- package/dist/services/continuation-outcome.js +1 -1
- package/dist/services/controller-fanout.js +3 -3
- package/dist/services/eval-snapshots.js +69 -41
- package/dist/services/execution-policy.js +12 -1
- package/dist/services/hooks.js +150 -42
- package/dist/services/lanes.js +23 -18
- package/dist/services/lifecycle-command.js +11 -2
- package/dist/services/lifecycle-invocations.js +291 -54
- package/dist/services/managed-processes.js +35 -23
- package/dist/services/merge-queue.js +71 -67
- package/dist/services/merge-server.js +3 -11
- package/dist/services/migrations.js +28 -12
- package/dist/services/plan-runtime.js +8 -4
- package/dist/services/projects.js +3 -11
- package/dist/services/protocols.js +98 -68
- package/dist/services/recovery-snapshot-database.js +43 -8
- package/dist/services/run-control-worker.js +15 -32
- package/dist/services/run-control.js +3 -17
- package/dist/services/run-controller-capture.js +58 -27
- package/dist/services/run-controller-process.js +21 -30
- package/dist/services/run-controller-recovery.js +36 -37
- package/dist/services/run-controller.js +81 -86
- package/dist/services/run-fork.js +10 -6
- package/dist/services/run-projection.js +19 -64
- package/dist/services/run-recovery-runtime.js +24 -14
- package/dist/services/run-recovery.js +56 -54
- package/dist/services/runs.js +218 -160
- package/dist/services/runtime-budget.js +19 -45
- package/dist/services/runtime-scope-capture.js +8 -6
- package/dist/services/runtime-scope-control.js +49 -39
- package/dist/services/runtime-scope-resume.js +134 -122
- package/dist/services/runtime-scope-worker.js +48 -49
- package/dist/services/scope-writer-cleanup.js +26 -0
- package/dist/services/stage-context.js +6 -1
- package/dist/services/stage-lifecycle.js +1 -1
- package/dist/services/stage-pause.js +13 -29
- package/dist/services/stage-report-renderer.js +116 -6
- package/dist/services/usage.js +2 -9
- package/dist/services/vnext-code-review.js +10 -7
- package/dist/services/vnext-code.js +9 -17
- package/dist/services/vnext-merge.js +64 -21
- package/dist/services/vnext-plan-review.js +43 -30
- package/dist/services/vnext-plan.js +28 -24
- package/dist/services/vnext-protocolize.js +183 -20
- package/dist/services/vnext-specify.js +27 -19
- package/dist/services/work-registry.js +229 -150
- package/dist/storage/database.js +286 -56
- package/dist/storage/foreign-key-transaction.js +39 -0
- package/dist/storage/immutable-input.js +45 -0
- package/dist/storage/import-guard.js +102 -0
- package/dist/storage/native-cleanup.js +52 -0
- package/dist/storage/projection-lock.js +73 -0
- package/dist/storage/writer-contract.js +3 -3
- package/dist/storage/writer-migration.js +24 -41
- package/package.json +1 -1
- package/tools/repair-paused-run-status.mjs +48 -23
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import crypto from "node:crypto";
|
|
3
4
|
import { spawnSync } from "node:child_process";
|
|
4
5
|
import { parse as parseYaml } from "yaml";
|
|
5
6
|
import { canTransition } from "../domain/contracts.js";
|
|
@@ -9,8 +10,9 @@ import { AppError } from "../shared/errors.js";
|
|
|
9
10
|
import { prepareJsonFile } from "../cli/input-preparation.js";
|
|
10
11
|
import { ensureDir, planJsonPath, resolveProjectRoot, resolveProtocolPlanPath, runtimeProtocolDir, runtimeStateJsonPath } from "../storage/paths.js";
|
|
11
12
|
import { appendAudit, getAuditEvents } from "./audit.js";
|
|
13
|
+
import { withProjectionLock } from "../storage/projection-lock.js";
|
|
12
14
|
import { requireProjectByRoot } from "./projects.js";
|
|
13
|
-
import {
|
|
15
|
+
import { prepareRuntimeProtocolFiles, publishRuntimeProtocolMetadata, readStateFile, writeState } from "../protocol/local-files.js";
|
|
14
16
|
import { planSummary, planWithProgress, readCanonicalPlan } from "./plan-runtime.js";
|
|
15
17
|
import { activeCodexSessionBindingsForProject, activeFlowSessionBindingsForProject, codexHomeProfilesForProject, codexHookEventsForProject, hookStatusForProject } from "./hooks.js";
|
|
16
18
|
import { buildProtocolFlowGuidance } from "./flow-guidance.js";
|
|
@@ -30,7 +32,7 @@ export function prepareProtocolRegistration(context, input) {
|
|
|
30
32
|
export function registerProtocol(context, input) {
|
|
31
33
|
const { projectRoot, project, workspacePath, protocolId, runtimeDir } = prepareProtocolRegistration(context, input);
|
|
32
34
|
const now = context.now();
|
|
33
|
-
const
|
|
35
|
+
const initialized = prepareRuntimeProtocolFiles({
|
|
34
36
|
runtimeDir,
|
|
35
37
|
protocolId,
|
|
36
38
|
projectRoot,
|
|
@@ -38,25 +40,27 @@ export function registerProtocol(context, input) {
|
|
|
38
40
|
workspacePath,
|
|
39
41
|
now
|
|
40
42
|
});
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
protocolId
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
43
|
+
return context.db.writeTransaction(() => {
|
|
44
|
+
const existing = findProtocol(context, protocolId, project.id);
|
|
45
|
+
const state = existing ? latestProtocolProjection(context, project.id, protocolId) ?? initialized : initialized;
|
|
46
|
+
const params = [
|
|
47
|
+
protocolId,
|
|
48
|
+
input.handshakeId,
|
|
49
|
+
project.id,
|
|
50
|
+
projectRoot,
|
|
51
|
+
state.status,
|
|
52
|
+
state.stage,
|
|
53
|
+
state.next_action,
|
|
54
|
+
JSON.stringify(state.route),
|
|
55
|
+
JSON.stringify(state.workspace),
|
|
56
|
+
JSON.stringify(state.blockers),
|
|
57
|
+
JSON.stringify(state.active_def),
|
|
58
|
+
runtimeStateJsonPath(context.ddFlowHome, project.id, protocolId),
|
|
59
|
+
planJsonPath(workspacePath ?? projectRoot, protocolId),
|
|
60
|
+
existing?.created_at ?? now,
|
|
61
|
+
now
|
|
62
|
+
];
|
|
63
|
+
context.db.run(`INSERT INTO protocols
|
|
60
64
|
(id, handshake_id, project_id, project_root, status, stage, next_action,
|
|
61
65
|
route_json, workspace_json, blockers_json, active_def_json, state_path, plan_path,
|
|
62
66
|
created_at, updated_at)
|
|
@@ -75,19 +79,23 @@ export function registerProtocol(context, input) {
|
|
|
75
79
|
state_path = excluded.state_path,
|
|
76
80
|
plan_path = excluded.plan_path,
|
|
77
81
|
updated_at = excluded.updated_at`, params);
|
|
78
|
-
|
|
79
|
-
protocolId
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
82
|
+
appendAudit(context, { protocolId, projectId: project.id, eventType: "protocol.state_projection", payload: { state } });
|
|
83
|
+
context.db.afterCommit(() => publishProtocolProjection(context, project.id, protocolId));
|
|
84
|
+
context.db.afterCommit(() => publishRuntimeProtocolMetadata({ runtimeDir, protocolId, handshakeId: input.handshakeId, projectRoot }));
|
|
85
|
+
appendAudit(context, {
|
|
86
|
+
protocolId,
|
|
87
|
+
projectId: project.id,
|
|
88
|
+
eventType: existing ? "protocol.registration_refreshed" : "protocol.registered",
|
|
89
|
+
payload: {
|
|
90
|
+
protocol_id: protocolId,
|
|
91
|
+
handshake_id: input.handshakeId,
|
|
92
|
+
project_root: projectRoot,
|
|
93
|
+
workspace_path: workspacePath ?? null,
|
|
94
|
+
runtime_dir: runtimeDir
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
return { ok: true, protocol: protocolStatusPayload(requireProtocol(context, protocolId, project.id)), state };
|
|
98
|
+
}, "protocol.register");
|
|
91
99
|
}
|
|
92
100
|
export function getProtocolStatus(context, input) {
|
|
93
101
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
@@ -438,8 +446,7 @@ export function syncProtocolFromRun(context, input) {
|
|
|
438
446
|
next_action: nextActionForSyncedTarget(targetStage),
|
|
439
447
|
updated_at: now
|
|
440
448
|
};
|
|
441
|
-
context.db.
|
|
442
|
-
try {
|
|
449
|
+
context.db.writeTransaction(() => {
|
|
443
450
|
persistProtocolState(context, protocol, nextState);
|
|
444
451
|
if (targetStage === "ready_for_merge") {
|
|
445
452
|
context.db.run(`INSERT INTO merge_queue
|
|
@@ -471,12 +478,7 @@ export function syncProtocolFromRun(context, input) {
|
|
|
471
478
|
stage_chain: stageChain(index)
|
|
472
479
|
}
|
|
473
480
|
});
|
|
474
|
-
|
|
475
|
-
}
|
|
476
|
-
catch (error) {
|
|
477
|
-
context.db.exec("ROLLBACK");
|
|
478
|
-
throw error;
|
|
479
|
-
}
|
|
481
|
+
}, "protocols.syncFromRun");
|
|
480
482
|
return {
|
|
481
483
|
ok: true,
|
|
482
484
|
protocol_id: protocol.id,
|
|
@@ -508,8 +510,7 @@ export function cancelProtocol(context, input) {
|
|
|
508
510
|
const now = context.now();
|
|
509
511
|
const worktree = context.db.get("SELECT worktree_path, status FROM worktree_records WHERE project_id = ? AND protocol_id = ?", [protocol.project_id, protocol.id]);
|
|
510
512
|
const worktreeRemoval = input.worktree === "remove" ? removeProtocolWorktreeBeforeCancel(protocol, state, worktree, input.force) : undefined;
|
|
511
|
-
context.db.
|
|
512
|
-
try {
|
|
513
|
+
return context.db.writeTransaction(() => {
|
|
513
514
|
const nextState = {
|
|
514
515
|
...state,
|
|
515
516
|
stage: "cancelled",
|
|
@@ -570,7 +571,6 @@ export function cancelProtocol(context, input) {
|
|
|
570
571
|
worktree_outcome: worktreeOutcome
|
|
571
572
|
}
|
|
572
573
|
});
|
|
573
|
-
context.db.exec("COMMIT");
|
|
574
574
|
return {
|
|
575
575
|
ok: true,
|
|
576
576
|
protocol_id: protocol.id,
|
|
@@ -582,11 +582,7 @@ export function cancelProtocol(context, input) {
|
|
|
582
582
|
worktree_outcome: worktreeOutcome,
|
|
583
583
|
worktree: context.db.get("SELECT * FROM worktree_records WHERE project_id = ? AND protocol_id = ?", [protocol.project_id, protocol.id]) ?? null
|
|
584
584
|
};
|
|
585
|
-
}
|
|
586
|
-
catch (error) {
|
|
587
|
-
context.db.exec("ROLLBACK");
|
|
588
|
-
throw error;
|
|
589
|
-
}
|
|
585
|
+
}, "protocols.cancel");
|
|
590
586
|
}
|
|
591
587
|
export function requireProtocol(context, protocolId, projectId) {
|
|
592
588
|
const matches = projectId
|
|
@@ -761,6 +757,9 @@ export function readProtocolRuntimeState(context, protocol) {
|
|
|
761
757
|
const expected = runtimeStateJsonPath(context.ddFlowHome, protocol.project_id, protocol.id);
|
|
762
758
|
if (protocol.state_path !== expected)
|
|
763
759
|
throw new AppError("protocol_state_migration_required", "Protocol runtime path requires explicit cleanup scan/apply before use", 1, { protocol_id: protocol.id, path: protocol.state_path, expected_path: expected });
|
|
760
|
+
const accepted = latestProtocolProjection(context, protocol.project_id, protocol.id);
|
|
761
|
+
if (accepted)
|
|
762
|
+
return { state: accepted, diagnostics: [] };
|
|
764
763
|
return resolveProtocolRuntimeState(context, { ...protocol }, false);
|
|
765
764
|
}
|
|
766
765
|
/** Explicit cleanup migration; productive readers accept only the current path. */
|
|
@@ -772,6 +771,19 @@ export function repairProtocolRuntimeState(context, protocol) {
|
|
|
772
771
|
Object.assign(protocol, stored);
|
|
773
772
|
return resolveProtocolRuntimeState(context, protocol, true);
|
|
774
773
|
}
|
|
774
|
+
/** Read-only repair preparation; caller CASes paths, then persists this state. */
|
|
775
|
+
export function prepareProtocolRuntimeRepair(context, protocol) {
|
|
776
|
+
const stored = context.db.get("SELECT * FROM protocols WHERE project_id = ? AND id = ?", [protocol.project_id, protocol.id]);
|
|
777
|
+
if (!stored)
|
|
778
|
+
throw new AppError("not_found", "Protocol is not registered", 1, { protocol_id: protocol.id });
|
|
779
|
+
const state = latestProtocolProjection(context, stored.project_id, stored.id) ?? resolveProtocolRuntimeState(context, { ...stored }, false).state;
|
|
780
|
+
return {
|
|
781
|
+
state,
|
|
782
|
+
statePath: runtimeStateJsonPath(context.ddFlowHome, stored.project_id, stored.id),
|
|
783
|
+
planPath: planJsonPath(stored.project_root, stored.id),
|
|
784
|
+
previousStatePath: stored.state_path
|
|
785
|
+
};
|
|
786
|
+
}
|
|
775
787
|
function resolveProtocolRuntimeState(context, protocol, repair) {
|
|
776
788
|
const diagnostics = [];
|
|
777
789
|
normalizeProtocolPlanPath(context, protocol, diagnostics, repair);
|
|
@@ -890,31 +902,49 @@ export function protocolRunDiagnostics(context, protocol, state) {
|
|
|
890
902
|
return { latest_run: latestWithStages, diagnostics };
|
|
891
903
|
}
|
|
892
904
|
export function persistProtocolState(context, protocol, state) {
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
context.db.run(`UPDATE protocols SET
|
|
905
|
+
context.db.writeTransaction(() => {
|
|
906
|
+
context.db.run(`UPDATE protocols SET
|
|
896
907
|
status = ?, stage = ?, next_action = ?, route_json = ?, workspace_json = ?,
|
|
897
908
|
blockers_json = ?, active_def_json = ?, updated_at = ?
|
|
898
909
|
WHERE project_id = ? AND id = ?`, [
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
910
|
+
state.status,
|
|
911
|
+
state.stage,
|
|
912
|
+
state.next_action,
|
|
913
|
+
JSON.stringify(state.route),
|
|
914
|
+
JSON.stringify(state.workspace),
|
|
915
|
+
JSON.stringify(state.blockers),
|
|
916
|
+
JSON.stringify(state.active_def),
|
|
917
|
+
state.updated_at,
|
|
918
|
+
protocol.project_id,
|
|
919
|
+
protocol.id
|
|
920
|
+
]);
|
|
921
|
+
appendAudit(context, { protocolId: protocol.id, projectId: protocol.project_id, eventType: "protocol.state_projection", payload: { state } });
|
|
922
|
+
if (state.status === "closed" || state.stage === "closed") {
|
|
923
|
+
const now = context.now();
|
|
924
|
+
context.db.run(`UPDATE sessions
|
|
913
925
|
SET status = 'stopped', stop_reason = 'protocol closed', updated_at = ?, stopped_at = ?
|
|
914
926
|
WHERE project_id = ?
|
|
915
927
|
AND protocol_id = ?
|
|
916
928
|
AND status IN ('pending', 'active', 'waiting_user', 'blocked', 'stopping')`, [now, now, protocol.project_id, protocol.id]);
|
|
917
|
-
|
|
929
|
+
}
|
|
930
|
+
context.db.afterCommit(() => publishProtocolProjection(context, protocol.project_id, protocol.id));
|
|
931
|
+
}, "protocols.persistProtocolState");
|
|
932
|
+
}
|
|
933
|
+
function latestProtocolProjection(context, projectId, protocolId) {
|
|
934
|
+
const row = context.db.get("SELECT payload_json FROM audit_events WHERE project_id = ? AND protocol_id = ? AND event_type = 'protocol.state_projection' ORDER BY id DESC LIMIT 1", [projectId, protocolId]);
|
|
935
|
+
return row ? JSON.parse(row.payload_json).state : null;
|
|
936
|
+
}
|
|
937
|
+
export function publishProtocolProjection(context, projectId, protocolId) {
|
|
938
|
+
const protocol = requireProtocol(context, protocolId, projectId);
|
|
939
|
+
ensureDir(path.dirname(protocol.state_path));
|
|
940
|
+
withProjectionLock(`${protocol.state_path}.publish-lock`, () => {
|
|
941
|
+
const accepted = latestProtocolProjection(context, projectId, protocolId);
|
|
942
|
+
if (!accepted)
|
|
943
|
+
throw new AppError("runtime_state_missing", "Protocol has no accepted SQL state", 1, { protocol_id: protocolId });
|
|
944
|
+
const temporary = `${protocol.state_path}.tmp-${process.pid}-${crypto.randomUUID()}`;
|
|
945
|
+
fs.writeFileSync(temporary, `${JSON.stringify(accepted, null, 2)}\n`);
|
|
946
|
+
fs.renameSync(temporary, protocol.state_path);
|
|
947
|
+
});
|
|
918
948
|
}
|
|
919
949
|
function findProtocol(context, protocolId, projectId) {
|
|
920
950
|
if (projectId) {
|
|
@@ -4,14 +4,16 @@ import path from "node:path";
|
|
|
4
4
|
import { AppError } from "../shared/errors.js";
|
|
5
5
|
import { registerStoreWriter } from "../storage/writer-contract.js";
|
|
6
6
|
import { managedProcessRuntimeHome } from "./managed-processes.js";
|
|
7
|
+
import { cleanupScopeWriters } from "./scope-writer-cleanup.js";
|
|
7
8
|
/** Export one RUN's relational evidence, never mutate the live authority. */
|
|
8
9
|
export function captureRecoveryDatabase(source, output, projectId, runId, protocolIds = []) {
|
|
9
10
|
source.exec(`VACUUM INTO '${output.replace(/'/g, "''")}'`);
|
|
10
11
|
const db = new DatabaseSync(output);
|
|
11
|
-
let complete = false;
|
|
12
|
+
let complete = false, held = false, failed = false, primary;
|
|
12
13
|
try {
|
|
13
14
|
registerStoreWriter(db);
|
|
14
15
|
db.exec("PRAGMA foreign_keys = OFF; PRAGMA secure_delete = ON; BEGIN IMMEDIATE");
|
|
16
|
+
held = true;
|
|
15
17
|
db.exec("CREATE TEMP TABLE capture_scope (project_id TEXT, run_id TEXT)");
|
|
16
18
|
db.prepare("INSERT INTO capture_scope VALUES (?, ?)").run(projectId, runId);
|
|
17
19
|
db.exec("CREATE TEMP TABLE capture_protocols (id TEXT)");
|
|
@@ -40,6 +42,7 @@ export function captureRecoveryDatabase(source, output, projectId, runId, protoc
|
|
|
40
42
|
work_recovery_segments: `work_id IN (${works})`,
|
|
41
43
|
run_recovery_bindings: `recovery_id IN (SELECT recovery_id FROM run_recovery_guards WHERE ${ownRun})`,
|
|
42
44
|
lifecycle_invocations: invocations,
|
|
45
|
+
lifecycle_packet_assignments: `invocation_id IN (SELECT id FROM lifecycle_invocations WHERE ${invocations})`,
|
|
43
46
|
hook_events: `${ownProject} AND (session_id IN (${sessions}) OR (harness, provider_session_id) IN (${nativeSessions})${invocationReceipts})`,
|
|
44
47
|
harness_usage_snapshots: `${ownProject} AND (harness, provider_session_id) IN (${nativeSessions})`,
|
|
45
48
|
protocols: `${ownProject} AND id IN (${protocols})`,
|
|
@@ -71,13 +74,31 @@ export function captureRecoveryDatabase(source, output, projectId, runId, protoc
|
|
|
71
74
|
const issues = db.prepare("PRAGMA foreign_key_check").all();
|
|
72
75
|
if (issues.length)
|
|
73
76
|
throw new AppError("snapshot_scope_inconsistent", "Scoped recovery export has unresolved relational references", 1, { issues });
|
|
74
|
-
db.exec("COMMIT
|
|
77
|
+
db.exec("COMMIT");
|
|
78
|
+
held = false;
|
|
79
|
+
db.exec("VACUUM");
|
|
75
80
|
complete = true;
|
|
76
81
|
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
82
|
+
catch (error) {
|
|
83
|
+
primary = error;
|
|
84
|
+
failed = true;
|
|
85
|
+
}
|
|
86
|
+
const cleanup = cleanupScopeWriters([{ name: output, db, rollback: held, close: "always" }]);
|
|
87
|
+
if (!complete) {
|
|
88
|
+
try {
|
|
80
89
|
fs.rmSync(output, { force: true });
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
cleanup.push({ target: output, phase: "remove_incomplete_output", message: String(error) });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
if (failed || cleanup.length) {
|
|
96
|
+
if (primary instanceof AppError) {
|
|
97
|
+
if (cleanup.length)
|
|
98
|
+
primary.details.snapshot_cleanup_errors = cleanup;
|
|
99
|
+
throw primary;
|
|
100
|
+
}
|
|
101
|
+
throw new AppError("snapshot_database_failed", primary instanceof Error ? primary.message : "Snapshot database cleanup failed", 1, { ...(failed ? { cause: String(primary) } : {}), snapshot_cleanup_errors: cleanup });
|
|
81
102
|
}
|
|
82
103
|
}
|
|
83
104
|
/** Physical leases are evidence, not authority to resurrect on another host. */
|
|
@@ -87,8 +108,10 @@ export function readRecoveryResourceEvidence(resourceHome, projectId, runId, aut
|
|
|
87
108
|
if (!fs.existsSync(file))
|
|
88
109
|
return { ...scope, database_present: false, processes: [], resources: [] };
|
|
89
110
|
const db = new DatabaseSync(file, { readOnly: true });
|
|
111
|
+
let held = false, failed = false, primary, result;
|
|
90
112
|
try {
|
|
91
113
|
db.exec("PRAGMA busy_timeout = 4000; BEGIN");
|
|
114
|
+
held = true;
|
|
92
115
|
const candidates = db.prepare("SELECT * FROM managed_processes WHERE project_id = ? AND run_id = ? ORDER BY id").all(projectId, runId).map(record => ({ record, home: managedProcessRuntimeHome(record, authority) }));
|
|
93
116
|
if (candidates.some(item => item.home === null))
|
|
94
117
|
throw new AppError("snapshot_resource_scope_unproven", "A retained process has no proven runtime-home identity", 1, { resource_home: resourceHome });
|
|
@@ -99,9 +122,21 @@ export function readRecoveryResourceEvidence(resourceHome, projectId, runId, aut
|
|
|
99
122
|
if (unbound)
|
|
100
123
|
throw new AppError("snapshot_resource_scope_unproven", "A retained resource has no process binding proving its RUN scope", 1, { resource_home: resourceHome });
|
|
101
124
|
db.exec("COMMIT");
|
|
102
|
-
|
|
125
|
+
held = false;
|
|
126
|
+
result = { ...scope, database_present: true, processes, resources };
|
|
103
127
|
}
|
|
104
|
-
|
|
105
|
-
|
|
128
|
+
catch (error) {
|
|
129
|
+
primary = error;
|
|
130
|
+
failed = true;
|
|
131
|
+
}
|
|
132
|
+
const cleanup = cleanupScopeWriters([{ name: file, db, rollback: held, close: "always" }]);
|
|
133
|
+
if (failed || cleanup.length) {
|
|
134
|
+
if (primary instanceof AppError) {
|
|
135
|
+
if (cleanup.length)
|
|
136
|
+
primary.details.snapshot_cleanup_errors = cleanup;
|
|
137
|
+
throw primary;
|
|
138
|
+
}
|
|
139
|
+
throw new AppError("snapshot_resource_evidence_failed", primary instanceof Error ? primary.message : "Resource evidence cleanup failed", 1, { ...(failed ? { cause: String(primary) } : {}), snapshot_cleanup_errors: cleanup });
|
|
106
140
|
}
|
|
141
|
+
return result;
|
|
107
142
|
}
|
|
@@ -56,21 +56,16 @@ async function launchControlWorker(context, input) {
|
|
|
56
56
|
const home = resourceHome(context);
|
|
57
57
|
// Reclaim only a proven dead launcher or admitted worker. Lease expiry
|
|
58
58
|
// never proves death or authorizes a second physical owner.
|
|
59
|
+
const launcherBirth = processStartedAt(process.pid);
|
|
59
60
|
let claimed = false;
|
|
60
|
-
context.db.
|
|
61
|
-
try {
|
|
61
|
+
context.db.writeTransaction(() => {
|
|
62
62
|
claimed = context.db.run("INSERT OR IGNORE INTO run_control_workers (control_id, owner_token, status, resource_home, state_dir, created_at, updated_at) VALUES (?, ?, 'starting', ?, ?, ?, ?)", [input.controlId, token, home, directory, now, now]).changes === 1;
|
|
63
63
|
if (claimed) {
|
|
64
|
-
recordControlLauncher(context, input.controlId, token);
|
|
64
|
+
recordControlLauncher(context, input.controlId, token, launcherBirth);
|
|
65
65
|
if (input.reconcileId)
|
|
66
66
|
context.db.run("UPDATE run_control_workers SET snapshot_json = ? WHERE control_id = ?", [JSON.stringify(renewRecoveryObservation({}, input.reconcileId)), input.controlId]);
|
|
67
67
|
}
|
|
68
|
-
|
|
69
|
-
}
|
|
70
|
-
catch (error) {
|
|
71
|
-
context.db.exec("ROLLBACK");
|
|
72
|
-
throw error;
|
|
73
|
-
}
|
|
68
|
+
});
|
|
74
69
|
if (!claimed && !reclaimControlWorker(context, input, project.id, token, home))
|
|
75
70
|
return;
|
|
76
71
|
try {
|
|
@@ -123,9 +118,9 @@ export async function waitForControlWorkerAdmission(context, controlId, ownerTok
|
|
|
123
118
|
}
|
|
124
119
|
}
|
|
125
120
|
const launchOperationId = (controlId, token) => `${controlId}:worker-launch:${crypto.createHash("sha256").update(token).digest("hex")}`;
|
|
126
|
-
function recordControlLauncher(context, controlId, token) {
|
|
121
|
+
function recordControlLauncher(context, controlId, token, launcherBirth) {
|
|
127
122
|
const now = context.now();
|
|
128
|
-
context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [launchOperationId(controlId, token), controlId, JSON.stringify({ kind: "control_worker_launch_claim" }), JSON.stringify({ pid: process.pid, pid_started_at:
|
|
123
|
+
context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [launchOperationId(controlId, token), controlId, JSON.stringify({ kind: "control_worker_launch_claim" }), JSON.stringify({ pid: process.pid, pid_started_at: launcherBirth, registration: "worker_self@1" }), now, now]);
|
|
129
124
|
}
|
|
130
125
|
function reclaimControlWorker(context, input, projectId, token, home) {
|
|
131
126
|
const previous = context.db.get("SELECT * FROM run_control_workers WHERE control_id = ?", [input.controlId]);
|
|
@@ -187,32 +182,25 @@ function reclaimControlWorker(context, input, projectId, token, home) {
|
|
|
187
182
|
// and a crash before the RUN CAS leaves the same dead owner reconcilable.
|
|
188
183
|
if (record && !finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: "control_owner_crash_reconciled" }))
|
|
189
184
|
return false;
|
|
190
|
-
|
|
191
|
-
|
|
185
|
+
const launcherBirth = processStartedAt(process.pid);
|
|
186
|
+
return context.db.writeTransaction(() => {
|
|
192
187
|
const changed = context.db.run(`UPDATE run_control_workers SET owner_token = ?, owner_pid = NULL, status = 'starting', process_id = NULL, process_lease_token = NULL, updated_at = ?
|
|
193
188
|
WHERE control_id = ? AND owner_token = ? AND process_id IS ? AND owner_pid IS ? AND status = ?
|
|
194
189
|
AND (SELECT receipt_json FROM run_control_operations WHERE operation_id = ?) IS ?
|
|
195
190
|
AND EXISTS (SELECT 1 FROM run_controls c JOIN run_recovery_guards g ON g.recovery_id = c.recovery_id
|
|
196
191
|
WHERE c.control_id = run_control_workers.control_id AND c.status = 'requested' AND g.status IN ('draining','sealed')
|
|
197
192
|
AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [token, context.now(), input.controlId, previous.owner_token, previous.process_id, previous.owner_pid, previous.status, operationId, launch?.receipt_json ?? null]);
|
|
198
|
-
if (changed.changes !== 1)
|
|
199
|
-
context.db.exec("ROLLBACK");
|
|
193
|
+
if (changed.changes !== 1)
|
|
200
194
|
return false;
|
|
201
|
-
}
|
|
202
195
|
if (input.reconcileId) {
|
|
203
196
|
const snapshot = renewRecoveryObservation(JSON.parse(previous.snapshot_json ?? "{}"), input.reconcileId);
|
|
204
197
|
context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = NULL WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), input.controlId, token]);
|
|
205
198
|
}
|
|
206
199
|
const now = context.now();
|
|
207
200
|
context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, previous.owner_token)}:reconciled`, input.controlId, JSON.stringify({ kind: "control_worker_reconciliation", process_id: record?.id ?? null, resource_home: home }), JSON.stringify({ process_id: record?.id ?? null, pid: record?.pid ?? null, pid_started_at: record?.pid_started_at ?? null, prior_status: previous.status, snapshot: JSON.parse(previous.snapshot_json ?? "null"), error: JSON.parse(previous.error_json ?? "null") }), now, now]);
|
|
208
|
-
recordControlLauncher(context, input.controlId, token);
|
|
209
|
-
context.db.exec("COMMIT");
|
|
201
|
+
recordControlLauncher(context, input.controlId, token, launcherBirth);
|
|
210
202
|
return true;
|
|
211
|
-
}
|
|
212
|
-
catch (error) {
|
|
213
|
-
context.db.exec("ROLLBACK");
|
|
214
|
-
throw error;
|
|
215
|
-
}
|
|
203
|
+
});
|
|
216
204
|
}
|
|
217
205
|
export async function serveRunControl(context, input) {
|
|
218
206
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
@@ -226,19 +214,14 @@ export async function serveRunControl(context, input) {
|
|
|
226
214
|
throw new AppError("run_control_owner_mismatch", "Control registration is not authorized", 1);
|
|
227
215
|
// Physical identity is committed with the sole RUN claim, before the worker
|
|
228
216
|
// registers itself in the separate resource store or performs any dispatch.
|
|
229
|
-
|
|
230
|
-
|
|
217
|
+
const workerBirth = processStartedAt(process.pid);
|
|
218
|
+
context.db.writeTransaction(() => {
|
|
231
219
|
const claimed = context.db.run("UPDATE run_control_workers SET status = 'running', owner_pid = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [process.pid, context.now(), input.controlId, worker.owner_token]);
|
|
232
220
|
if (claimed.changes !== 1)
|
|
233
221
|
throw new AppError("run_control_already_owned", "A control owner has already claimed this request", 1);
|
|
234
222
|
const now = context.now();
|
|
235
|
-
context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, worker.owner_token)}:worker-claim`, input.controlId, JSON.stringify({ kind: "control_worker_self_claim" }), JSON.stringify({ pid: process.pid, pid_started_at:
|
|
236
|
-
|
|
237
|
-
}
|
|
238
|
-
catch (error) {
|
|
239
|
-
context.db.exec("ROLLBACK");
|
|
240
|
-
throw error;
|
|
241
|
-
}
|
|
223
|
+
context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, worker.owner_token)}:worker-claim`, input.controlId, JSON.stringify({ kind: "control_worker_self_claim" }), JSON.stringify({ pid: process.pid, pid_started_at: workerBirth }), now, now]);
|
|
224
|
+
});
|
|
242
225
|
const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: worker.resource_home } };
|
|
243
226
|
let leaseLost = false;
|
|
244
227
|
const heartbeat = setInterval(() => {
|
|
@@ -19,13 +19,7 @@ export function runControlStatus(context, input) {
|
|
|
19
19
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
20
20
|
// One read snapshot prevents a concurrent stop/resume from mixing generations
|
|
21
21
|
// across the request, arbiter and worker projections. Savepoints also nest.
|
|
22
|
-
context.db.
|
|
23
|
-
try {
|
|
24
|
-
return readControlStatus(context, input, project);
|
|
25
|
-
}
|
|
26
|
-
finally {
|
|
27
|
-
context.db.exec("RELEASE run_control_status");
|
|
28
|
-
}
|
|
22
|
+
return context.db.readSnapshot(() => readControlStatus(context, input, project));
|
|
29
23
|
}
|
|
30
24
|
function readControlStatus(context, input, project) {
|
|
31
25
|
const run = context.db.get("SELECT status FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
|
|
@@ -822,20 +816,17 @@ export function recordRunControlIntent(context, input) {
|
|
|
822
816
|
const request = { project_root: project.root, run_id: input.runId, action: input.action, ...(input.action === "pause" ? { grace_ms: input.graceMs ?? 30_000, escalate: input.escalate ?? null } : { force: input.force ?? false }) };
|
|
823
817
|
const requestHash = crypto.createHash("sha256").update(JSON.stringify(request)).digest("hex");
|
|
824
818
|
ensureRunControlStorage(context.db);
|
|
825
|
-
context.db.
|
|
826
|
-
try {
|
|
819
|
+
return context.db.writeTransaction(() => {
|
|
827
820
|
const existing = context.db.get("SELECT * FROM run_controls WHERE project_id = ? AND run_id = ? AND request_id = ?", [project.id, input.runId, input.requestId]);
|
|
828
821
|
if (existing) {
|
|
829
822
|
if (existing.request_hash !== requestHash)
|
|
830
823
|
throw new AppError("runtime_request_conflict", "Control request id has different inputs", 1);
|
|
831
|
-
context.db.exec("COMMIT");
|
|
832
824
|
return { ok: true, control_id: existing.control_id, recovery_id: existing.recovery_id, action: existing.action, status: existing.status, reused: true };
|
|
833
825
|
}
|
|
834
826
|
const run = context.db.get("SELECT status FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
|
|
835
827
|
if (!run)
|
|
836
828
|
throw new AppError("not_found", "RUN is not registered", 1);
|
|
837
829
|
if (["done", "failed", "cancelled"].includes(run.status)) {
|
|
838
|
-
context.db.exec("COMMIT");
|
|
839
830
|
return { ok: true, terminal: true, status: run.status, control_id: null };
|
|
840
831
|
}
|
|
841
832
|
const prior = recoveryGuard(context, project.id, input.runId);
|
|
@@ -866,11 +857,6 @@ export function recordRunControlIntent(context, input) {
|
|
|
866
857
|
WHERE r.project_id = ? AND r.id = ? AND json_extract(stage.value, '$.status') = 'running'
|
|
867
858
|
AND EXISTS (SELECT 1 FROM run_control_admissions a JOIN sessions s ON s.session_id = a.session_id AND s.project_id = r.project_id WHERE a.control_id = ? AND s.current_stage = json_extract(stage.value, '$.stage'))`, [controlId, controlId, project.id, input.runId, controlId]);
|
|
868
859
|
setFlowRunRecoveryPaused(context, project.id, input.runId, true);
|
|
869
|
-
context.db.exec("COMMIT");
|
|
870
860
|
return { ok: true, control_id: controlId, recovery_id: recoveryId, generation, action: input.action, status: "requested", settled: false };
|
|
871
|
-
}
|
|
872
|
-
catch (error) {
|
|
873
|
-
context.db.exec("ROLLBACK");
|
|
874
|
-
throw error;
|
|
875
|
-
}
|
|
861
|
+
});
|
|
876
862
|
}
|
|
@@ -8,6 +8,25 @@ import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
|
|
|
8
8
|
import { appendEvent, digest, errorRecord, requireController, requireRun } from "./run-controller-state.js";
|
|
9
9
|
import { getFlowRunStatus } from "./runs.js";
|
|
10
10
|
import { recoveryObservationBudget, withRecoveryObservation } from "./recovery-observation-budget.js";
|
|
11
|
+
import { getResourceDatabase } from "../storage/database.js";
|
|
12
|
+
import { managedProcessBinding, resourceHome } from "./managed-processes.js";
|
|
13
|
+
/** Physical probes precede reservation; the accepted publisher still checks
|
|
14
|
+
* exact owner identity and fresh lease expiry after waiting for the writer. */
|
|
15
|
+
export function prepareControllerPublication(context, row, assertPhysicalOwner) {
|
|
16
|
+
assertPhysicalOwner();
|
|
17
|
+
const registry = getResourceDatabase(resourceHome(context));
|
|
18
|
+
const owner = registry.get("SELECT * FROM managed_processes WHERE id = ?", [row.process_id]);
|
|
19
|
+
if (!owner || owner.lease_token !== row.process_lease_token || owner.pid !== row.owner_pid || owner.owner_id !== row.controller_id)
|
|
20
|
+
throw new AppError("controller_owner_mismatch", "Controller publication lacks its physical owner", 1, { process_id: row.process_id, controller_id: row.controller_id, owner_found: Boolean(owner), lease_matches: owner?.lease_token === row.process_lease_token, pid_matches: owner?.pid === row.owner_pid, owner_matches: owner?.owner_id === row.controller_id, resource_home: resourceHome(context) });
|
|
21
|
+
const binding = managedProcessBinding(owner, resourceHome(context));
|
|
22
|
+
return () => {
|
|
23
|
+
const current = requireController(context, row.controller_id), process = registry.get("SELECT * FROM managed_processes WHERE id = ?", [owner.id]);
|
|
24
|
+
if (current.owner_token !== row.owner_token || current.owner_pid !== row.owner_pid || current.process_id !== row.process_id || current.process_lease_token !== row.process_lease_token || current.generation !== row.generation
|
|
25
|
+
|| !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status)
|
|
26
|
+
|| !process || process.state !== "running" || process.lease_expires_at <= context.now() || digest(managedProcessBinding(process, resourceHome(context))) !== digest(binding))
|
|
27
|
+
throw new AppError("controller_owner_mismatch", "Controller publication lost its current physical lease", 1);
|
|
28
|
+
};
|
|
29
|
+
}
|
|
11
30
|
/** Capture is performed by the productive owner, before it releases the next
|
|
12
31
|
* Stage. An observer never races the controller to copy a completed boundary. */
|
|
13
32
|
export async function captureControllerBoundary(context, row, manifest, stage, nextStage, assertPhysicalOwner) {
|
|
@@ -30,27 +49,29 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
|
|
|
30
49
|
return;
|
|
31
50
|
}
|
|
32
51
|
catch (error) {
|
|
33
|
-
|
|
34
|
-
|
|
52
|
+
try {
|
|
53
|
+
assertPhysicalOwner();
|
|
54
|
+
context.db.run("UPDATE run_controllers SET status = 'capture_failed', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
|
|
55
|
+
}
|
|
56
|
+
catch (recordError) {
|
|
57
|
+
if (error instanceof AppError)
|
|
58
|
+
error.details.capture_error_recording = errorRecord(recordError);
|
|
59
|
+
}
|
|
35
60
|
throw error;
|
|
36
61
|
}
|
|
37
62
|
}
|
|
38
|
-
context
|
|
39
|
-
|
|
40
|
-
|
|
63
|
+
const assertStartOwner = prepareControllerPublication(context, row, assertPhysicalOwner);
|
|
64
|
+
context.db.writeTransaction(() => {
|
|
65
|
+
assertStartOwner();
|
|
41
66
|
assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
|
|
42
67
|
const now = context.now();
|
|
43
68
|
context.db.run("UPDATE run_controllers SET status = 'capturing', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [now, row.controller_id, row.owner_token]);
|
|
44
69
|
context.db.run("INSERT INTO run_controller_operations (operation_id,controller_id,generation,input_hash,input_json,status,created_at,updated_at) VALUES (?,?,?,?,?,'capturing',?,?) ON CONFLICT(operation_id) DO UPDATE SET generation = excluded.generation, status = 'capturing', error_json = NULL, updated_at = excluded.updated_at", [operationId, row.controller_id, row.generation, boundaryKey, JSON.stringify({ action: "boundary_capture", ...identity(stage), stage_entry: nextStage, output }), now, now]);
|
|
45
70
|
appendEvent(context, row.controller_id, "boundary_capture_requested", { operation_id: operationId, boundary_key: boundaryKey, stage: stage.stage, stage_entry: nextStage });
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
throw error;
|
|
51
|
-
}
|
|
52
|
-
const assertCurrent = () => {
|
|
53
|
-
assertPhysicalOwner();
|
|
71
|
+
});
|
|
72
|
+
const assertCurrent = (physical = true) => {
|
|
73
|
+
if (physical)
|
|
74
|
+
assertPhysicalOwner();
|
|
54
75
|
const guard = recoveryGuard(context, row.project_id, row.run_id);
|
|
55
76
|
if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
|
|
56
77
|
throw new AppError("run_recovery_guarded", "Recovery or operator control superseded boundary capture", 1);
|
|
@@ -60,9 +81,11 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
|
|
|
60
81
|
const current = index.stage_runs.find(item => item.stage === stage.stage);
|
|
61
82
|
if (!current || !["done", "skipped"].includes(current.status) || digest(identity(current)) !== boundaryKey)
|
|
62
83
|
throw new AppError("controller_boundary_changed", "Stage identity changed after its capture fence", 1);
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
84
|
+
if (physical) {
|
|
85
|
+
const continuation = getFlowRunStatus(context, scope).continuation;
|
|
86
|
+
if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal")
|
|
87
|
+
throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
|
|
88
|
+
}
|
|
66
89
|
};
|
|
67
90
|
let priorObservation = null;
|
|
68
91
|
try {
|
|
@@ -114,23 +137,31 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
|
|
|
114
137
|
await delay(observationBudget.nextDelay());
|
|
115
138
|
}
|
|
116
139
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
140
|
+
assertCurrent();
|
|
141
|
+
const assertFinishOwner = prepareControllerPublication(context, row, assertPhysicalOwner);
|
|
142
|
+
const preparedIndex = requireRun(context, row.project_id, row.run_id).index_json;
|
|
143
|
+
context.db.writeTransaction(() => {
|
|
144
|
+
assertFinishOwner();
|
|
145
|
+
assertCurrent(false);
|
|
146
|
+
if (requireRun(context, row.project_id, row.run_id).index_json !== preparedIndex)
|
|
147
|
+
throw new AppError("controller_boundary_changed", "RUN changed before capture publication", 1);
|
|
120
148
|
context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
|
|
121
149
|
appendEvent(context, row.controller_id, "boundary_captured", { ...identity(stage), boundary_key: boundaryKey, operation_id: operationId, ...receipt });
|
|
122
150
|
context.db.run("UPDATE run_controllers SET status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
|
|
123
|
-
|
|
124
|
-
}
|
|
125
|
-
catch (error) {
|
|
126
|
-
context.db.exec("ROLLBACK");
|
|
127
|
-
throw error;
|
|
128
|
-
}
|
|
151
|
+
});
|
|
129
152
|
}
|
|
130
153
|
catch (error) {
|
|
131
154
|
observationBudget.exhausted();
|
|
132
|
-
|
|
133
|
-
|
|
155
|
+
try {
|
|
156
|
+
context.db.writeTransaction(() => {
|
|
157
|
+
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)) WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), operationId]);
|
|
158
|
+
context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
catch (recordError) {
|
|
162
|
+
if (error instanceof AppError)
|
|
163
|
+
error.details.capture_error_recording = errorRecord(recordError);
|
|
164
|
+
}
|
|
134
165
|
throw error;
|
|
135
166
|
}
|
|
136
167
|
}
|