@deksden-com/dd-flow-cli 0.9.0-beta.124 → 0.9.0-beta.127
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +10 -1
- package/dist/build-info.json +7 -7
- package/dist/cli/argument-values.js +38 -0
- package/dist/cli/command-inputs.js +4 -3
- package/dist/cli/help.js +1 -0
- package/dist/cli/hook-ingress.js +1 -1
- package/dist/cli/run-cli.js +126 -193
- package/dist/cli/runtime-maintenance-diagnostics.js +46 -0
- package/dist/cli/runtime-maintenance-ingress.js +51 -0
- package/dist/cli/runtime-process.js +128 -0
- package/dist/cli.js +8 -5
- package/dist/harness-runtime/bin/dd-agy.mjs +3 -2
- package/dist/harness-runtime/bin/dd-codex.mjs +3 -2
- package/dist/harness-runtime/bin/dd-droid.mjs +3 -2
- package/dist/harness-runtime/bin/dd-grok.mjs +16 -6
- package/dist/harness-runtime/bin/dd-opencode.mjs +3 -2
- package/dist/harness-runtime/bin/dd-zcode.mjs +20 -16
- package/dist/harness-runtime/lib/adapter-timeouts.d.mts +5 -0
- package/dist/harness-runtime/lib/adapter-timeouts.mjs +34 -0
- package/dist/harness-runtime/lib/daemon-observation.d.mts +2 -0
- package/dist/harness-runtime/lib/daemon-observation.mjs +163 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +60 -13
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +151 -72
- package/dist/harness-runtime/lib/dd-agy.mjs +11 -23
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +9 -14
- package/dist/harness-runtime/lib/dd-codex.mjs +252 -80
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +14 -19
- package/dist/harness-runtime/lib/dd-droid.mjs +117 -31
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +56 -30
- package/dist/harness-runtime/lib/dd-grok.mjs +15 -13
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +18 -27
- package/dist/harness-runtime/lib/dd-opencode.mjs +44 -14
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +56 -53
- package/dist/harness-runtime/lib/dd-zcode.mjs +200 -111
- package/dist/harness-runtime/lib/lease-renewal.d.mts +8 -0
- package/dist/harness-runtime/lib/lease-renewal.mjs +23 -0
- package/dist/harness-runtime/lib/lifecycle-command.d.mts +3 -3
- package/dist/harness-runtime/lib/lifecycle-command.mjs +74 -20
- package/dist/harness-runtime/lib/managed-daemon.d.mts +10 -1
- package/dist/harness-runtime/lib/managed-daemon.mjs +228 -40
- package/dist/harness-runtime/lib/native-children.d.mts +13 -0
- package/dist/harness-runtime/lib/native-children.mjs +78 -0
- package/dist/harness-runtime/lib/native-hook-command.d.mts +4 -0
- package/dist/harness-runtime/lib/native-hook-command.mjs +115 -11
- package/dist/harness-runtime/lib/native-streams.mjs +45 -0
- package/dist/harness-runtime/lib/observation-clock.d.mts +18 -0
- package/dist/harness-runtime/lib/observation-clock.mjs +73 -17
- package/dist/harness-runtime/lib/operation-errors.d.mts +14 -0
- package/dist/harness-runtime/lib/operation-errors.mjs +105 -8
- package/dist/harness-runtime/lib/operation-progress.d.mts +7 -0
- package/dist/harness-runtime/lib/operation-progress.mjs +112 -0
- package/dist/harness-runtime/lib/process-json.mjs +136 -38
- package/dist/harness-runtime/lib/tool-observations.d.mts +1 -0
- package/dist/harness-runtime/lib/tool-observations.mjs +5 -2
- package/dist/harness-runtime/lib/zcode-usage.mjs +3 -2
- package/dist/runtime/context.js +8 -4
- package/dist/schemas/run-control-receipt.schema.json +19 -0
- package/dist/services/code-checks.js +79 -9
- package/dist/services/codex-capacity.js +117 -41
- package/dist/services/codex-hook-delivery.js +2 -2
- package/dist/services/compatibility-preflight.js +4 -1
- package/dist/services/controller-fanout.js +9 -76
- package/dist/services/engines.js +70 -20
- package/dist/services/eval-snapshots.js +179 -40
- package/dist/services/external-work-launch.js +13 -8
- package/dist/services/harness-adapter.js +175 -11
- package/dist/services/hooks.js +65 -21
- package/dist/services/lifecycle-invocations.js +404 -32
- package/dist/services/managed-lease-monitor.js +128 -0
- package/dist/services/managed-processes.js +103 -47
- package/dist/services/merge-server.js +58 -21
- package/dist/services/recovery-observation-budget.js +67 -7
- package/dist/services/recovery-snapshot-database.js +7 -4
- package/dist/services/run-control-receipt.js +1 -1
- package/dist/services/run-control-worker.js +177 -118
- package/dist/services/run-control.js +99 -8
- package/dist/services/run-controller-adapter.js +169 -46
- package/dist/services/run-controller-capture.js +14 -2
- package/dist/services/run-controller-process.js +11 -4
- package/dist/services/run-controller-state.js +68 -7
- package/dist/services/run-controller.js +166 -94
- package/dist/services/run-fork.js +8 -3
- package/dist/services/run-recovery-guard.js +75 -0
- package/dist/services/run-recovery-runtime.js +1 -56
- package/dist/services/run-recovery.js +17 -13
- package/dist/services/runs.js +39 -6
- package/dist/services/runtime-budget.js +32 -15
- package/dist/services/runtime-scope-control.js +8 -0
- package/dist/services/runtime-scope-resume.js +30 -8
- package/dist/services/runtime-scope-worker.js +118 -73
- package/dist/services/runtime-service.js +6 -7
- package/dist/services/scope-writer-cleanup.js +3 -2
- package/dist/services/stage-context.js +135 -0
- package/dist/services/stage-lifecycle.js +3 -15
- package/dist/services/stage-pause.js +3 -2
- package/dist/services/usage.js +14 -3
- package/dist/services/vnext-code-review.js +10 -9
- package/dist/services/vnext-code.js +10 -8
- package/dist/services/vnext-contracts.js +10 -1
- package/dist/services/vnext-execution-profile.js +75 -2
- package/dist/services/vnext-fanout.js +5 -0
- package/dist/services/vnext-merge.js +10 -9
- package/dist/services/vnext-plan-review.js +21 -17
- package/dist/services/vnext-plan.js +9 -7
- package/dist/services/vnext-protocolize.js +15 -9
- package/dist/services/vnext-specify.js +6 -6
- package/dist/services/work-registry.js +34 -4
- package/dist/services/workspace-bootstrap.js +14 -3
- package/dist/shared/errors.js +11 -6
- package/dist/storage/agent-output.js +26 -0
- package/dist/storage/database.js +80 -21
- package/package.json +7 -4
|
@@ -6,7 +6,7 @@ import { AppError } from "../shared/errors.js";
|
|
|
6
6
|
import { canonicalPath } from "../storage/paths.js";
|
|
7
7
|
import { importGuardPath, rearmCompletedSnapshotImport } from "../storage/import-guard.js";
|
|
8
8
|
import { installCurrentEngine, upgradeForkedRunEngine, prepareForkEngine } from "./engines.js";
|
|
9
|
-
import { restoreEvalRunSnapshot, prepareEvalRunSnapshot, prepareEvalRunRestore, restoredWorkspaceRoot } from "./eval-snapshots.js";
|
|
9
|
+
import { restoreEvalRunSnapshot, prepareEvalRunSnapshot, prepareEvalRunRestore, prepareSnapshotOperationalDecision, restoredWorkspaceRoot } from "./eval-snapshots.js";
|
|
10
10
|
export function prepareRunFork(input) {
|
|
11
11
|
const requestId = requireId(input.requestId, "request-id");
|
|
12
12
|
const snapshot = canonicalPath(input.snapshot, false);
|
|
@@ -21,7 +21,8 @@ export function prepareRunFork(input) {
|
|
|
21
21
|
if (fs.existsSync(receiptFile)) {
|
|
22
22
|
const receipt = readJson(receiptFile);
|
|
23
23
|
const prior = receipt?.intent;
|
|
24
|
-
if (!prior || prior.request_id !== requestId || prior.snapshot !== snapshot || prior.engine_version !== engineVersion || prior.integrity_checksum !== (input.integrityChecksum ?? null)
|
|
24
|
+
if (!prior || prior.request_id !== requestId || prior.snapshot !== snapshot || prior.engine_version !== engineVersion || prior.integrity_checksum !== (input.integrityChecksum ?? null)
|
|
25
|
+
|| (prior.operational_decision_sha256 ?? null) !== (input.operationalDecisionFile ? fileHash(input.operationalDecisionFile) : null))
|
|
25
26
|
throw new AppError("fork_request_conflict", "Fork output already belongs to another request", 1, { output });
|
|
26
27
|
return { receipt, output };
|
|
27
28
|
}
|
|
@@ -30,6 +31,7 @@ export function prepareRunFork(input) {
|
|
|
30
31
|
if (manifest.schema_id !== "dd-flow/eval-run-snapshot@5" || manifest.purpose !== "stage_entry" || typeof manifest.stage_entry !== "string" || typeof manifest.run_id !== "string") {
|
|
31
32
|
throw new AppError("fork_checkpoint_invalid", "Fork requires a sealed stage-entry snapshot", 2, { snapshot });
|
|
32
33
|
}
|
|
34
|
+
const operational = prepareSnapshotOperationalDecision({ snapshot, runId: manifest.run_id, projectRoot: path.join(output, "project"), now: new Date().toISOString(), ...(input.operationalDecisionFile ? { file: input.operationalDecisionFile } : {}) });
|
|
33
35
|
const intent = {
|
|
34
36
|
schema_id: "dd-flow/run-fork-intent@1",
|
|
35
37
|
request_id: requestId,
|
|
@@ -37,6 +39,7 @@ export function prepareRunFork(input) {
|
|
|
37
39
|
snapshot_sha256: fileHash(manifestFile),
|
|
38
40
|
engine_version: engineVersion,
|
|
39
41
|
integrity_checksum: input.integrityChecksum ?? null,
|
|
42
|
+
...(operational ? { operational_decision_sha256: crypto.createHash("sha256").update(operational.bytes).digest("hex") } : {}),
|
|
40
43
|
target_stage: manifest.stage_entry
|
|
41
44
|
};
|
|
42
45
|
assertForkOutput(output, intentFile, intent);
|
|
@@ -58,6 +61,8 @@ export function forkRunFromSnapshot(context, input, prepared = prepareRunFork(in
|
|
|
58
61
|
if ("receipt" in prepared)
|
|
59
62
|
return prepared.receipt;
|
|
60
63
|
const { manifest, snapshot, output, intent, intentFile, receiptFile, requestId } = prepared;
|
|
64
|
+
if (input.operationalDecisionFile && fileHash(input.operationalDecisionFile) !== intent.operational_decision_sha256)
|
|
65
|
+
throw new AppError("operational_decision_changed", "Accepted fork decision changed after preparation", 2);
|
|
61
66
|
assertForkOutput(output, intentFile, intent);
|
|
62
67
|
const projectRoot = path.join(output, "project");
|
|
63
68
|
const home = path.join(output, "dd-flow-home");
|
|
@@ -69,7 +74,7 @@ export function forkRunFromSnapshot(context, input, prepared = prepareRunFork(in
|
|
|
69
74
|
writeJsonAtomic(intentFile, { schema_id: "dd-flow/run-fork-intent@1", intent });
|
|
70
75
|
fs.mkdirSync(projectRoot, { recursive: true, mode: 0o700 });
|
|
71
76
|
const targetContext = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_ENGINE_MODE: "1" }, "initialize", prearmedImportId ?? undefined);
|
|
72
|
-
const restoreInput = { snapshot, projectRoot };
|
|
77
|
+
const restoreInput = { snapshot, projectRoot, ...(input.operationalDecisionFile ? { operationalDecisionFile: input.operationalDecisionFile } : {}) };
|
|
73
78
|
const restored = restoreEvalRunSnapshot(targetContext, restoreInput, prepareEvalRunRestore(targetContext, restoreInput, manifest), prearmedImportId ?? undefined);
|
|
74
79
|
const restoredContext = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_ENGINE_MODE: "1" });
|
|
75
80
|
try {
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { AppError } from "../shared/errors.js";
|
|
2
|
+
import { CAPACITY_POLICY, overloadBurst } from "../harness-runtime/lib/codex-capacity-policy.mjs";
|
|
3
|
+
export function recoveryGuard(context, projectId, runId) { return context.db.get("SELECT recovery_id, project_id, run_id, generation, status, interruption_json, settlement_json, capture_path, created_at, updated_at, sealed_at, resumed_at FROM run_recovery_guards WHERE project_id = ? AND run_id = ? ORDER BY generation DESC LIMIT 1", [projectId, runId]) ?? null; }
|
|
4
|
+
export function assertRunRecoveryAllowed(context, projectId, runId, expectedGeneration) {
|
|
5
|
+
const guard = recoveryGuard(context, projectId, runId);
|
|
6
|
+
const generation = guard?.generation ?? 0;
|
|
7
|
+
if (expectedGeneration !== undefined && generation !== expectedGeneration)
|
|
8
|
+
throw new AppError("run_recovery_generation_stale", "RUN execution generation changed while the operation was in flight", 1, { run_id: runId, expected_generation: expectedGeneration, generation });
|
|
9
|
+
if (guard && ["draining", "sealed", "resuming"].includes(guard.status))
|
|
10
|
+
throw new AppError("run_recovery_guarded", "RUN is retained for recovery; reconcile or resume its recovery operation before accepting a new lifecycle mutation", 1, { run_id: runId, recovery_id: guard.recovery_id, generation, status: guard.status });
|
|
11
|
+
return generation;
|
|
12
|
+
}
|
|
13
|
+
export function assertRunMutationAllowed(context, projectId, runId, expectedGeneration) {
|
|
14
|
+
const generation = assertRunRecoveryAllowed(context, projectId, runId, expectedGeneration);
|
|
15
|
+
if (context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'")
|
|
16
|
+
&& context.db.get("SELECT 1 FROM run_controllers WHERE project_id = ? AND run_id = ? AND status IN ('capturing','capture_failed') LIMIT 1", [projectId, runId]))
|
|
17
|
+
throw new AppError("run_boundary_capturing", "RUN writers are fenced while its controller captures a Stage boundary", 1, { run_id: runId });
|
|
18
|
+
return generation;
|
|
19
|
+
}
|
|
20
|
+
/** The caller has already checked PID/start/lease and runtime-home identity. */
|
|
21
|
+
export function recoveryRuntimeAdmission(context, record, input) {
|
|
22
|
+
if (!record.project_id || !record.run_id || !input.daemonId)
|
|
23
|
+
return false;
|
|
24
|
+
const guard = recoveryGuard(context, record.project_id, record.run_id);
|
|
25
|
+
if (!guard || !["resuming", "recovered"].includes(guard.status))
|
|
26
|
+
return false;
|
|
27
|
+
const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ? AND daemon_id = ?", [guard.recovery_id, input.daemonId]);
|
|
28
|
+
if (!binding?.runtime_dispatch_json)
|
|
29
|
+
return false;
|
|
30
|
+
const dispatch = JSON.parse(binding.runtime_dispatch_json);
|
|
31
|
+
if (dispatch.process_id !== record.id || dispatch.owner_id !== record.owner_id || dispatch.state_dir !== input.stateDir || dispatch.daemon_id !== input.daemonId)
|
|
32
|
+
return false;
|
|
33
|
+
const metadata = JSON.parse(record.metadata_json);
|
|
34
|
+
if (metadata.run_generation !== guard.generation)
|
|
35
|
+
return false;
|
|
36
|
+
if (guard.status === "recovered" && binding.status === "accepted")
|
|
37
|
+
return !input.operationId?.startsWith(`${dispatch.operation_id}:capacity:`);
|
|
38
|
+
if (guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.resume")
|
|
39
|
+
return typeof dispatch.resume_operation_id === "string" && input.operationId === dispatch.resume_operation_id && input.nativeSessionId === binding.provider_session_id;
|
|
40
|
+
const active = dispatch.capacity_intents?.at(-1);
|
|
41
|
+
if (active) {
|
|
42
|
+
const ordinal = dispatch.capacity_intents.length;
|
|
43
|
+
const predecessor = ordinal === 1 ? dispatch.operation_id : dispatch.capacity_intents[ordinal - 2].operation_id;
|
|
44
|
+
if (active.policy !== CAPACITY_POLICY || active.ordinal !== ordinal || active.root_operation_id !== dispatch.operation_id || active.session_id !== binding.provider_session_id
|
|
45
|
+
|| active.predecessor !== predecessor || active.operation_id !== `${dispatch.operation_id}:capacity:${ordinal}:session.prompt`
|
|
46
|
+
|| active.refusal?.policy !== CAPACITY_POLICY || active.refusal.ordinal !== ordinal - 1 || active.refusal.turn_id !== active.turn_id
|
|
47
|
+
|| active.refusal.session_id !== binding.provider_session_id || active.deadline !== null || !Number.isFinite(Date.parse(active.not_before)))
|
|
48
|
+
return false;
|
|
49
|
+
const failure = dispatch.capacity_failures?.find(item => item.ordinal === ordinal - 1);
|
|
50
|
+
if (!failure || JSON.stringify(failure) !== JSON.stringify(active.refusal))
|
|
51
|
+
return false;
|
|
52
|
+
try {
|
|
53
|
+
if (overloadBurst(dispatch.capacity_failures))
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
const predecessorRow = context.db.get("SELECT status,receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [predecessor, record.owner_id, guard.generation]);
|
|
60
|
+
if (predecessorRow?.status !== "failed")
|
|
61
|
+
return false;
|
|
62
|
+
try {
|
|
63
|
+
if (JSON.stringify(JSON.parse(predecessorRow.receipt_json).capacity_intent) !== JSON.stringify(active))
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (active && active.not_before > context.now())
|
|
71
|
+
return false;
|
|
72
|
+
return guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.prompt"
|
|
73
|
+
&& input.operationId === (active?.operation_id ?? dispatch.operation_id) && input.nativeSessionId === binding.provider_session_id
|
|
74
|
+
&& input.promptSha256 === (active?.prompt_sha256 ?? dispatch.prompt_sha256);
|
|
75
|
+
}
|
|
@@ -109,59 +109,4 @@ export function retainRecoveryCapacityIntent(context, input) {
|
|
|
109
109
|
throw new AppError("recovery_capacity_intent_conflict", "Recovery predecessor changed before successor publication", 1);
|
|
110
110
|
});
|
|
111
111
|
}
|
|
112
|
-
|
|
113
|
-
export function recoveryRuntimeAdmission(context, record, input) {
|
|
114
|
-
if (!record.project_id || !record.run_id || !input.daemonId)
|
|
115
|
-
return false;
|
|
116
|
-
const guard = recoveryGuard(context, record.project_id, record.run_id);
|
|
117
|
-
if (!guard || !["resuming", "recovered"].includes(guard.status))
|
|
118
|
-
return false;
|
|
119
|
-
const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ? AND daemon_id = ?", [guard.recovery_id, input.daemonId]);
|
|
120
|
-
if (!binding?.runtime_dispatch_json)
|
|
121
|
-
return false;
|
|
122
|
-
const dispatch = JSON.parse(binding.runtime_dispatch_json);
|
|
123
|
-
if (dispatch.process_id !== record.id || dispatch.owner_id !== record.owner_id || dispatch.state_dir !== input.stateDir || dispatch.daemon_id !== input.daemonId)
|
|
124
|
-
return false;
|
|
125
|
-
const metadata = JSON.parse(record.metadata_json);
|
|
126
|
-
if (metadata.run_generation !== guard.generation)
|
|
127
|
-
return false;
|
|
128
|
-
if (guard.status === "recovered" && binding.status === "accepted")
|
|
129
|
-
return !input.operationId?.startsWith(`${dispatch.operation_id}:capacity:`);
|
|
130
|
-
if (guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.resume")
|
|
131
|
-
return typeof dispatch.resume_operation_id === "string" && input.operationId === dispatch.resume_operation_id && input.nativeSessionId === binding.provider_session_id;
|
|
132
|
-
const active = dispatch.capacity_intents?.at(-1);
|
|
133
|
-
if (active) {
|
|
134
|
-
const ordinal = dispatch.capacity_intents.length;
|
|
135
|
-
const predecessor = ordinal === 1 ? dispatch.operation_id : dispatch.capacity_intents[ordinal - 2].operation_id;
|
|
136
|
-
if (active.policy !== CAPACITY_POLICY || active.ordinal !== ordinal || active.root_operation_id !== dispatch.operation_id || active.session_id !== binding.provider_session_id
|
|
137
|
-
|| active.predecessor !== predecessor || active.operation_id !== `${dispatch.operation_id}:capacity:${ordinal}:session.prompt`
|
|
138
|
-
|| active.refusal?.policy !== CAPACITY_POLICY || active.refusal.ordinal !== ordinal - 1 || active.refusal.turn_id !== active.turn_id
|
|
139
|
-
|| active.refusal.session_id !== binding.provider_session_id || active.deadline !== null || !Number.isFinite(Date.parse(active.not_before)))
|
|
140
|
-
return false;
|
|
141
|
-
const failure = dispatch.capacity_failures?.find(item => item.ordinal === ordinal - 1);
|
|
142
|
-
if (!failure || JSON.stringify(failure) !== JSON.stringify(active.refusal))
|
|
143
|
-
return false;
|
|
144
|
-
try {
|
|
145
|
-
if (overloadBurst(dispatch.capacity_failures))
|
|
146
|
-
return false;
|
|
147
|
-
}
|
|
148
|
-
catch {
|
|
149
|
-
return false;
|
|
150
|
-
}
|
|
151
|
-
const predecessorRow = context.db.get("SELECT status,receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [predecessor, record.owner_id, guard.generation]);
|
|
152
|
-
if (predecessorRow?.status !== "failed")
|
|
153
|
-
return false;
|
|
154
|
-
try {
|
|
155
|
-
if (JSON.stringify(JSON.parse(predecessorRow.receipt_json).capacity_intent) !== JSON.stringify(active))
|
|
156
|
-
return false;
|
|
157
|
-
}
|
|
158
|
-
catch {
|
|
159
|
-
return false;
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
if (active && active.not_before > context.now())
|
|
163
|
-
return false;
|
|
164
|
-
return guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.prompt"
|
|
165
|
-
&& input.operationId === (active?.operation_id ?? dispatch.operation_id) && input.nativeSessionId === binding.provider_session_id
|
|
166
|
-
&& input.promptSha256 === (active?.prompt_sha256 ?? dispatch.prompt_sha256);
|
|
167
|
-
}
|
|
112
|
+
export { recoveryRuntimeAdmission } from "./run-recovery-guard.js";
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { hasLifecyclePacketAssignments, managedLifecycleCommand, recoverLifecycleCommands } from "./lifecycle-invocations.js";
|
|
2
|
+
import { recoveryGuard, assertRunMutationAllowed } from "./run-recovery-guard.js";
|
|
3
|
+
export { recoveryGuard, assertRunMutationAllowed, assertRunRecoveryAllowed } from "./run-recovery-guard.js";
|
|
2
4
|
import crypto from "node:crypto";
|
|
3
5
|
import fs from "node:fs";
|
|
4
6
|
import path from "node:path";
|
|
@@ -11,6 +13,7 @@ import { verifyRecoverySnapshot } from "./eval-snapshots.js";
|
|
|
11
13
|
import { findRecentMatchingHookEvent, hookSessionIdentity, recoveryAcceptMatchKey } from "./hooks.js";
|
|
12
14
|
import { flowCommand } from "./stage-pause.js";
|
|
13
15
|
import { materializePendingWorkResults } from "./work-registry.js";
|
|
16
|
+
import { currentStagePrompt } from "./stage-context.js";
|
|
14
17
|
export function recoveryPacketReceipt(binding) {
|
|
15
18
|
if (!binding.packet_json || !binding.packet_sha256 || crypto.createHash("sha256").update(binding.packet_json).digest("hex") !== binding.packet_sha256)
|
|
16
19
|
throw new AppError("recovery_packet_missing", "Recovery requires its original verified Work packet", 1);
|
|
@@ -30,12 +33,15 @@ function rootRecoveryPacket(context, run, guard, owner) {
|
|
|
30
33
|
throw new AppError("recovery_evidence_missing", "Captured root prompt is missing or uses an unsafe link", 1);
|
|
31
34
|
}
|
|
32
35
|
const prompt = fs.readFileSync(promptPath, "utf8");
|
|
36
|
+
const currentPrompt = currentStagePrompt(context, session.prompt_path, run.run_root, prompt);
|
|
33
37
|
const dependencies = JSON.parse(work.depends_on_json);
|
|
34
38
|
return JSON.stringify({
|
|
35
39
|
schema_id: "dd-flow/root-recovery-packet@1", recovery_id: guard.recovery_id, generation: guard.generation, run_id: run.id, work_id: owner.work_id,
|
|
36
40
|
mode: "same-session", continuation: boundary ? "stage_boundary" : "unfinished_work", retained_work_status: work.status, retained_session_status: session.status, retained_result: work.result,
|
|
37
41
|
original_task: work.task, original_task_sha256: crypto.createHash("sha256").update(work.task).digest("hex"),
|
|
38
42
|
original_prompt: { path: promptPath, text: prompt, sha256: crypto.createHash("sha256").update(prompt).digest("hex") },
|
|
43
|
+
...(currentPrompt.worker_prompt_markdown !== prompt ? { current_prompt: { path: currentPrompt.prompt_path, text: currentPrompt.worker_prompt_markdown, sha256: crypto.createHash("sha256").update(currentPrompt.worker_prompt_markdown).digest("hex") } } : {}),
|
|
44
|
+
...(currentPrompt.worker_prompt_markdown !== prompt ? { current_prompt_instruction: "Use current_prompt for current operational authority and input paths; original_prompt is immutable historical provenance. Neither prompt grants new command authority: retain the existing lifecycle recovery acknowledgement and replacement-command rules." } : {}),
|
|
39
45
|
result_schema: work.result_schema, payload: work.payload_json ? JSON.parse(work.payload_json) : null, paused: work.status === "paused",
|
|
40
46
|
dependency_results: dependencies.map((id) => context.db.get("SELECT work_id, status, result FROM works WHERE work_id = ?", [id])),
|
|
41
47
|
capture_path: guard.capture_path, observed_at: guard.created_at, sealed_at: guard.sealed_at,
|
|
@@ -67,19 +73,6 @@ export function prepareRecoveryPayload(value, label) {
|
|
|
67
73
|
throw new AppError("recovery_not_settled", "Recovery capture requires an explicit settled tree receipt", 1);
|
|
68
74
|
return payload;
|
|
69
75
|
}
|
|
70
|
-
export function recoveryGuard(context, projectId, runId) { return context.db.get("SELECT recovery_id, project_id, run_id, generation, status, interruption_json, settlement_json, capture_path, created_at, updated_at, sealed_at, resumed_at FROM run_recovery_guards WHERE project_id = ? AND run_id = ? ORDER BY generation DESC LIMIT 1", [projectId, runId]) ?? null; }
|
|
71
|
-
export function assertRunMutationAllowed(context, projectId, runId, expectedGeneration) {
|
|
72
|
-
const guard = recoveryGuard(context, projectId, runId);
|
|
73
|
-
const generation = guard?.generation ?? 0;
|
|
74
|
-
if (expectedGeneration !== undefined && generation !== expectedGeneration)
|
|
75
|
-
throw new AppError("run_recovery_generation_stale", "RUN execution generation changed while the operation was in flight", 1, { run_id: runId, expected_generation: expectedGeneration, generation });
|
|
76
|
-
if (guard && ["draining", "sealed", "resuming"].includes(guard.status))
|
|
77
|
-
throw new AppError("run_recovery_guarded", "RUN is retained for recovery; reconcile or resume its recovery operation before accepting a new lifecycle mutation", 1, { run_id: runId, recovery_id: guard.recovery_id, generation, status: guard.status });
|
|
78
|
-
if (context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'")
|
|
79
|
-
&& context.db.get("SELECT 1 FROM run_controllers WHERE project_id = ? AND run_id = ? AND status IN ('capturing','capture_failed') LIMIT 1", [projectId, runId]))
|
|
80
|
-
throw new AppError("run_boundary_capturing", "RUN writers are fenced while its controller captures a Stage boundary", 1, { run_id: runId });
|
|
81
|
-
return generation;
|
|
82
|
-
}
|
|
83
76
|
/** A cooperative drain permits only the WorkSession present at its fence. */
|
|
84
77
|
export function assertWorkSettlementAllowed(context, projectId, runId, workId) {
|
|
85
78
|
const guard = recoveryGuard(context, projectId, runId);
|
|
@@ -188,6 +181,7 @@ export function prepareRunRecoveryResume(context, input) {
|
|
|
188
181
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
189
182
|
const run = requireRecoveryRun(context, project.id, input.runId);
|
|
190
183
|
const guard = recoveryGuard(context, project.id, run.id);
|
|
184
|
+
assertRecoveryRunCanResume(context, project.id, run.id);
|
|
191
185
|
if (!guard || guard.recovery_id !== input.recoveryId)
|
|
192
186
|
throw new AppError("recovery_not_found", "Recovery operation is not registered for this RUN", 1, { recovery_id: input.recoveryId, run_id: run.id });
|
|
193
187
|
if (["resuming", "recovered"].includes(guard.status)) {
|
|
@@ -212,6 +206,7 @@ export function prepareRunRecoveryResume(context, input) {
|
|
|
212
206
|
}
|
|
213
207
|
export function resumeRunRecovery(context, input, prepared = prepareRunRecoveryResume(context, input)) {
|
|
214
208
|
const { project, run, guard } = prepared;
|
|
209
|
+
assertRecoveryRunCanResume(context, project.id, run.id);
|
|
215
210
|
const acceptCommand = acceptanceCommand(context, run.id, guard.recovery_id, run.project_root);
|
|
216
211
|
if (prepared.reused)
|
|
217
212
|
return { ok: true, recovery_id: guard.recovery_id, generation: guard.generation, resumed: guard.status === "recovered", awaiting_acceptance: guard.status === "resuming", accept_command: acceptCommand, reused: true };
|
|
@@ -226,6 +221,7 @@ export function resumeRunRecovery(context, input, prepared = prepareRunRecoveryR
|
|
|
226
221
|
context.db.all("SELECT b.recovery_id FROM run_recovery_bindings b JOIN run_recovery_guards g ON g.recovery_id = b.recovery_id WHERE g.project_id = ? AND b.daemon_id = ? ORDER BY b.recovery_id", [project.id, input.daemonId])
|
|
227
222
|
]);
|
|
228
223
|
const publication = context.db.readSnapshot(() => {
|
|
224
|
+
assertRecoveryRunCanResume(context, project.id, run.id);
|
|
229
225
|
const signature = authority();
|
|
230
226
|
const current = recoveryGuard(context, project.id, run.id);
|
|
231
227
|
if (!current || current.recovery_id !== guard.recovery_id || current.generation !== guard.generation || current.status !== "sealed" || current.capture_path !== guard.capture_path)
|
|
@@ -258,6 +254,7 @@ export function resumeRunRecovery(context, input, prepared = prepareRunRecoveryR
|
|
|
258
254
|
context.db.writeTransaction(() => {
|
|
259
255
|
if (authority() !== publication.signature)
|
|
260
256
|
throw new AppError("recovery_state_changed", "Recovery inputs changed while its packet was prepared", 1, { recovery_id: guard.recovery_id });
|
|
257
|
+
assertRecoveryRunCanResume(context, project.id, run.id);
|
|
261
258
|
const now = context.now();
|
|
262
259
|
context.db.run("INSERT INTO run_recovery_bindings (recovery_id, work_id, prior_work_session_id, session_id, harness, provider_session_id, daemon_id, status, created_at, packet_json, packet_sha256) VALUES (?, ?, ?, ?, ?, ?, ?, 'prepared', ?, ?, ?)", [guard.recovery_id, owner.work_id, owner.id, owner.session_id, input.harness, input.nativeSessionId, input.daemonId, now, packet, crypto.createHash("sha256").update(packet).digest("hex")]);
|
|
263
260
|
for (const work of running)
|
|
@@ -276,11 +273,13 @@ export function resumeRunRecovery(context, input, prepared = prepareRunRecoveryR
|
|
|
276
273
|
export function acceptRunRecovery(context, input) {
|
|
277
274
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
278
275
|
const run = requireRecoveryRun(context, project.id, input.runId);
|
|
276
|
+
assertRecoveryRunCanResume(context, project.id, run.id);
|
|
279
277
|
const matchKey = entityReferenceVariants(run.id).map((reference) => recoveryAcceptMatchKey(reference, input.recoveryId, run.project_root));
|
|
280
278
|
const eventKey = input.hookEventId ?? findRecentMatchingHookEvent(context, { projectId: project.id, matchKey, errorCode: "trusted_recovery_acceptance_required", operation: "run recovery accept" }).eventKey;
|
|
281
279
|
const identity = hookSessionIdentity(context, project.id, eventKey);
|
|
282
280
|
const now = context.now();
|
|
283
281
|
const result = context.db.writeTransaction(() => {
|
|
282
|
+
assertRecoveryRunCanResume(context, project.id, run.id);
|
|
284
283
|
const guard = recoveryGuard(context, project.id, run.id);
|
|
285
284
|
const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ?", [input.recoveryId]);
|
|
286
285
|
const hook = context.db.get("SELECT status, match_key FROM hook_events WHERE id = ?", [identity.hookEventId]);
|
|
@@ -324,6 +323,11 @@ export function acceptRunRecovery(context, input) {
|
|
|
324
323
|
appendFlowRunTimelineEvent(context, project.id, run.id, { type: "run_recovery_accepted", recovery_id: input.recoveryId, daemon_id: identity.daemonId, hook_event_key: eventKey });
|
|
325
324
|
return result;
|
|
326
325
|
}
|
|
326
|
+
function assertRecoveryRunCanResume(context, projectId, runId) {
|
|
327
|
+
const run = context.db.get("SELECT status FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
|
|
328
|
+
if (run && ["done", "failed", "cancelled"].includes(run.status))
|
|
329
|
+
throw new AppError("invalid_run_state", "A terminal RUN permits settlement, not productive recovery", 1, { run_id: runId, status: run.status });
|
|
330
|
+
}
|
|
327
331
|
function recoveryEvidencePath(context, run, guard, workId) {
|
|
328
332
|
// The verified capture is already immutable evidence. Referencing it avoids
|
|
329
333
|
// a second filesystem copy whose partial result could survive SQL rollback.
|
package/dist/services/runs.js
CHANGED
|
@@ -16,7 +16,7 @@ import { recalculateRunUsage } from "./usage.js";
|
|
|
16
16
|
import { refreshRunSessionProjection } from "./run-projection.js";
|
|
17
17
|
import { resolveCanonRoot } from "./canon.js";
|
|
18
18
|
import { bindCurrentEngineToRun } from "./engines.js";
|
|
19
|
-
import { executionProfilePath,
|
|
19
|
+
import { assertOperationalDecisionSource, executionProfilePath, loadVnextExecutionProfileSnapshot } from "./vnext-execution-profile.js";
|
|
20
20
|
import { loadVnextWorkspacePolicy } from "./vnext-workspace-policy.js";
|
|
21
21
|
import { assertSameSessionMerge, loadExecutionRoutingInput, resolveStageTransition, snapshotAgentProfiles } from "./execution-policy.js";
|
|
22
22
|
import { isLegalVnextTransition, vnextStageDirectory, vnextStages } from "../domain/stage-catalog.js";
|
|
@@ -54,7 +54,7 @@ export function prepareFlowRunStart(context, input) {
|
|
|
54
54
|
if (input.executionRoutingFile !== undefined && !vnext)
|
|
55
55
|
throw new AppError("usage", "Execution routing input is supported only for vNext RUNs", 2);
|
|
56
56
|
const routingInput = input.executionRoutingFile !== undefined ? loadExecutionRoutingInput(input.executionRoutingFile) : undefined;
|
|
57
|
-
const executionProfile = vnext ? snapshotVnextExecutionProfile(projectRoot, context.ddFlowHome, routingInput) : undefined;
|
|
57
|
+
const executionProfile = vnext ? snapshotVnextExecutionProfile(projectRoot, context.ddFlowHome, routingInput, context.now()) : undefined;
|
|
58
58
|
const flowContract = input.flowKind === "mb-upgrade"
|
|
59
59
|
? loadUpgradeFlowContract(context, projectRoot)
|
|
60
60
|
: loadProjectFlowContract(projectRoot);
|
|
@@ -73,6 +73,14 @@ export function prepareFlowRunStart(context, input) {
|
|
|
73
73
|
}
|
|
74
74
|
export function startFlowRun(context, input, prepared = prepareFlowRunStart(context, input)) {
|
|
75
75
|
const { projectRoot, flowKind, routingInput, executionProfile, workspaceRoot, slug, subjectId, subjectType, now, flowFlags } = prepared;
|
|
76
|
+
// Ordinary settings are already frozen by preparation; only scoped authority
|
|
77
|
+
// requires its live acceptance source to survive final binding unchanged.
|
|
78
|
+
if (executionProfile?.operational_decision) {
|
|
79
|
+
if (crypto.createHash("sha256").update(fs.readFileSync(executionProfile.source_path)).digest("hex") !== executionProfile.source_checksum)
|
|
80
|
+
throw new AppError("execution_profile_changed", "Execution profile changed after RUN preparation", 2);
|
|
81
|
+
if (executionProfile.operational_decision)
|
|
82
|
+
assertOperationalDecisionSource(executionProfile.operational_decision, projectRoot, context.now());
|
|
83
|
+
}
|
|
76
84
|
registerProject(context, { root: projectRoot });
|
|
77
85
|
const project = requireProjectByRoot(context, projectRoot);
|
|
78
86
|
const runId = nextRunId(context, project.id, slug);
|
|
@@ -80,6 +88,14 @@ export function startFlowRun(context, input, prepared = prepareFlowRunStart(cont
|
|
|
80
88
|
const runHome = projectRunHome(context.ddFlowHome, project.id, runId);
|
|
81
89
|
const runDirRelative = path.posix.join("runs", runId);
|
|
82
90
|
const runtimePath = projectRunJsonPath(context.ddFlowHome, project.id, runId);
|
|
91
|
+
const policyBytes = executionProfile?.operational_decision ? fs.readFileSync(path.resolve(projectRoot, executionProfile.operational_decision.policy.path)) : undefined;
|
|
92
|
+
const policySource = policyBytes ? { path: "operational-policy.source.md", sha256: crypto.createHash("sha256").update(policyBytes).digest("hex") } : undefined;
|
|
93
|
+
if (policySource && policySource.sha256 !== executionProfile?.operational_decision?.policy.sha256)
|
|
94
|
+
throw new AppError("operational_decision_policy_changed", "Policy changed during final RUN binding", 2);
|
|
95
|
+
const binding = { project_id: project.id, run_id: runId, project_root: projectRoot, ...(executionProfile?.operational_decision?.scope.eval_id ? { eval_id: executionProfile.operational_decision.scope.eval_id } : {}), ...(executionProfile?.operational_decision?.scope.execution_id ? { execution_id: executionProfile.operational_decision.scope.execution_id } : {}) };
|
|
96
|
+
const operationalBytes = executionProfile ? `${JSON.stringify({ schema_id: "dd-flow/run-operational-contract@1", binding, settings: { merge_delivery: executionProfile.settings.merge_delivery, merge_cleanup: executionProfile.settings.merge_cleanup }, ...(executionProfile.operational_decision ? { decision: executionProfile.operational_decision, policy_source: policySource } : {}) }, null, 2)}\n` : undefined;
|
|
97
|
+
if (executionProfile && operationalBytes)
|
|
98
|
+
executionProfile.operational_contract = { path: "operational-contract.json", sha256: crypto.createHash("sha256").update(operationalBytes).digest("hex"), ...(policySource ? { policy_source: policySource } : {}), binding };
|
|
83
99
|
const index = {
|
|
84
100
|
schema_id: runSchemaId,
|
|
85
101
|
run_id: runId,
|
|
@@ -144,6 +160,10 @@ export function startFlowRun(context, input, prepared = prepareFlowRunStart(cont
|
|
|
144
160
|
};
|
|
145
161
|
ensureDir(path.dirname(runtimePath));
|
|
146
162
|
ensureDir(runHome);
|
|
163
|
+
if (policyBytes && policySource)
|
|
164
|
+
fs.writeFileSync(path.join(runHome, policySource.path), policyBytes, { flag: "wx" });
|
|
165
|
+
if (operationalBytes)
|
|
166
|
+
fs.writeFileSync(path.join(runHome, "operational-contract.json"), operationalBytes, { flag: "wx" });
|
|
147
167
|
if (routingInput)
|
|
148
168
|
fs.writeFileSync(path.join(runHome, "execution-routing.input.json"), routingInput.bytes, { flag: "wx", mode: 0o600 });
|
|
149
169
|
bindCurrentEngineToRun(context, { projectRoot, runId, runRoot: runHome });
|
|
@@ -193,13 +213,16 @@ export function startFlowRun(context, input, prepared = prepareFlowRunStart(cont
|
|
|
193
213
|
});
|
|
194
214
|
return { ok: true, run: flowRunSummary(requireRunById(context, project.id, runId)), index };
|
|
195
215
|
}
|
|
196
|
-
function snapshotVnextExecutionProfile(projectRoot, ddFlowHome, routingInput) {
|
|
197
|
-
const profile =
|
|
216
|
+
function snapshotVnextExecutionProfile(projectRoot, ddFlowHome, routingInput, now = new Date().toISOString()) {
|
|
217
|
+
const { profile, bytes } = loadVnextExecutionProfileSnapshot(projectRoot);
|
|
218
|
+
if (profile.operational_decision)
|
|
219
|
+
assertOperationalDecisionSource(profile.operational_decision, projectRoot, now);
|
|
198
220
|
const sourcePath = executionProfilePath(projectRoot);
|
|
199
221
|
const frozen = freezeRunExecutionRouting(ddFlowHome, {
|
|
200
222
|
schema_id: "dd-flow/run-execution-profile@1",
|
|
201
223
|
source_path: sourcePath,
|
|
202
|
-
source_checksum: crypto.createHash("sha256").update(
|
|
224
|
+
source_checksum: crypto.createHash("sha256").update(bytes).digest("hex"),
|
|
225
|
+
...(profile.operational_decision ? { operational_decision: profile.operational_decision } : {}),
|
|
203
226
|
settings: {
|
|
204
227
|
stage_session_mode: profile.stage_session_mode,
|
|
205
228
|
plan_review_mode: profile.plan_review_mode,
|
|
@@ -659,6 +682,16 @@ export function prepareFlowRunStageCompletion(context, input) {
|
|
|
659
682
|
throw new AppError("validation", "Stage data SHA-256 is invalid", 2);
|
|
660
683
|
return { project, run, index, stage, status, existing, dataSha256, git: input.git ?? gitFacts(run.workspace_root), generation: ["done", "failed", "skipped"].includes(status) ? assertStageSettlementAllowed(context, project.id, run.id, stage) : assertRunMutationAllowed(context, project.id, run.id) };
|
|
661
684
|
}
|
|
685
|
+
function stageUsageProjection(context, run, stage, startedAt) {
|
|
686
|
+
if (!startedAt)
|
|
687
|
+
return { status: "unavailable", input_tokens: null, cached_input_tokens: null, output_tokens: null, total_tokens: null, session_count: 0, observed_at: context.now() };
|
|
688
|
+
const value = recalculateRunUsage(context, { projectId: run.project_id, runId: run.id, stage: { name: stage, startedAt, completedAt: context.now() } });
|
|
689
|
+
const totals = value.totals, coverage = value.coverage;
|
|
690
|
+
const count = (name) => typeof coverage?.[name] === "number" ? coverage[name] : 0;
|
|
691
|
+
const measured = count("measured"), other = Object.values(coverage ?? {}).reduce((total, item) => total + (typeof item === "number" ? item : 0), 0) - measured;
|
|
692
|
+
const tokens = (name) => typeof totals?.[name] === "number" ? totals[name] : null;
|
|
693
|
+
return { status: measured === 0 ? "unavailable" : other > 0 || ["input_tokens", "cache_read_input_tokens", "output_tokens", "total_tokens"].some(name => tokens(name) === null) ? "partial" : "measured", input_tokens: tokens("input_tokens"), cached_input_tokens: tokens("cache_read_input_tokens"), output_tokens: tokens("output_tokens"), total_tokens: tokens("total_tokens"), session_count: Array.isArray(value.sessions) ? value.sessions.length : 0, observed_at: typeof value.observed_at === "string" ? value.observed_at : context.now() };
|
|
694
|
+
}
|
|
662
695
|
export function completeFlowRunStage(context, input, prepared = prepareFlowRunStageCompletion(context, input)) {
|
|
663
696
|
const { project, run, stage, status, dataSha256, git, generation } = prepared;
|
|
664
697
|
const completed = context.db.writeTransaction(() => {
|
|
@@ -681,7 +714,7 @@ export function completeFlowRunStage(context, input, prepared = prepareFlowRunSt
|
|
|
681
714
|
...(dataSha256 ? { data_sha256: dataSha256 } : {}),
|
|
682
715
|
...(input.dataSchemaId ? { data_schema_id: input.dataSchemaId } : {}),
|
|
683
716
|
...(input.report ? { report: input.report } : {}),
|
|
684
|
-
|
|
717
|
+
usage: input.usage ?? stageUsageProjection(context, latest, stage, existing.started_at),
|
|
685
718
|
...(input.aliases && input.aliases.length > 0 ? { artifact_aliases: input.aliases } : {}),
|
|
686
719
|
updated_at: now,
|
|
687
720
|
...(status === "done" || status === "waiting_for_user" || status === "blocked" || status === "failed" || status === "skipped"
|
|
@@ -6,8 +6,8 @@ import { setTimeout as delay } from "node:timers/promises";
|
|
|
6
6
|
import { getResourceDatabase, withResourceReadSnapshot } from "../storage/database.js";
|
|
7
7
|
import { AppError } from "../shared/errors.js";
|
|
8
8
|
import { harnessKey } from "./harness-config.js";
|
|
9
|
-
import { managedProcessBinding, resourceHome, processIsAlive } from "./managed-processes.js";
|
|
10
|
-
import { recoveryRuntimeAdmission } from "./run-recovery-
|
|
9
|
+
import { managedProcessBinding, assertManagedProcessAdmission, resourceHome, processIsAlive } from "./managed-processes.js";
|
|
10
|
+
import { recoveryRuntimeAdmission } from "./run-recovery-guard.js";
|
|
11
11
|
import { prepareTextFile, prepareJsonFile, prepareError } from "../cli/input-preparation.js";
|
|
12
12
|
const digest = (value) => crypto.createHash("sha256").update(value).digest("hex");
|
|
13
13
|
export function validateRuntimeScopeManifest(value, scopeId) {
|
|
@@ -47,7 +47,7 @@ export function validateRetainedRuntimeScopeManifest(value, scopeId) {
|
|
|
47
47
|
/** Prepared native owners cannot send the first recovery Turn until the scope
|
|
48
48
|
* arbiter publishes their exact release. Fence disappearance alone is not ACK. */
|
|
49
49
|
export async function waitForRuntimeScopeRecovery(context, input, assertOwner) {
|
|
50
|
-
const db = getResourceDatabase(resourceHome(context));
|
|
50
|
+
const db = getResourceDatabase(resourceHome(context), context.resourceDatabaseMode, context.databaseDeadlineAt);
|
|
51
51
|
for (;;) {
|
|
52
52
|
assertOwner();
|
|
53
53
|
const snapshot = db.readSnapshot(() => {
|
|
@@ -110,7 +110,7 @@ export function requestRuntimeScopeControl(context, scopeId, requestId, mode, ma
|
|
|
110
110
|
if (prepared.receipt)
|
|
111
111
|
return prepared.receipt;
|
|
112
112
|
const declaration = prepared.manifest;
|
|
113
|
-
const db = getResourceDatabase(resourceHome(context));
|
|
113
|
+
const db = getResourceDatabase(resourceHome(context), context.resourceDatabaseMode, context.databaseDeadlineAt);
|
|
114
114
|
const requestKey = digest(JSON.stringify([scopeId, requestId]));
|
|
115
115
|
return db.writeTransaction(() => {
|
|
116
116
|
const prior = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-request' AND resource_key = ?", [requestKey]);
|
|
@@ -145,7 +145,7 @@ export function requestRuntimeScopeControl(context, scopeId, requestId, mode, ma
|
|
|
145
145
|
});
|
|
146
146
|
}
|
|
147
147
|
function assertScopeAdmission(context, scopeId) {
|
|
148
|
-
const db = getResourceDatabase(resourceHome(context));
|
|
148
|
+
const db = getResourceDatabase(resourceHome(context), context.resourceDatabaseMode, context.databaseDeadlineAt);
|
|
149
149
|
if (db.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]))
|
|
150
150
|
throw new AppError("runtime_scope_stopped", "This runtime scope no longer admits productive dispatch", 1, { scope_id: scopeId });
|
|
151
151
|
const control = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
|
|
@@ -166,7 +166,7 @@ export function scopeLocalProcessRole(record) {
|
|
|
166
166
|
export function admitRuntimeCheck(context, id, leaseToken) {
|
|
167
167
|
if (typeof id !== "string" || !id.trim() || typeof leaseToken !== "string" || !leaseToken.trim())
|
|
168
168
|
throw new AppError("validation", "Check admission requires a process ID and lease token", 2);
|
|
169
|
-
const db = getResourceDatabase(resourceHome(context), "read_existing");
|
|
169
|
+
const db = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
|
|
170
170
|
try {
|
|
171
171
|
// One statement observes the owner and both admission fences together,
|
|
172
172
|
// without taking a writer lock or initializing an absent registry.
|
|
@@ -176,6 +176,7 @@ export function admitRuntimeCheck(context, id, leaseToken) {
|
|
|
176
176
|
FROM managed_processes p WHERE p.id = ?`, [id]);
|
|
177
177
|
if (!record || record.lease_token !== leaseToken || !["eval-baseline", "eval-observer", "check"].includes(record.kind) || record.state !== "running" || !processIsAlive(record))
|
|
178
178
|
throw new AppError("process_ownership_unknown", "Local process requires its confirmed live owner", 1);
|
|
179
|
+
assertManagedProcessAdmission(context, record);
|
|
179
180
|
let budget;
|
|
180
181
|
try {
|
|
181
182
|
const metadata = JSON.parse(record.metadata_json);
|
|
@@ -231,7 +232,7 @@ export function prepareRuntimeScopeFence(scopeId, requestId) {
|
|
|
231
232
|
}
|
|
232
233
|
export function fenceRuntimeBudget(context, scopeId, requestId) {
|
|
233
234
|
prepareRuntimeScopeFence(scopeId, requestId);
|
|
234
|
-
const db = getResourceDatabase(resourceHome(context));
|
|
235
|
+
const db = getResourceDatabase(resourceHome(context), context.resourceDatabaseMode, context.databaseDeadlineAt);
|
|
235
236
|
return db.writeTransaction(() => {
|
|
236
237
|
const existing = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]);
|
|
237
238
|
const now = context.now();
|
|
@@ -277,19 +278,32 @@ export function parseRuntimeBudget(raw) {
|
|
|
277
278
|
export function reserveProviderTurn(context, record, input) {
|
|
278
279
|
const metadata = JSON.parse(record.metadata_json);
|
|
279
280
|
const budget = parseRuntimeBudget(metadata.budget === undefined ? undefined : JSON.stringify(metadata.budget));
|
|
280
|
-
if (!budget)
|
|
281
|
-
|
|
281
|
+
if (!budget) {
|
|
282
|
+
const reader = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
|
|
283
|
+
try {
|
|
284
|
+
const current = reader.get("SELECT * FROM managed_processes WHERE id=?", [record.id]);
|
|
285
|
+
if (!current || current.state !== "running" || !processIsAlive(current) || managedProcessBinding(current, resourceHome(context)).registration_sha256 !== managedProcessBinding(record, resourceHome(context)).registration_sha256)
|
|
286
|
+
throw new AppError("process_ownership_unknown", "Owner changed before unbudgeted admission", 1);
|
|
287
|
+
assertManagedProcessAdmission(context, current);
|
|
288
|
+
return { admitted: true };
|
|
289
|
+
}
|
|
290
|
+
finally {
|
|
291
|
+
reader.close?.();
|
|
292
|
+
}
|
|
293
|
+
}
|
|
282
294
|
if (!metadata.harness || !metadata.state_dir || !input.operationId || !["session.create", "session.prompt", "session.start", "session.fork", "session.resume"].includes(input.operation))
|
|
283
295
|
throw new AppError("runtime_budget_invalid", "Budget admission requires a frozen harness and productive operation", 2);
|
|
284
296
|
const harness = harnessKey(metadata.harness), limit = budget.per_harness[harness] ?? 1;
|
|
285
|
-
const db = getResourceDatabase(resourceHome(context));
|
|
297
|
+
const db = getResourceDatabase(resourceHome(context), context.resourceDatabaseMode, context.databaseDeadlineAt);
|
|
286
298
|
const key = digest(JSON.stringify([record.id, input.operationId]));
|
|
287
299
|
const policy = JSON.stringify(budget), now = context.now();
|
|
288
|
-
const
|
|
300
|
+
const ownerAlive = processIsAlive(record);
|
|
301
|
+
const recoveryOwnerAlive = input.operation === "session.resume" && ownerAlive;
|
|
289
302
|
return db.writeTransaction(() => {
|
|
290
303
|
const current = db.get("SELECT * FROM managed_processes WHERE id = ?", [record.id]);
|
|
291
|
-
if (current?.lease_token !== record.lease_token || current.state !== "running" ||
|
|
304
|
+
if (current?.lease_token !== record.lease_token || current.state !== "running" || !ownerAlive || managedProcessBinding(current, resourceHome(context)).registration_sha256 !== managedProcessBinding(record, resourceHome(context)).registration_sha256)
|
|
292
305
|
throw new AppError("process_ownership_unknown", "Budget owner changed before admission", 1);
|
|
306
|
+
assertManagedProcessAdmission(context, current);
|
|
293
307
|
try {
|
|
294
308
|
assertScopeAdmission(context, budget.scope_id);
|
|
295
309
|
}
|
|
@@ -306,6 +320,9 @@ export function reserveProviderTurn(context, record, input) {
|
|
|
306
320
|
if (prior) {
|
|
307
321
|
if (prior.lease_token !== record.lease_token)
|
|
308
322
|
throw new AppError("process_ownership_unknown", "Retained budget claim belongs to another physical owner", 1);
|
|
323
|
+
const retained = JSON.parse(prior.metadata_json);
|
|
324
|
+
if (retained.operation_id !== input.operationId || retained.operation !== input.operation || retained.session_id !== (input.sessionId ?? null) || retained.state_dir !== metadata.state_dir || retained.daemon_id !== (input.daemonId ?? null) || retained.prompt_sha256 !== (input.promptSha256 ?? null) || retained.registration_sha256 !== managedProcessBinding(record, resourceHome(context)).registration_sha256)
|
|
325
|
+
throw new AppError("runtime_budget_conflict", "Retained admission fingerprint differs; reconcile rather than replay", 1, { operation_id: input.operationId });
|
|
309
326
|
return { admitted: true, budget_scope: budget.scope_id, reused: true };
|
|
310
327
|
}
|
|
311
328
|
const activeTurns = db.all("SELECT process_id, metadata_json FROM managed_resources WHERE resource_kind = 'provider-turn' AND json_extract(metadata_json, '$.scope_id') = ? AND json_extract(metadata_json, '$.harness') = ? ORDER BY created_at, resource_key", [budget.scope_id, harness]);
|
|
@@ -317,7 +334,7 @@ export function reserveProviderTurn(context, record, input) {
|
|
|
317
334
|
});
|
|
318
335
|
return { admitted: false, reason: "provider_capacity", budget_scope: budget.scope_id, harness, limit, active, blocking_turns };
|
|
319
336
|
}
|
|
320
|
-
db.run("INSERT INTO managed_resources (resource_kind, resource_key, owner_id, lease_token, lease_expires_at, process_id, metadata_json, created_at, updated_at) VALUES ('provider-turn', ?, ?, ?, ?, ?, ?, ?, ?)", [key, record.owner_id, record.lease_token, record.lease_expires_at, record.id, JSON.stringify({ scope_id: budget.scope_id, harness, operation_id: input.operationId, operation: input.operation, session_id: input.sessionId ?? null, state_dir: metadata.state_dir }), now, now]);
|
|
337
|
+
db.run("INSERT INTO managed_resources (resource_kind, resource_key, owner_id, lease_token, lease_expires_at, process_id, metadata_json, created_at, updated_at) VALUES ('provider-turn', ?, ?, ?, ?, ?, ?, ?, ?)", [key, record.owner_id, record.lease_token, record.lease_expires_at, record.id, JSON.stringify({ scope_id: budget.scope_id, harness, operation_id: input.operationId, operation: input.operation, session_id: input.sessionId ?? null, state_dir: metadata.state_dir, daemon_id: input.daemonId ?? null, prompt_sha256: input.promptSha256 ?? null, registration_sha256: managedProcessBinding(record, resourceHome(context)).registration_sha256 }), now, now]);
|
|
321
338
|
return { admitted: true, budget_scope: budget.scope_id, harness, limit, active: active + 1 };
|
|
322
339
|
});
|
|
323
340
|
}
|
|
@@ -325,7 +342,7 @@ function scopeRecoveryLoadAllowed(context, scopeId, record, input, physicalOwner
|
|
|
325
342
|
const metadata = JSON.parse(record.metadata_json);
|
|
326
343
|
if (input.operation !== "session.resume" || !input.daemonId || !metadata.state_dir || metadata.dd_flow_home !== context.ddFlowHome || !physicalOwnerAlive)
|
|
327
344
|
return false;
|
|
328
|
-
const db = getResourceDatabase(resourceHome(context));
|
|
345
|
+
const db = getResourceDatabase(resourceHome(context), context.resourceDatabaseMode, context.databaseDeadlineAt);
|
|
329
346
|
const control = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
|
|
330
347
|
const row = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-resume' AND resource_key = ?", [scopeId]);
|
|
331
348
|
const intent = row ? JSON.parse(row.metadata_json) : null;
|
|
@@ -405,7 +422,7 @@ export function releaseProviderTurns(context, record, input, prepared = prepareP
|
|
|
405
422
|
throw new AppError("runtime_budget_evidence_invalid", "Only a completed Session creation can release without a tree observation", 1);
|
|
406
423
|
operationIds = [input.operationId];
|
|
407
424
|
}
|
|
408
|
-
const db = getResourceDatabase(resourceHome(context));
|
|
425
|
+
const db = getResourceDatabase(resourceHome(context), context.resourceDatabaseMode, context.databaseDeadlineAt);
|
|
409
426
|
let released = 0;
|
|
410
427
|
for (const operationId of operationIds)
|
|
411
428
|
released += db.run("DELETE FROM managed_resources WHERE resource_kind = 'provider-turn' AND process_id = ? AND lease_token = ? AND json_extract(metadata_json, '$.operation_id') = ? AND (? IS NULL OR json_extract(metadata_json, '$.session_id') = ?)", [record.id, record.lease_token, operationId, sessionId, sessionId]).changes;
|
|
@@ -21,6 +21,7 @@ const scopeInventoryIdentity = (records) => JSON.stringify(records.map(record =>
|
|
|
21
21
|
* allowed while publishing: RUN resume and late registration cannot interleave.
|
|
22
22
|
*/
|
|
23
23
|
export function retainScopeDrain(context, control, processes, runs, native, probes) {
|
|
24
|
+
context.assertControlOwner?.();
|
|
24
25
|
const registry = getResourceDatabase(resourceHome(context));
|
|
25
26
|
const targets = new Map();
|
|
26
27
|
const heldTargets = new Set();
|
|
@@ -111,6 +112,7 @@ export function retainScopeDrain(context, control, processes, runs, native, prob
|
|
|
111
112
|
heldTargets.add(target);
|
|
112
113
|
}
|
|
113
114
|
const current = runtimeBudgetStatus(context, control.scope_id);
|
|
115
|
+
context.assertControlOwner?.();
|
|
114
116
|
if (current.fence || current.control?.generation !== control.generation
|
|
115
117
|
|| scopeInventoryIdentity(processes) !== scopeInventoryIdentity(current.processes) || current.provider_turns.length
|
|
116
118
|
|| JSON.stringify(current.manifest) !== JSON.stringify(latest.manifest))
|
|
@@ -173,6 +175,7 @@ export async function controlRuntimeScopeLocalProbes(context, scopeId, generatio
|
|
|
173
175
|
return await Promise.all(inventory.filter(record => ["eval-baseline", "eval-observer"].includes(record.kind)).map(async (record) => {
|
|
174
176
|
try {
|
|
175
177
|
const validate = () => {
|
|
178
|
+
context.assertControlOwner?.();
|
|
176
179
|
const head = registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
|
|
177
180
|
if (!head || JSON.parse(head.metadata_json).generation !== generation || JSON.parse(head.metadata_json).requested_mode !== mode || registry.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]))
|
|
178
181
|
throw new AppError("runtime_scope_control_superseded", "Local probe control belongs to an old generation", 1);
|
|
@@ -210,6 +213,7 @@ export async function controlRuntimeScopeLocalProbes(context, scopeId, generatio
|
|
|
210
213
|
export async function controlRuntimeScopeNativeOwners(context, scopeId, generation, action = "observe") {
|
|
211
214
|
const registry = getResourceDatabase(resourceHome(context));
|
|
212
215
|
const assertCurrent = () => {
|
|
216
|
+
context.assertControlOwner?.();
|
|
213
217
|
const row = registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
|
|
214
218
|
if (!row || JSON.parse(row.metadata_json).generation !== generation || action === "stop" && JSON.parse(row.metadata_json).requested_mode !== "stop" || registry.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]))
|
|
215
219
|
throw new AppError("runtime_scope_control_superseded", "Scope observation belongs to an old generation", 1);
|
|
@@ -323,6 +327,7 @@ export async function controlRuntimeScopeNativeOwners(context, scopeId, generati
|
|
|
323
327
|
}
|
|
324
328
|
/** Scope routing only: existing RUN workers own physical drain and capture. */
|
|
325
329
|
export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode, reconcileId) {
|
|
330
|
+
context.assertControlOwner?.();
|
|
326
331
|
const control = requestRuntimeScopeControl(context, scopeId, requestId, mode);
|
|
327
332
|
const registry = getResourceDatabase(resourceHome(context));
|
|
328
333
|
const inventory = runtimeBudgetStatus(context, scopeId).processes;
|
|
@@ -348,6 +353,7 @@ export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode,
|
|
|
348
353
|
// Lock order is resource registry then RUN database. No provider or await
|
|
349
354
|
// occurs here; a newer scope request cannot interleave RUN intent creation.
|
|
350
355
|
withDeferredPublications(() => registry.writeTransaction(() => {
|
|
356
|
+
context.assertControlOwner?.();
|
|
351
357
|
const current = registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
|
|
352
358
|
if (!current || JSON.parse(current.metadata_json).generation !== control.generation || registry.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]))
|
|
353
359
|
throw new AppError("runtime_scope_control_superseded", "Scope request no longer owns admission", 1);
|
|
@@ -363,6 +369,7 @@ export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode,
|
|
|
363
369
|
recordRunControlIntent(target, input);
|
|
364
370
|
}));
|
|
365
371
|
seen.add(key);
|
|
372
|
+
context.assertControlOwner?.();
|
|
366
373
|
const receipt = await requestRunControl(target, { ...input, waitMs: 0, ...(reconcileId ? { reconcileId: crypto.createHash("sha256").update(JSON.stringify([scopeId, control.generation, reconcileId])).digest("hex") } : {}) });
|
|
367
374
|
runs.push({ runtime_home: metadata.dd_flow_home, project_id: record.project_id, run_id: record.run_id, request_id: input.requestId, receipt });
|
|
368
375
|
}
|
|
@@ -379,6 +386,7 @@ export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode,
|
|
|
379
386
|
controlRuntimeScopeNativeOwners(context, scopeId, control.generation, mode === "stop" ? "stop" : "observe"),
|
|
380
387
|
controlRuntimeScopeLocalProbes(context, scopeId, control.generation, mode)
|
|
381
388
|
]);
|
|
389
|
+
context.assertControlOwner?.();
|
|
382
390
|
for (const run of runs) {
|
|
383
391
|
if (run.pending || typeof run.runtime_home !== "string" || typeof run.run_id !== "string")
|
|
384
392
|
continue;
|