@deksden-com/dd-flow-cli 0.9.0-beta.8 → 0.9.0-beta.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +402 -0
- package/README.md +65 -0
- package/dist/build-info.json +8 -8
- package/dist/cli/command-inputs.js +300 -0
- package/dist/cli/help.js +81 -9
- package/dist/cli/hook-ingress.js +82 -0
- package/dist/cli/input-preparation.js +115 -0
- package/dist/cli/run-cli.js +1344 -376
- package/dist/cli.js +6 -2
- package/dist/harness-runtime/bin/dd-agy.mjs +37 -0
- package/dist/harness-runtime/bin/dd-codex.mjs +25 -0
- package/dist/harness-runtime/bin/dd-droid.mjs +33 -0
- package/dist/harness-runtime/bin/dd-grok.mjs +30 -0
- package/dist/harness-runtime/bin/dd-opencode.mjs +21 -0
- package/dist/harness-runtime/bin/dd-zcode.mjs +83 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +215 -0
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +423 -0
- package/dist/harness-runtime/lib/dd-agy.mjs +59 -0
- package/dist/harness-runtime/lib/dd-codex-daemon.d.mts +1 -0
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +146 -0
- package/dist/harness-runtime/lib/dd-codex.mjs +562 -0
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +125 -0
- package/dist/harness-runtime/lib/dd-droid.mjs +471 -0
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +274 -0
- package/dist/harness-runtime/lib/dd-grok.mjs +174 -0
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +198 -0
- package/dist/harness-runtime/lib/dd-opencode.mjs +98 -0
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +644 -0
- package/dist/harness-runtime/lib/dd-zcode.mjs +910 -0
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +10 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +130 -0
- package/dist/harness-runtime/lib/dispatch-fence.mjs +14 -0
- package/dist/harness-runtime/lib/driver-recovery.mjs +129 -0
- package/dist/harness-runtime/lib/droid-observation.mjs +66 -0
- package/dist/harness-runtime/lib/managed-daemon.mjs +260 -0
- package/dist/harness-runtime/lib/model-observations.mjs +77 -0
- package/dist/harness-runtime/lib/native-hook-command.d.mts +1 -0
- package/dist/harness-runtime/lib/native-hook-command.mjs +34 -0
- package/dist/harness-runtime/lib/observation-clock.mjs +36 -0
- package/dist/harness-runtime/lib/operation-context.mjs +5 -0
- package/dist/harness-runtime/lib/operation-errors.mjs +19 -0
- package/dist/harness-runtime/lib/process-json.mjs +70 -0
- package/dist/harness-runtime/lib/process-snapshot.mjs +10 -0
- package/dist/harness-runtime/lib/runner-events.mjs +200 -0
- package/dist/harness-runtime/lib/runner-lock.mjs +42 -0
- package/dist/harness-runtime/lib/session-settlement.mjs +30 -0
- package/dist/harness-runtime/lib/tool-observations.d.mts +17 -0
- package/dist/harness-runtime/lib/tool-observations.mjs +153 -0
- package/dist/schemas/agent-profile.schema.json +1 -1
- package/dist/schemas/code-review-result.schema.json +2 -2
- package/dist/schemas/code-work-batch.schema.json +4 -3
- package/dist/schemas/code-work-result.schema.json +4 -4
- package/dist/schemas/plan-review-result.schema.json +2 -2
- package/dist/schemas/run-control-receipt.schema.json +99 -0
- package/dist/schemas/run-control-request.schema.json +36 -0
- package/dist/schemas/vnext-protocol-plan.schema.json +2 -2
- package/dist/services/canon.js +11 -8
- package/dist/services/cleanup.js +57 -13
- package/dist/services/cli-operation-classifier.js +22 -24
- package/dist/services/code-checks.js +543 -96
- package/dist/services/codex-hook-delivery.js +28 -0
- package/dist/services/config.js +11 -8
- package/dist/services/controller-fanout.js +116 -0
- package/dist/services/dashboard.js +103 -31
- package/dist/services/engines.js +121 -76
- package/dist/services/eval-snapshots.js +750 -73
- package/dist/services/execution-policy.js +170 -0
- package/dist/services/external-work-launch.js +110 -0
- package/dist/services/harness-adapter.js +147 -24
- package/dist/services/harness-config.js +2 -0
- package/dist/services/hooks.js +575 -293
- package/dist/services/lanes.js +61 -53
- package/dist/services/lifecycle-command.js +87 -7
- package/dist/services/lifecycle-invocations.js +835 -0
- package/dist/services/managed-daemon-binding.js +38 -0
- package/dist/services/managed-processes.js +283 -27
- package/dist/services/merge-queue.js +179 -104
- package/dist/services/merge-server.js +73 -28
- package/dist/services/migrations.js +14 -9
- package/dist/services/native-daemon-history.js +49 -0
- package/dist/services/native-session-control.js +39 -0
- package/dist/services/plan-runtime.js +19 -8
- package/dist/services/plans.js +28 -25
- package/dist/services/projects.js +18 -5
- package/dist/services/prompts.js +23 -16
- package/dist/services/protocols.js +64 -20
- package/dist/services/recovery-observation-budget.js +61 -0
- package/dist/services/recovery-snapshot-database.js +107 -0
- package/dist/services/run-control-receipt.js +56 -0
- package/dist/services/run-control-worker.js +363 -0
- package/dist/services/run-control.js +876 -0
- package/dist/services/run-controller-adapter.js +222 -0
- package/dist/services/run-controller-capture.js +134 -0
- package/dist/services/run-controller-process.js +189 -0
- package/dist/services/run-controller-recovery.js +222 -0
- package/dist/services/run-controller-state.js +36 -0
- package/dist/services/run-controller.js +779 -0
- package/dist/services/run-engine-bindings.js +1 -1
- package/dist/services/run-fork.js +138 -0
- package/dist/services/run-observations.js +112 -0
- package/dist/services/run-recovery-runtime.js +69 -0
- package/dist/services/run-recovery.js +343 -0
- package/dist/services/runs.js +368 -74
- package/dist/services/runtime-budget.js +434 -0
- package/dist/services/runtime-scope-capture.js +50 -0
- package/dist/services/runtime-scope-control.js +450 -0
- package/dist/services/runtime-scope-resume.js +565 -0
- package/dist/services/runtime-scope-stop.js +99 -0
- package/dist/services/runtime-scope-worker.js +243 -0
- package/dist/services/runtime-service.js +97 -0
- package/dist/services/schema-validation.js +10 -8
- package/dist/services/sessions.js +35 -69
- package/dist/services/stage-blocker.js +17 -7
- package/dist/services/stage-context.js +44 -18
- package/dist/services/stage-lifecycle.js +99 -102
- package/dist/services/stage-pause.js +69 -28
- package/dist/services/usage.js +37 -93
- package/dist/services/vnext-code-review.js +189 -85
- package/dist/services/vnext-code.js +257 -243
- package/dist/services/vnext-execution-profile.js +5 -3
- package/dist/services/vnext-fanout.js +134 -19
- package/dist/services/vnext-merge.js +285 -102
- package/dist/services/vnext-plan-review.js +148 -94
- package/dist/services/vnext-plan.js +141 -72
- package/dist/services/vnext-protocolize.js +129 -56
- package/dist/services/vnext-specify.js +87 -64
- package/dist/services/work-registry.js +701 -164
- package/dist/services/workspace-bootstrap.js +76 -0
- package/dist/services/worktrees.js +85 -35
- package/dist/shared/entity-references.js +8 -0
- package/dist/shared/errors.js +12 -0
- package/dist/storage/database.js +444 -32
- package/dist/storage/paths.js +17 -4
- package/dist/storage/work-references.js +23 -0
- package/dist/storage/writer-contract.js +77 -0
- package/dist/storage/writer-migration.js +102 -0
- package/package.json +11 -4
- package/tools/repair-paused-run-status.mjs +59 -0
|
@@ -6,9 +6,8 @@ import { canTransition } from "../domain/contracts.js";
|
|
|
6
6
|
import { defaultFlowContract, flowContractForState, loadProjectFlowContract } from "../domain/flow-contract.js";
|
|
7
7
|
import { requireStage } from "../domain/validation.js";
|
|
8
8
|
import { AppError } from "../shared/errors.js";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { ensureDir, planJsonPath, resolveProjectRoot, runtimeProtocolDir, runtimeStateJsonPath } from "../storage/paths.js";
|
|
9
|
+
import { prepareJsonFile } from "../cli/input-preparation.js";
|
|
10
|
+
import { ensureDir, planJsonPath, resolveProjectRoot, resolveProtocolPlanPath, runtimeProtocolDir, runtimeStateJsonPath } from "../storage/paths.js";
|
|
12
11
|
import { appendAudit, getAuditEvents } from "./audit.js";
|
|
13
12
|
import { requireProjectByRoot } from "./projects.js";
|
|
14
13
|
import { ensureRuntimeProtocolFiles, readStateFile, writeState } from "../protocol/local-files.js";
|
|
@@ -17,14 +16,20 @@ import { activeCodexSessionBindingsForProject, activeFlowSessionBindingsForProje
|
|
|
17
16
|
import { buildProtocolFlowGuidance } from "./flow-guidance.js";
|
|
18
17
|
import { lifecycleIsTerminalFailure, lifecycleIsTerminalSuccess, normalizeProtocolLifecycle } from "./protocol-lifecycle.js";
|
|
19
18
|
import { getProtocolBranchContext } from "./branch-context.js";
|
|
20
|
-
export function
|
|
19
|
+
export function prepareProtocolRegistration(context, input) {
|
|
20
|
+
if (typeof input.handshakeId !== "string" || !input.handshakeId.trim() || /[/\\\0]/.test(input.handshakeId) || [".", ".."].includes(input.handshakeId))
|
|
21
|
+
throw new AppError("validation", "Protocol handshake must be a non-empty identifier, not a path", 2);
|
|
21
22
|
const projectRoot = resolveProjectRoot(input.projectRoot);
|
|
22
23
|
const project = requireProjectByRoot(context, projectRoot);
|
|
23
|
-
const now = context.now();
|
|
24
24
|
const workspacePath = input.workspacePath ? resolveProjectRoot(input.workspacePath) : undefined;
|
|
25
25
|
const idRoot = workspacePath ?? projectRoot;
|
|
26
26
|
const protocolId = inferProtocolId(idRoot, input.handshakeId);
|
|
27
27
|
const runtimeDir = runtimeProtocolDir(context.ddFlowHome, project.id, protocolId);
|
|
28
|
+
return { projectRoot, project, workspacePath, protocolId, runtimeDir };
|
|
29
|
+
}
|
|
30
|
+
export function registerProtocol(context, input) {
|
|
31
|
+
const { projectRoot, project, workspacePath, protocolId, runtimeDir } = prepareProtocolRegistration(context, input);
|
|
32
|
+
const now = context.now();
|
|
28
33
|
const state = ensureRuntimeProtocolFiles({
|
|
29
34
|
runtimeDir,
|
|
30
35
|
protocolId,
|
|
@@ -243,7 +248,22 @@ export function implementProtocol(context, input) {
|
|
|
243
248
|
}
|
|
244
249
|
};
|
|
245
250
|
}
|
|
246
|
-
export function
|
|
251
|
+
export function prepareProtocolTransition(context, input) {
|
|
252
|
+
const { value } = prepareJsonFile(input.jsonFile, "Protocol transition payload");
|
|
253
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
254
|
+
throw new AppError("validation", "Protocol transition payload must be an object", 2);
|
|
255
|
+
const payload = value;
|
|
256
|
+
for (const [key, field] of Object.entries(payload)) {
|
|
257
|
+
const valid = key === "next_action" ? typeof field === "string"
|
|
258
|
+
: ["route", "workspace"].includes(key) ? Boolean(field && typeof field === "object" && !Array.isArray(field))
|
|
259
|
+
: ["blockers", "active_def"].includes(key) && Array.isArray(field);
|
|
260
|
+
if (!valid)
|
|
261
|
+
throw new AppError("validation", `Unsupported field or invalid value in protocol transition: ${key}`, 2, { file: input.jsonFile, parameter: key });
|
|
262
|
+
}
|
|
263
|
+
protocolTransitionState(context, input);
|
|
264
|
+
return payload;
|
|
265
|
+
}
|
|
266
|
+
function protocolTransitionState(context, input) {
|
|
247
267
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
248
268
|
const protocol = requireProtocol(context, input.protocolId, project.id);
|
|
249
269
|
const state = readProtocolRuntimeState(context, protocol).state;
|
|
@@ -256,8 +276,10 @@ export function transitionProtocol(context, input) {
|
|
|
256
276
|
if (input.force && (!input.reason || input.reason.trim().length === 0)) {
|
|
257
277
|
throw new AppError("validation", "Forced transitions require --reason", 2);
|
|
258
278
|
}
|
|
259
|
-
|
|
260
|
-
|
|
279
|
+
return { project, protocol, state, flowContract, from, to };
|
|
280
|
+
}
|
|
281
|
+
export function transitionProtocol(context, input, payload = prepareProtocolTransition(context, input)) {
|
|
282
|
+
const { project, protocol, state, flowContract, from, to } = protocolTransitionState(context, input);
|
|
261
283
|
const nextState = {
|
|
262
284
|
...state,
|
|
263
285
|
stage: to,
|
|
@@ -416,7 +438,7 @@ export function syncProtocolFromRun(context, input) {
|
|
|
416
438
|
next_action: nextActionForSyncedTarget(targetStage),
|
|
417
439
|
updated_at: now
|
|
418
440
|
};
|
|
419
|
-
context.db.
|
|
441
|
+
context.db.beginWriteTransaction();
|
|
420
442
|
try {
|
|
421
443
|
persistProtocolState(context, protocol, nextState);
|
|
422
444
|
if (targetStage === "ready_for_merge") {
|
|
@@ -486,7 +508,7 @@ export function cancelProtocol(context, input) {
|
|
|
486
508
|
const now = context.now();
|
|
487
509
|
const worktree = context.db.get("SELECT worktree_path, status FROM worktree_records WHERE project_id = ? AND protocol_id = ?", [protocol.project_id, protocol.id]);
|
|
488
510
|
const worktreeRemoval = input.worktree === "remove" ? removeProtocolWorktreeBeforeCancel(protocol, state, worktree, input.force) : undefined;
|
|
489
|
-
context.db.
|
|
511
|
+
context.db.beginWriteTransaction();
|
|
490
512
|
try {
|
|
491
513
|
const nextState = {
|
|
492
514
|
...state,
|
|
@@ -579,7 +601,10 @@ export function requireProtocol(context, protocolId, projectId) {
|
|
|
579
601
|
candidates: matches.map((protocol) => ({ project_id: protocol.project_id, project_root: protocol.project_root }))
|
|
580
602
|
});
|
|
581
603
|
}
|
|
582
|
-
return matches[0];
|
|
604
|
+
return resolvedProtocolRecord(matches[0]);
|
|
605
|
+
}
|
|
606
|
+
function resolvedProtocolRecord(protocol) {
|
|
607
|
+
return { ...protocol, plan_path: resolveProtocolPlanPath(protocol.project_root, protocol.id, protocol.plan_path) };
|
|
583
608
|
}
|
|
584
609
|
function cancelQueueForProtocol(context, protocol, reason, force) {
|
|
585
610
|
const job = context.db.get("SELECT id, status, claimed_by_session_id FROM merge_queue WHERE project_id = ? AND protocol_id = ?", [protocol.project_id, protocol.id]);
|
|
@@ -733,9 +758,26 @@ function releaseRelatedMergeLocks(context, protocol, reason) {
|
|
|
733
758
|
return { ok: true, released: released.length, locks: released };
|
|
734
759
|
}
|
|
735
760
|
export function readProtocolRuntimeState(context, protocol) {
|
|
761
|
+
const expected = runtimeStateJsonPath(context.ddFlowHome, protocol.project_id, protocol.id);
|
|
762
|
+
if (protocol.state_path !== expected)
|
|
763
|
+
throw new AppError("protocol_state_migration_required", "Protocol runtime path requires explicit cleanup scan/apply before use", 1, { protocol_id: protocol.id, path: protocol.state_path, expected_path: expected });
|
|
764
|
+
return resolveProtocolRuntimeState(context, { ...protocol }, false);
|
|
765
|
+
}
|
|
766
|
+
/** Explicit cleanup migration; productive readers accept only the current path. */
|
|
767
|
+
export function repairProtocolRuntimeState(context, protocol) {
|
|
768
|
+
// Readers project the canonical plan path without migrating storage. Repair
|
|
769
|
+
// must start from the stored locations, not that already-normalized view.
|
|
770
|
+
const stored = context.db.get("SELECT * FROM protocols WHERE project_id = ? AND id = ?", [protocol.project_id, protocol.id]);
|
|
771
|
+
if (stored)
|
|
772
|
+
Object.assign(protocol, stored);
|
|
773
|
+
return resolveProtocolRuntimeState(context, protocol, true);
|
|
774
|
+
}
|
|
775
|
+
function resolveProtocolRuntimeState(context, protocol, repair) {
|
|
736
776
|
const diagnostics = [];
|
|
737
|
-
normalizeProtocolPlanPath(context, protocol, diagnostics);
|
|
777
|
+
normalizeProtocolPlanPath(context, protocol, diagnostics, repair);
|
|
738
778
|
try {
|
|
779
|
+
if (protocol.state_path !== runtimeStateJsonPath(context.ddFlowHome, protocol.project_id, protocol.id))
|
|
780
|
+
throw new AppError("protocol_state_migration_required", "Retired state files are not runtime authority", 1);
|
|
739
781
|
return { state: readStateFile(protocol.state_path, protocol.id), diagnostics };
|
|
740
782
|
}
|
|
741
783
|
catch {
|
|
@@ -750,7 +792,8 @@ export function readProtocolRuntimeState(context, protocol) {
|
|
|
750
792
|
recommended_action: "legacy protocol state_path was moved to stable runtime storage"
|
|
751
793
|
});
|
|
752
794
|
const stablePlanPath = planJsonPath(protocol.project_root, protocol.id);
|
|
753
|
-
|
|
795
|
+
if (repair && context.db.writable)
|
|
796
|
+
context.db.run("UPDATE protocols SET state_path = ?, plan_path = ? WHERE project_id = ? AND id = ?", [stableStatePath, stablePlanPath, protocol.project_id, protocol.id]);
|
|
754
797
|
protocol.state_path = stableStatePath;
|
|
755
798
|
protocol.plan_path = stablePlanPath;
|
|
756
799
|
return { state: readStateFile(stableStatePath, protocol.id), diagnostics };
|
|
@@ -783,7 +826,7 @@ export function readProtocolRuntimeState(context, protocol) {
|
|
|
783
826
|
flow_contract: flowContract,
|
|
784
827
|
updated_at: protocol.updated_at
|
|
785
828
|
};
|
|
786
|
-
if (!context.db.writable) {
|
|
829
|
+
if (!repair || !context.db.writable) {
|
|
787
830
|
return { state, diagnostics };
|
|
788
831
|
}
|
|
789
832
|
const stableStatePath = runtimeStateJsonPath(context.ddFlowHome, protocol.project_id, protocol.id);
|
|
@@ -802,9 +845,9 @@ export function readProtocolRuntimeState(context, protocol) {
|
|
|
802
845
|
});
|
|
803
846
|
return { state, diagnostics };
|
|
804
847
|
}
|
|
805
|
-
function normalizeProtocolPlanPath(context, protocol, diagnostics) {
|
|
806
|
-
const canonicalPlanPath =
|
|
807
|
-
if (protocol.plan_path === canonicalPlanPath
|
|
848
|
+
function normalizeProtocolPlanPath(context, protocol, diagnostics, repair) {
|
|
849
|
+
const canonicalPlanPath = resolveProtocolPlanPath(protocol.project_root, protocol.id, protocol.plan_path);
|
|
850
|
+
if (protocol.plan_path === canonicalPlanPath)
|
|
808
851
|
return;
|
|
809
852
|
diagnostics.push({
|
|
810
853
|
code: "protocol_plan_relocated",
|
|
@@ -814,7 +857,7 @@ function normalizeProtocolPlanPath(context, protocol, diagnostics) {
|
|
|
814
857
|
source: "project_canonical_plan",
|
|
815
858
|
recommended_action: "use the project canonical plan as the sole semantic plan source"
|
|
816
859
|
});
|
|
817
|
-
if (context.db.writable) {
|
|
860
|
+
if (repair && context.db.writable) {
|
|
818
861
|
context.db.run("UPDATE protocols SET plan_path = ?, updated_at = ? WHERE project_id = ? AND id = ?", [canonicalPlanPath, context.now(), protocol.project_id, protocol.id]);
|
|
819
862
|
}
|
|
820
863
|
protocol.plan_path = canonicalPlanPath;
|
|
@@ -875,10 +918,11 @@ export function persistProtocolState(context, protocol, state) {
|
|
|
875
918
|
}
|
|
876
919
|
function findProtocol(context, protocolId, projectId) {
|
|
877
920
|
if (projectId) {
|
|
878
|
-
|
|
921
|
+
const protocol = context.db.get("SELECT * FROM protocols WHERE project_id = ? AND id = ?", [projectId, protocolId]);
|
|
922
|
+
return protocol ? resolvedProtocolRecord(protocol) : undefined;
|
|
879
923
|
}
|
|
880
924
|
const matches = context.db.all("SELECT * FROM protocols WHERE id = ? ORDER BY project_id", [protocolId]);
|
|
881
|
-
return matches.length === 1 ? matches[0] : undefined;
|
|
925
|
+
return matches.length === 1 ? resolvedProtocolRecord(matches[0]) : undefined;
|
|
882
926
|
}
|
|
883
927
|
function protocolDocumentsForProject(projectRoot) {
|
|
884
928
|
const protocolRoot = path.join(projectRoot, ".memory-bank", "protocol");
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { AppError } from "../shared/errors.js";
|
|
3
|
+
export const recoveryObservationBudgetMs = 120_000;
|
|
4
|
+
const activeObservation = new AsyncLocalStorage();
|
|
5
|
+
export const withRecoveryObservation = (budget, action) => activeObservation.run(budget, action);
|
|
6
|
+
export const recoveryRpcTimeout = (maximum) => {
|
|
7
|
+
const observation = activeObservation.getStore();
|
|
8
|
+
// An unbounded productive transport wait is still bounded while recovery is
|
|
9
|
+
// actively reconciling. Outside recovery, its owner policy decides when to
|
|
10
|
+
// stop rather than a hidden adapter timer.
|
|
11
|
+
if (maximum === null)
|
|
12
|
+
return observation ? observation.timeout() : null;
|
|
13
|
+
return observation?.timeout(maximum ?? 30_000) ?? maximum ?? 30_000;
|
|
14
|
+
};
|
|
15
|
+
/** Call only while holding the existing owner-claim transaction. A replay of
|
|
16
|
+
* an explicit request never replenishes time, even after another request. */
|
|
17
|
+
export function renewRecoveryObservation(snapshot, requestId) {
|
|
18
|
+
if (typeof requestId !== "string" || !requestId.trim() || requestId.length > 200)
|
|
19
|
+
throw new AppError("usage", "Reconciliation requires a nonempty request ID of at most 200 characters", 2);
|
|
20
|
+
const previous = snapshot.recovery_observation;
|
|
21
|
+
const ids = previous?.reconciliation_ids ?? [];
|
|
22
|
+
if (!Array.isArray(ids) || ids.some(id => typeof id !== "string"))
|
|
23
|
+
throw new AppError("recovery_observation_invalid", "Retained reconciliation identities are malformed", 1);
|
|
24
|
+
if (ids.includes(requestId))
|
|
25
|
+
return snapshot;
|
|
26
|
+
return { ...snapshot, recovery_observation: { budget_ms: recoveryObservationBudgetMs, remaining_ms: recoveryObservationBudgetMs, reconciliation_ids: [...ids, requestId], poll_index: 0 } };
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Measures only time this owner can actually observe. Large scheduler gaps
|
|
30
|
+
* reset the sample instead of turning host sleep into evidence about a remote
|
|
31
|
+
* process. Persisting the remaining budget makes a recovered owner continue
|
|
32
|
+
* reconciliation rather than silently receiving a new unlimited wait.
|
|
33
|
+
*/
|
|
34
|
+
export function recoveryObservationBudget(saved, monotonic = () => performance.now()) {
|
|
35
|
+
const prior = saved && typeof saved === "object" ? saved.recovery_observation : null;
|
|
36
|
+
let remaining = typeof prior?.remaining_ms === "number" && Number.isFinite(prior.remaining_ms)
|
|
37
|
+
? Math.max(0, Math.min(recoveryObservationBudgetMs, prior.remaining_ms))
|
|
38
|
+
: prior == null ? recoveryObservationBudgetMs : 0;
|
|
39
|
+
let last = monotonic();
|
|
40
|
+
let pollIndex = Number.isSafeInteger(prior?.poll_index) && prior.poll_index >= 0 ? prior.poll_index : 0;
|
|
41
|
+
const observation = {
|
|
42
|
+
exhausted: () => {
|
|
43
|
+
const current = monotonic();
|
|
44
|
+
const elapsed = current - last;
|
|
45
|
+
last = current;
|
|
46
|
+
// A gap means this process was not observing; it never proves progress
|
|
47
|
+
// or death and therefore does not consume the reconciliation budget.
|
|
48
|
+
if (elapsed >= 0 && elapsed <= 60_000)
|
|
49
|
+
remaining = Math.max(0, remaining - elapsed);
|
|
50
|
+
return remaining === 0;
|
|
51
|
+
},
|
|
52
|
+
state: () => ({ budget_ms: recoveryObservationBudgetMs, remaining_ms: Math.ceil(remaining), reconciliation_ids: prior?.reconciliation_ids ?? [], poll_index: pollIndex }),
|
|
53
|
+
timeout: (maximum = 30_000) => {
|
|
54
|
+
if (observation.exhausted())
|
|
55
|
+
throw new AppError("recovery_observation_budget_exhausted", "Recovery observation budget exhausted; explicit reconciliation is required", 1);
|
|
56
|
+
return Math.max(1, Math.min(maximum, Math.ceil(remaining)));
|
|
57
|
+
},
|
|
58
|
+
nextDelay: () => Math.min([1000, 2000, 5000, 10000][Math.min(pollIndex++, 3)], Math.ceil(remaining)),
|
|
59
|
+
};
|
|
60
|
+
return observation;
|
|
61
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { AppError } from "../shared/errors.js";
|
|
5
|
+
import { registerStoreWriter } from "../storage/writer-contract.js";
|
|
6
|
+
import { managedProcessRuntimeHome } from "./managed-processes.js";
|
|
7
|
+
/** Export one RUN's relational evidence, never mutate the live authority. */
|
|
8
|
+
export function captureRecoveryDatabase(source, output, projectId, runId, protocolIds = []) {
|
|
9
|
+
source.exec(`VACUUM INTO '${output.replace(/'/g, "''")}'`);
|
|
10
|
+
const db = new DatabaseSync(output);
|
|
11
|
+
let complete = false;
|
|
12
|
+
try {
|
|
13
|
+
registerStoreWriter(db);
|
|
14
|
+
db.exec("PRAGMA foreign_keys = OFF; PRAGMA secure_delete = ON; BEGIN IMMEDIATE");
|
|
15
|
+
db.exec("CREATE TEMP TABLE capture_scope (project_id TEXT, run_id TEXT)");
|
|
16
|
+
db.prepare("INSERT INTO capture_scope VALUES (?, ?)").run(projectId, runId);
|
|
17
|
+
db.exec("CREATE TEMP TABLE capture_protocols (id TEXT)");
|
|
18
|
+
for (const id of protocolIds)
|
|
19
|
+
db.prepare("INSERT INTO capture_protocols VALUES (?)").run(id);
|
|
20
|
+
if (!db.prepare("SELECT 1 FROM runs WHERE (project_id, id) IN (SELECT * FROM capture_scope)").get())
|
|
21
|
+
throw new AppError("snapshot_run_missing", "Recovery export requires the selected RUN", 1);
|
|
22
|
+
const ownRun = "(project_id, run_id) IN (SELECT * FROM capture_scope)";
|
|
23
|
+
const ownProject = "project_id IN (SELECT project_id FROM capture_scope)";
|
|
24
|
+
const works = `SELECT work_id FROM works WHERE ${ownRun}`;
|
|
25
|
+
if (db.prepare(`SELECT 1 FROM sessions WHERE ${ownProject} AND run_id IS NOT NULL AND run_id NOT IN (SELECT run_id FROM capture_scope) AND session_id IN (SELECT session_id FROM work_sessions WHERE work_id IN (${works})) LIMIT 1`).get())
|
|
26
|
+
throw new AppError("snapshot_scope_conflict", "Selected Work refers to a Session currently bound to another RUN", 1);
|
|
27
|
+
const sessions = `SELECT session_id FROM sessions WHERE ${ownProject} AND (run_id IN (SELECT run_id FROM capture_scope) OR session_id IN (SELECT session_id FROM work_sessions WHERE work_id IN (${works})))`;
|
|
28
|
+
const nativeSessions = `SELECT harness, provider_session_id FROM sessions WHERE ${ownProject} AND session_id IN (${sessions})`;
|
|
29
|
+
const invocations = "json_extract(scope_json, '$.runId') IN (SELECT run_id FROM capture_scope) AND json_extract(scope_json, '$.projectRoot') IN (SELECT root FROM projects WHERE id IN (SELECT project_id FROM capture_scope))";
|
|
30
|
+
const invocationReceipts = db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'lifecycle_invocations'").get()
|
|
31
|
+
? ` OR event_key IN (SELECT event_key FROM lifecycle_invocations WHERE ${invocations})` : "";
|
|
32
|
+
const protocols = `SELECT id FROM capture_protocols UNION SELECT subject_id FROM runs WHERE (project_id, id) IN (SELECT * FROM capture_scope) UNION SELECT value FROM merge_requests, json_each(merge_requests.protocol_ids_json) WHERE ${ownRun} UNION SELECT protocol_id FROM flow_jobs WHERE ${ownRun}`;
|
|
33
|
+
const rules = {
|
|
34
|
+
dd_flow_writer_contract: "1",
|
|
35
|
+
dd_flow_writer_migrations: "0",
|
|
36
|
+
projects: "id IN (SELECT project_id FROM capture_scope)",
|
|
37
|
+
runs: "(project_id, id) IN (SELECT * FROM capture_scope)",
|
|
38
|
+
sessions: `${ownProject} AND session_id IN (${sessions})`,
|
|
39
|
+
work_sessions: `work_id IN (${works})`, work_launches: `work_id IN (${works})`,
|
|
40
|
+
work_recovery_segments: `work_id IN (${works})`,
|
|
41
|
+
run_recovery_bindings: `recovery_id IN (SELECT recovery_id FROM run_recovery_guards WHERE ${ownRun})`,
|
|
42
|
+
lifecycle_invocations: invocations,
|
|
43
|
+
hook_events: `${ownProject} AND (session_id IN (${sessions}) OR (harness, provider_session_id) IN (${nativeSessions})${invocationReceipts})`,
|
|
44
|
+
harness_usage_snapshots: `${ownProject} AND (harness, provider_session_id) IN (${nativeSessions})`,
|
|
45
|
+
protocols: `${ownProject} AND id IN (${protocols})`,
|
|
46
|
+
project_config: ownProject,
|
|
47
|
+
// Host installations, caches, queue owners and leases are not portable RUN authority.
|
|
48
|
+
runtime_config: "0", hook_installations: "0", codex_home_profiles: "0", codex_session_bindings: "0",
|
|
49
|
+
merge_servers: "0", merge_sessions: "0", merge_queue: "0", lanes: "0", lane_locks: "0", lane_waiters: "0",
|
|
50
|
+
sqlite_sequence: "0"
|
|
51
|
+
};
|
|
52
|
+
for (const table of ["works", "check_receipts", "merge_requests", "flow_run_flag_mutations", "usage", "flow_session_segments", "flow_jobs", "run_recovery_guards", "run_controllers", "run_controls"])
|
|
53
|
+
rules[table] = ownRun;
|
|
54
|
+
for (const table of ["plan_bindings", "plan_progress", "worktree_records", "audit_events"])
|
|
55
|
+
rules[table] = `${ownProject} AND protocol_id IN (${protocols})`;
|
|
56
|
+
for (const table of ["run_controller_events", "run_controller_operations"])
|
|
57
|
+
rules[table] = `controller_id IN (SELECT controller_id FROM run_controllers WHERE ${ownRun})`;
|
|
58
|
+
for (const table of ["run_control_admissions", "run_control_stage_admissions", "run_control_operations", "run_control_workers"])
|
|
59
|
+
rules[table] = `control_id IN (SELECT control_id FROM run_controls WHERE ${ownRun})`;
|
|
60
|
+
const tables = db.prepare("SELECT name FROM sqlite_schema WHERE type = 'table'").all();
|
|
61
|
+
const quote = (name) => `"${name.replaceAll('"', '""')}"`;
|
|
62
|
+
// Evaluate every relationship before deleting parents, including historical bindings.
|
|
63
|
+
for (const { name } of tables) {
|
|
64
|
+
const predicate = rules[name];
|
|
65
|
+
if (predicate === undefined && db.prepare(`SELECT 1 FROM ${quote(name)} LIMIT 1`).get())
|
|
66
|
+
throw new AppError("snapshot_scope_unknown", "Recovery export has no scope rule for a populated table", 1, { table: name });
|
|
67
|
+
db.exec(`CREATE TEMP TABLE ${quote(`keep_${name}`)} AS SELECT rowid AS retained_id FROM ${quote(name)} WHERE ${predicate ?? "0"}`);
|
|
68
|
+
}
|
|
69
|
+
for (const { name } of tables)
|
|
70
|
+
db.exec(`DELETE FROM ${quote(name)} WHERE rowid NOT IN (SELECT retained_id FROM ${quote(`keep_${name}`)})`);
|
|
71
|
+
const issues = db.prepare("PRAGMA foreign_key_check").all();
|
|
72
|
+
if (issues.length)
|
|
73
|
+
throw new AppError("snapshot_scope_inconsistent", "Scoped recovery export has unresolved relational references", 1, { issues });
|
|
74
|
+
db.exec("COMMIT; VACUUM");
|
|
75
|
+
complete = true;
|
|
76
|
+
}
|
|
77
|
+
finally {
|
|
78
|
+
db.close();
|
|
79
|
+
if (!complete)
|
|
80
|
+
fs.rmSync(output, { force: true });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** Physical leases are evidence, not authority to resurrect on another host. */
|
|
84
|
+
export function readRecoveryResourceEvidence(resourceHome, projectId, runId, authority) {
|
|
85
|
+
const file = path.join(resourceHome, "runtime.sqlite");
|
|
86
|
+
const scope = { schema_id: "dd-flow/recovery-resource-evidence@1", source_home: resourceHome, dd_flow_home: authority.ddFlowHome, project_id: projectId, run_id: runId };
|
|
87
|
+
if (!fs.existsSync(file))
|
|
88
|
+
return { ...scope, database_present: false, processes: [], resources: [] };
|
|
89
|
+
const db = new DatabaseSync(file, { readOnly: true });
|
|
90
|
+
try {
|
|
91
|
+
db.exec("PRAGMA busy_timeout = 4000; BEGIN");
|
|
92
|
+
const candidates = db.prepare("SELECT * FROM managed_processes WHERE project_id = ? AND run_id = ? ORDER BY id").all(projectId, runId).map(record => ({ record, home: managedProcessRuntimeHome(record, authority) }));
|
|
93
|
+
if (candidates.some(item => item.home === null))
|
|
94
|
+
throw new AppError("snapshot_resource_scope_unproven", "A retained process has no proven runtime-home identity", 1, { resource_home: resourceHome });
|
|
95
|
+
const processes = candidates.filter(item => item.home === authority.ddFlowHome).map(item => item.record);
|
|
96
|
+
const ids = new Set(processes.map(record => record.id)), owners = new Set(processes.map(record => record.owner_id));
|
|
97
|
+
const resources = db.prepare("SELECT r.* FROM managed_resources r JOIN managed_processes p ON p.id = r.process_id WHERE p.project_id = ? AND p.run_id = ? ORDER BY r.resource_kind, r.resource_key").all(projectId, runId).filter(row => typeof row.process_id === "string" && ids.has(row.process_id));
|
|
98
|
+
const unbound = db.prepare("SELECT owner_id FROM managed_resources WHERE process_id IS NULL AND owner_id IN (SELECT owner_id FROM managed_processes WHERE project_id = ? AND run_id = ?)").all(projectId, runId).some(row => typeof row.owner_id === "string" && owners.has(row.owner_id));
|
|
99
|
+
if (unbound)
|
|
100
|
+
throw new AppError("snapshot_resource_scope_unproven", "A retained resource has no process binding proving its RUN scope", 1, { resource_home: resourceHome });
|
|
101
|
+
db.exec("COMMIT");
|
|
102
|
+
return { ...scope, database_present: true, processes, resources };
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
db.close();
|
|
106
|
+
}
|
|
107
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
const object = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
3
|
+
const records = (value) => Array.isArray(value) ? value.map(object) : [];
|
|
4
|
+
const text = (value) => typeof value === "string" && value.length ? value : null;
|
|
5
|
+
/** Pure projection of durable evidence. Never infer quiescence from an ACK,
|
|
6
|
+
* a terminal RUN label, or a settlement captured before a subsequent resume. */
|
|
7
|
+
export function controlStatusReceipt(observation, request = {}) {
|
|
8
|
+
const control = object(observation.control), worker = object(observation.worker), resume = object(observation.resume_operation);
|
|
9
|
+
const present = Boolean(control.control_id), current = control.current === true;
|
|
10
|
+
const activeResume = Boolean(resume.operation_id);
|
|
11
|
+
const historical = !current || activeResume || ["resuming", "resumed"].includes(String(control.status));
|
|
12
|
+
const settlement = object(control.settlement);
|
|
13
|
+
const snapshot = historical ? {} : control.admission === "sealed" ? settlement : object(worker.snapshot);
|
|
14
|
+
const nodes = records(snapshot.nodes).map((node, index) => {
|
|
15
|
+
const identity = Object.fromEntries(["operation_id", "controller_id", "process_id", "check_id", "merge_request_id", "work_id", "work_session_id", "session_id", "harness", "state_dir", "resource_home"].filter(key => node[key] !== undefined).map(key => [key, node[key]]));
|
|
16
|
+
const settled = node.status === "completed";
|
|
17
|
+
const role = ["subject", "worker", "merge", "judge", "probe", "check", "process"].includes(String(node.role)) ? node.role : node.check_id ? "check" : node.merge_request_id ? "merge" : node.process_id ? "process" : "unknown";
|
|
18
|
+
return { id: `node:${crypto.createHash("sha256").update(JSON.stringify(Object.keys(identity).length ? identity : { index, reason: node.reason })).digest("hex")}`, role, status: settled ? "completed" : node.status === "outcome_unknown" ? "unknown" : "pending", acknowledged: settled || object(node.receipt).cancelled === true, settled, reason: settled ? null : text(node.reason) ?? "node_settlement_required", details: node };
|
|
19
|
+
});
|
|
20
|
+
const settled = present && current && !historical && control.status === "settled" && control.admission === "sealed" && Boolean(control.capture_path) && settlement.settled === true && Array.isArray(settlement.nodes) && nodes.every(node => node.settled);
|
|
21
|
+
let state;
|
|
22
|
+
if (!present)
|
|
23
|
+
state = request.action && ["done", "failed", "cancelled"].includes(String(observation.run_status)) ? "terminal_noop" : "not_requested";
|
|
24
|
+
else if (!current)
|
|
25
|
+
state = "superseded";
|
|
26
|
+
else if (resume.status === "failed" || worker.status === "failed")
|
|
27
|
+
state = "recovery_required";
|
|
28
|
+
else if (control.status === "resumed" && control.admission === "recovered" && resume.status === "completed")
|
|
29
|
+
state = "running";
|
|
30
|
+
else if (activeResume || control.status === "resuming")
|
|
31
|
+
state = "resuming";
|
|
32
|
+
else if (settled)
|
|
33
|
+
state = control.requested_mode === "pause" ? "paused" : "stopped";
|
|
34
|
+
else
|
|
35
|
+
state = control.requested_mode === "pause" ? "pause_requested" : "stop_requested";
|
|
36
|
+
const reasons = settled ? [] : [...new Set([
|
|
37
|
+
...nodes.filter(node => !node.settled).map(node => node.reason),
|
|
38
|
+
...(Array.isArray(snapshot.pending_reasons) ? snapshot.pending_reasons.filter((reason) => typeof reason === "string" && Boolean(reason)) : []),
|
|
39
|
+
...(!present ? [state === "terminal_noop" ? "terminal_settlement_not_observed" : "control_not_requested"]
|
|
40
|
+
: !current ? ["control_superseded"]
|
|
41
|
+
: state === "running" ? ["run_active"]
|
|
42
|
+
: state === "resuming" ? ["recovery_acceptance_pending"]
|
|
43
|
+
: state === "recovery_required" ? [text(object(resume.error).code) ?? text(object(worker.error).code) ?? "control_recovery_required"]
|
|
44
|
+
: control.admission === "sealed" ? ["recovery_capture_required"] : ["settlement_observation_required"])
|
|
45
|
+
])];
|
|
46
|
+
return {
|
|
47
|
+
...observation, schema_id: "dd-flow/run-control-receipt@1", ok: true,
|
|
48
|
+
request_id: request.requestId ?? text(resume.request_id) ?? text(control.request_id), control_id: text(control.control_id),
|
|
49
|
+
generation: Number.isSafeInteger(control.generation) ? control.generation : 0,
|
|
50
|
+
requested_mode: request.action ?? (activeResume ? "resume" : text(control.requested_mode)),
|
|
51
|
+
applied_mode: settled ? control.requested_mode : state === "running" ? "resume" : null,
|
|
52
|
+
state, settled, nodes, unsettled_reasons: reasons,
|
|
53
|
+
resumability: settled || state === "recovery_required" ? "recovery_required" : ["running", "not_requested", "terminal_noop", "superseded"].includes(state) ? "not_applicable" : "pending",
|
|
54
|
+
recovery_id: text(control.recovery_id)
|
|
55
|
+
};
|
|
56
|
+
}
|