@deksden-com/dd-flow-cli 0.9.0-beta.151 → 0.9.0-beta.155
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/build-info.json +8 -8
- package/dist/cli/command-inputs.js +33 -3
- package/dist/cli/command-metadata.js +73 -2
- package/dist/cli/delivery.js +108 -0
- package/dist/cli/diagnostic-invocation.js +23 -7
- package/dist/cli/diagnostic.js +14 -3
- package/dist/cli/help.js +117 -1
- package/dist/cli/hook-ingress.js +36 -49
- package/dist/cli/interactive-invocation.js +33 -0
- package/dist/cli/interactive.js +114 -0
- package/dist/cli/memory-fix-invocation.js +52 -0
- package/dist/cli/memory-fix.js +100 -0
- package/dist/cli/memory-flow-admission.js +7 -3
- package/dist/cli/memory-init.js +5 -4
- package/dist/cli/native-hook-ingress.js +5 -1
- package/dist/cli/native-hook-preparation.js +31 -4
- package/dist/cli/run-cli.js +273 -19
- package/dist/cli.js +17 -3
- package/dist/domain/delivery-flows.js +8 -0
- package/dist/domain/distill-flow.js +20 -0
- package/dist/domain/interactive-flow.js +1 -0
- package/dist/domain/memory-fix-flow.js +2 -0
- package/dist/domain/review-fix-flow.js +2 -0
- package/dist/domain/source-snapshot-flows.js +4 -0
- package/dist/harness-runtime/bin/dd-agy.mjs +18 -3
- package/dist/harness-runtime/bin/dd-grok.mjs +15 -5
- package/dist/harness-runtime/bin/dd-zcode.mjs +15 -7
- package/dist/harness-runtime/lib/agy-error-boundary.mjs +18 -10
- package/dist/harness-runtime/lib/agy-native-inspection.mjs +12 -3
- package/dist/harness-runtime/lib/agy-native-inventory.mjs +19 -0
- package/dist/harness-runtime/lib/agy-terminal-envelope.mjs +13 -6
- package/dist/harness-runtime/lib/agy-transcript-stream.mjs +54 -0
- package/dist/harness-runtime/lib/daemon-observation.d.mts +1 -0
- package/dist/harness-runtime/lib/daemon-observation.mjs +12 -4
- package/dist/harness-runtime/lib/daemon-operations.mjs +20 -5
- package/dist/harness-runtime/lib/dd-agy-daemon.d.mts +1 -0
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +371 -114
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +43 -3
- package/dist/harness-runtime/lib/dd-codex.mjs +36 -4
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +25 -11
- package/dist/harness-runtime/lib/dd-droid.mjs +124 -11
- package/dist/harness-runtime/lib/dd-grok-daemon.d.mts +1 -0
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +137 -25
- package/dist/harness-runtime/lib/dd-grok.mjs +4 -2
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +170 -25
- package/dist/harness-runtime/lib/dd-opencode.mjs +40 -19
- package/dist/harness-runtime/lib/dd-zcode-daemon.d.mts +1 -0
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +115 -22
- package/dist/harness-runtime/lib/dd-zcode.mjs +75 -36
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +1 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +10 -3
- package/dist/harness-runtime/lib/delivery-command.mjs +84 -0
- package/dist/harness-runtime/lib/driver-recovery.mjs +0 -19
- package/dist/harness-runtime/lib/droid-observation.mjs +9 -5
- package/dist/harness-runtime/lib/lifecycle-contract.d.mts +1 -0
- package/dist/harness-runtime/lib/lifecycle-contract.mjs +2 -1
- package/dist/harness-runtime/lib/native-hook-command.d.mts +3 -2
- package/dist/harness-runtime/lib/native-hook-command.mjs +63 -8
- package/dist/harness-runtime/lib/native-streams.mjs +3 -10
- package/dist/harness-runtime/lib/opencode-plugin.mjs +36 -9
- package/dist/harness-runtime/lib/operation-errors.mjs +13 -0
- package/dist/harness-runtime/lib/operation-progress.mjs +6 -2
- package/dist/harness-runtime/lib/process-json.mjs +4 -5
- package/dist/harness-runtime/lib/tool-observations.mjs +20 -10
- package/dist/harness-runtime/lib/zcode-native-hook-ownership.mjs +109 -0
- package/dist/schemas/code-work-result.schema.json +2 -1
- package/dist/schemas/delivery-stage-result.schema.json +286 -0
- package/dist/schemas/distill-stage-result.schema.json +667 -0
- package/dist/schemas/distill-work-result.schema.json +428 -0
- package/dist/schemas/execution-bundle.schema.json +6 -2
- package/dist/schemas/interactive-stage-result.schema.json +47 -0
- package/dist/schemas/interactive-work-result.schema.json +63 -0
- package/dist/schemas/mb-fix-stage-result.schema.json +93 -0
- package/dist/schemas/project-flow-pack-manifest.schema.json +2 -2
- package/dist/schemas/review-fix-stage-result.schema.json +210 -0
- package/dist/schemas/run-control-receipt.schema.json +1 -0
- package/dist/services/assigned-lifecycle.js +382 -0
- package/dist/services/check-runner-evidence.js +77 -10
- package/dist/services/check-runner-proof.js +69 -1
- package/dist/services/cli-operation-classifier.js +6 -1
- package/dist/services/code-checks.js +56 -23
- package/dist/services/codex-hook-delivery.js +11 -2
- package/dist/services/controller-execution.js +4 -1
- package/dist/services/controller-fanout.js +9 -4
- package/dist/services/controller-memory-worker.js +6 -4
- package/dist/services/controller-result-authority.js +122 -0
- package/dist/services/dashboard.js +4 -2
- package/dist/services/delivery-access.js +123 -0
- package/dist/services/delivery-approval.js +68 -0
- package/dist/services/delivery-artifact.js +86 -0
- package/dist/services/delivery-checks.js +178 -0
- package/dist/services/delivery-execution.js +181 -0
- package/dist/services/delivery-intents.js +212 -0
- package/dist/services/delivery-operation.js +81 -0
- package/dist/services/delivery-policy.js +91 -0
- package/dist/services/delivery-procedures.js +43 -0
- package/dist/services/delivery-request.js +99 -0
- package/dist/services/delivery-result.js +144 -0
- package/dist/services/delivery-stage-commands.js +26 -0
- package/dist/services/delivery-stage-entry.js +56 -0
- package/dist/services/delivery-stage-handlers.js +135 -0
- package/dist/services/delivery-stage-results.js +94 -0
- package/dist/services/descriptor-coordinator-session.js +3 -1
- package/dist/services/descriptor-run-start.js +24 -4
- package/dist/services/descriptor-stage-coordinator.js +78 -10
- package/dist/services/descriptor-stage-settlement.js +59 -14
- package/dist/services/diagnostic-repair-intake.js +123 -0
- package/dist/services/diagnostic-repair-source-intake.js +102 -0
- package/dist/services/diagnostic-report.js +30 -0
- package/dist/services/diagnostic-run-bundle.js +48 -9
- package/dist/services/diagnostic-snapshot.js +2 -2
- package/dist/services/diagnostic-stage-handlers.js +3 -2
- package/dist/services/diagnostic-stage-results.js +26 -0
- package/dist/services/diagnostic-start.js +56 -6
- package/dist/services/diagnostic-work-collection.js +3 -3
- package/dist/services/distill-document-publication.js +280 -0
- package/dist/services/distill-promotion.js +197 -0
- package/dist/services/distill-publication-custody.js +162 -0
- package/dist/services/distill-stage-handlers.js +249 -0
- package/dist/services/distill-stage-results.js +105 -0
- package/dist/services/distill-user-inputs.js +4 -0
- package/dist/services/distill-work-batch.js +42 -0
- package/dist/services/distill-work-collection.js +74 -0
- package/dist/services/distill-work-registration.js +93 -0
- package/dist/services/distill-work-results.js +98 -0
- package/dist/services/eval-snapshots.js +17 -7
- package/dist/services/external-work-launch.js +8 -2
- package/dist/services/flow-catalog.js +24 -0
- package/dist/services/frozen-engine.js +19 -2
- package/dist/services/generic-execution.js +4 -3
- package/dist/services/harness-adapter.js +2 -14
- package/dist/services/hook-memory-routing.js +1 -1
- package/dist/services/hooks.js +31 -6
- package/dist/services/hybrid-contracts.js +6 -1
- package/dist/services/installed-flow-source.js +37 -2
- package/dist/services/interactive-attached-session.js +215 -0
- package/dist/services/interactive-baseline-custody.js +54 -0
- package/dist/services/interactive-before-versions.js +148 -0
- package/dist/services/interactive-bundle-data.js +85 -0
- package/dist/services/interactive-checks.js +179 -0
- package/dist/services/interactive-execution.js +33 -0
- package/dist/services/interactive-finish-integration.js +236 -0
- package/dist/services/interactive-git-baseline.js +230 -0
- package/dist/services/interactive-git-dispatch.js +278 -0
- package/dist/services/interactive-input-delivery.js +82 -0
- package/dist/services/interactive-mutation-scope.js +231 -0
- package/dist/services/interactive-request.js +287 -0
- package/dist/services/interactive-run-bundle.js +109 -0
- package/dist/services/interactive-stage-entry.js +41 -0
- package/dist/services/interactive-stage-handlers.js +289 -0
- package/dist/services/interactive-stage-observation.js +136 -0
- package/dist/services/interactive-stage-results.js +111 -0
- package/dist/services/interactive-user-inputs.js +142 -0
- package/dist/services/interactive-work-batch.js +43 -0
- package/dist/services/interactive-work-collection.js +119 -0
- package/dist/services/interactive-work-completion.js +180 -0
- package/dist/services/interactive-work-context.js +249 -0
- package/dist/services/interactive-work-history.js +112 -0
- package/dist/services/interactive-work-registration.js +124 -0
- package/dist/services/interactive-work-results.js +115 -0
- package/dist/services/interactive-work-start.js +155 -0
- package/dist/services/lifecycle-invocations.js +214 -24
- package/dist/services/managed-daemon-binding.js +52 -1
- package/dist/services/memory-compose-handler.js +2 -22
- package/dist/services/memory-finish-integration.js +58 -17
- package/dist/services/memory-fix-finish-integration.js +129 -0
- package/dist/services/memory-fix-git-dispatch.js +139 -0
- package/dist/services/memory-fix-intake-custody.js +11 -0
- package/dist/services/memory-fix-intake-settlement.js +127 -0
- package/dist/services/memory-fix-publication.js +196 -0
- package/dist/services/memory-fix-repair-admission.js +116 -0
- package/dist/services/memory-fix-request.js +141 -0
- package/dist/services/memory-fix-run-bundle.js +83 -0
- package/dist/services/memory-fix-scope.js +78 -0
- package/dist/services/memory-fix-sources.js +85 -0
- package/dist/services/memory-fix-stage-entry.js +24 -0
- package/dist/services/memory-fix-stage-handlers.js +282 -0
- package/dist/services/memory-fix-stage-results.js +106 -0
- package/dist/services/memory-fix-workspace.js +127 -0
- package/dist/services/memory-flow-user-inputs.js +88 -0
- package/dist/services/memory-init-contexts.js +55 -5
- package/dist/services/memory-init-controller-start.js +20 -4
- package/dist/services/memory-layer.js +7 -3
- package/dist/services/memory-native-review-copy.js +27 -3
- package/dist/services/memory-repair-document-delta.js +1 -0
- package/dist/services/project-flow-source.js +195 -0
- package/dist/services/project-git-integration.js +92 -0
- package/dist/services/repair-intents.js +2 -4
- package/dist/services/retained-native-hook.js +245 -0
- package/dist/services/review-fix-handoff.js +82 -0
- package/dist/services/review-fix-intake-custody.js +17 -0
- package/dist/services/review-fix-stage-entry.js +43 -0
- package/dist/services/review-fix-stage-handlers.js +176 -0
- package/dist/services/review-fix-stage-results.js +86 -0
- package/dist/services/run-control-worker.js +42 -1
- package/dist/services/run-control.js +40 -5
- package/dist/services/run-controller-adapter.js +37 -3
- package/dist/services/run-controller-process.js +12 -4
- package/dist/services/run-controller-recovery.js +46 -6
- package/dist/services/run-controller-state.js +3 -2
- package/dist/services/run-controller.js +209 -43
- package/dist/services/run-recovery.js +31 -2
- package/dist/services/run-timeline.js +59 -0
- package/dist/services/runs.js +18 -31
- package/dist/services/sessions.js +2 -1
- package/dist/services/stage-handlers.js +55 -3
- package/dist/services/stage-lifecycle.js +4 -3
- package/dist/services/stage-pause.js +26 -9
- package/dist/services/stage-report-renderer.js +2 -6
- package/dist/services/technical-notes.js +31 -0
- package/dist/services/verification-matrix.js +1 -1
- package/dist/services/version-status.js +1 -1
- package/dist/services/vnext-code-review.js +74 -5
- package/dist/services/vnext-code.js +9 -32
- package/dist/services/vnext-fanout.js +53 -8
- package/dist/services/vnext-merge.js +10 -5
- package/dist/services/vnext-plan-review.js +1 -1
- package/dist/services/vnext-plan.js +1 -1
- package/dist/services/vnext-protocolize.js +1 -1
- package/dist/services/vnext-specify.js +10 -5
- package/dist/services/work-registry.js +252 -66
- package/dist/services/worktrees.js +7 -4
- package/dist/storage/database.js +2 -1
- package/dist/verification-components.json +82 -0
- package/package.json +4 -3
- package/tools/verification-jobs.mjs +342 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { Ajv2020 } from "ajv/dist/2020.js";
|
|
2
|
+
import { AppError } from "../shared/errors.js";
|
|
3
|
+
import { hybridDigest } from "./hybrid-contracts.js";
|
|
4
|
+
function refuse(reason) {
|
|
5
|
+
throw new AppError("review_fix_stage_result_invalid", "Review-fix requires its selected findings and original delivered user scope", 2, { reason, effect: "no_effect", ...(reason === "invalid_schema" ? { phase: "prepare", recoverable: true } : {}) });
|
|
6
|
+
}
|
|
7
|
+
/** Proposal validation, not semantic approval or physical reviewer independence.
|
|
8
|
+
* Registered handlers must assess actual agreement and independent ownership.
|
|
9
|
+
* There is intentionally no writable scope, specification or SDLC launch field. */
|
|
10
|
+
export function validateReviewFixStageResult(bytes, schemaBytes, binding) {
|
|
11
|
+
if (!bytes.length || bytes.length > 1024 * 1024 || !schemaBytes.length || schemaBytes.length > 1024 * 1024)
|
|
12
|
+
refuse("result_exceeds_bound");
|
|
13
|
+
let result;
|
|
14
|
+
try {
|
|
15
|
+
result = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return refuse("invalid_schema");
|
|
19
|
+
}
|
|
20
|
+
const validate = (() => {
|
|
21
|
+
try {
|
|
22
|
+
const schema = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(schemaBytes));
|
|
23
|
+
if (schema.$id !== "dd-flow/review-fix-stage-result@1")
|
|
24
|
+
refuse("frozen_schema_invalid");
|
|
25
|
+
return new Ajv2020({ strict: true, validateFormats: false }).compile(schema);
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return refuse("frozen_schema_invalid");
|
|
29
|
+
}
|
|
30
|
+
})();
|
|
31
|
+
if (!validate(result) || result.schema_id !== "dd-flow/review-fix-stage-result@1" || result.flow_id !== "review-fix"
|
|
32
|
+
|| !result.summary.trim())
|
|
33
|
+
refuse("invalid_schema");
|
|
34
|
+
if (result.run_id !== binding.runId || result.stage !== binding.stage || result.input_sha256 !== binding.inputSha256
|
|
35
|
+
|| result.intake_sha256 !== binding.intakeSha256 || result.snapshot_sha256 !== binding.snapshotSha256
|
|
36
|
+
|| result.user_inputs_sha256 !== binding.userInputsSha256
|
|
37
|
+
|| hybridDigest(result.predecessors) !== hybridDigest(binding.predecessors))
|
|
38
|
+
refuse("foreign_stage_binding");
|
|
39
|
+
const ids = binding.findingIds;
|
|
40
|
+
const complete = (items, expected) => {
|
|
41
|
+
if (new Set(expected).size !== expected.length || items.length !== expected.length
|
|
42
|
+
|| hybridDigest(items.map(item => item.finding_id)) !== hybridDigest(expected))
|
|
43
|
+
refuse("incomplete_finding_decisions");
|
|
44
|
+
};
|
|
45
|
+
if (!ids.length || ids.length > 4096)
|
|
46
|
+
refuse("selected_findings_required");
|
|
47
|
+
complete(result.details.decisions, ids);
|
|
48
|
+
complete(result.details.review, binding.stage === "intake" ? [] : ids);
|
|
49
|
+
for (const decision of result.details.decisions) {
|
|
50
|
+
if (!decision.reason.trim())
|
|
51
|
+
refuse("invalid_schema");
|
|
52
|
+
if (decision.disposition === "approved") {
|
|
53
|
+
if (!decision.approval_source || !binding.userSources.some(source => source.role !== "question"
|
|
54
|
+
&& hybridDigest({ root: source.root, path: source.path, sha256: source.sha256 }) === hybridDigest(decision.approval_source)))
|
|
55
|
+
refuse("approval_not_delivered_by_user");
|
|
56
|
+
}
|
|
57
|
+
else if (decision.approval_source !== null)
|
|
58
|
+
refuse("nonapproved_scope_claim");
|
|
59
|
+
}
|
|
60
|
+
if (result.details.review.some(item => !item.reason.trim()))
|
|
61
|
+
refuse("invalid_schema");
|
|
62
|
+
return result;
|
|
63
|
+
}
|
|
64
|
+
export function assertReviewFixStageScope(result, prior) {
|
|
65
|
+
// A real answer delivered during scope-review may revise the intake proposal.
|
|
66
|
+
// The handler must bind the newly read full input set; handoff cannot revise
|
|
67
|
+
// an already reviewed decision or silently consume later user messages.
|
|
68
|
+
const revisedByUser = prior.stage === "intake" && result.stage === "scope-review"
|
|
69
|
+
&& result.user_inputs_sha256 !== prior.user_inputs_sha256;
|
|
70
|
+
if (result.intake_sha256 !== prior.intake_sha256 || result.snapshot_sha256 !== prior.snapshot_sha256
|
|
71
|
+
|| !revisedByUser && (result.user_inputs_sha256 !== prior.user_inputs_sha256
|
|
72
|
+
|| hybridDigest(result.details.decisions) !== hybridDigest(prior.details.decisions))
|
|
73
|
+
|| prior.details.limitations.some(value => !result.details.limitations.includes(value))
|
|
74
|
+
|| prior.stage === "scope-review" && hybridDigest(result.details.review) !== hybridDigest(prior.details.review))
|
|
75
|
+
refuse("accepted_scope_changed");
|
|
76
|
+
}
|
|
77
|
+
/** Honest handoff disposition only; even ready still needs explicit formalization. */
|
|
78
|
+
export function reviewFixHandoffDisposition(result) {
|
|
79
|
+
if (result.stage !== "handoff")
|
|
80
|
+
refuse("handoff_required");
|
|
81
|
+
if (result.details.review.some(item => item.verdict === "revise"))
|
|
82
|
+
return "needs_scope";
|
|
83
|
+
if (result.details.decisions.some(item => item.disposition === "approved"))
|
|
84
|
+
return "ready";
|
|
85
|
+
return result.details.decisions.some(item => item.disposition === "deferred") ? "deferred" : "declined";
|
|
86
|
+
}
|
|
@@ -15,16 +15,57 @@ import { interruptControlledSessions, observeControlledSessions, recordRunContro
|
|
|
15
15
|
import { recoveryGuard, recordRecoveryCapture, sealRunRecovery } from "./run-recovery.js";
|
|
16
16
|
import { createEvalRunSnapshot, verifyRecoverySnapshot } from "./eval-snapshots.js";
|
|
17
17
|
import { recoveryObservationBudget, renewRecoveryObservation, withRecoveryObservation } from "./recovery-observation-budget.js";
|
|
18
|
+
import { acceptedTechnicalWorkControl } from "./controller-result-authority.js";
|
|
19
|
+
import { resumeRunController } from "./run-controller-recovery.js";
|
|
20
|
+
import { managedDaemonExecutionAbsent, readNativeDaemonState } from "./managed-daemon-binding.js";
|
|
18
21
|
export async function reconcileRunControl(context, input) {
|
|
19
22
|
const status = runControlStatus(context, input);
|
|
20
23
|
const control = status.control;
|
|
21
|
-
if (!control?.current || !["requested", "settled"].includes(control.status))
|
|
24
|
+
if (!control?.current || !["requested", "settled", "resuming"].includes(control.status))
|
|
22
25
|
throw new AppError("run_control_state_changed", "Reconciliation requires the current draining/sealed control", 1);
|
|
23
26
|
renewRecoveryObservation({}, input.requestId); // Validate before claiming anything.
|
|
24
27
|
if (control.status === "requested")
|
|
25
28
|
await launchControlWorker(context, { ...input, reconcileId: input.requestId });
|
|
29
|
+
else
|
|
30
|
+
await reconcileAcceptedTechnicalWork(context, input);
|
|
26
31
|
return runControlStatus(context, input);
|
|
27
32
|
}
|
|
33
|
+
/** Explicit, repeatable reconciliation; never converts a human pause to resume. */
|
|
34
|
+
async function reconcileAcceptedTechnicalWork(context, input) {
|
|
35
|
+
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
36
|
+
const row = context.db.get("SELECT * FROM run_controllers WHERE project_id=? AND run_id=? AND status NOT IN ('completed','cancelled','superseded')", [project.id, input.runId]);
|
|
37
|
+
if (!row)
|
|
38
|
+
return;
|
|
39
|
+
const requestId = `technical-work:${crypto.createHash("sha256").update(input.controlId).digest("hex")}`;
|
|
40
|
+
const control = context.db.get("SELECT status,recovery_id FROM run_controls WHERE control_id=? AND project_id=? AND run_id=?", [input.controlId, project.id, input.runId]);
|
|
41
|
+
const guard = recoveryGuard(context, project.id, input.runId);
|
|
42
|
+
if (!control || !guard || control.recovery_id !== guard.recovery_id)
|
|
43
|
+
return;
|
|
44
|
+
if (control.status === "settled") {
|
|
45
|
+
if (!acceptedTechnicalWorkControl(context, row, input.controlId))
|
|
46
|
+
return;
|
|
47
|
+
try {
|
|
48
|
+
const state = JSON.parse(row.state_json), session = state.sessions[state.current_session];
|
|
49
|
+
const manifest = JSON.parse(row.manifest_json);
|
|
50
|
+
const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
|
|
51
|
+
if (readNativeDaemonState(session.state_dir).shutdown_state !== "clean" || !managedDaemonExecutionAbsent(resourceContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id }))
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const validate = () => {
|
|
58
|
+
const current = context.db.get("SELECT * FROM run_controllers WHERE controller_id=?", [row.controller_id]);
|
|
59
|
+
return Boolean(current && current.owner_token === row.owner_token && current.generation === row.generation && acceptedTechnicalWorkControl(context, current, input.controlId));
|
|
60
|
+
};
|
|
61
|
+
await resumeRunController(context, { projectRoot: input.projectRoot, runId: input.runId, controlId: input.controlId, requestId, waitMs: 0 }, validate);
|
|
62
|
+
}
|
|
63
|
+
else if (control.status === "resuming" && guard.generation === row.generation && ["sealed", "resuming", "recovered"].includes(guard.status)) {
|
|
64
|
+
const operation = context.db.get("SELECT status FROM run_controller_operations WHERE controller_id=? AND generation=? AND json_extract(input_json,'$.action')='control_resume' AND json_extract(input_json,'$.request_id')=? AND json_extract(input_json,'$.control_id')=?", [row.controller_id, row.generation, requestId, input.controlId]);
|
|
65
|
+
if (operation?.status === "accepted")
|
|
66
|
+
await resumeRunController(context, { projectRoot: input.projectRoot, runId: input.runId, controlId: input.controlId, requestId, waitMs: 0 });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
28
69
|
/** The caller publishes intent and launches a control owner; it never owns drain. */
|
|
29
70
|
export async function requestRunControl(context, input) {
|
|
30
71
|
if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { setFlowRunRecoveryPaused } from "./runs.js";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import { nativeShutdownConfirmed } from "../harness-runtime/lib/native-children.mjs";
|
|
4
|
+
import { readTechnicalNotes, recordTechnicalNote } from "./technical-notes.js";
|
|
4
5
|
import fs from "node:fs";
|
|
5
6
|
import { ensureRunControlStorage } from "../storage/database.js";
|
|
6
7
|
import { canonicalPath, resolveProjectRoot } from "../storage/paths.js";
|
|
@@ -17,13 +18,15 @@ import { controlStatusReceipt } from "./run-control-receipt.js";
|
|
|
17
18
|
import { assertNativeCancellationReceipt, cancelControlledNativeSession, inspectControlledNativeSession } from "./native-session-control.js";
|
|
18
19
|
import { terminalControllerFinalization } from "./run-controller-state.js";
|
|
19
20
|
import { canonicalNativeOperation } from "../harness-runtime/lib/adapter-timeouts.mjs";
|
|
20
|
-
import { requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
21
|
+
import { requireManagedDaemonBinding, managedDaemonExecutionAbsent } from "./managed-daemon-binding.js";
|
|
22
|
+
import { controllerObligationAccepted, technicalDependencyControl } from "./controller-result-authority.js";
|
|
23
|
+
import { requireController } from "./run-controller-state.js";
|
|
21
24
|
/** Durable projection only: status never probes a provider or acquires an owner. */
|
|
22
25
|
export function runControlStatus(context, input) {
|
|
23
26
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
24
27
|
// One read snapshot prevents a concurrent stop/resume from mixing generations
|
|
25
28
|
// across the request, arbiter and worker projections. Savepoints also nest.
|
|
26
|
-
return context.db.readSnapshot(() => readControlStatus(context, input, project));
|
|
29
|
+
return context.db.readSnapshot(() => ({ ...readControlStatus(context, input, project), technical_notes: readTechnicalNotes(context, project.id, input.runId) }));
|
|
27
30
|
}
|
|
28
31
|
function readControlStatus(context, input, project) {
|
|
29
32
|
const run = context.db.get("SELECT status FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
|
|
@@ -78,6 +81,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
78
81
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
79
82
|
const control = input.controlId ? context.db.get("SELECT * FROM run_controls WHERE project_id = ? AND run_id = ? AND control_id = ?", [project.id, input.runId, input.controlId]) : undefined;
|
|
80
83
|
const request = control && JSON.parse(control.request_json);
|
|
84
|
+
const technicalPause = Boolean(control && technicalDependencyControl(context, project.id, input.runId, control.control_id));
|
|
81
85
|
const pauseEscalated = control?.action === "pause" && request?.escalate === "interrupt" && Number.isSafeInteger(request.grace_ms) && Date.now() >= Date.parse(control.created_at) + request.grace_ms;
|
|
82
86
|
if (boundary ? interrupt || control : !control || interrupt && !(control.action === "stop" || pauseEscalated) || control.status !== "requested")
|
|
83
87
|
throw new AppError("run_control_invalid_state", "Interrupt requires the current stop intent or its explicitly elapsed pause escalation", 1);
|
|
@@ -293,6 +297,29 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
293
297
|
const nativeVersions = new Map();
|
|
294
298
|
for (const [directory, { state, records }] of daemonInventories) {
|
|
295
299
|
let failedStart = null;
|
|
300
|
+
// Snapshot execution safety is not a claim of clean native shutdown.
|
|
301
|
+
// A technical pause can seal after its real writer is gone even when the
|
|
302
|
+
// third-party session/card never acknowledged completion.
|
|
303
|
+
const physicalContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: registeredProcesses.find(item => records.includes(item.record))?.home ?? resourceHome(context) } };
|
|
304
|
+
let executionAbsent = false;
|
|
305
|
+
if ((boundary || technicalPause) && !interrupt && !conflicts.has(directory) && !unsafeDaemons.has(canonicalPath(directory, false))) {
|
|
306
|
+
try {
|
|
307
|
+
executionAbsent = managedDaemonExecutionAbsent(physicalContext, { stateDir: directory, projectId: project.id, runId: input.runId });
|
|
308
|
+
}
|
|
309
|
+
catch { /* Unknown physical ownership remains on strict reconciliation. */ }
|
|
310
|
+
}
|
|
311
|
+
if (executionAbsent) {
|
|
312
|
+
const leader = records.find(record => record.kind.endsWith("-daemon") && record.id === state.resource_process?.id && record.pid === state.pid && record.lease_token === state.resource_process.lease_token);
|
|
313
|
+
if (leader && (state.daemon_id ?? state.config?.daemonId)) {
|
|
314
|
+
nativeVersions.set(directory, nativeInventoryVersion(directory));
|
|
315
|
+
closedDaemons.set(directory, { source: "owned_execution_absent", daemon_id: state.daemon_id ?? state.config?.daemonId,
|
|
316
|
+
session_ids: state.sessions.map(item => typeof item === "string" ? item : adapterSessionId(item)), execution_settled: true,
|
|
317
|
+
native_cleanup_confirmed: state.shutdown_state === "clean", processes: records.map(record => managedProcessBinding(record, registeredProcesses.find(item => item.record === record).home)) });
|
|
318
|
+
if (state.shutdown_state !== "clean")
|
|
319
|
+
recordTechnicalNote(context, project.id, input.runId, { code: "native_cleanup_pending_after_execution_exit", message: "Owned execution is physically absent; native cleanup acknowledgement remains unavailable.", phase: "capture_observation", generation: recoveryGuard(context, project.id, input.runId)?.generation ?? 0, resource_identity: directory, diagnostic_reference: directory, consequence: "cleanup_pending" });
|
|
320
|
+
continue;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
296
323
|
// Explicit stop can reconcile a retired owner whose last bookkeeping ACK
|
|
297
324
|
// failed. This is physical retirement, not a fabricated clean shutdown.
|
|
298
325
|
if (control?.action === "stop" && state.shutdown_state === "cleanup_failed")
|
|
@@ -450,6 +477,11 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
450
477
|
if (!Array.isArray(args) || args.some(value => typeof value !== "string") || !saved.state_dir || !saved.executable || saved.owner !== operation.controller_id || conflicts.has(saved.state_dir) || directoryAdapters.get(saved.state_dir) !== saved.executable)
|
|
451
478
|
throw new AppError("controller_operation_binding_unknown", "Unfinished controller operation has no matching frozen runtime binding", 1);
|
|
452
479
|
const method = `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`;
|
|
480
|
+
if ((boundary || technicalPause) && closedDaemons.get(saved.state_dir)?.source === "owned_execution_absent"
|
|
481
|
+
&& (["daemon.stop", "session.cancel", "session.inspect"].includes(method)
|
|
482
|
+
|| saved.obligation && controllerObligationAccepted(context, requireController(context, operation.controller_id), saved.obligation))) {
|
|
483
|
+
return { operation_id: operation.operation_id, status: "completed", receipt: { source: "accepted_boundary_execution_absent", native_outcome: "unavailable", cleanup_pending: true } };
|
|
484
|
+
}
|
|
453
485
|
if (!["session.create", "session.prompt", "session.start", "session.fork", "session.cancel", "session.resume", "session.inspect", "daemon.start", "daemon.stop"].includes(method))
|
|
454
486
|
throw new AppError("controller_operation_receipt_required", "Unfinished controller operation needs its original receipt", 1);
|
|
455
487
|
const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
|
|
@@ -1032,7 +1064,7 @@ export function runControlDirective(context, projectId, runId) {
|
|
|
1032
1064
|
return { control_id: current.control_id, action: current.action, status: current.status, stop_turn: true, instruction: "The operator has fenced this RUN. Preserve the recorded result and end this Turn now. Do not start another Work, check, Stage or provider operation, and do not wait for resume in a sleep loop. Only the runtime may authorize continuation.", acknowledged: false, settled: false };
|
|
1033
1065
|
}
|
|
1034
1066
|
/** The durable fence is committed before observation, steering or interruption. */
|
|
1035
|
-
export function recordRunControlIntent(context, input) {
|
|
1067
|
+
export function recordRunControlIntent(context, input, onRecorded) {
|
|
1036
1068
|
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !["pause", "stop"].includes(input.action)
|
|
1037
1069
|
|| input.graceMs !== undefined && (!Number.isSafeInteger(input.graceMs) || input.graceMs < 0 || input.graceMs > 3_600_000)
|
|
1038
1070
|
|| input.force !== undefined && typeof input.force !== "boolean"
|
|
@@ -1045,11 +1077,14 @@ export function recordRunControlIntent(context, input) {
|
|
|
1045
1077
|
const requestHash = crypto.createHash("sha256").update(JSON.stringify(request)).digest("hex");
|
|
1046
1078
|
ensureRunControlStorage(context.db);
|
|
1047
1079
|
return context.db.writeTransaction(() => {
|
|
1080
|
+
// Internal SQL-only metadata can be published atomically with the intent,
|
|
1081
|
+
// before any detached control worker observes it. A throw rolls back both.
|
|
1082
|
+
const publish = (receipt) => { onRecorded?.(receipt); return receipt; };
|
|
1048
1083
|
const existing = context.db.get("SELECT * FROM run_controls WHERE project_id = ? AND run_id = ? AND request_id = ?", [project.id, input.runId, input.requestId]);
|
|
1049
1084
|
if (existing) {
|
|
1050
1085
|
if (existing.request_hash !== requestHash)
|
|
1051
1086
|
throw new AppError("runtime_request_conflict", "Control request id has different inputs", 1);
|
|
1052
|
-
return { ok: true, control_id: existing.control_id, recovery_id: existing.recovery_id, action: existing.action, status: existing.status, reused: true };
|
|
1087
|
+
return publish({ ok: true, control_id: existing.control_id, recovery_id: existing.recovery_id, action: existing.action, status: existing.status, reused: true });
|
|
1053
1088
|
}
|
|
1054
1089
|
const run = context.db.get("SELECT status FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
|
|
1055
1090
|
if (!run)
|
|
@@ -1087,6 +1122,6 @@ export function recordRunControlIntent(context, input) {
|
|
|
1087
1122
|
WHERE r.project_id = ? AND r.id = ? AND json_extract(stage.value, '$.status') = 'running'
|
|
1088
1123
|
AND EXISTS (SELECT 1 FROM run_control_admissions a JOIN sessions s ON s.session_id = a.session_id AND s.project_id = r.project_id WHERE a.control_id = ? AND s.current_stage = json_extract(stage.value, '$.stage'))`, [controlId, controlId, project.id, input.runId, controlId]);
|
|
1089
1124
|
setFlowRunRecoveryPaused(context, project.id, input.runId, true);
|
|
1090
|
-
return { ok: true, control_id: controlId, recovery_id: recoveryId, generation, action: input.action, status: "requested", settled: false };
|
|
1125
|
+
return publish({ ok: true, control_id: controlId, recovery_id: recoveryId, generation, action: input.action, status: "requested", settled: false });
|
|
1091
1126
|
});
|
|
1092
1127
|
}
|
|
@@ -13,6 +13,8 @@ import { retainedDaemonReply } from "../harness-runtime/lib/daemon-observation.m
|
|
|
13
13
|
import { readOperationRecord } from "../harness-runtime/lib/operation-progress.mjs";
|
|
14
14
|
import { readNativeDaemonState } from "./managed-daemon-binding.js";
|
|
15
15
|
import { validateCheckEnvironment } from "./check-environment.js";
|
|
16
|
+
import { controllerObligationAccepted, lateNativeFailure } from "./controller-result-authority.js";
|
|
17
|
+
import { recordTechnicalNote } from "./technical-notes.js";
|
|
16
18
|
/** A paused RUN admits only its controller's exact accepted answer Turn. */
|
|
17
19
|
export function controllerHitlAdmission(context, record, input) {
|
|
18
20
|
if (input.operation !== "session.prompt" || !input.operationId || !input.promptSha256)
|
|
@@ -286,11 +288,16 @@ async function awaitSettlement(context, row, manifest, session, executable, args
|
|
|
286
288
|
}
|
|
287
289
|
return adapterSessionId(current) ? { ...initial, ...current, ...(initial.assistant_text !== undefined ? { assistant_text: initial.assistant_text } : {}) } : { ...initial, settlement: settlement(current) };
|
|
288
290
|
}
|
|
289
|
-
export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal, reconfirmOwner) {
|
|
291
|
+
export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal, reconfirmOwner, obligation) {
|
|
290
292
|
const executable = manifest.adapters[runtimeHarness(session.profile)].executable;
|
|
291
293
|
const promptFile = args.indexOf("--prompt-file"), inlinePrompt = args.indexOf("--prompt");
|
|
292
294
|
const promptSha256 = promptFile >= 0 ? digestBytes(fs.readFileSync(args[promptFile + 1])) : inlinePrompt >= 0 ? digestBytes(args[inlinePrompt + 1] ?? "") : undefined;
|
|
293
|
-
const
|
|
295
|
+
const priorInput = retainedOperationId ? context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id=? AND controller_id=? AND generation=?", [retainedOperationId, row.controller_id, row.generation]) : undefined;
|
|
296
|
+
const effectiveObligation = priorInput ? JSON.parse(priorInput.input_json).obligation : obligation;
|
|
297
|
+
// Committing an answer resumes a Stage; it does not finish the semantic
|
|
298
|
+
// work owed by that prompt. Preserve its provider error for same-Session recovery.
|
|
299
|
+
const acceptedPromptBoundary = (expected) => controllerObligationAccepted(context, row, expected.action === "resume" ? { ...expected, action: "finish" } : expected);
|
|
300
|
+
const input = { executable, args, owner: session.owner.owner_id, state_dir: session.state_dir, ...(args[0] === "session" && args[1] === "prompt" && promptSha256 ? { prompt_sha256: promptSha256 } : {}), ...(effectiveObligation ? { obligation: effectiveObligation } : {}) };
|
|
294
301
|
const prepared = retainedOperationId ? [] : context.db.all("SELECT operation_id FROM run_controller_operations WHERE controller_id = ? AND generation = ? AND input_hash = ? AND status = 'dispatching' AND json_extract(input_json, '$.dispatch_phase') = 'prepared'", [row.controller_id, row.generation, digest(input)]);
|
|
295
302
|
if (prepared.length > 1)
|
|
296
303
|
throw new AppError("controller_operation_conflict", "Multiple prepared intents require explicit reconciliation", 1);
|
|
@@ -369,6 +376,14 @@ export async function controllerAdapter(context, row, manifest, session, args, a
|
|
|
369
376
|
recoveredOperation = true;
|
|
370
377
|
}
|
|
371
378
|
validateReceipt(receipt, session, args);
|
|
379
|
+
const hookDiagnostics = Array.isArray(receipt.native_hook_diagnostics) ? receipt.native_hook_diagnostics : [];
|
|
380
|
+
if (hookDiagnostics.some(item => item && typeof item === "object" && ["native_hook_timeout", "native_hook_failed"].includes(String(item.code)))) {
|
|
381
|
+
recordTechnicalNote(context, row.project_id, row.run_id, { code: "native_hook_observation_unavailable",
|
|
382
|
+
message: "Native hook observation failed; the actual lifecycle command remains authoritative.", phase: "native_observation",
|
|
383
|
+
generation: row.generation, ...(effectiveObligation?.stage ? { stage: effectiveObligation.stage } : {}),
|
|
384
|
+
harness: runtimeHarness(session.profile), source_operation_id: operationId, resource_identity: session.state_dir,
|
|
385
|
+
diagnostic_reference: session.journal, consequence: "none" });
|
|
386
|
+
}
|
|
372
387
|
retainPhase("receipt_observed");
|
|
373
388
|
const persistReceipt = () => {
|
|
374
389
|
const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
|
|
@@ -376,7 +391,13 @@ export async function controllerAdapter(context, row, manifest, session, args, a
|
|
|
376
391
|
throw new AppError("controller_owner_mismatch", "Controller lost ownership before receipt retention", 1);
|
|
377
392
|
};
|
|
378
393
|
persistReceipt();
|
|
379
|
-
if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
|
|
394
|
+
if (["pending", "blocked"].includes(String(settlement(receipt)?.state)) && effectiveObligation && acceptedPromptBoundary(effectiveObligation)) {
|
|
395
|
+
// Required semantic work is already committed. Keep the honest pending
|
|
396
|
+
// native proof for resource cleanup without holding its accepted result.
|
|
397
|
+
recordTechnicalNote(context, row.project_id, row.run_id, { code: "accepted_boundary_native_settlement_pending", message: "The assigned result is committed; native settlement remains pending.", phase: "native_observation", generation: row.generation, stage: effectiveObligation.stage, harness: runtimeHarness(session.profile), source_operation_id: operationId, diagnostic_reference: session.state_dir, consequence: "cleanup_pending" });
|
|
398
|
+
receipt = { ...receipt, accepted_boundary: effectiveObligation };
|
|
399
|
+
}
|
|
400
|
+
else if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
|
|
380
401
|
const recovered = await awaitSettlement(context, row, manifest, session, executable, args, operationId, receipt, assertOwner, signal);
|
|
381
402
|
receipt = recovered;
|
|
382
403
|
}
|
|
@@ -389,6 +410,19 @@ export async function controllerAdapter(context, row, manifest, session, args, a
|
|
|
389
410
|
return receipt;
|
|
390
411
|
}
|
|
391
412
|
catch (error) {
|
|
413
|
+
// A lost provider/transport acknowledgement cannot revoke the exact
|
|
414
|
+
// boundary already accepted by the engine. Identity/integrity failures
|
|
415
|
+
// and unfinished siblings remain on the existing recovery path.
|
|
416
|
+
const expected = effectiveObligation;
|
|
417
|
+
if (expected && lateNativeFailure(error, session.id) && acceptedPromptBoundary(expected)) {
|
|
418
|
+
assertOwner?.();
|
|
419
|
+
const receipt = { adapter_session_id: session.id, accepted_boundary: expected, native_observation_unavailable: true, settlement: { state: "pending", reason: "native_observation_unavailable" } };
|
|
420
|
+
const completed = context.db.run("UPDATE run_controller_operations SET status='completed',receipt_json=?,error_json=NULL,updated_at=? WHERE operation_id=? AND controller_id=? AND generation=? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id=? AND owner_token=? AND generation=?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
|
|
421
|
+
if (completed.changes !== 1)
|
|
422
|
+
throw new AppError("controller_owner_mismatch", "Controller lost ownership before accepting the committed boundary", 1);
|
|
423
|
+
recordTechnicalNote(context, row.project_id, row.run_id, { code: "accepted_boundary_native_observation_unavailable", message: "The assigned result is committed; the native acknowledgement is unavailable.", phase: "native_observation", generation: row.generation, stage: expected.stage, harness: runtimeHarness(session.profile), source_operation_id: operationId, diagnostic_reference: session.state_dir, consequence: "cleanup_pending" });
|
|
424
|
+
return receipt;
|
|
425
|
+
}
|
|
392
426
|
try {
|
|
393
427
|
context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' AND ? <> 'prepared' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [phase ?? "legacy_unknown", JSON.stringify(errorRecord(error)), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
|
|
394
428
|
}
|
|
@@ -7,6 +7,7 @@ import { validateCheckEnvironment } from "./check-environment.js";
|
|
|
7
7
|
import { getResourceDatabase } from "../storage/database.js";
|
|
8
8
|
import { finishManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess } from "./managed-processes.js";
|
|
9
9
|
import { setFlowRunRecoveryPaused } from "./runs.js";
|
|
10
|
+
import { physicalStartup } from "./physical-startup.js";
|
|
10
11
|
import { assertRunMutationAllowed, assertRunRecoveryAllowed, recoveryGuard } from "./run-recovery.js";
|
|
11
12
|
import { requireRun, requireController, controllerReceipt, appendEvent, digest, errorRecord } from "./run-controller-state.js";
|
|
12
13
|
const launchId = (row) => `${row.controller_id}:owner-launch:${digest(row.owner_token)}`;
|
|
@@ -15,6 +16,8 @@ const birth = () => ({ pid: process.pid, pid_started_at: processStartedAt(proces
|
|
|
15
16
|
const dead = (value) => Boolean(value && Number.isSafeInteger(value.pid) && value.pid > 0 && typeof value.pid_started_at === "string" && value.pid_started_at && !processIsAlive(value));
|
|
16
17
|
/** Called inside the RUN intent transaction, before the client can disappear. */
|
|
17
18
|
export function recordControllerLauncher(context, row, retired, identity = birth()) {
|
|
19
|
+
if (!identity.pid_started_at)
|
|
20
|
+
throw new AppError("process_ownership_unknown", "Controller launcher requires an observed physical identity", 1, { phase: "process_confirmation", effect: "no_effect", native_dispatch_started: false, reason: "physical_identity_unavailable" });
|
|
18
21
|
const request = { action: "controller_owner_launch", owner_id: launchId(row) };
|
|
19
22
|
const receipt = { ...identity, registration: "controller_self@1", ...(retired ? { retired_process: { id: retired.id, lease_token: retired.lease_token, pid: retired.pid, pid_started_at: retired.pid_started_at } } : {}) };
|
|
20
23
|
const now = context.now();
|
|
@@ -72,12 +75,18 @@ function retire(context, row, identity) {
|
|
|
72
75
|
throw new AppError("controller_owner_unsettled", "Prior controller identity is not proven dead and retired", 1);
|
|
73
76
|
}
|
|
74
77
|
/** Retry only before productive admission, with exact prior birth and journal CAS. */
|
|
75
|
-
function prepareSpawn(context, row) {
|
|
78
|
+
async function prepareSpawn(context, row) {
|
|
76
79
|
const launch = launchRecord(context, row);
|
|
77
80
|
if (!launch || launch.value.dispatch_ready || !["starting", "running", "recovery_required"].includes(row.status))
|
|
78
81
|
return null;
|
|
82
|
+
const launcher = await physicalStartup(context.databaseDeadlineAt).birth(() => {
|
|
83
|
+
const current = requireController(context, row.controller_id);
|
|
84
|
+
if (current.owner_token !== row.owner_token || current.generation !== row.generation)
|
|
85
|
+
throw new AppError("controller_owner_mismatch", "Controller launch owner changed during physical observation", 1);
|
|
86
|
+
assertLaunchAllowed(context, current);
|
|
87
|
+
});
|
|
79
88
|
const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: JSON.parse(row.manifest_json).resource_home } };
|
|
80
|
-
const ours = launch.value.pid ===
|
|
89
|
+
const ours = launch.value.pid === launcher.pid && launch.value.pid_started_at === launcher.pid_started_at;
|
|
81
90
|
let token = row.owner_token;
|
|
82
91
|
if (!ours) {
|
|
83
92
|
if (row.owner_pid) {
|
|
@@ -115,7 +124,6 @@ function prepareSpawn(context, row) {
|
|
|
115
124
|
return null;
|
|
116
125
|
if (launch.value.retired_process)
|
|
117
126
|
retire(resourceContext, row, launch.value.retired_process);
|
|
118
|
-
const launcher = birth();
|
|
119
127
|
return context.db.writeTransaction(() => {
|
|
120
128
|
assertLaunchAllowed(context, row);
|
|
121
129
|
const changed = context.db.run(`UPDATE run_controllers SET owner_token = ?, owner_pid = NULL, process_id = NULL, process_lease_token = NULL, status = 'starting', last_error_json = NULL, updated_at = ?
|
|
@@ -165,7 +173,7 @@ export function markControllerDispatchReady(context, row) {
|
|
|
165
173
|
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.dispatch_ready', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(row)]);
|
|
166
174
|
}
|
|
167
175
|
export async function spawnController(context, row) {
|
|
168
|
-
const admitted = prepareSpawn(context, row);
|
|
176
|
+
const admitted = await prepareSpawn(context, row);
|
|
169
177
|
if (!admitted)
|
|
170
178
|
return controllerReceipt(requireController(context, row.controller_id));
|
|
171
179
|
row = admitted;
|
|
@@ -18,16 +18,19 @@ import { controllerAdapter, controllerEnvironment, sessionArguments } from "./ru
|
|
|
18
18
|
import { runHarnessAdapter } from "./harness-adapter.js";
|
|
19
19
|
import { continueProviderCapacity } from "./provider-capacity.js";
|
|
20
20
|
import { spawnController, recordControllerLauncher } from "./run-controller-process.js";
|
|
21
|
-
import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
21
|
+
import { readNativeDaemonState, requireManagedDaemonBinding, managedDaemonExecutionAbsent } from "./managed-daemon-binding.js";
|
|
22
22
|
import { managedInvocationContext } from "./lifecycle-invocations.js";
|
|
23
23
|
import { runControlStatus } from "./run-control.js";
|
|
24
24
|
import { runtimeBudgetStatus, waitForRuntimeScopeRecovery } from "./runtime-budget.js";
|
|
25
25
|
import { prepareControllerPublication } from "./run-controller-capture.js";
|
|
26
|
+
import { technicalDependencyControl } from "./controller-result-authority.js";
|
|
27
|
+
import { recordTechnicalNote } from "./technical-notes.js";
|
|
28
|
+
import { setFlowRunRecoveryPaused } from "./runs.js";
|
|
26
29
|
/** CLI observation is bounded; the physical controller owns the recovery work. */
|
|
27
|
-
export async function resumeRunController(context, input) {
|
|
30
|
+
export async function resumeRunController(context, input, validate) {
|
|
28
31
|
if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
|
|
29
32
|
throw new AppError("usage", "--wait-ms must be between 0 and 60000", 2);
|
|
30
|
-
const intent =
|
|
33
|
+
const intent = prepareControllerResume(context, input, validate)();
|
|
31
34
|
const manifest = JSON.parse(intent.controller.manifest_json);
|
|
32
35
|
const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
|
|
33
36
|
await spawnController(resourceContext, intent.controller);
|
|
@@ -44,7 +47,7 @@ export function recordControllerResume(context, input) {
|
|
|
44
47
|
}
|
|
45
48
|
/** Verify immutable recovery evidence before a caller reserves the registry
|
|
46
49
|
* writer. The returned operation rechecks SQL ownership under the RUN CAS. */
|
|
47
|
-
export function prepareControllerResume(context, input) {
|
|
50
|
+
export function prepareControllerResume(context, input, validate) {
|
|
48
51
|
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !input.controlId.trim())
|
|
49
52
|
throw new AppError("usage", "Resume requires request-id and the exact source control", 2);
|
|
50
53
|
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
@@ -97,6 +100,8 @@ export function prepareControllerResume(context, input) {
|
|
|
97
100
|
throw new AppError("runtime_request_conflict", "Resume request id has different inputs", 1);
|
|
98
101
|
return { controller: requireController(context, row.controller_id), operationId };
|
|
99
102
|
}
|
|
103
|
+
if (validate && !validate())
|
|
104
|
+
throw new AppError("run_control_superseded", "Technical recovery evidence changed before resume admission", 1);
|
|
100
105
|
const current = recoveryGuard(context, project.id, run.id);
|
|
101
106
|
if (current?.recovery_id !== guard.recovery_id || current.generation !== guard.generation || current.status !== "sealed" || current.capture_path !== guard.capture_path)
|
|
102
107
|
throw new AppError("run_control_superseded", "Control changed before resume could be accepted", 1);
|
|
@@ -146,6 +151,39 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
146
151
|
assertRecoveryOwner();
|
|
147
152
|
const readDaemon = () => readNativeDaemonState(session.state_dir);
|
|
148
153
|
const { state: retired, process: record } = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
|
|
154
|
+
const run = requireRun(context, row.project_id, row.run_id);
|
|
155
|
+
const boundary = JSON.parse(run.index_json).stage_runs.find(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status));
|
|
156
|
+
if (boundary && technicalDependencyControl(context, row.project_id, row.run_id, recovery.control_id) && !scopeControl) {
|
|
157
|
+
if (!managedDaemonExecutionAbsent(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id }))
|
|
158
|
+
throw new AppError("recovery_native_shutdown_unproven", "Technical boundary recovery still has an unknown or live owned writer", 1);
|
|
159
|
+
const assertPublication = prepareControllerPublication(executionContext, row, assertPhysicalOwner);
|
|
160
|
+
context.db.writeTransaction(() => {
|
|
161
|
+
assertPublication();
|
|
162
|
+
assertRecoveryOwner(false);
|
|
163
|
+
if (requireRun(context, row.project_id, row.run_id).index_json !== run.index_json)
|
|
164
|
+
throw new AppError("controller_boundary_changed", "Accepted boundary changed before technical continuation", 1);
|
|
165
|
+
const now = context.now();
|
|
166
|
+
if (context.db.run("UPDATE run_recovery_guards SET status='recovered',resumed_at=?,updated_at=? WHERE recovery_id=? AND generation=? AND status='sealed'", [now, now, recovery.recovery_id, row.generation]).changes !== 1)
|
|
167
|
+
throw new AppError("run_recovery_generation_stale", "Technical boundary no longer owns its sealed recovery", 1);
|
|
168
|
+
if (context.db.run("UPDATE run_controls SET status='resumed',updated_at=? WHERE control_id=? AND status='resuming'", [now, recovery.control_id]).changes !== 1)
|
|
169
|
+
throw new AppError("run_control_superseded", "Technical recovery control changed before continuation", 1);
|
|
170
|
+
const receipt = { schema_id: "dd-flow/technical-boundary-resume@1", resumed: true, recovery_id: recovery.recovery_id, stage: boundary.stage, attempt: boundary.attempt, native_cleanup_confirmed: retired.shutdown_state === "clean", native_model_call: false };
|
|
171
|
+
if (context.db.run("UPDATE run_controller_operations SET status='completed',receipt_json=?,updated_at=? WHERE operation_id=? AND controller_id=? AND generation=? AND status='accepted'", [JSON.stringify(receipt), now, recovery.operation_id, row.controller_id, row.generation]).changes !== 1)
|
|
172
|
+
throw new AppError("run_control_superseded", "Technical recovery operation changed before continuation", 1);
|
|
173
|
+
session.retired = true;
|
|
174
|
+
session.cleanup_pending = retired.shutdown_state !== "clean";
|
|
175
|
+
state.current_session = null;
|
|
176
|
+
state.last_continuation = null;
|
|
177
|
+
delete state.last_receipt;
|
|
178
|
+
delete state.recovery;
|
|
179
|
+
setFlowRunRecoveryPaused(context, row.project_id, row.run_id, false);
|
|
180
|
+
saveState(context, row, state);
|
|
181
|
+
appendEvent(context, row.controller_id, "technical_boundary_resumed", { ...receipt, control_id: recovery.control_id, state_dir: session.state_dir });
|
|
182
|
+
});
|
|
183
|
+
if (retired.shutdown_state !== "clean")
|
|
184
|
+
recordTechnicalNote(context, row.project_id, row.run_id, { code: "native_cleanup_pending_after_execution_exit", message: "The accepted Stage continues in a fresh Session; old native cleanup remains unconfirmed.", phase: "recovery", generation: row.generation, resource_identity: session.state_dir, diagnostic_reference: session.state_dir, consequence: "cleanup_pending" });
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
149
187
|
const call = (args, operationId) => controllerAdapter(executionContext, row, manifest, session, args, assertRecoveryOwner, operationId);
|
|
150
188
|
if (processTreeIsAlive(record)) {
|
|
151
189
|
const stopped = await call(["daemon", "stop", "--state-dir", session.state_dir, "--json"]);
|
|
@@ -157,8 +195,10 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
157
195
|
assertRecoveryOwner();
|
|
158
196
|
}
|
|
159
197
|
}
|
|
160
|
-
if (processTreeIsAlive(record)
|
|
161
|
-
throw new AppError("recovery_native_shutdown_unproven", "The previous native daemon
|
|
198
|
+
if (processTreeIsAlive(record))
|
|
199
|
+
throw new AppError("recovery_native_shutdown_unproven", "The previous native daemon still owns execution", 1);
|
|
200
|
+
if (readDaemon().shutdown_state !== "clean")
|
|
201
|
+
throw new AppError("recovery_native_shutdown_unproven", "An unfinished coordinator requires its retained native Session to be cleanly resumable", 1);
|
|
162
202
|
assertRecoveryOwner();
|
|
163
203
|
session.stopped = false;
|
|
164
204
|
saveState(context, row, state);
|
|
@@ -2,6 +2,7 @@ import crypto from "node:crypto";
|
|
|
2
2
|
import { AppError, errorRecord as structuredErrorRecord } from "../shared/errors.js";
|
|
3
3
|
import { runtimeHarness } from "./execution-policy.js";
|
|
4
4
|
import { managedProcessRuntimeHome, managedProcessStatus, processTreeIsAlive } from "./managed-processes.js";
|
|
5
|
+
import { readTechnicalNotes } from "./technical-notes.js";
|
|
5
6
|
export function requireRun(context, projectId, runId) {
|
|
6
7
|
const run = context.db.get("SELECT id, project_id, flow_kind, project_root, workspace_root, run_root, status, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
|
|
7
8
|
if (!run?.run_root)
|
|
@@ -14,10 +15,10 @@ export function requireController(context, id) {
|
|
|
14
15
|
throw new AppError("not_found", "Managed controller was not found", 1, { controller_id: id });
|
|
15
16
|
return row;
|
|
16
17
|
}
|
|
17
|
-
export function controllerReceipt(row) {
|
|
18
|
+
export function controllerReceipt(row, context) {
|
|
18
19
|
const state = JSON.parse(row.state_json);
|
|
19
20
|
const manifest = JSON.parse(row.manifest_json);
|
|
20
|
-
return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, journal: session.journal, stopped: session.stopped === true })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
|
|
21
|
+
return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", ...(context ? { technical_notes: readTechnicalNotes(context, row.project_id, row.run_id) } : {}), controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, journal: session.journal, stopped: session.stopped === true, ...(session.cleanup_pending ? { cleanup_pending: true } : {}) })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
|
|
21
22
|
}
|
|
22
23
|
/** Read-only causal diagnostics, not a settlement or capture authorization. */
|
|
23
24
|
export function terminalControllerFinalization(context, project, runId, runStatus) {
|