@deksden-com/dd-flow-cli 0.9.0-beta.137 → 0.9.0-beta.139
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +9 -0
- package/dist/build-info.json +6 -6
- package/dist/cli/argument-values.js +5 -5
- package/dist/cli/command-inputs.js +89 -20
- package/dist/cli/command-metadata.js +197 -0
- package/dist/cli/command-option-descriptions.js +191 -0
- package/dist/cli/completion-cli.js +70 -0
- package/dist/cli/completion.js +57 -0
- package/dist/cli/error-output.js +52 -0
- package/dist/cli/executor-onboarding.js +76 -0
- package/dist/cli/flows.js +29 -0
- package/dist/cli/harnesses.js +84 -0
- package/dist/cli/help.js +240 -6
- package/dist/cli/input-preparation.js +54 -5
- package/dist/cli/memory-flow-admission.js +44 -0
- package/dist/cli/memory-init-invocation.js +78 -0
- package/dist/cli/memory-init.js +240 -0
- package/dist/cli/memory-storage-admission.js +53 -0
- package/dist/cli/models.js +39 -0
- package/dist/cli/native-hook-ingress.js +1 -1
- package/dist/cli/operator-profiles.js +61 -0
- package/dist/cli/output-options.js +35 -0
- package/dist/cli/response-file.js +71 -0
- package/dist/cli/run-cli.js +283 -160
- package/dist/cli/runtime-process.js +1 -1
- package/dist/cli/setup.js +149 -0
- package/dist/cli.js +56 -2
- package/dist/domain/stage-handler-identities.js +20 -0
- package/dist/harness-runtime/lib/daemon-operations.d.mts +1 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +4 -3
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +53 -22
- package/dist/harness-runtime/lib/dd-agy.mjs +16 -5
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +12 -6
- package/dist/harness-runtime/lib/dd-codex.mjs +24 -12
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +2 -1
- package/dist/harness-runtime/lib/dd-droid.mjs +32 -9
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +11 -3
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +100 -15
- package/dist/harness-runtime/lib/dd-opencode.mjs +51 -20
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +22 -10
- package/dist/harness-runtime/lib/dd-zcode.mjs +17 -4
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +1 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +16 -3
- package/dist/harness-runtime/lib/lease-renewal.mjs +2 -1
- package/dist/harness-runtime/lib/lifecycle-command.mjs +1 -0
- package/dist/harness-runtime/lib/managed-daemon.d.mts +1 -0
- package/dist/harness-runtime/lib/managed-daemon.mjs +50 -15
- package/dist/harness-runtime/lib/managed-flow-env.mjs +15 -0
- package/dist/harness-runtime/lib/native-children.d.mts +2 -0
- package/dist/harness-runtime/lib/native-children.mjs +20 -1
- package/dist/harness-runtime/lib/native-hook-command.mjs +1 -1
- package/dist/harness-runtime/lib/native-request-proof.d.mts +2 -0
- package/dist/harness-runtime/lib/native-request-proof.mjs +65 -0
- package/dist/harness-runtime/lib/opencode-plugin.mjs +32 -9
- package/dist/harness-runtime/lib/operation-progress.mjs +3 -1
- package/dist/harness-runtime/lib/provider-overload-policy.d.mts +13 -0
- package/dist/harness-runtime/lib/provider-overload-policy.mjs +138 -0
- package/dist/runtime/context.js +1 -1
- package/dist/schemas/discovery-result.schema.json +184 -0
- package/dist/schemas/execution-bundle.schema.json +205 -0
- package/dist/schemas/execution-settings.schema.json +162 -0
- package/dist/schemas/flow-descriptor.schema.json +54 -0
- package/dist/schemas/flow-run.schema.json +26 -0
- package/dist/schemas/harness-catalog.schema.json +80 -0
- package/dist/schemas/mb-init-compose-result.schema.json +457 -0
- package/dist/schemas/mb-init-discover-result.schema.json +500 -0
- package/dist/schemas/mb-init-finish-result.schema.json +422 -0
- package/dist/schemas/mb-init-preflight-result.schema.json +367 -0
- package/dist/schemas/mb-init-review-result.schema.json +396 -0
- package/dist/schemas/mb-init-verify-result.schema.json +391 -0
- package/dist/schemas/memory-bootstrap-receipt-v2.schema.json +67 -0
- package/dist/schemas/memory-bootstrap-receipt.schema.json +354 -0
- package/dist/schemas/memory-composition-work-result.schema.json +38 -0
- package/dist/schemas/memory-repair-work-result.schema.json +49 -0
- package/dist/schemas/memory-review-correction-plan.schema.json +30 -0
- package/dist/schemas/memory-review-work-result.schema.json +43 -0
- package/dist/schemas/model-catalog.schema.json +153 -0
- package/dist/schemas/profile-settings.schema.json +122 -0
- package/dist/schemas/release-manifest.schema.json +291 -0
- package/dist/schemas/run-engine-binding.schema.json +1 -1
- package/dist/services/check-runner-evidence.js +51 -4
- package/dist/services/cli-operation-classifier.js +8 -47
- package/dist/services/cli-upgrade-allowlist.js +27 -0
- package/dist/services/code-checks.js +387 -34
- package/dist/services/controller-child-capacity.js +13 -0
- package/dist/services/controller-child-observer.js +18 -2
- package/dist/services/controller-execution.js +88 -0
- package/dist/services/controller-fanout.js +71 -12
- package/dist/services/dashboard.js +8 -2
- package/dist/services/descriptor-coordinator-session.js +35 -0
- package/dist/services/descriptor-root-work.js +53 -0
- package/dist/services/descriptor-run-settlement.js +105 -0
- package/dist/services/descriptor-run-start.js +449 -0
- package/dist/services/descriptor-stage-coordinator.js +197 -0
- package/dist/services/descriptor-stage-settlement.js +222 -0
- package/dist/services/descriptor-stage-start.js +139 -0
- package/dist/services/discovery/agy-discovery.js +2 -0
- package/dist/services/discovery/codex-discovery.js +2 -0
- package/dist/services/discovery/droid-discovery.js +2 -0
- package/dist/services/discovery/grok-discovery.js +2 -0
- package/dist/services/discovery/opencode-discovery.js +2 -0
- package/dist/services/discovery/static.js +31 -0
- package/dist/services/discovery/zcode-discovery.js +2 -0
- package/dist/services/engines.js +39 -22
- package/dist/services/eval-snapshots.js +83 -25
- package/dist/services/execution-bundles.js +263 -0
- package/dist/services/execution-policy.js +20 -16
- package/dist/services/executor-onboarding.js +56 -0
- package/dist/services/external-work-launch.js +93 -15
- package/dist/services/flow-catalog.js +21 -0
- package/dist/services/flow-descriptors.js +82 -0
- package/dist/services/frozen-engine.js +250 -0
- package/dist/services/generic-execution.js +164 -0
- package/dist/services/harness-adapter.js +28 -2
- package/dist/services/harness-discovery.js +29 -0
- package/dist/services/harness-doctor.js +75 -0
- package/dist/services/harness-probe-effects.js +51 -0
- package/dist/services/harness-version-probe.js +100 -0
- package/dist/services/hooks.js +24 -6
- package/dist/services/hybrid-contracts.js +483 -0
- package/dist/services/ids.js +29 -1
- package/dist/services/installation-lock.js +95 -0
- package/dist/services/installed-flow-source.js +183 -0
- package/dist/services/lifecycle-invocations.js +257 -78
- package/dist/services/managed-processes.js +31 -10
- package/dist/services/memory-accepted-composition.js +77 -0
- package/dist/services/memory-baseline-preservation.js +65 -0
- package/dist/services/memory-completed-correction-wave.js +320 -0
- package/dist/services/memory-compose-handler.js +258 -0
- package/dist/services/memory-composition-acceptance.js +199 -0
- package/dist/services/memory-composition-graph.js +134 -0
- package/dist/services/memory-composition-registration.js +146 -0
- package/dist/services/memory-composition-work-custody.js +151 -0
- package/dist/services/memory-composition-work-results.js +96 -0
- package/dist/services/memory-corrected-review-acceptance.js +129 -0
- package/dist/services/memory-corrected-review-batch.js +30 -0
- package/dist/services/memory-corrected-review-findings.js +106 -0
- package/dist/services/memory-corrected-review-inputs.js +210 -0
- package/dist/services/memory-corrected-review-projection.js +78 -0
- package/dist/services/memory-corrected-review-receipt.js +170 -0
- package/dist/services/memory-corrected-review-registration.js +191 -0
- package/dist/services/memory-correction-check-receipt.js +219 -0
- package/dist/services/memory-correction-input-owner.js +6 -0
- package/dist/services/memory-discover-handler.js +104 -0
- package/dist/services/memory-finish-evidence.js +435 -0
- package/dist/services/memory-finish-handler.js +215 -0
- package/dist/services/memory-finish-integration.js +280 -0
- package/dist/services/memory-finish-readiness.js +122 -0
- package/dist/services/memory-git-integration.js +151 -0
- package/dist/services/memory-historical-corrected-review.js +206 -0
- package/dist/services/memory-historical-original-review.js +130 -0
- package/dist/services/memory-historical-repair-registration.js +99 -0
- package/dist/services/memory-historical-second-review.js +128 -0
- package/dist/services/memory-init-bootstrap-readiness.js +210 -0
- package/dist/services/memory-init-contexts.js +89 -0
- package/dist/services/memory-init-controller-start.js +84 -0
- package/dist/services/memory-init-engine-release.js +223 -0
- package/dist/services/memory-init-git-policy.js +136 -0
- package/dist/services/memory-init-lint-capability.js +185 -0
- package/dist/services/memory-init-native-capacity.js +296 -0
- package/dist/services/memory-init-preflight.js +330 -0
- package/dist/services/memory-init-project-bootstrap.js +599 -0
- package/dist/services/memory-init-request-candidate.js +157 -0
- package/dist/services/memory-init-request-input.js +66 -0
- package/dist/services/memory-init-resource-storage.js +125 -0
- package/dist/services/memory-init-run-allocation.js +81 -0
- package/dist/services/memory-init-run-bundle.js +98 -0
- package/dist/services/memory-init-start.js +181 -0
- package/dist/services/memory-init-startup-custody.js +80 -0
- package/dist/services/memory-init-startup-input.js +240 -0
- package/dist/services/memory-init-storage.js +81 -0
- package/dist/services/memory-init-stores.js +359 -0
- package/dist/services/memory-init-worktrunk-capability.js +178 -0
- package/dist/services/memory-layer-publication.js +611 -0
- package/dist/services/memory-layer.js +156 -0
- package/dist/services/memory-lint-config.js +85 -0
- package/dist/services/memory-native-review-copy.js +165 -0
- package/dist/services/memory-normal-flow-entry.js +75 -0
- package/dist/services/memory-ordinary-defaults.js +112 -0
- package/dist/services/memory-preflight-bootstrap-custody.js +115 -0
- package/dist/services/memory-preflight-handler.js +259 -0
- package/dist/services/memory-repair-current-view.js +299 -0
- package/dist/services/memory-repair-document-delta.js +451 -0
- package/dist/services/memory-repair-history.js +491 -0
- package/dist/services/memory-repair-physical-binding.js +312 -0
- package/dist/services/memory-repair-publication-files.js +152 -0
- package/dist/services/memory-repair-publication-intent.js +27 -0
- package/dist/services/memory-repair-publication-plan.js +160 -0
- package/dist/services/memory-repair-publication-source.js +225 -0
- package/dist/services/memory-repair-publication.js +794 -0
- package/dist/services/memory-repair-registration.js +239 -0
- package/dist/services/memory-repair-work-baseline.js +413 -0
- package/dist/services/memory-repair-work-completion.js +97 -0
- package/dist/services/memory-repair-work-custody.js +73 -0
- package/dist/services/memory-repair-work-results.js +89 -0
- package/dist/services/memory-repair-workspace.js +252 -0
- package/dist/services/memory-repair-writer-copy.js +735 -0
- package/dist/services/memory-result-binding.js +173 -0
- package/dist/services/memory-retained-review-authority.js +129 -0
- package/dist/services/memory-retained-review-results.js +70 -0
- package/dist/services/memory-retained-second-baseline.js +102 -0
- package/dist/services/memory-retained-second-repairs.js +156 -0
- package/dist/services/memory-review-acceptance.js +214 -0
- package/dist/services/memory-review-correction-plan.js +171 -0
- package/dist/services/memory-review-handler.js +585 -0
- package/dist/services/memory-review-inputs.js +200 -0
- package/dist/services/memory-review-receipt.js +180 -0
- package/dist/services/memory-review-registration.js +108 -0
- package/dist/services/memory-review-work-custody.js +124 -0
- package/dist/services/memory-review-work-input.js +72 -0
- package/dist/services/memory-review-work-results.js +73 -0
- package/dist/services/memory-second-corrected-review-inputs.js +165 -0
- package/dist/services/memory-second-correction-plan.js +76 -0
- package/dist/services/memory-second-repair-baseline.js +91 -0
- package/dist/services/memory-second-repair-launch.js +614 -0
- package/dist/services/memory-second-repair-registration.js +88 -0
- package/dist/services/memory-second-repair-source.js +187 -0
- package/dist/services/memory-second-review-acceptance.js +115 -0
- package/dist/services/memory-source-observations.js +117 -0
- package/dist/services/memory-stage-dispatch.js +28 -0
- package/dist/services/memory-stage-results.js +199 -0
- package/dist/services/memory-verification-acceptance.js +135 -0
- package/dist/services/memory-verification-checks.js +252 -0
- package/dist/services/memory-verification-receipt.js +159 -0
- package/dist/services/memory-verify-handler.js +135 -0
- package/dist/services/memory-work-result-preparation.js +63 -0
- package/dist/services/merge-server.js +2 -4
- package/dist/services/model-catalog.js +109 -0
- package/dist/services/model-discovery.js +46 -0
- package/dist/services/operator-profiles.js +236 -0
- package/dist/services/projects.js +17 -2
- package/dist/services/{codex-capacity.js → provider-capacity.js} +34 -16
- package/dist/services/recovery-snapshot-database.js +417 -2
- package/dist/services/release-archive-producer.js +52 -0
- package/dist/services/release-archive.js +143 -0
- package/dist/services/release-cache.js +142 -0
- package/dist/services/release-compatibility.js +18 -0
- package/dist/services/release-download.js +67 -0
- package/dist/services/release-envelope.js +104 -0
- package/dist/services/release-manifest.js +45 -0
- package/dist/services/release-payload.js +114 -0
- package/dist/services/release-producer.js +63 -0
- package/dist/services/release-provenance.js +160 -0
- package/dist/services/release-registry.js +66 -0
- package/dist/services/release-source.js +194 -0
- package/dist/services/repair-intents.js +1 -1
- package/dist/services/review-copy.js +10 -2
- package/dist/services/run-control.js +80 -9
- package/dist/services/run-controller-adapter.js +1 -1
- package/dist/services/run-controller-capture.js +10 -3
- package/dist/services/run-controller-process.js +21 -0
- package/dist/services/run-controller-recovery.js +5 -5
- package/dist/services/run-controller-state.js +1 -1
- package/dist/services/run-controller.js +158 -64
- package/dist/services/run-recovery-guard.js +5 -3
- package/dist/services/run-recovery-runtime.js +3 -3
- package/dist/services/run-recovery.js +145 -5
- package/dist/services/run-start-custody.js +177 -0
- package/dist/services/runs.js +413 -30
- package/dist/services/setup.js +377 -0
- package/dist/services/stage-context.js +9 -1
- package/dist/services/stage-handlers.js +74 -0
- package/dist/services/stage-report-renderer.js +63 -0
- package/dist/services/stage-work-graph.js +17 -0
- package/dist/services/vnext-code-review.js +1 -1
- package/dist/services/vnext-code.js +6 -6
- package/dist/services/vnext-execution-profile.js +2 -2
- package/dist/services/vnext-fanout.js +73 -11
- package/dist/services/vnext-merge.js +8 -3
- package/dist/services/vnext-plan-review.js +7 -5
- package/dist/services/vnext-plan.js +3 -3
- package/dist/services/vnext-protocolize.js +20 -6
- package/dist/services/vnext-workspace-policy.js +7 -2
- package/dist/services/work-registry.js +616 -71
- package/dist/services/workspace-files.js +1 -1
- package/dist/storage/database.js +84 -6
- package/dist/storage/writer-contract.js +11 -7
- package/package.json +13 -3
- package/tools/create-canon-release.mjs +33 -0
- package/tools/produce-hybrid-release.mjs +41 -0
- package/dist/harness-runtime/lib/codex-capacity-policy.d.mts +0 -10
- package/dist/harness-runtime/lib/codex-capacity-policy.mjs +0 -102
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import { assertLifecycleOutcomes, currentLifecycleFailure, managedInvocationContext, managedLifecycleCommand } from "./lifecycle-invocations.js";
|
|
2
|
+
export function recoveryPacketMatchesStage(packet, scope) {
|
|
3
|
+
return packet.schema_id === 'dd-flow/root-recovery-packet@1' && packet.continuation === 'unfinished_work'
|
|
4
|
+
&& packet.run_id === scope.runId && packet.generation === scope.generation && packet.stage === scope.stage
|
|
5
|
+
&& packet.work_id === scope.workId && packet.paused === false;
|
|
6
|
+
}
|
|
2
7
|
import crypto from "node:crypto";
|
|
3
8
|
import fs from "node:fs";
|
|
4
9
|
import path from "node:path";
|
|
@@ -13,10 +18,11 @@ import { prepareFileBytes, prepareJsonText } from "../cli/input-preparation.js";
|
|
|
13
18
|
import { parseRuntimeBudget } from "./runtime-budget.js";
|
|
14
19
|
import { requireProjectByRoot } from "./projects.js";
|
|
15
20
|
import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
|
|
16
|
-
import { getFlowRunStatus, setFlowRunRecoveryPaused } from "./runs.js";
|
|
21
|
+
import { getFlowRunStatus, setFlowRunRecoveryPaused, assertPreparedRunBundleSqlAuthority } from "./runs.js";
|
|
17
22
|
import { loadExternalStageContext } from "./stage-context.js";
|
|
18
23
|
import { flowCommand } from "./stage-pause.js";
|
|
19
|
-
import {
|
|
24
|
+
import { sameSessionProfile, runtimeHarness } from "./execution-policy.js";
|
|
25
|
+
import { selectControllerExecution } from "./controller-execution.js";
|
|
20
26
|
import { resolveHarnessCommand, harnessRuntimeArguments } from "./harness-config.js";
|
|
21
27
|
import { adapterSessionId, prepareHarnessFlowExecutable, runHarnessAdapter } from "./harness-adapter.js";
|
|
22
28
|
import { assertManagedProcessLease, confirmManagedProcess, renewManagedProcess, finishManagedProcess, processStartedAt, processTreeIsAlive, resourceHome } from "./managed-processes.js";
|
|
@@ -29,17 +35,24 @@ import { claimQueuedMergeRequest } from "./merge-server.js";
|
|
|
29
35
|
import { prepareVnextCodeWorkspace } from "./vnext-code.js";
|
|
30
36
|
import { requireRun, requireController, controllerReceipt, appendEvent, saveState, retainControllerContinuation, retainControllerCapacityIntent, digest, digestBytes, errorRecord } from "./run-controller-state.js";
|
|
31
37
|
import { assertNoCorrelatedRejection } from "./continuation-outcome.js";
|
|
32
|
-
import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady,
|
|
38
|
+
import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady, assertControllerLeaseAllowed } from "./run-controller-process.js";
|
|
33
39
|
import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
|
|
34
40
|
import { retainedChildObserver } from "./controller-child-observer.js";
|
|
35
|
-
import { capacityDelay,
|
|
41
|
+
import { capacityDelay, continueProviderCapacity, eligibleProviderChildOverload, providerChildFailure } from "./provider-capacity.js";
|
|
36
42
|
import { resumeControllerSession, failControllerRecovery } from "./run-controller-recovery.js";
|
|
37
43
|
import { captureControllerBoundary } from "./run-controller-capture.js";
|
|
38
44
|
import { requestRunControl } from "./run-control-worker.js";
|
|
39
45
|
import { storageSessionId } from "./session-identity.js";
|
|
40
|
-
import { CAPACITY_POLICY, nativeItemsSettled,
|
|
46
|
+
import { CAPACITY_POLICY, nativeItemsSettled, overloadExhausted, refusalObservation, terminalProviderOverload } from "../harness-runtime/lib/provider-overload-policy.mjs";
|
|
47
|
+
import { renderNativeChildFollowupInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
|
|
48
|
+
import { closeSuccessfulChildCapacity } from "./controller-child-capacity.js";
|
|
41
49
|
import { isObservationLoss } from "../harness-runtime/lib/operation-errors.mjs";
|
|
42
50
|
export { resumeRunController } from "./run-controller-recovery.js";
|
|
51
|
+
const preparedLaunchInputs = new WeakMap();
|
|
52
|
+
function launchInputDigest(input) {
|
|
53
|
+
return digest([input.projectRoot, input.runId, input.requestId, input.contextFile, input.contextSha256,
|
|
54
|
+
input.stopAfter ?? null, input.captureRoot ?? null]);
|
|
55
|
+
}
|
|
43
56
|
export function prepareRunControllerLaunch(context, input) {
|
|
44
57
|
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !/^[a-f0-9]{64}$/.test(input.contextSha256))
|
|
45
58
|
throw new AppError("usage", "Managed launch needs a request id and context SHA-256", 2);
|
|
@@ -60,58 +73,79 @@ export function prepareRunControllerLaunch(context, input) {
|
|
|
60
73
|
if (existing) {
|
|
61
74
|
if (existing.request_hash !== requestHash)
|
|
62
75
|
throw new AppError("runtime_request_conflict", "Managed launch request id has different inputs", 1);
|
|
63
|
-
|
|
76
|
+
selectControllerExecution(context, run, JSON.parse(existing.manifest_json));
|
|
77
|
+
const prepared = { kind: "replay", existing };
|
|
78
|
+
preparedLaunchInputs.set(prepared, { home: context.ddFlowHome, sha256: launchInputDigest(input) });
|
|
79
|
+
return prepared;
|
|
64
80
|
}
|
|
65
81
|
const generation = assertRunMutationAllowed(context, project.id, run.id);
|
|
66
82
|
if (run.status !== "running")
|
|
67
83
|
throw new AppError("invalid_run_state", "Managed launch requires a running logical RUN", 1, { status: run.status });
|
|
68
|
-
const
|
|
69
|
-
const execution = index.execution_profile;
|
|
70
|
-
if (!execution?.settings.execution || !execution.agent_profiles)
|
|
71
|
-
throw new AppError("execution_profile_not_frozen", "Managed launch requires the RUN's frozen execution profiles", 1);
|
|
84
|
+
const routing = selectControllerExecution(context, run);
|
|
72
85
|
const contexts = readContexts(input.contextFile, input.contextSha256);
|
|
86
|
+
if (routing.bundle) {
|
|
87
|
+
for (const stage of [...Object.keys(contexts), ...(input.stopAfter ? [input.stopAfter] : [])])
|
|
88
|
+
routing.bundle.descriptor.stage(stage);
|
|
89
|
+
}
|
|
73
90
|
const next = getFlowRunStatus(context, { projectRoot: project.root, runId: run.id }).continuation;
|
|
74
91
|
if (next.kind !== "start_stage" || !next.stage)
|
|
75
92
|
throw new AppError("controller_attach_required", "Managed launch cannot replace an attached or unfinished coordinator; reconcile its existing owner", 1, { continuation: next });
|
|
76
93
|
if (!contexts[next.stage])
|
|
77
94
|
throw new AppError("controller_context_missing", "Managed context omits the first stage", 2, { stage: next.stage });
|
|
78
95
|
const adapters = {};
|
|
79
|
-
for (const profile of Object.values(
|
|
96
|
+
for (const profile of Object.values(routing.profiles)) {
|
|
80
97
|
const harness = runtimeHarness(profile);
|
|
81
98
|
adapters[harness] = { executable: resolveHarnessCommand(context.ddFlowHome, harness, "adapter"), runtime_args: harnessRuntimeArguments(context.ddFlowHome, harness) };
|
|
82
99
|
}
|
|
83
100
|
const entry = fileURLToPath(new URL("../cli.js", import.meta.url));
|
|
84
101
|
if (!fs.existsSync(entry))
|
|
85
102
|
throw new AppError("harness_runtime_unbuilt", "Build the CLI before launching its detached controller", 1);
|
|
86
|
-
|
|
103
|
+
const prepared = { kind: "create", project, run, request, requestHash, generation, budget, routing, contexts, adapters };
|
|
104
|
+
preparedLaunchInputs.set(prepared, { home: context.ddFlowHome, sha256: launchInputDigest(input) });
|
|
105
|
+
return prepared;
|
|
87
106
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
const
|
|
107
|
+
/** Retain the existing launch intent and frozen inputs without starting a
|
|
108
|
+
* process. Dispatch remains exclusively owned by launchRunController. */
|
|
109
|
+
export function retainRunControllerLaunch(context, input, prepared = prepareRunControllerLaunch(context, input)) {
|
|
110
|
+
const selection = preparedLaunchInputs.get(prepared);
|
|
111
|
+
if (!selection || selection.home !== context.ddFlowHome || selection.sha256 !== launchInputDigest(input)) {
|
|
112
|
+
throw new AppError("runtime_request_conflict", "Controller launch differs from its prepared request", 1, { effect: "no_effect" });
|
|
113
|
+
}
|
|
114
|
+
if (prepared.kind === "replay") {
|
|
115
|
+
const current = requireController(context, prepared.existing.controller_id);
|
|
116
|
+
if (current.request_hash !== prepared.existing.request_hash)
|
|
117
|
+
throw new AppError("runtime_request_conflict", "Controller launch authority changed after replay preparation", 1);
|
|
118
|
+
selectControllerExecution(context, requireRun(context, current.project_id, current.run_id), JSON.parse(current.manifest_json));
|
|
119
|
+
return { controller: current, reused: true };
|
|
120
|
+
}
|
|
121
|
+
const { project, run, request, requestHash, generation, budget, routing, contexts, adapters } = prepared;
|
|
92
122
|
assertRunMutationAllowed(context, project.id, run.id, generation);
|
|
93
123
|
ensureRunControllerStorage(context.db);
|
|
94
124
|
const controllerId = `DRV-${crypto.randomUUID()}`;
|
|
95
125
|
const root = path.join(run.run_root, "controllers", controllerId);
|
|
96
126
|
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
97
|
-
const flowExecutable = prepareHarnessFlowExecutable(context, root);
|
|
127
|
+
const flowExecutable = prepareHarnessFlowExecutable(context, root, routing.bundle?.engine());
|
|
98
128
|
const frozenContexts = {};
|
|
99
129
|
for (const [stage, value] of Object.entries(contexts)) {
|
|
100
130
|
const file = path.join(root, `${stage}.context.json`);
|
|
101
131
|
fs.writeFileSync(file, value.bytes, { flag: "wx", mode: 0o600 });
|
|
102
132
|
frozenContexts[stage] = { file, sha256: value.sha256 };
|
|
103
133
|
}
|
|
104
|
-
const manifest = { schema_id: "dd-flow/run-controller-manifest@
|
|
134
|
+
const manifest = { schema_id: routing.kind === "descriptor" ? "dd-flow/run-controller-manifest@2" : "dd-flow/run-controller-manifest@1", request,
|
|
135
|
+
...(routing.kind === "descriptor" ? { execution_binding: routing.binding } : { execution_profile: routing.legacy }),
|
|
136
|
+
contexts: frozenContexts, adapters, flow_executable: flowExecutable, resource_home: resourceHome(context), budget };
|
|
105
137
|
const token = crypto.randomUUID();
|
|
106
138
|
const now = context.now();
|
|
107
139
|
const state = { stage: null, sessions: [], current_session: null, turns: 0, last_continuation: null };
|
|
108
140
|
const launcher = { pid: process.pid, pid_started_at: processStartedAt(process.pid) };
|
|
109
141
|
const duplicate = context.db.writeTransaction(() => {
|
|
142
|
+
if (routing.bundle)
|
|
143
|
+
assertPreparedRunBundleSqlAuthority(context, { projectId: project.id, runId: run.id }, routing.bundle);
|
|
110
144
|
const duplicate = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND request_id = ?", [project.id, run.id, input.requestId]);
|
|
111
145
|
if (duplicate) {
|
|
112
146
|
if (duplicate.request_hash !== requestHash)
|
|
113
147
|
throw new AppError("runtime_request_conflict", "Managed launch request id has different inputs", 1);
|
|
114
|
-
return
|
|
148
|
+
return duplicate;
|
|
115
149
|
}
|
|
116
150
|
assertRunMutationAllowed(context, project.id, run.id, generation);
|
|
117
151
|
if (requireRun(context, project.id, run.id).status !== "running")
|
|
@@ -126,9 +160,16 @@ export async function launchRunController(context, input, prepared = prepareRunC
|
|
|
126
160
|
appendEvent(context, controllerId, "launch_requested", { request_id: input.requestId, generation, request_hash: requestHash });
|
|
127
161
|
return null;
|
|
128
162
|
});
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
163
|
+
return { controller: duplicate ?? requireController(context, controllerId), reused: duplicate !== null };
|
|
164
|
+
}
|
|
165
|
+
export async function launchRunController(context, input, prepared = prepareRunControllerLaunch(context, input)) {
|
|
166
|
+
const retained = retainRunControllerLaunch(context, input, prepared);
|
|
167
|
+
// A simultaneous publisher already owns dispatch. Preserve the original
|
|
168
|
+
// race behavior: only an explicit replay may reconcile that existing owner.
|
|
169
|
+
if (prepared.kind === "create" && retained.reused)
|
|
170
|
+
return { ...controllerReceipt(retained.controller), reused: true };
|
|
171
|
+
const receipt = await spawnController(context, retained.controller);
|
|
172
|
+
return retained.reused ? { ...receipt, reused: true } : receipt;
|
|
132
173
|
}
|
|
133
174
|
/**
|
|
134
175
|
* A registry write can be temporarily unavailable while another runtime client
|
|
@@ -179,6 +220,7 @@ export async function serveRunController(context, input) {
|
|
|
179
220
|
throw new AppError("controller_owner_mismatch", "Managed owner has no resource lease", 1);
|
|
180
221
|
const confirmed = confirmManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
|
|
181
222
|
const lease = controllerLeaseMonitor(attemptMs => renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: row.process_id, leaseToken: row.process_lease_token }), confirmed.lease_expires_at);
|
|
223
|
+
const state = JSON.parse(row.state_json);
|
|
182
224
|
// Resource and RUN commits are distinct effects. A RUN keepalive failure must
|
|
183
225
|
// neither erase a resource receipt nor grant RUN dispatch authority.
|
|
184
226
|
const mergeLease = controllerLeaseMonitor(attemptMs => {
|
|
@@ -194,22 +236,23 @@ export async function serveRunController(context, input) {
|
|
|
194
236
|
return { owned: true, lease_expires_at: expiry };
|
|
195
237
|
});
|
|
196
238
|
const assertLeaseFence = () => {
|
|
197
|
-
|
|
198
|
-
if (current.owner_token !== row.owner_token || current.generation !== row.generation)
|
|
199
|
-
throw new AppError("controller_owner_mismatch", "Lease renewal no longer owns this controller generation", 1);
|
|
200
|
-
assertControllerRecoveryAllowed(context, current);
|
|
239
|
+
assertControllerLeaseAllowed(context, row);
|
|
201
240
|
};
|
|
202
241
|
const heartbeat = setInterval(() => {
|
|
203
242
|
lease.background(assertLeaseFence, () => { });
|
|
204
|
-
|
|
243
|
+
if (state.merge_claim)
|
|
244
|
+
mergeLease.background(assertLeaseFence, () => { });
|
|
205
245
|
}, 30_000);
|
|
206
246
|
heartbeat.unref();
|
|
207
247
|
const manifest = JSON.parse(row.manifest_json);
|
|
208
|
-
const state = JSON.parse(row.state_json);
|
|
209
248
|
const assertPhysicalOwner = () => {
|
|
210
249
|
lease.assertNoRenewal();
|
|
211
|
-
|
|
212
|
-
|
|
250
|
+
if (state.merge_claim)
|
|
251
|
+
mergeLease.assertNoRenewal();
|
|
252
|
+
// This isolated read already bounds SQLite contention. A wall deadline
|
|
253
|
+
// before opening the reader mistakes scheduler delay for heartbeat loss;
|
|
254
|
+
// the observed row's actual expiry/token still fences productive dispatch.
|
|
255
|
+
assertManagedProcessLease(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid });
|
|
213
256
|
const current = requireController(context, row.controller_id);
|
|
214
257
|
if (current.owner_pid !== process.pid || current.owner_token !== row.owner_token || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
|
|
215
258
|
throw new AppError("controller_owner_mismatch", "Controller no longer owns productive dispatch", 1);
|
|
@@ -221,7 +264,8 @@ export async function serveRunController(context, input) {
|
|
|
221
264
|
};
|
|
222
265
|
try {
|
|
223
266
|
await lease.ensureConfirmed(assertLeaseFence);
|
|
224
|
-
|
|
267
|
+
if (state.merge_claim)
|
|
268
|
+
await mergeLease.ensureConfirmed(assertLeaseFence);
|
|
225
269
|
context.db.writeTransaction(() => {
|
|
226
270
|
markControllerDispatchReady(context, row);
|
|
227
271
|
if (!state.recovery && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation && requireRun(context, row.project_id, row.run_id).status === "paused")
|
|
@@ -230,7 +274,8 @@ export async function serveRunController(context, input) {
|
|
|
230
274
|
appendEvent(context, row.controller_id, "owner_started", { pid: process.pid, generation: row.generation });
|
|
231
275
|
if (state.recovery && !["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
|
|
232
276
|
await resumeControllerSession(context, row, manifest, state, assertPhysicalOwner);
|
|
233
|
-
await executeController(context, row, manifest, state, assertOwner, assertPhysicalOwner, async () => { await lease.ensureConfirmed(assertLeaseFence);
|
|
277
|
+
await executeController(context, row, manifest, state, assertOwner, assertPhysicalOwner, async () => { await lease.ensureConfirmed(assertLeaseFence); if (state.merge_claim)
|
|
278
|
+
await mergeLease.ensureConfirmed(assertLeaseFence); });
|
|
234
279
|
return controllerReceipt(requireController(context, row.controller_id));
|
|
235
280
|
}
|
|
236
281
|
catch (error) {
|
|
@@ -449,8 +494,9 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
449
494
|
}
|
|
450
495
|
}
|
|
451
496
|
};
|
|
452
|
-
const
|
|
453
|
-
const
|
|
497
|
+
const routing = selectControllerExecution(context, requireRun(context, row.project_id, row.run_id), manifest);
|
|
498
|
+
const policy = routing.policy;
|
|
499
|
+
const profiles = routing.profiles;
|
|
454
500
|
let executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined, DD_FLOW_CODE_READINESS_OWNER: row.controller_id } };
|
|
455
501
|
const bindInvocationScope = (session) => {
|
|
456
502
|
executionContext = managedInvocationContext(executionContext, { harness: runtimeHarness(session.profile), stateDir: session.state_dir, sessionId: session.id,
|
|
@@ -640,14 +686,12 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
640
686
|
}
|
|
641
687
|
}
|
|
642
688
|
catch (error) {
|
|
643
|
-
if (!["codex-desktop", "opencode-server"].includes(runtimeHarness(session.profile)))
|
|
644
|
-
throw error;
|
|
645
689
|
const resumeCommand = hitl ? text.split("\n\n")[1] : undefined;
|
|
646
690
|
if (hitl && !resumeCommand?.includes(" stage resume "))
|
|
647
691
|
throw error;
|
|
648
692
|
const inspect = (args, id) => runHarnessAdapter({ executable: manifest.adapters[runtimeHarness(session.profile)].executable,
|
|
649
693
|
args, env: controllerEnvironment(context, manifest, session), runtimeOwner: session.owner, operationId: id, timeoutMs: 30_000, signal: cancellation.signal });
|
|
650
|
-
const resumed = await
|
|
694
|
+
const resumed = await continueProviderCapacity({ error, sessionId: session.id, operationId: firstOperationId, signal: cancellation.signal, allowSettledDescendants: true, atomicIntent: true,
|
|
651
695
|
retainedIntents: () => context.db.all("SELECT receipt_json FROM run_controller_operations WHERE controller_id = ? AND generation = ?", [row.controller_id, row.generation])
|
|
652
696
|
.flatMap(item => { const retained = JSON.parse(item.receipt_json || "null"); return retained?.capacity_intent?.root_operation_id === firstOperationId ? [retained.capacity_intent] : []; }).sort((a, b) => a.ordinal - b.ordinal),
|
|
653
697
|
retainedFailures: () => context.db.all("SELECT receipt_json FROM run_controller_operations WHERE controller_id = ? AND generation = ?", [row.controller_id, row.generation])
|
|
@@ -745,7 +789,10 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
745
789
|
if (state.current_session !== null && !state.sessions[state.current_session].stopped)
|
|
746
790
|
bindInvocationScope(state.sessions[state.current_session]);
|
|
747
791
|
const run = requireRun(context, row.project_id, row.run_id);
|
|
748
|
-
|
|
792
|
+
const view = getFlowRunStatus(executionContext, { projectRoot: run.project_root, runId: run.id });
|
|
793
|
+
const next = view.continuation;
|
|
794
|
+
terminalFinalization = next.kind === "terminal";
|
|
795
|
+
if (state.pending_capacity && next.kind !== "terminal" && !["done", "failed", "cancelled"].includes(run.status)) {
|
|
749
796
|
const pending = state.pending_capacity;
|
|
750
797
|
const session = state.sessions.find(item => item.id === pending.session_id && !item.stopped);
|
|
751
798
|
const operation = context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [pending.root_operation_id, row.controller_id, row.generation]);
|
|
@@ -757,9 +804,6 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
757
804
|
await prompt(session, fs.readFileSync(file, "utf8"), pending.root_operation_id, pending.kind === "stage" ? undefined : pending.kind);
|
|
758
805
|
continue;
|
|
759
806
|
}
|
|
760
|
-
const view = getFlowRunStatus(executionContext, { projectRoot: run.project_root, runId: run.id });
|
|
761
|
-
const next = view.continuation;
|
|
762
|
-
terminalFinalization = next.kind === "terminal";
|
|
763
807
|
const completed = state.stage && view.index.stage_runs.find(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status));
|
|
764
808
|
if (state.current_session !== null && (completed || next.stage !== state.stage) && childObserver(state.sessions[state.current_session])?.failed().length)
|
|
765
809
|
throw new AppError("fanout_dispatch_fenced", "A retained child failure forbids completing or advancing its Stage", 1);
|
|
@@ -833,8 +877,8 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
833
877
|
loadExternalStageContext({ file: stageContext.file, expectedSha256: stageContext.sha256, stage: next.stage });
|
|
834
878
|
const continuation = JSON.stringify(["start_stage", next.stage, next.attempt ?? null]);
|
|
835
879
|
retainControllerContinuation(state, continuation, { code: "stage_entry_nonprogressing", message: "The coordinator returned without starting the assigned Stage", details: { stage: next.stage, attempt: next.attempt ?? null, last_receipt: state.last_receipt ?? null } });
|
|
836
|
-
const transition =
|
|
837
|
-
if (next.stage === "merge" &&
|
|
880
|
+
const transition = routing.transition(next.stage, state.stage ?? undefined, run);
|
|
881
|
+
if (routing.kind === "sdlc" && next.stage === "merge" && routing.mergeMode === "server" && !state.merge_claim) {
|
|
838
882
|
const request = context.db.get("SELECT merge_request_id, status FROM merge_requests WHERE project_id = ? AND run_id = ? ORDER BY created_at DESC, merge_request_id DESC LIMIT 1", [row.project_id, row.run_id]);
|
|
839
883
|
if (!request || request.status !== "queued")
|
|
840
884
|
throw new AppError("controller_attach_required", "Managed MERGE needs an unclaimed queued request; reconcile an existing owner", 1, { request });
|
|
@@ -847,10 +891,26 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
847
891
|
state.merge_claim = { request_id: claim.request.merge_request_id, token: claim.token, target_workspace: claim.request.target_workspace, work_id: claim.request.executor_work_id };
|
|
848
892
|
saveState(context, row, state);
|
|
849
893
|
appendEvent(context, row.controller_id, "merge_claimed", { request_id: claim.request.merge_request_id });
|
|
894
|
+
await renewLease();
|
|
850
895
|
}
|
|
851
|
-
const stageCwd =
|
|
896
|
+
const stageCwd = routing.cwd(next.stage, state.merge_claim?.target_workspace, run);
|
|
897
|
+
const routeBinding = (current) => {
|
|
898
|
+
const index = JSON.parse(current.index_json);
|
|
899
|
+
return JSON.stringify([current.project_root, current.workspace_root, index.workspace_route ?? null, state.merge_claim ?? null]);
|
|
900
|
+
};
|
|
901
|
+
const preparedRoute = routeBinding(run);
|
|
902
|
+
const assertRoute = async () => {
|
|
903
|
+
assertOwner();
|
|
904
|
+
if (routeBinding(requireRun(context, row.project_id, row.run_id)) !== preparedRoute) {
|
|
905
|
+
const current = state.current_session === null ? null : state.sessions[state.current_session];
|
|
906
|
+
if (current && !current.stopped)
|
|
907
|
+
await stopSession(current);
|
|
908
|
+
throw new AppError("workspace_route_changed", "RUN workspace changed during native Session preparation", 1, { effect: "no_effect", stage: next.stage });
|
|
909
|
+
}
|
|
910
|
+
};
|
|
911
|
+
await assertRoute();
|
|
852
912
|
let session = state.current_session === null ? null : state.sessions[state.current_session];
|
|
853
|
-
if (session && next.stage === "merge" &&
|
|
913
|
+
if (session && routing.kind === "sdlc" && next.stage === "merge" && routing.mergeMode === "same_session" && path.resolve(session.cwd) !== stageCwd) {
|
|
854
914
|
throw new AppError("merge_session_workspace_conflict", "Same-session MERGE cannot change the native Session working directory", 1, { session_cwd: session.cwd, target_cwd: stageCwd });
|
|
855
915
|
}
|
|
856
916
|
if (!session || transition.session_mode === "new_session" || !sameSessionProfile(session.profile, transition.coordinator) || path.resolve(session.cwd) !== stageCwd) {
|
|
@@ -890,7 +950,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
890
950
|
state.child_capacity = {};
|
|
891
951
|
saveState(context, row, state);
|
|
892
952
|
setStatus("running");
|
|
893
|
-
if (next.stage === "code" || next.stage === "code-review") {
|
|
953
|
+
if (routing.kind === "sdlc" && (next.stage === "code" || next.stage === "code-review")) {
|
|
894
954
|
// A CODE_REVIEW fork has source files but intentionally no installed
|
|
895
955
|
// dependencies. Its repair Work executes checks just like CODE Work.
|
|
896
956
|
const readiness = await prepareVnextCodeWorkspace(executionContext, { projectRoot: run.project_root, runId: run.id, ownerId: row.controller_id, stage: next.stage });
|
|
@@ -900,6 +960,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
900
960
|
}
|
|
901
961
|
const responseFile = path.join(row.state_dir, `${next.stage}-${next.attempt ?? 1}.stage-start-response.json`);
|
|
902
962
|
const command = managedLifecycleCommand(executionContext, `${flowCommand(executionContext)} stage start ${run.id} --stage ${next.stage} --project-root ${JSON.stringify(run.project_root)} --context-file ${JSON.stringify(stageContext.file)} --context-sha256 ${stageContext.sha256} --require-session-binding --response-file ${JSON.stringify(responseFile)} --json --progress-jsonl`);
|
|
963
|
+
await assertRoute();
|
|
903
964
|
appendEvent(context, row.controller_id, "stage_entered", { stage: next.stage, session_id: session.id, context_sha256: stageContext.sha256 });
|
|
904
965
|
await prompt(session, controllerStageEntryPrompt(next.stage, command));
|
|
905
966
|
continue;
|
|
@@ -907,11 +968,19 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
907
968
|
if (state.current_session === null)
|
|
908
969
|
throw new AppError("controller_attach_required", "An unfinished Stage has no managed coordinator", 1);
|
|
909
970
|
const session = state.sessions[state.current_session];
|
|
971
|
+
const childHarness = runtimeHarness(session.profile);
|
|
910
972
|
const baseline = new Set(state.native_baseline ?? []);
|
|
911
973
|
let children = controllerNativeChildren(state.last_receipt ?? {}, session.id).filter(child => !baseline.has(child.session_id));
|
|
974
|
+
childObserver(session)?.observeSettledChildren(children);
|
|
912
975
|
const retainedFailures = childObserver(session)?.failed() ?? [];
|
|
913
976
|
children = controllerNativeChildren({ descendants: [...children, ...retainedFailures] }, session.id);
|
|
914
|
-
|
|
977
|
+
const closedChildEpisodes = closeSuccessfulChildCapacity(state, children, session.id);
|
|
978
|
+
if (closedChildEpisodes.length)
|
|
979
|
+
await prepareMutation(() => {
|
|
980
|
+
saveState(context, row, state);
|
|
981
|
+
appendEvent(context, row.controller_id, "capacity_child_episodes_completed", { child_session_ids: closedChildEpisodes });
|
|
982
|
+
});
|
|
983
|
+
if (children.some(child => child.status === "failed" && renderNativeChildFollowupInstruction(childHarness, child.session_id))) {
|
|
915
984
|
const inspected = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
916
985
|
state.last_receipt = inspected;
|
|
917
986
|
saveState(context, row, state);
|
|
@@ -919,32 +988,31 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
919
988
|
const failed = children.filter(child => child.status === "failed");
|
|
920
989
|
// Freeze each distinct child refusal on its first exact observation,
|
|
921
990
|
// even while a healthy sibling still owns an active native Turn.
|
|
922
|
-
for (const failedChild of failed.filter(item =>
|
|
923
|
-
const linked = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id:
|
|
991
|
+
for (const failedChild of failed.filter(item => eligibleProviderChildOverload(item, session.id))) {
|
|
992
|
+
const linked = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: childHarness, session_id: failedChild.session_id })]);
|
|
924
993
|
if (linked.length > 1 || linked.some(link => link.status !== "running"))
|
|
925
994
|
continue;
|
|
926
995
|
const retained = state.child_capacity?.[failedChild.session_id];
|
|
927
996
|
const refusals = [...(retained?.refusals ?? [])];
|
|
928
997
|
if (refusals.some(item => item.turn_id === failedChild.terminal_turn_id))
|
|
929
998
|
continue;
|
|
930
|
-
const nativeFailure =
|
|
999
|
+
const nativeFailure = providerChildFailure(failedChild);
|
|
931
1000
|
refusals.push(refusalObservation(nativeFailure, retained?.attempts ?? 0, failedChild.terminal_observed_at ?? context.now()));
|
|
932
1001
|
state.child_capacity ??= {};
|
|
933
1002
|
state.child_capacity[failedChild.session_id] = { ...retained, turn_id: retained?.turn_id ?? failedChild.terminal_turn_id, attempts: retained?.attempts ?? 0, policy: CAPACITY_POLICY, refusals };
|
|
934
1003
|
await prepareMutation(() => { saveState(context, row, state); appendEvent(context, row.controller_id, "capacity_child_refusal", { child_session_id: failedChild.session_id, refusal: refusals.at(-1) }); });
|
|
935
|
-
const
|
|
936
|
-
if (
|
|
937
|
-
throw new AppError("
|
|
1004
|
+
const exhausted = overloadExhausted(refusals);
|
|
1005
|
+
if (exhausted)
|
|
1006
|
+
throw new AppError("provider_unavailable", "Native child exhausted its overload continuation attempts", 1, { ...exhausted, reason: "overload_retries_exhausted", child_session_id: failedChild.session_id, primary_error: nativeFailure });
|
|
938
1007
|
}
|
|
939
1008
|
// Continue one existing child at a time; another failed child does not
|
|
940
1009
|
// create a second fanout wave or share this child's refusal counter.
|
|
941
|
-
const child = children.every(item => item.status === "failed" || item.status === "completed") ? failed.find(item =>
|
|
1010
|
+
const child = children.every(item => item.status === "failed" || item.status === "completed") ? failed.find(item => renderNativeChildFollowupInstruction(childHarness, item.session_id) && eligibleProviderChildOverload(item, session.id)) : undefined;
|
|
942
1011
|
const turnId = child?.terminal_turn_id;
|
|
943
|
-
const native = child && turnId ?
|
|
944
|
-
terminal_status: "failed", provider_error: child.provider_error, observed_at: child.terminal_observed_at } } : null;
|
|
1012
|
+
const native = child && turnId ? providerChildFailure(child) : null;
|
|
945
1013
|
const prior = child && state.child_capacity?.[child.session_id];
|
|
946
|
-
const links = child ? context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id:
|
|
947
|
-
if (child && turnId && native &&
|
|
1014
|
+
const links = child ? context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: childHarness, session_id: child.session_id })]) : [];
|
|
1015
|
+
if (child && turnId && native && terminalProviderOverload(native, child.session_id, turnId)
|
|
948
1016
|
&& nativeItemsSettled(child.native_turn_items)
|
|
949
1017
|
&& inspected.settled === true && child.parent_session_id === session.id && links.length <= 1
|
|
950
1018
|
&& (!links.length || links[0].status === "running")) {
|
|
@@ -962,10 +1030,17 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
962
1030
|
state.child_capacity[child.session_id] = { ...prior, turn_id: prior?.turn_id ?? turnId, attempts: prior?.attempts ?? 0, policy: CAPACITY_POLICY, refusals };
|
|
963
1031
|
saveState(context, row, state);
|
|
964
1032
|
}
|
|
965
|
-
const
|
|
966
|
-
if (
|
|
967
|
-
throw new AppError("
|
|
968
|
-
|
|
1033
|
+
const exhausted = overloadExhausted(refusals);
|
|
1034
|
+
if (exhausted)
|
|
1035
|
+
throw new AppError("provider_unavailable", "Native child exhausted its overload continuation attempts", 1, { ...exhausted, reason: "overload_retries_exhausted", child_session_id: child.session_id, primary_error: native });
|
|
1036
|
+
let wait;
|
|
1037
|
+
try {
|
|
1038
|
+
wait = prepared ? Math.max(0, Date.parse(prepared.not_before) - Date.now()) : capacityDelay(native, attempt);
|
|
1039
|
+
}
|
|
1040
|
+
catch (error) {
|
|
1041
|
+
const failure = error;
|
|
1042
|
+
throw new AppError(failure.code ?? "capacity_retry_metadata_invalid", failure.message ?? "Invalid child retry delay", 1, { ...failure.details, child_session_id: child.session_id, root_operation_id: prior?.root_operation_id ?? null, attempts: refusals.length, primary_error: native });
|
|
1043
|
+
}
|
|
969
1044
|
if (!Number.isFinite(wait) || Date.now() + wait > 8.64e15)
|
|
970
1045
|
throw new AppError("capacity_retry_metadata_invalid", "Native child retry time is invalid", 1, { child_session_id: child.session_id, delay_ms: wait });
|
|
971
1046
|
const rootOperationId = prior?.root_operation_id ?? `${row.controller_id}:child:${digest([child.session_id, turnId]).slice(0, 16)}`;
|
|
@@ -988,7 +1063,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
988
1063
|
throw new AppError("capacity_child_followup_blocked", "RUN changed during native child capacity wait", 1, { child_session_id: child.session_id });
|
|
989
1064
|
const fresh = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
990
1065
|
const still = controllerNativeChildren(fresh, session.id).find(item => item.session_id === child.session_id);
|
|
991
|
-
const currentLinks = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id:
|
|
1066
|
+
const currentLinks = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: childHarness, session_id: child.session_id })]);
|
|
992
1067
|
if (currentLinks.length === 1 && ["completed", "cancelled", "paused"].includes(currentLinks[0].status)) {
|
|
993
1068
|
state.last_receipt = fresh;
|
|
994
1069
|
saveState(context, row, state);
|
|
@@ -1000,7 +1075,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
1000
1075
|
throw new AppError("capacity_child_followup_blocked", "Native child or Work changed during capacity wait", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
1001
1076
|
await renewLease();
|
|
1002
1077
|
assertOwner();
|
|
1003
|
-
const text =
|
|
1078
|
+
const text = renderNativeChildFollowupInstruction(childHarness, child.session_id);
|
|
1004
1079
|
await prompt(session, text, operationId, "child");
|
|
1005
1080
|
state.child_capacity[child.session_id].dispatched = true;
|
|
1006
1081
|
saveState(context, row, state);
|
|
@@ -1016,6 +1091,25 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
1016
1091
|
const fanout = await nextControllerFanout(executionContext, { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles, dispatchFence: retainedFailures.length > 0 });
|
|
1017
1092
|
assertOwner();
|
|
1018
1093
|
if (fanout.kind === "none") {
|
|
1094
|
+
// An ACK-only recovery Turn may leave the original root Work unfinished.
|
|
1095
|
+
// Continue that retained packet once; never restart its Stage or repeat
|
|
1096
|
+
// an unknown productive dispatch merely because the observer reattached.
|
|
1097
|
+
const recovery = context.db.get(`SELECT o.operation_id,b.packet_json,b.packet_sha256,b.work_id
|
|
1098
|
+
FROM run_controller_operations o JOIN run_recovery_bindings b ON b.recovery_id=json_extract(o.receipt_json,'$.recovery_id')
|
|
1099
|
+
JOIN run_recovery_guards g ON g.recovery_id=b.recovery_id
|
|
1100
|
+
WHERE o.controller_id=? AND o.generation=? AND o.status='completed' AND json_extract(o.input_json,'$.action')='control_resume'
|
|
1101
|
+
AND b.status='accepted' AND g.status='recovered' AND g.project_id=? AND g.run_id=?
|
|
1102
|
+
AND b.provider_session_id=? AND b.daemon_id=json_extract(o.receipt_json,'$.daemon_id') ORDER BY o.created_at DESC LIMIT 1`, [row.controller_id, row.generation, row.project_id, row.run_id, session.id]);
|
|
1103
|
+
if (recovery && digestBytes(recovery.packet_json) === recovery.packet_sha256) {
|
|
1104
|
+
const packet = JSON.parse(recovery.packet_json);
|
|
1105
|
+
const operationId = `${recovery.operation_id}:unfinished-root`;
|
|
1106
|
+
const work = context.db.get('SELECT status FROM works WHERE work_id=? AND project_id=? AND run_id=? AND parent_work_id IS NULL', [recovery.work_id, row.project_id, row.run_id]);
|
|
1107
|
+
if (recoveryPacketMatchesStage(packet, { runId: row.run_id, generation: row.generation, stage: next.stage, workId: recovery.work_id }) && work?.status === 'running'
|
|
1108
|
+
&& !context.db.get('SELECT 1 FROM run_controller_operations WHERE operation_id=?', [operationId])) {
|
|
1109
|
+
await prompt(session, `Continue the original unfinished root Work from this accepted recovery packet. Inspect retained receipts and partial effects first. Do not restart the Stage, repeat observed lifecycle operations, create another Work or start a successor Stage. Preserve the packet write scope and result schema; stop at its original boundary.\n\nAccepted recovery packet (JSON data):\n${recovery.packet_json}`, operationId);
|
|
1110
|
+
continue;
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1019
1113
|
assertNoCorrelatedRejection(context, { projectId: row.project_id, runId: row.run_id, generation: row.generation, stage: next.stage, turn: state.turns });
|
|
1020
1114
|
const cause = currentLifecycleFailure(executionContext, { runId: run.id, generation: row.generation, stage: next.stage, turn: state.turns, projectId: row.project_id, projectRoot: run.project_root });
|
|
1021
1115
|
if (cause)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { AppError } from "../shared/errors.js";
|
|
2
|
-
import { CAPACITY_POLICY,
|
|
2
|
+
import { CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, overloadExhausted } from "../harness-runtime/lib/provider-overload-policy.mjs";
|
|
3
|
+
import { assertRunStartupSettled } from "./run-start-custody.js";
|
|
3
4
|
export function recoveryGuard(context, projectId, runId) { return context.db.get("SELECT recovery_id, project_id, run_id, generation, status, interruption_json, settlement_json, capture_path, created_at, updated_at, sealed_at, resumed_at FROM run_recovery_guards WHERE project_id = ? AND run_id = ? ORDER BY generation DESC LIMIT 1", [projectId, runId]) ?? null; }
|
|
4
5
|
export function assertRunRecoveryAllowed(context, projectId, runId, expectedGeneration) {
|
|
5
6
|
const guard = recoveryGuard(context, projectId, runId);
|
|
@@ -11,6 +12,7 @@ export function assertRunRecoveryAllowed(context, projectId, runId, expectedGene
|
|
|
11
12
|
return generation;
|
|
12
13
|
}
|
|
13
14
|
export function assertRunMutationAllowed(context, projectId, runId, expectedGeneration) {
|
|
15
|
+
assertRunStartupSettled(context, projectId, runId);
|
|
14
16
|
const generation = assertRunRecoveryAllowed(context, projectId, runId, expectedGeneration);
|
|
15
17
|
if (context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'")
|
|
16
18
|
&& context.db.get("SELECT 1 FROM run_controllers WHERE project_id = ? AND run_id = ? AND status IN ('capturing','capture_failed') LIMIT 1", [projectId, runId]))
|
|
@@ -41,7 +43,7 @@ export function recoveryRuntimeAdmission(context, record, input) {
|
|
|
41
43
|
if (active) {
|
|
42
44
|
const ordinal = dispatch.capacity_intents.length;
|
|
43
45
|
const predecessor = ordinal === 1 ? dispatch.operation_id : dispatch.capacity_intents[ordinal - 2].operation_id;
|
|
44
|
-
if (active.policy !== CAPACITY_POLICY || active.ordinal !== ordinal || active.root_operation_id !== dispatch.operation_id || active.session_id !== binding.provider_session_id
|
|
46
|
+
if (ordinal > MAX_OVERLOAD_RETRIES || active.policy !== CAPACITY_POLICY || active.ordinal !== ordinal || active.root_operation_id !== dispatch.operation_id || active.session_id !== binding.provider_session_id
|
|
45
47
|
|| active.predecessor !== predecessor || active.operation_id !== `${dispatch.operation_id}:capacity:${ordinal}:session.prompt`
|
|
46
48
|
|| active.refusal?.policy !== CAPACITY_POLICY || active.refusal.ordinal !== ordinal - 1 || active.refusal.turn_id !== active.turn_id
|
|
47
49
|
|| active.refusal.session_id !== binding.provider_session_id || active.deadline !== null || !Number.isFinite(Date.parse(active.not_before)))
|
|
@@ -50,7 +52,7 @@ export function recoveryRuntimeAdmission(context, record, input) {
|
|
|
50
52
|
if (!failure || JSON.stringify(failure) !== JSON.stringify(active.refusal))
|
|
51
53
|
return false;
|
|
52
54
|
try {
|
|
53
|
-
if (
|
|
55
|
+
if (overloadExhausted(dispatch.capacity_failures))
|
|
54
56
|
return false;
|
|
55
57
|
}
|
|
56
58
|
catch {
|
|
@@ -8,7 +8,7 @@ import { managedProcessBinding, processIsAlive, resourceHome } from "./managed-p
|
|
|
8
8
|
import { getResourceDatabase } from "../storage/database.js";
|
|
9
9
|
import { requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
10
10
|
import { lifecycleRetryInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
|
|
11
|
-
import { CAPACITY_POLICY,
|
|
11
|
+
import { CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, overloadExhausted } from "../harness-runtime/lib/provider-overload-policy.mjs";
|
|
12
12
|
/** Pin retained Session loading and its first acknowledgement Turn, never a general fence bypass. */
|
|
13
13
|
export function prepareRunRecoveryDispatch(context, input) {
|
|
14
14
|
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/.test(input.operationId) || !path.isAbsolute(input.stateDir))
|
|
@@ -60,7 +60,7 @@ export function prepareRunRecoveryDispatch(context, input) {
|
|
|
60
60
|
/** Publish only an exact successor to a failed, effect-free recovery ACK Turn. */
|
|
61
61
|
export function retainRecoveryCapacityIntent(context, input) {
|
|
62
62
|
const { intent } = input;
|
|
63
|
-
if (!Number.isSafeInteger(intent.ordinal) || intent.ordinal < 1 || !intent.turn_id || !Number.isFinite(Date.parse(intent.not_before)) || intent.policy !== CAPACITY_POLICY
|
|
63
|
+
if (!Number.isSafeInteger(intent.ordinal) || intent.ordinal < 1 || intent.ordinal > MAX_OVERLOAD_RETRIES || !intent.turn_id || !Number.isFinite(Date.parse(intent.not_before)) || intent.policy !== CAPACITY_POLICY
|
|
64
64
|
|| !intent.refusal || intent.refusal.policy !== CAPACITY_POLICY || intent.refusal.ordinal !== intent.ordinal - 1 || intent.refusal.turn_id !== intent.turn_id
|
|
65
65
|
|| !Number.isFinite(Date.parse(intent.refusal.first_observed_at)) || intent.deadline !== null)
|
|
66
66
|
throw new AppError("recovery_capacity_intent_invalid", "Recovery capacity successor is invalid", 1);
|
|
@@ -91,7 +91,7 @@ export function retainRecoveryCapacityIntent(context, input) {
|
|
|
91
91
|
}
|
|
92
92
|
const refusals = dispatch.capacity_failures ?? [];
|
|
93
93
|
const observed = refusals.find(item => item.ordinal === intent.refusal.ordinal);
|
|
94
|
-
if (!observed || JSON.stringify(observed) !== JSON.stringify(intent.refusal) ||
|
|
94
|
+
if (!observed || JSON.stringify(observed) !== JSON.stringify(intent.refusal) || overloadExhausted(refusals))
|
|
95
95
|
throw new AppError("recovery_capacity_intent_conflict", "Recovery successor lacks an immutable non-burst refusal binding", 1);
|
|
96
96
|
const predecessor = intent.ordinal === 1 ? dispatch.operation_id : prior[intent.ordinal - 2]?.operation_id;
|
|
97
97
|
if (prior.length !== intent.ordinal - 1 || predecessor !== intent.predecessor || intent.operation_id !== `${dispatch.operation_id}:capacity:${intent.ordinal}:session.prompt`)
|