@deksden-com/dd-flow-cli 0.9.0-beta.14 → 0.9.0-beta.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +733 -0
- package/README.md +143 -1
- package/dist/build-info.json +8 -8
- package/dist/cli/argument-values.js +38 -0
- package/dist/cli/command-inputs.js +376 -0
- package/dist/cli/command-metadata.js +197 -0
- package/dist/cli/command-option-descriptions.js +191 -0
- package/dist/cli/completion-cli.js +70 -0
- package/dist/cli/completion.js +57 -0
- package/dist/cli/error-output.js +52 -0
- package/dist/cli/executor-onboarding.js +76 -0
- package/dist/cli/flows.js +29 -0
- package/dist/cli/harnesses.js +84 -0
- package/dist/cli/help.js +328 -18
- package/dist/cli/hook-ingress.js +78 -0
- package/dist/cli/input-preparation.js +165 -0
- package/dist/cli/memory-flow-admission.js +44 -0
- package/dist/cli/memory-init-invocation.js +78 -0
- package/dist/cli/memory-init.js +240 -0
- package/dist/cli/memory-storage-admission.js +53 -0
- package/dist/cli/models.js +39 -0
- package/dist/cli/native-hook-ingress.js +80 -0
- package/dist/cli/operator-profiles.js +61 -0
- package/dist/cli/output-options.js +35 -0
- package/dist/cli/response-file.js +71 -0
- package/dist/cli/run-cli.js +1689 -545
- package/dist/cli/runtime-maintenance-diagnostics.js +46 -0
- package/dist/cli/runtime-maintenance-ingress.js +52 -0
- package/dist/cli/runtime-process.js +153 -0
- package/dist/cli/setup.js +149 -0
- package/dist/cli.js +71 -2
- package/dist/domain/stage-catalog.js +1 -1
- package/dist/domain/stage-handler-identities.js +20 -0
- package/dist/harness-runtime/DD-ZCODE.md +112 -0
- package/dist/harness-runtime/OPENCODE-LICENSE.txt +24 -0
- package/dist/harness-runtime/bin/dd-agy.mjs +56 -0
- package/dist/harness-runtime/bin/dd-codex.mjs +27 -0
- package/dist/harness-runtime/bin/dd-droid.mjs +34 -0
- package/dist/harness-runtime/bin/dd-grok.mjs +40 -0
- package/dist/harness-runtime/bin/dd-opencode.mjs +22 -0
- package/dist/harness-runtime/bin/dd-zcode.mjs +87 -0
- package/dist/harness-runtime/lib/adapter-timeouts.d.mts +5 -0
- package/dist/harness-runtime/lib/adapter-timeouts.mjs +34 -0
- package/dist/harness-runtime/lib/agy-hooks.d.mts +1 -0
- package/dist/harness-runtime/lib/agy-hooks.mjs +9 -0
- package/dist/harness-runtime/lib/daemon-observation.d.mts +2 -0
- package/dist/harness-runtime/lib/daemon-observation.mjs +163 -0
- package/dist/harness-runtime/lib/daemon-operations.d.mts +1 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +371 -0
- package/dist/harness-runtime/lib/dd-agy-daemon.d.mts +6 -0
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +724 -0
- package/dist/harness-runtime/lib/dd-agy.mjs +70 -0
- package/dist/harness-runtime/lib/dd-codex-daemon.d.mts +1 -0
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +196 -0
- package/dist/harness-runtime/lib/dd-codex.d.mts +1 -0
- package/dist/harness-runtime/lib/dd-codex.mjs +817 -0
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +129 -0
- package/dist/harness-runtime/lib/dd-droid.mjs +584 -0
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +498 -0
- package/dist/harness-runtime/lib/dd-grok.mjs +177 -0
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +507 -0
- package/dist/harness-runtime/lib/dd-opencode.mjs +354 -0
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +750 -0
- package/dist/harness-runtime/lib/dd-zcode.mjs +1145 -0
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +14 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +157 -0
- package/dist/harness-runtime/lib/dispatch-fence.mjs +14 -0
- package/dist/harness-runtime/lib/driver-recovery.mjs +139 -0
- package/dist/harness-runtime/lib/droid-observation.mjs +66 -0
- package/dist/harness-runtime/lib/lease-renewal.d.mts +8 -0
- package/dist/harness-runtime/lib/lease-renewal.mjs +37 -0
- package/dist/harness-runtime/lib/lifecycle-command.d.mts +20 -0
- package/dist/harness-runtime/lib/lifecycle-command.mjs +273 -0
- package/dist/harness-runtime/lib/lifecycle-contract.d.mts +10 -0
- package/dist/harness-runtime/lib/lifecycle-contract.mjs +19 -0
- package/dist/harness-runtime/lib/managed-daemon.d.mts +13 -0
- package/dist/harness-runtime/lib/managed-daemon.mjs +654 -0
- package/dist/harness-runtime/lib/managed-flow-env.mjs +15 -0
- package/dist/harness-runtime/lib/model-observations.mjs +101 -0
- package/dist/harness-runtime/lib/native-children.d.mts +16 -0
- package/dist/harness-runtime/lib/native-children.mjs +113 -0
- package/dist/harness-runtime/lib/native-hook-command.d.mts +7 -0
- package/dist/harness-runtime/lib/native-hook-command.mjs +251 -0
- package/dist/harness-runtime/lib/native-request-proof.d.mts +2 -0
- package/dist/harness-runtime/lib/native-request-proof.mjs +65 -0
- package/dist/harness-runtime/lib/native-streams.mjs +45 -0
- package/dist/harness-runtime/lib/observation-clock.d.mts +18 -0
- package/dist/harness-runtime/lib/observation-clock.mjs +92 -0
- package/dist/harness-runtime/lib/opencode-plugin.mjs +85 -0
- package/dist/harness-runtime/lib/operation-context.mjs +5 -0
- package/dist/harness-runtime/lib/operation-errors.d.mts +14 -0
- package/dist/harness-runtime/lib/operation-errors.mjs +116 -0
- package/dist/harness-runtime/lib/operation-progress.d.mts +7 -0
- package/dist/harness-runtime/lib/operation-progress.mjs +114 -0
- package/dist/harness-runtime/lib/process-json.d.mts +5 -0
- package/dist/harness-runtime/lib/process-json.mjs +201 -0
- package/dist/harness-runtime/lib/process-snapshot.mjs +10 -0
- package/dist/harness-runtime/lib/provider-overload-policy.d.mts +13 -0
- package/dist/harness-runtime/lib/provider-overload-policy.mjs +138 -0
- package/dist/harness-runtime/lib/runner-events.mjs +200 -0
- package/dist/harness-runtime/lib/runner-lock.mjs +42 -0
- package/dist/harness-runtime/lib/session-settlement.mjs +30 -0
- package/dist/harness-runtime/lib/setup-progress.d.mts +9 -0
- package/dist/harness-runtime/lib/setup-progress.mjs +51 -0
- package/dist/harness-runtime/lib/tool-observations.d.mts +21 -0
- package/dist/harness-runtime/lib/tool-observations.mjs +228 -0
- package/dist/harness-runtime/lib/zcode-usage.mjs +55 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-1y4a697q.js +81 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-2ny4vxct.js +92 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-6mptkzbc.js +0 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-7x67cf1q.js +52 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-9rqn6x4v.js +16 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-9vdz0zwn.js +1661 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-ecs5ftka.js +1466 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-frbwqjmf.js +145 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-gne7h7mq.js +50 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-gx9wnkzr.js +105 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-m2eywc8w.js +0 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-nwcpakzx.js +12 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-q4g3afnw.js +623 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-qav6mfdz.js +0 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-sxt88q69.js +12 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-t1xa31vm.js +39 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-ve4aws36.js +103 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-vmytr4q8.js +4 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-vt9jbch8.js +36 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-vyas3vgt.js +0 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-wjyeq25y.js +30 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-xgnj2w4v.js +20 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-y2etg5bs.js +9 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-ydhz1zrh.js +238 -0
- package/dist/harness-runtime/vendor/opencode-client/chunks/service-probe-z6qdwcsf.js +62 -0
- package/dist/harness-runtime/vendor/opencode-client/contract.d.ts +1 -0
- package/dist/harness-runtime/vendor/opencode-client/contract.js +12 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/api/api.d.ts +2806 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/api/api.js +0 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/api.d.ts +4 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/api.js +0 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/client.d.ts +2530 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/client.js +15 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/generated/client-error.d.ts +7 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/generated/client-error.js +6 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/generated/client.d.ts +2529 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/generated/client.js +8 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/generated/index.d.ts +2 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/generated/index.js +12 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/index.d.ts +35 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/index.js +75 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/rpc.d.ts +20 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/rpc.js +11 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/service.d.ts +90 -0
- package/dist/harness-runtime/vendor/opencode-client/effect/service.js +187 -0
- package/dist/harness-runtime/vendor/opencode-client/package.json +84 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/api.d.ts +21 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/api.js +0 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/client.d.ts +252 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/client.js +14 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/generated/client-error.d.ts +8 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/generated/client-error.js +6 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/generated/client.d.ts +259 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/generated/client.js +7 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/generated/index.d.ts +3 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/generated/index.js +79 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/generated/types.d.ts +8932 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/generated/types.js +70 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/index.d.ts +6 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/index.js +83 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/rpc.d.ts +32 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/rpc.js +7 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/service.d.ts +24 -0
- package/dist/harness-runtime/vendor/opencode-client/promise/service.js +187 -0
- package/dist/harness-runtime/vendor/opencode-client/pty-handoff.d.ts +12 -0
- package/dist/harness-runtime/vendor/opencode-client/pty-handoff.js +14 -0
- package/dist/harness-runtime/vendor/opencode-client/rpc-runtime.d.ts +20 -0
- package/dist/harness-runtime/vendor/opencode-client/rpc-runtime.js +14 -0
- package/dist/harness-runtime/vendor/opencode-client/service-contender.d.ts +37 -0
- package/dist/harness-runtime/vendor/opencode-client/service-contender.js +10 -0
- package/dist/harness-runtime/vendor/opencode-client/service-probe.d.ts +70 -0
- package/dist/harness-runtime/vendor/opencode-client/service-probe.js +16 -0
- package/dist/harness-runtime/vendor/opencode-client/service-timing.d.ts +12 -0
- package/dist/harness-runtime/vendor/opencode-client/service-timing.js +10 -0
- package/dist/harness-runtime/vendor/opencode-client/service-version.d.ts +2 -0
- package/dist/harness-runtime/vendor/opencode-client/service-version.js +6 -0
- package/dist/harness-runtime/vendor/opencode-client/service.d.ts +52 -0
- package/dist/harness-runtime/vendor/opencode-client/service.js +1 -0
- package/dist/harness-runtime/vendor/opencode-client/shared-events.d.ts +11 -0
- package/dist/harness-runtime/vendor/opencode-client/shared-events.js +8 -0
- package/dist/harness-runtime/vendor/opencode-client/solid/connection.d.ts +37 -0
- package/dist/harness-runtime/vendor/opencode-client/solid/connection.js +10 -0
- package/dist/harness-runtime/vendor/opencode-client/solid/data.d.ts +211 -0
- package/dist/harness-runtime/vendor/opencode-client/solid/data.js +18 -0
- package/dist/harness-runtime/vendor/opencode-client/solid/index.d.ts +3 -0
- package/dist/harness-runtime/vendor/opencode-client/solid/index.js +32 -0
- package/dist/harness-runtime/vendor/opencode-client/solid/pty.d.ts +22 -0
- package/dist/harness-runtime/vendor/opencode-client/solid/pty.js +8 -0
- package/dist/harness-runtime/vendor/opencode-plugin.mjs +3 -0
- package/dist/protocol/local-files.js +18 -9
- package/dist/runtime/context.js +8 -4
- package/dist/schemas/agent-profile.schema.json +1 -1
- package/dist/schemas/code-review-result.schema.json +1 -1
- package/dist/schemas/code-verification.schema.json +7 -5
- package/dist/schemas/code-work-batch.schema.json +2 -2
- package/dist/schemas/discovery-result.schema.json +184 -0
- package/dist/schemas/execution-bundle.schema.json +205 -0
- package/dist/schemas/execution-settings.schema.json +162 -0
- package/dist/schemas/flow-descriptor.schema.json +54 -0
- package/dist/schemas/flow-run.schema.json +28 -0
- package/dist/schemas/harness-catalog.schema.json +80 -0
- package/dist/schemas/mb-init-compose-result.schema.json +457 -0
- package/dist/schemas/mb-init-discover-result.schema.json +500 -0
- package/dist/schemas/mb-init-finish-result.schema.json +422 -0
- package/dist/schemas/mb-init-preflight-result.schema.json +367 -0
- package/dist/schemas/mb-init-review-result.schema.json +396 -0
- package/dist/schemas/mb-init-verify-result.schema.json +391 -0
- package/dist/schemas/memory-bootstrap-receipt-v2.schema.json +67 -0
- package/dist/schemas/memory-bootstrap-receipt.schema.json +354 -0
- package/dist/schemas/memory-composition-work-result.schema.json +38 -0
- package/dist/schemas/memory-repair-work-result.schema.json +49 -0
- package/dist/schemas/memory-review-correction-plan.schema.json +30 -0
- package/dist/schemas/memory-review-work-result.schema.json +43 -0
- package/dist/schemas/model-catalog.schema.json +153 -0
- package/dist/schemas/plan-review-decision.schema.json +3 -3
- package/dist/schemas/plan-review-result.schema.json +2 -2
- package/dist/schemas/profile-settings.schema.json +122 -0
- package/dist/schemas/release-manifest.schema.json +291 -0
- package/dist/schemas/run-control-receipt.schema.json +118 -0
- package/dist/schemas/run-control-request.schema.json +36 -0
- package/dist/schemas/run-engine-binding.schema.json +1 -1
- package/dist/schemas/vnext-protocol-plan.schema.json +2 -2
- package/dist/services/branch-context.js +3 -3
- package/dist/services/canon.js +11 -8
- package/dist/services/check-runner-evidence.js +168 -0
- package/dist/services/cleanup.js +133 -40
- package/dist/services/cli-operation-classifier.js +25 -62
- package/dist/services/cli-upgrade-allowlist.js +27 -0
- package/dist/services/code-checks.js +1156 -107
- package/dist/services/codex-hook-delivery.js +32 -0
- package/dist/services/command-context.js +80 -0
- package/dist/services/compatibility-preflight.js +13 -3
- package/dist/services/config.js +11 -8
- package/dist/services/continuation-outcome.js +20 -0
- package/dist/services/controller-child-capacity.js +13 -0
- package/dist/services/controller-child-observer.js +115 -0
- package/dist/services/controller-execution.js +88 -0
- package/dist/services/controller-fanout.js +202 -0
- package/dist/services/dashboard.js +143 -58
- package/dist/services/descriptor-coordinator-session.js +35 -0
- package/dist/services/descriptor-root-work.js +53 -0
- package/dist/services/descriptor-run-settlement.js +105 -0
- package/dist/services/descriptor-run-start.js +449 -0
- package/dist/services/descriptor-stage-coordinator.js +197 -0
- package/dist/services/descriptor-stage-settlement.js +222 -0
- package/dist/services/descriptor-stage-start.js +139 -0
- package/dist/services/discovery/agy-discovery.js +2 -0
- package/dist/services/discovery/codex-discovery.js +2 -0
- package/dist/services/discovery/droid-discovery.js +2 -0
- package/dist/services/discovery/grok-discovery.js +2 -0
- package/dist/services/discovery/opencode-discovery.js +2 -0
- package/dist/services/discovery/static.js +31 -0
- package/dist/services/discovery/zcode-discovery.js +2 -0
- package/dist/services/engines.js +294 -153
- package/dist/services/eval-snapshots.js +1215 -116
- package/dist/services/execution-bundles.js +263 -0
- package/dist/services/execution-policy.js +185 -0
- package/dist/services/executor-onboarding.js +56 -0
- package/dist/services/external-work-launch.js +308 -0
- package/dist/services/flow-catalog.js +21 -0
- package/dist/services/flow-descriptors.js +82 -0
- package/dist/services/frozen-engine.js +250 -0
- package/dist/services/generic-execution.js +164 -0
- package/dist/services/harness-adapter.js +355 -36
- package/dist/services/harness-config.js +2 -0
- package/dist/services/harness-discovery.js +29 -0
- package/dist/services/harness-doctor.js +75 -0
- package/dist/services/harness-probe-effects.js +51 -0
- package/dist/services/harness-version-probe.js +100 -0
- package/dist/services/hooks.js +771 -325
- package/dist/services/hybrid-contracts.js +483 -0
- package/dist/services/ids.js +29 -1
- package/dist/services/installation-lock.js +95 -0
- package/dist/services/installed-flow-source.js +183 -0
- package/dist/services/lanes.js +78 -65
- package/dist/services/lifecycle-command.js +1 -288
- package/dist/services/lifecycle-contract.js +1 -0
- package/dist/services/lifecycle-invocations.js +2007 -0
- package/dist/services/managed-daemon-binding.js +38 -0
- package/dist/services/managed-lease-monitor.js +133 -0
- package/dist/services/managed-processes.js +475 -55
- package/dist/services/memory-accepted-composition.js +77 -0
- package/dist/services/memory-baseline-preservation.js +65 -0
- package/dist/services/memory-completed-correction-wave.js +320 -0
- package/dist/services/memory-compose-handler.js +258 -0
- package/dist/services/memory-composition-acceptance.js +199 -0
- package/dist/services/memory-composition-graph.js +134 -0
- package/dist/services/memory-composition-registration.js +146 -0
- package/dist/services/memory-composition-work-custody.js +151 -0
- package/dist/services/memory-composition-work-results.js +96 -0
- package/dist/services/memory-corrected-review-acceptance.js +129 -0
- package/dist/services/memory-corrected-review-batch.js +30 -0
- package/dist/services/memory-corrected-review-findings.js +106 -0
- package/dist/services/memory-corrected-review-inputs.js +210 -0
- package/dist/services/memory-corrected-review-projection.js +78 -0
- package/dist/services/memory-corrected-review-receipt.js +170 -0
- package/dist/services/memory-corrected-review-registration.js +191 -0
- package/dist/services/memory-correction-check-receipt.js +219 -0
- package/dist/services/memory-correction-input-owner.js +6 -0
- package/dist/services/memory-discover-handler.js +104 -0
- package/dist/services/memory-finish-evidence.js +435 -0
- package/dist/services/memory-finish-handler.js +215 -0
- package/dist/services/memory-finish-integration.js +280 -0
- package/dist/services/memory-finish-readiness.js +122 -0
- package/dist/services/memory-git-integration.js +151 -0
- package/dist/services/memory-historical-corrected-review.js +206 -0
- package/dist/services/memory-historical-original-review.js +130 -0
- package/dist/services/memory-historical-repair-registration.js +99 -0
- package/dist/services/memory-historical-second-review.js +128 -0
- package/dist/services/memory-init-bootstrap-readiness.js +210 -0
- package/dist/services/memory-init-contexts.js +89 -0
- package/dist/services/memory-init-controller-start.js +84 -0
- package/dist/services/memory-init-engine-release.js +223 -0
- package/dist/services/memory-init-git-policy.js +136 -0
- package/dist/services/memory-init-lint-capability.js +185 -0
- package/dist/services/memory-init-native-capacity.js +296 -0
- package/dist/services/memory-init-preflight.js +330 -0
- package/dist/services/memory-init-project-bootstrap.js +599 -0
- package/dist/services/memory-init-request-candidate.js +157 -0
- package/dist/services/memory-init-request-input.js +66 -0
- package/dist/services/memory-init-resource-storage.js +125 -0
- package/dist/services/memory-init-run-allocation.js +81 -0
- package/dist/services/memory-init-run-bundle.js +98 -0
- package/dist/services/memory-init-start.js +181 -0
- package/dist/services/memory-init-startup-custody.js +80 -0
- package/dist/services/memory-init-startup-input.js +240 -0
- package/dist/services/memory-init-storage.js +81 -0
- package/dist/services/memory-init-stores.js +359 -0
- package/dist/services/memory-init-worktrunk-capability.js +178 -0
- package/dist/services/memory-layer-publication.js +611 -0
- package/dist/services/memory-layer.js +156 -0
- package/dist/services/memory-lint-config.js +85 -0
- package/dist/services/memory-native-review-copy.js +165 -0
- package/dist/services/memory-normal-flow-entry.js +75 -0
- package/dist/services/memory-ordinary-defaults.js +112 -0
- package/dist/services/memory-preflight-bootstrap-custody.js +115 -0
- package/dist/services/memory-preflight-handler.js +259 -0
- package/dist/services/memory-repair-current-view.js +299 -0
- package/dist/services/memory-repair-document-delta.js +451 -0
- package/dist/services/memory-repair-history.js +491 -0
- package/dist/services/memory-repair-physical-binding.js +312 -0
- package/dist/services/memory-repair-publication-files.js +152 -0
- package/dist/services/memory-repair-publication-intent.js +27 -0
- package/dist/services/memory-repair-publication-plan.js +160 -0
- package/dist/services/memory-repair-publication-source.js +225 -0
- package/dist/services/memory-repair-publication.js +794 -0
- package/dist/services/memory-repair-registration.js +239 -0
- package/dist/services/memory-repair-work-baseline.js +413 -0
- package/dist/services/memory-repair-work-completion.js +97 -0
- package/dist/services/memory-repair-work-custody.js +73 -0
- package/dist/services/memory-repair-work-results.js +89 -0
- package/dist/services/memory-repair-workspace.js +252 -0
- package/dist/services/memory-repair-writer-copy.js +735 -0
- package/dist/services/memory-result-binding.js +173 -0
- package/dist/services/memory-retained-review-authority.js +129 -0
- package/dist/services/memory-retained-review-results.js +70 -0
- package/dist/services/memory-retained-second-baseline.js +102 -0
- package/dist/services/memory-retained-second-repairs.js +156 -0
- package/dist/services/memory-review-acceptance.js +214 -0
- package/dist/services/memory-review-correction-plan.js +171 -0
- package/dist/services/memory-review-handler.js +585 -0
- package/dist/services/memory-review-inputs.js +200 -0
- package/dist/services/memory-review-receipt.js +180 -0
- package/dist/services/memory-review-registration.js +108 -0
- package/dist/services/memory-review-work-custody.js +124 -0
- package/dist/services/memory-review-work-input.js +72 -0
- package/dist/services/memory-review-work-results.js +73 -0
- package/dist/services/memory-second-corrected-review-inputs.js +165 -0
- package/dist/services/memory-second-correction-plan.js +76 -0
- package/dist/services/memory-second-repair-baseline.js +91 -0
- package/dist/services/memory-second-repair-launch.js +614 -0
- package/dist/services/memory-second-repair-registration.js +88 -0
- package/dist/services/memory-second-repair-source.js +187 -0
- package/dist/services/memory-second-review-acceptance.js +115 -0
- package/dist/services/memory-source-observations.js +117 -0
- package/dist/services/memory-stage-dispatch.js +28 -0
- package/dist/services/memory-stage-results.js +199 -0
- package/dist/services/memory-verification-acceptance.js +135 -0
- package/dist/services/memory-verification-checks.js +252 -0
- package/dist/services/memory-verification-receipt.js +159 -0
- package/dist/services/memory-verify-handler.js +135 -0
- package/dist/services/memory-work-result-preparation.js +63 -0
- package/dist/services/merge-queue.js +228 -149
- package/dist/services/merge-server.js +161 -35
- package/dist/services/migrations.js +37 -16
- package/dist/services/model-catalog.js +109 -0
- package/dist/services/model-discovery.js +46 -0
- package/dist/services/native-daemon-history.js +49 -0
- package/dist/services/native-session-control.js +39 -0
- package/dist/services/operator-profiles.js +236 -0
- package/dist/services/plan-runtime.js +26 -11
- package/dist/services/plans.js +28 -25
- package/dist/services/projects.js +37 -17
- package/dist/services/prompts.js +28 -17
- package/dist/services/protocols.js +163 -87
- package/dist/services/provider-capacity.js +385 -0
- package/dist/services/recovery-observation-budget.js +121 -0
- package/dist/services/recovery-snapshot-database.js +560 -0
- package/dist/services/release-archive-producer.js +52 -0
- package/dist/services/release-archive.js +143 -0
- package/dist/services/release-cache.js +142 -0
- package/dist/services/release-compatibility.js +18 -0
- package/dist/services/release-download.js +67 -0
- package/dist/services/release-envelope.js +104 -0
- package/dist/services/release-manifest.js +45 -0
- package/dist/services/release-payload.js +114 -0
- package/dist/services/release-producer.js +63 -0
- package/dist/services/release-provenance.js +160 -0
- package/dist/services/release-registry.js +66 -0
- package/dist/services/release-source.js +194 -0
- package/dist/services/repair-intents.js +42 -0
- package/dist/services/review-copy.js +165 -0
- package/dist/services/review-deferral.js +23 -0
- package/dist/services/run-control-receipt.js +56 -0
- package/dist/services/run-control-worker.js +411 -0
- package/dist/services/run-control.js +1056 -0
- package/dist/services/run-controller-adapter.js +399 -0
- package/dist/services/run-controller-capture.js +186 -0
- package/dist/services/run-controller-process.js +208 -0
- package/dist/services/run-controller-recovery.js +289 -0
- package/dist/services/run-controller-state.js +97 -0
- package/dist/services/run-controller.js +1205 -0
- package/dist/services/run-engine-bindings.js +1 -1
- package/dist/services/run-fork.js +147 -0
- package/dist/services/run-health.js +149 -0
- package/dist/services/run-observations.js +152 -0
- package/dist/services/run-projection.js +22 -64
- package/dist/services/run-recovery-guard.js +77 -0
- package/dist/services/run-recovery-runtime.js +112 -0
- package/dist/services/run-recovery.js +489 -0
- package/dist/services/run-start-custody.js +177 -0
- package/dist/services/runs.js +1019 -247
- package/dist/services/runtime-budget.js +430 -0
- package/dist/services/runtime-command.js +43 -0
- package/dist/services/runtime-scope-capture.js +52 -0
- package/dist/services/runtime-scope-control.js +468 -0
- package/dist/services/runtime-scope-resume.js +599 -0
- package/dist/services/runtime-scope-stop.js +210 -0
- package/dist/services/runtime-scope-worker.js +287 -0
- package/dist/services/runtime-service.js +96 -0
- package/dist/services/schema-validation.js +10 -8
- package/dist/services/scope-writer-cleanup.js +27 -0
- package/dist/services/sessions.js +35 -69
- package/dist/services/setup.js +377 -0
- package/dist/services/stage-blocker.js +17 -7
- package/dist/services/stage-context.js +193 -19
- package/dist/services/stage-handlers.js +74 -0
- package/dist/services/stage-lifecycle.js +100 -108
- package/dist/services/stage-pause.js +89 -75
- package/dist/services/stage-report-renderer.js +234 -6
- package/dist/services/stage-work-graph.js +52 -0
- package/dist/services/tooling.js +32 -23
- package/dist/services/usage.js +49 -101
- package/dist/services/verification-matrix.js +374 -0
- package/dist/services/vnext-code-review.js +223 -88
- package/dist/services/vnext-code.js +327 -226
- package/dist/services/vnext-contracts.js +10 -1
- package/dist/services/vnext-execution-profile.js +80 -5
- package/dist/services/vnext-fanout.js +311 -19
- package/dist/services/vnext-merge.js +332 -111
- package/dist/services/vnext-plan-review.js +279 -127
- package/dist/services/vnext-plan.js +229 -104
- package/dist/services/vnext-protocolize.js +411 -86
- package/dist/services/vnext-specify.js +119 -81
- package/dist/services/vnext-workspace-policy.js +7 -2
- package/dist/services/work-registry.js +1627 -199
- package/dist/services/workspace-bootstrap.js +52 -15
- package/dist/services/workspace-files.js +85 -0
- package/dist/services/worktrees.js +98 -44
- package/dist/shared/entity-references.js +8 -0
- package/dist/shared/errors.js +17 -0
- package/dist/storage/agent-output.js +26 -0
- package/dist/storage/database.js +891 -39
- package/dist/storage/foreign-key-transaction.js +39 -0
- package/dist/storage/immutable-input.js +45 -0
- package/dist/storage/import-guard.js +102 -0
- package/dist/storage/native-cleanup.js +52 -0
- package/dist/storage/paths.js +17 -4
- package/dist/storage/projection-lock.js +73 -0
- package/dist/storage/work-references.js +23 -0
- package/dist/storage/writer-contract.js +89 -0
- package/dist/storage/writer-migration.js +85 -0
- package/package.json +43 -13
- package/tools/create-canon-release.mjs +33 -0
- package/tools/produce-hybrid-release.mjs +41 -0
- package/tools/repair-paused-run-status.mjs +84 -0
|
@@ -0,0 +1,1205 @@
|
|
|
1
|
+
import { assertLifecycleOutcomes, currentLifecycleFailure, managedInvocationContext, managedLifecycleCommand } from "./lifecycle-invocations.js";
|
|
2
|
+
export function recoveryPacketMatchesStage(packet, scope) {
|
|
3
|
+
return packet.schema_id === 'dd-flow/root-recovery-packet@1' && packet.continuation === 'unfinished_work'
|
|
4
|
+
&& packet.run_id === scope.runId && packet.generation === scope.generation && packet.stage === scope.stage
|
|
5
|
+
&& packet.work_id === scope.workId && packet.paused === false;
|
|
6
|
+
}
|
|
7
|
+
import crypto from "node:crypto";
|
|
8
|
+
import fs from "node:fs";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { quote } from "shell-quote";
|
|
13
|
+
import { ensureRunControllerStorage, getDatabase } from "../storage/database.js";
|
|
14
|
+
import { canonicalPath, resolveProjectRoot } from "../storage/paths.js";
|
|
15
|
+
import { publishImmutableInput } from "../storage/immutable-input.js";
|
|
16
|
+
import { AppError } from "../shared/errors.js";
|
|
17
|
+
import { prepareFileBytes, prepareJsonText } from "../cli/input-preparation.js";
|
|
18
|
+
import { parseRuntimeBudget } from "./runtime-budget.js";
|
|
19
|
+
import { requireProjectByRoot } from "./projects.js";
|
|
20
|
+
import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
|
|
21
|
+
import { getFlowRunStatus, setFlowRunRecoveryPaused, assertPreparedRunBundleSqlAuthority } from "./runs.js";
|
|
22
|
+
import { loadExternalStageContext } from "./stage-context.js";
|
|
23
|
+
import { flowCommand } from "./stage-pause.js";
|
|
24
|
+
import { sameSessionProfile, runtimeHarness } from "./execution-policy.js";
|
|
25
|
+
import { selectControllerExecution } from "./controller-execution.js";
|
|
26
|
+
import { resolveHarnessCommand, harnessRuntimeArguments } from "./harness-config.js";
|
|
27
|
+
import { adapterSessionId, prepareHarnessFlowExecutable, runHarnessAdapter } from "./harness-adapter.js";
|
|
28
|
+
import { assertManagedProcessLease, confirmManagedProcess, renewManagedProcess, finishManagedProcess, processStartedAt, processTreeIsAlive, resourceHome } from "./managed-processes.js";
|
|
29
|
+
import { managedLeaseMonitor } from "./managed-lease-monitor.js";
|
|
30
|
+
import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
31
|
+
import { initCodexHome } from "./hooks.js";
|
|
32
|
+
import { launchExternalWork } from "./external-work-launch.js";
|
|
33
|
+
import { controllerNativeChildren, controllerStageEntryPrompt, controllerStageResumePrompt, nextControllerFanout } from "./controller-fanout.js";
|
|
34
|
+
import { claimQueuedMergeRequest } from "./merge-server.js";
|
|
35
|
+
import { prepareVnextCodeWorkspace } from "./vnext-code.js";
|
|
36
|
+
import { requireRun, requireController, controllerReceipt, appendEvent, saveState, retainControllerContinuation, retainControllerCapacityIntent, digest, digestBytes, errorRecord } from "./run-controller-state.js";
|
|
37
|
+
import { assertNoCorrelatedRejection } from "./continuation-outcome.js";
|
|
38
|
+
import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady, assertControllerLeaseAllowed } from "./run-controller-process.js";
|
|
39
|
+
import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
|
|
40
|
+
import { retainedChildObserver } from "./controller-child-observer.js";
|
|
41
|
+
import { capacityDelay, continueProviderCapacity, eligibleProviderChildOverload, providerChildFailure } from "./provider-capacity.js";
|
|
42
|
+
import { resumeControllerSession, failControllerRecovery } from "./run-controller-recovery.js";
|
|
43
|
+
import { captureControllerBoundary } from "./run-controller-capture.js";
|
|
44
|
+
import { requestRunControl } from "./run-control-worker.js";
|
|
45
|
+
import { storageSessionId } from "./session-identity.js";
|
|
46
|
+
import { CAPACITY_POLICY, nativeItemsSettled, overloadExhausted, refusalObservation, terminalProviderOverload } from "../harness-runtime/lib/provider-overload-policy.mjs";
|
|
47
|
+
import { renderNativeChildFollowupInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
|
|
48
|
+
import { closeSuccessfulChildCapacity } from "./controller-child-capacity.js";
|
|
49
|
+
import { isObservationLoss } from "../harness-runtime/lib/operation-errors.mjs";
|
|
50
|
+
export { resumeRunController } from "./run-controller-recovery.js";
|
|
51
|
+
const preparedLaunchInputs = new WeakMap();
|
|
52
|
+
function launchInputDigest(input) {
|
|
53
|
+
return digest([input.projectRoot, input.runId, input.requestId, input.contextFile, input.contextSha256,
|
|
54
|
+
input.stopAfter ?? null, input.captureRoot ?? null]);
|
|
55
|
+
}
|
|
56
|
+
export function prepareRunControllerLaunch(context, input) {
|
|
57
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !/^[a-f0-9]{64}$/.test(input.contextSha256))
|
|
58
|
+
throw new AppError("usage", "Managed launch needs a request id and context SHA-256", 2);
|
|
59
|
+
if (input.stopAfter !== undefined && !/^[a-z][a-z-]*$/.test(input.stopAfter))
|
|
60
|
+
throw new AppError("usage", "Invalid stop-after stage", 2);
|
|
61
|
+
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
62
|
+
const run = requireRun(context, project.id, input.runId);
|
|
63
|
+
const captureRoot = input.captureRoot === undefined ? undefined : canonicalPath(input.captureRoot, false);
|
|
64
|
+
if (captureRoot && [context.ddFlowHome, project.root, run.workspace_root].some(root => { const relative = path.relative(canonicalPath(root, false), captureRoot); return relative === "" || relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); }))
|
|
65
|
+
throw new AppError("snapshot_output_inside_runtime", "Controller captures require a root outside the runtime, project and workspace", 2);
|
|
66
|
+
if (captureRoot && fs.existsSync(captureRoot) && !fs.statSync(captureRoot).isDirectory())
|
|
67
|
+
throw new AppError("usage", "Controller capture root must be a directory", 2);
|
|
68
|
+
const budget = parseRuntimeBudget(context.env.DD_FLOW_RUNTIME_BUDGET);
|
|
69
|
+
const request = { project_root: project.root, run_id: run.id, context_sha256: input.contextSha256, stop_after: input.stopAfter ?? null, ...(captureRoot ? { capture_root: captureRoot } : {}), ...(budget ? { budget_sha256: digest(budget) } : {}) };
|
|
70
|
+
const requestHash = digest(request);
|
|
71
|
+
const existing = context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'")
|
|
72
|
+
? context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND request_id = ?", [project.id, run.id, input.requestId]) : undefined;
|
|
73
|
+
if (existing) {
|
|
74
|
+
if (existing.request_hash !== requestHash)
|
|
75
|
+
throw new AppError("runtime_request_conflict", "Managed launch request id has different inputs", 1);
|
|
76
|
+
selectControllerExecution(context, run, JSON.parse(existing.manifest_json));
|
|
77
|
+
const prepared = { kind: "replay", existing };
|
|
78
|
+
preparedLaunchInputs.set(prepared, { home: context.ddFlowHome, sha256: launchInputDigest(input) });
|
|
79
|
+
return prepared;
|
|
80
|
+
}
|
|
81
|
+
const generation = assertRunMutationAllowed(context, project.id, run.id);
|
|
82
|
+
if (run.status !== "running")
|
|
83
|
+
throw new AppError("invalid_run_state", "Managed launch requires a running logical RUN", 1, { status: run.status });
|
|
84
|
+
const routing = selectControllerExecution(context, run);
|
|
85
|
+
const contexts = readContexts(input.contextFile, input.contextSha256);
|
|
86
|
+
if (routing.bundle) {
|
|
87
|
+
for (const stage of [...Object.keys(contexts), ...(input.stopAfter ? [input.stopAfter] : [])])
|
|
88
|
+
routing.bundle.descriptor.stage(stage);
|
|
89
|
+
}
|
|
90
|
+
const next = getFlowRunStatus(context, { projectRoot: project.root, runId: run.id }).continuation;
|
|
91
|
+
if (next.kind !== "start_stage" || !next.stage)
|
|
92
|
+
throw new AppError("controller_attach_required", "Managed launch cannot replace an attached or unfinished coordinator; reconcile its existing owner", 1, { continuation: next });
|
|
93
|
+
if (!contexts[next.stage])
|
|
94
|
+
throw new AppError("controller_context_missing", "Managed context omits the first stage", 2, { stage: next.stage });
|
|
95
|
+
const adapters = {};
|
|
96
|
+
for (const profile of Object.values(routing.profiles)) {
|
|
97
|
+
const harness = runtimeHarness(profile);
|
|
98
|
+
adapters[harness] = { executable: resolveHarnessCommand(context.ddFlowHome, harness, "adapter"), runtime_args: harnessRuntimeArguments(context.ddFlowHome, harness) };
|
|
99
|
+
}
|
|
100
|
+
const entry = fileURLToPath(new URL("../cli.js", import.meta.url));
|
|
101
|
+
if (!fs.existsSync(entry))
|
|
102
|
+
throw new AppError("harness_runtime_unbuilt", "Build the CLI before launching its detached controller", 1);
|
|
103
|
+
const prepared = { kind: "create", project, run, request, requestHash, generation, budget, routing, contexts, adapters };
|
|
104
|
+
preparedLaunchInputs.set(prepared, { home: context.ddFlowHome, sha256: launchInputDigest(input) });
|
|
105
|
+
return prepared;
|
|
106
|
+
}
|
|
107
|
+
/** Retain the existing launch intent and frozen inputs without starting a
|
|
108
|
+
* process. Dispatch remains exclusively owned by launchRunController. */
|
|
109
|
+
export function retainRunControllerLaunch(context, input, prepared = prepareRunControllerLaunch(context, input)) {
|
|
110
|
+
const selection = preparedLaunchInputs.get(prepared);
|
|
111
|
+
if (!selection || selection.home !== context.ddFlowHome || selection.sha256 !== launchInputDigest(input)) {
|
|
112
|
+
throw new AppError("runtime_request_conflict", "Controller launch differs from its prepared request", 1, { effect: "no_effect" });
|
|
113
|
+
}
|
|
114
|
+
if (prepared.kind === "replay") {
|
|
115
|
+
const current = requireController(context, prepared.existing.controller_id);
|
|
116
|
+
if (current.request_hash !== prepared.existing.request_hash)
|
|
117
|
+
throw new AppError("runtime_request_conflict", "Controller launch authority changed after replay preparation", 1);
|
|
118
|
+
selectControllerExecution(context, requireRun(context, current.project_id, current.run_id), JSON.parse(current.manifest_json));
|
|
119
|
+
return { controller: current, reused: true };
|
|
120
|
+
}
|
|
121
|
+
const { project, run, request, requestHash, generation, budget, routing, contexts, adapters } = prepared;
|
|
122
|
+
assertRunMutationAllowed(context, project.id, run.id, generation);
|
|
123
|
+
ensureRunControllerStorage(context.db);
|
|
124
|
+
const controllerId = `DRV-${crypto.randomUUID()}`;
|
|
125
|
+
const root = path.join(run.run_root, "controllers", controllerId);
|
|
126
|
+
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
127
|
+
const flowExecutable = prepareHarnessFlowExecutable(context, root, routing.bundle?.engine());
|
|
128
|
+
const frozenContexts = {};
|
|
129
|
+
for (const [stage, value] of Object.entries(contexts)) {
|
|
130
|
+
const file = path.join(root, `${stage}.context.json`);
|
|
131
|
+
fs.writeFileSync(file, value.bytes, { flag: "wx", mode: 0o600 });
|
|
132
|
+
frozenContexts[stage] = { file, sha256: value.sha256 };
|
|
133
|
+
}
|
|
134
|
+
const manifest = { schema_id: routing.kind === "descriptor" ? "dd-flow/run-controller-manifest@2" : "dd-flow/run-controller-manifest@1", request,
|
|
135
|
+
...(routing.kind === "descriptor" ? { execution_binding: routing.binding } : { execution_profile: routing.legacy }),
|
|
136
|
+
contexts: frozenContexts, adapters, flow_executable: flowExecutable, resource_home: resourceHome(context), budget };
|
|
137
|
+
const token = crypto.randomUUID();
|
|
138
|
+
const now = context.now();
|
|
139
|
+
const state = { stage: null, sessions: [], current_session: null, turns: 0, last_continuation: null };
|
|
140
|
+
const launcher = { pid: process.pid, pid_started_at: processStartedAt(process.pid) };
|
|
141
|
+
const duplicate = context.db.writeTransaction(() => {
|
|
142
|
+
if (routing.bundle)
|
|
143
|
+
assertPreparedRunBundleSqlAuthority(context, { projectId: project.id, runId: run.id }, routing.bundle);
|
|
144
|
+
const duplicate = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND request_id = ?", [project.id, run.id, input.requestId]);
|
|
145
|
+
if (duplicate) {
|
|
146
|
+
if (duplicate.request_hash !== requestHash)
|
|
147
|
+
throw new AppError("runtime_request_conflict", "Managed launch request id has different inputs", 1);
|
|
148
|
+
return duplicate;
|
|
149
|
+
}
|
|
150
|
+
assertRunMutationAllowed(context, project.id, run.id, generation);
|
|
151
|
+
if (requireRun(context, project.id, run.id).status !== "running")
|
|
152
|
+
throw new AppError("invalid_run_state", "Managed launch requires a running logical RUN", 1);
|
|
153
|
+
const owner = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND status NOT IN ('completed','cancelled','superseded')", [project.id, run.id]);
|
|
154
|
+
if (owner)
|
|
155
|
+
throw new AppError("controller_already_owned", "RUN already has an unresolved managed owner; observe or reconcile it", 1, { controller_id: owner.controller_id });
|
|
156
|
+
if (context.db.get("SELECT 1 FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.status = 'running' LIMIT 1", [project.id, run.id]))
|
|
157
|
+
throw new AppError("controller_attach_required", "A WorkSession acquired the RUN before managed launch", 1);
|
|
158
|
+
context.db.run("INSERT INTO run_controllers (controller_id, project_id, run_id, request_id, request_hash, generation, status, owner_token, manifest_json, state_json, state_dir, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 'starting', ?, ?, ?, ?, ?, ?)", [controllerId, project.id, run.id, input.requestId, requestHash, generation, token, JSON.stringify(manifest), JSON.stringify(state), root, now, now]);
|
|
159
|
+
recordControllerLauncher(context, requireController(context, controllerId), undefined, launcher);
|
|
160
|
+
appendEvent(context, controllerId, "launch_requested", { request_id: input.requestId, generation, request_hash: requestHash });
|
|
161
|
+
return null;
|
|
162
|
+
});
|
|
163
|
+
return { controller: duplicate ?? requireController(context, controllerId), reused: duplicate !== null };
|
|
164
|
+
}
|
|
165
|
+
export async function launchRunController(context, input, prepared = prepareRunControllerLaunch(context, input)) {
|
|
166
|
+
const retained = retainRunControllerLaunch(context, input, prepared);
|
|
167
|
+
// A simultaneous publisher already owns dispatch. Preserve the original
|
|
168
|
+
// race behavior: only an explicit replay may reconcile that existing owner.
|
|
169
|
+
if (prepared.kind === "create" && retained.reused)
|
|
170
|
+
return { ...controllerReceipt(retained.controller), reused: true };
|
|
171
|
+
const receipt = await spawnController(context, retained.controller);
|
|
172
|
+
return retained.reused ? { ...receipt, reused: true } : receipt;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* A registry write can be temporarily unavailable while another runtime client
|
|
176
|
+
* owns SQLite's writer lock. That is not evidence that this controller lost
|
|
177
|
+
* its lease. Productive dispatch still requires a successful recheck before
|
|
178
|
+
* proceeding, while an explicit negative renewal remains a hard ownership
|
|
179
|
+
* loss.
|
|
180
|
+
*/
|
|
181
|
+
export const controllerLeaseMonitor = managedLeaseMonitor;
|
|
182
|
+
/** Unconfirmed ownership is not evidence authorizing a native cancellation. */
|
|
183
|
+
export function retainControllerLeaseUncertainty(context, row, error) {
|
|
184
|
+
if (!(error instanceof AppError) || error.code !== "controller_lease_unconfirmed")
|
|
185
|
+
return false;
|
|
186
|
+
try {
|
|
187
|
+
const db = context.db.withDeadline(Date.now() + 5_000);
|
|
188
|
+
db.writeTransaction(() => {
|
|
189
|
+
const changed = db.run("UPDATE run_controllers SET status = CASE WHEN status IN ('capturing','capture_failed') THEN 'capture_failed' ELSE 'recovery_required' END, last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND generation = ? AND status NOT IN ('completed','cancelled','superseded','recovery_required','capture_failed')", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token, row.generation]);
|
|
190
|
+
if (changed.changes)
|
|
191
|
+
appendEvent({ ...context, db }, row.controller_id, "owner_confirmation_required", { ...errorRecord(error), native_cancelled: false });
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
catch (persistence) {
|
|
195
|
+
process.stderr.write(`${JSON.stringify({ controller_id: row.controller_id, primary: errorRecord(error), persistence: errorRecord(persistence), native_cancelled: false })}\n`);
|
|
196
|
+
}
|
|
197
|
+
return true;
|
|
198
|
+
}
|
|
199
|
+
/** No initialization, provider calls or lease takeover on the observation path. */
|
|
200
|
+
export function runControllerStatus(context, input) {
|
|
201
|
+
if (input.after !== undefined && (!Number.isSafeInteger(input.after) || input.after < 0))
|
|
202
|
+
throw new AppError("usage", "--after must be a non-negative cursor", 2);
|
|
203
|
+
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
204
|
+
const run = requireRun(context, project.id, input.runId);
|
|
205
|
+
if (!context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'"))
|
|
206
|
+
return { ok: true, run_id: run.id, controller: null, events: [], next_cursor: input.after ?? 0 };
|
|
207
|
+
const row = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? ORDER BY created_at DESC, controller_id DESC LIMIT 1", [project.id, run.id]);
|
|
208
|
+
if (!row)
|
|
209
|
+
return { ok: true, run_id: run.id, controller: null, events: [], next_cursor: input.after ?? 0 };
|
|
210
|
+
const events = context.db.all("SELECT sequence, event_type, data_json, created_at FROM run_controller_events WHERE controller_id = ? AND sequence > ? ORDER BY sequence LIMIT 100", [row.controller_id, input.after ?? 0]);
|
|
211
|
+
return { ...controllerReceipt(row), events: events.map(event => ({ sequence: event.sequence, type: event.event_type, data: JSON.parse(event.data_json), at: event.created_at })), next_cursor: events.at(-1)?.sequence ?? input.after ?? 0 };
|
|
212
|
+
}
|
|
213
|
+
export async function serveRunController(context, input) {
|
|
214
|
+
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
215
|
+
const row = requireController(context, input.controllerId);
|
|
216
|
+
if (row.project_id !== project.id || row.run_id !== input.runId || row.owner_token !== context.env.DD_FLOW_CONTROLLER_TOKEN)
|
|
217
|
+
throw new AppError("controller_owner_mismatch", "Managed serve does not own this launch", 1);
|
|
218
|
+
claimControllerProcess(context, row);
|
|
219
|
+
if (!row.process_id || !row.process_lease_token)
|
|
220
|
+
throw new AppError("controller_owner_mismatch", "Managed owner has no resource lease", 1);
|
|
221
|
+
const confirmed = confirmManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
|
|
222
|
+
const lease = controllerLeaseMonitor(attemptMs => renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: row.process_id, leaseToken: row.process_lease_token }), confirmed.lease_expires_at);
|
|
223
|
+
const state = JSON.parse(row.state_json);
|
|
224
|
+
// Resource and RUN commits are distinct effects. A RUN keepalive failure must
|
|
225
|
+
// neither erase a resource receipt nor grant RUN dispatch authority.
|
|
226
|
+
const mergeLease = controllerLeaseMonitor(attemptMs => {
|
|
227
|
+
const expiry = new Date(Date.now() + 120_000).toISOString();
|
|
228
|
+
const db = context.db.withDeadline(Date.now() + attemptMs);
|
|
229
|
+
if (db.get("SELECT 1 FROM merge_requests WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching' LIMIT 1", [row.project_id, row.run_id, row.controller_id]))
|
|
230
|
+
db.writeTransaction(() => {
|
|
231
|
+
const guard = recoveryGuard({ ...context, db }, row.project_id, row.run_id);
|
|
232
|
+
if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
|
|
233
|
+
throw new AppError("controller_lease_lost", "RUN generation changed before MERGE keepalive", 1);
|
|
234
|
+
db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching'", [expiry, context.now(), row.project_id, row.run_id, row.controller_id]);
|
|
235
|
+
});
|
|
236
|
+
return { owned: true, lease_expires_at: expiry };
|
|
237
|
+
});
|
|
238
|
+
const assertLeaseFence = () => {
|
|
239
|
+
assertControllerLeaseAllowed(context, row);
|
|
240
|
+
};
|
|
241
|
+
const heartbeat = setInterval(() => {
|
|
242
|
+
lease.background(assertLeaseFence, () => { });
|
|
243
|
+
if (state.merge_claim)
|
|
244
|
+
mergeLease.background(assertLeaseFence, () => { });
|
|
245
|
+
}, 30_000);
|
|
246
|
+
heartbeat.unref();
|
|
247
|
+
const manifest = JSON.parse(row.manifest_json);
|
|
248
|
+
const assertPhysicalOwner = () => {
|
|
249
|
+
lease.assertNoRenewal();
|
|
250
|
+
if (state.merge_claim)
|
|
251
|
+
mergeLease.assertNoRenewal();
|
|
252
|
+
// This isolated read already bounds SQLite contention. A wall deadline
|
|
253
|
+
// before opening the reader mistakes scheduler delay for heartbeat loss;
|
|
254
|
+
// the observed row's actual expiry/token still fences productive dispatch.
|
|
255
|
+
assertManagedProcessLease(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid });
|
|
256
|
+
const current = requireController(context, row.controller_id);
|
|
257
|
+
if (current.owner_pid !== process.pid || current.owner_token !== row.owner_token || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
|
|
258
|
+
throw new AppError("controller_owner_mismatch", "Controller no longer owns productive dispatch", 1);
|
|
259
|
+
};
|
|
260
|
+
const assertOwner = () => { assertPhysicalOwner(); assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation); };
|
|
261
|
+
let primaryFailure;
|
|
262
|
+
const retainSecondary = (phase, error) => {
|
|
263
|
+
process.stderr.write(`${JSON.stringify({ controller_id: row.controller_id, phase, primary: errorRecord(primaryFailure), secondary: errorRecord(error) })}\n`);
|
|
264
|
+
};
|
|
265
|
+
try {
|
|
266
|
+
await lease.ensureConfirmed(assertLeaseFence);
|
|
267
|
+
if (state.merge_claim)
|
|
268
|
+
await mergeLease.ensureConfirmed(assertLeaseFence);
|
|
269
|
+
context.db.writeTransaction(() => {
|
|
270
|
+
markControllerDispatchReady(context, row);
|
|
271
|
+
if (!state.recovery && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation && requireRun(context, row.project_id, row.run_id).status === "paused")
|
|
272
|
+
setFlowRunRecoveryPaused(context, row.project_id, row.run_id, false);
|
|
273
|
+
});
|
|
274
|
+
appendEvent(context, row.controller_id, "owner_started", { pid: process.pid, generation: row.generation });
|
|
275
|
+
if (state.recovery && !["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
|
|
276
|
+
await resumeControllerSession(context, row, manifest, state, assertPhysicalOwner);
|
|
277
|
+
await executeController(context, row, manifest, state, assertOwner, assertPhysicalOwner, async () => { await lease.ensureConfirmed(assertLeaseFence); if (state.merge_claim)
|
|
278
|
+
await mergeLease.ensureConfirmed(assertLeaseFence); });
|
|
279
|
+
return controllerReceipt(requireController(context, row.controller_id));
|
|
280
|
+
}
|
|
281
|
+
catch (error) {
|
|
282
|
+
primaryFailure = error;
|
|
283
|
+
try {
|
|
284
|
+
try {
|
|
285
|
+
failControllerRecovery(context, row, state, error);
|
|
286
|
+
}
|
|
287
|
+
catch (secondary) {
|
|
288
|
+
retainSecondary("fail_controller_recovery", secondary);
|
|
289
|
+
}
|
|
290
|
+
if ((error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale"].includes(error.code) || (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) !== row.generation)
|
|
291
|
+
&& context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'")) {
|
|
292
|
+
const guard = recoveryGuard(context, row.project_id, row.run_id);
|
|
293
|
+
const control = guard && ["draining", "sealed"].includes(guard.status) && context.db.get("SELECT control_id, action FROM run_controls WHERE project_id = ? AND run_id = ? AND recovery_id = ? AND status IN ('requested','settled')", [row.project_id, row.run_id, guard.recovery_id]);
|
|
294
|
+
if (control) {
|
|
295
|
+
// Control is a later state transition, not evidence that the causal
|
|
296
|
+
// failure which fenced admission disappeared.
|
|
297
|
+
context.db.run("UPDATE run_controllers SET status = CASE WHEN last_error_json IS NULL THEN 'control_requested' ELSE status END, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
|
|
298
|
+
appendEvent(context, row.controller_id, "control_boundary_reached", { control_id: control.control_id, action: control.action, generation: guard.generation, settled: false });
|
|
299
|
+
return controllerReceipt(requireController(context, row.controller_id));
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
try {
|
|
303
|
+
context.db.writeTransaction(() => {
|
|
304
|
+
// A failed capture must not reopen admission when this process exits.
|
|
305
|
+
const changed = context.db.run("UPDATE run_controllers SET status = CASE WHEN status IN ('capturing','capture_failed') THEN 'capture_failed' ELSE 'recovery_required' END, last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status NOT IN ('completed','cancelled','superseded')", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
|
|
306
|
+
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status IN ('dispatching','active')", [JSON.stringify(errorRecord(error)), context.now(), row.project_id, row.run_id, row.controller_id]);
|
|
307
|
+
if (changed.changes && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation) {
|
|
308
|
+
setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
|
|
309
|
+
}
|
|
310
|
+
appendEvent(context, row.controller_id, "recovery_required", errorRecord(error));
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
catch (failure) {
|
|
314
|
+
retainSecondary("persist_controller_failure", failure);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
catch (secondary) {
|
|
318
|
+
retainSecondary("controller_failure_boundary", secondary);
|
|
319
|
+
}
|
|
320
|
+
throw error;
|
|
321
|
+
}
|
|
322
|
+
finally {
|
|
323
|
+
lease.close();
|
|
324
|
+
mergeLease.close();
|
|
325
|
+
clearInterval(heartbeat);
|
|
326
|
+
finishOwner();
|
|
327
|
+
}
|
|
328
|
+
function finishOwner() {
|
|
329
|
+
try {
|
|
330
|
+
finishManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token, state: "stopped", reason: "controller_owner_exited" });
|
|
331
|
+
}
|
|
332
|
+
catch (cleanup) {
|
|
333
|
+
if (!primaryFailure)
|
|
334
|
+
throw cleanup;
|
|
335
|
+
retainSecondary("finish_controller_process", cleanup);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
export function prepareRunControllerAnswer(context, input) {
|
|
340
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !input.pauseId.trim())
|
|
341
|
+
throw new AppError("usage", "Answer requires request-id and pause-id", 2);
|
|
342
|
+
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
343
|
+
const prior = context.db.get(`SELECT o.operation_id, o.status, o.input_json
|
|
344
|
+
FROM run_controller_operations o JOIN run_controllers c ON c.controller_id = o.controller_id
|
|
345
|
+
WHERE c.project_id = ? AND c.run_id = ? AND o.operation_id = c.controller_id || ':answer:' || ?`, [project.id, input.runId, input.requestId]);
|
|
346
|
+
if (prior) {
|
|
347
|
+
const retained = JSON.parse(prior.input_json);
|
|
348
|
+
if (retained.pause_id !== input.pauseId)
|
|
349
|
+
throw new AppError("runtime_request_conflict", "HITL answer request id belongs to another pause", 1, { operation_id: prior.operation_id });
|
|
350
|
+
return { kind: "replay", operationId: prior.operation_id, status: prior.status };
|
|
351
|
+
}
|
|
352
|
+
const { bytes } = prepareFileBytes(input.answerFile, "HITL answer");
|
|
353
|
+
if (!bytes.toString("utf8").trim())
|
|
354
|
+
throw new AppError("validation", "HITL answer must not be empty", 2);
|
|
355
|
+
const row = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND status NOT IN ('completed','cancelled','superseded')", [project.id, input.runId]);
|
|
356
|
+
if (!row)
|
|
357
|
+
throw new AppError("not_found", "RUN has no managed controller for this answer", 1);
|
|
358
|
+
const operationId = `${row.controller_id}:answer:${input.requestId}`;
|
|
359
|
+
const answerHash = digestBytes(bytes);
|
|
360
|
+
const hash = digest({ pause_id: input.pauseId, answer_sha256: answerHash });
|
|
361
|
+
const previous = prepareControllerAnswerAcceptance(context, input, project.id, row, operationId, hash);
|
|
362
|
+
if (previous.prior)
|
|
363
|
+
return { kind: "replay", operationId, status: previous.prior.status };
|
|
364
|
+
return { kind: "create", project, row, bytes, operationId, answerHash, hash };
|
|
365
|
+
}
|
|
366
|
+
function prepareControllerAnswerAcceptance(context, input, projectId, row, operationId, hash) {
|
|
367
|
+
const prior = context.db.get("SELECT input_hash, status FROM run_controller_operations WHERE operation_id = ?", [operationId]);
|
|
368
|
+
if (prior) {
|
|
369
|
+
if (prior.input_hash !== hash)
|
|
370
|
+
throw new AppError("runtime_request_conflict", "HITL answer request id has different raw input", 1);
|
|
371
|
+
return { prior };
|
|
372
|
+
}
|
|
373
|
+
const currentOwner = requireController(context, row.controller_id);
|
|
374
|
+
if (["completed", "cancelled", "superseded"].includes(currentOwner.status))
|
|
375
|
+
throw new AppError("invalid_run_state", "The answer's controller is no longer current", 1);
|
|
376
|
+
const competing = context.db.get("SELECT operation_id FROM run_controller_operations WHERE controller_id = ? AND json_extract(input_json, '$.action') = 'user_answer' AND json_extract(input_json, '$.pause_id') = ? LIMIT 1", [row.controller_id, input.pauseId]);
|
|
377
|
+
if (competing)
|
|
378
|
+
throw new AppError("runtime_request_conflict", "This HITL pause already has an accepted answer", 1, { operation_id: competing.operation_id });
|
|
379
|
+
const run = requireRun(context, projectId, input.runId);
|
|
380
|
+
if (["done", "failed", "cancelled"].includes(run.status))
|
|
381
|
+
throw new AppError("invalid_run_state", "A terminal RUN cannot accept a new answer", 1);
|
|
382
|
+
const index = JSON.parse(run.index_json);
|
|
383
|
+
const stage = index.stage_runs.find(stage => stage.pause?.id === input.pauseId && ["paused", "waiting_for_user"].includes(stage.status));
|
|
384
|
+
if (!stage?.pause?.work_id)
|
|
385
|
+
throw new AppError("hitl_pause_mismatch", "Answer does not match the RUN's current HITL pause", 1);
|
|
386
|
+
return { run, stage, workId: stage.pause.work_id };
|
|
387
|
+
}
|
|
388
|
+
export function answerRunController(context, input, prepared = prepareRunControllerAnswer(context, input)) {
|
|
389
|
+
if (prepared.kind === "replay")
|
|
390
|
+
return { ok: true, operation_id: prepared.operationId, status: prepared.status, reused: true };
|
|
391
|
+
const { project, row, bytes, operationId, answerHash, hash } = prepared;
|
|
392
|
+
const previous = prepareControllerAnswerAcceptance(context, input, project.id, row, operationId, hash);
|
|
393
|
+
if (previous.prior)
|
|
394
|
+
return { ok: true, operation_id: operationId, status: previous.prior.status, reused: true };
|
|
395
|
+
const file = path.join(row.state_dir, `answer-${digest(operationId)}-${answerHash}.md`);
|
|
396
|
+
publishImmutableInput(file, bytes, "runtime_request_conflict");
|
|
397
|
+
return context.db.writeTransaction(() => {
|
|
398
|
+
// Recheck only mutable acceptance state under the write transaction; bytes are already prepared.
|
|
399
|
+
const acceptance = prepareControllerAnswerAcceptance(context, input, project.id, row, operationId, hash);
|
|
400
|
+
if (acceptance.prior) {
|
|
401
|
+
return { ok: true, operation_id: operationId, status: acceptance.prior.status, reused: true };
|
|
402
|
+
}
|
|
403
|
+
const { run, stage, workId } = acceptance;
|
|
404
|
+
if (requireController(context, row.controller_id).state_dir !== row.state_dir)
|
|
405
|
+
throw new AppError("controller_owner_mismatch", "Controller input directory changed before acceptance", 1);
|
|
406
|
+
const now = context.now();
|
|
407
|
+
const generation = recoveryGuard(context, project.id, run.id)?.generation ?? row.generation;
|
|
408
|
+
context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'accepted', ?, ?)", [operationId, row.controller_id, generation, hash, JSON.stringify({ action: "user_answer", pause_id: input.pauseId, stage: stage.stage, work_id: workId, file, sha256: answerHash }), now, now]);
|
|
409
|
+
appendEvent(context, row.controller_id, "answer_accepted", { operation_id: operationId, pause_id: input.pauseId, answer_sha256: answerHash });
|
|
410
|
+
return { ok: true, operation_id: operationId, status: "accepted" };
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
export function prepareRunControllerContext(context, input) {
|
|
414
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !/^[a-z][a-z-]*$/.test(input.stage) || !/^[a-f0-9]{64}$/.test(input.contextSha256))
|
|
415
|
+
throw new AppError("usage", "Context submission requires request id, stage and SHA-256", 2);
|
|
416
|
+
const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
|
|
417
|
+
const operationId = `${input.controllerId}:context:${input.requestId}`;
|
|
418
|
+
const hash = digest({ stage: input.stage, sha256: input.contextSha256 });
|
|
419
|
+
const row = requireController(context, input.controllerId);
|
|
420
|
+
if (row.project_id !== project.id || row.run_id !== input.runId)
|
|
421
|
+
throw new AppError("controller_owner_mismatch", "Context submission belongs to another RUN", 1);
|
|
422
|
+
const prior = context.db.get("SELECT input_hash, status FROM run_controller_operations WHERE operation_id = ?", [operationId]);
|
|
423
|
+
if (prior) {
|
|
424
|
+
if (prior.input_hash !== hash)
|
|
425
|
+
throw new AppError("runtime_request_conflict", "Context request id has different inputs", 1);
|
|
426
|
+
return { kind: "replay", operationId, status: prior.status };
|
|
427
|
+
}
|
|
428
|
+
availableControllerContext(context, input, row);
|
|
429
|
+
const { bytes } = loadExternalStageContext({ file: input.contextFile, expectedSha256: input.contextSha256, stage: input.stage });
|
|
430
|
+
return { kind: "create", project, operationId, hash, bytes };
|
|
431
|
+
}
|
|
432
|
+
export function supplyRunControllerContext(context, input, prepared = prepareRunControllerContext(context, input)) {
|
|
433
|
+
if (prepared.kind === "replay")
|
|
434
|
+
return { ok: true, operation_id: prepared.operationId, status: prepared.status, reused: true };
|
|
435
|
+
const { project, operationId, hash, bytes } = prepared;
|
|
436
|
+
const preparedOwner = requireController(context, input.controllerId);
|
|
437
|
+
if (preparedOwner.project_id !== project.id || preparedOwner.run_id !== input.runId)
|
|
438
|
+
throw new AppError("controller_owner_mismatch", "Context submission belongs to another RUN", 1);
|
|
439
|
+
const existing = context.db.get("SELECT input_hash, status FROM run_controller_operations WHERE operation_id = ?", [operationId]);
|
|
440
|
+
if (existing) {
|
|
441
|
+
if (existing.input_hash !== hash)
|
|
442
|
+
throw new AppError("runtime_request_conflict", "Context request id has different inputs", 1);
|
|
443
|
+
return { ok: true, operation_id: operationId, status: existing.status, reused: true };
|
|
444
|
+
}
|
|
445
|
+
availableControllerContext(context, input, preparedOwner);
|
|
446
|
+
const file = path.join(preparedOwner.state_dir, `${input.stage}-${input.contextSha256}.context.json`);
|
|
447
|
+
publishImmutableInput(file, bytes, "stage_context_replacement");
|
|
448
|
+
return context.db.writeTransaction(() => {
|
|
449
|
+
const row = requireController(context, input.controllerId);
|
|
450
|
+
if (row.project_id !== project.id || row.run_id !== input.runId)
|
|
451
|
+
throw new AppError("controller_owner_mismatch", "Context submission belongs to another RUN", 1);
|
|
452
|
+
const prior = context.db.get("SELECT input_hash, status FROM run_controller_operations WHERE operation_id = ?", [operationId]);
|
|
453
|
+
if (prior) {
|
|
454
|
+
if (prior.input_hash !== hash)
|
|
455
|
+
throw new AppError("runtime_request_conflict", "Context request id has different inputs", 1);
|
|
456
|
+
return { ok: true, operation_id: operationId, status: prior.status, reused: true };
|
|
457
|
+
}
|
|
458
|
+
const manifest = availableControllerContext(context, input, row);
|
|
459
|
+
if (row.state_dir !== preparedOwner.state_dir)
|
|
460
|
+
throw new AppError("controller_owner_mismatch", "Controller input directory changed before acceptance", 1);
|
|
461
|
+
manifest.contexts[input.stage] = { file, sha256: input.contextSha256 };
|
|
462
|
+
const now = context.now();
|
|
463
|
+
context.db.run("UPDATE run_controllers SET manifest_json = ?, updated_at = ? WHERE controller_id = ?", [JSON.stringify(manifest), now, row.controller_id]);
|
|
464
|
+
context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'accepted', ?, ?)", [operationId, row.controller_id, row.generation, hash, JSON.stringify({ action: "stage_context", stage: input.stage, file, sha256: input.contextSha256 }), now, now]);
|
|
465
|
+
appendEvent(context, row.controller_id, "context_accepted", { operation_id: operationId, stage: input.stage, context_sha256: input.contextSha256 });
|
|
466
|
+
return { ok: true, operation_id: operationId, status: "accepted" };
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
function availableControllerContext(context, input, row) {
|
|
470
|
+
assertRunMutationAllowed(context, row.project_id, input.runId, row.generation);
|
|
471
|
+
if (!["starting", "running", "waiting_for_context", "waiting_for_user", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(row.status))
|
|
472
|
+
throw new AppError("invalid_run_state", "Controller cannot accept a new context in its current state", 1);
|
|
473
|
+
const manifest = JSON.parse(row.manifest_json);
|
|
474
|
+
if (manifest.contexts[input.stage])
|
|
475
|
+
throw new AppError("stage_context_replacement", "Controller already has an immutable context for this stage", 1);
|
|
476
|
+
return manifest;
|
|
477
|
+
}
|
|
478
|
+
async function executeController(context, row, manifest, state, assertOwnership, assertPhysicalOwner, renewLease) {
|
|
479
|
+
assertOwnership();
|
|
480
|
+
const observer = { ...context, db: getDatabase(context.ddFlowHome, "read_existing") };
|
|
481
|
+
const checkOutcomes = () => assertLifecycleOutcomes(observer, { projectRoot: manifest.request.project_root, runId: row.run_id, generation: row.generation });
|
|
482
|
+
const assertOwner = () => { if (fatal)
|
|
483
|
+
throw fatal; checkOutcomes(); assertOwnership(); };
|
|
484
|
+
const prepareMutation = async (action) => {
|
|
485
|
+
for (;;) {
|
|
486
|
+
await renewLease();
|
|
487
|
+
try {
|
|
488
|
+
context.db.writeTransaction(() => { assertOwner(); action(); });
|
|
489
|
+
return;
|
|
490
|
+
}
|
|
491
|
+
catch (error) {
|
|
492
|
+
if (!(error instanceof AppError) || error.code !== "controller_lease_unconfirmed")
|
|
493
|
+
throw error;
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
};
|
|
497
|
+
const routing = selectControllerExecution(context, requireRun(context, row.project_id, row.run_id), manifest);
|
|
498
|
+
const policy = routing.policy;
|
|
499
|
+
const profiles = routing.profiles;
|
|
500
|
+
let executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined, DD_FLOW_CODE_READINESS_OWNER: row.controller_id } };
|
|
501
|
+
const bindInvocationScope = (session) => {
|
|
502
|
+
executionContext = managedInvocationContext(executionContext, { harness: runtimeHarness(session.profile), stateDir: session.state_dir, sessionId: session.id,
|
|
503
|
+
projectRoot: manifest.request.project_root, runId: row.run_id, generation: row.generation });
|
|
504
|
+
};
|
|
505
|
+
let lastCapture = null;
|
|
506
|
+
let terminalFinalization = false;
|
|
507
|
+
let fatal;
|
|
508
|
+
let stopping;
|
|
509
|
+
const cancellation = new AbortController();
|
|
510
|
+
const observationCancellation = new AbortController();
|
|
511
|
+
const childObservers = new Map();
|
|
512
|
+
let promptPending = false;
|
|
513
|
+
let observedPromptOperation = null;
|
|
514
|
+
const childObserver = (session) => {
|
|
515
|
+
if (runtimeHarness(session.profile) !== "zcode-acp" || !state.stage || !["plan-review", "code", "code-review"].includes(state.stage))
|
|
516
|
+
return undefined;
|
|
517
|
+
if (readNativeDaemonState(session.state_dir).capabilities?.retained_child_observation !== true)
|
|
518
|
+
return undefined;
|
|
519
|
+
const scope = { controller_id: row.controller_id, generation: row.generation, stage: state.stage, root_session_id: session.id };
|
|
520
|
+
const key = JSON.stringify(scope);
|
|
521
|
+
let retained = childObservers.get(key);
|
|
522
|
+
if (!retained) {
|
|
523
|
+
retained = retainedChildObserver({ file: path.join(row.state_dir, "native-child-observations.jsonl"), scope, assertOwner: assertOwnership,
|
|
524
|
+
gap: error => { process.stderr.write(`${JSON.stringify({ event: "native_child_observation_gap", scope, error_code: String(error?.code ?? "observation_unavailable") })}\n`); },
|
|
525
|
+
read: async () => {
|
|
526
|
+
assertOwnership();
|
|
527
|
+
const operation = observedPromptOperation;
|
|
528
|
+
const binding = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id });
|
|
529
|
+
let receipt;
|
|
530
|
+
try {
|
|
531
|
+
receipt = await runHarnessAdapter({ executable: manifest.adapters["zcode-acp"].executable,
|
|
532
|
+
args: ["session", "observe-children", "--session-id", session.id, ...sessionArguments(context, manifest, session)],
|
|
533
|
+
env: controllerEnvironment(context, manifest, session), operationId: crypto.randomUUID(), harness: "zcode", timeoutMs: 30_000,
|
|
534
|
+
signal: AbortSignal.any([cancellation.signal, observationCancellation.signal]) });
|
|
535
|
+
}
|
|
536
|
+
catch (error) {
|
|
537
|
+
if (!promptPending || observedPromptOperation !== operation)
|
|
538
|
+
return null;
|
|
539
|
+
throw error;
|
|
540
|
+
}
|
|
541
|
+
if (!promptPending || observedPromptOperation !== operation || receipt.active_operation_id === null)
|
|
542
|
+
return null;
|
|
543
|
+
if (receipt.schema_id !== "dd-zcode/child-observation@1" || receipt.provider_session_id !== session.id || receipt.daemon_id !== binding.state.daemon_id)
|
|
544
|
+
throw new AppError("native_child_observation_conflict", "Retained child observation has foreign daemon/root identity", 1);
|
|
545
|
+
if (receipt.active_operation_id !== operation)
|
|
546
|
+
throw new AppError("native_child_observation_conflict", "Retained observation has a foreign productive operation", 1);
|
|
547
|
+
const children = controllerNativeChildren(receipt, session.id).filter(child => !(state.native_baseline ?? []).includes(child.session_id));
|
|
548
|
+
if (children.some(child => child.status === "failed" && !child.terminal_turn_id))
|
|
549
|
+
throw new AppError("operation_observation_lost", "Retained child terminal identity is unavailable", 1);
|
|
550
|
+
return children;
|
|
551
|
+
} });
|
|
552
|
+
childObservers.set(key, retained);
|
|
553
|
+
}
|
|
554
|
+
return retained;
|
|
555
|
+
};
|
|
556
|
+
const capacityUncertain = (error) => error instanceof AppError && ["capacity_continuation_blocked", "operation_observation_lost"].includes(error.code) && error.details.capacity_uncertainty === true || Boolean(state.pending_capacity) && (String(error?.code) === "capacity_continuation_blocked" || isObservationLoss(error));
|
|
557
|
+
const stopAfterFailure = (error) => {
|
|
558
|
+
// A retained successor may already be running. Fencing/recovery records
|
|
559
|
+
// uncertainty; cancellation is not evidence of its missing outcome.
|
|
560
|
+
if (capacityUncertain(error))
|
|
561
|
+
return Promise.resolve();
|
|
562
|
+
if (retainControllerLeaseUncertainty(context, row, error))
|
|
563
|
+
return Promise.resolve();
|
|
564
|
+
if (stopping)
|
|
565
|
+
return stopping;
|
|
566
|
+
fatal = error;
|
|
567
|
+
// Persist the cause before fencing/cancellation; later control is not an
|
|
568
|
+
// operator pause and cannot replace this execution failure.
|
|
569
|
+
stopping = (async () => {
|
|
570
|
+
try {
|
|
571
|
+
context.db.run("UPDATE run_controllers SET status = CASE WHEN status IN ('capturing','capture_failed') THEN 'capture_failed' ELSE 'recovery_required' END, last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
|
|
572
|
+
}
|
|
573
|
+
catch (persistence) {
|
|
574
|
+
process.stderr.write(`${JSON.stringify({ primary: errorRecord(error), persistence: errorRecord(persistence) })}\n`);
|
|
575
|
+
}
|
|
576
|
+
await requestRunControl(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id, requestId: `fatal:${row.controller_id}:${row.generation}`, action: "stop", waitMs: 0 });
|
|
577
|
+
})().catch(cleanup => {
|
|
578
|
+
// The caller still receives the primary failure if control cannot start.
|
|
579
|
+
try {
|
|
580
|
+
appendEvent(context, row.controller_id, "failure_control_error", { primary: errorRecord(error), cleanup: errorRecord(cleanup) });
|
|
581
|
+
}
|
|
582
|
+
catch {
|
|
583
|
+
process.stderr.write(`${JSON.stringify({ primary: errorRecord(error), cleanup: errorRecord(cleanup) })}\n`);
|
|
584
|
+
}
|
|
585
|
+
}).finally(() => cancellation.abort());
|
|
586
|
+
return stopping;
|
|
587
|
+
};
|
|
588
|
+
const setStatus = (status, terminalData) => {
|
|
589
|
+
context.db.writeTransaction(() => {
|
|
590
|
+
const changed = context.db.run("UPDATE run_controllers SET status = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status <> ?", [status, context.now(), row.controller_id, row.owner_token, status]);
|
|
591
|
+
if (changed.changes) {
|
|
592
|
+
appendEvent(context, row.controller_id, "status_changed", { status, stage: state.stage });
|
|
593
|
+
if (terminalData)
|
|
594
|
+
appendEvent(context, row.controller_id, status, terminalData);
|
|
595
|
+
}
|
|
596
|
+
});
|
|
597
|
+
};
|
|
598
|
+
const call = (session, args, productive = true) => controllerAdapter(executionContext, row, manifest, session, args, productive ? assertOwner : undefined, undefined, productive ? cancellation.signal : undefined, productive ? renewLease : undefined);
|
|
599
|
+
const confirmStopped = async (stateDir, receipt) => {
|
|
600
|
+
const fail = () => new AppError("controller_native_shutdown_unproven", "The managed daemon is not proven clean and closed", 1, { state_dir: stateDir, receipt });
|
|
601
|
+
if (receipt.stopped !== true || receipt.clean !== true)
|
|
602
|
+
throw fail();
|
|
603
|
+
const binding = requireManagedDaemonBinding(executionContext, { stateDir, projectId: row.project_id, runId: row.run_id });
|
|
604
|
+
const deadline = performance.now() + 30_000;
|
|
605
|
+
while (processTreeIsAlive(binding.process) && performance.now() < deadline)
|
|
606
|
+
await delay(100);
|
|
607
|
+
const current = readNativeDaemonState(stateDir);
|
|
608
|
+
if (processTreeIsAlive(binding.process) || current.shutdown_state !== "clean"
|
|
609
|
+
|| current.pid !== binding.process.pid || (current.daemon_id ?? current.config?.daemonId) !== (binding.state.daemon_id ?? binding.state.config?.daemonId)
|
|
610
|
+
|| current.resource_process?.id !== binding.process.id || current.resource_process?.lease_token !== binding.process.lease_token)
|
|
611
|
+
throw fail();
|
|
612
|
+
};
|
|
613
|
+
const stopSession = async (session) => {
|
|
614
|
+
if (session.stopped)
|
|
615
|
+
return;
|
|
616
|
+
const receipt = await call(session, ["daemon", "stop", "--state-dir", session.state_dir, "--json"], false);
|
|
617
|
+
await confirmStopped(session.state_dir, receipt);
|
|
618
|
+
session.stopped = true;
|
|
619
|
+
saveState(context, row, state);
|
|
620
|
+
appendEvent(context, row.controller_id, "session_stopped", { session_id: session.id || null, state_dir: session.state_dir, receipt });
|
|
621
|
+
};
|
|
622
|
+
const settle = async () => {
|
|
623
|
+
const results = await Promise.allSettled(state.sessions.map(stopSession));
|
|
624
|
+
const failure = results.find((result) => result.status === "rejected");
|
|
625
|
+
if (failure)
|
|
626
|
+
throw failure.reason;
|
|
627
|
+
};
|
|
628
|
+
const reconcilePromptPredecessor = async (session) => {
|
|
629
|
+
const prior = state.last_receipt?.settlement;
|
|
630
|
+
if (!prior || typeof prior !== "object" || Array.isArray(prior) || !["pending", "blocked"].includes(String(prior.state)))
|
|
631
|
+
return;
|
|
632
|
+
const receipt = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
633
|
+
state.last_receipt = receipt;
|
|
634
|
+
saveState(context, row, state);
|
|
635
|
+
const current = receipt.settlement;
|
|
636
|
+
if (current && typeof current === "object" && !Array.isArray(current) && ["pending", "blocked"].includes(String(current.state))) {
|
|
637
|
+
throw new AppError("settlement_reconciliation_required", "The managed Session still has an unsettled productive predecessor", 1, { session_id: session.id, settlement: current });
|
|
638
|
+
}
|
|
639
|
+
};
|
|
640
|
+
const prompt = async (session, text, operationId, kind) => {
|
|
641
|
+
const hitl = kind === "hitl";
|
|
642
|
+
const pending = state.pending_capacity;
|
|
643
|
+
if (pending && (pending.session_id !== session.id || pending.generation !== row.generation || pending.kind !== (kind ?? "stage") || operationId && operationId !== pending.root_operation_id))
|
|
644
|
+
throw new AppError("capacity_continuation_conflict", "Retained capacity owner reference differs from its Session/task", 1);
|
|
645
|
+
if (!pending)
|
|
646
|
+
await reconcilePromptPredecessor(session);
|
|
647
|
+
const firstOperationId = pending?.root_operation_id ?? operationId ?? `${row.controller_id}:prompt:${crypto.randomUUID()}`;
|
|
648
|
+
const send = (value, id) => {
|
|
649
|
+
const retained = context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [id, row.controller_id, row.generation]);
|
|
650
|
+
if (retained) {
|
|
651
|
+
const dispatch = JSON.parse(retained.input_json);
|
|
652
|
+
if (![value, `${value}\n`].some(text => dispatch.prompt_sha256 === crypto.createHash("sha256").update(text).digest("hex")))
|
|
653
|
+
throw new AppError("capacity_continuation_conflict", "Retained prompt differs from continuation", 1);
|
|
654
|
+
return controllerAdapter(executionContext, row, manifest, session, dispatch.args, assertOwner, id, cancellation.signal, renewLease);
|
|
655
|
+
}
|
|
656
|
+
const file = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
|
|
657
|
+
fs.writeFileSync(file, value, { flag: "wx", mode: 0o600 });
|
|
658
|
+
state.turns += 1;
|
|
659
|
+
saveState(context, row, state);
|
|
660
|
+
return controllerAdapter(executionContext, row, manifest, session, ["session", "prompt", "--session-id", session.id, "--prompt-file", file, ...sessionArguments(context, manifest, session)], assertOwner, id, cancellation.signal, renewLease);
|
|
661
|
+
};
|
|
662
|
+
let receipt;
|
|
663
|
+
try {
|
|
664
|
+
try {
|
|
665
|
+
if (pending) {
|
|
666
|
+
const predecessor = context.db.get("SELECT error_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'failed'", [pending.predecessor, row.controller_id, row.generation]);
|
|
667
|
+
if (!predecessor?.error_json)
|
|
668
|
+
throw new AppError("capacity_continuation_conflict", "Retained capacity predecessor has no failure receipt", 1);
|
|
669
|
+
// The root identifies the whole chain; inspect its frozen failure,
|
|
670
|
+
// then reconcile all retained successors before any dispatch.
|
|
671
|
+
const root = context.db.get("SELECT error_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [firstOperationId, row.controller_id, row.generation]);
|
|
672
|
+
if (!root?.error_json)
|
|
673
|
+
throw new AppError("capacity_continuation_conflict", "Retained capacity root has no failure receipt", 1);
|
|
674
|
+
throw JSON.parse(root.error_json);
|
|
675
|
+
}
|
|
676
|
+
if (childObserver(session)?.failed().length)
|
|
677
|
+
throw new AppError("fanout_dispatch_fenced", "A retained child failure forbids another coordinator dispatch", 1);
|
|
678
|
+
promptPending = true;
|
|
679
|
+
observedPromptOperation = firstOperationId;
|
|
680
|
+
try {
|
|
681
|
+
receipt = await send(text, firstOperationId);
|
|
682
|
+
}
|
|
683
|
+
finally {
|
|
684
|
+
promptPending = false;
|
|
685
|
+
observedPromptOperation = null;
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
catch (error) {
|
|
689
|
+
const resumeCommand = hitl ? text.split("\n\n")[1] : undefined;
|
|
690
|
+
if (hitl && !resumeCommand?.includes(" stage resume "))
|
|
691
|
+
throw error;
|
|
692
|
+
const inspect = (args, id) => runHarnessAdapter({ executable: manifest.adapters[runtimeHarness(session.profile)].executable,
|
|
693
|
+
args, env: controllerEnvironment(context, manifest, session), runtimeOwner: session.owner, operationId: id, timeoutMs: 30_000, signal: cancellation.signal });
|
|
694
|
+
const resumed = await continueProviderCapacity({ error, sessionId: session.id, operationId: firstOperationId, signal: cancellation.signal, allowSettledDescendants: true, atomicIntent: true,
|
|
695
|
+
retainedIntents: () => context.db.all("SELECT receipt_json FROM run_controller_operations WHERE controller_id = ? AND generation = ?", [row.controller_id, row.generation])
|
|
696
|
+
.flatMap(item => { const retained = JSON.parse(item.receipt_json || "null"); return retained?.capacity_intent?.root_operation_id === firstOperationId ? [retained.capacity_intent] : []; }).sort((a, b) => a.ordinal - b.ordinal),
|
|
697
|
+
retainedFailures: () => context.db.all("SELECT receipt_json FROM run_controller_operations WHERE controller_id = ? AND generation = ?", [row.controller_id, row.generation])
|
|
698
|
+
.flatMap(item => { const retained = JSON.parse(item.receipt_json || "null"); return retained?.capacity_root === firstOperationId && retained.capacity_failure ? [retained.capacity_failure] : []; }),
|
|
699
|
+
retainFailure: (id, refusal) => {
|
|
700
|
+
const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'outcome_unknown'", [JSON.stringify({ capacity_root: firstOperationId, capacity_failure: refusal }), context.now(), id, row.controller_id, row.generation]);
|
|
701
|
+
if (changed.changes !== 1)
|
|
702
|
+
throw new AppError("capacity_continuation_blocked", "Failed Turn owner changed before its refusal was retained", 1);
|
|
703
|
+
},
|
|
704
|
+
inspectOperation: id => inspect(["daemon", "operation", "--state-dir", session.state_dir, "--operation-id", id, "--json"], `${id}:capacity-observe`),
|
|
705
|
+
inspectSession: turnId => inspect(["session", "inspect", "--session-id", session.id, ...(turnId ? ["--turn-id", turnId] : []), ...sessionArguments(context, manifest, session)], `${firstOperationId}:capacity-inspect`),
|
|
706
|
+
assertCurrent: () => {
|
|
707
|
+
assertOwner();
|
|
708
|
+
},
|
|
709
|
+
decision: async () => {
|
|
710
|
+
if (kind === "child") {
|
|
711
|
+
const target = Object.entries(state.child_capacity ?? {}).find(([, retained]) => retained.operation_id === firstOperationId);
|
|
712
|
+
if (!target)
|
|
713
|
+
return "reconciliation_blocked";
|
|
714
|
+
const seen = await inspect(["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], `${firstOperationId}:child-followup-observe`);
|
|
715
|
+
const child = controllerNativeChildren(seen, session.id).find(item => item.session_id === target[0]);
|
|
716
|
+
if (child?.parent_session_id === session.id && (child.status === "running" || child.terminal_turn_id && child.terminal_turn_id !== target[1].turn_id))
|
|
717
|
+
return "completed";
|
|
718
|
+
}
|
|
719
|
+
const view = getFlowRunStatus(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id });
|
|
720
|
+
const next = view.continuation;
|
|
721
|
+
if (next.kind === "blocked")
|
|
722
|
+
return "reconciliation_blocked";
|
|
723
|
+
if (next.kind === "paused" && next.stage !== state.stage)
|
|
724
|
+
return "paused";
|
|
725
|
+
if (next.kind === "terminal" || next.stage !== state.stage)
|
|
726
|
+
return "completed";
|
|
727
|
+
if (hitl) {
|
|
728
|
+
const answerId = firstOperationId.slice(0, -`:prompt:${row.generation}`.length);
|
|
729
|
+
const answer = context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND status = 'accepted'", [answerId, row.controller_id]);
|
|
730
|
+
const accepted = answer && JSON.parse(answer.input_json);
|
|
731
|
+
const pause = view.index.stage_runs.find(item => item.stage === accepted?.stage)?.pause;
|
|
732
|
+
if (next.kind !== "paused")
|
|
733
|
+
return "continue";
|
|
734
|
+
if (!accepted || pause?.id !== accepted.pause_id || pause.work_id !== accepted.work_id || pause.answer_path)
|
|
735
|
+
return "paused";
|
|
736
|
+
}
|
|
737
|
+
else if (next.kind === "paused")
|
|
738
|
+
return "paused";
|
|
739
|
+
return next.kind === "wait" && !["native_children_unsettled", "work_receipts_pending", "qualified_capacity_required"].includes(next.reason) ? "reconciliation_blocked" : "continue";
|
|
740
|
+
},
|
|
741
|
+
retainIntent: intent => retainControllerCapacityIntent(context, row, state, intent, session.id, kind ?? "stage", assertOwner),
|
|
742
|
+
prompt: (id, value) => send(value, id),
|
|
743
|
+
...(resumeCommand ? { continuationText: `Continue the same accepted HITL task in this Session. First inspect current durable RUN and pause state. Only if the exact pause still awaits the accepted answer, run this previously issued command once:\n\n${resumeCommand}\n\nNever replay an already committed answer. Otherwise use only the current issued successor.` } : {}) });
|
|
744
|
+
if (!resumed)
|
|
745
|
+
throw error;
|
|
746
|
+
receipt = resumed;
|
|
747
|
+
}
|
|
748
|
+
checkOutcomes();
|
|
749
|
+
await childObserver(session)?.drain();
|
|
750
|
+
childObserver(session)?.completePostTurn(receipt);
|
|
751
|
+
const observationGap = childObserver(session)?.gap();
|
|
752
|
+
if (observationGap)
|
|
753
|
+
throw new AppError("operation_observation_lost", "In-flight child observation remained unavailable at the safe Turn boundary", 1, { observation_error_code: observationGap, provider_session_id: session.id, stage: state.stage });
|
|
754
|
+
if (fatal)
|
|
755
|
+
throw fatal;
|
|
756
|
+
}
|
|
757
|
+
catch (error) {
|
|
758
|
+
if (!fatal && error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale", "controller_owner_mismatch"].includes(error.code))
|
|
759
|
+
throw error;
|
|
760
|
+
await stopAfterFailure(error);
|
|
761
|
+
if (!capacityUncertain(error) && !(error instanceof AppError && error.code === "controller_lease_unconfirmed"))
|
|
762
|
+
cancellation.abort();
|
|
763
|
+
throw fatal ?? error;
|
|
764
|
+
}
|
|
765
|
+
state.last_receipt = receipt;
|
|
766
|
+
delete state.pending_capacity;
|
|
767
|
+
saveState(context, row, state);
|
|
768
|
+
return receipt;
|
|
769
|
+
};
|
|
770
|
+
const monitor = setInterval(() => {
|
|
771
|
+
if (fatal)
|
|
772
|
+
return;
|
|
773
|
+
try {
|
|
774
|
+
checkOutcomes();
|
|
775
|
+
if (promptPending && state.current_session !== null)
|
|
776
|
+
childObserver(state.sessions[state.current_session])?.tick();
|
|
777
|
+
}
|
|
778
|
+
catch (error) {
|
|
779
|
+
void stopAfterFailure(error);
|
|
780
|
+
}
|
|
781
|
+
}, 1000);
|
|
782
|
+
monitor.unref();
|
|
783
|
+
try {
|
|
784
|
+
for (;;) {
|
|
785
|
+
// Renewal touches the resource registry; never do it from a callback that
|
|
786
|
+
// may already own the RUN writer.
|
|
787
|
+
await renewLease();
|
|
788
|
+
assertOwner();
|
|
789
|
+
if (state.current_session !== null && !state.sessions[state.current_session].stopped)
|
|
790
|
+
bindInvocationScope(state.sessions[state.current_session]);
|
|
791
|
+
const run = requireRun(context, row.project_id, row.run_id);
|
|
792
|
+
const view = getFlowRunStatus(executionContext, { projectRoot: run.project_root, runId: run.id });
|
|
793
|
+
const next = view.continuation;
|
|
794
|
+
terminalFinalization = next.kind === "terminal";
|
|
795
|
+
if (state.pending_capacity && next.kind !== "terminal" && !["done", "failed", "cancelled"].includes(run.status)) {
|
|
796
|
+
const pending = state.pending_capacity;
|
|
797
|
+
const session = state.sessions.find(item => item.id === pending.session_id && !item.stopped);
|
|
798
|
+
const operation = context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [pending.root_operation_id, row.controller_id, row.generation]);
|
|
799
|
+
const dispatch = operation && JSON.parse(operation.input_json);
|
|
800
|
+
const args = dispatch?.args;
|
|
801
|
+
const file = args?.includes("--prompt-file") ? args[args.indexOf("--prompt-file") + 1] : undefined;
|
|
802
|
+
if (!session || pending.generation !== row.generation || !file || digestBytes(fs.readFileSync(file)) !== dispatch?.prompt_sha256)
|
|
803
|
+
throw new AppError("capacity_continuation_conflict", "Retained capacity resume reference has no exact owned prompt", 1);
|
|
804
|
+
await prompt(session, fs.readFileSync(file, "utf8"), pending.root_operation_id, pending.kind === "stage" ? undefined : pending.kind);
|
|
805
|
+
continue;
|
|
806
|
+
}
|
|
807
|
+
const completed = state.stage && view.index.stage_runs.find(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status));
|
|
808
|
+
if (state.current_session !== null && (completed || next.stage !== state.stage) && childObserver(state.sessions[state.current_session])?.failed().length)
|
|
809
|
+
throw new AppError("fanout_dispatch_fenced", "A retained child failure forbids completing or advancing its Stage", 1);
|
|
810
|
+
if (completed && manifest.request.stop_after === state.stage) {
|
|
811
|
+
// A bounded run is final evidence, not an entry fixture for its successor.
|
|
812
|
+
// Stop owned observers before capture; keep the controller lease/fence.
|
|
813
|
+
await settle();
|
|
814
|
+
if (manifest.request.capture_root)
|
|
815
|
+
await captureControllerBoundary(executionContext, row, manifest, completed, null, assertPhysicalOwner);
|
|
816
|
+
setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
|
|
817
|
+
setStatus("stop_target_reached", { stage: state.stage, attempt: completed.attempt ?? null });
|
|
818
|
+
return;
|
|
819
|
+
}
|
|
820
|
+
if (completed && manifest.request.capture_root && ["start_stage", "terminal"].includes(next.kind)) {
|
|
821
|
+
const key = JSON.stringify([completed.stage, completed.attempt, completed.started_at, completed.completed_at]);
|
|
822
|
+
if (key !== lastCapture) {
|
|
823
|
+
await captureControllerBoundary(executionContext, row, manifest, completed, next.kind === "start_stage" ? next.stage : null, assertPhysicalOwner);
|
|
824
|
+
lastCapture = key;
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
if (next.kind === "terminal") {
|
|
828
|
+
await settle();
|
|
829
|
+
setStatus(run.status === "cancelled" ? "cancelled" : "completed");
|
|
830
|
+
appendEvent(context, row.controller_id, "terminal", { reason: next.reason });
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
if (next.kind === "paused") {
|
|
834
|
+
setStatus("waiting_for_user");
|
|
835
|
+
const pause = view.index.stage_runs.find(stage => stage.stage === next.stage)?.pause;
|
|
836
|
+
if (!pause || state.current_session === null)
|
|
837
|
+
throw new AppError("controller_hitl_missing", "Paused RUN has no managed Session or HITL receipt", 1);
|
|
838
|
+
const answer = context.db.get("SELECT operation_id, input_json, status FROM run_controller_operations WHERE controller_id = ? AND status IN ('accepted','completed') AND json_extract(input_json, '$.action') = 'user_answer' AND json_extract(input_json, '$.pause_id') = ? ORDER BY created_at LIMIT 1", [row.controller_id, pause.id]);
|
|
839
|
+
if (!answer) {
|
|
840
|
+
await delay(1000);
|
|
841
|
+
continue;
|
|
842
|
+
}
|
|
843
|
+
if (answer.status === "completed")
|
|
844
|
+
throw new AppError("controller_answer_not_applied", "Completed answer operation still has its original HITL pause", 1, { pause_id: pause.id, operation_id: answer.operation_id, stage: next.stage });
|
|
845
|
+
const input = JSON.parse(answer.input_json);
|
|
846
|
+
if (digestBytes(fs.readFileSync(input.file)) !== input.sha256)
|
|
847
|
+
throw new AppError("context_checksum_mismatch", "Accepted HITL answer bytes changed", 1);
|
|
848
|
+
await renewLease();
|
|
849
|
+
assertOwner();
|
|
850
|
+
// The accepted answer is logical intent; only its generation-bound
|
|
851
|
+
// adapter operation represents a physical dispatch that stop must drain.
|
|
852
|
+
const command = managedLifecycleCommand(executionContext, `${flowCommand(executionContext)} stage resume ${run.id} --stage ${next.stage} --work ${input.work_id} --project-root ${quote([run.project_root])} --answer-file ${quote([input.file])} --json`, { replaceUnobservedAnswer: true });
|
|
853
|
+
const receipt = await prompt(state.sessions[state.current_session], controllerStageResumePrompt(pause.id, command), `${answer.operation_id}:prompt:${row.generation}`, "hitl");
|
|
854
|
+
const afterAnswer = getFlowRunStatus(executionContext, { projectRoot: run.project_root, runId: run.id });
|
|
855
|
+
if (afterAnswer.index.stage_runs.some(stage => stage.status === "paused" && stage.pause?.id === pause.id)) {
|
|
856
|
+
throw new AppError("controller_answer_not_applied", "The answer Turn completed without resuming the assigned HITL pause", 1, { pause_id: pause.id, operation_id: answer.operation_id, stage: next.stage, effect: "no_progress" });
|
|
857
|
+
}
|
|
858
|
+
context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify({ delivered: true, session_id: adapterSessionId(receipt) }), context.now(), answer.operation_id]);
|
|
859
|
+
state.last_continuation = null;
|
|
860
|
+
saveState(context, row, state);
|
|
861
|
+
setStatus("running");
|
|
862
|
+
continue;
|
|
863
|
+
}
|
|
864
|
+
if (next.kind === "blocked" || next.kind === "wait" || !next.stage)
|
|
865
|
+
throw new AppError("controller_continuation_blocked", "RUN has no legal managed continuation", 1, { continuation: next });
|
|
866
|
+
if (next.kind === "start_stage") {
|
|
867
|
+
// Only context slices can be added; profiles and launch inputs stay frozen.
|
|
868
|
+
manifest.contexts = JSON.parse(requireController(context, row.controller_id).manifest_json).contexts;
|
|
869
|
+
const stageContext = manifest.contexts[next.stage];
|
|
870
|
+
if (!stageContext) {
|
|
871
|
+
if (requireController(context, row.controller_id).status !== "waiting_for_context")
|
|
872
|
+
appendEvent(context, row.controller_id, "context_required", { stage: next.stage, attempt: next.attempt ?? null });
|
|
873
|
+
setStatus("waiting_for_context");
|
|
874
|
+
await delay(1000);
|
|
875
|
+
continue;
|
|
876
|
+
}
|
|
877
|
+
loadExternalStageContext({ file: stageContext.file, expectedSha256: stageContext.sha256, stage: next.stage });
|
|
878
|
+
const continuation = JSON.stringify(["start_stage", next.stage, next.attempt ?? null]);
|
|
879
|
+
retainControllerContinuation(state, continuation, { code: "stage_entry_nonprogressing", message: "The coordinator returned without starting the assigned Stage", details: { stage: next.stage, attempt: next.attempt ?? null, last_receipt: state.last_receipt ?? null } });
|
|
880
|
+
const transition = routing.transition(next.stage, state.stage ?? undefined, run);
|
|
881
|
+
if (routing.kind === "sdlc" && next.stage === "merge" && routing.mergeMode === "server" && !state.merge_claim) {
|
|
882
|
+
const request = context.db.get("SELECT merge_request_id, status FROM merge_requests WHERE project_id = ? AND run_id = ? ORDER BY created_at DESC, merge_request_id DESC LIMIT 1", [row.project_id, row.run_id]);
|
|
883
|
+
if (!request || request.status !== "queued")
|
|
884
|
+
throw new AppError("controller_attach_required", "Managed MERGE needs an unclaimed queued request; reconcile an existing owner", 1, { request });
|
|
885
|
+
const claim = claimQueuedMergeRequest(context, { requestId: request.merge_request_id, ownerId: row.controller_id, expectedGeneration: row.generation });
|
|
886
|
+
if (!claim) {
|
|
887
|
+
setStatus("waiting_for_merge");
|
|
888
|
+
await delay(1000);
|
|
889
|
+
continue;
|
|
890
|
+
}
|
|
891
|
+
state.merge_claim = { request_id: claim.request.merge_request_id, token: claim.token, target_workspace: claim.request.target_workspace, work_id: claim.request.executor_work_id };
|
|
892
|
+
saveState(context, row, state);
|
|
893
|
+
appendEvent(context, row.controller_id, "merge_claimed", { request_id: claim.request.merge_request_id });
|
|
894
|
+
await renewLease();
|
|
895
|
+
}
|
|
896
|
+
const stageCwd = routing.cwd(next.stage, state.merge_claim?.target_workspace, run);
|
|
897
|
+
const routeBinding = (current) => {
|
|
898
|
+
const index = JSON.parse(current.index_json);
|
|
899
|
+
return JSON.stringify([current.project_root, current.workspace_root, index.workspace_route ?? null, state.merge_claim ?? null]);
|
|
900
|
+
};
|
|
901
|
+
const preparedRoute = routeBinding(run);
|
|
902
|
+
const assertRoute = async () => {
|
|
903
|
+
assertOwner();
|
|
904
|
+
if (routeBinding(requireRun(context, row.project_id, row.run_id)) !== preparedRoute) {
|
|
905
|
+
const current = state.current_session === null ? null : state.sessions[state.current_session];
|
|
906
|
+
if (current && !current.stopped)
|
|
907
|
+
await stopSession(current);
|
|
908
|
+
throw new AppError("workspace_route_changed", "RUN workspace changed during native Session preparation", 1, { effect: "no_effect", stage: next.stage });
|
|
909
|
+
}
|
|
910
|
+
};
|
|
911
|
+
await assertRoute();
|
|
912
|
+
let session = state.current_session === null ? null : state.sessions[state.current_session];
|
|
913
|
+
if (session && routing.kind === "sdlc" && next.stage === "merge" && routing.mergeMode === "same_session" && path.resolve(session.cwd) !== stageCwd) {
|
|
914
|
+
throw new AppError("merge_session_workspace_conflict", "Same-session MERGE cannot change the native Session working directory", 1, { session_cwd: session.cwd, target_cwd: stageCwd });
|
|
915
|
+
}
|
|
916
|
+
if (!session || transition.session_mode === "new_session" || !sameSessionProfile(session.profile, transition.coordinator) || path.resolve(session.cwd) !== stageCwd) {
|
|
917
|
+
if (session)
|
|
918
|
+
await stopSession(session);
|
|
919
|
+
await renewLease();
|
|
920
|
+
assertOwner();
|
|
921
|
+
const directory = path.join(row.state_dir, `session-${state.sessions.length + 1}`);
|
|
922
|
+
fs.mkdirSync(directory, { mode: 0o700 });
|
|
923
|
+
const owner = { schema_id: "dd-flow/runtime-owner@1", owner_id: row.controller_id, role: "subject", project_id: row.project_id, run_id: row.run_id, operation_id: `${row.controller_id}:session:${state.sessions.length + 1}`, state_dir: directory, dd_flow_home: context.ddFlowHome, dd_flow_bin: manifest.flow_executable, resource_home: manifest.resource_home, budget: manifest.budget };
|
|
924
|
+
if (next.stage === "merge" && state.merge_claim) {
|
|
925
|
+
owner.role = "merge";
|
|
926
|
+
owner.work_id = state.merge_claim.work_id;
|
|
927
|
+
}
|
|
928
|
+
session = { id: "", profile: transition.coordinator, state_dir: directory, cwd: stageCwd, journal: path.join(directory, "adapter.events.jsonl"), owner };
|
|
929
|
+
if (runtimeHarness(session.profile) === "codex-desktop") {
|
|
930
|
+
session.codex_home = path.join(directory, "codex-home");
|
|
931
|
+
initCodexHome(context, { projectRoot: run.project_root, profile: `controller-${digest(directory).slice(0, 16)}`, targetHome: session.codex_home, ...(context.env.CODEX_HOME ? { sourceHome: context.env.CODEX_HOME } : {}) });
|
|
932
|
+
}
|
|
933
|
+
state.sessions.push(session);
|
|
934
|
+
state.current_session = state.sessions.length - 1;
|
|
935
|
+
saveState(context, row, state);
|
|
936
|
+
await call(session, ["daemon", "start", ...sessionArguments(context, manifest, session)]);
|
|
937
|
+
const created = await call(session, ["session", "create", ...sessionArguments(context, manifest, session)]);
|
|
938
|
+
const sessionId = adapterSessionId(created);
|
|
939
|
+
if (!sessionId)
|
|
940
|
+
throw new AppError("harness_adapter_invalid", "Managed adapter did not return its Session id", 1);
|
|
941
|
+
session.id = sessionId;
|
|
942
|
+
saveState(context, row, state);
|
|
943
|
+
bindInvocationScope(session);
|
|
944
|
+
if (next.stage === "merge" && state.merge_claim)
|
|
945
|
+
context.db.run("UPDATE merge_requests SET adapter_receipt_json = ?, updated_at = ? WHERE merge_request_id = ? AND dispatch_owner = ? AND dispatch_lease_token = ?", [JSON.stringify({ controller_id: row.controller_id, profile: session.profile, session_id: session.id, state_dir: session.state_dir, create: created }), context.now(), state.merge_claim.request_id, row.controller_id, state.merge_claim.token]);
|
|
946
|
+
appendEvent(context, row.controller_id, "session_created", { stage: next.stage, session_id: sessionId, harness: runtimeHarness(session.profile), profile: session.profile });
|
|
947
|
+
}
|
|
948
|
+
state.native_baseline = controllerNativeChildren(state.last_receipt ?? {}, session.id).map(child => child.session_id);
|
|
949
|
+
state.stage = next.stage;
|
|
950
|
+
state.child_capacity = {};
|
|
951
|
+
saveState(context, row, state);
|
|
952
|
+
setStatus("running");
|
|
953
|
+
if (routing.kind === "sdlc" && (next.stage === "code" || next.stage === "code-review")) {
|
|
954
|
+
// A CODE_REVIEW fork has source files but intentionally no installed
|
|
955
|
+
// dependencies. Its repair Work executes checks just like CODE Work.
|
|
956
|
+
const readiness = await prepareVnextCodeWorkspace(executionContext, { projectRoot: run.project_root, runId: run.id, ownerId: row.controller_id, stage: next.stage });
|
|
957
|
+
await renewLease();
|
|
958
|
+
assertOwner();
|
|
959
|
+
appendEvent(context, row.controller_id, "code_workspace_prepared", { stage: next.stage, readiness_status: readiness.status ?? null });
|
|
960
|
+
}
|
|
961
|
+
const responseFile = path.join(row.state_dir, `${next.stage}-${next.attempt ?? 1}.stage-start-response.json`);
|
|
962
|
+
const command = managedLifecycleCommand(executionContext, `${flowCommand(executionContext)} stage start ${run.id} --stage ${next.stage} --project-root ${JSON.stringify(run.project_root)} --context-file ${JSON.stringify(stageContext.file)} --context-sha256 ${stageContext.sha256} --require-session-binding --response-file ${JSON.stringify(responseFile)} --json --progress-jsonl`);
|
|
963
|
+
await assertRoute();
|
|
964
|
+
appendEvent(context, row.controller_id, "stage_entered", { stage: next.stage, session_id: session.id, context_sha256: stageContext.sha256 });
|
|
965
|
+
await prompt(session, controllerStageEntryPrompt(next.stage, command));
|
|
966
|
+
continue;
|
|
967
|
+
}
|
|
968
|
+
if (state.current_session === null)
|
|
969
|
+
throw new AppError("controller_attach_required", "An unfinished Stage has no managed coordinator", 1);
|
|
970
|
+
const session = state.sessions[state.current_session];
|
|
971
|
+
const childHarness = runtimeHarness(session.profile);
|
|
972
|
+
const baseline = new Set(state.native_baseline ?? []);
|
|
973
|
+
let children = controllerNativeChildren(state.last_receipt ?? {}, session.id).filter(child => !baseline.has(child.session_id));
|
|
974
|
+
childObserver(session)?.observeSettledChildren(children);
|
|
975
|
+
const retainedFailures = childObserver(session)?.failed() ?? [];
|
|
976
|
+
children = controllerNativeChildren({ descendants: [...children, ...retainedFailures] }, session.id);
|
|
977
|
+
const closedChildEpisodes = closeSuccessfulChildCapacity(state, children, session.id);
|
|
978
|
+
if (closedChildEpisodes.length)
|
|
979
|
+
await prepareMutation(() => {
|
|
980
|
+
saveState(context, row, state);
|
|
981
|
+
appendEvent(context, row.controller_id, "capacity_child_episodes_completed", { child_session_ids: closedChildEpisodes });
|
|
982
|
+
});
|
|
983
|
+
if (children.some(child => child.status === "failed" && renderNativeChildFollowupInstruction(childHarness, child.session_id))) {
|
|
984
|
+
const inspected = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
985
|
+
state.last_receipt = inspected;
|
|
986
|
+
saveState(context, row, state);
|
|
987
|
+
children = controllerNativeChildren(inspected, session.id).filter(child => !baseline.has(child.session_id));
|
|
988
|
+
const failed = children.filter(child => child.status === "failed");
|
|
989
|
+
// Freeze each distinct child refusal on its first exact observation,
|
|
990
|
+
// even while a healthy sibling still owns an active native Turn.
|
|
991
|
+
for (const failedChild of failed.filter(item => eligibleProviderChildOverload(item, session.id))) {
|
|
992
|
+
const linked = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: childHarness, session_id: failedChild.session_id })]);
|
|
993
|
+
if (linked.length > 1 || linked.some(link => link.status !== "running"))
|
|
994
|
+
continue;
|
|
995
|
+
const retained = state.child_capacity?.[failedChild.session_id];
|
|
996
|
+
const refusals = [...(retained?.refusals ?? [])];
|
|
997
|
+
if (refusals.some(item => item.turn_id === failedChild.terminal_turn_id))
|
|
998
|
+
continue;
|
|
999
|
+
const nativeFailure = providerChildFailure(failedChild);
|
|
1000
|
+
refusals.push(refusalObservation(nativeFailure, retained?.attempts ?? 0, failedChild.terminal_observed_at ?? context.now()));
|
|
1001
|
+
state.child_capacity ??= {};
|
|
1002
|
+
state.child_capacity[failedChild.session_id] = { ...retained, turn_id: retained?.turn_id ?? failedChild.terminal_turn_id, attempts: retained?.attempts ?? 0, policy: CAPACITY_POLICY, refusals };
|
|
1003
|
+
await prepareMutation(() => { saveState(context, row, state); appendEvent(context, row.controller_id, "capacity_child_refusal", { child_session_id: failedChild.session_id, refusal: refusals.at(-1) }); });
|
|
1004
|
+
const exhausted = overloadExhausted(refusals);
|
|
1005
|
+
if (exhausted)
|
|
1006
|
+
throw new AppError("provider_unavailable", "Native child exhausted its overload continuation attempts", 1, { ...exhausted, reason: "overload_retries_exhausted", child_session_id: failedChild.session_id, primary_error: nativeFailure });
|
|
1007
|
+
}
|
|
1008
|
+
// Continue one existing child at a time; another failed child does not
|
|
1009
|
+
// create a second fanout wave or share this child's refusal counter.
|
|
1010
|
+
const child = children.every(item => item.status === "failed" || item.status === "completed") ? failed.find(item => renderNativeChildFollowupInstruction(childHarness, item.session_id) && eligibleProviderChildOverload(item, session.id)) : undefined;
|
|
1011
|
+
const turnId = child?.terminal_turn_id;
|
|
1012
|
+
const native = child && turnId ? providerChildFailure(child) : null;
|
|
1013
|
+
const prior = child && state.child_capacity?.[child.session_id];
|
|
1014
|
+
const links = child ? context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: childHarness, session_id: child.session_id })]) : [];
|
|
1015
|
+
if (child && turnId && native && terminalProviderOverload(native, child.session_id, turnId)
|
|
1016
|
+
&& nativeItemsSettled(child.native_turn_items)
|
|
1017
|
+
&& inspected.settled === true && child.parent_session_id === session.id && links.length <= 1
|
|
1018
|
+
&& (!links.length || links[0].status === "running")) {
|
|
1019
|
+
const prepared = prior?.turn_id === turnId && prior.dispatched === false && prior.operation_id && prior.not_before ? prior : null;
|
|
1020
|
+
if (prior?.turn_id === turnId && prior.dispatched === true && !prepared)
|
|
1021
|
+
throw new AppError("capacity_child_followup_unconfirmed", "Parent continuation did not start a new native child Turn", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
1022
|
+
const attempt = prepared ? prepared.attempts : (prior?.attempts ?? 0) + 1;
|
|
1023
|
+
if (!Number.isSafeInteger(attempt) || prior && prior.policy !== CAPACITY_POLICY)
|
|
1024
|
+
throw new AppError("capacity_continuation_conflict", "Native child continuation policy or ordinal is incompatible", 1);
|
|
1025
|
+
const refusals = [...(prior?.refusals ?? [])];
|
|
1026
|
+
if (!refusals.some(item => item.turn_id === turnId)) {
|
|
1027
|
+
const refusal = refusalObservation(native, attempt - 1, child.terminal_observed_at ?? context.now());
|
|
1028
|
+
refusals.push(refusal);
|
|
1029
|
+
state.child_capacity ??= {};
|
|
1030
|
+
state.child_capacity[child.session_id] = { ...prior, turn_id: prior?.turn_id ?? turnId, attempts: prior?.attempts ?? 0, policy: CAPACITY_POLICY, refusals };
|
|
1031
|
+
saveState(context, row, state);
|
|
1032
|
+
}
|
|
1033
|
+
const exhausted = overloadExhausted(refusals);
|
|
1034
|
+
if (exhausted)
|
|
1035
|
+
throw new AppError("provider_unavailable", "Native child exhausted its overload continuation attempts", 1, { ...exhausted, reason: "overload_retries_exhausted", child_session_id: child.session_id, primary_error: native });
|
|
1036
|
+
let wait;
|
|
1037
|
+
try {
|
|
1038
|
+
wait = prepared ? Math.max(0, Date.parse(prepared.not_before) - Date.now()) : capacityDelay(native, attempt);
|
|
1039
|
+
}
|
|
1040
|
+
catch (error) {
|
|
1041
|
+
const failure = error;
|
|
1042
|
+
throw new AppError(failure.code ?? "capacity_retry_metadata_invalid", failure.message ?? "Invalid child retry delay", 1, { ...failure.details, child_session_id: child.session_id, root_operation_id: prior?.root_operation_id ?? null, attempts: refusals.length, primary_error: native });
|
|
1043
|
+
}
|
|
1044
|
+
if (!Number.isFinite(wait) || Date.now() + wait > 8.64e15)
|
|
1045
|
+
throw new AppError("capacity_retry_metadata_invalid", "Native child retry time is invalid", 1, { child_session_id: child.session_id, delay_ms: wait });
|
|
1046
|
+
const rootOperationId = prior?.root_operation_id ?? `${row.controller_id}:child:${digest([child.session_id, turnId]).slice(0, 16)}`;
|
|
1047
|
+
const operationId = prepared?.operation_id ?? `${rootOperationId}:followup:${attempt}`;
|
|
1048
|
+
state.child_capacity ??= {};
|
|
1049
|
+
state.child_capacity[child.session_id] = { turn_id: turnId, attempts: attempt, operation_id: operationId, root_operation_id: rootOperationId, policy: CAPACITY_POLICY, refusals, not_before: prepared?.not_before ?? new Date(Date.now() + wait).toISOString(), dispatched: false };
|
|
1050
|
+
await prepareMutation(() => { saveState(context, row, state); appendEvent(context, row.controller_id, "capacity_child_followup", { child_session_id: child.session_id, turn_id: turnId, attempt, operation_id: operationId }); });
|
|
1051
|
+
const until = performance.now() + wait;
|
|
1052
|
+
while (performance.now() < until) {
|
|
1053
|
+
await delay(Math.min(250, Math.max(0, until - performance.now())), undefined, { signal: cancellation.signal });
|
|
1054
|
+
await renewLease();
|
|
1055
|
+
assertOwner();
|
|
1056
|
+
}
|
|
1057
|
+
await renewLease();
|
|
1058
|
+
assertOwner();
|
|
1059
|
+
const current = getFlowRunStatus(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id });
|
|
1060
|
+
if (current.continuation.stage !== state.stage || ["terminal", "paused"].includes(current.continuation.kind))
|
|
1061
|
+
continue;
|
|
1062
|
+
if (["blocked", "wait"].includes(current.continuation.kind))
|
|
1063
|
+
throw new AppError("capacity_child_followup_blocked", "RUN changed during native child capacity wait", 1, { child_session_id: child.session_id });
|
|
1064
|
+
const fresh = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
1065
|
+
const still = controllerNativeChildren(fresh, session.id).find(item => item.session_id === child.session_id);
|
|
1066
|
+
const currentLinks = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: childHarness, session_id: child.session_id })]);
|
|
1067
|
+
if (currentLinks.length === 1 && ["completed", "cancelled", "paused"].includes(currentLinks[0].status)) {
|
|
1068
|
+
state.last_receipt = fresh;
|
|
1069
|
+
saveState(context, row, state);
|
|
1070
|
+
continue;
|
|
1071
|
+
}
|
|
1072
|
+
if (fresh.settled !== true || !still || still.parent_session_id !== session.id || still.status !== "failed" || still.terminal_turn_id !== turnId
|
|
1073
|
+
|| !nativeItemsSettled(still.native_turn_items) || currentLinks.length > 1
|
|
1074
|
+
|| currentLinks.some(link => link.status !== "running"))
|
|
1075
|
+
throw new AppError("capacity_child_followup_blocked", "Native child or Work changed during capacity wait", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
1076
|
+
await renewLease();
|
|
1077
|
+
assertOwner();
|
|
1078
|
+
const text = renderNativeChildFollowupInstruction(childHarness, child.session_id);
|
|
1079
|
+
await prompt(session, text, operationId, "child");
|
|
1080
|
+
state.child_capacity[child.session_id].dispatched = true;
|
|
1081
|
+
saveState(context, row, state);
|
|
1082
|
+
const after = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
1083
|
+
state.last_receipt = after;
|
|
1084
|
+
saveState(context, row, state);
|
|
1085
|
+
const observed = controllerNativeChildren(after, session.id).find(item => item.session_id === child.session_id);
|
|
1086
|
+
if (!observed || observed.status === "failed" && observed.terminal_turn_id === turnId)
|
|
1087
|
+
throw new AppError("capacity_child_followup_unconfirmed", "Native child did not advance to a new Turn after parent followup", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
1088
|
+
continue;
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
const fanout = await nextControllerFanout(executionContext, { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles, dispatchFence: retainedFailures.length > 0 });
|
|
1092
|
+
assertOwner();
|
|
1093
|
+
if (fanout.kind === "none") {
|
|
1094
|
+
// An ACK-only recovery Turn may leave the original root Work unfinished.
|
|
1095
|
+
// Continue that retained packet once; never restart its Stage or repeat
|
|
1096
|
+
// an unknown productive dispatch merely because the observer reattached.
|
|
1097
|
+
const recovery = context.db.get(`SELECT o.operation_id,b.packet_json,b.packet_sha256,b.work_id
|
|
1098
|
+
FROM run_controller_operations o JOIN run_recovery_bindings b ON b.recovery_id=json_extract(o.receipt_json,'$.recovery_id')
|
|
1099
|
+
JOIN run_recovery_guards g ON g.recovery_id=b.recovery_id
|
|
1100
|
+
WHERE o.controller_id=? AND o.generation=? AND o.status='completed' AND json_extract(o.input_json,'$.action')='control_resume'
|
|
1101
|
+
AND b.status='accepted' AND g.status='recovered' AND g.project_id=? AND g.run_id=?
|
|
1102
|
+
AND b.provider_session_id=? AND b.daemon_id=json_extract(o.receipt_json,'$.daemon_id') ORDER BY o.created_at DESC LIMIT 1`, [row.controller_id, row.generation, row.project_id, row.run_id, session.id]);
|
|
1103
|
+
if (recovery && digestBytes(recovery.packet_json) === recovery.packet_sha256) {
|
|
1104
|
+
const packet = JSON.parse(recovery.packet_json);
|
|
1105
|
+
const operationId = `${recovery.operation_id}:unfinished-root`;
|
|
1106
|
+
const work = context.db.get('SELECT status FROM works WHERE work_id=? AND project_id=? AND run_id=? AND parent_work_id IS NULL', [recovery.work_id, row.project_id, row.run_id]);
|
|
1107
|
+
if (recoveryPacketMatchesStage(packet, { runId: row.run_id, generation: row.generation, stage: next.stage, workId: recovery.work_id }) && work?.status === 'running'
|
|
1108
|
+
&& !context.db.get('SELECT 1 FROM run_controller_operations WHERE operation_id=?', [operationId])) {
|
|
1109
|
+
await prompt(session, `Continue the original unfinished root Work from this accepted recovery packet. Inspect retained receipts and partial effects first. Do not restart the Stage, repeat observed lifecycle operations, create another Work or start a successor Stage. Preserve the packet write scope and result schema; stop at its original boundary.\n\nAccepted recovery packet (JSON data):\n${recovery.packet_json}`, operationId);
|
|
1110
|
+
continue;
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
assertNoCorrelatedRejection(context, { projectId: row.project_id, runId: row.run_id, generation: row.generation, stage: next.stage, turn: state.turns });
|
|
1114
|
+
const cause = currentLifecycleFailure(executionContext, { runId: run.id, generation: row.generation, stage: next.stage, turn: state.turns, projectId: row.project_id, projectRoot: run.project_root });
|
|
1115
|
+
if (cause)
|
|
1116
|
+
throw new AppError(cause.code, cause.message, 1, { ...cause.details, ...(cause.invocation_id ? { invocation_id: cause.invocation_id } : {}), ...(cause.hook_event_id ? { hook_event_id: cause.hook_event_id } : {}), controller_wrapper: "incomplete_subject_turn", stage: next.stage });
|
|
1117
|
+
throw new AppError("incomplete_subject_turn", "Managed Turn ended without a completed Stage or declared continuation", 1, { stage: next.stage });
|
|
1118
|
+
}
|
|
1119
|
+
if (fanout.kind === "wait") {
|
|
1120
|
+
setStatus(fanout.reason === "qualified_capacity_required" ? "waiting_for_capacity" : "waiting_for_children");
|
|
1121
|
+
await delay(1000);
|
|
1122
|
+
state.last_receipt = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
1123
|
+
saveState(context, row, state);
|
|
1124
|
+
continue;
|
|
1125
|
+
}
|
|
1126
|
+
setStatus("running");
|
|
1127
|
+
if (fanout.kind === "external") {
|
|
1128
|
+
const results = await Promise.allSettled(fanout.works.map(async (work) => {
|
|
1129
|
+
try {
|
|
1130
|
+
await renewLease();
|
|
1131
|
+
assertOwner();
|
|
1132
|
+
const receipt = await launchExternalWork(executionContext, { workId: work.work_id, stage: next.stage, signal: cancellation.signal });
|
|
1133
|
+
if (fatal)
|
|
1134
|
+
throw fatal;
|
|
1135
|
+
appendEvent(context, row.controller_id, "external_work_observed", { work_id: work.work_id, receipt });
|
|
1136
|
+
const launch = context.db.get("SELECT state_dir, profile_json FROM work_launches WHERE work_id = ?", [work.work_id]);
|
|
1137
|
+
const status = context.db.get("SELECT status FROM works WHERE work_id = ?", [work.work_id])?.status;
|
|
1138
|
+
if (!launch || !["completed", "failed", "cancelled"].includes(status ?? ""))
|
|
1139
|
+
throw new AppError("external_work_unsettled", "External launch returned without its terminal Work receipt; observe it rather than relaunching", 1, { work_id: work.work_id, status });
|
|
1140
|
+
const resolved = JSON.parse(launch.profile_json);
|
|
1141
|
+
const executable = manifest.adapters[runtimeHarness(resolved.worker)].executable;
|
|
1142
|
+
const stopped = await runHarnessAdapter({ executable, args: ["daemon", "stop", "--state-dir", launch.state_dir, "--json"], env: controllerEnvironment(context, manifest), operationId: `${row.controller_id}:${work.work_id}:settle` });
|
|
1143
|
+
await confirmStopped(launch.state_dir, stopped);
|
|
1144
|
+
appendEvent(context, row.controller_id, "external_work_settled", { work_id: work.work_id, receipt: stopped });
|
|
1145
|
+
}
|
|
1146
|
+
catch (error) {
|
|
1147
|
+
await stopAfterFailure(error);
|
|
1148
|
+
throw error;
|
|
1149
|
+
}
|
|
1150
|
+
}));
|
|
1151
|
+
const rejected = results.find((result) => result.status === "rejected");
|
|
1152
|
+
if (rejected)
|
|
1153
|
+
throw fatal ?? rejected.reason;
|
|
1154
|
+
state.last_continuation = null;
|
|
1155
|
+
saveState(context, row, state);
|
|
1156
|
+
continue;
|
|
1157
|
+
}
|
|
1158
|
+
if (state.last_continuation === fanout.fingerprint)
|
|
1159
|
+
assertNoCorrelatedRejection(context, { projectId: row.project_id, runId: row.run_id, generation: row.generation, stage: next.stage, turn: state.turns });
|
|
1160
|
+
retainControllerContinuation(state, fanout.fingerprint, { code: "fanout_stage_nonprogressing", message: "The coordinator repeated an unchanged Work-graph continuation", details: { stage: next.stage, action: fanout.action, before: state.last_continuation, after: fanout.fingerprint, controller_turn: state.turns } });
|
|
1161
|
+
saveState(context, row, state);
|
|
1162
|
+
await prompt(session, fanout.prompt);
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
catch (error) {
|
|
1166
|
+
if (!fatal && error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale", "controller_owner_mismatch"].includes(error.code))
|
|
1167
|
+
throw error;
|
|
1168
|
+
let cause = fatal ?? error;
|
|
1169
|
+
if (!fatal && terminalFinalization) {
|
|
1170
|
+
const record = errorRecord(error);
|
|
1171
|
+
const provenance = { phase: "terminal_finalization", project_root: manifest.request.project_root, run_id: row.run_id, controller_id: row.controller_id };
|
|
1172
|
+
if (error instanceof AppError)
|
|
1173
|
+
Object.assign(error.details, provenance);
|
|
1174
|
+
else
|
|
1175
|
+
cause = new AppError(String(record.code), String(record.message), 1, { ...record.details, ...provenance, cause: record });
|
|
1176
|
+
}
|
|
1177
|
+
await stopAfterFailure(cause);
|
|
1178
|
+
throw fatal ?? cause;
|
|
1179
|
+
}
|
|
1180
|
+
finally {
|
|
1181
|
+
clearInterval(monitor);
|
|
1182
|
+
const drainedObservations = [...childObservers.values()].map(retained => retained.close());
|
|
1183
|
+
observationCancellation.abort();
|
|
1184
|
+
await Promise.allSettled(drainedObservations);
|
|
1185
|
+
if (stopping)
|
|
1186
|
+
await stopping;
|
|
1187
|
+
observer.db.close?.();
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
function readContexts(file, expected) {
|
|
1191
|
+
const { bytes } = prepareFileBytes(file, "Managed context package");
|
|
1192
|
+
if (digestBytes(bytes) !== expected)
|
|
1193
|
+
throw new AppError("context_checksum_mismatch", "Managed context package SHA-256 differs", 2);
|
|
1194
|
+
const value = prepareJsonText(bytes.toString("utf8"), { file, label: "Managed context package" }).value;
|
|
1195
|
+
if (!value || value.schema_id !== "dd-flow/managed-context@1" || !value.stages || typeof value.stages !== "object" || Array.isArray(value.stages) || Object.keys(value).some(key => !["schema_id", "stages"].includes(key)))
|
|
1196
|
+
throw new AppError("validation", "Expected dd-flow/managed-context@1 with stage file/hash references", 2);
|
|
1197
|
+
const result = {};
|
|
1198
|
+
for (const [stage, source] of Object.entries(value.stages)) {
|
|
1199
|
+
if (!/^[a-z][a-z-]*$/.test(stage) || !source || typeof source !== "object" || Object.keys(source).some(key => !["file", "sha256"].includes(key)) || typeof source.file !== "string" || !path.isAbsolute(source.file) || typeof source.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(source.sha256))
|
|
1200
|
+
throw new AppError("validation", "Invalid managed stage context reference", 2);
|
|
1201
|
+
const { bytes: content } = loadExternalStageContext({ file: source.file, expectedSha256: source.sha256, stage });
|
|
1202
|
+
result[stage] = { bytes: content, sha256: source.sha256 };
|
|
1203
|
+
}
|
|
1204
|
+
return result;
|
|
1205
|
+
}
|