clinkx 0.3.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +119 -0
- package/LICENSE +10 -0
- package/README.md +154 -26
- package/clinkx-workflows/dist/allowed-roots.d.ts +1 -0
- package/clinkx-workflows/dist/allowed-roots.js +5 -0
- package/clinkx-workflows/dist/allowed-roots.js.map +1 -0
- package/clinkx-workflows/dist/artifacts.d.ts +20 -1
- package/clinkx-workflows/dist/artifacts.js +38 -9
- package/clinkx-workflows/dist/artifacts.js.map +1 -1
- package/clinkx-workflows/dist/backend.d.ts +4 -0
- package/clinkx-workflows/dist/chairman-knobs.d.ts +24 -0
- package/clinkx-workflows/dist/chairman-knobs.js +27 -0
- package/clinkx-workflows/dist/chairman-knobs.js.map +1 -0
- package/clinkx-workflows/dist/citation-check.d.ts +40 -0
- package/clinkx-workflows/dist/citation-check.js +348 -0
- package/clinkx-workflows/dist/citation-check.js.map +1 -0
- package/clinkx-workflows/dist/clink-client.d.ts +12 -1
- package/clinkx-workflows/dist/clink-client.js +56 -5
- package/clinkx-workflows/dist/clink-client.js.map +1 -1
- package/clinkx-workflows/dist/config.d.ts +173 -16
- package/clinkx-workflows/dist/config.js +417 -26
- package/clinkx-workflows/dist/config.js.map +1 -1
- package/clinkx-workflows/dist/contained-file.d.ts +172 -0
- package/clinkx-workflows/dist/contained-file.js +483 -0
- package/clinkx-workflows/dist/contained-file.js.map +1 -0
- package/clinkx-workflows/dist/context-builder-lifecycle-contract.d.ts +416 -0
- package/clinkx-workflows/dist/context-builder-lifecycle-contract.js +328 -0
- package/clinkx-workflows/dist/context-builder-lifecycle-contract.js.map +1 -0
- package/clinkx-workflows/dist/context-builder-lifecycle-router.d.ts +24 -0
- package/clinkx-workflows/dist/context-builder-lifecycle-router.js +39 -0
- package/clinkx-workflows/dist/context-builder-lifecycle-router.js.map +1 -0
- package/clinkx-workflows/dist/context-cache-key.d.ts +53 -0
- package/clinkx-workflows/dist/context-cache-key.js +91 -0
- package/clinkx-workflows/dist/context-cache-key.js.map +1 -0
- package/clinkx-workflows/dist/context-mode-resolver.d.ts +27 -0
- package/clinkx-workflows/dist/context-mode-resolver.js +126 -0
- package/clinkx-workflows/dist/context-mode-resolver.js.map +1 -0
- package/clinkx-workflows/dist/context-preflight.d.ts +189 -0
- package/clinkx-workflows/dist/context-preflight.js +91 -0
- package/clinkx-workflows/dist/context-preflight.js.map +1 -0
- package/clinkx-workflows/dist/council-execution-policy.d.ts +68 -0
- package/clinkx-workflows/dist/council-execution-policy.js +110 -0
- package/clinkx-workflows/dist/council-execution-policy.js.map +1 -0
- package/clinkx-workflows/dist/council-expander.d.ts +10 -9
- package/clinkx-workflows/dist/council-expander.js +20 -10
- package/clinkx-workflows/dist/council-expander.js.map +1 -1
- package/clinkx-workflows/dist/dashboard/activity-contract.d.ts +5718 -0
- package/clinkx-workflows/dist/dashboard/activity-contract.js +157 -0
- package/clinkx-workflows/dist/dashboard/activity-contract.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/activity-lease.d.ts +25 -0
- package/clinkx-workflows/dist/dashboard/activity-lease.js +34 -0
- package/clinkx-workflows/dist/dashboard/activity-lease.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/activity-reader.d.ts +90 -0
- package/clinkx-workflows/dist/dashboard/activity-reader.js +493 -0
- package/clinkx-workflows/dist/dashboard/activity-reader.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/api-routes.d.ts +40 -0
- package/clinkx-workflows/dist/dashboard/api-routes.js +1155 -0
- package/clinkx-workflows/dist/dashboard/api-routes.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/artifact-index.d.ts +37 -0
- package/clinkx-workflows/dist/dashboard/artifact-index.js +145 -0
- package/clinkx-workflows/dist/dashboard/artifact-index.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/auth/challenge-store.d.ts +31 -0
- package/clinkx-workflows/dist/dashboard/auth/challenge-store.js +65 -0
- package/clinkx-workflows/dist/dashboard/auth/challenge-store.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/auth/controller.d.ts +26 -0
- package/clinkx-workflows/dist/dashboard/auth/controller.js +596 -0
- package/clinkx-workflows/dist/dashboard/auth/controller.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/auth/credential-store.d.ts +60 -0
- package/clinkx-workflows/dist/dashboard/auth/credential-store.js +776 -0
- package/clinkx-workflows/dist/dashboard/auth/credential-store.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/auth/enrollment-gate.d.ts +9 -0
- package/clinkx-workflows/dist/dashboard/auth/enrollment-gate.js +44 -0
- package/clinkx-workflows/dist/dashboard/auth/enrollment-gate.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/auth/session-store.d.ts +17 -0
- package/clinkx-workflows/dist/dashboard/auth/session-store.js +77 -0
- package/clinkx-workflows/dist/dashboard/auth/session-store.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/auth/webauthn-service.d.ts +25 -0
- package/clinkx-workflows/dist/dashboard/auth/webauthn-service.js +71 -0
- package/clinkx-workflows/dist/dashboard/auth/webauthn-service.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/context-builder-invocation-contract.d.ts +3865 -0
- package/clinkx-workflows/dist/dashboard/context-builder-invocation-contract.js +69 -0
- package/clinkx-workflows/dist/dashboard/context-builder-invocation-contract.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/context-run-liveness.d.ts +37 -0
- package/clinkx-workflows/dist/dashboard/context-run-liveness.js +49 -0
- package/clinkx-workflows/dist/dashboard/context-run-liveness.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/context-run-reader.d.ts +256 -0
- package/clinkx-workflows/dist/dashboard/context-run-reader.js +1301 -0
- package/clinkx-workflows/dist/dashboard/context-run-reader.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/contracts.d.ts +28483 -0
- package/clinkx-workflows/dist/dashboard/contracts.js +1284 -0
- package/clinkx-workflows/dist/dashboard/contracts.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/generation.d.ts +14 -0
- package/clinkx-workflows/dist/dashboard/generation.js +41 -0
- package/clinkx-workflows/dist/dashboard/generation.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/index.d.ts +32 -0
- package/clinkx-workflows/dist/dashboard/index.js +20 -0
- package/clinkx-workflows/dist/dashboard/index.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/invocation-contract.d.ts +786 -0
- package/clinkx-workflows/dist/dashboard/invocation-contract.js +201 -0
- package/clinkx-workflows/dist/dashboard/invocation-contract.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/lazy-content.d.ts +114 -0
- package/clinkx-workflows/dist/dashboard/lazy-content.js +799 -0
- package/clinkx-workflows/dist/dashboard/lazy-content.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/native-transcript/contract.d.ts +90 -0
- package/clinkx-workflows/dist/dashboard/native-transcript/contract.js +31 -0
- package/clinkx-workflows/dist/dashboard/native-transcript/contract.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/native-transcript/projection.d.ts +12 -0
- package/clinkx-workflows/dist/dashboard/native-transcript/projection.js +286 -0
- package/clinkx-workflows/dist/dashboard/native-transcript/projection.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/progress-report-contract.d.ts +446 -0
- package/clinkx-workflows/dist/dashboard/progress-report-contract.js +221 -0
- package/clinkx-workflows/dist/dashboard/progress-report-contract.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/progress-report-reader.d.ts +50 -0
- package/clinkx-workflows/dist/dashboard/progress-report-reader.js +529 -0
- package/clinkx-workflows/dist/dashboard/progress-report-reader.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/progress-report-status-overlay.d.ts +89 -0
- package/clinkx-workflows/dist/dashboard/progress-report-status-overlay.js +71 -0
- package/clinkx-workflows/dist/dashboard/progress-report-status-overlay.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/progress-report-status-store.d.ts +29 -0
- package/clinkx-workflows/dist/dashboard/progress-report-status-store.js +111 -0
- package/clinkx-workflows/dist/dashboard/progress-report-status-store.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/prompt-builder-library-contract.d.ts +194 -0
- package/clinkx-workflows/dist/dashboard/prompt-builder-library-contract.js +89 -0
- package/clinkx-workflows/dist/dashboard/prompt-builder-library-contract.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/prompt-builder-library-store.d.ts +26 -0
- package/clinkx-workflows/dist/dashboard/prompt-builder-library-store.js +102 -0
- package/clinkx-workflows/dist/dashboard/prompt-builder-library-store.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/read-model.d.ts +86 -0
- package/clinkx-workflows/dist/dashboard/read-model.js +440 -0
- package/clinkx-workflows/dist/dashboard/read-model.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/request-admission.d.ts +31 -0
- package/clinkx-workflows/dist/dashboard/request-admission.js +130 -0
- package/clinkx-workflows/dist/dashboard/request-admission.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/scanner.d.ts +78 -0
- package/clinkx-workflows/dist/dashboard/scanner.js +1154 -0
- package/clinkx-workflows/dist/dashboard/scanner.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/server-factory.d.ts +63 -0
- package/clinkx-workflows/dist/dashboard/server-factory.js +483 -0
- package/clinkx-workflows/dist/dashboard/server-factory.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/session-log-contract.d.ts +2778 -0
- package/clinkx-workflows/dist/dashboard/session-log-contract.js +506 -0
- package/clinkx-workflows/dist/dashboard/session-log-contract.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/session-log-identity.d.ts +21 -0
- package/clinkx-workflows/dist/dashboard/session-log-identity.js +101 -0
- package/clinkx-workflows/dist/dashboard/session-log-identity.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/session-log-journal.d.ts +43 -0
- package/clinkx-workflows/dist/dashboard/session-log-journal.js +529 -0
- package/clinkx-workflows/dist/dashboard/session-log-journal.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/session-log-reader.d.ts +61 -0
- package/clinkx-workflows/dist/dashboard/session-log-reader.js +378 -0
- package/clinkx-workflows/dist/dashboard/session-log-reader.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/session-log-redaction.d.ts +14 -0
- package/clinkx-workflows/dist/dashboard/session-log-redaction.js +1101 -0
- package/clinkx-workflows/dist/dashboard/session-log-redaction.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/session-log-retention.d.ts +23 -0
- package/clinkx-workflows/dist/dashboard/session-log-retention.js +212 -0
- package/clinkx-workflows/dist/dashboard/session-log-retention.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/static-responses.d.ts +10 -0
- package/clinkx-workflows/dist/dashboard/static-responses.js +94 -0
- package/clinkx-workflows/dist/dashboard/static-responses.js.map +1 -0
- package/clinkx-workflows/dist/dashboard/worker-lock-observer.d.ts +28 -0
- package/clinkx-workflows/dist/dashboard/worker-lock-observer.js +241 -0
- package/clinkx-workflows/dist/dashboard/worker-lock-observer.js.map +1 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-sans-latin-400-normal-CDDApCn2.woff2 +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-sans-latin-400-normal-CYLoc0-x.woff +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-sans-latin-500-normal-6ng42L7E.woff2 +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-sans-latin-500-normal-BgVn5rGT.woff +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-sans-latin-600-normal-Cu4Hd6ag.woff +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/ibm-plex-sans-latin-600-normal-CuJfVYMP.woff2 +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/index-JPHQVtn9.js +106 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/jetbrains-mono-latin-400-normal-6-qcROiO.woff +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/jetbrains-mono-latin-400-normal-V6pRDFza.woff2 +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/jetbrains-mono-latin-500-normal-BWZEU5yA.woff2 +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/jetbrains-mono-latin-500-normal-CJOVTJB7.woff +0 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/style-DOMHk-j6.css +1 -0
- package/clinkx-workflows/dist/dashboard-ui/contracts/council-plan-review-input-v1.json +158 -0
- package/clinkx-workflows/dist/dashboard-ui/index.html +15 -0
- package/clinkx-workflows/dist/dashboard-ui/licenses/IBM-Plex-Mono-OFL.txt +93 -0
- package/clinkx-workflows/dist/dashboard-ui/licenses/IBM-Plex-Sans-OFL.txt +93 -0
- package/clinkx-workflows/dist/dashboard-ui/licenses/JetBrains-Mono-OFL.txt +93 -0
- package/clinkx-workflows/dist/dashboard-ui/licenses/NOTICE.md +32 -0
- package/clinkx-workflows/dist/dashboard-ui/theme-init.js +8 -0
- package/clinkx-workflows/dist/definition-normalizer.d.ts +19 -0
- package/clinkx-workflows/dist/definition-normalizer.js.map +1 -1
- package/clinkx-workflows/dist/delegate-finalizer-policy.d.ts +26 -0
- package/clinkx-workflows/dist/delegate-finalizer-policy.js +72 -0
- package/clinkx-workflows/dist/delegate-finalizer-policy.js.map +1 -0
- package/clinkx-workflows/dist/delegate-signature.d.ts +14 -0
- package/clinkx-workflows/dist/delegate-signature.js +14 -0
- package/clinkx-workflows/dist/delegate-signature.js.map +1 -0
- package/clinkx-workflows/dist/delegate-validate.d.ts +108 -0
- package/clinkx-workflows/dist/delegate-validate.js +148 -0
- package/clinkx-workflows/dist/delegate-validate.js.map +1 -0
- package/clinkx-workflows/dist/engine.d.ts +251 -0
- package/clinkx-workflows/dist/engine.js +1342 -92
- package/clinkx-workflows/dist/engine.js.map +1 -1
- package/clinkx-workflows/dist/errors.d.ts +24 -2
- package/clinkx-workflows/dist/errors.js +30 -2
- package/clinkx-workflows/dist/errors.js.map +1 -1
- package/clinkx-workflows/dist/fidelity.d.ts +47 -1
- package/clinkx-workflows/dist/fidelity.js +54 -1
- package/clinkx-workflows/dist/fidelity.js.map +1 -1
- package/clinkx-workflows/dist/fingerprint.d.ts +25 -7
- package/clinkx-workflows/dist/fingerprint.js +33 -10
- package/clinkx-workflows/dist/fingerprint.js.map +1 -1
- package/clinkx-workflows/dist/index.d.ts +19 -0
- package/clinkx-workflows/dist/index.js +68 -1
- package/clinkx-workflows/dist/index.js.map +1 -1
- package/clinkx-workflows/dist/loader.d.ts +9 -3
- package/clinkx-workflows/dist/loader.js +92 -21
- package/clinkx-workflows/dist/loader.js.map +1 -1
- package/clinkx-workflows/dist/logger.d.ts +1 -1
- package/clinkx-workflows/dist/logger.js +17 -3
- package/clinkx-workflows/dist/logger.js.map +1 -1
- package/clinkx-workflows/dist/monotonic-clock.d.ts +19 -0
- package/clinkx-workflows/dist/monotonic-clock.js +22 -0
- package/clinkx-workflows/dist/monotonic-clock.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/admission.d.ts +160 -0
- package/clinkx-workflows/dist/plan-review/admission.js +290 -0
- package/clinkx-workflows/dist/plan-review/admission.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/context-policy.d.ts +55 -0
- package/clinkx-workflows/dist/plan-review/context-policy.js +146 -0
- package/clinkx-workflows/dist/plan-review/context-policy.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/contract.d.ts +763 -0
- package/clinkx-workflows/dist/plan-review/contract.js +289 -0
- package/clinkx-workflows/dist/plan-review/contract.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/digest.d.ts +68 -0
- package/clinkx-workflows/dist/plan-review/digest.js +241 -0
- package/clinkx-workflows/dist/plan-review/digest.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/discovery.d.ts +45 -0
- package/clinkx-workflows/dist/plan-review/discovery.js +36 -0
- package/clinkx-workflows/dist/plan-review/discovery.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/execution-contract.d.ts +298 -0
- package/clinkx-workflows/dist/plan-review/execution-contract.js +194 -0
- package/clinkx-workflows/dist/plan-review/execution-contract.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/expander.d.ts +48 -0
- package/clinkx-workflows/dist/plan-review/expander.js +243 -0
- package/clinkx-workflows/dist/plan-review/expander.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/projector.d.ts +126 -0
- package/clinkx-workflows/dist/plan-review/projector.js +437 -0
- package/clinkx-workflows/dist/plan-review/projector.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/publication.d.ts +95 -0
- package/clinkx-workflows/dist/plan-review/publication.js +182 -0
- package/clinkx-workflows/dist/plan-review/publication.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/ranking-reducer.d.ts +34 -0
- package/clinkx-workflows/dist/plan-review/ranking-reducer.js +139 -0
- package/clinkx-workflows/dist/plan-review/ranking-reducer.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/resume.d.ts +38 -0
- package/clinkx-workflows/dist/plan-review/resume.js +122 -0
- package/clinkx-workflows/dist/plan-review/resume.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/rubric.d.ts +7 -0
- package/clinkx-workflows/dist/plan-review/rubric.js +24 -0
- package/clinkx-workflows/dist/plan-review/rubric.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/run-state.d.ts +398 -0
- package/clinkx-workflows/dist/plan-review/run-state.js +183 -0
- package/clinkx-workflows/dist/plan-review/run-state.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/runtime-policy.d.ts +74 -0
- package/clinkx-workflows/dist/plan-review/runtime-policy.js +186 -0
- package/clinkx-workflows/dist/plan-review/runtime-policy.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/snapshot.d.ts +81 -0
- package/clinkx-workflows/dist/plan-review/snapshot.js +170 -0
- package/clinkx-workflows/dist/plan-review/snapshot.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/stage-error.d.ts +19 -0
- package/clinkx-workflows/dist/plan-review/stage-error.js +20 -0
- package/clinkx-workflows/dist/plan-review/stage-error.js.map +1 -0
- package/clinkx-workflows/dist/post-processors/index.d.ts +20 -0
- package/clinkx-workflows/dist/post-processors/index.js +21 -0
- package/clinkx-workflows/dist/post-processors/index.js.map +1 -0
- package/clinkx-workflows/dist/prompt-budget.d.ts +7 -0
- package/clinkx-workflows/dist/prompt-budget.js +7 -1
- package/clinkx-workflows/dist/prompt-budget.js.map +1 -1
- package/clinkx-workflows/dist/reducers/index.js +2 -0
- package/clinkx-workflows/dist/reducers/index.js.map +1 -1
- package/clinkx-workflows/dist/roster-contract.d.ts +889 -0
- package/clinkx-workflows/dist/roster-contract.js +180 -0
- package/clinkx-workflows/dist/roster-contract.js.map +1 -0
- package/clinkx-workflows/dist/rosters.d.ts +14 -0
- package/clinkx-workflows/dist/rosters.js +169 -0
- package/clinkx-workflows/dist/rosters.js.map +1 -0
- package/clinkx-workflows/dist/run-health.d.ts +7 -0
- package/clinkx-workflows/dist/run-health.js +25 -0
- package/clinkx-workflows/dist/run-health.js.map +1 -0
- package/clinkx-workflows/dist/run-lock-heartbeat.d.ts +2 -0
- package/clinkx-workflows/dist/run-lock-heartbeat.js +3 -0
- package/clinkx-workflows/dist/run-lock-heartbeat.js.map +1 -0
- package/clinkx-workflows/dist/run-summary/cards/council-plan-review.d.ts +34 -0
- package/clinkx-workflows/dist/run-summary/cards/council-plan-review.js +74 -0
- package/clinkx-workflows/dist/run-summary/cards/council-plan-review.js.map +1 -0
- package/clinkx-workflows/dist/run-summary/cards/index.js +5 -0
- package/clinkx-workflows/dist/run-summary/cards/index.js.map +1 -1
- package/clinkx-workflows/dist/run-summary.d.ts +13 -0
- package/clinkx-workflows/dist/run-summary.js +29 -1
- package/clinkx-workflows/dist/run-summary.js.map +1 -1
- package/clinkx-workflows/dist/schema.d.ts +140 -21
- package/clinkx-workflows/dist/schema.js +34 -1
- package/clinkx-workflows/dist/schema.js.map +1 -1
- package/clinkx-workflows/dist/seed-terms.d.ts +30 -0
- package/clinkx-workflows/dist/seed-terms.js +176 -0
- package/clinkx-workflows/dist/seed-terms.js.map +1 -0
- package/clinkx-workflows/dist/server.d.ts +5 -2
- package/clinkx-workflows/dist/server.js +84 -52
- package/clinkx-workflows/dist/server.js.map +1 -1
- package/clinkx-workflows/dist/shutdown.d.ts +29 -18
- package/clinkx-workflows/dist/shutdown.js +90 -64
- package/clinkx-workflows/dist/shutdown.js.map +1 -1
- package/clinkx-workflows/dist/state-schema.d.ts +61 -0
- package/clinkx-workflows/dist/state.d.ts +175 -3
- package/clinkx-workflows/dist/state.js +1398 -44
- package/clinkx-workflows/dist/state.js.map +1 -1
- package/clinkx-workflows/dist/stdio-watchdog.d.ts +52 -0
- package/clinkx-workflows/dist/stdio-watchdog.js +212 -0
- package/clinkx-workflows/dist/stdio-watchdog.js.map +1 -0
- package/clinkx-workflows/dist/transport.d.ts +15 -0
- package/clinkx-workflows/dist/transport.js +19 -10
- package/clinkx-workflows/dist/transport.js.map +1 -1
- package/clinkx-workflows/dist/types.d.ts +20 -2
- package/clinkx-workflows/dist/validators/council.d.ts +334 -101
- package/clinkx-workflows/dist/validators/council.js +52 -13
- package/clinkx-workflows/dist/validators/council.js.map +1 -1
- package/clinkx-workflows/dist/validators/index.d.ts +1 -0
- package/clinkx-workflows/dist/validators/index.js +3 -0
- package/clinkx-workflows/dist/validators/index.js.map +1 -1
- package/clinkx-workflows/dist/validators/plan-review.d.ts +398 -0
- package/clinkx-workflows/dist/validators/plan-review.js +316 -0
- package/clinkx-workflows/dist/validators/plan-review.js.map +1 -0
- package/clinkx-workflows/dist/workflow-receipt.js +188 -4
- package/clinkx-workflows/dist/workflow-receipt.js.map +1 -1
- package/clinkx-workflows/dist/workflow-tools.d.ts +118 -3
- package/clinkx-workflows/dist/workflow-tools.js +2110 -175
- package/clinkx-workflows/dist/workflow-tools.js.map +1 -1
- package/clinkx-workflows/dist/workspace-signature.d.ts +16 -0
- package/clinkx-workflows/dist/workspace-signature.js +388 -0
- package/clinkx-workflows/dist/workspace-signature.js.map +1 -0
- package/clinkx-workflows/templates/council/answer-chairman.txt +3 -0
- package/clinkx-workflows/templates/council/answer-cross-reviewer.txt +3 -0
- package/clinkx-workflows/templates/council/answer-specialist.txt +5 -1
- package/clinkx-workflows/templates/council/chairman.txt +3 -0
- package/clinkx-workflows/templates/council/code-review-chairman.txt +16 -0
- package/clinkx-workflows/templates/council/code-review-cross-reviewer.txt +7 -1
- package/clinkx-workflows/templates/council/code-review-reviewer.txt +22 -3
- package/clinkx-workflows/templates/council/debug-chairman.txt +3 -0
- package/clinkx-workflows/templates/council/debug-cross-reviewer.txt +3 -0
- package/clinkx-workflows/templates/council/debug-debugger.txt +5 -1
- package/clinkx-workflows/templates/council/discover-analyst.txt +5 -1
- package/clinkx-workflows/templates/council/discover-chairman.txt +3 -0
- package/clinkx-workflows/templates/council/discover-cross-reviewer.txt +3 -0
- package/clinkx-workflows/templates/council/member.txt +5 -1
- package/clinkx-workflows/templates/council/plan-review-chairman.txt +71 -0
- package/clinkx-workflows/templates/council/plan-review-cross-reviewer.txt +54 -0
- package/clinkx-workflows/templates/council/plan-review-reviewer.txt +63 -0
- package/clinkx-workflows/templates/council/reviewer.txt +3 -0
- package/clinkx-workflows/templates/plan-review/chairman.txt +48 -0
- package/clinkx-workflows/templates/plan-review/cross-review.txt +52 -0
- package/clinkx-workflows/templates/plan-review/ranking.txt +16 -0
- package/clinkx-workflows/templates/plan-review/reviewer.txt +57 -0
- package/clinkx-workflows/workflows/council-answer.yaml +100 -8
- package/clinkx-workflows/workflows/council-code-review.yaml +106 -8
- package/clinkx-workflows/workflows/council-debug.yaml +100 -8
- package/clinkx-workflows/workflows/council-default.yaml +104 -8
- package/clinkx-workflows/workflows/council-discover.yaml +100 -8
- package/clinkx-workflows/workflows/council-plan-review.yaml +253 -0
- package/clinkx-workflows/workflows/plan-review-receipts.yaml +230 -0
- package/conf/adapters/agy.json +56 -0
- package/conf/adapters/claude.json +12 -0
- package/conf/adapters/codex-app-server.json +44 -0
- package/conf/adapters/codex.json +14 -2
- package/conf/adapters/glm.json +24 -3
- package/conf/adapters/grok.json +67 -0
- package/conf/adapters/hapi/agy.json +59 -0
- package/conf/adapters/hapi/claude.json +12 -0
- package/conf/adapters/hapi/codex.json +14 -2
- package/conf/adapters/hapi/glm.json +25 -4
- package/conf/rosters/default-council.json +21 -0
- package/dist/activity-journal.d.ts +56 -0
- package/dist/activity-journal.js +706 -0
- package/dist/activity-journal.js.map +1 -0
- package/dist/allowed-roots.d.ts +41 -0
- package/dist/allowed-roots.js +125 -3
- package/dist/allowed-roots.js.map +1 -1
- package/dist/codex/activity-session-log.d.ts +14 -0
- package/dist/codex/activity-session-log.js +206 -0
- package/dist/codex/activity-session-log.js.map +1 -0
- package/dist/codex/agent-tools.d.ts +23 -0
- package/dist/codex/agent-tools.js +1306 -0
- package/dist/codex/agent-tools.js.map +1 -0
- package/dist/codex/app-server-client.d.ts +79 -0
- package/dist/codex/app-server-client.js +593 -0
- package/dist/codex/app-server-client.js.map +1 -0
- package/dist/codex/app-server-registry.d.ts +407 -0
- package/dist/codex/app-server-registry.js +1493 -0
- package/dist/codex/app-server-registry.js.map +1 -0
- package/dist/codex/client-types.d.ts +115 -0
- package/dist/codex/client-types.js +2 -0
- package/dist/codex/client-types.js.map +1 -0
- package/dist/codex/protocol.d.ts +5375 -0
- package/dist/codex/protocol.js +614 -0
- package/dist/codex/protocol.js.map +1 -0
- package/dist/codex/session.d.ts +263 -0
- package/dist/codex/session.js +466 -0
- package/dist/codex/session.js.map +1 -0
- package/dist/config.d.ts +386 -23
- package/dist/config.js +184 -15
- package/dist/config.js.map +1 -1
- package/dist/context-builder/anchors.d.ts +1 -0
- package/dist/context-builder/anchors.js +3 -0
- package/dist/context-builder/anchors.js.map +1 -1
- package/dist/context-builder/bundle.d.ts +48 -2
- package/dist/context-builder/bundle.js +388 -2417
- package/dist/context-builder/bundle.js.map +1 -1
- package/dist/context-builder/codemap.d.ts +5 -0
- package/dist/context-builder/codemap.js +387 -13
- package/dist/context-builder/codemap.js.map +1 -1
- package/dist/context-builder/config.d.ts +1 -0
- package/dist/context-builder/config.js +12 -0
- package/dist/context-builder/config.js.map +1 -1
- package/dist/context-builder/context-run-catalog.d.ts +17 -0
- package/dist/context-builder/context-run-catalog.js +99 -0
- package/dist/context-builder/context-run-catalog.js.map +1 -0
- package/dist/context-builder/delegate-finalizer-effective.d.ts +33 -0
- package/dist/context-builder/delegate-finalizer-effective.js +82 -0
- package/dist/context-builder/delegate-finalizer-effective.js.map +1 -0
- package/dist/context-builder/delegate-packet.d.ts +55 -0
- package/dist/context-builder/delegate-packet.js +430 -74
- package/dist/context-builder/delegate-packet.js.map +1 -1
- package/dist/context-builder/delegate-reconstruction.d.ts +29 -0
- package/dist/context-builder/delegate-reconstruction.js +110 -41
- package/dist/context-builder/delegate-reconstruction.js.map +1 -1
- package/dist/context-builder/delegate-validation.d.ts +41 -1
- package/dist/context-builder/delegate-validation.js +237 -48
- package/dist/context-builder/delegate-validation.js.map +1 -1
- package/dist/context-builder/delegate.d.ts +11 -0
- package/dist/context-builder/delegate.js +210 -58
- package/dist/context-builder/delegate.js.map +1 -1
- package/dist/context-builder/diagnostics.d.ts +11 -2
- package/dist/context-builder/diagnostics.js +39 -6
- package/dist/context-builder/diagnostics.js.map +1 -1
- package/dist/context-builder/discovery-progress.d.ts +25 -0
- package/dist/context-builder/discovery-progress.js +64 -0
- package/dist/context-builder/discovery-progress.js.map +1 -0
- package/dist/context-builder/discovery-watchdog.d.ts +15 -0
- package/dist/context-builder/discovery-watchdog.js +39 -0
- package/dist/context-builder/discovery-watchdog.js.map +1 -0
- package/dist/context-builder/discovery.d.ts +36 -1
- package/dist/context-builder/discovery.js +584 -172
- package/dist/context-builder/discovery.js.map +1 -1
- package/dist/context-builder/external-gaps.d.ts +73 -0
- package/dist/context-builder/external-gaps.js +248 -0
- package/dist/context-builder/external-gaps.js.map +1 -0
- package/dist/context-builder/globs.d.ts +1 -0
- package/dist/context-builder/globs.js +25 -10
- package/dist/context-builder/globs.js.map +1 -1
- package/dist/context-builder/handler.d.ts +67 -2
- package/dist/context-builder/handler.js +1810 -2440
- package/dist/context-builder/handler.js.map +1 -1
- package/dist/context-builder/invocation-sidecar.d.ts +41 -0
- package/dist/context-builder/invocation-sidecar.js +189 -0
- package/dist/context-builder/invocation-sidecar.js.map +1 -0
- package/dist/context-builder/lifecycle-contract.d.ts +424 -0
- package/dist/context-builder/lifecycle-contract.js +523 -0
- package/dist/context-builder/lifecycle-contract.js.map +1 -0
- package/dist/context-builder/lifecycle-journal.d.ts +62 -0
- package/dist/context-builder/lifecycle-journal.js +88 -0
- package/dist/context-builder/lifecycle-journal.js.map +1 -0
- package/dist/context-builder/lifecycle-observer.d.ts +102 -0
- package/dist/context-builder/lifecycle-observer.js +258 -0
- package/dist/context-builder/lifecycle-observer.js.map +1 -0
- package/dist/context-builder/lifecycle-reconcile.d.ts +32 -0
- package/dist/context-builder/lifecycle-reconcile.js +191 -0
- package/dist/context-builder/lifecycle-reconcile.js.map +1 -0
- package/dist/context-builder/lifecycle-recovery.d.ts +2 -0
- package/dist/context-builder/lifecycle-recovery.js +14 -0
- package/dist/context-builder/lifecycle-recovery.js.map +1 -0
- package/dist/context-builder/mutation-lock.d.ts +32 -0
- package/dist/context-builder/mutation-lock.js +321 -0
- package/dist/context-builder/mutation-lock.js.map +1 -0
- package/dist/context-builder/path-signals.js +9 -5
- package/dist/context-builder/path-signals.js.map +1 -1
- package/dist/context-builder/progress.d.ts +3 -1
- package/dist/context-builder/progress.js +23 -0
- package/dist/context-builder/progress.js.map +1 -1
- package/dist/context-builder/receipt.d.ts +6 -4
- package/dist/context-builder/receipt.js +12 -1
- package/dist/context-builder/receipt.js.map +1 -1
- package/dist/context-builder/refine-handler.d.ts +14 -0
- package/dist/context-builder/refine-handler.js +790 -0
- package/dist/context-builder/refine-handler.js.map +1 -0
- package/dist/context-builder/review-coverage.d.ts +116 -0
- package/dist/context-builder/review-coverage.js +544 -0
- package/dist/context-builder/review-coverage.js.map +1 -0
- package/dist/context-builder/schema.d.ts +315 -2
- package/dist/context-builder/schema.js +133 -7
- package/dist/context-builder/schema.js.map +1 -1
- package/dist/context-builder/secrets.d.ts +59 -0
- package/dist/context-builder/secrets.js +2255 -44
- package/dist/context-builder/secrets.js.map +1 -1
- package/dist/context-builder/selection.d.ts +7 -3
- package/dist/context-builder/selection.js +200 -738
- package/dist/context-builder/selection.js.map +1 -1
- package/dist/context-builder/session-log-identity.d.ts +1 -0
- package/dist/context-builder/session-log-identity.js +66 -0
- package/dist/context-builder/session-log-identity.js.map +1 -0
- package/dist/context-builder/slice-range-math.d.ts +16 -0
- package/dist/context-builder/slice-range-math.js +137 -0
- package/dist/context-builder/slice-range-math.js.map +1 -0
- package/dist/context-builder/status.d.ts +23 -2
- package/dist/context-builder/status.js +60 -4
- package/dist/context-builder/status.js.map +1 -1
- package/dist/context-builder/storage.d.ts +59 -2
- package/dist/context-builder/storage.js +500 -168
- package/dist/context-builder/storage.js.map +1 -1
- package/dist/context-builder/token-estimate.d.ts +13 -2
- package/dist/context-builder/token-estimate.js +77 -4
- package/dist/context-builder/token-estimate.js.map +1 -1
- package/dist/context-builder/versions.d.ts +44 -0
- package/dist/context-builder/versions.js +45 -0
- package/dist/context-builder/versions.js.map +1 -0
- package/dist/context-builder/workflow-runner.d.ts +75 -0
- package/dist/context-builder/workflow-runner.js +1241 -0
- package/dist/context-builder/workflow-runner.js.map +1 -0
- package/dist/context-builder/write-watchdog.d.ts +14 -0
- package/dist/context-builder/write-watchdog.js +38 -0
- package/dist/context-builder/write-watchdog.js.map +1 -0
- package/dist/dashboard-adapter-catalog.d.ts +9 -0
- package/dist/dashboard-adapter-catalog.js +53 -0
- package/dist/dashboard-adapter-catalog.js.map +1 -0
- package/dist/dashboard-cli.d.ts +81 -0
- package/dist/dashboard-cli.js +623 -0
- package/dist/dashboard-cli.js.map +1 -0
- package/dist/dashboard-companion/client.d.ts +37 -0
- package/dist/dashboard-companion/client.js +229 -0
- package/dist/dashboard-companion/client.js.map +1 -0
- package/dist/dashboard-companion/config.d.ts +8 -0
- package/dist/dashboard-companion/config.js +52 -0
- package/dist/dashboard-companion/config.js.map +1 -0
- package/dist/dashboard-companion/http-proxy.d.ts +10 -0
- package/dist/dashboard-companion/http-proxy.js +234 -0
- package/dist/dashboard-companion/http-proxy.js.map +1 -0
- package/dist/dashboard-companion/protocol.d.ts +80 -0
- package/dist/dashboard-companion/protocol.js +458 -0
- package/dist/dashboard-companion/protocol.js.map +1 -0
- package/dist/dashboard-companion/standalone-assets.d.ts +12 -0
- package/dist/dashboard-companion/standalone-assets.js +72 -0
- package/dist/dashboard-companion/standalone-assets.js.map +1 -0
- package/dist/dashboard-companion/standalone-entry.d.ts +1 -0
- package/dist/dashboard-companion/standalone-entry.js +17 -0
- package/dist/dashboard-companion/standalone-entry.js.map +1 -0
- package/dist/dashboard-gateway/config.d.ts +9 -0
- package/dist/dashboard-gateway/config.js +44 -0
- package/dist/dashboard-gateway/config.js.map +1 -0
- package/dist/dashboard-gateway/index.d.ts +12 -0
- package/dist/dashboard-gateway/index.js +786 -0
- package/dist/dashboard-gateway/index.js.map +1 -0
- package/dist/dashboard-gateway/main.d.ts +1 -0
- package/dist/dashboard-gateway/main.js +25 -0
- package/dist/dashboard-gateway/main.js.map +1 -0
- package/dist/dashboard-pairing-token.d.ts +1 -0
- package/dist/dashboard-pairing-token.js +5 -0
- package/dist/dashboard-pairing-token.js.map +1 -0
- package/dist/elaris/client.d.ts +73 -0
- package/dist/elaris/client.js +825 -0
- package/dist/elaris/client.js.map +1 -0
- package/dist/elaris/config.d.ts +18 -0
- package/dist/elaris/config.js +75 -0
- package/dist/elaris/config.js.map +1 -0
- package/dist/elaris/errors.d.ts +41 -0
- package/dist/elaris/errors.js +205 -0
- package/dist/elaris/errors.js.map +1 -0
- package/dist/elaris/github-repo.d.ts +18 -0
- package/dist/elaris/github-repo.js +89 -0
- package/dist/elaris/github-repo.js.map +1 -0
- package/dist/elaris/index.d.ts +10 -0
- package/dist/elaris/index.js +11 -0
- package/dist/elaris/index.js.map +1 -0
- package/dist/elaris/redact.d.ts +89 -0
- package/dist/elaris/redact.js +423 -0
- package/dist/elaris/redact.js.map +1 -0
- package/dist/elaris/run-id.d.ts +5 -0
- package/dist/elaris/run-id.js +20 -0
- package/dist/elaris/run-id.js.map +1 -0
- package/dist/elaris/status.d.ts +45 -0
- package/dist/elaris/status.js +122 -0
- package/dist/elaris/status.js.map +1 -0
- package/dist/elaris/tool-schemas.d.ts +314 -0
- package/dist/elaris/tool-schemas.js +208 -0
- package/dist/elaris/tool-schemas.js.map +1 -0
- package/dist/elaris/tools.d.ts +73 -0
- package/dist/elaris/tools.js +1065 -0
- package/dist/elaris/tools.js.map +1 -0
- package/dist/elaris/types.d.ts +1541 -0
- package/dist/elaris/types.js +252 -0
- package/dist/elaris/types.js.map +1 -0
- package/dist/errors.d.ts +16 -0
- package/dist/errors.js +19 -0
- package/dist/errors.js.map +1 -1
- package/dist/handler.d.ts +4 -0
- package/dist/handler.js +15 -2
- package/dist/handler.js.map +1 -1
- package/dist/index.d.ts +20 -2
- package/dist/index.js +154 -6
- package/dist/index.js.map +1 -1
- package/dist/local-clink-backend.d.ts +3 -0
- package/dist/local-clink-backend.js +4 -0
- package/dist/local-clink-backend.js.map +1 -1
- package/dist/logger.d.ts +1 -1
- package/dist/logger.js +18 -3
- package/dist/logger.js.map +1 -1
- package/dist/package-info.d.ts +2 -0
- package/dist/package-info.js +18 -0
- package/dist/package-info.js.map +1 -0
- package/dist/parsers/agy-stream-json.d.ts +14 -0
- package/dist/parsers/agy-stream-json.js +172 -0
- package/dist/parsers/agy-stream-json.js.map +1 -0
- package/dist/parsers/codex-jsonl.js +48 -3
- package/dist/parsers/codex-jsonl.js.map +1 -1
- package/dist/parsers/extract.d.ts +10 -1
- package/dist/parsers/extract.js +16 -4
- package/dist/parsers/extract.js.map +1 -1
- package/dist/parsers/observed-invocation.d.ts +20 -0
- package/dist/parsers/observed-invocation.js +42 -0
- package/dist/parsers/observed-invocation.js.map +1 -0
- package/dist/pipeline.d.ts +17 -0
- package/dist/pipeline.js +181 -10
- package/dist/pipeline.js.map +1 -1
- package/dist/progress-report/lock.d.ts +74 -0
- package/dist/progress-report/lock.js +343 -0
- package/dist/progress-report/lock.js.map +1 -0
- package/dist/progress-report/schema.d.ts +808 -0
- package/dist/progress-report/schema.js +119 -0
- package/dist/progress-report/schema.js.map +1 -0
- package/dist/progress-report/store.d.ts +66 -0
- package/dist/progress-report/store.js +766 -0
- package/dist/progress-report/store.js.map +1 -0
- package/dist/progress-report/tools.d.ts +20 -0
- package/dist/progress-report/tools.js +141 -0
- package/dist/progress-report/tools.js.map +1 -0
- package/dist/prompt-mode.d.ts +1 -1
- package/dist/prompt-mode.js +7 -2
- package/dist/prompt-mode.js.map +1 -1
- package/dist/run-dir.d.ts +24 -0
- package/dist/run-dir.js +201 -2
- package/dist/run-dir.js.map +1 -1
- package/dist/runner.d.ts +20 -0
- package/dist/runner.js +192 -16
- package/dist/runner.js.map +1 -1
- package/dist/safety.d.ts +54 -4
- package/dist/safety.js +186 -5
- package/dist/safety.js.map +1 -1
- package/dist/schema.d.ts +15 -7
- package/dist/schema.js +20 -0
- package/dist/schema.js.map +1 -1
- package/dist/server.d.ts +7 -1
- package/dist/server.js +451 -23
- package/dist/server.js.map +1 -1
- package/dist/session-log-stream.d.ts +18 -0
- package/dist/session-log-stream.js +566 -0
- package/dist/session-log-stream.js.map +1 -0
- package/dist/stdio-activity.d.ts +28 -0
- package/dist/stdio-activity.js +72 -0
- package/dist/stdio-activity.js.map +1 -0
- package/package.json +47 -5
- package/conf/adapters/gemini.json +0 -44
- package/conf/adapters/hapi/gemini.json +0 -47
- package/dist/parsers/gemini-json.d.ts +0 -2
- package/dist/parsers/gemini-json.js +0 -72
- package/dist/parsers/gemini-json.js.map +0 -1
|
@@ -1,15 +1,30 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
2
2
|
import { execFileSync } from "node:child_process";
|
|
3
|
-
import { readFileSync } from "node:fs";
|
|
4
|
-
import { mkdir, open, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
|
|
3
|
+
import { constants as fsConstants, readFileSync } from "node:fs";
|
|
4
|
+
import { lstat, mkdir, open, readFile, readdir, realpath, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
|
|
5
5
|
import { dirname, basename, isAbsolute, join, relative, resolve } from "node:path";
|
|
6
6
|
import { hostname } from "node:os";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
import { WorkflowError } from "./errors.js";
|
|
9
|
+
import { computeRunHealth } from "./run-health.js";
|
|
9
10
|
import { buildFidelityMetadata, } from "./fidelity.js";
|
|
10
11
|
import { fingerprintsMatch, generateFingerprint, generateWorkflowFingerprint, } from "./fingerprint.js";
|
|
11
|
-
import {
|
|
12
|
+
import { effectiveFinalizerEnabled } from "./delegate-finalizer-policy.js";
|
|
13
|
+
import { parseWorkflowContext, } from "./context-preflight.js";
|
|
14
|
+
import { isLegalContextBuilderLifecycleTransition, parseContextBuilderLifecycleStateV1, } from "./context-builder-lifecycle-contract.js";
|
|
15
|
+
import { ResolvedRosterSchema, RosterNameSchema } from "./roster-contract.js";
|
|
16
|
+
import { PlanReviewRunStateSchema, } from "./plan-review/run-state.js";
|
|
17
|
+
import { CouncilPlanReviewErrorSchema, createPlanReviewPublicationError, serializePlanReviewResult, writeSerializedPlanReviewResultSidecar, } from "./plan-review/publication.js";
|
|
18
|
+
import { isPlanReviewPublicationEligible, projectPlanReviewRun, } from "./plan-review/projector.js";
|
|
19
|
+
import { isPlanReviewRunState } from "./plan-review/admission.js";
|
|
20
|
+
import { restorePlanReviewRun } from "./plan-review/resume.js";
|
|
21
|
+
import { PlanReviewStageError } from "./plan-review/stage-error.js";
|
|
22
|
+
import { DEFAULT_RETAIN_ASYNC_DAYS, DEFAULT_RETAIN_CANCELLED_DAYS, DEFAULT_RETAIN_DEBUG_DAYS, DEFAULT_RETAIN_FAILED_DAYS, DEFAULT_RETAIN_SUCCEEDED_DAYS, deriveAllowedRoots, getArtifactsDir, getChildConfigPath, getChildMaxConcurrent, getChildMaxOutputBytes, getChildMaxPromptBytes, getChildMaxResponseChars, getLogLevel, getRetentionConfig, getRunsDir, getRunStateDir, getStateDir, invocationAuditV2PersistenceEnabled, observedInvocationPersistenceEnabled, resolveWorkspaceRoot, } from "./config.js";
|
|
12
23
|
import { logger } from "./logger.js";
|
|
24
|
+
import { resolveOpenedDescriptorPath } from "./contained-file.js";
|
|
25
|
+
import { deriveWorkflowRunGeneration } from "./dashboard/generation.js";
|
|
26
|
+
import { removeSessionLogGeneration } from "./dashboard/session-log-retention.js";
|
|
27
|
+
import { createSessionLogGeneration, readSessionLogGeneration, } from "./dashboard/session-log-identity.js";
|
|
13
28
|
const STATE_FILENAME = "state.json";
|
|
14
29
|
const STATE_TMP_FILENAME = "state.json.tmp";
|
|
15
30
|
const WORKFLOW_SNAPSHOT_FILENAME = "workflow.snapshot.json";
|
|
@@ -17,34 +32,201 @@ const INPUTS_SNAPSHOT_FILENAME = "inputs.snapshot.json";
|
|
|
17
32
|
const CHILD_CONFIG_DIRNAME = "child-config";
|
|
18
33
|
const CHILD_CONFIG_SNAPSHOT_FILENAME = "snapshot.json";
|
|
19
34
|
const RUN_LOCK_FILENAME = "run.lock";
|
|
35
|
+
/**
|
|
36
|
+
* Producer-side refresh cadence for a held `run.lock`'s mtime. Mirrors
|
|
37
|
+
* `dashboard/auth/credential-store.ts`'s LOCK_HEARTBEAT_MS.
|
|
38
|
+
*/
|
|
39
|
+
const RUN_LOCK_HEARTBEAT_MS = 10_000;
|
|
40
|
+
/**
|
|
41
|
+
* Reader-side staleness floor the shipped cadence is designed against; the
|
|
42
|
+
* dashboard observer derives its own threshold from the declared interval.
|
|
43
|
+
* Re-exported for tests and cross-module documentation only — nothing in this
|
|
44
|
+
* module consults it, because run-lock CONTROL decisions
|
|
45
|
+
* (`classifyRunLockSync`, `reapRunCancelled`, pruning) must never depend on
|
|
46
|
+
* mtime freshness.
|
|
47
|
+
*/
|
|
48
|
+
export { RUN_LOCK_HEARTBEAT_STALE_FLOOR_MS } from "./run-lock-heartbeat.js";
|
|
49
|
+
/** Same cap as the dashboard observer's contained-read `maxBytes`. */
|
|
50
|
+
const RUN_LOCK_MAX_BYTES = 4_096;
|
|
20
51
|
const OUTPUT_INLINE_THRESHOLD_BYTES = 10 * 1024;
|
|
52
|
+
/**
|
|
53
|
+
* Cap for error strings persisted inline in state.json (run-level `error` and
|
|
54
|
+
* per-call `errorMessage`). Unlike call outputs (which spill to sidecar files
|
|
55
|
+
* via persistOutput), errors always live inline, and provider stderr/exception
|
|
56
|
+
* chains can be arbitrarily large — keep a bounded head plus an explicit
|
|
57
|
+
* truncation marker recording how many bytes were cut.
|
|
58
|
+
*/
|
|
59
|
+
const MAX_PERSISTED_ERROR_BYTES = 16 * 1024;
|
|
21
60
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
61
|
+
const STALE_RUNNING_RECOVERY_ERROR = "Recovered stale running run during resume";
|
|
62
|
+
/**
|
|
63
|
+
* Run-level error written by reconcileStaleRunDirectories. Deliberately
|
|
64
|
+
* distinct from STALE_RUNNING_RECOVERY_ERROR (the resume-path recovery string)
|
|
65
|
+
* so the two reconcile paths remain distinguishable in persisted state.
|
|
66
|
+
*/
|
|
67
|
+
export const STALE_RUN_RECONCILE_ERROR = "Reconciled stale running run: no live worker process";
|
|
68
|
+
const STALE_RUN_RECONCILE_CALL_ERROR = "Reconciled: worker terminated before call completion";
|
|
69
|
+
function resolvePlanReviewChairmanCallId(definition) {
|
|
70
|
+
const stage = definition.stages.find((entry) => entry.id === "chairman");
|
|
71
|
+
const callId = stage?.calls[0]?.id;
|
|
72
|
+
if (callId === undefined || stage?.calls.length !== 1) {
|
|
73
|
+
throw new Error("Persisted plan-review workflow definition must have exactly one chairman call");
|
|
74
|
+
}
|
|
75
|
+
return callId;
|
|
76
|
+
}
|
|
77
|
+
async function readPersistedPlanReviewChairmanCallId(runDir, state) {
|
|
78
|
+
const workflowSnapshotPath = join(runDir, state.workflowSnapshotPath ?? WORKFLOW_SNAPSHOT_FILENAME);
|
|
79
|
+
await verifySnapshotHash(workflowSnapshotPath, state.workflowSnapshotHash, "workflow snapshot");
|
|
80
|
+
const definition = await readJsonFile(workflowSnapshotPath);
|
|
81
|
+
return resolvePlanReviewChairmanCallId(definition);
|
|
82
|
+
}
|
|
83
|
+
function withPlanReviewPublication(state, publication) {
|
|
84
|
+
const nextState = structuredClone(state);
|
|
85
|
+
nextState.plan_review_state = PlanReviewRunStateSchema.parse({
|
|
86
|
+
...nextState.plan_review_state,
|
|
87
|
+
publication,
|
|
88
|
+
});
|
|
89
|
+
return nextState;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* C-9 chain-internal finalizer. It never calls a public store writer or
|
|
93
|
+
* enqueueWrite. The committed marker is persisted on the pre-terminal state,
|
|
94
|
+
* after the sidecar and before the caller commits the terminal state.
|
|
95
|
+
*/
|
|
96
|
+
const finalizePlanReviewTerminalState = async ({ runDir, previousState, intendedState, intent, chairmanCallId, serializedResult, }) => {
|
|
97
|
+
const existing = previousState.plan_review_state?.publication;
|
|
98
|
+
if (existing?.committed === true) {
|
|
99
|
+
return { state: withPlanReviewPublication(intendedState, existing) };
|
|
100
|
+
}
|
|
101
|
+
if (!isPlanReviewPublicationEligible(previousState)) {
|
|
102
|
+
return { state: intendedState };
|
|
103
|
+
}
|
|
104
|
+
let restored;
|
|
105
|
+
try {
|
|
106
|
+
const resolvedChairmanCallId = chairmanCallId ??
|
|
107
|
+
await readPersistedPlanReviewChairmanCallId(runDir, previousState);
|
|
108
|
+
restored = await restorePlanReviewRun(runDir, previousState.plan_review_state, resolvedChairmanCallId);
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
112
|
+
const stageError = err instanceof PlanReviewStageError
|
|
113
|
+
? err
|
|
114
|
+
: new PlanReviewStageError("execution_contract", detail, "internal_error");
|
|
115
|
+
const finalStatus = intent.status === "succeeded" ? "failed" : intent.status;
|
|
116
|
+
const failed = structuredClone(intendedState);
|
|
117
|
+
failed.status = finalStatus;
|
|
118
|
+
if (intent.status === "succeeded") {
|
|
119
|
+
failed.error = `Plan-review persisted-state verification failed: ${detail}`;
|
|
120
|
+
}
|
|
121
|
+
failed.plan_review_error = intent.planReviewError ??
|
|
122
|
+
CouncilPlanReviewErrorSchema.parse({
|
|
123
|
+
schema_version: "council-plan-review-error-v1",
|
|
124
|
+
run_id: failed.runId,
|
|
125
|
+
execution_status: finalStatus,
|
|
126
|
+
stage: stageError.stage,
|
|
127
|
+
error: {
|
|
128
|
+
reason: stageError.reason,
|
|
129
|
+
detail: detail.slice(0, 2_048),
|
|
130
|
+
},
|
|
131
|
+
});
|
|
132
|
+
return {
|
|
133
|
+
state: failed,
|
|
134
|
+
...(intent.status === "succeeded"
|
|
135
|
+
? { throwAfterCommit: stageError }
|
|
136
|
+
: {}),
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
try {
|
|
140
|
+
const projected = serializedResult ?? serializePlanReviewResult(await projectPlanReviewRun({
|
|
141
|
+
runDir,
|
|
142
|
+
state: previousState,
|
|
143
|
+
contract: restored.contract,
|
|
144
|
+
execution_contract_digest: restored.execution_contract_digest,
|
|
145
|
+
}));
|
|
146
|
+
const marker = await writeSerializedPlanReviewResultSidecar(runDir, projected);
|
|
147
|
+
// Durable crash boundary: a process death after this write still leaves a
|
|
148
|
+
// contained, marker-addressable result even though terminal intent has not
|
|
149
|
+
// yet been committed.
|
|
150
|
+
const intermediate = withPlanReviewPublication(previousState, marker);
|
|
151
|
+
validateStrictStateSubtrees(intermediate, join(runDir, STATE_FILENAME));
|
|
152
|
+
await writeJsonAtomic(join(runDir, STATE_FILENAME), intermediate);
|
|
153
|
+
return { state: withPlanReviewPublication(intendedState, marker) };
|
|
154
|
+
}
|
|
155
|
+
catch (err) {
|
|
156
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
157
|
+
const finalStatus = intent.status === "succeeded" ? "failed" : intent.status;
|
|
158
|
+
const failed = withPlanReviewPublication(intendedState, { committed: false });
|
|
159
|
+
failed.status = finalStatus;
|
|
160
|
+
if (intent.status === "succeeded") {
|
|
161
|
+
failed.error = `Plan-review publication failed: ${detail}`;
|
|
162
|
+
}
|
|
163
|
+
failed.plan_review_error = createPlanReviewPublicationError(failed.runId, finalStatus, err instanceof PlanReviewStageError ? err.reason : "internal_error", detail);
|
|
164
|
+
return {
|
|
165
|
+
state: failed,
|
|
166
|
+
...(intent.status === "succeeded"
|
|
167
|
+
? {
|
|
168
|
+
throwAfterCommit: new PlanReviewStageError("publication", failed.error, failed.plan_review_error.error.reason),
|
|
169
|
+
}
|
|
170
|
+
: {}),
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
};
|
|
174
|
+
class RunLockOwnershipUnknownError extends Error {
|
|
175
|
+
constructor(lockPath, detail) {
|
|
176
|
+
super(`Run lock ownership is unknown at ${lockPath}; refusing to reclaim it: ${detail}`);
|
|
177
|
+
this.name = "RunLockOwnershipUnknownError";
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Constructed ONLY after a complete, valid lock record has been read and its
|
|
182
|
+
* token proves the lock is no longer ours. Never for open/stat/read/parse/type/
|
|
183
|
+
* size errors — those stay "ordinary" and must not revoke unlink authority.
|
|
184
|
+
*/
|
|
185
|
+
class RunLockOwnershipLostError extends Error {
|
|
186
|
+
constructor(lockPath, detail) {
|
|
187
|
+
super(`Run lock ownership was lost at ${lockPath}: ${detail}`);
|
|
188
|
+
this.name = "RunLockOwnershipLostError";
|
|
189
|
+
}
|
|
190
|
+
}
|
|
22
191
|
export function classifyRunDirectory(runDirPath) {
|
|
192
|
+
return inspectRunDirectory(runDirPath).classification;
|
|
193
|
+
}
|
|
194
|
+
function inspectRunDirectory(runDirPath) {
|
|
23
195
|
try {
|
|
24
196
|
const state = readRunStateSync(runDirPath);
|
|
25
197
|
const lockStatus = classifyRunLockSync(join(runDirPath, RUN_LOCK_FILENAME));
|
|
26
198
|
if (lockStatus === "held") {
|
|
27
|
-
return "active_locked";
|
|
199
|
+
return { classification: "active_locked" };
|
|
28
200
|
}
|
|
29
201
|
if (lockStatus === "unknown") {
|
|
30
|
-
return "unknown";
|
|
202
|
+
return { classification: "unknown" };
|
|
31
203
|
}
|
|
32
204
|
switch (state.status) {
|
|
33
205
|
case "failed":
|
|
34
|
-
return "failed";
|
|
206
|
+
return { classification: "failed", state };
|
|
35
207
|
case "cancelled":
|
|
36
|
-
return "cancelled";
|
|
37
|
-
case "succeeded":
|
|
38
|
-
|
|
208
|
+
return { classification: "cancelled", state };
|
|
209
|
+
case "succeeded": {
|
|
210
|
+
if ((state.debugKeepArtifacts !== undefined && typeof state.debugKeepArtifacts !== "boolean") ||
|
|
211
|
+
(state.async !== undefined && typeof state.async !== "boolean")) {
|
|
212
|
+
return { classification: "unknown" };
|
|
213
|
+
}
|
|
214
|
+
return {
|
|
215
|
+
classification: state.debugKeepArtifacts === true
|
|
216
|
+
? "debug_kept"
|
|
217
|
+
: (state.async === true ? "succeeded_async" : "succeeded_sync"),
|
|
218
|
+
state,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
39
221
|
case "running":
|
|
40
222
|
case "pending":
|
|
41
|
-
return "running_or_pending";
|
|
223
|
+
return { classification: "running_or_pending" };
|
|
42
224
|
default:
|
|
43
|
-
return "unknown";
|
|
225
|
+
return { classification: "unknown" };
|
|
44
226
|
}
|
|
45
227
|
}
|
|
46
228
|
catch {
|
|
47
|
-
return "unknown";
|
|
229
|
+
return { classification: "unknown" };
|
|
48
230
|
}
|
|
49
231
|
}
|
|
50
232
|
export async function pruneRunDirectories(runsRootDir, options) {
|
|
@@ -73,8 +255,17 @@ export async function pruneRunDirectories(runsRootDir, options) {
|
|
|
73
255
|
}
|
|
74
256
|
const runDirPath = join(runsRootDir, entry.name);
|
|
75
257
|
try {
|
|
76
|
-
const classification =
|
|
77
|
-
|
|
258
|
+
const { classification, state } = inspectRunDirectory(runDirPath);
|
|
259
|
+
let retainDays = retentionDaysForClassification(classification, retention);
|
|
260
|
+
if ((classification === "succeeded_async" || classification === "succeeded_sync") &&
|
|
261
|
+
state != null && retainDays != null && retainDays > 0 &&
|
|
262
|
+
computeRunHealth(state, state.context_builder_lifecycle).classification === "degraded") {
|
|
263
|
+
// Zero means unlimited retention, not the smallest TTL. Keep explicit
|
|
264
|
+
// debug policy separate and never shorten a longer success override.
|
|
265
|
+
retainDays = retention.retainFailedDays === 0
|
|
266
|
+
? 0
|
|
267
|
+
: Math.max(retainDays, retention.retainFailedDays);
|
|
268
|
+
}
|
|
78
269
|
if (retainDays == null || retainDays === 0) {
|
|
79
270
|
result.skipped += 1;
|
|
80
271
|
continue;
|
|
@@ -91,9 +282,40 @@ export async function pruneRunDirectories(runsRootDir, options) {
|
|
|
91
282
|
result.skipped += 1;
|
|
92
283
|
continue;
|
|
93
284
|
}
|
|
285
|
+
const identity = await readSessionLogGeneration(runDirPath);
|
|
286
|
+
let sessionLogGeneration;
|
|
287
|
+
if (identity.kind === "available") {
|
|
288
|
+
sessionLogGeneration = identity.sessionLogGeneration;
|
|
289
|
+
}
|
|
290
|
+
else if (identity.kind === "legacy") {
|
|
291
|
+
try {
|
|
292
|
+
sessionLogGeneration = await admitWorkflowRunGeneration(runsRootDir, runDirPath, entry.name);
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
logger.warn({ run_id: entry.name }, "legacy workflow session-log generation could not be admitted before parent pruning");
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
else {
|
|
299
|
+
logger.warn({ run_id: entry.name }, "workflow session-log identity was unsafe before parent pruning");
|
|
300
|
+
}
|
|
94
301
|
await rm(runDirPath, { recursive: true, force: true });
|
|
95
302
|
pruned = true;
|
|
96
303
|
result.pruned += 1;
|
|
304
|
+
if (sessionLogGeneration !== undefined) {
|
|
305
|
+
try {
|
|
306
|
+
const cleanup = await removeSessionLogGeneration({
|
|
307
|
+
rootDir: join(dirname(resolve(runsRootDir)), "session-logs"),
|
|
308
|
+
family: "workflow",
|
|
309
|
+
generation: sessionLogGeneration,
|
|
310
|
+
});
|
|
311
|
+
if (cleanup.kind === "unsafe" || cleanup.kind === "unavailable") {
|
|
312
|
+
logger.warn({ run_id: entry.name, cleanup: cleanup.kind }, "workflow parent pruned but exact session-log cleanup was unavailable");
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
catch {
|
|
316
|
+
logger.warn({ run_id: entry.name }, "workflow parent pruned but exact session-log cleanup failed");
|
|
317
|
+
}
|
|
318
|
+
}
|
|
97
319
|
logger.info({
|
|
98
320
|
run_dir: runDirPath,
|
|
99
321
|
classification,
|
|
@@ -115,8 +337,64 @@ export async function pruneRunDirectories(runsRootDir, options) {
|
|
|
115
337
|
logger.info({ runs_root_dir: runsRootDir, ...result }, "workflow run directory pruning complete");
|
|
116
338
|
return result;
|
|
117
339
|
}
|
|
340
|
+
function isTerminalRunStatus(status) {
|
|
341
|
+
return status === "succeeded" || status === "failed" || status === "cancelled";
|
|
342
|
+
}
|
|
343
|
+
function isPlanReviewRun(state) {
|
|
344
|
+
// Presence-based typed detection (see isPlanReviewRunState): legacy runs
|
|
345
|
+
// persisted under the pre-rename name "council-plan-review" keep routing
|
|
346
|
+
// through the plan-review terminal-intent protocol, while runs of the
|
|
347
|
+
// GENERIC council-plan-review workflow never do.
|
|
348
|
+
return isPlanReviewRunState(state);
|
|
349
|
+
}
|
|
350
|
+
function isStaleRunningRecoveryCheckpoint(state) {
|
|
351
|
+
return state.status === "failed" &&
|
|
352
|
+
state.failureClass === "transient" &&
|
|
353
|
+
state.error === STALE_RUNNING_RECOVERY_ERROR &&
|
|
354
|
+
state.plan_review_error === undefined &&
|
|
355
|
+
state.stages.every((stage) => stage.calls.every((call) => call.state !== "running"));
|
|
356
|
+
}
|
|
357
|
+
function canReplaceStaleRunningRecovery(state, intent) {
|
|
358
|
+
return isStaleRunningRecoveryCheckpoint(state) &&
|
|
359
|
+
intent.status === "failed" &&
|
|
360
|
+
intent.planReviewError?.stage === "execution_contract";
|
|
361
|
+
}
|
|
362
|
+
function applyPlanReviewTerminalIntent(state, intent) {
|
|
363
|
+
const replacingRecovery = canReplaceStaleRunningRecovery(state, intent);
|
|
364
|
+
if (isTerminalRunStatus(state.status) && !replacingRecovery) {
|
|
365
|
+
return undefined;
|
|
366
|
+
}
|
|
367
|
+
const nextState = structuredClone(state);
|
|
368
|
+
nextState.status = intent.status;
|
|
369
|
+
nextState.completedAt = intent.completedAt;
|
|
370
|
+
nextState.error = intent.error;
|
|
371
|
+
if (intent.replaceFailureClass || replacingRecovery) {
|
|
372
|
+
nextState.failureClass = intent.failureClass;
|
|
373
|
+
}
|
|
374
|
+
if (intent.planReviewError === undefined) {
|
|
375
|
+
nextState.plan_review_error = undefined;
|
|
376
|
+
}
|
|
377
|
+
else {
|
|
378
|
+
const planReviewError = CouncilPlanReviewErrorSchema.parse(intent.planReviewError);
|
|
379
|
+
if (planReviewError.run_id !== state.runId) {
|
|
380
|
+
throw new Error(`plan_review_error.run_id ${planReviewError.run_id} does not match run ${state.runId}`);
|
|
381
|
+
}
|
|
382
|
+
if (planReviewError.execution_status !== intent.status) {
|
|
383
|
+
throw new Error(`plan_review_error.execution_status ${planReviewError.execution_status} does not match terminal intent ${intent.status}`);
|
|
384
|
+
}
|
|
385
|
+
nextState.plan_review_error = planReviewError;
|
|
386
|
+
}
|
|
387
|
+
return nextState;
|
|
388
|
+
}
|
|
118
389
|
export class WorkflowStateStore {
|
|
119
390
|
runDir;
|
|
391
|
+
/** Portable journal identity; absent only for a legacy or failed-open run. */
|
|
392
|
+
sessionLogGeneration;
|
|
393
|
+
/**
|
|
394
|
+
* Warn-only duplicate-replay signal detected at create(); undefined on
|
|
395
|
+
* resume and when no permanently-failed identical sibling was found.
|
|
396
|
+
*/
|
|
397
|
+
duplicateOfFailedRun;
|
|
120
398
|
definition;
|
|
121
399
|
variables;
|
|
122
400
|
hooks;
|
|
@@ -126,22 +404,55 @@ export class WorkflowStateStore {
|
|
|
126
404
|
debugKeepArtifacts;
|
|
127
405
|
writeChain = Promise.resolve();
|
|
128
406
|
lockHeld = true;
|
|
407
|
+
planReviewTerminalResult;
|
|
408
|
+
lockToken;
|
|
409
|
+
heartbeat;
|
|
410
|
+
heartbeatInFlight;
|
|
411
|
+
heartbeatError;
|
|
412
|
+
persistInvocationAuditV2;
|
|
413
|
+
persistObservedInvocation;
|
|
129
414
|
constructor(options) {
|
|
130
415
|
this.runDir = options.runDir;
|
|
416
|
+
this.sessionLogGeneration = options.sessionLogGeneration;
|
|
417
|
+
this.duplicateOfFailedRun = options.duplicateOfFailedRun;
|
|
131
418
|
this.definition = options.definition;
|
|
132
419
|
this.variables = options.variables;
|
|
133
420
|
this.state = options.state;
|
|
134
421
|
this.resume = options.resume;
|
|
135
422
|
this.debugKeepArtifacts = options.debugKeepArtifacts;
|
|
136
423
|
this.lockPath = join(options.runDir, RUN_LOCK_FILENAME);
|
|
424
|
+
this.lockToken = options.lockToken;
|
|
425
|
+
this.persistInvocationAuditV2 = invocationAuditV2PersistenceEnabled();
|
|
426
|
+
this.persistObservedInvocation = observedInvocationPersistenceEnabled();
|
|
427
|
+
this.startHeartbeat();
|
|
137
428
|
this.hooks = {
|
|
138
429
|
onRunStateChange: async (event) => {
|
|
139
430
|
await this.enqueueWrite(async () => {
|
|
431
|
+
const cappedError = capPersistedError(event.error);
|
|
432
|
+
if (isPlanReviewRun(this.state) && isTerminalRunStatus(event.state)) {
|
|
433
|
+
await this.finalizePlanReviewTerminalIntent({
|
|
434
|
+
status: event.state,
|
|
435
|
+
completedAt: event.completedAt,
|
|
436
|
+
error: cappedError,
|
|
437
|
+
failureClass: event.failureClass,
|
|
438
|
+
replaceFailureClass: true,
|
|
439
|
+
planReviewError: event.planReviewError,
|
|
440
|
+
});
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
if (isPlanReviewRun(this.state) &&
|
|
444
|
+
isTerminalRunStatus(this.state.status) &&
|
|
445
|
+
!(event.state === "running" &&
|
|
446
|
+
isStaleRunningRecoveryCheckpoint(this.state))) {
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
140
449
|
const nextState = structuredClone(this.state);
|
|
141
450
|
nextState.status = event.state;
|
|
142
|
-
|
|
451
|
+
// First emit to set startedAt wins: a fresh run fills an empty
|
|
452
|
+
// field, and a resume can never overwrite the original run start.
|
|
453
|
+
nextState.startedAt = nextState.startedAt ?? event.startedAt;
|
|
143
454
|
nextState.completedAt = event.completedAt;
|
|
144
|
-
nextState.error =
|
|
455
|
+
nextState.error = cappedError;
|
|
145
456
|
nextState.failureClass = event.failureClass;
|
|
146
457
|
this.state = nextState;
|
|
147
458
|
await this.writeState();
|
|
@@ -156,6 +467,10 @@ export class WorkflowStateStore {
|
|
|
156
467
|
callEntry.materializedInputPath = inputPath;
|
|
157
468
|
callEntry.contextProducing = event.contextProducing;
|
|
158
469
|
callEntry.retryCount = event.retryCount;
|
|
470
|
+
if (this.persistInvocationAuditV2 &&
|
|
471
|
+
event.invocation !== undefined) {
|
|
472
|
+
callEntry.invocation = event.invocation;
|
|
473
|
+
}
|
|
159
474
|
this.state = nextState;
|
|
160
475
|
await this.writeState();
|
|
161
476
|
});
|
|
@@ -166,13 +481,40 @@ export class WorkflowStateStore {
|
|
|
166
481
|
const callEntry = getCallEntry(nextState, event.stageId, event.callId);
|
|
167
482
|
callEntry.state = event.state;
|
|
168
483
|
callEntry.retryCount = event.retryCount;
|
|
484
|
+
if (event.state === "running") {
|
|
485
|
+
callEntry.attemptsConsumed = Math.max(callEntry.attemptsConsumed ?? 0, event.attempt);
|
|
486
|
+
}
|
|
169
487
|
callEntry.startedAt = event.startedAt;
|
|
170
488
|
callEntry.completedAt = event.completedAt;
|
|
171
489
|
callEntry.durationMs = event.durationMs;
|
|
172
|
-
callEntry.errorMessage = event.error;
|
|
173
|
-
|
|
490
|
+
callEntry.errorMessage = capPersistedError(event.error);
|
|
491
|
+
// Two opposite hazards, hence three branches:
|
|
492
|
+
// (1) ERASURE — a terminal emit that omits failureClass would, under
|
|
493
|
+
// an unconditional assignment, wipe a class the engine did
|
|
494
|
+
// classify. Only assign from the event when it carries one.
|
|
495
|
+
// (2) STALE CARRY-OVER — the succeeded emit carries no failureClass
|
|
496
|
+
// (engine.ts), while applyRetryUpdate persists one on every
|
|
497
|
+
// scheduled retry. Without the explicit clear, a call that
|
|
498
|
+
// finally succeeds after a retry stays stamped with the failed
|
|
499
|
+
// attempt's class forever, and is read as such by
|
|
500
|
+
// dashboard/scanner.ts and run-summary-derived.ts.
|
|
501
|
+
// "running" is deliberately in neither branch: a running emit must
|
|
502
|
+
// not clear the prior attempt's class, which resume reads.
|
|
503
|
+
if (event.failureClass !== undefined) {
|
|
504
|
+
callEntry.failureClass = event.failureClass;
|
|
505
|
+
}
|
|
506
|
+
else if (event.state === "succeeded") {
|
|
507
|
+
callEntry.failureClass = undefined;
|
|
508
|
+
}
|
|
174
509
|
callEntry.childDebugMetadata = event.childDebugMetadata;
|
|
175
510
|
callEntry.contextProducing = callEntry.contextProducing ?? false;
|
|
511
|
+
// Observed-invocation telemetry rides only the terminal succeeded
|
|
512
|
+
// emit; running/failed events carry none and must not write it.
|
|
513
|
+
if (this.persistObservedInvocation &&
|
|
514
|
+
event.observedInvocation !== undefined &&
|
|
515
|
+
event.state === "succeeded") {
|
|
516
|
+
callEntry.observedInvocation = event.observedInvocation;
|
|
517
|
+
}
|
|
176
518
|
if (event.outputText != null) {
|
|
177
519
|
const { outputInline, outputPath } = await persistOutput(this.runDir, event.stageId, event.callId, event.outputText);
|
|
178
520
|
callEntry.outputInline = outputInline;
|
|
@@ -205,20 +547,35 @@ export class WorkflowStateStore {
|
|
|
205
547
|
const recycled = await prepareRunDirForCreate(runDir, options.runId);
|
|
206
548
|
let lockHeld = false;
|
|
207
549
|
try {
|
|
208
|
-
await acquireRunLock(runDir);
|
|
550
|
+
const lock = await acquireRunLock(runDir, { declareHeartbeat: true });
|
|
209
551
|
lockHeld = true;
|
|
552
|
+
const sessionLogGeneration = await createSessionLogGeneration(runDir);
|
|
210
553
|
const workflowSnapshotHash = await writeJsonSnapshot(join(runDir, WORKFLOW_SNAPSHOT_FILENAME), options.definition);
|
|
211
554
|
const inputsSnapshotHash = await writeJsonSnapshot(join(runDir, INPUTS_SNAPSHOT_FILENAME), options.variables);
|
|
555
|
+
const duplicateOfFailedRun = await detectDuplicateReplay(runDir, workflowSnapshotHash, inputsSnapshotHash);
|
|
556
|
+
if (duplicateOfFailedRun !== undefined) {
|
|
557
|
+
logger.warn({
|
|
558
|
+
run_id: options.runId,
|
|
559
|
+
duplicate_of_run_id: duplicateOfFailedRun.runId,
|
|
560
|
+
duplicate_completed_at: duplicateOfFailedRun.completedAt,
|
|
561
|
+
}, "run replays workflow and inputs identical to a permanently-failed prior run");
|
|
562
|
+
}
|
|
212
563
|
const childConfigPath = getChildConfigPath();
|
|
213
564
|
const resumable = childConfigPath != null;
|
|
214
565
|
const fingerprintData = resumable
|
|
215
|
-
? await buildChildConfigFingerprint(options.definition)
|
|
566
|
+
? await buildChildConfigFingerprint(options.definition, options.initialContextPolicy, options.variables)
|
|
216
567
|
: undefined;
|
|
217
568
|
if (fingerprintData != null) {
|
|
218
569
|
const childConfigDir = join(runDir, CHILD_CONFIG_DIRNAME);
|
|
219
570
|
await mkdir(childConfigDir, { recursive: true });
|
|
220
571
|
await writeJsonAtomic(join(childConfigDir, CHILD_CONFIG_SNAPSHOT_FILENAME), fingerprintData.snapshot);
|
|
221
572
|
}
|
|
573
|
+
const resolvedRoster = options.resolvedRoster !== undefined
|
|
574
|
+
? ResolvedRosterSchema.parse(options.resolvedRoster)
|
|
575
|
+
: undefined;
|
|
576
|
+
const rosterName = options.rosterName !== undefined
|
|
577
|
+
? RosterNameSchema.parse(options.rosterName)
|
|
578
|
+
: undefined;
|
|
222
579
|
const state = {
|
|
223
580
|
state_version: 1,
|
|
224
581
|
runId: options.runId,
|
|
@@ -242,15 +599,23 @@ export class WorkflowStateStore {
|
|
|
242
599
|
callId: call.id,
|
|
243
600
|
state: "pending",
|
|
244
601
|
retryCount: 0,
|
|
602
|
+
attemptsConsumed: 0,
|
|
245
603
|
contextProducing: isContextProducing(options.definition, stage.id, call.id),
|
|
246
604
|
})),
|
|
247
605
|
})),
|
|
248
606
|
variables: options.variables,
|
|
607
|
+
initialContextPolicy: options.initialContextPolicy,
|
|
249
608
|
debugKeepArtifacts,
|
|
609
|
+
async: options.async ?? false,
|
|
610
|
+
...(resolvedRoster !== undefined ? { resolvedRoster } : {}),
|
|
611
|
+
...(rosterName !== undefined ? { rosterName } : {}),
|
|
250
612
|
};
|
|
251
613
|
await writeJsonAtomic(join(runDir, STATE_FILENAME), state);
|
|
252
|
-
if (recycled) {
|
|
614
|
+
if (recycled.recycled) {
|
|
253
615
|
await rm(`${runDir}.recycled`, { recursive: true, force: true }).catch(() => { });
|
|
616
|
+
if (recycled.retiredSessionLogGeneration !== undefined) {
|
|
617
|
+
await cleanupWorkflowSessionLogGeneration(dirname(runDir), options.runId, recycled.retiredSessionLogGeneration);
|
|
618
|
+
}
|
|
254
619
|
}
|
|
255
620
|
return new WorkflowStateStore({
|
|
256
621
|
runDir,
|
|
@@ -259,13 +624,16 @@ export class WorkflowStateStore {
|
|
|
259
624
|
state,
|
|
260
625
|
resume: {},
|
|
261
626
|
debugKeepArtifacts,
|
|
627
|
+
lockToken: lock.token,
|
|
628
|
+
sessionLogGeneration,
|
|
629
|
+
duplicateOfFailedRun,
|
|
262
630
|
});
|
|
263
631
|
}
|
|
264
632
|
catch (error) {
|
|
265
633
|
if (lockHeld) {
|
|
266
634
|
await safeUnlink(join(runDir, RUN_LOCK_FILENAME));
|
|
267
635
|
}
|
|
268
|
-
if (recycled) {
|
|
636
|
+
if (recycled.recycled) {
|
|
269
637
|
try {
|
|
270
638
|
await rm(runDir, { recursive: true, force: true });
|
|
271
639
|
await rename(`${runDir}.recycled`, runDir);
|
|
@@ -283,9 +651,19 @@ export class WorkflowStateStore {
|
|
|
283
651
|
static async resume(options) {
|
|
284
652
|
const runDir = options.runDir ?? getRunStateDir(options.runId);
|
|
285
653
|
const debugKeepArtifacts = options.debugKeepArtifacts ?? false;
|
|
286
|
-
await acquireRunLock(runDir);
|
|
654
|
+
const lock = await acquireRunLock(runDir, { declareHeartbeat: true });
|
|
287
655
|
try {
|
|
288
656
|
const state = await readRunState(runDir);
|
|
657
|
+
const identity = await readSessionLogGeneration(runDir);
|
|
658
|
+
if (identity.kind === "unavailable") {
|
|
659
|
+
throw new Error(`Run ${options.runId} has an unsafe session-log identity`);
|
|
660
|
+
}
|
|
661
|
+
if (isPlanReviewRun(state) &&
|
|
662
|
+
(state.status === "failed" ||
|
|
663
|
+
state.status === "cancelled" ||
|
|
664
|
+
state.status === "succeeded")) {
|
|
665
|
+
throw new Error(`Run ${options.runId} is terminal (${state.status}) and cannot be resumed; start a new run instead.`);
|
|
666
|
+
}
|
|
289
667
|
if (!state.resumable) {
|
|
290
668
|
throw new Error(`Run ${options.runId} is non-resumable because it was started from ambient child config`);
|
|
291
669
|
}
|
|
@@ -317,10 +695,12 @@ export class WorkflowStateStore {
|
|
|
317
695
|
// Also verify live config hasn't drifted from original run
|
|
318
696
|
const childConfigPath = getChildConfigPath();
|
|
319
697
|
if (childConfigPath != null) {
|
|
320
|
-
const currentFingerprint = await buildChildConfigFingerprint(definition);
|
|
698
|
+
const currentFingerprint = await buildChildConfigFingerprint(definition, state.initialContextPolicy, state.variables);
|
|
321
699
|
if (!fingerprintsMatch(state.fingerprint, currentFingerprint.fingerprint)) {
|
|
322
700
|
throw new Error(`Run ${options.runId} cannot be resumed: live child config at ${childConfigPath} has changed since the original run. ` +
|
|
323
|
-
`Restore the original config or start a new run
|
|
701
|
+
`Restore the original config or start a new run. ` +
|
|
702
|
+
`The fingerprint can also mismatch after a ClinkX upgrade that changes fingerprint composition ` +
|
|
703
|
+
`(for example context-delegate curator/finalizer refs), not only after live adapter JSON edits.`);
|
|
324
704
|
}
|
|
325
705
|
}
|
|
326
706
|
else {
|
|
@@ -361,6 +741,10 @@ export class WorkflowStateStore {
|
|
|
361
741
|
state: recovered,
|
|
362
742
|
resume,
|
|
363
743
|
debugKeepArtifacts,
|
|
744
|
+
lockToken: lock.token,
|
|
745
|
+
...(identity.kind === "available"
|
|
746
|
+
? { sessionLogGeneration: identity.sessionLogGeneration }
|
|
747
|
+
: {}),
|
|
364
748
|
});
|
|
365
749
|
}
|
|
366
750
|
catch (error) {
|
|
@@ -371,8 +755,54 @@ export class WorkflowStateStore {
|
|
|
371
755
|
get runState() {
|
|
372
756
|
return structuredClone(this.state);
|
|
373
757
|
}
|
|
758
|
+
/**
|
|
759
|
+
* Open a new generic resume attempt before any context preflight or provider
|
|
760
|
+
* setup. Provider-call attempt counters remain untouched; only the run-level
|
|
761
|
+
* terminal envelope is cleared so a pre-dispatch failure/cancellation can be
|
|
762
|
+
* persisted as the outcome of this resume attempt.
|
|
763
|
+
*
|
|
764
|
+
* Plan-review stale-run recovery has its own terminal-intent protocol and is
|
|
765
|
+
* deliberately left unchanged here.
|
|
766
|
+
*/
|
|
767
|
+
async beginResumeAttempt() {
|
|
768
|
+
await this.enqueueWrite(async () => {
|
|
769
|
+
if (isPlanReviewRun(this.state))
|
|
770
|
+
return;
|
|
771
|
+
const nextState = structuredClone(this.state);
|
|
772
|
+
if (isTerminalRunStatus(this.state.status)) {
|
|
773
|
+
nextState.previousTerminal = {
|
|
774
|
+
status: this.state.status,
|
|
775
|
+
...(this.state.completedAt === undefined
|
|
776
|
+
? {}
|
|
777
|
+
: { completedAt: this.state.completedAt }),
|
|
778
|
+
...(this.state.error === undefined ? {} : { error: this.state.error }),
|
|
779
|
+
...(this.state.failureClass === undefined
|
|
780
|
+
? {}
|
|
781
|
+
: { failureClass: this.state.failureClass }),
|
|
782
|
+
};
|
|
783
|
+
}
|
|
784
|
+
nextState.status = "pending";
|
|
785
|
+
nextState.completedAt = undefined;
|
|
786
|
+
nextState.error = undefined;
|
|
787
|
+
nextState.failureClass = undefined;
|
|
788
|
+
this.state = nextState;
|
|
789
|
+
await this.writeState();
|
|
790
|
+
});
|
|
791
|
+
}
|
|
374
792
|
async markCancelled(reason = "Workflow cancelled") {
|
|
375
793
|
await this.enqueueWrite(async () => {
|
|
794
|
+
if (isPlanReviewRun(this.state)) {
|
|
795
|
+
await this.finalizePlanReviewTerminalIntent({
|
|
796
|
+
status: "cancelled",
|
|
797
|
+
completedAt: new Date().toISOString(),
|
|
798
|
+
error: reason,
|
|
799
|
+
replaceFailureClass: false,
|
|
800
|
+
});
|
|
801
|
+
return;
|
|
802
|
+
}
|
|
803
|
+
if (this.state.status !== "pending" && this.state.status !== "running") {
|
|
804
|
+
return;
|
|
805
|
+
}
|
|
376
806
|
const nextState = structuredClone(this.state);
|
|
377
807
|
nextState.status = "cancelled";
|
|
378
808
|
nextState.completedAt = new Date().toISOString();
|
|
@@ -381,14 +811,308 @@ export class WorkflowStateStore {
|
|
|
381
811
|
await this.writeState();
|
|
382
812
|
});
|
|
383
813
|
}
|
|
814
|
+
/**
|
|
815
|
+
* Idempotently mark the run failed. Only applies when the current status is
|
|
816
|
+
* non-terminal (pending/running) so it never clobbers a richer engine-emitted
|
|
817
|
+
* terminal status (succeeded/failed/cancelled).
|
|
818
|
+
*/
|
|
819
|
+
async markFailed(error, failureClass, planReviewError) {
|
|
820
|
+
const cappedError = capPersistedError(error);
|
|
821
|
+
await this.enqueueWrite(async () => {
|
|
822
|
+
if (isPlanReviewRun(this.state)) {
|
|
823
|
+
await this.finalizePlanReviewTerminalIntent({
|
|
824
|
+
status: "failed",
|
|
825
|
+
completedAt: new Date().toISOString(),
|
|
826
|
+
error: cappedError,
|
|
827
|
+
failureClass,
|
|
828
|
+
replaceFailureClass: failureClass !== undefined,
|
|
829
|
+
planReviewError,
|
|
830
|
+
});
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
if (this.state.status !== "pending" && this.state.status !== "running") {
|
|
834
|
+
return;
|
|
835
|
+
}
|
|
836
|
+
const nextState = structuredClone(this.state);
|
|
837
|
+
nextState.status = "failed";
|
|
838
|
+
nextState.completedAt = new Date().toISOString();
|
|
839
|
+
nextState.error = cappedError;
|
|
840
|
+
if (failureClass !== undefined) {
|
|
841
|
+
nextState.failureClass = failureClass;
|
|
842
|
+
}
|
|
843
|
+
this.state = nextState;
|
|
844
|
+
await this.writeState();
|
|
845
|
+
});
|
|
846
|
+
}
|
|
384
847
|
async flush() {
|
|
385
848
|
await this.writeChain;
|
|
386
849
|
}
|
|
850
|
+
async commitPreflightContext({ variables, workflow_context, }) {
|
|
851
|
+
await this.enqueueWrite(async () => {
|
|
852
|
+
const lifecycle = this.state.context_builder_lifecycle;
|
|
853
|
+
if (lifecycle !== undefined) {
|
|
854
|
+
if ((lifecycle.status !== "succeeded" && lifecycle.status !== "partial") ||
|
|
855
|
+
lifecycle.phase !== "complete" ||
|
|
856
|
+
lifecycle.artifact_run_id !== workflow_context.context_builder.source_run_id) {
|
|
857
|
+
throw new Error("workflow_context cannot commit before a matching successful lifecycle completes");
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
const committedWorkflowContext = lifecycle === undefined
|
|
861
|
+
? workflow_context
|
|
862
|
+
: {
|
|
863
|
+
...workflow_context,
|
|
864
|
+
context_builder: {
|
|
865
|
+
...workflow_context.context_builder,
|
|
866
|
+
lifecycle: structuredClone(lifecycle),
|
|
867
|
+
},
|
|
868
|
+
};
|
|
869
|
+
const inputsSnapshotPath = this.state.inputsSnapshotPath ?? INPUTS_SNAPSHOT_FILENAME;
|
|
870
|
+
const inputsSnapshotHash = await writeJsonSnapshot(join(this.runDir, inputsSnapshotPath), variables);
|
|
871
|
+
const nextState = structuredClone(this.state);
|
|
872
|
+
nextState.inputsSnapshotPath = inputsSnapshotPath;
|
|
873
|
+
nextState.inputsSnapshotHash = inputsSnapshotHash;
|
|
874
|
+
nextState.variables = variables;
|
|
875
|
+
nextState.workflow_context = structuredClone(committedWorkflowContext);
|
|
876
|
+
this.variables = variables;
|
|
877
|
+
this.state = nextState;
|
|
878
|
+
await this.writeState();
|
|
879
|
+
try {
|
|
880
|
+
await writeJsonAtomic(join(getStateDir(), "workflow-contexts", `${this.state.runId}.json`), committedWorkflowContext);
|
|
881
|
+
}
|
|
882
|
+
catch (error) {
|
|
883
|
+
logger.warn({ runId: this.state.runId, err: error }, "failed to write workflow context provenance");
|
|
884
|
+
}
|
|
885
|
+
});
|
|
886
|
+
}
|
|
887
|
+
async observeContextBuilderLifecycleTransition(transition) {
|
|
888
|
+
const previous = transition.previous === null
|
|
889
|
+
? null
|
|
890
|
+
: parseContextBuilderLifecycleStateV1(transition.previous);
|
|
891
|
+
const snapshot = parseContextBuilderLifecycleStateV1(transition.snapshot);
|
|
892
|
+
if (snapshot.operation_kind !== "workflow_preflight") {
|
|
893
|
+
throw new Error("workflow state accepts only workflow_preflight lifecycle snapshots");
|
|
894
|
+
}
|
|
895
|
+
if (previous !== null && !isLegalContextBuilderLifecycleTransition(previous, snapshot)) {
|
|
896
|
+
throw new Error("illegal workflow context-builder lifecycle transition");
|
|
897
|
+
}
|
|
898
|
+
await this.enqueueWrite(async () => {
|
|
899
|
+
const current = this.state.context_builder_lifecycle;
|
|
900
|
+
if (previous !== null) {
|
|
901
|
+
if (current === undefined || !lifecycleSnapshotsEqual(current, previous)) {
|
|
902
|
+
throw new Error("workflow context-builder lifecycle predecessor mismatch");
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
const nextState = structuredClone(this.state);
|
|
906
|
+
nextState.context_builder_lifecycle = structuredClone(snapshot);
|
|
907
|
+
this.state = nextState;
|
|
908
|
+
await this.writeState();
|
|
909
|
+
});
|
|
910
|
+
}
|
|
911
|
+
/**
|
|
912
|
+
* Persist the plan-review aux subtree (A-5).
|
|
913
|
+
*
|
|
914
|
+
* Called immediately after `create()` and BEFORE any seat can be dispatched.
|
|
915
|
+
* It cannot be a `create()` argument in the runtime path: the subtree binds
|
|
916
|
+
* the canonical snapshot's hash, and the snapshot can only be materialized
|
|
917
|
+
* into a run directory that `create()` itself makes — and the plan requires a
|
|
918
|
+
* snapshot/digest failure to be a POST-state `failed` run, not a pre-state
|
|
919
|
+
* reject. `create()` still validates+persists the subtree when a caller
|
|
920
|
+
* already has one (e.g. a rebuild), so both entry points are strict.
|
|
921
|
+
*/
|
|
922
|
+
async commitPlanReviewState(state) {
|
|
923
|
+
const validated = PlanReviewRunStateSchema.parse(state);
|
|
924
|
+
await this.enqueueWrite(async () => {
|
|
925
|
+
const nextState = structuredClone(this.state);
|
|
926
|
+
nextState.plan_review_state = validated;
|
|
927
|
+
this.state = nextState;
|
|
928
|
+
await this.writeState();
|
|
929
|
+
});
|
|
930
|
+
}
|
|
931
|
+
/**
|
|
932
|
+
* Record that the plan-review context policy was actually enforced at the
|
|
933
|
+
* preflight seam, together with the post-filtered copied paths (A-7).
|
|
934
|
+
*
|
|
935
|
+
* Committed before stage dispatch so a resumed run replays the same enforced
|
|
936
|
+
* exclusion instead of re-deriving it from caller variables.
|
|
937
|
+
*/
|
|
938
|
+
async commitPlanReviewContextProvenance(provenance) {
|
|
939
|
+
await this.enqueueWrite(async () => {
|
|
940
|
+
const current = this.state.plan_review_state;
|
|
941
|
+
if (current === undefined) {
|
|
942
|
+
return;
|
|
943
|
+
}
|
|
944
|
+
const nextState = structuredClone(this.state);
|
|
945
|
+
nextState.plan_review_state = PlanReviewRunStateSchema.parse({
|
|
946
|
+
...current,
|
|
947
|
+
context_provenance: provenance,
|
|
948
|
+
});
|
|
949
|
+
this.state = nextState;
|
|
950
|
+
await this.writeState();
|
|
951
|
+
});
|
|
952
|
+
}
|
|
953
|
+
/** Persisted-state terminal projection used only by the engine's opt-in policy. */
|
|
954
|
+
async projectPlanReviewTerminalResult() {
|
|
955
|
+
if (this.planReviewTerminalResult !== undefined) {
|
|
956
|
+
return this.planReviewTerminalResult;
|
|
957
|
+
}
|
|
958
|
+
const state = this.runState;
|
|
959
|
+
const restored = await restorePlanReviewRun(this.runDir, state.plan_review_state, resolvePlanReviewChairmanCallId(this.definition));
|
|
960
|
+
const envelope = await projectPlanReviewRun({
|
|
961
|
+
runDir: this.runDir,
|
|
962
|
+
state,
|
|
963
|
+
contract: restored.contract,
|
|
964
|
+
execution_contract_digest: restored.execution_contract_digest,
|
|
965
|
+
});
|
|
966
|
+
this.planReviewTerminalResult = serializePlanReviewResult(envelope);
|
|
967
|
+
return this.planReviewTerminalResult;
|
|
968
|
+
}
|
|
969
|
+
/**
|
|
970
|
+
* Refresh this store's own `run.lock` mtime so a reader that cannot verify our
|
|
971
|
+
* PID namespace (a containerized or bind-mounted dashboard) has portable
|
|
972
|
+
* liveness evidence. Observation only — nothing in this module reads the
|
|
973
|
+
* mtime, and no control path consults it.
|
|
974
|
+
*/
|
|
975
|
+
startHeartbeat() {
|
|
976
|
+
this.heartbeat = setInterval(() => {
|
|
977
|
+
// Single-flight: never overlap beats. Skip this tick if we no longer hold
|
|
978
|
+
// the lock, a prior beat latched an error, or a beat is still running. The
|
|
979
|
+
// callback must NOT await — a slow fstat/read must not stall the interval.
|
|
980
|
+
if (!this.lockHeld ||
|
|
981
|
+
this.heartbeatError != null ||
|
|
982
|
+
this.heartbeatInFlight != null) {
|
|
983
|
+
return;
|
|
984
|
+
}
|
|
985
|
+
const beat = this.beatRunLock()
|
|
986
|
+
.catch((error) => {
|
|
987
|
+
// Losing liveness REPORTING must never lose the run: latch and log, and
|
|
988
|
+
// never let this reach enqueueWrite. This deliberately diverges from
|
|
989
|
+
// credential-store's assertHealthy(), where the lock protects a shared
|
|
990
|
+
// file; here it only decorates a dashboard observation.
|
|
991
|
+
if (error instanceof RunLockOwnershipLostError) {
|
|
992
|
+
// PROVEN ownership loss (a complete valid record whose token is not
|
|
993
|
+
// ours): revoke unlink authority BEFORE latching/logging so close()
|
|
994
|
+
// cannot delete a lock a different owner now holds.
|
|
995
|
+
this.lockHeld = false;
|
|
996
|
+
}
|
|
997
|
+
// ENOENT / EACCES / EIO / ELOOP / non-regular-file / oversized /
|
|
998
|
+
// malformed JSON / invalid record / short read stay "ordinary" and DO
|
|
999
|
+
// NOT clear lockHeld. That is safe ONLY under the confirmed
|
|
1000
|
+
// SINGLE-PRODUCER topology: no other writer holds this exact lock, so a
|
|
1001
|
+
// transient read failure is not evidence of a foreign owner and ENOENT
|
|
1002
|
+
// is our own recycled/removed path, not a takeover. A future
|
|
1003
|
+
// multi-producer change MUST revisit this asymmetry.
|
|
1004
|
+
//
|
|
1005
|
+
// Latching is PERMANENT for this run: a single ordinary transient
|
|
1006
|
+
// failure (e.g. a rejected utimes / EIO) halts THIS run's heartbeat
|
|
1007
|
+
// with NO retry — retry was deliberately omitted as over-engineering.
|
|
1008
|
+
// A foreign-host dashboard may therefore age a live-but-transiently-
|
|
1009
|
+
// erroring run to `orphaned` and never self-correct until the run ends.
|
|
1010
|
+
// That is fail-safe: it only stops a liveness DECORATION, never mutates
|
|
1011
|
+
// run state and never fails the run.
|
|
1012
|
+
this.heartbeatError = error instanceof Error ? error : new Error(String(error));
|
|
1013
|
+
logger.warn({ run_dir: this.runDir, err: this.heartbeatError.message }, "workflow run lock heartbeat stopped");
|
|
1014
|
+
})
|
|
1015
|
+
.finally(() => {
|
|
1016
|
+
// Identity guard: only an obsolete completion of the CURRENT in-flight
|
|
1017
|
+
// beat clears the slot; a newer promise is never clobbered.
|
|
1018
|
+
if (this.heartbeatInFlight === beat) {
|
|
1019
|
+
this.heartbeatInFlight = undefined;
|
|
1020
|
+
}
|
|
1021
|
+
});
|
|
1022
|
+
this.heartbeatInFlight = beat;
|
|
1023
|
+
}, RUN_LOCK_HEARTBEAT_MS);
|
|
1024
|
+
this.heartbeat.unref();
|
|
1025
|
+
}
|
|
1026
|
+
/**
|
|
1027
|
+
* One descriptor-bound heartbeat beat: open the lock O_NOFOLLOW read-only,
|
|
1028
|
+
* fstat + bounded positional read through the SAME handle, verify it is still
|
|
1029
|
+
* our token, then utimes that still-open handle. Throws
|
|
1030
|
+
* `RunLockOwnershipLostError` only on a complete valid record whose token
|
|
1031
|
+
* differs; every other failure is an ordinary Error.
|
|
1032
|
+
*/
|
|
1033
|
+
async beatRunLock() {
|
|
1034
|
+
if (!this.lockHeld || this.heartbeatError != null)
|
|
1035
|
+
return;
|
|
1036
|
+
const handle = await open(this.lockPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
1037
|
+
let propagating;
|
|
1038
|
+
try {
|
|
1039
|
+
const stats = await handle.stat();
|
|
1040
|
+
if (!stats.isFile()) {
|
|
1041
|
+
throw new Error(`run.lock at ${this.lockPath} is not a regular file`);
|
|
1042
|
+
}
|
|
1043
|
+
if (stats.size > RUN_LOCK_MAX_BYTES) {
|
|
1044
|
+
throw new Error(`run.lock at ${this.lockPath} exceeds ${RUN_LOCK_MAX_BYTES} bytes`);
|
|
1045
|
+
}
|
|
1046
|
+
const size = Number(stats.size);
|
|
1047
|
+
const buffer = Buffer.allocUnsafe(size);
|
|
1048
|
+
let offset = 0;
|
|
1049
|
+
while (offset < size) {
|
|
1050
|
+
const { bytesRead } = await handle.read(buffer, offset, size - offset, offset);
|
|
1051
|
+
if (bytesRead === 0) {
|
|
1052
|
+
throw new Error(`run.lock at ${this.lockPath} ended prematurely after ${offset} of ${size} bytes`);
|
|
1053
|
+
}
|
|
1054
|
+
offset += bytesRead;
|
|
1055
|
+
}
|
|
1056
|
+
const value = JSON.parse(buffer.toString("utf-8"));
|
|
1057
|
+
if (!isRunLockRecord(value)) {
|
|
1058
|
+
throw new Error(`run.lock at ${this.lockPath} is not a valid lock record`);
|
|
1059
|
+
}
|
|
1060
|
+
if (typeof value.token !== "string") {
|
|
1061
|
+
// A present-but-mistyped token is a corrupt/invalid record, NOT proven
|
|
1062
|
+
// ownership loss: treat it as an ordinary heartbeat error so it does not
|
|
1063
|
+
// revoke unlink authority. Under single-producer this producer always
|
|
1064
|
+
// writes a string token; this only guards hostile/corrupt lock bodies.
|
|
1065
|
+
throw new Error(`run.lock token is not a string at ${this.lockPath}`);
|
|
1066
|
+
}
|
|
1067
|
+
if (value.token !== this.lockToken) {
|
|
1068
|
+
throw new RunLockOwnershipLostError(this.lockPath, "lock token no longer matches this store");
|
|
1069
|
+
}
|
|
1070
|
+
const now = new Date();
|
|
1071
|
+
await handle.utimes(now, now);
|
|
1072
|
+
}
|
|
1073
|
+
catch (error) {
|
|
1074
|
+
propagating = error;
|
|
1075
|
+
throw error;
|
|
1076
|
+
}
|
|
1077
|
+
finally {
|
|
1078
|
+
try {
|
|
1079
|
+
await handle.close();
|
|
1080
|
+
}
|
|
1081
|
+
catch (closeError) {
|
|
1082
|
+
// Swallow a close failure only when another error is already
|
|
1083
|
+
// propagating; otherwise surface it.
|
|
1084
|
+
if (propagating === undefined)
|
|
1085
|
+
throw closeError;
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
async stopHeartbeat() {
|
|
1090
|
+
if (this.heartbeat !== undefined) {
|
|
1091
|
+
clearInterval(this.heartbeat);
|
|
1092
|
+
this.heartbeat = undefined;
|
|
1093
|
+
}
|
|
1094
|
+
// At most one beat is ever active. Await it, swallowing the already-latched
|
|
1095
|
+
// failure; do NOT loop.
|
|
1096
|
+
const inFlight = this.heartbeatInFlight;
|
|
1097
|
+
if (inFlight !== undefined) {
|
|
1098
|
+
await inFlight.catch(() => { });
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
387
1101
|
async close() {
|
|
388
1102
|
await this.flush();
|
|
1103
|
+
// Ordering is load-bearing in both directions. Not before flush(): draining
|
|
1104
|
+
// the write chain can take time, and a lock aging while the run is provably
|
|
1105
|
+
// alive reads `orphaned` to a foreign host. Not after the unlink: a beat
|
|
1106
|
+
// racing the unlink would utimes a deleted — or recycled — path.
|
|
1107
|
+
await this.stopHeartbeat();
|
|
389
1108
|
if (this.state.status === "succeeded" && !this.debugKeepArtifacts) {
|
|
390
|
-
|
|
391
|
-
|
|
1109
|
+
// Successful runs remain observable until their retention window expires.
|
|
1110
|
+
// Release ownership so classification/pruning and immediate run-ID recycling
|
|
1111
|
+
// can safely operate on both synchronous and asynchronous successes.
|
|
1112
|
+
if (this.lockHeld) {
|
|
1113
|
+
await safeUnlink(this.lockPath);
|
|
1114
|
+
this.lockHeld = false;
|
|
1115
|
+
}
|
|
392
1116
|
return;
|
|
393
1117
|
}
|
|
394
1118
|
if (this.lockHeld) {
|
|
@@ -401,12 +1125,34 @@ export class WorkflowStateStore {
|
|
|
401
1125
|
const nextState = structuredClone(this.state);
|
|
402
1126
|
const callEntry = getCallEntry(nextState, event.stageId, event.callId);
|
|
403
1127
|
callEntry.retryCount = event.nextRetryCount;
|
|
404
|
-
callEntry.errorMessage = event.error;
|
|
1128
|
+
callEntry.errorMessage = capPersistedError(event.error);
|
|
405
1129
|
callEntry.failureClass = event.failureClass;
|
|
406
1130
|
this.state = nextState;
|
|
407
1131
|
await this.writeState();
|
|
408
1132
|
});
|
|
409
1133
|
}
|
|
1134
|
+
/** Runs inside the caller's existing queue task; never enqueue from here. */
|
|
1135
|
+
async finalizePlanReviewTerminalIntent(intent) {
|
|
1136
|
+
const previousState = this.state;
|
|
1137
|
+
const intended = applyPlanReviewTerminalIntent(previousState, intent);
|
|
1138
|
+
if (intended === undefined) {
|
|
1139
|
+
return;
|
|
1140
|
+
}
|
|
1141
|
+
const finalized = await finalizePlanReviewTerminalState({
|
|
1142
|
+
runDir: this.runDir,
|
|
1143
|
+
previousState,
|
|
1144
|
+
intendedState: intended,
|
|
1145
|
+
intent,
|
|
1146
|
+
chairmanCallId: resolvePlanReviewChairmanCallId(this.definition),
|
|
1147
|
+
serializedResult: this.planReviewTerminalResult,
|
|
1148
|
+
});
|
|
1149
|
+
validateStrictStateSubtrees(finalized.state, join(this.runDir, STATE_FILENAME));
|
|
1150
|
+
this.state = finalized.state;
|
|
1151
|
+
await this.writeState();
|
|
1152
|
+
if (finalized.throwAfterCommit !== undefined) {
|
|
1153
|
+
throw finalized.throwAfterCommit;
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
410
1156
|
async enqueueWrite(task) {
|
|
411
1157
|
const next = this.writeChain.then(task, task);
|
|
412
1158
|
this.writeChain = next.catch(() => { });
|
|
@@ -420,7 +1166,7 @@ async function prepareRunDirForCreate(runDir, runId) {
|
|
|
420
1166
|
await mkdir(dirname(runDir), { recursive: true });
|
|
421
1167
|
try {
|
|
422
1168
|
await mkdir(runDir);
|
|
423
|
-
return false;
|
|
1169
|
+
return { recycled: false };
|
|
424
1170
|
}
|
|
425
1171
|
catch (error) {
|
|
426
1172
|
if (!(error instanceof Error) || !("code" in error) || error.code !== "EEXIST") {
|
|
@@ -428,7 +1174,7 @@ async function prepareRunDirForCreate(runDir, runId) {
|
|
|
428
1174
|
}
|
|
429
1175
|
}
|
|
430
1176
|
const recycled = await recycleSucceededRunDir(runDir);
|
|
431
|
-
if (recycled) {
|
|
1177
|
+
if (recycled.recycled) {
|
|
432
1178
|
const recycledPath = `${runDir}.recycled`;
|
|
433
1179
|
try {
|
|
434
1180
|
await mkdir(runDir);
|
|
@@ -445,7 +1191,7 @@ async function prepareRunDirForCreate(runDir, runId) {
|
|
|
445
1191
|
}
|
|
446
1192
|
throw error;
|
|
447
1193
|
}
|
|
448
|
-
return
|
|
1194
|
+
return recycled;
|
|
449
1195
|
}
|
|
450
1196
|
throw duplicateRunDirError(runId);
|
|
451
1197
|
}
|
|
@@ -453,6 +1199,14 @@ async function pruneRunDirectoriesBeforeCreate(runDir) {
|
|
|
453
1199
|
const runsRoot = getRunsDir();
|
|
454
1200
|
if (dirname(runDir) !== runsRoot)
|
|
455
1201
|
return;
|
|
1202
|
+
// Reconcile before pruning so a stale running run terminalized here becomes
|
|
1203
|
+
// prunable by a later create once past its retention window.
|
|
1204
|
+
try {
|
|
1205
|
+
await reconcileStaleRunDirectories(runsRoot);
|
|
1206
|
+
}
|
|
1207
|
+
catch (error) {
|
|
1208
|
+
logger.warn({ runs_root_dir: runsRoot, err: error instanceof Error ? error.message : String(error) }, "stale workflow run reconcile failed before create");
|
|
1209
|
+
}
|
|
456
1210
|
try {
|
|
457
1211
|
await pruneRunDirectories(runsRoot, getRetentionConfig());
|
|
458
1212
|
}
|
|
@@ -460,11 +1214,89 @@ async function pruneRunDirectoriesBeforeCreate(runDir) {
|
|
|
460
1214
|
logger.warn({ runs_root_dir: runsRoot, err: error instanceof Error ? error.message : String(error) }, "workflow run directory pruning failed before create");
|
|
461
1215
|
}
|
|
462
1216
|
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Bounds the per-create duplicate-replay scan: at most this many sibling run
|
|
1219
|
+
* directories have their persisted state read. Best-effort by design — a runs
|
|
1220
|
+
* root larger than the cap simply yields a possibly-incomplete signal.
|
|
1221
|
+
*/
|
|
1222
|
+
const MAX_DUPLICATE_REPLAY_SCAN_DIRS = 1_000;
|
|
1223
|
+
/**
|
|
1224
|
+
* Warn-only duplicate-replay detection (failure investigation F6): find a
|
|
1225
|
+
* sibling run whose persisted `workflowSnapshotHash` AND `inputsSnapshotHash`
|
|
1226
|
+
* both equal the run being created, and whose status is terminal `failed`
|
|
1227
|
+
* with `failureClass:"permanent"`. Cancelled runs and transient failures are
|
|
1228
|
+
* deliberately excluded; with multiple matches the most recent `completedAt`
|
|
1229
|
+
* wins.
|
|
1230
|
+
*
|
|
1231
|
+
* WARN-ONLY, never a gate: identical hashes do NOT imply an identical outcome.
|
|
1232
|
+
* The hashes cover only the workflow definition and resolved input variables —
|
|
1233
|
+
* not workspace content, which may have changed since the prior run — and
|
|
1234
|
+
* providers are nondeterministic, so an exact replay can legitimately succeed.
|
|
1235
|
+
* This function therefore never throws and never blocks creation: any scan or
|
|
1236
|
+
* per-sibling read error degrades silently (or with a warn log) to "no
|
|
1237
|
+
* signal", and admission, mutex, cap, and state-creation behavior are
|
|
1238
|
+
* unchanged.
|
|
1239
|
+
*/
|
|
1240
|
+
async function detectDuplicateReplay(runDir, workflowSnapshotHash, inputsSnapshotHash) {
|
|
1241
|
+
try {
|
|
1242
|
+
const runsRoot = dirname(runDir);
|
|
1243
|
+
const selfName = basename(runDir);
|
|
1244
|
+
let entries;
|
|
1245
|
+
try {
|
|
1246
|
+
entries = await readdir(runsRoot, { withFileTypes: true });
|
|
1247
|
+
}
|
|
1248
|
+
catch {
|
|
1249
|
+
return undefined;
|
|
1250
|
+
}
|
|
1251
|
+
let best;
|
|
1252
|
+
let bestCompletedAtMs = Number.NEGATIVE_INFINITY;
|
|
1253
|
+
let examined = 0;
|
|
1254
|
+
for (const entry of entries) {
|
|
1255
|
+
if (!entry.isDirectory() || entry.name === selfName)
|
|
1256
|
+
continue;
|
|
1257
|
+
if (examined >= MAX_DUPLICATE_REPLAY_SCAN_DIRS)
|
|
1258
|
+
break;
|
|
1259
|
+
examined += 1;
|
|
1260
|
+
let state;
|
|
1261
|
+
try {
|
|
1262
|
+
state = readRunStateSync(join(runsRoot, entry.name));
|
|
1263
|
+
}
|
|
1264
|
+
catch {
|
|
1265
|
+
// Unreadable siblings are skipped silently — mid-write, corrupt, or
|
|
1266
|
+
// foreign directories must never affect the new run.
|
|
1267
|
+
continue;
|
|
1268
|
+
}
|
|
1269
|
+
if (state.status !== "failed" ||
|
|
1270
|
+
state.failureClass !== "permanent" ||
|
|
1271
|
+
state.workflowSnapshotHash !== workflowSnapshotHash ||
|
|
1272
|
+
state.inputsSnapshotHash !== inputsSnapshotHash) {
|
|
1273
|
+
continue;
|
|
1274
|
+
}
|
|
1275
|
+
const parsedCompletedAt = state.completedAt !== undefined
|
|
1276
|
+
? Date.parse(state.completedAt)
|
|
1277
|
+
: Number.NaN;
|
|
1278
|
+
const orderKey = Number.isFinite(parsedCompletedAt)
|
|
1279
|
+
? parsedCompletedAt
|
|
1280
|
+
: Number.NEGATIVE_INFINITY;
|
|
1281
|
+
if (best === undefined || orderKey > bestCompletedAtMs) {
|
|
1282
|
+
best = { runId: state.runId, completedAt: state.completedAt };
|
|
1283
|
+
bestCompletedAtMs = orderKey;
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
return best;
|
|
1287
|
+
}
|
|
1288
|
+
catch (error) {
|
|
1289
|
+
logger.warn({ run_dir: runDir, err: error instanceof Error ? error.message : String(error) }, "duplicate-replay scan failed; proceeding without signal");
|
|
1290
|
+
return undefined;
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
463
1293
|
function normalizePruneOptions(options) {
|
|
464
1294
|
return {
|
|
465
1295
|
retainFailedDays: normalizeRetentionDays(options?.retainFailedDays, DEFAULT_RETAIN_FAILED_DAYS),
|
|
466
1296
|
retainCancelledDays: normalizeRetentionDays(options?.retainCancelledDays, DEFAULT_RETAIN_CANCELLED_DAYS),
|
|
467
1297
|
retainDebugDays: normalizeRetentionDays(options?.retainDebugDays, DEFAULT_RETAIN_DEBUG_DAYS),
|
|
1298
|
+
retainAsyncDays: normalizeRetentionDays(options?.retainAsyncDays, DEFAULT_RETAIN_ASYNC_DAYS),
|
|
1299
|
+
retainSucceededDays: normalizeRetentionDays(options?.retainSucceededDays, DEFAULT_RETAIN_SUCCEEDED_DAYS),
|
|
468
1300
|
};
|
|
469
1301
|
}
|
|
470
1302
|
function normalizeRetentionDays(value, fallback) {
|
|
@@ -478,6 +1310,10 @@ function retentionDaysForClassification(classification, options) {
|
|
|
478
1310
|
return options.retainCancelledDays;
|
|
479
1311
|
case "debug_kept":
|
|
480
1312
|
return options.retainDebugDays;
|
|
1313
|
+
case "succeeded_async":
|
|
1314
|
+
return options.retainAsyncDays;
|
|
1315
|
+
case "succeeded_sync":
|
|
1316
|
+
return options.retainSucceededDays;
|
|
481
1317
|
case "active_locked":
|
|
482
1318
|
case "running_or_pending":
|
|
483
1319
|
case "unknown":
|
|
@@ -493,6 +1329,44 @@ async function tryAcquirePruneLock(runDirPath) {
|
|
|
493
1329
|
return false;
|
|
494
1330
|
}
|
|
495
1331
|
}
|
|
1332
|
+
async function admitWorkflowRunGeneration(runsRootDir, runDirPath, runId) {
|
|
1333
|
+
const requestedRoot = resolve(runsRootDir);
|
|
1334
|
+
const requestedRun = resolve(runDirPath);
|
|
1335
|
+
if (requestedRun !== join(requestedRoot, runId)) {
|
|
1336
|
+
throw new Error("workflow run identity is not contained");
|
|
1337
|
+
}
|
|
1338
|
+
const rootLinked = await lstat(requestedRoot);
|
|
1339
|
+
if (rootLinked.isSymbolicLink() || !rootLinked.isDirectory()) {
|
|
1340
|
+
throw new Error("workflow runs root identity is unsafe");
|
|
1341
|
+
}
|
|
1342
|
+
const canonicalRoot = await realpath(requestedRoot);
|
|
1343
|
+
const canonicalRun = join(canonicalRoot, runId);
|
|
1344
|
+
const flags = fsConstants.O_RDONLY |
|
|
1345
|
+
(fsConstants.O_DIRECTORY ?? 0) |
|
|
1346
|
+
(fsConstants.O_NOFOLLOW ?? 0);
|
|
1347
|
+
const rootHandle = await open(canonicalRoot, flags);
|
|
1348
|
+
let runHandle;
|
|
1349
|
+
try {
|
|
1350
|
+
runHandle = await open(canonicalRun, flags);
|
|
1351
|
+
const rootStats = await rootHandle.stat({ bigint: true });
|
|
1352
|
+
const runStats = await runHandle.stat({ bigint: true });
|
|
1353
|
+
if (!rootStats.isDirectory() ||
|
|
1354
|
+
!runStats.isDirectory() ||
|
|
1355
|
+
rootStats.nlink === 0n ||
|
|
1356
|
+
runStats.nlink === 0n ||
|
|
1357
|
+
BigInt(rootLinked.dev) !== rootStats.dev ||
|
|
1358
|
+
BigInt(rootLinked.ino) !== rootStats.ino ||
|
|
1359
|
+
await resolveOpenedDescriptorPath(rootHandle.fd, canonicalRoot) !== canonicalRoot ||
|
|
1360
|
+
await resolveOpenedDescriptorPath(runHandle.fd, canonicalRun) !== canonicalRun) {
|
|
1361
|
+
throw new Error("workflow run identity changed during admission");
|
|
1362
|
+
}
|
|
1363
|
+
return deriveWorkflowRunGeneration(runId, rootStats, runStats);
|
|
1364
|
+
}
|
|
1365
|
+
finally {
|
|
1366
|
+
await runHandle?.close().catch(() => undefined);
|
|
1367
|
+
await rootHandle.close().catch(() => undefined);
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
496
1370
|
async function getRunDirectoryAgeMs(runDirPath) {
|
|
497
1371
|
const completedAt = await readRunCompletedAt(runDirPath);
|
|
498
1372
|
if (completedAt != null) {
|
|
@@ -522,8 +1396,101 @@ function readRunStateSync(runDir) {
|
|
|
522
1396
|
if (state.state_version !== 1) {
|
|
523
1397
|
throw new Error(`Unsupported state_version ${String(state.state_version)} in ${statePath}`);
|
|
524
1398
|
}
|
|
1399
|
+
validateStrictStateSubtrees(state, statePath);
|
|
525
1400
|
return state;
|
|
526
1401
|
}
|
|
1402
|
+
/**
|
|
1403
|
+
* Strictly validate optional state subtrees that are resume or lifecycle
|
|
1404
|
+
* authorities. Older state_version:1 files may omit either subtree.
|
|
1405
|
+
*
|
|
1406
|
+
* The rest of RunState is still validated only by `state_version`. These are
|
|
1407
|
+
* exceptions because resume derives execution bindings from plan_review_state,
|
|
1408
|
+
* while lifecycle is a strict bounded H0 contract and must never acquire
|
|
1409
|
+
* arbitrary producer fields.
|
|
1410
|
+
*/
|
|
1411
|
+
function validateStrictStateSubtrees(state, statePath) {
|
|
1412
|
+
if (state.workflow_context !== undefined) {
|
|
1413
|
+
try {
|
|
1414
|
+
state = {
|
|
1415
|
+
...state,
|
|
1416
|
+
workflow_context: parseWorkflowContext(state.workflow_context),
|
|
1417
|
+
};
|
|
1418
|
+
}
|
|
1419
|
+
catch (error) {
|
|
1420
|
+
throw new Error(`Invalid workflow_context in ${statePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
let parsedLifecycle;
|
|
1424
|
+
if (state.context_builder_lifecycle !== undefined) {
|
|
1425
|
+
try {
|
|
1426
|
+
parsedLifecycle = parseContextBuilderLifecycleStateV1(state.context_builder_lifecycle);
|
|
1427
|
+
}
|
|
1428
|
+
catch (error) {
|
|
1429
|
+
throw new Error(`Invalid context_builder_lifecycle in ${statePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
const nestedLifecycle = state.workflow_context?.context_builder.lifecycle;
|
|
1433
|
+
let parsedNestedLifecycle;
|
|
1434
|
+
if (nestedLifecycle !== undefined) {
|
|
1435
|
+
try {
|
|
1436
|
+
parsedNestedLifecycle = parseContextBuilderLifecycleStateV1(nestedLifecycle);
|
|
1437
|
+
}
|
|
1438
|
+
catch (error) {
|
|
1439
|
+
throw new Error(`Invalid workflow_context.context_builder.lifecycle in ${statePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
if (state.workflow_context !== undefined) {
|
|
1443
|
+
const sourceRunId = state.workflow_context.context_builder.source_run_id;
|
|
1444
|
+
for (const lifecycle of [parsedLifecycle, parsedNestedLifecycle]) {
|
|
1445
|
+
if (lifecycle !== undefined &&
|
|
1446
|
+
((lifecycle.status !== "succeeded" && lifecycle.status !== "partial") ||
|
|
1447
|
+
lifecycle.phase !== "complete" ||
|
|
1448
|
+
lifecycle.artifact_run_id !== sourceRunId)) {
|
|
1449
|
+
throw new Error(`Inconsistent workflow context lifecycle in ${statePath}`);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
if (parsedLifecycle !== undefined &&
|
|
1453
|
+
parsedNestedLifecycle !== undefined &&
|
|
1454
|
+
!lifecycleSnapshotsEqual(parsedLifecycle, parsedNestedLifecycle)) {
|
|
1455
|
+
throw new Error(`Inconsistent workflow context lifecycle in ${statePath}`);
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
if (state.plan_review_state !== undefined) {
|
|
1459
|
+
const parsed = PlanReviewRunStateSchema.safeParse(state.plan_review_state);
|
|
1460
|
+
if (!parsed.success) {
|
|
1461
|
+
throw new Error(`Invalid plan_review_state in ${statePath}: ${parsed.error.issues
|
|
1462
|
+
.map((issue) => `${issue.path.join(".") || "plan_review_state"}: ${issue.message}`)
|
|
1463
|
+
.join("; ")}`);
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
if (state.plan_review_error !== undefined) {
|
|
1467
|
+
const parsed = CouncilPlanReviewErrorSchema.safeParse(state.plan_review_error);
|
|
1468
|
+
if (!parsed.success) {
|
|
1469
|
+
throw new Error(`Invalid plan_review_error in ${statePath}: ${parsed.error.issues
|
|
1470
|
+
.map((issue) => `${issue.path.join(".") || "plan_review_error"}: ${issue.message}`)
|
|
1471
|
+
.join("; ")}`);
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
if (state.resolvedRoster !== undefined) {
|
|
1475
|
+
const parsed = ResolvedRosterSchema.safeParse(state.resolvedRoster);
|
|
1476
|
+
if (!parsed.success) {
|
|
1477
|
+
throw new Error(`Invalid resolvedRoster in ${statePath}: ${parsed.error.issues
|
|
1478
|
+
.map((issue) => `${issue.path.join(".") || "resolvedRoster"}: ${issue.message}`)
|
|
1479
|
+
.join("; ")}`);
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
if (state.rosterName !== undefined) {
|
|
1483
|
+
const parsed = RosterNameSchema.safeParse(state.rosterName);
|
|
1484
|
+
if (!parsed.success) {
|
|
1485
|
+
throw new Error(`Invalid rosterName in ${statePath}: ${parsed.error.issues
|
|
1486
|
+
.map((issue) => `${issue.path.join(".") || "rosterName"}: ${issue.message}`)
|
|
1487
|
+
.join("; ")}`);
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
function lifecycleSnapshotsEqual(left, right) {
|
|
1492
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
1493
|
+
}
|
|
527
1494
|
function classifyRunLockSync(lockPath) {
|
|
528
1495
|
try {
|
|
529
1496
|
const lock = JSON.parse(readFileSync(lockPath, "utf-8"));
|
|
@@ -540,11 +1507,275 @@ function classifyRunLockSync(lockPath) {
|
|
|
540
1507
|
return code === "ENOENT" ? "stale_or_absent" : "unknown";
|
|
541
1508
|
}
|
|
542
1509
|
}
|
|
1510
|
+
export async function readRunStateForStatus(runId) {
|
|
1511
|
+
const runDir = getRunStateDir(runId);
|
|
1512
|
+
let state;
|
|
1513
|
+
try {
|
|
1514
|
+
state = await readRunState(runDir);
|
|
1515
|
+
}
|
|
1516
|
+
catch (error) {
|
|
1517
|
+
const code = error.code;
|
|
1518
|
+
if (code === "ENOENT") {
|
|
1519
|
+
return null;
|
|
1520
|
+
}
|
|
1521
|
+
throw error;
|
|
1522
|
+
}
|
|
1523
|
+
const lockStatus = classifyRunLockSync(join(runDir, RUN_LOCK_FILENAME));
|
|
1524
|
+
return {
|
|
1525
|
+
state,
|
|
1526
|
+
lock: lockStatus,
|
|
1527
|
+
};
|
|
1528
|
+
}
|
|
1529
|
+
/**
|
|
1530
|
+
* Reap a run that has no live worker (orphaned/stale/prior-process) by writing a
|
|
1531
|
+
* terminal "cancelled" state and unlinking its lock. Used by cancel_workflow when
|
|
1532
|
+
* the run is not in this process's in-memory cancel registry.
|
|
1533
|
+
*
|
|
1534
|
+
* The run.lock is the single source of truth for ownership: this acquires the
|
|
1535
|
+
* lock itself (reusing acquireRunLock, which atomically creates the lock and
|
|
1536
|
+
* reclaims a stale same-host/dead-PID lock) BEFORE mutating, closing the TOCTOU
|
|
1537
|
+
* where a concurrent resume_workflow could grab the lock between a caller's
|
|
1538
|
+
* status read and the reap write.
|
|
1539
|
+
*
|
|
1540
|
+
* Returns:
|
|
1541
|
+
* - "locked" — a LIVE lock blocks acquisition (held by a running worker, or a
|
|
1542
|
+
* same-process blocking run); nothing is mutated.
|
|
1543
|
+
* - "ownership_unknown" — the lock is malformed/unreadable; nothing is mutated
|
|
1544
|
+
* or reclaimed.
|
|
1545
|
+
* - "skipped" — lock acquired but the run is already terminal (not
|
|
1546
|
+
* pending/running); the lock is released and nothing is mutated.
|
|
1547
|
+
* The caller re-reads to report already_terminal.
|
|
1548
|
+
* - "cancelled" — lock acquired and run was pending/running; terminal
|
|
1549
|
+
* "cancelled" state is written and the lock removed.
|
|
1550
|
+
*
|
|
1551
|
+
* NOTE: This intentionally does NOT use WorkflowStateStore.resume() (which
|
|
1552
|
+
* recovers + re-locks the run — the wrong intent here).
|
|
1553
|
+
*/
|
|
1554
|
+
export async function reapRunCancelled(runId) {
|
|
1555
|
+
const runDir = getRunStateDir(runId);
|
|
1556
|
+
try {
|
|
1557
|
+
await acquireRunLock(runDir);
|
|
1558
|
+
}
|
|
1559
|
+
catch (error) {
|
|
1560
|
+
if (error instanceof RunLockOwnershipUnknownError) {
|
|
1561
|
+
return "ownership_unknown";
|
|
1562
|
+
}
|
|
1563
|
+
// A live lock holder (running worker or same-process blocking run) prevents
|
|
1564
|
+
// acquisition; a truly stale/dead lock would have been reclaimed instead.
|
|
1565
|
+
return "locked";
|
|
1566
|
+
}
|
|
1567
|
+
const lockPath = join(runDir, RUN_LOCK_FILENAME);
|
|
1568
|
+
let lockHeld = true;
|
|
1569
|
+
try {
|
|
1570
|
+
const state = await readRunState(runDir);
|
|
1571
|
+
if (state.status !== "pending" && state.status !== "running") {
|
|
1572
|
+
return "skipped";
|
|
1573
|
+
}
|
|
1574
|
+
let nextState = structuredClone(state);
|
|
1575
|
+
if (isPlanReviewRun(state)) {
|
|
1576
|
+
const intent = {
|
|
1577
|
+
status: "cancelled",
|
|
1578
|
+
completedAt: new Date().toISOString(),
|
|
1579
|
+
error: "Reaped: no live worker process",
|
|
1580
|
+
replaceFailureClass: false,
|
|
1581
|
+
};
|
|
1582
|
+
const intended = applyPlanReviewTerminalIntent(state, intent);
|
|
1583
|
+
if (intended === undefined) {
|
|
1584
|
+
return "skipped";
|
|
1585
|
+
}
|
|
1586
|
+
const finalized = await finalizePlanReviewTerminalState({
|
|
1587
|
+
runDir,
|
|
1588
|
+
previousState: state,
|
|
1589
|
+
intendedState: intended,
|
|
1590
|
+
intent,
|
|
1591
|
+
});
|
|
1592
|
+
nextState = finalized.state;
|
|
1593
|
+
validateStrictStateSubtrees(nextState, join(runDir, STATE_FILENAME));
|
|
1594
|
+
}
|
|
1595
|
+
else {
|
|
1596
|
+
nextState.status = "cancelled";
|
|
1597
|
+
nextState.completedAt = new Date().toISOString();
|
|
1598
|
+
nextState.error = "Reaped: no live worker process";
|
|
1599
|
+
}
|
|
1600
|
+
await writeJsonAtomic(join(runDir, STATE_FILENAME), nextState);
|
|
1601
|
+
await safeUnlink(lockPath);
|
|
1602
|
+
lockHeld = false;
|
|
1603
|
+
return "cancelled";
|
|
1604
|
+
}
|
|
1605
|
+
finally {
|
|
1606
|
+
if (lockHeld) {
|
|
1607
|
+
await safeUnlink(lockPath);
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
/**
|
|
1612
|
+
* Terminalize run directories frozen as running/pending with no live worker.
|
|
1613
|
+
*
|
|
1614
|
+
* A worker that dies between state writes (e.g. during a retry backoff delay,
|
|
1615
|
+
* where applyRetryUpdate has already persisted errorMessage/failureClass while
|
|
1616
|
+
* the call state stays "running") leaves the run persisted as `running`
|
|
1617
|
+
* forever: the only other reconcile lives in WorkflowStateStore.resume(),
|
|
1618
|
+
* which non-resumable runs never reach, and pruning skips running_or_pending
|
|
1619
|
+
* directories. This scan converts such runs to terminal `failed` so status
|
|
1620
|
+
* projection and retention can move on. No new persisted status is introduced;
|
|
1621
|
+
* STALE_RUN_RECONCILE_ERROR is the distinct terminal representation.
|
|
1622
|
+
*
|
|
1623
|
+
* Liveness authority is classifyRunLockSync's same-host PID probe only:
|
|
1624
|
+
* a run is touched solely when its lock is "stale_or_absent" (missing lock or
|
|
1625
|
+
* same-host dead PID). "held" (live PID, or ANY foreign-host lock) and
|
|
1626
|
+
* "unknown" (malformed lock) are skipped entirely, and no decision here
|
|
1627
|
+
* consults heartbeat mtime freshness. TOCTOU-safety mirrors reapRunCancelled:
|
|
1628
|
+
* acquire the run.lock (no heartbeat declaration, like other reap
|
|
1629
|
+
* acquisitions), re-read state under the lock, re-verify non-terminal, write
|
|
1630
|
+
* once, then release. Idempotent: reconciled runs are terminal `failed` and
|
|
1631
|
+
* count as skipped_terminal on re-scan.
|
|
1632
|
+
*/
|
|
1633
|
+
export async function reconcileStaleRunDirectories(runsRootDir) {
|
|
1634
|
+
const result = {
|
|
1635
|
+
scanned: 0,
|
|
1636
|
+
reconciled: 0,
|
|
1637
|
+
skipped_held: 0,
|
|
1638
|
+
skipped_terminal: 0,
|
|
1639
|
+
errors: 0,
|
|
1640
|
+
};
|
|
1641
|
+
let entries;
|
|
1642
|
+
try {
|
|
1643
|
+
entries = await readdir(runsRootDir, { withFileTypes: true });
|
|
1644
|
+
}
|
|
1645
|
+
catch (error) {
|
|
1646
|
+
const code = error.code;
|
|
1647
|
+
if (code !== "ENOENT") {
|
|
1648
|
+
result.errors += 1;
|
|
1649
|
+
logger.warn({ runs_root_dir: runsRootDir, err: error instanceof Error ? error.message : String(error) }, "failed to read workflow runs directory for stale-run reconcile");
|
|
1650
|
+
}
|
|
1651
|
+
return result;
|
|
1652
|
+
}
|
|
1653
|
+
for (const entry of entries) {
|
|
1654
|
+
if (!entry.isDirectory()) {
|
|
1655
|
+
continue;
|
|
1656
|
+
}
|
|
1657
|
+
const runDirPath = join(runsRootDir, entry.name);
|
|
1658
|
+
result.scanned += 1;
|
|
1659
|
+
let state;
|
|
1660
|
+
try {
|
|
1661
|
+
state = readRunStateSync(runDirPath);
|
|
1662
|
+
}
|
|
1663
|
+
catch (error) {
|
|
1664
|
+
result.errors += 1;
|
|
1665
|
+
logger.warn({ run_dir: runDirPath, err: error instanceof Error ? error.message : String(error) }, "skipping run with unreadable state during stale-run reconcile");
|
|
1666
|
+
continue;
|
|
1667
|
+
}
|
|
1668
|
+
if (state.status !== "running" && state.status !== "pending") {
|
|
1669
|
+
result.skipped_terminal += 1;
|
|
1670
|
+
continue;
|
|
1671
|
+
}
|
|
1672
|
+
const lockStatus = classifyRunLockSync(join(runDirPath, RUN_LOCK_FILENAME));
|
|
1673
|
+
if (lockStatus !== "stale_or_absent") {
|
|
1674
|
+
// "held" (live same-host PID or any foreign-host lock) and "unknown"
|
|
1675
|
+
// (malformed lock) are both untouchable — never infer liveness beyond
|
|
1676
|
+
// the same-host PID probe.
|
|
1677
|
+
result.skipped_held += 1;
|
|
1678
|
+
continue;
|
|
1679
|
+
}
|
|
1680
|
+
try {
|
|
1681
|
+
const outcome = await reconcileStaleRun(runDirPath);
|
|
1682
|
+
result[outcome] += 1;
|
|
1683
|
+
}
|
|
1684
|
+
catch (error) {
|
|
1685
|
+
result.errors += 1;
|
|
1686
|
+
logger.warn({ run_dir: runDirPath, err: error instanceof Error ? error.message : String(error) }, "failed to reconcile stale workflow run directory");
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
logger.info({ runs_root_dir: runsRootDir, ...result }, "stale workflow run reconcile complete");
|
|
1690
|
+
return result;
|
|
1691
|
+
}
|
|
1692
|
+
/**
|
|
1693
|
+
* Terminalize one pre-screened stale run under its own lock. See
|
|
1694
|
+
* reconcileStaleRunDirectories for the authority and safety model.
|
|
1695
|
+
*/
|
|
1696
|
+
async function reconcileStaleRun(runDir) {
|
|
1697
|
+
try {
|
|
1698
|
+
await acquireRunLock(runDir);
|
|
1699
|
+
}
|
|
1700
|
+
catch {
|
|
1701
|
+
// A live lock holder appeared between the pre-screen and acquisition, or
|
|
1702
|
+
// ownership is unknown; either way nothing is mutated.
|
|
1703
|
+
return "skipped_held";
|
|
1704
|
+
}
|
|
1705
|
+
const lockPath = join(runDir, RUN_LOCK_FILENAME);
|
|
1706
|
+
try {
|
|
1707
|
+
const state = await readRunState(runDir);
|
|
1708
|
+
if (state.status !== "pending" && state.status !== "running") {
|
|
1709
|
+
return "skipped_terminal";
|
|
1710
|
+
}
|
|
1711
|
+
const reconciledAt = new Date().toISOString();
|
|
1712
|
+
let nextState = structuredClone(state);
|
|
1713
|
+
for (const stage of nextState.stages) {
|
|
1714
|
+
for (const call of stage.calls) {
|
|
1715
|
+
if (call.state === "running") {
|
|
1716
|
+
call.state = "failed";
|
|
1717
|
+
call.completedAt = reconciledAt;
|
|
1718
|
+
// Preserve an already-recorded error verbatim: a retry-scheduled
|
|
1719
|
+
// "permanent" classification must stay permanent (deliberately NOT
|
|
1720
|
+
// resume()'s downgrade-to-transient recovery behavior).
|
|
1721
|
+
if (call.errorMessage === undefined) {
|
|
1722
|
+
call.errorMessage = STALE_RUN_RECONCILE_CALL_ERROR;
|
|
1723
|
+
}
|
|
1724
|
+
if (call.failureClass === undefined) {
|
|
1725
|
+
call.failureClass = "transient";
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1728
|
+
}
|
|
1729
|
+
stage.state = deriveStageState(stage.calls);
|
|
1730
|
+
stage.completedAt = deriveStageCompletedAt(stage.calls);
|
|
1731
|
+
}
|
|
1732
|
+
if (isPlanReviewRun(state)) {
|
|
1733
|
+
const intent = {
|
|
1734
|
+
status: "failed",
|
|
1735
|
+
completedAt: reconciledAt,
|
|
1736
|
+
error: STALE_RUN_RECONCILE_ERROR,
|
|
1737
|
+
failureClass: "transient",
|
|
1738
|
+
replaceFailureClass: true,
|
|
1739
|
+
};
|
|
1740
|
+
const intended = applyPlanReviewTerminalIntent(nextState, intent);
|
|
1741
|
+
if (intended === undefined) {
|
|
1742
|
+
return "skipped_terminal";
|
|
1743
|
+
}
|
|
1744
|
+
const finalized = await finalizePlanReviewTerminalState({
|
|
1745
|
+
runDir,
|
|
1746
|
+
previousState: state,
|
|
1747
|
+
intendedState: intended,
|
|
1748
|
+
intent,
|
|
1749
|
+
});
|
|
1750
|
+
nextState = finalized.state;
|
|
1751
|
+
validateStrictStateSubtrees(nextState, join(runDir, STATE_FILENAME));
|
|
1752
|
+
}
|
|
1753
|
+
else {
|
|
1754
|
+
nextState.status = "failed";
|
|
1755
|
+
nextState.completedAt = reconciledAt;
|
|
1756
|
+
nextState.error = STALE_RUN_RECONCILE_ERROR;
|
|
1757
|
+
nextState.failureClass = "transient";
|
|
1758
|
+
}
|
|
1759
|
+
await writeJsonAtomic(join(runDir, STATE_FILENAME), nextState);
|
|
1760
|
+
logger.info({ run_dir: runDir, run_id: state.runId }, "reconciled stale running workflow run to terminal failed");
|
|
1761
|
+
return "reconciled";
|
|
1762
|
+
}
|
|
1763
|
+
finally {
|
|
1764
|
+
await safeUnlink(lockPath);
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
543
1767
|
function isRunLockRecord(value) {
|
|
544
1768
|
const record = asRecord(value);
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
1769
|
+
const pid = record["pid"];
|
|
1770
|
+
const host = record["hostname"];
|
|
1771
|
+
const startedAt = record["startedAt"];
|
|
1772
|
+
return typeof pid === "number" &&
|
|
1773
|
+
Number.isSafeInteger(pid) &&
|
|
1774
|
+
pid > 0 &&
|
|
1775
|
+
typeof host === "string" &&
|
|
1776
|
+
host.length > 0 &&
|
|
1777
|
+
typeof startedAt === "string" &&
|
|
1778
|
+
Number.isFinite(Date.parse(startedAt));
|
|
548
1779
|
}
|
|
549
1780
|
function duplicateRunDirError(runId) {
|
|
550
1781
|
return new WorkflowError(`Run directory for run_id "${runId}" already exists. Use a unique run_id for new runs. If this is a failed/cancelled resumable run, try resume_workflow.`);
|
|
@@ -557,19 +1788,40 @@ async function recycleSucceededRunDir(runDir) {
|
|
|
557
1788
|
lockHeld = true;
|
|
558
1789
|
}
|
|
559
1790
|
catch {
|
|
560
|
-
return false;
|
|
1791
|
+
return { recycled: false };
|
|
561
1792
|
}
|
|
562
1793
|
try {
|
|
563
1794
|
const state = await readRunState(runDir);
|
|
564
1795
|
if (state.status !== "succeeded") {
|
|
565
|
-
return false;
|
|
1796
|
+
return { recycled: false };
|
|
1797
|
+
}
|
|
1798
|
+
const identity = await readSessionLogGeneration(runDir);
|
|
1799
|
+
let retiredSessionLogGeneration;
|
|
1800
|
+
if (identity.kind === "available") {
|
|
1801
|
+
retiredSessionLogGeneration = identity.sessionLogGeneration;
|
|
1802
|
+
}
|
|
1803
|
+
else if (identity.kind === "legacy") {
|
|
1804
|
+
try {
|
|
1805
|
+
retiredSessionLogGeneration = await admitWorkflowRunGeneration(dirname(runDir), runDir, basename(runDir));
|
|
1806
|
+
}
|
|
1807
|
+
catch {
|
|
1808
|
+
logger.warn({ run_id: basename(runDir) }, "legacy recycled workflow session-log generation could not be admitted");
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1811
|
+
else {
|
|
1812
|
+
logger.warn({ run_id: basename(runDir) }, "recycled workflow session-log identity was unsafe");
|
|
566
1813
|
}
|
|
567
1814
|
await rename(runDir, `${runDir}.recycled`);
|
|
568
1815
|
lockHeld = false;
|
|
569
|
-
return
|
|
1816
|
+
return {
|
|
1817
|
+
recycled: true,
|
|
1818
|
+
...(retiredSessionLogGeneration === undefined
|
|
1819
|
+
? {}
|
|
1820
|
+
: { retiredSessionLogGeneration }),
|
|
1821
|
+
};
|
|
570
1822
|
}
|
|
571
1823
|
catch {
|
|
572
|
-
return false;
|
|
1824
|
+
return { recycled: false };
|
|
573
1825
|
}
|
|
574
1826
|
finally {
|
|
575
1827
|
if (lockHeld) {
|
|
@@ -577,6 +1829,21 @@ async function recycleSucceededRunDir(runDir) {
|
|
|
577
1829
|
}
|
|
578
1830
|
}
|
|
579
1831
|
}
|
|
1832
|
+
async function cleanupWorkflowSessionLogGeneration(runsRoot, runId, sessionLogGeneration) {
|
|
1833
|
+
try {
|
|
1834
|
+
const cleanup = await removeSessionLogGeneration({
|
|
1835
|
+
rootDir: join(dirname(resolve(runsRoot)), "session-logs"),
|
|
1836
|
+
family: "workflow",
|
|
1837
|
+
generation: sessionLogGeneration,
|
|
1838
|
+
});
|
|
1839
|
+
if (cleanup.kind === "unsafe" || cleanup.kind === "unavailable") {
|
|
1840
|
+
logger.warn({ run_id: runId, cleanup: cleanup.kind }, "recycled workflow parent removed but exact session-log cleanup was unavailable");
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
catch {
|
|
1844
|
+
logger.warn({ run_id: runId }, "recycled workflow parent removed but exact session-log cleanup failed");
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
580
1847
|
function getCallEntry(state, stageId, callId) {
|
|
581
1848
|
const stage = state.stages.find((entry) => entry.stageId === stageId);
|
|
582
1849
|
if (stage == null) {
|
|
@@ -610,6 +1877,27 @@ function deriveStageCompletedAt(calls) {
|
|
|
610
1877
|
.sort();
|
|
611
1878
|
return completedValues.length > 0 ? completedValues[completedValues.length - 1] : undefined;
|
|
612
1879
|
}
|
|
1880
|
+
function capPersistedError(message) {
|
|
1881
|
+
if (message === undefined) {
|
|
1882
|
+
return undefined;
|
|
1883
|
+
}
|
|
1884
|
+
const encoded = Buffer.from(message, "utf-8");
|
|
1885
|
+
if (encoded.length <= MAX_PERSISTED_ERROR_BYTES) {
|
|
1886
|
+
return message;
|
|
1887
|
+
}
|
|
1888
|
+
let end = MAX_PERSISTED_ERROR_BYTES;
|
|
1889
|
+
// Never split a multi-byte UTF-8 sequence: back off over continuation bytes
|
|
1890
|
+
// so the kept head decodes cleanly.
|
|
1891
|
+
while (end > 0) {
|
|
1892
|
+
const boundaryByte = encoded[end];
|
|
1893
|
+
if (boundaryByte === undefined || (boundaryByte & 0b1100_0000) !== 0b1000_0000) {
|
|
1894
|
+
break;
|
|
1895
|
+
}
|
|
1896
|
+
end -= 1;
|
|
1897
|
+
}
|
|
1898
|
+
const omitted = encoded.length - end;
|
|
1899
|
+
return `${encoded.subarray(0, end).toString("utf-8")} …[truncated ${omitted} bytes]`;
|
|
1900
|
+
}
|
|
613
1901
|
async function persistOutput(runDir, stageId, callId, outputText) {
|
|
614
1902
|
const sizeBytes = Buffer.byteLength(outputText, "utf-8");
|
|
615
1903
|
if (sizeBytes <= OUTPUT_INLINE_THRESHOLD_BYTES) {
|
|
@@ -629,6 +1917,7 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
629
1917
|
const completedCalls = new Map();
|
|
630
1918
|
const preparedInputs = new Map();
|
|
631
1919
|
const failedCalls = new Map();
|
|
1920
|
+
const nextRetryCounts = new Map();
|
|
632
1921
|
const firstIncompleteStageIndex = definition.stages.findIndex((stage) => {
|
|
633
1922
|
const persistedStage = state.stages.find((entry) => entry.stageId === stage.id);
|
|
634
1923
|
return !persistedStage?.calls.every((call) => call.state === "succeeded");
|
|
@@ -653,6 +1942,7 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
653
1942
|
stageId: stage.id,
|
|
654
1943
|
callId: call.id,
|
|
655
1944
|
error: entry.errorMessage,
|
|
1945
|
+
terminal: true,
|
|
656
1946
|
});
|
|
657
1947
|
}
|
|
658
1948
|
}
|
|
@@ -670,10 +1960,16 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
670
1960
|
stageId: stage.id,
|
|
671
1961
|
callId: call.id,
|
|
672
1962
|
error: entry.errorMessage,
|
|
1963
|
+
// Only terminal failures are persisted as `failed`; retryable
|
|
1964
|
+
// intermediate attempts remain `running` with attemptsConsumed.
|
|
1965
|
+
// Never redispatch a persisted failed call after process restart.
|
|
1966
|
+
terminal: true,
|
|
673
1967
|
});
|
|
674
1968
|
}
|
|
675
1969
|
if (entry?.materializedInputPath != null) {
|
|
676
1970
|
preparedInputs.set(`${stage.id}.${call.id}`, await readJsonFile(join(runDir, entry.materializedInputPath)));
|
|
1971
|
+
nextRetryCounts.set(`${stage.id}.${call.id}`, entry.attemptsConsumed ??
|
|
1972
|
+
(entry.state === "running" ? entry.retryCount + 1 : entry.retryCount));
|
|
677
1973
|
}
|
|
678
1974
|
}
|
|
679
1975
|
}
|
|
@@ -682,6 +1978,7 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
682
1978
|
completedCalls,
|
|
683
1979
|
preparedInputs,
|
|
684
1980
|
failedCalls,
|
|
1981
|
+
nextRetryCounts,
|
|
685
1982
|
};
|
|
686
1983
|
}
|
|
687
1984
|
async function rehydrateExecutedCall(runDir, entry) {
|
|
@@ -739,7 +2036,27 @@ function buildPersistedFidelity(fidelity, contextProducing) {
|
|
|
739
2036
|
resultContractEnabled: fidelity.parseSource === "result_contract",
|
|
740
2037
|
});
|
|
741
2038
|
}
|
|
742
|
-
|
|
2039
|
+
function resolveEnabledDelegateFingerprintRefs(policy, variables) {
|
|
2040
|
+
const delegate = policy?.effectiveDelegate;
|
|
2041
|
+
if (delegate?.enabled !== true) {
|
|
2042
|
+
return undefined;
|
|
2043
|
+
}
|
|
2044
|
+
const curator = {
|
|
2045
|
+
cli_name: delegate.curator.cli_name,
|
|
2046
|
+
role: delegate.curator.role,
|
|
2047
|
+
};
|
|
2048
|
+
if (!effectiveFinalizerEnabled(delegate, variables)) {
|
|
2049
|
+
return { curator };
|
|
2050
|
+
}
|
|
2051
|
+
return {
|
|
2052
|
+
curator,
|
|
2053
|
+
finalizer: {
|
|
2054
|
+
cli_name: delegate.finalizer.cli_name,
|
|
2055
|
+
role: delegate.finalizer.role,
|
|
2056
|
+
},
|
|
2057
|
+
};
|
|
2058
|
+
}
|
|
2059
|
+
async function buildChildConfigFingerprint(definition, policy, variables) {
|
|
743
2060
|
const configPath = getChildConfigPath();
|
|
744
2061
|
if (configPath == null) {
|
|
745
2062
|
throw new Error("Persistent resume requires CLINKX_WORKFLOWS_CHILD_CONFIG_PATH");
|
|
@@ -756,7 +2073,10 @@ async function buildChildConfigFingerprint(definition) {
|
|
|
756
2073
|
CLINKX_ALLOWED_ROOTS: deriveAllowedRoots(workspaceRoot, getStateDir(), getArtifactsDir()),
|
|
757
2074
|
CLINKX_LOG_LEVEL: getLogLevel(),
|
|
758
2075
|
};
|
|
759
|
-
|
|
2076
|
+
const delegateRefs = resolveEnabledDelegateFingerprintRefs(policy, variables);
|
|
2077
|
+
return delegateRefs === undefined
|
|
2078
|
+
? generateWorkflowFingerprint(definition, adapterConfigs, resolvedPrompts, childEnv)
|
|
2079
|
+
: generateWorkflowFingerprint(definition, adapterConfigs, resolvedPrompts, childEnv, delegateRefs);
|
|
760
2080
|
}
|
|
761
2081
|
async function loadNormalizedAdapterConfigs(configPath) {
|
|
762
2082
|
const paths = await listConfigFiles(configPath);
|
|
@@ -820,6 +2140,8 @@ async function resolveRolePrompt(configFilePath, role) {
|
|
|
820
2140
|
function normalizeAdapterConfig(raw) {
|
|
821
2141
|
const rolesRecord = asRecord(raw["roles"]);
|
|
822
2142
|
const roles = Object.fromEntries(Object.entries(rolesRecord).map(([name, value]) => [name, normalizeRoleConfig(asRecord(value))]));
|
|
2143
|
+
const invocationOverrides = raw["invocation_overrides"];
|
|
2144
|
+
const promptFileFlag = asOptionalString(raw["prompt_file_flag"]);
|
|
823
2145
|
return {
|
|
824
2146
|
name: expectString(raw["name"], "adapter.name"),
|
|
825
2147
|
command: expectString(raw["command"], "adapter.command"),
|
|
@@ -842,6 +2164,10 @@ function normalizeAdapterConfig(raw) {
|
|
|
842
2164
|
stdin_placeholder_arg: asOptionalString(raw["stdin_placeholder_arg"]),
|
|
843
2165
|
native_output_file_flag: asOptionalString(raw["native_output_file_flag"]),
|
|
844
2166
|
suppress_stdout: asOptionalBoolean(raw["suppress_stdout"]) ?? false,
|
|
2167
|
+
...(invocationOverrides !== undefined
|
|
2168
|
+
? { invocation_overrides: invocationOverrides }
|
|
2169
|
+
: {}),
|
|
2170
|
+
...(promptFileFlag !== undefined ? { prompt_file_flag: promptFileFlag } : {}),
|
|
845
2171
|
};
|
|
846
2172
|
}
|
|
847
2173
|
function normalizeRoleConfig(raw) {
|
|
@@ -878,6 +2204,7 @@ async function readRunState(runDir) {
|
|
|
878
2204
|
if (state.state_version !== 1) {
|
|
879
2205
|
throw new Error(`Unsupported state_version ${String(state.state_version)} in ${statePath}`);
|
|
880
2206
|
}
|
|
2207
|
+
validateStrictStateSubtrees(state, statePath);
|
|
881
2208
|
return state;
|
|
882
2209
|
}
|
|
883
2210
|
async function verifySnapshotHash(filePath, expectedHash, label) {
|
|
@@ -913,12 +2240,25 @@ function hashContent(content) {
|
|
|
913
2240
|
async function readJsonFile(filePath) {
|
|
914
2241
|
return JSON.parse(await readFile(filePath, "utf-8"));
|
|
915
2242
|
}
|
|
916
|
-
|
|
2243
|
+
/**
|
|
2244
|
+
* Atomically create `run.lock`, reclaiming a same-host dead-PID lock.
|
|
2245
|
+
*
|
|
2246
|
+
* `declareHeartbeat` is deliberately opt-in: only a caller that goes on to hold
|
|
2247
|
+
* the lock behind a live `WorkflowStateStore` (create/resume) beats it. The
|
|
2248
|
+
* prune/reap/recycle callers hold the lock briefly with no beating owner, so
|
|
2249
|
+
* declaring a heartbeat there would publish a capability claim nobody honors —
|
|
2250
|
+
* and a foreign-host reader would then age it into a false `orphaned`.
|
|
2251
|
+
*/
|
|
2252
|
+
async function acquireRunLock(runDir, options = {}) {
|
|
917
2253
|
const lockPath = join(runDir, RUN_LOCK_FILENAME);
|
|
918
2254
|
const record = {
|
|
919
2255
|
pid: process.pid,
|
|
920
2256
|
hostname: hostname(),
|
|
921
2257
|
startedAt: new Date().toISOString(),
|
|
2258
|
+
token: randomBytes(16).toString("base64url"),
|
|
2259
|
+
...(options.declareHeartbeat === true
|
|
2260
|
+
? { heartbeatIntervalMs: RUN_LOCK_HEARTBEAT_MS }
|
|
2261
|
+
: {}),
|
|
922
2262
|
};
|
|
923
2263
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
924
2264
|
try {
|
|
@@ -929,13 +2269,26 @@ async function acquireRunLock(runDir) {
|
|
|
929
2269
|
finally {
|
|
930
2270
|
await handle.close();
|
|
931
2271
|
}
|
|
932
|
-
return;
|
|
2272
|
+
return record;
|
|
933
2273
|
}
|
|
934
2274
|
catch (error) {
|
|
935
2275
|
if (!(error instanceof Error) || !("code" in error) || error.code !== "EEXIST") {
|
|
936
2276
|
throw error;
|
|
937
2277
|
}
|
|
938
|
-
|
|
2278
|
+
let existingValue;
|
|
2279
|
+
try {
|
|
2280
|
+
existingValue = await readJsonFile(lockPath);
|
|
2281
|
+
}
|
|
2282
|
+
catch (readError) {
|
|
2283
|
+
if (readError.code === "ENOENT") {
|
|
2284
|
+
continue;
|
|
2285
|
+
}
|
|
2286
|
+
throw new RunLockOwnershipUnknownError(lockPath, readError instanceof Error ? readError.message : String(readError));
|
|
2287
|
+
}
|
|
2288
|
+
if (!isRunLockRecord(existingValue)) {
|
|
2289
|
+
throw new RunLockOwnershipUnknownError(lockPath, "record must contain a positive integer pid, non-empty hostname, and valid startedAt timestamp");
|
|
2290
|
+
}
|
|
2291
|
+
const existing = existingValue;
|
|
939
2292
|
if (existing.hostname === hostname() && !isPidLive(existing.pid)) {
|
|
940
2293
|
await unlink(lockPath);
|
|
941
2294
|
continue;
|
|
@@ -943,6 +2296,7 @@ async function acquireRunLock(runDir) {
|
|
|
943
2296
|
throw new Error(`Run lock is held for ${runDir} by pid ${String(existing.pid)} on ${existing.hostname} since ${existing.startedAt}`);
|
|
944
2297
|
}
|
|
945
2298
|
}
|
|
2299
|
+
throw new Error(`Failed to acquire run lock for ${runDir} after 2 attempts; lock ownership was not established`);
|
|
946
2300
|
}
|
|
947
2301
|
function isPidLive(pid) {
|
|
948
2302
|
try {
|