clinkx 1.1.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +67 -14
- package/clinkx-workflows/dist/allowed-roots.js +2 -0
- package/clinkx-workflows/dist/allowed-roots.js.map +1 -1
- package/clinkx-workflows/dist/architecture-arena/admission.d.ts +16 -0
- package/clinkx-workflows/dist/architecture-arena/admission.js +107 -0
- package/clinkx-workflows/dist/architecture-arena/admission.js.map +1 -0
- package/clinkx-workflows/dist/architecture-arena/contract.d.ts +1368 -0
- package/clinkx-workflows/dist/architecture-arena/contract.js +254 -0
- package/clinkx-workflows/dist/architecture-arena/contract.js.map +1 -0
- package/clinkx-workflows/dist/architecture-arena/runtime.d.ts +53 -0
- package/clinkx-workflows/dist/architecture-arena/runtime.js +511 -0
- package/clinkx-workflows/dist/architecture-arena/runtime.js.map +1 -0
- package/clinkx-workflows/dist/architecture-comparison/admission.d.ts +17 -1
- package/clinkx-workflows/dist/architecture-comparison/admission.js +27 -4
- package/clinkx-workflows/dist/architecture-comparison/admission.js.map +1 -1
- package/clinkx-workflows/dist/architecture-comparison/contract.d.ts +59 -62
- package/clinkx-workflows/dist/architecture-comparison/contract.js +4 -21
- package/clinkx-workflows/dist/architecture-comparison/contract.js.map +1 -1
- package/clinkx-workflows/dist/architecture-comparison/runtime.d.ts +4 -3
- package/clinkx-workflows/dist/architecture-comparison/runtime.js +22 -67
- package/clinkx-workflows/dist/architecture-comparison/runtime.js.map +1 -1
- package/clinkx-workflows/dist/artifacts.d.ts +19 -4
- package/clinkx-workflows/dist/artifacts.js +16 -7
- package/clinkx-workflows/dist/artifacts.js.map +1 -1
- package/clinkx-workflows/dist/backend.d.ts +30 -0
- package/clinkx-workflows/dist/child-env.d.ts +3 -1
- package/clinkx-workflows/dist/child-env.js +6 -2
- package/clinkx-workflows/dist/child-env.js.map +1 -1
- package/clinkx-workflows/dist/clink-client.d.ts +8 -2
- package/clinkx-workflows/dist/clink-client.js +4 -2
- package/clinkx-workflows/dist/clink-client.js.map +1 -1
- package/clinkx-workflows/dist/config.d.ts +28 -5
- package/clinkx-workflows/dist/config.js +137 -20
- package/clinkx-workflows/dist/config.js.map +1 -1
- package/clinkx-workflows/dist/contained-file.js +9 -0
- package/clinkx-workflows/dist/contained-file.js.map +1 -1
- package/clinkx-workflows/dist/context-preflight.d.ts +6 -0
- package/clinkx-workflows/dist/context-preflight.js.map +1 -1
- package/clinkx-workflows/dist/council-execution-policy.d.ts +3 -1
- package/clinkx-workflows/dist/council-execution-policy.js +4 -0
- package/clinkx-workflows/dist/council-execution-policy.js.map +1 -1
- package/clinkx-workflows/dist/dashboard/contracts.d.ts +586 -586
- package/clinkx-workflows/dist/dashboard/progress-report-contract.d.ts +28 -28
- package/clinkx-workflows/dist/dashboard/session-log-contract.d.ts +32 -32
- package/clinkx-workflows/dist/dashboard/session-log-identity.js +2 -1
- package/clinkx-workflows/dist/dashboard/session-log-identity.js.map +1 -1
- package/clinkx-workflows/dist/dashboard-ui/assets/index-DRId2DVA.js +119 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/style-CHAw57y5.css +1 -0
- package/clinkx-workflows/dist/dashboard-ui/index.html +2 -2
- package/clinkx-workflows/dist/delegate-signature.d.ts +4 -3
- package/clinkx-workflows/dist/delegate-signature.js +4 -3
- package/clinkx-workflows/dist/delegate-signature.js.map +1 -1
- package/clinkx-workflows/dist/durable-file.d.ts +69 -0
- package/clinkx-workflows/dist/durable-file.js +145 -0
- package/clinkx-workflows/dist/durable-file.js.map +1 -0
- package/clinkx-workflows/dist/engine.d.ts +42 -7
- package/clinkx-workflows/dist/engine.js +196 -24
- package/clinkx-workflows/dist/engine.js.map +1 -1
- package/clinkx-workflows/dist/ping-pong/admission.d.ts +127 -0
- package/clinkx-workflows/dist/ping-pong/admission.js +522 -0
- package/clinkx-workflows/dist/ping-pong/admission.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/contract.d.ts +2539 -0
- package/clinkx-workflows/dist/ping-pong/contract.js +819 -0
- package/clinkx-workflows/dist/ping-pong/contract.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/control.d.ts +67 -0
- package/clinkx-workflows/dist/ping-pong/control.js +12 -0
- package/clinkx-workflows/dist/ping-pong/control.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/runtime-handle.d.ts +24 -0
- package/clinkx-workflows/dist/ping-pong/runtime-handle.js +55 -0
- package/clinkx-workflows/dist/ping-pong/runtime-handle.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/runtime.d.ts +235 -0
- package/clinkx-workflows/dist/ping-pong/runtime.js +1183 -0
- package/clinkx-workflows/dist/ping-pong/runtime.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/settle.d.ts +57 -0
- package/clinkx-workflows/dist/ping-pong/settle.js +58 -0
- package/clinkx-workflows/dist/ping-pong/settle.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/sidecar.d.ts +65 -0
- package/clinkx-workflows/dist/ping-pong/sidecar.js +138 -0
- package/clinkx-workflows/dist/ping-pong/sidecar.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/publication.d.ts +2 -1
- package/clinkx-workflows/dist/plan-review/publication.js +5 -6
- package/clinkx-workflows/dist/plan-review/publication.js.map +1 -1
- package/clinkx-workflows/dist/plan-review/snapshot.d.ts +3 -2
- package/clinkx-workflows/dist/plan-review/snapshot.js +8 -8
- package/clinkx-workflows/dist/plan-review/snapshot.js.map +1 -1
- package/clinkx-workflows/dist/recipe-host.d.ts +315 -0
- package/clinkx-workflows/dist/recipe-host.js +183 -0
- package/clinkx-workflows/dist/recipe-host.js.map +1 -0
- package/clinkx-workflows/dist/recipe-seats.d.ts +96 -0
- package/clinkx-workflows/dist/recipe-seats.js +163 -0
- package/clinkx-workflows/dist/recipe-seats.js.map +1 -0
- package/clinkx-workflows/dist/recipe-shared/evidence.d.ts +67 -0
- package/clinkx-workflows/dist/recipe-shared/evidence.js +111 -0
- package/clinkx-workflows/dist/recipe-shared/evidence.js.map +1 -0
- package/clinkx-workflows/dist/reducers/index.d.ts +9 -1
- package/clinkx-workflows/dist/reducers/index.js +30 -0
- package/clinkx-workflows/dist/reducers/index.js.map +1 -1
- package/clinkx-workflows/dist/root-path-validation.d.ts +12 -0
- package/clinkx-workflows/dist/root-path-validation.js +63 -0
- package/clinkx-workflows/dist/root-path-validation.js.map +1 -0
- package/clinkx-workflows/dist/rosters.d.ts +1 -0
- package/clinkx-workflows/dist/rosters.js +16 -6
- package/clinkx-workflows/dist/rosters.js.map +1 -1
- package/clinkx-workflows/dist/run-lock-reclaim.d.ts +44 -0
- package/clinkx-workflows/dist/run-lock-reclaim.js +86 -0
- package/clinkx-workflows/dist/run-lock-reclaim.js.map +1 -0
- package/clinkx-workflows/dist/server.js +5 -2
- package/clinkx-workflows/dist/server.js.map +1 -1
- package/clinkx-workflows/dist/state-schema.d.ts +24 -0
- package/clinkx-workflows/dist/state-schema.js +4 -1
- package/clinkx-workflows/dist/state-schema.js.map +1 -1
- package/clinkx-workflows/dist/state.d.ts +185 -4
- package/clinkx-workflows/dist/state.js +684 -88
- package/clinkx-workflows/dist/state.js.map +1 -1
- package/clinkx-workflows/dist/transport.d.ts +2 -1
- package/clinkx-workflows/dist/transport.js +5 -0
- package/clinkx-workflows/dist/transport.js.map +1 -1
- package/clinkx-workflows/dist/validators/architecture-arena.d.ts +6 -0
- package/clinkx-workflows/dist/validators/architecture-arena.js +16 -0
- package/clinkx-workflows/dist/validators/architecture-arena.js.map +1 -0
- package/clinkx-workflows/dist/validators/index.js +4 -0
- package/clinkx-workflows/dist/validators/index.js.map +1 -1
- package/clinkx-workflows/dist/validators/ping-pong.d.ts +6 -0
- package/clinkx-workflows/dist/validators/ping-pong.js +10 -0
- package/clinkx-workflows/dist/validators/ping-pong.js.map +1 -0
- package/clinkx-workflows/dist/workflow-completion.d.ts +83 -0
- package/clinkx-workflows/dist/workflow-completion.js +250 -0
- package/clinkx-workflows/dist/workflow-completion.js.map +1 -0
- package/clinkx-workflows/dist/workflow-tools.d.ts +53 -6
- package/clinkx-workflows/dist/workflow-tools.js +1123 -65
- package/clinkx-workflows/dist/workflow-tools.js.map +1 -1
- package/clinkx-workflows/dist/workflow-verification-contract.d.ts +4 -4
- package/clinkx-workflows/templates/architecture-arena/candidate.txt +39 -0
- package/clinkx-workflows/templates/architecture-arena/judge.txt +34 -0
- package/clinkx-workflows/templates/ping-pong/implement.txt +14 -0
- package/clinkx-workflows/templates/ping-pong/remediate.txt +14 -0
- package/clinkx-workflows/templates/ping-pong/review.txt +27 -0
- package/clinkx-workflows/workflows/architecture-arena.yaml +127 -0
- package/clinkx-workflows/workflows/ping-pong.yaml +158 -0
- package/dist/allowed-roots.d.ts +1 -1
- package/dist/allowed-roots.js +124 -14
- package/dist/allowed-roots.js.map +1 -1
- package/dist/context-builder/bundle.js +4 -3
- package/dist/context-builder/bundle.js.map +1 -1
- package/dist/context-builder/config.d.ts +1 -0
- package/dist/context-builder/config.js +21 -9
- package/dist/context-builder/config.js.map +1 -1
- package/dist/context-builder/delegate-packet.d.ts +1 -0
- package/dist/context-builder/delegate-packet.js +66 -7
- package/dist/context-builder/delegate-packet.js.map +1 -1
- package/dist/context-builder/delegate-validation.d.ts +12 -3
- package/dist/context-builder/delegate-validation.js +23 -13
- package/dist/context-builder/delegate-validation.js.map +1 -1
- package/dist/context-builder/external-gaps.d.ts +14 -3
- package/dist/context-builder/external-gaps.js +124 -18
- package/dist/context-builder/external-gaps.js.map +1 -1
- package/dist/context-builder/handler.js +108 -191
- package/dist/context-builder/handler.js.map +1 -1
- package/dist/context-builder/mutation-lock.d.ts +4 -0
- package/dist/context-builder/mutation-lock.js +7 -1
- package/dist/context-builder/mutation-lock.js.map +1 -1
- package/dist/context-builder/review-coverage.js +27 -1
- package/dist/context-builder/review-coverage.js.map +1 -1
- package/dist/context-builder/secrets.js +1 -1
- package/dist/context-builder/secrets.js.map +1 -1
- package/dist/context-builder/storage.d.ts +10 -2
- package/dist/context-builder/storage.js +213 -16
- package/dist/context-builder/storage.js.map +1 -1
- package/dist/context-builder/versions.d.ts +10 -4
- package/dist/context-builder/versions.js +10 -4
- package/dist/context-builder/versions.js.map +1 -1
- package/dist/context-builder/workflow-runner.js +33 -24
- package/dist/context-builder/workflow-runner.js.map +1 -1
- package/dist/dashboard-cli.d.ts +8 -0
- package/dist/dashboard-cli.js +155 -86
- package/dist/dashboard-cli.js.map +1 -1
- package/dist/dashboard-companion/client.d.ts +10 -0
- package/dist/dashboard-companion/client.js +43 -1
- package/dist/dashboard-companion/client.js.map +1 -1
- package/dist/dashboard-companion/config.d.ts +19 -1
- package/dist/dashboard-companion/config.js +104 -3
- package/dist/dashboard-companion/config.js.map +1 -1
- package/dist/dashboard-companion/protocol.d.ts +16 -0
- package/dist/dashboard-companion/protocol.js +38 -1
- package/dist/dashboard-companion/protocol.js.map +1 -1
- package/dist/dashboard-companion/workstation-binding.d.ts +69 -0
- package/dist/dashboard-companion/workstation-binding.js +192 -0
- package/dist/dashboard-companion/workstation-binding.js.map +1 -0
- package/dist/dashboard-gateway/admin-cli.d.ts +10 -0
- package/dist/dashboard-gateway/admin-cli.js +266 -0
- package/dist/dashboard-gateway/admin-cli.js.map +1 -0
- package/dist/dashboard-gateway/config.d.ts +50 -0
- package/dist/dashboard-gateway/config.js +235 -1
- package/dist/dashboard-gateway/config.js.map +1 -1
- package/dist/dashboard-gateway/connection-registry.d.ts +100 -0
- package/dist/dashboard-gateway/connection-registry.js +228 -0
- package/dist/dashboard-gateway/connection-registry.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/account-auth.d.ts +78 -0
- package/dist/dashboard-gateway/control-plane/account-auth.js +198 -0
- package/dist/dashboard-gateway/control-plane/account-auth.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/billing.d.ts +264 -0
- package/dist/dashboard-gateway/control-plane/billing.js +787 -0
- package/dist/dashboard-gateway/control-plane/billing.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/contracts.d.ts +1293 -0
- package/dist/dashboard-gateway/control-plane/contracts.js +280 -0
- package/dist/dashboard-gateway/control-plane/contracts.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/cookies.d.ts +13 -0
- package/dist/dashboard-gateway/control-plane/cookies.js +46 -0
- package/dist/dashboard-gateway/control-plane/cookies.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/credentials.d.ts +63 -0
- package/dist/dashboard-gateway/control-plane/credentials.js +66 -0
- package/dist/dashboard-gateway/control-plane/credentials.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/entitlements.d.ts +118 -0
- package/dist/dashboard-gateway/control-plane/entitlements.js +329 -0
- package/dist/dashboard-gateway/control-plane/entitlements.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/errors.d.ts +17 -0
- package/dist/dashboard-gateway/control-plane/errors.js +23 -0
- package/dist/dashboard-gateway/control-plane/errors.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/hosts.d.ts +6 -0
- package/dist/dashboard-gateway/control-plane/hosts.js +16 -0
- package/dist/dashboard-gateway/control-plane/hosts.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/http.d.ts +34 -0
- package/dist/dashboard-gateway/control-plane/http.js +331 -0
- package/dist/dashboard-gateway/control-plane/http.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/schema.d.ts +9 -0
- package/dist/dashboard-gateway/control-plane/schema.js +247 -0
- package/dist/dashboard-gateway/control-plane/schema.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/secrets.d.ts +20 -0
- package/dist/dashboard-gateway/control-plane/secrets.js +39 -0
- package/dist/dashboard-gateway/control-plane/secrets.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/store.d.ts +579 -0
- package/dist/dashboard-gateway/control-plane/store.js +1115 -0
- package/dist/dashboard-gateway/control-plane/store.js.map +1 -0
- package/dist/dashboard-gateway/cookie-boundary.d.ts +60 -0
- package/dist/dashboard-gateway/cookie-boundary.js +205 -0
- package/dist/dashboard-gateway/cookie-boundary.js.map +1 -0
- package/dist/dashboard-gateway/index.js +271 -494
- package/dist/dashboard-gateway/index.js.map +1 -1
- package/dist/dashboard-gateway/main.js +7 -2
- package/dist/dashboard-gateway/main.js.map +1 -1
- package/dist/dashboard-gateway/routing.d.ts +43 -0
- package/dist/dashboard-gateway/routing.js +65 -0
- package/dist/dashboard-gateway/routing.js.map +1 -0
- package/dist/dashboard-gateway/tenant-gateway.d.ts +56 -0
- package/dist/dashboard-gateway/tenant-gateway.js +989 -0
- package/dist/dashboard-gateway/tenant-gateway.js.map +1 -0
- package/dist/dashboard-gateway/tenant-main.d.ts +41 -0
- package/dist/dashboard-gateway/tenant-main.js +303 -0
- package/dist/dashboard-gateway/tenant-main.js.map +1 -0
- package/dist/dashboard-gateway/tunnel-errors.d.ts +16 -0
- package/dist/dashboard-gateway/tunnel-errors.js +22 -0
- package/dist/dashboard-gateway/tunnel-errors.js.map +1 -0
- package/dist/dashboard-gateway/tunnel-http.d.ts +21 -0
- package/dist/dashboard-gateway/tunnel-http.js +175 -0
- package/dist/dashboard-gateway/tunnel-http.js.map +1 -0
- package/dist/handler.d.ts +6 -0
- package/dist/handler.js +17 -1
- package/dist/handler.js.map +1 -1
- package/dist/local-clink-backend.d.ts +2 -1
- package/dist/local-clink-backend.js +28 -4
- package/dist/local-clink-backend.js.map +1 -1
- package/dist/parsers/extract.d.ts +19 -1
- package/dist/parsers/extract.js +68 -3
- package/dist/parsers/extract.js.map +1 -1
- package/dist/pipeline.d.ts +17 -1
- package/dist/pipeline.js +34 -6
- package/dist/pipeline.js.map +1 -1
- package/dist/process-group.d.ts +65 -0
- package/dist/process-group.js +153 -0
- package/dist/process-group.js.map +1 -0
- package/dist/progress-report/schema.d.ts +23 -23
- package/dist/project-verification/files.js +1 -1
- package/dist/project-verification/files.js.map +1 -1
- package/dist/project-verification/process.js +23 -44
- package/dist/project-verification/process.js.map +1 -1
- package/dist/project-verification/service.js +9 -1
- package/dist/project-verification/service.js.map +1 -1
- package/dist/prompt.d.ts +7 -0
- package/dist/prompt.js +18 -1
- package/dist/prompt.js.map +1 -1
- package/dist/recipe-host/git-trust.d.ts +43 -0
- package/dist/recipe-host/git-trust.js +360 -0
- package/dist/recipe-host/git-trust.js.map +1 -0
- package/dist/recipe-host/git.d.ts +42 -0
- package/dist/recipe-host/git.js +207 -0
- package/dist/recipe-host/git.js.map +1 -0
- package/dist/recipe-host/native-recipe-host.d.ts +127 -0
- package/dist/recipe-host/native-recipe-host.js +880 -0
- package/dist/recipe-host/native-recipe-host.js.map +1 -0
- package/dist/recipe-host/seat-cwd.d.ts +16 -0
- package/dist/recipe-host/seat-cwd.js +170 -0
- package/dist/recipe-host/seat-cwd.js.map +1 -0
- package/dist/registry.d.ts +1 -1
- package/dist/registry.js +5 -1
- package/dist/registry.js.map +1 -1
- package/dist/runner.d.ts +43 -1
- package/dist/runner.js +234 -35
- package/dist/runner.js.map +1 -1
- package/dist/server.js +21 -6
- package/dist/server.js.map +1 -1
- package/docs/configuration.md +181 -15
- package/docs/engineering-recipes.md +291 -0
- package/docs/security.md +87 -0
- package/docs/verification/project-checks.md +24 -2
- package/package.json +5 -1
- package/scripts/provider-doctor-core.mjs +21 -5
- package/clinkx-workflows/dist/dashboard-ui/assets/index-BdpmbPps.js +0 -115
- package/clinkx-workflows/dist/dashboard-ui/assets/style-Im5i1iAs.css +0 -1
|
@@ -1,17 +1,24 @@
|
|
|
1
1
|
import { createHash, randomBytes } from "node:crypto";
|
|
2
2
|
import { execFileSync } from "node:child_process";
|
|
3
3
|
import { constants as fsConstants, readFileSync } from "node:fs";
|
|
4
|
-
import { lstat, mkdir, open, readFile, readdir, realpath, rename, rm, stat, unlink
|
|
4
|
+
import { lstat, mkdir, open, readFile, readdir, realpath, rename, rm, rmdir, stat, unlink } from "node:fs/promises";
|
|
5
5
|
import { dirname, basename, isAbsolute, join, relative, resolve } from "node:path";
|
|
6
6
|
import { hostname } from "node:os";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
|
-
import { readContainedRelativeFile } from "./contained-file.js";
|
|
9
|
-
import {
|
|
8
|
+
import { ContainedFileChangedError, readContainedRelativeFile } from "./contained-file.js";
|
|
9
|
+
import { MAX_RECORD_BYTES } from "./architecture-comparison/contract.js";
|
|
10
|
+
import { usesContainedArchitectureEvidence } from "./architecture-arena/contract.js";
|
|
10
11
|
import { WorkflowError } from "./errors.js";
|
|
12
|
+
import { RECIPE_STATE_FILENAME, readRecipePruneSummary } from "./recipe-host.js";
|
|
13
|
+
import { PING_PONG_WORKFLOW, isCompleteAwaitingCheckpoint, isTerminalRecipePhase, runStatusForOutcome, } from "./ping-pong/contract.js";
|
|
14
|
+
import { RecipeStateInvariantError, isSidecarRecipeWorkflow, readRecipeState, recipeSidecarPresence, stampRecipeState, writeRecipeStateAtomic, } from "./ping-pong/sidecar.js";
|
|
15
|
+
import { classifyRecipeTerminalAuthority, recipeOutcomeEnvelope, recipeSettlementDraft, settlementNote, } from "./ping-pong/settle.js";
|
|
11
16
|
import { computeRunHealth } from "./run-health.js";
|
|
17
|
+
import { reclaimDeadRunLock } from "./run-lock-reclaim.js";
|
|
12
18
|
import { buildFidelityMetadata, } from "./fidelity.js";
|
|
13
19
|
import { fingerprintsMatch, generateFingerprint, generateWorkflowFingerprint, } from "./fingerprint.js";
|
|
14
20
|
import { effectiveFinalizerEnabled } from "./delegate-finalizer-policy.js";
|
|
21
|
+
import { parseCallFailureKind } from "./state-schema.js";
|
|
15
22
|
import { parseWorkflowContext, } from "./context-preflight.js";
|
|
16
23
|
import { isLegalContextBuilderLifecycleTransition, parseContextBuilderLifecycleStateV1, } from "./context-builder-lifecycle-contract.js";
|
|
17
24
|
import { ResolvedRosterSchema, RosterNameSchema } from "./roster-contract.js";
|
|
@@ -21,14 +28,14 @@ import { isPlanReviewPublicationEligible, projectPlanReviewRun, } from "./plan-r
|
|
|
21
28
|
import { isPlanReviewRunState } from "./plan-review/admission.js";
|
|
22
29
|
import { restorePlanReviewRun } from "./plan-review/resume.js";
|
|
23
30
|
import { PlanReviewStageError } from "./plan-review/stage-error.js";
|
|
24
|
-
import { DEFAULT_RETAIN_ASYNC_DAYS, DEFAULT_RETAIN_CANCELLED_DAYS, DEFAULT_RETAIN_DEBUG_DAYS, DEFAULT_RETAIN_FAILED_DAYS, DEFAULT_RETAIN_SUCCEEDED_DAYS, deriveAllowedRoots, getArtifactsDir, getChildConfigPath, getChildMaxConcurrent, getChildMaxOutputBytes, getChildMaxPromptBytes, getChildMaxResponseChars, getLogLevel, getRetentionConfig, getRunsDir, getRunStateDir, getStateDir, invocationAuditV2PersistenceEnabled, observedInvocationPersistenceEnabled, resolveWorkspaceRoot, } from "./config.js";
|
|
31
|
+
import { DEFAULT_DASHBOARD_MAX_STATE_BYTES, DEFAULT_RETAIN_ASYNC_DAYS, DEFAULT_RETAIN_CANCELLED_DAYS, DEFAULT_RETAIN_DEBUG_DAYS, DEFAULT_RETAIN_FAILED_DAYS, DEFAULT_RETAIN_SUCCEEDED_DAYS, deriveAllowedRoots, getArtifactsDir, getChildConfigPath, getChildMaxConcurrent, getChildMaxOutputBytes, getChildMaxPromptBytes, getChildMaxResponseChars, getLogLevel, getRetentionConfig, getRunsDir, getRunStateDir, getStateDir, invocationAuditV2PersistenceEnabled, observedInvocationPersistenceEnabled, resolveWorkspaceRoot, } from "./config.js";
|
|
25
32
|
import { logger } from "./logger.js";
|
|
33
|
+
import { DurablePublishError, mkdirDurable, resolveFileWriter } from "./durable-file.js";
|
|
26
34
|
import { resolveOpenedDescriptorPath } from "./contained-file.js";
|
|
27
35
|
import { deriveWorkflowRunGeneration } from "./dashboard/generation.js";
|
|
28
36
|
import { removeSessionLogGeneration } from "./dashboard/session-log-retention.js";
|
|
29
37
|
import { createSessionLogGeneration, readSessionLogGeneration, } from "./dashboard/session-log-identity.js";
|
|
30
38
|
const STATE_FILENAME = "state.json";
|
|
31
|
-
const STATE_TMP_FILENAME = "state.json.tmp";
|
|
32
39
|
const WORKFLOW_SNAPSHOT_FILENAME = "workflow.snapshot.json";
|
|
33
40
|
const INPUTS_SNAPSHOT_FILENAME = "inputs.snapshot.json";
|
|
34
41
|
const CHILD_CONFIG_DIRNAME = "child-config";
|
|
@@ -95,7 +102,7 @@ function withPlanReviewPublication(state, publication) {
|
|
|
95
102
|
* enqueueWrite. The committed marker is persisted on the pre-terminal state,
|
|
96
103
|
* after the sidecar and before the caller commits the terminal state.
|
|
97
104
|
*/
|
|
98
|
-
const finalizePlanReviewTerminalState = async ({ runDir, previousState, intendedState, intent, chairmanCallId, serializedResult, }) => {
|
|
105
|
+
const finalizePlanReviewTerminalState = async ({ runDir, previousState, intendedState, intent, chairmanCallId, serializedResult, fileWriter, }) => {
|
|
99
106
|
const existing = previousState.plan_review_state?.publication;
|
|
100
107
|
if (existing?.committed === true) {
|
|
101
108
|
return { state: withPlanReviewPublication(intendedState, existing) };
|
|
@@ -145,13 +152,13 @@ const finalizePlanReviewTerminalState = async ({ runDir, previousState, intended
|
|
|
145
152
|
contract: restored.contract,
|
|
146
153
|
execution_contract_digest: restored.execution_contract_digest,
|
|
147
154
|
}));
|
|
148
|
-
const marker = await writeSerializedPlanReviewResultSidecar(runDir, projected);
|
|
155
|
+
const marker = await writeSerializedPlanReviewResultSidecar(runDir, projected, fileWriter);
|
|
149
156
|
// Durable crash boundary: a process death after this write still leaves a
|
|
150
157
|
// contained, marker-addressable result even though terminal intent has not
|
|
151
158
|
// yet been committed.
|
|
152
159
|
const intermediate = withPlanReviewPublication(previousState, marker);
|
|
153
160
|
validateStrictStateSubtrees(intermediate, join(runDir, STATE_FILENAME));
|
|
154
|
-
await
|
|
161
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), intermediate);
|
|
155
162
|
return { state: withPlanReviewPublication(intendedState, marker) };
|
|
156
163
|
}
|
|
157
164
|
catch (err) {
|
|
@@ -190,6 +197,32 @@ class RunLockOwnershipLostError extends Error {
|
|
|
190
197
|
this.name = "RunLockOwnershipLostError";
|
|
191
198
|
}
|
|
192
199
|
}
|
|
200
|
+
/**
|
|
201
|
+
* The terminal recipe sidecar for a failed/cancelled settlement could not be
|
|
202
|
+
* committed, so `state.json` was left active rather than claiming an outcome
|
|
203
|
+
* the sidecar does not record. Once run.lock is released the run is an
|
|
204
|
+
* orphan: reconcile fails it, or cancel_workflow records it.
|
|
205
|
+
*/
|
|
206
|
+
export class RecipeTerminalUnrecordedError extends WorkflowError {
|
|
207
|
+
intent;
|
|
208
|
+
constructor(intent, cause) {
|
|
209
|
+
super(`terminal recipe sidecar (${intent}) could not be committed; state.json left active: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
210
|
+
this.intent = intent;
|
|
211
|
+
this.name = "RecipeTerminalUnrecordedError";
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
export class RecipeReopenError extends WorkflowError {
|
|
215
|
+
reason;
|
|
216
|
+
current;
|
|
217
|
+
constructor(reason, message,
|
|
218
|
+
/** Current sidecar position, when one could be read. */
|
|
219
|
+
current) {
|
|
220
|
+
super(message);
|
|
221
|
+
this.reason = reason;
|
|
222
|
+
this.current = current;
|
|
223
|
+
this.name = "RecipeReopenError";
|
|
224
|
+
}
|
|
225
|
+
}
|
|
193
226
|
export function classifyRunDirectory(runDirPath) {
|
|
194
227
|
return inspectRunDirectory(runDirPath).classification;
|
|
195
228
|
}
|
|
@@ -258,8 +291,14 @@ export async function pruneRunDirectories(runsRootDir, options) {
|
|
|
258
291
|
const runDirPath = join(runsRootDir, entry.name);
|
|
259
292
|
try {
|
|
260
293
|
const { classification, state } = inspectRunDirectory(runDirPath);
|
|
294
|
+
const recipeRun = await hasRecipeSidecar(runDirPath);
|
|
261
295
|
let retainDays = retentionDaysForClassification(classification, retention);
|
|
262
|
-
if (
|
|
296
|
+
if (recipeRun && retainDays !== undefined) {
|
|
297
|
+
// CONTRACTS C5: a paused or delivered recipe run keeps the failed-run TTL
|
|
298
|
+
// for every terminal status; that TTL is also the decision TTL.
|
|
299
|
+
retainDays = retention.retainFailedDays;
|
|
300
|
+
}
|
|
301
|
+
else if ((classification === "succeeded_async" || classification === "succeeded_sync") &&
|
|
263
302
|
state != null && retainDays != null && retainDays > 0 &&
|
|
264
303
|
computeRunHealth(state, state.context_builder_lifecycle).classification === "degraded") {
|
|
265
304
|
// Zero means unlimited retention, not the smallest TTL. Keep explicit
|
|
@@ -284,6 +323,10 @@ export async function pruneRunDirectories(runsRootDir, options) {
|
|
|
284
323
|
result.skipped += 1;
|
|
285
324
|
continue;
|
|
286
325
|
}
|
|
326
|
+
if (recipeRun && await recipeRunPrePruneDecision(entry.name, runDirPath) === "veto") {
|
|
327
|
+
result.skipped += 1;
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
287
330
|
const identity = await readSessionLogGeneration(runDirPath);
|
|
288
331
|
let sessionLogGeneration;
|
|
289
332
|
if (identity.kind === "available") {
|
|
@@ -339,6 +382,54 @@ export async function pruneRunDirectories(runsRootDir, options) {
|
|
|
339
382
|
logger.info({ runs_root_dir: runsRootDir, ...result }, "workflow run directory pruning complete");
|
|
340
383
|
return result;
|
|
341
384
|
}
|
|
385
|
+
let recipeRunPrePruneObserver;
|
|
386
|
+
/**
|
|
387
|
+
* Install the root host's pre-prune observer for recipe runs (CONTRACTS C5).
|
|
388
|
+
* It runs under the prune lock immediately before a run directory holding
|
|
389
|
+
* `recipe-state.json` is removed; "veto" keeps the directory. Process-local,
|
|
390
|
+
* like `setWorkflowActivityObserver`.
|
|
391
|
+
*/
|
|
392
|
+
export function setRecipeRunPrePruneObserver(observer) {
|
|
393
|
+
recipeRunPrePruneObserver = observer;
|
|
394
|
+
}
|
|
395
|
+
async function hasRecipeSidecar(runDirPath) {
|
|
396
|
+
try {
|
|
397
|
+
await lstat(join(runDirPath, RECIPE_STATE_FILENAME));
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
400
|
+
catch (error) {
|
|
401
|
+
if (error.code === "ENOENT")
|
|
402
|
+
return false;
|
|
403
|
+
throw error;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
async function recipeRunPrePruneDecision(runId, runDir) {
|
|
407
|
+
const observer = recipeRunPrePruneObserver;
|
|
408
|
+
let decision;
|
|
409
|
+
try {
|
|
410
|
+
if (observer !== undefined) {
|
|
411
|
+
decision = await observer({ runId, runDir });
|
|
412
|
+
}
|
|
413
|
+
else {
|
|
414
|
+
// No root host (standalone server): pruning would orphan any worktree,
|
|
415
|
+
// so allow only a sidecar that proves there is nothing to clean up.
|
|
416
|
+
const summary = await readRecipePruneSummary(runDir);
|
|
417
|
+
decision = summary.kind === "read" && summary.unresolvedOperations === 0 &&
|
|
418
|
+
summary.worktreePaths !== undefined && summary.worktreePaths.length === 0
|
|
419
|
+
? "allow"
|
|
420
|
+
: "veto";
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
catch (error) {
|
|
424
|
+
logger.warn({ run_id: runId, err: error instanceof Error ? error.message : String(error) }, "recipe pre-prune observer failed; keeping run directory");
|
|
425
|
+
return "veto";
|
|
426
|
+
}
|
|
427
|
+
if (decision !== "allow") {
|
|
428
|
+
logger.info({ run_id: runId }, "recipe run prune vetoed; keeping run directory");
|
|
429
|
+
return "veto";
|
|
430
|
+
}
|
|
431
|
+
return "allow";
|
|
432
|
+
}
|
|
342
433
|
function isTerminalRunStatus(status) {
|
|
343
434
|
return status === "succeeded" || status === "failed" || status === "cancelled";
|
|
344
435
|
}
|
|
@@ -413,7 +504,16 @@ export class WorkflowStateStore {
|
|
|
413
504
|
heartbeatError;
|
|
414
505
|
persistInvocationAuditV2;
|
|
415
506
|
persistObservedInvocation;
|
|
507
|
+
/** Last committed recipe sidecar; `undefined` until loaded or when absent. */
|
|
508
|
+
recipeStateCache;
|
|
509
|
+
recipeStateLoaded = false;
|
|
510
|
+
writer;
|
|
416
511
|
constructor(options) {
|
|
512
|
+
this.writer = options.fileWriter;
|
|
513
|
+
if (options.recipeState !== undefined) {
|
|
514
|
+
this.recipeStateCache = options.recipeState;
|
|
515
|
+
this.recipeStateLoaded = true;
|
|
516
|
+
}
|
|
417
517
|
this.runDir = options.runDir;
|
|
418
518
|
this.sessionLogGeneration = options.sessionLogGeneration;
|
|
419
519
|
this.duplicateOfFailedRun = options.duplicateOfFailedRun;
|
|
@@ -448,14 +548,28 @@ export class WorkflowStateStore {
|
|
|
448
548
|
isStaleRunningRecoveryCheckpoint(this.state))) {
|
|
449
549
|
return;
|
|
450
550
|
}
|
|
551
|
+
let recipeOverride;
|
|
552
|
+
if (isTerminalRunStatus(event.state)) {
|
|
553
|
+
try {
|
|
554
|
+
recipeOverride = await this.recipeTerminalOverrideLocked(event.state);
|
|
555
|
+
}
|
|
556
|
+
catch (error) {
|
|
557
|
+
if (!(error instanceof RecipeTerminalUnrecordedError))
|
|
558
|
+
throw error;
|
|
559
|
+
// Throwing here would replace the engine's own error; the
|
|
560
|
+
// owner's settlement retries through mark*.
|
|
561
|
+
logger.error({ run_id: this.state.runId, err: error.message }, "recipe terminal state left unpublished");
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
}
|
|
451
565
|
const nextState = structuredClone(this.state);
|
|
452
|
-
nextState.status = event.state;
|
|
566
|
+
nextState.status = recipeOverride?.status ?? event.state;
|
|
453
567
|
// First emit to set startedAt wins: a fresh run fills an empty
|
|
454
568
|
// field, and a resume can never overwrite the original run start.
|
|
455
569
|
nextState.startedAt = nextState.startedAt ?? event.startedAt;
|
|
456
570
|
nextState.completedAt = event.completedAt;
|
|
457
|
-
nextState.error = cappedError;
|
|
458
|
-
nextState.failureClass = event.failureClass;
|
|
571
|
+
nextState.error = recipeOverride === undefined ? cappedError : recipeOverride.error;
|
|
572
|
+
nextState.failureClass = recipeOverride === undefined ? event.failureClass : recipeOverride.failureClass;
|
|
459
573
|
this.state = nextState;
|
|
460
574
|
await this.writeState();
|
|
461
575
|
});
|
|
@@ -463,7 +577,7 @@ export class WorkflowStateStore {
|
|
|
463
577
|
onCallPrepared: async (event) => {
|
|
464
578
|
await this.enqueueWrite(async () => {
|
|
465
579
|
const inputPath = `${event.stageId}.${event.callId}.input.json`;
|
|
466
|
-
await writeJsonAtomic(join(this.runDir, inputPath), event.input);
|
|
580
|
+
await writeJsonAtomic(this.writer, join(this.runDir, inputPath), event.input);
|
|
467
581
|
const nextState = structuredClone(this.state);
|
|
468
582
|
const callEntry = getCallEntry(nextState, event.stageId, event.callId);
|
|
469
583
|
callEntry.materializedInputPath = inputPath;
|
|
@@ -509,6 +623,9 @@ export class WorkflowStateStore {
|
|
|
509
623
|
callEntry.failureClass = undefined;
|
|
510
624
|
}
|
|
511
625
|
callEntry.childDebugMetadata = event.childDebugMetadata;
|
|
626
|
+
// Same lifetime as childDebugMetadata: only the failed emit that
|
|
627
|
+
// observed the marker carries it, and any later emit clears it.
|
|
628
|
+
callEntry.failureKind = event.state === "failed" ? event.failureKind : undefined;
|
|
512
629
|
callEntry.contextProducing = callEntry.contextProducing ?? false;
|
|
513
630
|
// Observed-invocation telemetry rides only the terminal succeeded
|
|
514
631
|
// emit; running/failed events carry none and must not write it.
|
|
@@ -518,7 +635,7 @@ export class WorkflowStateStore {
|
|
|
518
635
|
callEntry.observedInvocation = event.observedInvocation;
|
|
519
636
|
}
|
|
520
637
|
if (event.outputText != null) {
|
|
521
|
-
const { outputInline, outputPath } = await persistOutput(this.runDir, event.stageId, event.callId, event.outputText);
|
|
638
|
+
const { outputInline, outputPath } = await persistOutput(this.writer, this.runDir, event.stageId, event.callId, event.outputText);
|
|
522
639
|
callEntry.outputInline = outputInline;
|
|
523
640
|
callEntry.outputPath = outputPath;
|
|
524
641
|
}
|
|
@@ -545,15 +662,16 @@ export class WorkflowStateStore {
|
|
|
545
662
|
static async create(options) {
|
|
546
663
|
const runDir = options.runDir ?? getRunStateDir(options.runId);
|
|
547
664
|
const debugKeepArtifacts = options.debugKeepArtifacts ?? false;
|
|
548
|
-
|
|
549
|
-
|
|
665
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
666
|
+
await pruneRunDirectoriesBeforeCreate(runDir, fileWriter);
|
|
667
|
+
const recycled = await prepareRunDirForCreate(runDir, options.runId, fileWriter);
|
|
550
668
|
let lockHeld = false;
|
|
551
669
|
try {
|
|
552
670
|
const lock = await acquireRunLock(runDir, { declareHeartbeat: true });
|
|
553
671
|
lockHeld = true;
|
|
554
672
|
const sessionLogGeneration = await createSessionLogGeneration(runDir);
|
|
555
|
-
const workflowSnapshotHash = await writeJsonSnapshot(join(runDir, WORKFLOW_SNAPSHOT_FILENAME), options.definition);
|
|
556
|
-
const inputsSnapshotHash = await writeJsonSnapshot(join(runDir, INPUTS_SNAPSHOT_FILENAME), options.variables);
|
|
673
|
+
const workflowSnapshotHash = await writeJsonSnapshot(fileWriter, join(runDir, WORKFLOW_SNAPSHOT_FILENAME), options.definition);
|
|
674
|
+
const inputsSnapshotHash = await writeJsonSnapshot(fileWriter, join(runDir, INPUTS_SNAPSHOT_FILENAME), options.variables);
|
|
557
675
|
const duplicateOfFailedRun = await detectDuplicateReplay(runDir, workflowSnapshotHash, inputsSnapshotHash);
|
|
558
676
|
if (duplicateOfFailedRun !== undefined) {
|
|
559
677
|
logger.warn({
|
|
@@ -569,8 +687,7 @@ export class WorkflowStateStore {
|
|
|
569
687
|
: undefined;
|
|
570
688
|
if (fingerprintData != null) {
|
|
571
689
|
const childConfigDir = join(runDir, CHILD_CONFIG_DIRNAME);
|
|
572
|
-
await
|
|
573
|
-
await writeJsonAtomic(join(childConfigDir, CHILD_CONFIG_SNAPSHOT_FILENAME), fingerprintData.snapshot);
|
|
690
|
+
await writeJsonAtomic(fileWriter, join(childConfigDir, CHILD_CONFIG_SNAPSHOT_FILENAME), fingerprintData.snapshot);
|
|
574
691
|
}
|
|
575
692
|
const resolvedRoster = options.resolvedRoster !== undefined
|
|
576
693
|
? ResolvedRosterSchema.parse(options.resolvedRoster)
|
|
@@ -612,7 +729,7 @@ export class WorkflowStateStore {
|
|
|
612
729
|
...(resolvedRoster !== undefined ? { resolvedRoster } : {}),
|
|
613
730
|
...(rosterName !== undefined ? { rosterName } : {}),
|
|
614
731
|
};
|
|
615
|
-
await
|
|
732
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), state);
|
|
616
733
|
if (recycled.recycled) {
|
|
617
734
|
await rm(`${runDir}.recycled`, { recursive: true, force: true }).catch(() => { });
|
|
618
735
|
if (recycled.retiredSessionLogGeneration !== undefined) {
|
|
@@ -629,6 +746,7 @@ export class WorkflowStateStore {
|
|
|
629
746
|
lockToken: lock.token,
|
|
630
747
|
sessionLogGeneration,
|
|
631
748
|
duplicateOfFailedRun,
|
|
749
|
+
fileWriter,
|
|
632
750
|
});
|
|
633
751
|
}
|
|
634
752
|
catch (error) {
|
|
@@ -653,9 +771,19 @@ export class WorkflowStateStore {
|
|
|
653
771
|
static async resume(options) {
|
|
654
772
|
const runDir = options.runDir ?? getRunStateDir(options.runId);
|
|
655
773
|
const debugKeepArtifacts = options.debugKeepArtifacts ?? false;
|
|
774
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
656
775
|
const lock = await acquireRunLock(runDir, { declareHeartbeat: true });
|
|
657
776
|
try {
|
|
777
|
+
// CONTRACTS 0-7: a recipe run is never generically resumed. Checked
|
|
778
|
+
// before the state read so no recovery rewrite can touch it; an lstat
|
|
779
|
+
// failure other than ENOENT also refuses (fail closed).
|
|
780
|
+
if (await recipeSidecarPresence(runDir) === "present") {
|
|
781
|
+
throw new Error(`Run ${options.runId} is a recipe run and cannot be resumed; use workflow_recipe_decide or cancel_workflow.`);
|
|
782
|
+
}
|
|
658
783
|
const state = await readRunState(runDir);
|
|
784
|
+
if (isSidecarRecipeWorkflow(state.workflowName)) {
|
|
785
|
+
throw new Error(`Run ${options.runId} is a recipe run and cannot be resumed; use workflow_recipe_decide or cancel_workflow.`);
|
|
786
|
+
}
|
|
659
787
|
const identity = await readSessionLogGeneration(runDir);
|
|
660
788
|
if (identity.kind === "unavailable") {
|
|
661
789
|
throw new Error(`Run ${options.runId} has an unsafe session-log identity`);
|
|
@@ -733,7 +861,7 @@ export class WorkflowStateStore {
|
|
|
733
861
|
stage.completedAt = deriveStageCompletedAt(stage.calls);
|
|
734
862
|
}
|
|
735
863
|
if (mutated) {
|
|
736
|
-
await
|
|
864
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), recovered);
|
|
737
865
|
}
|
|
738
866
|
const resume = await buildResumeState(definition, recovered, runDir);
|
|
739
867
|
return new WorkflowStateStore({
|
|
@@ -744,6 +872,95 @@ export class WorkflowStateStore {
|
|
|
744
872
|
resume,
|
|
745
873
|
debugKeepArtifacts,
|
|
746
874
|
lockToken: lock.token,
|
|
875
|
+
fileWriter,
|
|
876
|
+
...(identity.kind === "available"
|
|
877
|
+
? { sessionLogGeneration: identity.sessionLogGeneration }
|
|
878
|
+
: {}),
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
catch (error) {
|
|
882
|
+
await safeUnlink(join(runDir, RUN_LOCK_FILENAME));
|
|
883
|
+
throw error;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
/**
|
|
887
|
+
* Reopen a paused recipe run for a control or decision execution (CONTRACTS
|
|
888
|
+
* 0-2 step 4, 0-7). Acquires `run.lock` with a heartbeat, verifies the
|
|
889
|
+
* workflow and inputs snapshot hashes, and requires terminal `succeeded`
|
|
890
|
+
* plus a sidecar at `awaiting_decision`. Read-only: nothing is written until
|
|
891
|
+
* the caller validates and calls `beginRecipeExecution()`. Calls are never
|
|
892
|
+
* generically rewritten, and the child-config fingerprint is not consulted
|
|
893
|
+
* (a pause can outlive adapter edits; the snapshots pin the definition).
|
|
894
|
+
* On refusal the lock is released and a `RecipeReopenError` is thrown.
|
|
895
|
+
*/
|
|
896
|
+
static async reopen(options) {
|
|
897
|
+
const runDir = options.runDir ?? getRunStateDir(options.runId);
|
|
898
|
+
const debugKeepArtifacts = options.debugKeepArtifacts ?? false;
|
|
899
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
900
|
+
if (!await fileExists(join(runDir, STATE_FILENAME))) {
|
|
901
|
+
throw new RecipeReopenError("not_found", `Run ${options.runId} not found`);
|
|
902
|
+
}
|
|
903
|
+
let lock;
|
|
904
|
+
try {
|
|
905
|
+
lock = await acquireRunLock(runDir, { declareHeartbeat: true });
|
|
906
|
+
}
|
|
907
|
+
catch (error) {
|
|
908
|
+
throw new RecipeReopenError("run_busy", `Run ${options.runId} is owned by another execution: ${error instanceof Error ? error.message : String(error)}`);
|
|
909
|
+
}
|
|
910
|
+
try {
|
|
911
|
+
let state;
|
|
912
|
+
try {
|
|
913
|
+
state = await readRunState(runDir);
|
|
914
|
+
}
|
|
915
|
+
catch (error) {
|
|
916
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} state is unreadable: ${error instanceof Error ? error.message : String(error)}`);
|
|
917
|
+
}
|
|
918
|
+
if (state.workflowName !== PING_PONG_WORKFLOW || state.runId !== options.runId) {
|
|
919
|
+
throw new RecipeReopenError("not_recipe", `Run ${options.runId} is not a ${PING_PONG_WORKFLOW} run`);
|
|
920
|
+
}
|
|
921
|
+
const identity = await readSessionLogGeneration(runDir);
|
|
922
|
+
if (identity.kind === "unavailable") {
|
|
923
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} has an unsafe session-log identity`);
|
|
924
|
+
}
|
|
925
|
+
const workflowSnapshotPath = join(runDir, state.workflowSnapshotPath ?? WORKFLOW_SNAPSHOT_FILENAME);
|
|
926
|
+
const inputsSnapshotPath = join(runDir, state.inputsSnapshotPath ?? INPUTS_SNAPSHOT_FILENAME);
|
|
927
|
+
let definition;
|
|
928
|
+
let variables;
|
|
929
|
+
try {
|
|
930
|
+
await verifySnapshotHash(workflowSnapshotPath, state.workflowSnapshotHash, "workflow snapshot");
|
|
931
|
+
await verifySnapshotHash(inputsSnapshotPath, state.inputsSnapshotHash, "inputs snapshot");
|
|
932
|
+
definition = await readJsonFile(workflowSnapshotPath);
|
|
933
|
+
variables = await readJsonFile(inputsSnapshotPath);
|
|
934
|
+
}
|
|
935
|
+
catch (error) {
|
|
936
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} snapshot verification failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
937
|
+
}
|
|
938
|
+
const recipe = await readRecipeState(runDir);
|
|
939
|
+
if (recipe.kind === "invalid") {
|
|
940
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} recipe sidecar is unreadable: ${recipe.reason}`);
|
|
941
|
+
}
|
|
942
|
+
if (recipe.kind === "absent" || state.status !== "succeeded" || recipe.state.phase !== "awaiting_decision") {
|
|
943
|
+
throw new RecipeReopenError("not_awaiting_decision", `Run ${options.runId} is not awaiting a decision (status ${state.status}, phase ${recipe.kind === "read" ? recipe.state.phase : "none"})`, recipe.kind === "read"
|
|
944
|
+
? { phase: recipe.state.phase, revision: recipe.state.revision, status: state.status }
|
|
945
|
+
: undefined);
|
|
946
|
+
}
|
|
947
|
+
let resume;
|
|
948
|
+
try {
|
|
949
|
+
resume = await buildResumeState(definition, state, runDir);
|
|
950
|
+
}
|
|
951
|
+
catch (error) {
|
|
952
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} executed calls cannot be rehydrated: ${error instanceof Error ? error.message : String(error)}`);
|
|
953
|
+
}
|
|
954
|
+
return new WorkflowStateStore({
|
|
955
|
+
runDir,
|
|
956
|
+
definition,
|
|
957
|
+
variables,
|
|
958
|
+
state,
|
|
959
|
+
resume,
|
|
960
|
+
debugKeepArtifacts,
|
|
961
|
+
lockToken: lock.token,
|
|
962
|
+
recipeState: recipe.state,
|
|
963
|
+
fileWriter,
|
|
747
964
|
...(identity.kind === "available"
|
|
748
965
|
? { sessionLogGeneration: identity.sessionLogGeneration }
|
|
749
966
|
: {}),
|
|
@@ -757,6 +974,167 @@ export class WorkflowStateStore {
|
|
|
757
974
|
get runState() {
|
|
758
975
|
return structuredClone(this.state);
|
|
759
976
|
}
|
|
977
|
+
/** The writer every file of this run goes through (durable in production). */
|
|
978
|
+
get fileWriter() {
|
|
979
|
+
return this.writer;
|
|
980
|
+
}
|
|
981
|
+
/** Last committed recipe sidecar held by this store (reopen loads it; commits refresh it). */
|
|
982
|
+
get recipeState() {
|
|
983
|
+
return this.recipeStateCache === undefined ? undefined : structuredClone(this.recipeStateCache);
|
|
984
|
+
}
|
|
985
|
+
/**
|
|
986
|
+
* Turn a reopened terminal envelope into a new active execution: status
|
|
987
|
+
* `running`, terminal fields and the previous execution id cleared. The
|
|
988
|
+
* caller then binds the new id (`bindExecution` via the run owner) before
|
|
989
|
+
* any sidecar commit. A crash in between leaves `running` with no id, which
|
|
990
|
+
* reconcile fails rather than settling the old pin.
|
|
991
|
+
*/
|
|
992
|
+
async beginRecipeExecution() {
|
|
993
|
+
await this.enqueueWrite(async () => {
|
|
994
|
+
if (!isTerminalRunStatus(this.state.status)) {
|
|
995
|
+
throw new RecipeStateInvariantError(`beginRecipeExecution requires a terminal run, found ${this.state.status}`);
|
|
996
|
+
}
|
|
997
|
+
const nextState = structuredClone(this.state);
|
|
998
|
+
nextState.status = "running";
|
|
999
|
+
nextState.completedAt = undefined;
|
|
1000
|
+
nextState.error = undefined;
|
|
1001
|
+
nextState.failureClass = undefined;
|
|
1002
|
+
nextState.executionId = undefined;
|
|
1003
|
+
this.state = nextState;
|
|
1004
|
+
await this.writeState();
|
|
1005
|
+
});
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Commit `recipe-state.json` on this store's write chain under its
|
|
1009
|
+
* `run.lock` (CONTRACTS 0-7). The store stamps `execution_id` (the bound
|
|
1010
|
+
* id; unbound -> error), `revision` (previous + 1), `updated_at` and
|
|
1011
|
+
* `unresolved_operations`, enforces the phase machine and the 0-4 byte
|
|
1012
|
+
* budget, and returns the committed state. Over budget throws
|
|
1013
|
+
* `RecipeStateCapacityError`; a non-terminal write may use 1 MiB minus the
|
|
1014
|
+
* 64 KiB terminal reserve. State.json is not touched.
|
|
1015
|
+
*/
|
|
1016
|
+
async commitRecipeState(draft) {
|
|
1017
|
+
let committed;
|
|
1018
|
+
await this.enqueueWrite(async () => {
|
|
1019
|
+
committed = await this.writeRecipeStateLocked(draft);
|
|
1020
|
+
});
|
|
1021
|
+
return committed;
|
|
1022
|
+
}
|
|
1023
|
+
/**
|
|
1024
|
+
* Terminal recipe commit in ONE critical section: the terminal sidecar first,
|
|
1025
|
+
* then the matching terminal `state.json` (`qualified`/`stopped` ->
|
|
1026
|
+
* succeeded, `failed` -> failed, `cancelled` -> cancelled). A crash between
|
|
1027
|
+
* the two is rolled forward by reconcile. Used by control executions
|
|
1028
|
+
* (decision stop, deadline, paused cancel) and by the runtime for a terminal
|
|
1029
|
+
* stop it publishes itself.
|
|
1030
|
+
*/
|
|
1031
|
+
async commitRecipeTerminal(draft, options = {}) {
|
|
1032
|
+
let committed;
|
|
1033
|
+
await this.enqueueWrite(async () => {
|
|
1034
|
+
if (!isTerminalRecipePhase(draft.phase) || draft.outcome === undefined) {
|
|
1035
|
+
throw new RecipeStateInvariantError("commitRecipeTerminal requires a terminal phase and outcome");
|
|
1036
|
+
}
|
|
1037
|
+
if (!(this.state.status === "running" || this.state.status === "pending")) {
|
|
1038
|
+
throw new RecipeStateInvariantError(`commitRecipeTerminal requires an active execution, found ${this.state.status}`);
|
|
1039
|
+
}
|
|
1040
|
+
committed = await this.writeRecipeStateLocked(draft);
|
|
1041
|
+
const status = runStatusForOutcome(draft.outcome);
|
|
1042
|
+
const nextState = structuredClone(this.state);
|
|
1043
|
+
nextState.status = status;
|
|
1044
|
+
nextState.completedAt = new Date().toISOString();
|
|
1045
|
+
nextState.error = capPersistedError(options.error ?? (status === "cancelled" ? "Workflow cancelled" : status === "failed" ? `Recipe stopped: ${draft.stop_reason ?? "failed"}` : undefined));
|
|
1046
|
+
nextState.failureClass = status === "failed" ? options.failureClass ?? "permanent" : options.failureClass;
|
|
1047
|
+
this.state = nextState;
|
|
1048
|
+
await this.writeState();
|
|
1049
|
+
});
|
|
1050
|
+
return committed;
|
|
1051
|
+
}
|
|
1052
|
+
/**
|
|
1053
|
+
* D1 terminal precedence for a recipe run, inside the write chain: the
|
|
1054
|
+
* terminal fields to publish INSTEAD of `intent`, or `undefined` to publish
|
|
1055
|
+
* `intent` as the caller would. A committed terminal sidecar of this
|
|
1056
|
+
* execution decides; a non-terminal one is terminalized first (cancelled or
|
|
1057
|
+
* failed), so the sidecar is never behind the state that settles it. A
|
|
1058
|
+
* failed sidecar commit throws `RecipeTerminalUnrecordedError` and the
|
|
1059
|
+
* caller must leave state.json active, as after a crash between the two
|
|
1060
|
+
* writes. Non-recipe runs return `undefined` without any I/O.
|
|
1061
|
+
*/
|
|
1062
|
+
async recipeTerminalOverrideLocked(intent) {
|
|
1063
|
+
if (!isSidecarRecipeWorkflow(this.state.workflowName))
|
|
1064
|
+
return undefined;
|
|
1065
|
+
const authority = await this.recipeAuthorityLocked();
|
|
1066
|
+
switch (authority.kind) {
|
|
1067
|
+
case "none":
|
|
1068
|
+
case "unbound":
|
|
1069
|
+
case "foreign":
|
|
1070
|
+
return undefined;
|
|
1071
|
+
case "unreadable":
|
|
1072
|
+
logger.error({ run_id: this.state.runId, reason: authority.reason }, "recipe sidecar is unreadable; publishing the terminal intent without deriving an outcome");
|
|
1073
|
+
return undefined;
|
|
1074
|
+
case "terminal":
|
|
1075
|
+
return authority.status === intent
|
|
1076
|
+
? undefined
|
|
1077
|
+
: recipeOutcomeEnvelope(authority.outcome, authority.state.stop_reason);
|
|
1078
|
+
case "active":
|
|
1079
|
+
if (intent === "succeeded")
|
|
1080
|
+
return undefined;
|
|
1081
|
+
try {
|
|
1082
|
+
await this.writeRecipeStateLocked(recipeSettlementDraft(authority.state, intent, settlementNote(intent)));
|
|
1083
|
+
}
|
|
1084
|
+
catch (error) {
|
|
1085
|
+
// Published bytes count as committed (the cache already holds them).
|
|
1086
|
+
if (!(error instanceof DurablePublishError))
|
|
1087
|
+
throw new RecipeTerminalUnrecordedError(intent, error);
|
|
1088
|
+
}
|
|
1089
|
+
return undefined;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
/** Runs inside the write chain. */
|
|
1093
|
+
async recipeAuthorityLocked() {
|
|
1094
|
+
if (!this.recipeStateLoaded) {
|
|
1095
|
+
const current = await readRecipeState(this.runDir);
|
|
1096
|
+
if (current.kind === "invalid")
|
|
1097
|
+
return classifyRecipeTerminalAuthority(current, this.state.executionId);
|
|
1098
|
+
this.recipeStateCache = current.kind === "read" ? current.state : undefined;
|
|
1099
|
+
this.recipeStateLoaded = true;
|
|
1100
|
+
}
|
|
1101
|
+
return classifyRecipeTerminalAuthority(this.recipeStateCache === undefined ? { kind: "absent" } : { kind: "read", state: this.recipeStateCache }, this.state.executionId);
|
|
1102
|
+
}
|
|
1103
|
+
/** Runs inside the write chain; never enqueue from here. */
|
|
1104
|
+
async writeRecipeStateLocked(draft) {
|
|
1105
|
+
if (!this.lockHeld) {
|
|
1106
|
+
throw new RecipeStateInvariantError("recipe state can only be committed while this store holds run.lock");
|
|
1107
|
+
}
|
|
1108
|
+
const executionId = this.state.executionId;
|
|
1109
|
+
if (executionId === undefined) {
|
|
1110
|
+
throw new RecipeStateInvariantError("recipe state can only be committed by a bound execution");
|
|
1111
|
+
}
|
|
1112
|
+
if (!this.recipeStateLoaded) {
|
|
1113
|
+
const current = await readRecipeState(this.runDir);
|
|
1114
|
+
if (current.kind === "invalid") {
|
|
1115
|
+
throw new RecipeStateInvariantError(`existing recipe sidecar is unreadable: ${current.reason}`);
|
|
1116
|
+
}
|
|
1117
|
+
this.recipeStateCache = current.kind === "read" ? current.state : undefined;
|
|
1118
|
+
this.recipeStateLoaded = true;
|
|
1119
|
+
}
|
|
1120
|
+
const next = stampRecipeState(draft, {
|
|
1121
|
+
runId: this.state.runId,
|
|
1122
|
+
executionId,
|
|
1123
|
+
previous: this.recipeStateCache,
|
|
1124
|
+
});
|
|
1125
|
+
try {
|
|
1126
|
+
await writeRecipeStateAtomic(this.runDir, next, this.writer);
|
|
1127
|
+
}
|
|
1128
|
+
catch (error) {
|
|
1129
|
+
// The bytes are already visible; the next commit must build on them, or
|
|
1130
|
+
// it would reuse this revision.
|
|
1131
|
+
if (error instanceof DurablePublishError)
|
|
1132
|
+
this.recipeStateCache = next;
|
|
1133
|
+
throw error;
|
|
1134
|
+
}
|
|
1135
|
+
this.recipeStateCache = next;
|
|
1136
|
+
return structuredClone(next);
|
|
1137
|
+
}
|
|
760
1138
|
/**
|
|
761
1139
|
* Open a new generic resume attempt before any context preflight or provider
|
|
762
1140
|
* setup. Provider-call attempt counters remain untouched; only the run-level
|
|
@@ -787,6 +1165,23 @@ export class WorkflowStateStore {
|
|
|
787
1165
|
nextState.completedAt = undefined;
|
|
788
1166
|
nextState.error = undefined;
|
|
789
1167
|
nextState.failureClass = undefined;
|
|
1168
|
+
// The reopened record belongs to no execution until bind stamps the new
|
|
1169
|
+
// id; a crash in between must not let reconcile/reap settle the old pin.
|
|
1170
|
+
nextState.executionId = undefined;
|
|
1171
|
+
this.state = nextState;
|
|
1172
|
+
await this.writeState();
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
/**
|
|
1176
|
+
* Persist the bound execution identity. Every later write clones this.state,
|
|
1177
|
+
* so the id rides each subsequent publication; a resume overwrites it.
|
|
1178
|
+
*/
|
|
1179
|
+
async bindExecution(executionId) {
|
|
1180
|
+
await this.enqueueWrite(async () => {
|
|
1181
|
+
if (this.state.executionId === executionId)
|
|
1182
|
+
return;
|
|
1183
|
+
const nextState = structuredClone(this.state);
|
|
1184
|
+
nextState.executionId = executionId;
|
|
790
1185
|
this.state = nextState;
|
|
791
1186
|
await this.writeState();
|
|
792
1187
|
});
|
|
@@ -805,10 +1200,13 @@ export class WorkflowStateStore {
|
|
|
805
1200
|
if (this.state.status !== "pending" && this.state.status !== "running") {
|
|
806
1201
|
return;
|
|
807
1202
|
}
|
|
1203
|
+
const recipeOverride = await this.recipeTerminalOverrideLocked("cancelled");
|
|
808
1204
|
const nextState = structuredClone(this.state);
|
|
809
|
-
nextState.status = "cancelled";
|
|
1205
|
+
nextState.status = recipeOverride?.status ?? "cancelled";
|
|
810
1206
|
nextState.completedAt = new Date().toISOString();
|
|
811
|
-
nextState.error = reason;
|
|
1207
|
+
nextState.error = recipeOverride === undefined ? reason : recipeOverride.error;
|
|
1208
|
+
if (recipeOverride !== undefined)
|
|
1209
|
+
nextState.failureClass = recipeOverride.failureClass;
|
|
812
1210
|
this.state = nextState;
|
|
813
1211
|
await this.writeState();
|
|
814
1212
|
});
|
|
@@ -835,11 +1233,15 @@ export class WorkflowStateStore {
|
|
|
835
1233
|
if (this.state.status !== "pending" && this.state.status !== "running") {
|
|
836
1234
|
return;
|
|
837
1235
|
}
|
|
1236
|
+
const recipeOverride = await this.recipeTerminalOverrideLocked("failed");
|
|
838
1237
|
const nextState = structuredClone(this.state);
|
|
839
|
-
nextState.status = "failed";
|
|
1238
|
+
nextState.status = recipeOverride?.status ?? "failed";
|
|
840
1239
|
nextState.completedAt = new Date().toISOString();
|
|
841
|
-
nextState.error = cappedError;
|
|
842
|
-
if (
|
|
1240
|
+
nextState.error = recipeOverride === undefined ? cappedError : recipeOverride.error;
|
|
1241
|
+
if (recipeOverride !== undefined) {
|
|
1242
|
+
nextState.failureClass = recipeOverride.failureClass;
|
|
1243
|
+
}
|
|
1244
|
+
else if (failureClass !== undefined) {
|
|
843
1245
|
nextState.failureClass = failureClass;
|
|
844
1246
|
}
|
|
845
1247
|
this.state = nextState;
|
|
@@ -869,7 +1271,7 @@ export class WorkflowStateStore {
|
|
|
869
1271
|
},
|
|
870
1272
|
};
|
|
871
1273
|
const inputsSnapshotPath = this.state.inputsSnapshotPath ?? INPUTS_SNAPSHOT_FILENAME;
|
|
872
|
-
const inputsSnapshotHash = await writeJsonSnapshot(join(this.runDir, inputsSnapshotPath), variables);
|
|
1274
|
+
const inputsSnapshotHash = await writeJsonSnapshot(this.writer, join(this.runDir, inputsSnapshotPath), variables);
|
|
873
1275
|
const nextState = structuredClone(this.state);
|
|
874
1276
|
nextState.inputsSnapshotPath = inputsSnapshotPath;
|
|
875
1277
|
nextState.inputsSnapshotHash = inputsSnapshotHash;
|
|
@@ -879,7 +1281,7 @@ export class WorkflowStateStore {
|
|
|
879
1281
|
this.state = nextState;
|
|
880
1282
|
await this.writeState();
|
|
881
1283
|
try {
|
|
882
|
-
await writeJsonAtomic(join(getStateDir(), "workflow-contexts", `${this.state.runId}.json`), committedWorkflowContext);
|
|
1284
|
+
await writeJsonAtomic(this.writer, join(getStateDir(), "workflow-contexts", `${this.state.runId}.json`), committedWorkflowContext);
|
|
883
1285
|
}
|
|
884
1286
|
catch (error) {
|
|
885
1287
|
logger.warn({ runId: this.state.runId, err: error }, "failed to write workflow context provenance");
|
|
@@ -1147,6 +1549,7 @@ export class WorkflowStateStore {
|
|
|
1147
1549
|
intent,
|
|
1148
1550
|
chairmanCallId: resolvePlanReviewChairmanCallId(this.definition),
|
|
1149
1551
|
serializedResult: this.planReviewTerminalResult,
|
|
1552
|
+
fileWriter: this.writer,
|
|
1150
1553
|
});
|
|
1151
1554
|
validateStrictStateSubtrees(finalized.state, join(this.runDir, STATE_FILENAME));
|
|
1152
1555
|
this.state = finalized.state;
|
|
@@ -1161,20 +1564,32 @@ export class WorkflowStateStore {
|
|
|
1161
1564
|
await next;
|
|
1162
1565
|
}
|
|
1163
1566
|
async writeState() {
|
|
1164
|
-
await
|
|
1567
|
+
await writeRunStateAtomic(this.writer, join(this.runDir, STATE_FILENAME), this.state);
|
|
1165
1568
|
}
|
|
1166
1569
|
}
|
|
1167
|
-
async function prepareRunDirForCreate(runDir, runId) {
|
|
1168
|
-
await
|
|
1570
|
+
async function prepareRunDirForCreate(runDir, runId, fileWriter) {
|
|
1571
|
+
await mkdirDurable(fileWriter, dirname(runDir));
|
|
1572
|
+
let created = false;
|
|
1169
1573
|
try {
|
|
1170
1574
|
await mkdir(runDir);
|
|
1171
|
-
|
|
1575
|
+
created = true;
|
|
1172
1576
|
}
|
|
1173
1577
|
catch (error) {
|
|
1174
1578
|
if (!(error instanceof Error) || !("code" in error) || error.code !== "EEXIST") {
|
|
1175
1579
|
throw error;
|
|
1176
1580
|
}
|
|
1177
1581
|
}
|
|
1582
|
+
if (created) {
|
|
1583
|
+
try {
|
|
1584
|
+
await fileWriter.syncDirectory(dirname(runDir));
|
|
1585
|
+
}
|
|
1586
|
+
catch (error) {
|
|
1587
|
+
// Nothing is in the directory yet; leave no half-admitted run behind.
|
|
1588
|
+
await rmdir(runDir).catch(() => { });
|
|
1589
|
+
throw error;
|
|
1590
|
+
}
|
|
1591
|
+
return { recycled: false };
|
|
1592
|
+
}
|
|
1178
1593
|
const recycled = await recycleSucceededRunDir(runDir);
|
|
1179
1594
|
if (recycled.recycled) {
|
|
1180
1595
|
const recycledPath = `${runDir}.recycled`;
|
|
@@ -1193,18 +1608,32 @@ async function prepareRunDirForCreate(runDir, runId) {
|
|
|
1193
1608
|
}
|
|
1194
1609
|
throw error;
|
|
1195
1610
|
}
|
|
1611
|
+
// One fsync publishes both the retirement rename and the fresh directory.
|
|
1612
|
+
try {
|
|
1613
|
+
await fileWriter.syncDirectory(dirname(runDir));
|
|
1614
|
+
}
|
|
1615
|
+
catch (error) {
|
|
1616
|
+
await rmdir(runDir).catch(() => { });
|
|
1617
|
+
try {
|
|
1618
|
+
await rename(recycledPath, runDir);
|
|
1619
|
+
}
|
|
1620
|
+
catch {
|
|
1621
|
+
logger.warn({ recycledPath }, "failed to restore recycled directory after runs-root fsync failure");
|
|
1622
|
+
}
|
|
1623
|
+
throw error;
|
|
1624
|
+
}
|
|
1196
1625
|
return recycled;
|
|
1197
1626
|
}
|
|
1198
1627
|
throw duplicateRunDirError(runId);
|
|
1199
1628
|
}
|
|
1200
|
-
async function pruneRunDirectoriesBeforeCreate(runDir) {
|
|
1629
|
+
async function pruneRunDirectoriesBeforeCreate(runDir, fileWriter) {
|
|
1201
1630
|
const runsRoot = getRunsDir();
|
|
1202
1631
|
if (dirname(runDir) !== runsRoot)
|
|
1203
1632
|
return;
|
|
1204
1633
|
// Reconcile before pruning so a stale running run terminalized here becomes
|
|
1205
1634
|
// prunable by a later create once past its retention window.
|
|
1206
1635
|
try {
|
|
1207
|
-
await reconcileStaleRunDirectories(runsRoot);
|
|
1636
|
+
await reconcileStaleRunDirectories(runsRoot, { fileWriter });
|
|
1208
1637
|
}
|
|
1209
1638
|
catch (error) {
|
|
1210
1639
|
logger.warn({ runs_root_dir: runsRoot, err: error instanceof Error ? error.message : String(error) }, "stale workflow run reconcile failed before create");
|
|
@@ -1509,11 +1938,13 @@ function classifyRunLockSync(lockPath) {
|
|
|
1509
1938
|
return code === "ENOENT" ? "stale_or_absent" : "unknown";
|
|
1510
1939
|
}
|
|
1511
1940
|
}
|
|
1512
|
-
export async function readRunStateForStatus(runId) {
|
|
1941
|
+
export async function readRunStateForStatus(runId, containedRead = false) {
|
|
1513
1942
|
const runDir = getRunStateDir(runId);
|
|
1514
|
-
let
|
|
1943
|
+
let read;
|
|
1515
1944
|
try {
|
|
1516
|
-
|
|
1945
|
+
read = containedRead
|
|
1946
|
+
? await readContainedRunState(runDir)
|
|
1947
|
+
: { state: await readRunState(runDir) };
|
|
1517
1948
|
}
|
|
1518
1949
|
catch (error) {
|
|
1519
1950
|
const code = error.code;
|
|
@@ -1524,37 +1955,15 @@ export async function readRunStateForStatus(runId) {
|
|
|
1524
1955
|
}
|
|
1525
1956
|
const lockStatus = classifyRunLockSync(join(runDir, RUN_LOCK_FILENAME));
|
|
1526
1957
|
return {
|
|
1527
|
-
state,
|
|
1958
|
+
state: read.state,
|
|
1528
1959
|
lock: lockStatus,
|
|
1960
|
+
// Internal execution-fence input only; never projected onto the wire.
|
|
1961
|
+
...(read.digest !== undefined ? { stateDigest: read.digest } : {}),
|
|
1529
1962
|
};
|
|
1530
1963
|
}
|
|
1531
|
-
|
|
1532
|
-
* Reap a run that has no live worker (orphaned/stale/prior-process) by writing a
|
|
1533
|
-
* terminal "cancelled" state and unlinking its lock. Used by cancel_workflow when
|
|
1534
|
-
* the run is not in this process's in-memory cancel registry.
|
|
1535
|
-
*
|
|
1536
|
-
* The run.lock is the single source of truth for ownership: this acquires the
|
|
1537
|
-
* lock itself (reusing acquireRunLock, which atomically creates the lock and
|
|
1538
|
-
* reclaims a stale same-host/dead-PID lock) BEFORE mutating, closing the TOCTOU
|
|
1539
|
-
* where a concurrent resume_workflow could grab the lock between a caller's
|
|
1540
|
-
* status read and the reap write.
|
|
1541
|
-
*
|
|
1542
|
-
* Returns:
|
|
1543
|
-
* - "locked" — a LIVE lock blocks acquisition (held by a running worker, or a
|
|
1544
|
-
* same-process blocking run); nothing is mutated.
|
|
1545
|
-
* - "ownership_unknown" — the lock is malformed/unreadable; nothing is mutated
|
|
1546
|
-
* or reclaimed.
|
|
1547
|
-
* - "skipped" — lock acquired but the run is already terminal (not
|
|
1548
|
-
* pending/running); the lock is released and nothing is mutated.
|
|
1549
|
-
* The caller re-reads to report already_terminal.
|
|
1550
|
-
* - "cancelled" — lock acquired and run was pending/running; terminal
|
|
1551
|
-
* "cancelled" state is written and the lock removed.
|
|
1552
|
-
*
|
|
1553
|
-
* NOTE: This intentionally does NOT use WorkflowStateStore.resume() (which
|
|
1554
|
-
* recovers + re-locks the run — the wrong intent here).
|
|
1555
|
-
*/
|
|
1556
|
-
export async function reapRunCancelled(runId) {
|
|
1964
|
+
export async function reapRunCancelled(runId, options = {}) {
|
|
1557
1965
|
const runDir = getRunStateDir(runId);
|
|
1966
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
1558
1967
|
try {
|
|
1559
1968
|
await acquireRunLock(runDir);
|
|
1560
1969
|
}
|
|
@@ -1573,6 +1982,19 @@ export async function reapRunCancelled(runId) {
|
|
|
1573
1982
|
if (state.status !== "pending" && state.status !== "running") {
|
|
1574
1983
|
return "skipped";
|
|
1575
1984
|
}
|
|
1985
|
+
let disposition = { kind: "cancel" };
|
|
1986
|
+
if (options.beforeWrite !== undefined) {
|
|
1987
|
+
try {
|
|
1988
|
+
disposition = (await options.beforeWrite({ runDir, state, fileWriter })) ?? { kind: "cancel" };
|
|
1989
|
+
}
|
|
1990
|
+
catch (error) {
|
|
1991
|
+
disposition = { kind: "unavailable", reason: error instanceof Error ? error.message : String(error) };
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1994
|
+
if (disposition.kind === "unavailable") {
|
|
1995
|
+
logger.error({ run_id: runId, reason: disposition.reason }, "orphan cancel refused: the recipe sidecar could not be read or recorded; nothing written");
|
|
1996
|
+
return "recipe_state_unavailable";
|
|
1997
|
+
}
|
|
1576
1998
|
let nextState = structuredClone(state);
|
|
1577
1999
|
if (isPlanReviewRun(state)) {
|
|
1578
2000
|
const intent = {
|
|
@@ -1590,19 +2012,26 @@ export async function reapRunCancelled(runId) {
|
|
|
1590
2012
|
previousState: state,
|
|
1591
2013
|
intendedState: intended,
|
|
1592
2014
|
intent,
|
|
2015
|
+
fileWriter,
|
|
1593
2016
|
});
|
|
1594
2017
|
nextState = finalized.state;
|
|
1595
2018
|
validateStrictStateSubtrees(nextState, join(runDir, STATE_FILENAME));
|
|
1596
2019
|
}
|
|
2020
|
+
else if (disposition.kind === "settled") {
|
|
2021
|
+
nextState.status = disposition.envelope.status;
|
|
2022
|
+
nextState.completedAt = new Date().toISOString();
|
|
2023
|
+
nextState.error = disposition.envelope.error;
|
|
2024
|
+
nextState.failureClass = disposition.envelope.failureClass;
|
|
2025
|
+
}
|
|
1597
2026
|
else {
|
|
1598
2027
|
nextState.status = "cancelled";
|
|
1599
2028
|
nextState.completedAt = new Date().toISOString();
|
|
1600
2029
|
nextState.error = "Reaped: no live worker process";
|
|
1601
2030
|
}
|
|
1602
|
-
await
|
|
2031
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), nextState);
|
|
1603
2032
|
await safeUnlink(lockPath);
|
|
1604
2033
|
lockHeld = false;
|
|
1605
|
-
return "cancelled";
|
|
2034
|
+
return disposition.kind === "settled" ? "settled" : "cancelled";
|
|
1606
2035
|
}
|
|
1607
2036
|
finally {
|
|
1608
2037
|
if (lockHeld) {
|
|
@@ -1632,7 +2061,8 @@ export async function reapRunCancelled(runId) {
|
|
|
1632
2061
|
* once, then release. Idempotent: reconciled runs are terminal `failed` and
|
|
1633
2062
|
* count as skipped_terminal on re-scan.
|
|
1634
2063
|
*/
|
|
1635
|
-
export async function reconcileStaleRunDirectories(runsRootDir) {
|
|
2064
|
+
export async function reconcileStaleRunDirectories(runsRootDir, options = {}) {
|
|
2065
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
1636
2066
|
const result = {
|
|
1637
2067
|
scanned: 0,
|
|
1638
2068
|
reconciled: 0,
|
|
@@ -1680,7 +2110,7 @@ export async function reconcileStaleRunDirectories(runsRootDir) {
|
|
|
1680
2110
|
continue;
|
|
1681
2111
|
}
|
|
1682
2112
|
try {
|
|
1683
|
-
const outcome = await reconcileStaleRun(runDirPath);
|
|
2113
|
+
const outcome = await reconcileStaleRun(runDirPath, fileWriter);
|
|
1684
2114
|
result[outcome] += 1;
|
|
1685
2115
|
}
|
|
1686
2116
|
catch (error) {
|
|
@@ -1695,7 +2125,7 @@ export async function reconcileStaleRunDirectories(runsRootDir) {
|
|
|
1695
2125
|
* Terminalize one pre-screened stale run under its own lock. See
|
|
1696
2126
|
* reconcileStaleRunDirectories for the authority and safety model.
|
|
1697
2127
|
*/
|
|
1698
|
-
async function reconcileStaleRun(runDir) {
|
|
2128
|
+
async function reconcileStaleRun(runDir, fileWriter) {
|
|
1699
2129
|
try {
|
|
1700
2130
|
await acquireRunLock(runDir);
|
|
1701
2131
|
}
|
|
@@ -1711,6 +2141,7 @@ async function reconcileStaleRun(runDir) {
|
|
|
1711
2141
|
return "skipped_terminal";
|
|
1712
2142
|
}
|
|
1713
2143
|
const reconciledAt = new Date().toISOString();
|
|
2144
|
+
const recipeRollForward = await recipeReconcileStatus(runDir, state, fileWriter);
|
|
1714
2145
|
let nextState = structuredClone(state);
|
|
1715
2146
|
for (const stage of nextState.stages) {
|
|
1716
2147
|
for (const call of stage.calls) {
|
|
@@ -1731,6 +2162,19 @@ async function reconcileStaleRun(runDir) {
|
|
|
1731
2162
|
stage.state = deriveStageState(stage.calls);
|
|
1732
2163
|
stage.completedAt = deriveStageCompletedAt(stage.calls);
|
|
1733
2164
|
}
|
|
2165
|
+
if (recipeRollForward !== undefined) {
|
|
2166
|
+
// CONTRACTS 0-7: the sidecar already records how this execution ended
|
|
2167
|
+
// (a complete pause checkpoint, or a terminal outcome); publish it.
|
|
2168
|
+
nextState.status = recipeRollForward;
|
|
2169
|
+
nextState.completedAt = reconciledAt;
|
|
2170
|
+
nextState.error = recipeRollForward === "succeeded"
|
|
2171
|
+
? undefined
|
|
2172
|
+
: recipeRollForward === "cancelled" ? "Workflow cancelled" : RECIPE_ROLL_FORWARD_FAILED_ERROR;
|
|
2173
|
+
nextState.failureClass = recipeRollForward === "failed" ? "permanent" : undefined;
|
|
2174
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), nextState);
|
|
2175
|
+
logger.info({ run_dir: runDir, run_id: state.runId, status: recipeRollForward }, "rolled stale recipe run forward from its sidecar");
|
|
2176
|
+
return "reconciled";
|
|
2177
|
+
}
|
|
1734
2178
|
if (isPlanReviewRun(state)) {
|
|
1735
2179
|
const intent = {
|
|
1736
2180
|
status: "failed",
|
|
@@ -1748,6 +2192,7 @@ async function reconcileStaleRun(runDir) {
|
|
|
1748
2192
|
previousState: state,
|
|
1749
2193
|
intendedState: intended,
|
|
1750
2194
|
intent,
|
|
2195
|
+
fileWriter,
|
|
1751
2196
|
});
|
|
1752
2197
|
nextState = finalized.state;
|
|
1753
2198
|
validateStrictStateSubtrees(nextState, join(runDir, STATE_FILENAME));
|
|
@@ -1758,7 +2203,7 @@ async function reconcileStaleRun(runDir) {
|
|
|
1758
2203
|
nextState.error = STALE_RUN_RECONCILE_ERROR;
|
|
1759
2204
|
nextState.failureClass = "transient";
|
|
1760
2205
|
}
|
|
1761
|
-
await
|
|
2206
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), nextState);
|
|
1762
2207
|
logger.info({ run_dir: runDir, run_id: state.runId }, "reconciled stale running workflow run to terminal failed");
|
|
1763
2208
|
return "reconciled";
|
|
1764
2209
|
}
|
|
@@ -1766,6 +2211,92 @@ async function reconcileStaleRun(runDir) {
|
|
|
1766
2211
|
await safeUnlink(lockPath);
|
|
1767
2212
|
}
|
|
1768
2213
|
}
|
|
2214
|
+
export const RECIPE_ROLL_FORWARD_FAILED_ERROR = "Reconciled: recipe sidecar recorded a failed outcome";
|
|
2215
|
+
/**
|
|
2216
|
+
* Reconcile rule (CONTRACTS 0-7, D1), evaluated under the lock. Only a sidecar
|
|
2217
|
+
* of THIS execution counts: a complete `awaiting_decision` checkpoint with
|
|
2218
|
+
* zero unresolved operations publishes `succeeded`; a terminal sidecar
|
|
2219
|
+
* publishes its outcome's status. Any other active sidecar of this execution
|
|
2220
|
+
* is first terminalized `stopped(recovery_required)` (operations without a
|
|
2221
|
+
* result become uncertain), as `mark*` does, so the sidecar is never behind
|
|
2222
|
+
* the `failed` state that follows; if that record cannot be committed this
|
|
2223
|
+
* throws `RecipeTerminalUnrecordedError` and state.json stays active. No,
|
|
2224
|
+
* unreadable, or another execution's sidecar is left as is, and the run gets
|
|
2225
|
+
* the ordinary stale-run failure.
|
|
2226
|
+
*/
|
|
2227
|
+
async function recipeReconcileStatus(runDir, state, fileWriter) {
|
|
2228
|
+
if (state.workflowName !== PING_PONG_WORKFLOW)
|
|
2229
|
+
return undefined;
|
|
2230
|
+
const authority = classifyRecipeTerminalAuthority(await readRecipeState(runDir), state.executionId);
|
|
2231
|
+
if (authority.kind === "terminal")
|
|
2232
|
+
return authority.status;
|
|
2233
|
+
if (authority.kind !== "active")
|
|
2234
|
+
return undefined;
|
|
2235
|
+
if (isCompleteAwaitingCheckpoint(authority.state, state.executionId))
|
|
2236
|
+
return "succeeded";
|
|
2237
|
+
const next = stampRecipeState(recipeSettlementDraft(authority.state, "failed", settlementNote("failed")), {
|
|
2238
|
+
runId: state.runId,
|
|
2239
|
+
executionId: authority.state.execution_id,
|
|
2240
|
+
previous: authority.state,
|
|
2241
|
+
});
|
|
2242
|
+
try {
|
|
2243
|
+
await writeRecipeStateAtomic(runDir, next, fileWriter);
|
|
2244
|
+
}
|
|
2245
|
+
catch (error) {
|
|
2246
|
+
// Published bytes count as committed.
|
|
2247
|
+
if (!(error instanceof DurablePublishError))
|
|
2248
|
+
throw new RecipeTerminalUnrecordedError("failed", error);
|
|
2249
|
+
}
|
|
2250
|
+
return undefined;
|
|
2251
|
+
}
|
|
2252
|
+
/**
|
|
2253
|
+
* `reapRunCancelled` beforeWrite hook for an orphaned ping-pong run (CONTRACTS
|
|
2254
|
+
* 0-7, D1), under the lock. Only a sidecar of THIS execution is ever written
|
|
2255
|
+
* or decides:
|
|
2256
|
+
* - a terminal one settles the run with its own outcome's status;
|
|
2257
|
+
* - a non-terminal one records terminal `stopped(cancelled)` (every operation
|
|
2258
|
+
* without a result becomes `uncertain`; `effects_stopped` only when none
|
|
2259
|
+
* remain), then the run is cancelled;
|
|
2260
|
+
* - absent, another execution's, or an unbound state: the sidecar bytes stay
|
|
2261
|
+
* as they are and the run is cancelled;
|
|
2262
|
+
* - unreadable, or a record that cannot be committed: `unavailable`, and the
|
|
2263
|
+
* reap writes nothing.
|
|
2264
|
+
*/
|
|
2265
|
+
export async function recordRecipeOrphanCancel(context) {
|
|
2266
|
+
if (context.state.workflowName !== PING_PONG_WORKFLOW)
|
|
2267
|
+
return { kind: "cancel" };
|
|
2268
|
+
const authority = classifyRecipeTerminalAuthority(await readRecipeState(context.runDir), context.state.executionId);
|
|
2269
|
+
switch (authority.kind) {
|
|
2270
|
+
case "none":
|
|
2271
|
+
case "unbound":
|
|
2272
|
+
case "foreign":
|
|
2273
|
+
return { kind: "cancel" };
|
|
2274
|
+
case "unreadable":
|
|
2275
|
+
return { kind: "unavailable", reason: `recipe sidecar is unreadable: ${authority.reason}` };
|
|
2276
|
+
case "terminal":
|
|
2277
|
+
return { kind: "settled", envelope: recipeOutcomeEnvelope(authority.outcome, authority.state.stop_reason) };
|
|
2278
|
+
case "active": {
|
|
2279
|
+
const draft = recipeSettlementDraft(authority.state, "cancelled", "no live worker at cancel; result unknown");
|
|
2280
|
+
try {
|
|
2281
|
+
const next = stampRecipeState(draft, {
|
|
2282
|
+
runId: context.state.runId,
|
|
2283
|
+
executionId: authority.state.execution_id,
|
|
2284
|
+
previous: authority.state,
|
|
2285
|
+
});
|
|
2286
|
+
await writeRecipeStateAtomic(context.runDir, next, resolveFileWriter(context.fileWriter));
|
|
2287
|
+
}
|
|
2288
|
+
catch (error) {
|
|
2289
|
+
if (!(error instanceof DurablePublishError)) {
|
|
2290
|
+
return {
|
|
2291
|
+
kind: "unavailable",
|
|
2292
|
+
reason: `terminal recipe sidecar could not be committed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2293
|
+
};
|
|
2294
|
+
}
|
|
2295
|
+
}
|
|
2296
|
+
return { kind: "cancel" };
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
1769
2300
|
function isRunLockRecord(value) {
|
|
1770
2301
|
const record = asRecord(value);
|
|
1771
2302
|
const pid = record["pid"];
|
|
@@ -1797,6 +2328,11 @@ async function recycleSucceededRunDir(runDir) {
|
|
|
1797
2328
|
if (state.status !== "succeeded") {
|
|
1798
2329
|
return { recycled: false };
|
|
1799
2330
|
}
|
|
2331
|
+
// CONTRACTS 0-7 run-ID recycling: a recipe run (a pause is terminal
|
|
2332
|
+
// succeeded) is never recycled; fail closed on an uncertain sidecar probe.
|
|
2333
|
+
if (isSidecarRecipeWorkflow(state.workflowName) || await recipeSidecarPresence(runDir) === "present") {
|
|
2334
|
+
return { recycled: false };
|
|
2335
|
+
}
|
|
1800
2336
|
const identity = await readSessionLogGeneration(runDir);
|
|
1801
2337
|
let retiredSessionLogGeneration;
|
|
1802
2338
|
if (identity.kind === "available") {
|
|
@@ -1900,7 +2436,7 @@ function capPersistedError(message) {
|
|
|
1900
2436
|
const omitted = encoded.length - end;
|
|
1901
2437
|
return `${encoded.subarray(0, end).toString("utf-8")} …[truncated ${omitted} bytes]`;
|
|
1902
2438
|
}
|
|
1903
|
-
async function persistOutput(runDir, stageId, callId, outputText) {
|
|
2439
|
+
async function persistOutput(fileWriter, runDir, stageId, callId, outputText) {
|
|
1904
2440
|
const sizeBytes = Buffer.byteLength(outputText, "utf-8");
|
|
1905
2441
|
if (sizeBytes <= OUTPUT_INLINE_THRESHOLD_BYTES) {
|
|
1906
2442
|
return {
|
|
@@ -1909,12 +2445,21 @@ async function persistOutput(runDir, stageId, callId, outputText) {
|
|
|
1909
2445
|
};
|
|
1910
2446
|
}
|
|
1911
2447
|
const relativePath = `${stageId}.${callId}.output.txt`;
|
|
1912
|
-
await writeTextAtomic(join(runDir, relativePath), outputText);
|
|
2448
|
+
await writeTextAtomic(fileWriter, join(runDir, relativePath), outputText);
|
|
1913
2449
|
return {
|
|
1914
2450
|
outputInline: undefined,
|
|
1915
2451
|
outputPath: relativePath,
|
|
1916
2452
|
};
|
|
1917
2453
|
}
|
|
2454
|
+
/** Provenance from untrusted persisted state: only exact known literals survive. */
|
|
2455
|
+
function persistedFailureProvenance(entry) {
|
|
2456
|
+
const failureClass = entry.failureClass === "transient" || entry.failureClass === "permanent" ? entry.failureClass : undefined;
|
|
2457
|
+
const failureKind = parseCallFailureKind(entry.failureKind);
|
|
2458
|
+
return {
|
|
2459
|
+
...(failureClass !== undefined ? { failureClass } : {}),
|
|
2460
|
+
...(failureKind !== undefined ? { failureKind } : {}),
|
|
2461
|
+
};
|
|
2462
|
+
}
|
|
1918
2463
|
async function buildResumeState(definition, state, runDir) {
|
|
1919
2464
|
const completedCalls = new Map();
|
|
1920
2465
|
const preparedInputs = new Map();
|
|
@@ -1937,7 +2482,7 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
1937
2482
|
for (const call of stage.calls) {
|
|
1938
2483
|
const entry = persistedStage.calls.find((candidate) => candidate.callId === call.id);
|
|
1939
2484
|
if (entry?.state === "succeeded") {
|
|
1940
|
-
completedCalls.set(`${stage.id}.${call.id}`, await rehydrateExecutedCall(runDir, entry, definition.name
|
|
2485
|
+
completedCalls.set(`${stage.id}.${call.id}`, await rehydrateExecutedCall(runDir, entry, usesContainedArchitectureEvidence(definition.name)));
|
|
1941
2486
|
}
|
|
1942
2487
|
else if (entry?.state === "failed" && entry.errorMessage != null) {
|
|
1943
2488
|
failedCalls.set(`${stage.id}.${call.id}`, {
|
|
@@ -1945,6 +2490,7 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
1945
2490
|
callId: call.id,
|
|
1946
2491
|
error: entry.errorMessage,
|
|
1947
2492
|
terminal: true,
|
|
2493
|
+
...persistedFailureProvenance(entry),
|
|
1948
2494
|
});
|
|
1949
2495
|
}
|
|
1950
2496
|
}
|
|
@@ -1954,7 +2500,7 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
1954
2500
|
for (const call of stage.calls) {
|
|
1955
2501
|
const entry = persistedStage.calls.find((candidate) => candidate.callId === call.id);
|
|
1956
2502
|
if (entry?.state === "succeeded") {
|
|
1957
|
-
completedCalls.set(`${stage.id}.${call.id}`, await rehydrateExecutedCall(runDir, entry, definition.name
|
|
2503
|
+
completedCalls.set(`${stage.id}.${call.id}`, await rehydrateExecutedCall(runDir, entry, usesContainedArchitectureEvidence(definition.name)));
|
|
1958
2504
|
continue;
|
|
1959
2505
|
}
|
|
1960
2506
|
if (entry?.state === "failed" && entry.errorMessage != null) {
|
|
@@ -1966,10 +2512,11 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
1966
2512
|
// intermediate attempts remain `running` with attemptsConsumed.
|
|
1967
2513
|
// Never redispatch a persisted failed call after process restart.
|
|
1968
2514
|
terminal: true,
|
|
2515
|
+
...persistedFailureProvenance(entry),
|
|
1969
2516
|
});
|
|
1970
2517
|
}
|
|
1971
2518
|
if (entry?.materializedInputPath != null) {
|
|
1972
|
-
preparedInputs.set(`${stage.id}.${call.id}`, definition.name
|
|
2519
|
+
preparedInputs.set(`${stage.id}.${call.id}`, usesContainedArchitectureEvidence(definition.name)
|
|
1973
2520
|
? JSON.parse((await readContainedRelativeFile(runDir, entry.materializedInputPath, { wantContent: true, requireRegular: true, requireExactRequestedPath: true, maxBytes: MAX_RECORD_BYTES })).content)
|
|
1974
2521
|
: await readJsonFile(join(runDir, entry.materializedInputPath)));
|
|
1975
2522
|
nextRetryCounts.set(`${stage.id}.${call.id}`, entry.attemptsConsumed ??
|
|
@@ -2210,7 +2757,40 @@ async function listConfigFiles(configPath) {
|
|
|
2210
2757
|
}
|
|
2211
2758
|
async function readRunState(runDir) {
|
|
2212
2759
|
const statePath = join(runDir, STATE_FILENAME);
|
|
2213
|
-
|
|
2760
|
+
return validateRunState(await readJsonFile(statePath), statePath);
|
|
2761
|
+
}
|
|
2762
|
+
/**
|
|
2763
|
+
* One contained read: the digest covers exactly the raw bytes that were parsed,
|
|
2764
|
+
* so a guarded projection is fenced on the state it actually rendered.
|
|
2765
|
+
*/
|
|
2766
|
+
async function readContainedRunState(runDir) {
|
|
2767
|
+
// Atomic publication can retire an opened inode while a status read is in
|
|
2768
|
+
// flight. Retry only the reader's known change discriminator, with a fixed
|
|
2769
|
+
// immediate bound; policy, permission and malformed-state errors propagate.
|
|
2770
|
+
for (let attempt = 0;; attempt += 1) {
|
|
2771
|
+
let bytes;
|
|
2772
|
+
try {
|
|
2773
|
+
const file = await readContainedRelativeFile(runDir, STATE_FILENAME, {
|
|
2774
|
+
wantBytes: true,
|
|
2775
|
+
requireRegular: true,
|
|
2776
|
+
requireExactRequestedPath: true,
|
|
2777
|
+
maxBytes: DEFAULT_DASHBOARD_MAX_STATE_BYTES,
|
|
2778
|
+
});
|
|
2779
|
+
bytes = file.bytes;
|
|
2780
|
+
}
|
|
2781
|
+
catch (error) {
|
|
2782
|
+
if (!(error instanceof ContainedFileChangedError) || attempt >= 2)
|
|
2783
|
+
throw error;
|
|
2784
|
+
continue;
|
|
2785
|
+
}
|
|
2786
|
+
const state = JSON.parse(bytes.toString("utf-8"));
|
|
2787
|
+
return {
|
|
2788
|
+
state: validateRunState(state, join(runDir, STATE_FILENAME)),
|
|
2789
|
+
digest: createHash("sha256").update(bytes).digest("hex"),
|
|
2790
|
+
};
|
|
2791
|
+
}
|
|
2792
|
+
}
|
|
2793
|
+
function validateRunState(state, statePath) {
|
|
2214
2794
|
if (state.state_version !== 1) {
|
|
2215
2795
|
throw new Error(`Unsupported state_version ${String(state.state_version)} in ${statePath}`);
|
|
2216
2796
|
}
|
|
@@ -2227,19 +2807,28 @@ async function verifySnapshotHash(filePath, expectedHash, label) {
|
|
|
2227
2807
|
throw new Error(`Corrupted ${label}: hash mismatch for ${filePath}`);
|
|
2228
2808
|
}
|
|
2229
2809
|
}
|
|
2230
|
-
async function writeJsonSnapshot(filePath, value) {
|
|
2810
|
+
async function writeJsonSnapshot(fileWriter, filePath, value) {
|
|
2231
2811
|
const serialized = serializeJson(value);
|
|
2232
|
-
await writeTextAtomic(filePath, serialized);
|
|
2812
|
+
await writeTextAtomic(fileWriter, filePath, serialized);
|
|
2233
2813
|
return hashContent(serialized);
|
|
2234
2814
|
}
|
|
2235
|
-
async function
|
|
2236
|
-
|
|
2815
|
+
async function writeRunStateAtomic(fileWriter, filePath, state) {
|
|
2816
|
+
const { executionCompletedAt: _stale, ...rest } = state;
|
|
2817
|
+
const witnessed = rest.executionId !== undefined && isTerminalRunStatus(rest.status) && rest.completedAt !== undefined
|
|
2818
|
+
? { ...rest, executionCompletedAt: rest.completedAt }
|
|
2819
|
+
: rest;
|
|
2820
|
+
await writeJsonAtomic(fileWriter, filePath, witnessed);
|
|
2237
2821
|
}
|
|
2238
|
-
async function
|
|
2239
|
-
await
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2822
|
+
async function writeJsonAtomic(fileWriter, filePath, value) {
|
|
2823
|
+
await writeTextAtomic(fileWriter, filePath, serializeJson(value));
|
|
2824
|
+
}
|
|
2825
|
+
/**
|
|
2826
|
+
* Run files stay 0644 whatever the umask: the dashboard container reads them
|
|
2827
|
+
* as another user.
|
|
2828
|
+
*/
|
|
2829
|
+
async function writeTextAtomic(fileWriter, filePath, content) {
|
|
2830
|
+
await mkdirDurable(fileWriter, dirname(filePath));
|
|
2831
|
+
await fileWriter.writeFile(filePath, content, { mode: 0o644 });
|
|
2243
2832
|
}
|
|
2244
2833
|
function serializeJson(value) {
|
|
2245
2834
|
return `${JSON.stringify(value, null, 2)}\n`;
|
|
@@ -2251,7 +2840,8 @@ async function readJsonFile(filePath) {
|
|
|
2251
2840
|
return JSON.parse(await readFile(filePath, "utf-8"));
|
|
2252
2841
|
}
|
|
2253
2842
|
/**
|
|
2254
|
-
* Atomically create `run.lock`, reclaiming a same-host dead-PID lock
|
|
2843
|
+
* Atomically create `run.lock`, reclaiming a same-host dead-PID lock behind
|
|
2844
|
+
* `run.lock.steal` (see `reclaimDeadRunLock`).
|
|
2255
2845
|
*
|
|
2256
2846
|
* `declareHeartbeat` is deliberately opt-in: only a caller that goes on to hold
|
|
2257
2847
|
* the lock behind a live `WorkflowStateStore` (create/resume) beats it. The
|
|
@@ -2270,7 +2860,9 @@ async function acquireRunLock(runDir, options = {}) {
|
|
|
2270
2860
|
? { heartbeatIntervalMs: RUN_LOCK_HEARTBEAT_MS }
|
|
2271
2861
|
: {}),
|
|
2272
2862
|
};
|
|
2863
|
+
let stealBusyPath;
|
|
2273
2864
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
2865
|
+
stealBusyPath = undefined;
|
|
2274
2866
|
try {
|
|
2275
2867
|
const handle = await open(lockPath, "wx");
|
|
2276
2868
|
try {
|
|
@@ -2300,12 +2892,16 @@ async function acquireRunLock(runDir, options = {}) {
|
|
|
2300
2892
|
}
|
|
2301
2893
|
const existing = existingValue;
|
|
2302
2894
|
if (existing.hostname === hostname() && !isPidLive(existing.pid)) {
|
|
2303
|
-
await
|
|
2895
|
+
const outcome = await reclaimDeadRunLock(lockPath, existing);
|
|
2896
|
+
stealBusyPath = outcome.kind === "steal_lock_busy" ? outcome.stealPath : undefined;
|
|
2304
2897
|
continue;
|
|
2305
2898
|
}
|
|
2306
2899
|
throw new Error(`Run lock is held for ${runDir} by pid ${String(existing.pid)} on ${existing.hostname} since ${existing.startedAt}`);
|
|
2307
2900
|
}
|
|
2308
2901
|
}
|
|
2902
|
+
if (stealBusyPath !== undefined) {
|
|
2903
|
+
throw new Error(`Failed to acquire run lock for ${runDir}: its owner pid is dead but reclaim was declined because ${stealBusyPath} exists; if no other process is acquiring this run, remove ${stealBusyPath}`);
|
|
2904
|
+
}
|
|
2309
2905
|
throw new Error(`Failed to acquire run lock for ${runDir} after 2 attempts; lock ownership was not established`);
|
|
2310
2906
|
}
|
|
2311
2907
|
function isPidLive(pid) {
|