clinkx 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +4 -2
- package/clinkx-workflows/dist/architecture-arena/admission.d.ts +16 -0
- package/clinkx-workflows/dist/architecture-arena/admission.js +107 -0
- package/clinkx-workflows/dist/architecture-arena/admission.js.map +1 -0
- package/clinkx-workflows/dist/architecture-arena/contract.d.ts +1368 -0
- package/clinkx-workflows/dist/architecture-arena/contract.js +254 -0
- package/clinkx-workflows/dist/architecture-arena/contract.js.map +1 -0
- package/clinkx-workflows/dist/architecture-arena/runtime.d.ts +53 -0
- package/clinkx-workflows/dist/architecture-arena/runtime.js +511 -0
- package/clinkx-workflows/dist/architecture-arena/runtime.js.map +1 -0
- package/clinkx-workflows/dist/architecture-comparison/admission.d.ts +17 -1
- package/clinkx-workflows/dist/architecture-comparison/admission.js +27 -4
- package/clinkx-workflows/dist/architecture-comparison/admission.js.map +1 -1
- package/clinkx-workflows/dist/architecture-comparison/contract.d.ts +59 -62
- package/clinkx-workflows/dist/architecture-comparison/contract.js +4 -21
- package/clinkx-workflows/dist/architecture-comparison/contract.js.map +1 -1
- package/clinkx-workflows/dist/architecture-comparison/runtime.d.ts +4 -3
- package/clinkx-workflows/dist/architecture-comparison/runtime.js +22 -67
- package/clinkx-workflows/dist/architecture-comparison/runtime.js.map +1 -1
- package/clinkx-workflows/dist/artifacts.d.ts +19 -4
- package/clinkx-workflows/dist/artifacts.js +16 -7
- package/clinkx-workflows/dist/artifacts.js.map +1 -1
- package/clinkx-workflows/dist/backend.d.ts +30 -0
- package/clinkx-workflows/dist/clink-client.d.ts +8 -2
- package/clinkx-workflows/dist/clink-client.js +4 -2
- package/clinkx-workflows/dist/clink-client.js.map +1 -1
- package/clinkx-workflows/dist/context-preflight.d.ts +6 -0
- package/clinkx-workflows/dist/context-preflight.js.map +1 -1
- package/clinkx-workflows/dist/council-execution-policy.d.ts +3 -1
- package/clinkx-workflows/dist/council-execution-policy.js +4 -0
- package/clinkx-workflows/dist/council-execution-policy.js.map +1 -1
- package/clinkx-workflows/dist/dashboard/contracts.d.ts +586 -586
- package/clinkx-workflows/dist/dashboard/progress-report-contract.d.ts +28 -28
- package/clinkx-workflows/dist/dashboard/session-log-contract.d.ts +32 -32
- package/clinkx-workflows/dist/dashboard-ui/assets/index-DRId2DVA.js +119 -0
- package/clinkx-workflows/dist/dashboard-ui/assets/style-CHAw57y5.css +1 -0
- package/clinkx-workflows/dist/dashboard-ui/index.html +2 -2
- package/clinkx-workflows/dist/delegate-signature.d.ts +4 -3
- package/clinkx-workflows/dist/delegate-signature.js +4 -3
- package/clinkx-workflows/dist/delegate-signature.js.map +1 -1
- package/clinkx-workflows/dist/durable-file.d.ts +69 -0
- package/clinkx-workflows/dist/durable-file.js +145 -0
- package/clinkx-workflows/dist/durable-file.js.map +1 -0
- package/clinkx-workflows/dist/engine.d.ts +42 -7
- package/clinkx-workflows/dist/engine.js +196 -24
- package/clinkx-workflows/dist/engine.js.map +1 -1
- package/clinkx-workflows/dist/ping-pong/admission.d.ts +127 -0
- package/clinkx-workflows/dist/ping-pong/admission.js +522 -0
- package/clinkx-workflows/dist/ping-pong/admission.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/contract.d.ts +2539 -0
- package/clinkx-workflows/dist/ping-pong/contract.js +819 -0
- package/clinkx-workflows/dist/ping-pong/contract.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/control.d.ts +67 -0
- package/clinkx-workflows/dist/ping-pong/control.js +12 -0
- package/clinkx-workflows/dist/ping-pong/control.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/runtime-handle.d.ts +24 -0
- package/clinkx-workflows/dist/ping-pong/runtime-handle.js +55 -0
- package/clinkx-workflows/dist/ping-pong/runtime-handle.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/runtime.d.ts +235 -0
- package/clinkx-workflows/dist/ping-pong/runtime.js +1183 -0
- package/clinkx-workflows/dist/ping-pong/runtime.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/settle.d.ts +57 -0
- package/clinkx-workflows/dist/ping-pong/settle.js +58 -0
- package/clinkx-workflows/dist/ping-pong/settle.js.map +1 -0
- package/clinkx-workflows/dist/ping-pong/sidecar.d.ts +65 -0
- package/clinkx-workflows/dist/ping-pong/sidecar.js +138 -0
- package/clinkx-workflows/dist/ping-pong/sidecar.js.map +1 -0
- package/clinkx-workflows/dist/plan-review/publication.d.ts +2 -1
- package/clinkx-workflows/dist/plan-review/publication.js +5 -6
- package/clinkx-workflows/dist/plan-review/publication.js.map +1 -1
- package/clinkx-workflows/dist/plan-review/snapshot.d.ts +3 -2
- package/clinkx-workflows/dist/plan-review/snapshot.js +8 -8
- package/clinkx-workflows/dist/plan-review/snapshot.js.map +1 -1
- package/clinkx-workflows/dist/recipe-host.d.ts +315 -0
- package/clinkx-workflows/dist/recipe-host.js +183 -0
- package/clinkx-workflows/dist/recipe-host.js.map +1 -0
- package/clinkx-workflows/dist/recipe-seats.d.ts +96 -0
- package/clinkx-workflows/dist/recipe-seats.js +163 -0
- package/clinkx-workflows/dist/recipe-seats.js.map +1 -0
- package/clinkx-workflows/dist/recipe-shared/evidence.d.ts +67 -0
- package/clinkx-workflows/dist/recipe-shared/evidence.js +111 -0
- package/clinkx-workflows/dist/recipe-shared/evidence.js.map +1 -0
- package/clinkx-workflows/dist/reducers/index.d.ts +9 -1
- package/clinkx-workflows/dist/reducers/index.js +30 -0
- package/clinkx-workflows/dist/reducers/index.js.map +1 -1
- package/clinkx-workflows/dist/run-lock-reclaim.d.ts +44 -0
- package/clinkx-workflows/dist/run-lock-reclaim.js +86 -0
- package/clinkx-workflows/dist/run-lock-reclaim.js.map +1 -0
- package/clinkx-workflows/dist/state-schema.d.ts +11 -0
- package/clinkx-workflows/dist/state-schema.js +4 -1
- package/clinkx-workflows/dist/state-schema.js.map +1 -1
- package/clinkx-workflows/dist/state.d.ts +178 -3
- package/clinkx-workflows/dist/state.js +618 -83
- package/clinkx-workflows/dist/state.js.map +1 -1
- package/clinkx-workflows/dist/transport.d.ts +2 -1
- package/clinkx-workflows/dist/transport.js +5 -0
- package/clinkx-workflows/dist/transport.js.map +1 -1
- package/clinkx-workflows/dist/validators/architecture-arena.d.ts +6 -0
- package/clinkx-workflows/dist/validators/architecture-arena.js +16 -0
- package/clinkx-workflows/dist/validators/architecture-arena.js.map +1 -0
- package/clinkx-workflows/dist/validators/index.js +4 -0
- package/clinkx-workflows/dist/validators/index.js.map +1 -1
- package/clinkx-workflows/dist/validators/ping-pong.d.ts +6 -0
- package/clinkx-workflows/dist/validators/ping-pong.js +10 -0
- package/clinkx-workflows/dist/validators/ping-pong.js.map +1 -0
- package/clinkx-workflows/dist/workflow-tools.d.ts +51 -5
- package/clinkx-workflows/dist/workflow-tools.js +785 -34
- package/clinkx-workflows/dist/workflow-tools.js.map +1 -1
- package/clinkx-workflows/dist/workflow-verification-contract.d.ts +4 -4
- package/clinkx-workflows/templates/architecture-arena/candidate.txt +39 -0
- package/clinkx-workflows/templates/architecture-arena/judge.txt +34 -0
- package/clinkx-workflows/templates/ping-pong/implement.txt +14 -0
- package/clinkx-workflows/templates/ping-pong/remediate.txt +14 -0
- package/clinkx-workflows/templates/ping-pong/review.txt +27 -0
- package/clinkx-workflows/workflows/architecture-arena.yaml +127 -0
- package/clinkx-workflows/workflows/ping-pong.yaml +158 -0
- package/dist/context-builder/bundle.js +4 -3
- package/dist/context-builder/bundle.js.map +1 -1
- package/dist/context-builder/delegate-packet.d.ts +1 -0
- package/dist/context-builder/delegate-packet.js +66 -7
- package/dist/context-builder/delegate-packet.js.map +1 -1
- package/dist/context-builder/delegate-validation.d.ts +12 -3
- package/dist/context-builder/delegate-validation.js +23 -13
- package/dist/context-builder/delegate-validation.js.map +1 -1
- package/dist/context-builder/external-gaps.d.ts +14 -3
- package/dist/context-builder/external-gaps.js +124 -18
- package/dist/context-builder/external-gaps.js.map +1 -1
- package/dist/context-builder/handler.js +108 -191
- package/dist/context-builder/handler.js.map +1 -1
- package/dist/context-builder/review-coverage.js +27 -1
- package/dist/context-builder/review-coverage.js.map +1 -1
- package/dist/context-builder/secrets.js +1 -1
- package/dist/context-builder/secrets.js.map +1 -1
- package/dist/context-builder/versions.d.ts +10 -4
- package/dist/context-builder/versions.js +10 -4
- package/dist/context-builder/versions.js.map +1 -1
- package/dist/context-builder/workflow-runner.js +33 -24
- package/dist/context-builder/workflow-runner.js.map +1 -1
- package/dist/dashboard-cli.d.ts +2 -0
- package/dist/dashboard-cli.js +147 -57
- package/dist/dashboard-cli.js.map +1 -1
- package/dist/dashboard-companion/client.d.ts +10 -0
- package/dist/dashboard-companion/client.js +43 -1
- package/dist/dashboard-companion/client.js.map +1 -1
- package/dist/dashboard-companion/config.d.ts +19 -1
- package/dist/dashboard-companion/config.js +104 -3
- package/dist/dashboard-companion/config.js.map +1 -1
- package/dist/dashboard-companion/protocol.d.ts +16 -0
- package/dist/dashboard-companion/protocol.js +38 -1
- package/dist/dashboard-companion/protocol.js.map +1 -1
- package/dist/dashboard-companion/workstation-binding.d.ts +69 -0
- package/dist/dashboard-companion/workstation-binding.js +192 -0
- package/dist/dashboard-companion/workstation-binding.js.map +1 -0
- package/dist/dashboard-gateway/admin-cli.d.ts +10 -0
- package/dist/dashboard-gateway/admin-cli.js +266 -0
- package/dist/dashboard-gateway/admin-cli.js.map +1 -0
- package/dist/dashboard-gateway/config.d.ts +50 -0
- package/dist/dashboard-gateway/config.js +235 -1
- package/dist/dashboard-gateway/config.js.map +1 -1
- package/dist/dashboard-gateway/connection-registry.d.ts +100 -0
- package/dist/dashboard-gateway/connection-registry.js +228 -0
- package/dist/dashboard-gateway/connection-registry.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/account-auth.d.ts +78 -0
- package/dist/dashboard-gateway/control-plane/account-auth.js +198 -0
- package/dist/dashboard-gateway/control-plane/account-auth.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/billing.d.ts +264 -0
- package/dist/dashboard-gateway/control-plane/billing.js +787 -0
- package/dist/dashboard-gateway/control-plane/billing.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/contracts.d.ts +1293 -0
- package/dist/dashboard-gateway/control-plane/contracts.js +280 -0
- package/dist/dashboard-gateway/control-plane/contracts.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/cookies.d.ts +13 -0
- package/dist/dashboard-gateway/control-plane/cookies.js +46 -0
- package/dist/dashboard-gateway/control-plane/cookies.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/credentials.d.ts +63 -0
- package/dist/dashboard-gateway/control-plane/credentials.js +66 -0
- package/dist/dashboard-gateway/control-plane/credentials.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/entitlements.d.ts +118 -0
- package/dist/dashboard-gateway/control-plane/entitlements.js +329 -0
- package/dist/dashboard-gateway/control-plane/entitlements.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/errors.d.ts +17 -0
- package/dist/dashboard-gateway/control-plane/errors.js +23 -0
- package/dist/dashboard-gateway/control-plane/errors.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/hosts.d.ts +6 -0
- package/dist/dashboard-gateway/control-plane/hosts.js +16 -0
- package/dist/dashboard-gateway/control-plane/hosts.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/http.d.ts +34 -0
- package/dist/dashboard-gateway/control-plane/http.js +331 -0
- package/dist/dashboard-gateway/control-plane/http.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/schema.d.ts +9 -0
- package/dist/dashboard-gateway/control-plane/schema.js +247 -0
- package/dist/dashboard-gateway/control-plane/schema.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/secrets.d.ts +20 -0
- package/dist/dashboard-gateway/control-plane/secrets.js +39 -0
- package/dist/dashboard-gateway/control-plane/secrets.js.map +1 -0
- package/dist/dashboard-gateway/control-plane/store.d.ts +579 -0
- package/dist/dashboard-gateway/control-plane/store.js +1115 -0
- package/dist/dashboard-gateway/control-plane/store.js.map +1 -0
- package/dist/dashboard-gateway/cookie-boundary.d.ts +60 -0
- package/dist/dashboard-gateway/cookie-boundary.js +205 -0
- package/dist/dashboard-gateway/cookie-boundary.js.map +1 -0
- package/dist/dashboard-gateway/index.js +271 -494
- package/dist/dashboard-gateway/index.js.map +1 -1
- package/dist/dashboard-gateway/main.js +7 -2
- package/dist/dashboard-gateway/main.js.map +1 -1
- package/dist/dashboard-gateway/routing.d.ts +43 -0
- package/dist/dashboard-gateway/routing.js +65 -0
- package/dist/dashboard-gateway/routing.js.map +1 -0
- package/dist/dashboard-gateway/tenant-gateway.d.ts +56 -0
- package/dist/dashboard-gateway/tenant-gateway.js +989 -0
- package/dist/dashboard-gateway/tenant-gateway.js.map +1 -0
- package/dist/dashboard-gateway/tenant-main.d.ts +41 -0
- package/dist/dashboard-gateway/tenant-main.js +303 -0
- package/dist/dashboard-gateway/tenant-main.js.map +1 -0
- package/dist/dashboard-gateway/tunnel-errors.d.ts +16 -0
- package/dist/dashboard-gateway/tunnel-errors.js +22 -0
- package/dist/dashboard-gateway/tunnel-errors.js.map +1 -0
- package/dist/dashboard-gateway/tunnel-http.d.ts +21 -0
- package/dist/dashboard-gateway/tunnel-http.js +175 -0
- package/dist/dashboard-gateway/tunnel-http.js.map +1 -0
- package/dist/handler.d.ts +6 -0
- package/dist/handler.js +17 -1
- package/dist/handler.js.map +1 -1
- package/dist/local-clink-backend.d.ts +2 -1
- package/dist/local-clink-backend.js +28 -4
- package/dist/local-clink-backend.js.map +1 -1
- package/dist/parsers/extract.d.ts +19 -1
- package/dist/parsers/extract.js +68 -3
- package/dist/parsers/extract.js.map +1 -1
- package/dist/pipeline.d.ts +17 -1
- package/dist/pipeline.js +34 -6
- package/dist/pipeline.js.map +1 -1
- package/dist/process-group.d.ts +65 -0
- package/dist/process-group.js +153 -0
- package/dist/process-group.js.map +1 -0
- package/dist/progress-report/schema.d.ts +23 -23
- package/dist/project-verification/files.js +1 -1
- package/dist/project-verification/files.js.map +1 -1
- package/dist/project-verification/process.js +23 -44
- package/dist/project-verification/process.js.map +1 -1
- package/dist/project-verification/service.js +9 -1
- package/dist/project-verification/service.js.map +1 -1
- package/dist/prompt.d.ts +7 -0
- package/dist/prompt.js +18 -1
- package/dist/prompt.js.map +1 -1
- package/dist/recipe-host/git-trust.d.ts +43 -0
- package/dist/recipe-host/git-trust.js +360 -0
- package/dist/recipe-host/git-trust.js.map +1 -0
- package/dist/recipe-host/git.d.ts +42 -0
- package/dist/recipe-host/git.js +207 -0
- package/dist/recipe-host/git.js.map +1 -0
- package/dist/recipe-host/native-recipe-host.d.ts +127 -0
- package/dist/recipe-host/native-recipe-host.js +880 -0
- package/dist/recipe-host/native-recipe-host.js.map +1 -0
- package/dist/recipe-host/seat-cwd.d.ts +16 -0
- package/dist/recipe-host/seat-cwd.js +170 -0
- package/dist/recipe-host/seat-cwd.js.map +1 -0
- package/dist/runner.d.ts +43 -1
- package/dist/runner.js +234 -35
- package/dist/runner.js.map +1 -1
- package/dist/server.js +9 -1
- package/dist/server.js.map +1 -1
- package/docs/configuration.md +24 -3
- package/docs/engineering-recipes.md +291 -0
- package/docs/security.md +84 -0
- package/docs/verification/project-checks.md +24 -2
- package/package.json +5 -1
- package/clinkx-workflows/dist/dashboard-ui/assets/index-BdpmbPps.js +0 -115
- package/clinkx-workflows/dist/dashboard-ui/assets/style-Im5i1iAs.css +0 -1
|
@@ -1,17 +1,24 @@
|
|
|
1
1
|
import { createHash, randomBytes } from "node:crypto";
|
|
2
2
|
import { execFileSync } from "node:child_process";
|
|
3
3
|
import { constants as fsConstants, readFileSync } from "node:fs";
|
|
4
|
-
import { lstat, mkdir, open, readFile, readdir, realpath, rename, rm, stat, unlink
|
|
4
|
+
import { lstat, mkdir, open, readFile, readdir, realpath, rename, rm, rmdir, stat, unlink } from "node:fs/promises";
|
|
5
5
|
import { dirname, basename, isAbsolute, join, relative, resolve } from "node:path";
|
|
6
6
|
import { hostname } from "node:os";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
import { ContainedFileChangedError, readContainedRelativeFile } from "./contained-file.js";
|
|
9
|
-
import {
|
|
9
|
+
import { MAX_RECORD_BYTES } from "./architecture-comparison/contract.js";
|
|
10
|
+
import { usesContainedArchitectureEvidence } from "./architecture-arena/contract.js";
|
|
10
11
|
import { WorkflowError } from "./errors.js";
|
|
12
|
+
import { RECIPE_STATE_FILENAME, readRecipePruneSummary } from "./recipe-host.js";
|
|
13
|
+
import { PING_PONG_WORKFLOW, isCompleteAwaitingCheckpoint, isTerminalRecipePhase, runStatusForOutcome, } from "./ping-pong/contract.js";
|
|
14
|
+
import { RecipeStateInvariantError, isSidecarRecipeWorkflow, readRecipeState, recipeSidecarPresence, stampRecipeState, writeRecipeStateAtomic, } from "./ping-pong/sidecar.js";
|
|
15
|
+
import { classifyRecipeTerminalAuthority, recipeOutcomeEnvelope, recipeSettlementDraft, settlementNote, } from "./ping-pong/settle.js";
|
|
11
16
|
import { computeRunHealth } from "./run-health.js";
|
|
17
|
+
import { reclaimDeadRunLock } from "./run-lock-reclaim.js";
|
|
12
18
|
import { buildFidelityMetadata, } from "./fidelity.js";
|
|
13
19
|
import { fingerprintsMatch, generateFingerprint, generateWorkflowFingerprint, } from "./fingerprint.js";
|
|
14
20
|
import { effectiveFinalizerEnabled } from "./delegate-finalizer-policy.js";
|
|
21
|
+
import { parseCallFailureKind } from "./state-schema.js";
|
|
15
22
|
import { parseWorkflowContext, } from "./context-preflight.js";
|
|
16
23
|
import { isLegalContextBuilderLifecycleTransition, parseContextBuilderLifecycleStateV1, } from "./context-builder-lifecycle-contract.js";
|
|
17
24
|
import { ResolvedRosterSchema, RosterNameSchema } from "./roster-contract.js";
|
|
@@ -23,12 +30,12 @@ import { restorePlanReviewRun } from "./plan-review/resume.js";
|
|
|
23
30
|
import { PlanReviewStageError } from "./plan-review/stage-error.js";
|
|
24
31
|
import { DEFAULT_DASHBOARD_MAX_STATE_BYTES, DEFAULT_RETAIN_ASYNC_DAYS, DEFAULT_RETAIN_CANCELLED_DAYS, DEFAULT_RETAIN_DEBUG_DAYS, DEFAULT_RETAIN_FAILED_DAYS, DEFAULT_RETAIN_SUCCEEDED_DAYS, deriveAllowedRoots, getArtifactsDir, getChildConfigPath, getChildMaxConcurrent, getChildMaxOutputBytes, getChildMaxPromptBytes, getChildMaxResponseChars, getLogLevel, getRetentionConfig, getRunsDir, getRunStateDir, getStateDir, invocationAuditV2PersistenceEnabled, observedInvocationPersistenceEnabled, resolveWorkspaceRoot, } from "./config.js";
|
|
25
32
|
import { logger } from "./logger.js";
|
|
33
|
+
import { DurablePublishError, mkdirDurable, resolveFileWriter } from "./durable-file.js";
|
|
26
34
|
import { resolveOpenedDescriptorPath } from "./contained-file.js";
|
|
27
35
|
import { deriveWorkflowRunGeneration } from "./dashboard/generation.js";
|
|
28
36
|
import { removeSessionLogGeneration } from "./dashboard/session-log-retention.js";
|
|
29
37
|
import { createSessionLogGeneration, readSessionLogGeneration, } from "./dashboard/session-log-identity.js";
|
|
30
38
|
const STATE_FILENAME = "state.json";
|
|
31
|
-
const STATE_TMP_FILENAME = "state.json.tmp";
|
|
32
39
|
const WORKFLOW_SNAPSHOT_FILENAME = "workflow.snapshot.json";
|
|
33
40
|
const INPUTS_SNAPSHOT_FILENAME = "inputs.snapshot.json";
|
|
34
41
|
const CHILD_CONFIG_DIRNAME = "child-config";
|
|
@@ -95,7 +102,7 @@ function withPlanReviewPublication(state, publication) {
|
|
|
95
102
|
* enqueueWrite. The committed marker is persisted on the pre-terminal state,
|
|
96
103
|
* after the sidecar and before the caller commits the terminal state.
|
|
97
104
|
*/
|
|
98
|
-
const finalizePlanReviewTerminalState = async ({ runDir, previousState, intendedState, intent, chairmanCallId, serializedResult, }) => {
|
|
105
|
+
const finalizePlanReviewTerminalState = async ({ runDir, previousState, intendedState, intent, chairmanCallId, serializedResult, fileWriter, }) => {
|
|
99
106
|
const existing = previousState.plan_review_state?.publication;
|
|
100
107
|
if (existing?.committed === true) {
|
|
101
108
|
return { state: withPlanReviewPublication(intendedState, existing) };
|
|
@@ -145,13 +152,13 @@ const finalizePlanReviewTerminalState = async ({ runDir, previousState, intended
|
|
|
145
152
|
contract: restored.contract,
|
|
146
153
|
execution_contract_digest: restored.execution_contract_digest,
|
|
147
154
|
}));
|
|
148
|
-
const marker = await writeSerializedPlanReviewResultSidecar(runDir, projected);
|
|
155
|
+
const marker = await writeSerializedPlanReviewResultSidecar(runDir, projected, fileWriter);
|
|
149
156
|
// Durable crash boundary: a process death after this write still leaves a
|
|
150
157
|
// contained, marker-addressable result even though terminal intent has not
|
|
151
158
|
// yet been committed.
|
|
152
159
|
const intermediate = withPlanReviewPublication(previousState, marker);
|
|
153
160
|
validateStrictStateSubtrees(intermediate, join(runDir, STATE_FILENAME));
|
|
154
|
-
await writeRunStateAtomic(join(runDir, STATE_FILENAME), intermediate);
|
|
161
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), intermediate);
|
|
155
162
|
return { state: withPlanReviewPublication(intendedState, marker) };
|
|
156
163
|
}
|
|
157
164
|
catch (err) {
|
|
@@ -190,6 +197,32 @@ class RunLockOwnershipLostError extends Error {
|
|
|
190
197
|
this.name = "RunLockOwnershipLostError";
|
|
191
198
|
}
|
|
192
199
|
}
|
|
200
|
+
/**
|
|
201
|
+
* The terminal recipe sidecar for a failed/cancelled settlement could not be
|
|
202
|
+
* committed, so `state.json` was left active rather than claiming an outcome
|
|
203
|
+
* the sidecar does not record. Once run.lock is released the run is an
|
|
204
|
+
* orphan: reconcile fails it, or cancel_workflow records it.
|
|
205
|
+
*/
|
|
206
|
+
export class RecipeTerminalUnrecordedError extends WorkflowError {
|
|
207
|
+
intent;
|
|
208
|
+
constructor(intent, cause) {
|
|
209
|
+
super(`terminal recipe sidecar (${intent}) could not be committed; state.json left active: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
210
|
+
this.intent = intent;
|
|
211
|
+
this.name = "RecipeTerminalUnrecordedError";
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
export class RecipeReopenError extends WorkflowError {
|
|
215
|
+
reason;
|
|
216
|
+
current;
|
|
217
|
+
constructor(reason, message,
|
|
218
|
+
/** Current sidecar position, when one could be read. */
|
|
219
|
+
current) {
|
|
220
|
+
super(message);
|
|
221
|
+
this.reason = reason;
|
|
222
|
+
this.current = current;
|
|
223
|
+
this.name = "RecipeReopenError";
|
|
224
|
+
}
|
|
225
|
+
}
|
|
193
226
|
export function classifyRunDirectory(runDirPath) {
|
|
194
227
|
return inspectRunDirectory(runDirPath).classification;
|
|
195
228
|
}
|
|
@@ -258,8 +291,14 @@ export async function pruneRunDirectories(runsRootDir, options) {
|
|
|
258
291
|
const runDirPath = join(runsRootDir, entry.name);
|
|
259
292
|
try {
|
|
260
293
|
const { classification, state } = inspectRunDirectory(runDirPath);
|
|
294
|
+
const recipeRun = await hasRecipeSidecar(runDirPath);
|
|
261
295
|
let retainDays = retentionDaysForClassification(classification, retention);
|
|
262
|
-
if (
|
|
296
|
+
if (recipeRun && retainDays !== undefined) {
|
|
297
|
+
// CONTRACTS C5: a paused or delivered recipe run keeps the failed-run TTL
|
|
298
|
+
// for every terminal status; that TTL is also the decision TTL.
|
|
299
|
+
retainDays = retention.retainFailedDays;
|
|
300
|
+
}
|
|
301
|
+
else if ((classification === "succeeded_async" || classification === "succeeded_sync") &&
|
|
263
302
|
state != null && retainDays != null && retainDays > 0 &&
|
|
264
303
|
computeRunHealth(state, state.context_builder_lifecycle).classification === "degraded") {
|
|
265
304
|
// Zero means unlimited retention, not the smallest TTL. Keep explicit
|
|
@@ -284,6 +323,10 @@ export async function pruneRunDirectories(runsRootDir, options) {
|
|
|
284
323
|
result.skipped += 1;
|
|
285
324
|
continue;
|
|
286
325
|
}
|
|
326
|
+
if (recipeRun && await recipeRunPrePruneDecision(entry.name, runDirPath) === "veto") {
|
|
327
|
+
result.skipped += 1;
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
287
330
|
const identity = await readSessionLogGeneration(runDirPath);
|
|
288
331
|
let sessionLogGeneration;
|
|
289
332
|
if (identity.kind === "available") {
|
|
@@ -339,6 +382,54 @@ export async function pruneRunDirectories(runsRootDir, options) {
|
|
|
339
382
|
logger.info({ runs_root_dir: runsRootDir, ...result }, "workflow run directory pruning complete");
|
|
340
383
|
return result;
|
|
341
384
|
}
|
|
385
|
+
let recipeRunPrePruneObserver;
|
|
386
|
+
/**
|
|
387
|
+
* Install the root host's pre-prune observer for recipe runs (CONTRACTS C5).
|
|
388
|
+
* It runs under the prune lock immediately before a run directory holding
|
|
389
|
+
* `recipe-state.json` is removed; "veto" keeps the directory. Process-local,
|
|
390
|
+
* like `setWorkflowActivityObserver`.
|
|
391
|
+
*/
|
|
392
|
+
export function setRecipeRunPrePruneObserver(observer) {
|
|
393
|
+
recipeRunPrePruneObserver = observer;
|
|
394
|
+
}
|
|
395
|
+
async function hasRecipeSidecar(runDirPath) {
|
|
396
|
+
try {
|
|
397
|
+
await lstat(join(runDirPath, RECIPE_STATE_FILENAME));
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
400
|
+
catch (error) {
|
|
401
|
+
if (error.code === "ENOENT")
|
|
402
|
+
return false;
|
|
403
|
+
throw error;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
async function recipeRunPrePruneDecision(runId, runDir) {
|
|
407
|
+
const observer = recipeRunPrePruneObserver;
|
|
408
|
+
let decision;
|
|
409
|
+
try {
|
|
410
|
+
if (observer !== undefined) {
|
|
411
|
+
decision = await observer({ runId, runDir });
|
|
412
|
+
}
|
|
413
|
+
else {
|
|
414
|
+
// No root host (standalone server): pruning would orphan any worktree,
|
|
415
|
+
// so allow only a sidecar that proves there is nothing to clean up.
|
|
416
|
+
const summary = await readRecipePruneSummary(runDir);
|
|
417
|
+
decision = summary.kind === "read" && summary.unresolvedOperations === 0 &&
|
|
418
|
+
summary.worktreePaths !== undefined && summary.worktreePaths.length === 0
|
|
419
|
+
? "allow"
|
|
420
|
+
: "veto";
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
catch (error) {
|
|
424
|
+
logger.warn({ run_id: runId, err: error instanceof Error ? error.message : String(error) }, "recipe pre-prune observer failed; keeping run directory");
|
|
425
|
+
return "veto";
|
|
426
|
+
}
|
|
427
|
+
if (decision !== "allow") {
|
|
428
|
+
logger.info({ run_id: runId }, "recipe run prune vetoed; keeping run directory");
|
|
429
|
+
return "veto";
|
|
430
|
+
}
|
|
431
|
+
return "allow";
|
|
432
|
+
}
|
|
342
433
|
function isTerminalRunStatus(status) {
|
|
343
434
|
return status === "succeeded" || status === "failed" || status === "cancelled";
|
|
344
435
|
}
|
|
@@ -413,7 +504,16 @@ export class WorkflowStateStore {
|
|
|
413
504
|
heartbeatError;
|
|
414
505
|
persistInvocationAuditV2;
|
|
415
506
|
persistObservedInvocation;
|
|
507
|
+
/** Last committed recipe sidecar; `undefined` until loaded or when absent. */
|
|
508
|
+
recipeStateCache;
|
|
509
|
+
recipeStateLoaded = false;
|
|
510
|
+
writer;
|
|
416
511
|
constructor(options) {
|
|
512
|
+
this.writer = options.fileWriter;
|
|
513
|
+
if (options.recipeState !== undefined) {
|
|
514
|
+
this.recipeStateCache = options.recipeState;
|
|
515
|
+
this.recipeStateLoaded = true;
|
|
516
|
+
}
|
|
417
517
|
this.runDir = options.runDir;
|
|
418
518
|
this.sessionLogGeneration = options.sessionLogGeneration;
|
|
419
519
|
this.duplicateOfFailedRun = options.duplicateOfFailedRun;
|
|
@@ -448,14 +548,28 @@ export class WorkflowStateStore {
|
|
|
448
548
|
isStaleRunningRecoveryCheckpoint(this.state))) {
|
|
449
549
|
return;
|
|
450
550
|
}
|
|
551
|
+
let recipeOverride;
|
|
552
|
+
if (isTerminalRunStatus(event.state)) {
|
|
553
|
+
try {
|
|
554
|
+
recipeOverride = await this.recipeTerminalOverrideLocked(event.state);
|
|
555
|
+
}
|
|
556
|
+
catch (error) {
|
|
557
|
+
if (!(error instanceof RecipeTerminalUnrecordedError))
|
|
558
|
+
throw error;
|
|
559
|
+
// Throwing here would replace the engine's own error; the
|
|
560
|
+
// owner's settlement retries through mark*.
|
|
561
|
+
logger.error({ run_id: this.state.runId, err: error.message }, "recipe terminal state left unpublished");
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
}
|
|
451
565
|
const nextState = structuredClone(this.state);
|
|
452
|
-
nextState.status = event.state;
|
|
566
|
+
nextState.status = recipeOverride?.status ?? event.state;
|
|
453
567
|
// First emit to set startedAt wins: a fresh run fills an empty
|
|
454
568
|
// field, and a resume can never overwrite the original run start.
|
|
455
569
|
nextState.startedAt = nextState.startedAt ?? event.startedAt;
|
|
456
570
|
nextState.completedAt = event.completedAt;
|
|
457
|
-
nextState.error = cappedError;
|
|
458
|
-
nextState.failureClass = event.failureClass;
|
|
571
|
+
nextState.error = recipeOverride === undefined ? cappedError : recipeOverride.error;
|
|
572
|
+
nextState.failureClass = recipeOverride === undefined ? event.failureClass : recipeOverride.failureClass;
|
|
459
573
|
this.state = nextState;
|
|
460
574
|
await this.writeState();
|
|
461
575
|
});
|
|
@@ -463,7 +577,7 @@ export class WorkflowStateStore {
|
|
|
463
577
|
onCallPrepared: async (event) => {
|
|
464
578
|
await this.enqueueWrite(async () => {
|
|
465
579
|
const inputPath = `${event.stageId}.${event.callId}.input.json`;
|
|
466
|
-
await writeJsonAtomic(join(this.runDir, inputPath), event.input);
|
|
580
|
+
await writeJsonAtomic(this.writer, join(this.runDir, inputPath), event.input);
|
|
467
581
|
const nextState = structuredClone(this.state);
|
|
468
582
|
const callEntry = getCallEntry(nextState, event.stageId, event.callId);
|
|
469
583
|
callEntry.materializedInputPath = inputPath;
|
|
@@ -509,6 +623,9 @@ export class WorkflowStateStore {
|
|
|
509
623
|
callEntry.failureClass = undefined;
|
|
510
624
|
}
|
|
511
625
|
callEntry.childDebugMetadata = event.childDebugMetadata;
|
|
626
|
+
// Same lifetime as childDebugMetadata: only the failed emit that
|
|
627
|
+
// observed the marker carries it, and any later emit clears it.
|
|
628
|
+
callEntry.failureKind = event.state === "failed" ? event.failureKind : undefined;
|
|
512
629
|
callEntry.contextProducing = callEntry.contextProducing ?? false;
|
|
513
630
|
// Observed-invocation telemetry rides only the terminal succeeded
|
|
514
631
|
// emit; running/failed events carry none and must not write it.
|
|
@@ -518,7 +635,7 @@ export class WorkflowStateStore {
|
|
|
518
635
|
callEntry.observedInvocation = event.observedInvocation;
|
|
519
636
|
}
|
|
520
637
|
if (event.outputText != null) {
|
|
521
|
-
const { outputInline, outputPath } = await persistOutput(this.runDir, event.stageId, event.callId, event.outputText);
|
|
638
|
+
const { outputInline, outputPath } = await persistOutput(this.writer, this.runDir, event.stageId, event.callId, event.outputText);
|
|
522
639
|
callEntry.outputInline = outputInline;
|
|
523
640
|
callEntry.outputPath = outputPath;
|
|
524
641
|
}
|
|
@@ -545,15 +662,16 @@ export class WorkflowStateStore {
|
|
|
545
662
|
static async create(options) {
|
|
546
663
|
const runDir = options.runDir ?? getRunStateDir(options.runId);
|
|
547
664
|
const debugKeepArtifacts = options.debugKeepArtifacts ?? false;
|
|
548
|
-
|
|
549
|
-
|
|
665
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
666
|
+
await pruneRunDirectoriesBeforeCreate(runDir, fileWriter);
|
|
667
|
+
const recycled = await prepareRunDirForCreate(runDir, options.runId, fileWriter);
|
|
550
668
|
let lockHeld = false;
|
|
551
669
|
try {
|
|
552
670
|
const lock = await acquireRunLock(runDir, { declareHeartbeat: true });
|
|
553
671
|
lockHeld = true;
|
|
554
672
|
const sessionLogGeneration = await createSessionLogGeneration(runDir);
|
|
555
|
-
const workflowSnapshotHash = await writeJsonSnapshot(join(runDir, WORKFLOW_SNAPSHOT_FILENAME), options.definition);
|
|
556
|
-
const inputsSnapshotHash = await writeJsonSnapshot(join(runDir, INPUTS_SNAPSHOT_FILENAME), options.variables);
|
|
673
|
+
const workflowSnapshotHash = await writeJsonSnapshot(fileWriter, join(runDir, WORKFLOW_SNAPSHOT_FILENAME), options.definition);
|
|
674
|
+
const inputsSnapshotHash = await writeJsonSnapshot(fileWriter, join(runDir, INPUTS_SNAPSHOT_FILENAME), options.variables);
|
|
557
675
|
const duplicateOfFailedRun = await detectDuplicateReplay(runDir, workflowSnapshotHash, inputsSnapshotHash);
|
|
558
676
|
if (duplicateOfFailedRun !== undefined) {
|
|
559
677
|
logger.warn({
|
|
@@ -569,8 +687,7 @@ export class WorkflowStateStore {
|
|
|
569
687
|
: undefined;
|
|
570
688
|
if (fingerprintData != null) {
|
|
571
689
|
const childConfigDir = join(runDir, CHILD_CONFIG_DIRNAME);
|
|
572
|
-
await
|
|
573
|
-
await writeJsonAtomic(join(childConfigDir, CHILD_CONFIG_SNAPSHOT_FILENAME), fingerprintData.snapshot);
|
|
690
|
+
await writeJsonAtomic(fileWriter, join(childConfigDir, CHILD_CONFIG_SNAPSHOT_FILENAME), fingerprintData.snapshot);
|
|
574
691
|
}
|
|
575
692
|
const resolvedRoster = options.resolvedRoster !== undefined
|
|
576
693
|
? ResolvedRosterSchema.parse(options.resolvedRoster)
|
|
@@ -612,7 +729,7 @@ export class WorkflowStateStore {
|
|
|
612
729
|
...(resolvedRoster !== undefined ? { resolvedRoster } : {}),
|
|
613
730
|
...(rosterName !== undefined ? { rosterName } : {}),
|
|
614
731
|
};
|
|
615
|
-
await writeRunStateAtomic(join(runDir, STATE_FILENAME), state);
|
|
732
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), state);
|
|
616
733
|
if (recycled.recycled) {
|
|
617
734
|
await rm(`${runDir}.recycled`, { recursive: true, force: true }).catch(() => { });
|
|
618
735
|
if (recycled.retiredSessionLogGeneration !== undefined) {
|
|
@@ -629,6 +746,7 @@ export class WorkflowStateStore {
|
|
|
629
746
|
lockToken: lock.token,
|
|
630
747
|
sessionLogGeneration,
|
|
631
748
|
duplicateOfFailedRun,
|
|
749
|
+
fileWriter,
|
|
632
750
|
});
|
|
633
751
|
}
|
|
634
752
|
catch (error) {
|
|
@@ -653,9 +771,19 @@ export class WorkflowStateStore {
|
|
|
653
771
|
static async resume(options) {
|
|
654
772
|
const runDir = options.runDir ?? getRunStateDir(options.runId);
|
|
655
773
|
const debugKeepArtifacts = options.debugKeepArtifacts ?? false;
|
|
774
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
656
775
|
const lock = await acquireRunLock(runDir, { declareHeartbeat: true });
|
|
657
776
|
try {
|
|
777
|
+
// CONTRACTS 0-7: a recipe run is never generically resumed. Checked
|
|
778
|
+
// before the state read so no recovery rewrite can touch it; an lstat
|
|
779
|
+
// failure other than ENOENT also refuses (fail closed).
|
|
780
|
+
if (await recipeSidecarPresence(runDir) === "present") {
|
|
781
|
+
throw new Error(`Run ${options.runId} is a recipe run and cannot be resumed; use workflow_recipe_decide or cancel_workflow.`);
|
|
782
|
+
}
|
|
658
783
|
const state = await readRunState(runDir);
|
|
784
|
+
if (isSidecarRecipeWorkflow(state.workflowName)) {
|
|
785
|
+
throw new Error(`Run ${options.runId} is a recipe run and cannot be resumed; use workflow_recipe_decide or cancel_workflow.`);
|
|
786
|
+
}
|
|
659
787
|
const identity = await readSessionLogGeneration(runDir);
|
|
660
788
|
if (identity.kind === "unavailable") {
|
|
661
789
|
throw new Error(`Run ${options.runId} has an unsafe session-log identity`);
|
|
@@ -733,7 +861,7 @@ export class WorkflowStateStore {
|
|
|
733
861
|
stage.completedAt = deriveStageCompletedAt(stage.calls);
|
|
734
862
|
}
|
|
735
863
|
if (mutated) {
|
|
736
|
-
await writeRunStateAtomic(join(runDir, STATE_FILENAME), recovered);
|
|
864
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), recovered);
|
|
737
865
|
}
|
|
738
866
|
const resume = await buildResumeState(definition, recovered, runDir);
|
|
739
867
|
return new WorkflowStateStore({
|
|
@@ -744,6 +872,95 @@ export class WorkflowStateStore {
|
|
|
744
872
|
resume,
|
|
745
873
|
debugKeepArtifacts,
|
|
746
874
|
lockToken: lock.token,
|
|
875
|
+
fileWriter,
|
|
876
|
+
...(identity.kind === "available"
|
|
877
|
+
? { sessionLogGeneration: identity.sessionLogGeneration }
|
|
878
|
+
: {}),
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
catch (error) {
|
|
882
|
+
await safeUnlink(join(runDir, RUN_LOCK_FILENAME));
|
|
883
|
+
throw error;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
/**
|
|
887
|
+
* Reopen a paused recipe run for a control or decision execution (CONTRACTS
|
|
888
|
+
* 0-2 step 4, 0-7). Acquires `run.lock` with a heartbeat, verifies the
|
|
889
|
+
* workflow and inputs snapshot hashes, and requires terminal `succeeded`
|
|
890
|
+
* plus a sidecar at `awaiting_decision`. Read-only: nothing is written until
|
|
891
|
+
* the caller validates and calls `beginRecipeExecution()`. Calls are never
|
|
892
|
+
* generically rewritten, and the child-config fingerprint is not consulted
|
|
893
|
+
* (a pause can outlive adapter edits; the snapshots pin the definition).
|
|
894
|
+
* On refusal the lock is released and a `RecipeReopenError` is thrown.
|
|
895
|
+
*/
|
|
896
|
+
static async reopen(options) {
|
|
897
|
+
const runDir = options.runDir ?? getRunStateDir(options.runId);
|
|
898
|
+
const debugKeepArtifacts = options.debugKeepArtifacts ?? false;
|
|
899
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
900
|
+
if (!await fileExists(join(runDir, STATE_FILENAME))) {
|
|
901
|
+
throw new RecipeReopenError("not_found", `Run ${options.runId} not found`);
|
|
902
|
+
}
|
|
903
|
+
let lock;
|
|
904
|
+
try {
|
|
905
|
+
lock = await acquireRunLock(runDir, { declareHeartbeat: true });
|
|
906
|
+
}
|
|
907
|
+
catch (error) {
|
|
908
|
+
throw new RecipeReopenError("run_busy", `Run ${options.runId} is owned by another execution: ${error instanceof Error ? error.message : String(error)}`);
|
|
909
|
+
}
|
|
910
|
+
try {
|
|
911
|
+
let state;
|
|
912
|
+
try {
|
|
913
|
+
state = await readRunState(runDir);
|
|
914
|
+
}
|
|
915
|
+
catch (error) {
|
|
916
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} state is unreadable: ${error instanceof Error ? error.message : String(error)}`);
|
|
917
|
+
}
|
|
918
|
+
if (state.workflowName !== PING_PONG_WORKFLOW || state.runId !== options.runId) {
|
|
919
|
+
throw new RecipeReopenError("not_recipe", `Run ${options.runId} is not a ${PING_PONG_WORKFLOW} run`);
|
|
920
|
+
}
|
|
921
|
+
const identity = await readSessionLogGeneration(runDir);
|
|
922
|
+
if (identity.kind === "unavailable") {
|
|
923
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} has an unsafe session-log identity`);
|
|
924
|
+
}
|
|
925
|
+
const workflowSnapshotPath = join(runDir, state.workflowSnapshotPath ?? WORKFLOW_SNAPSHOT_FILENAME);
|
|
926
|
+
const inputsSnapshotPath = join(runDir, state.inputsSnapshotPath ?? INPUTS_SNAPSHOT_FILENAME);
|
|
927
|
+
let definition;
|
|
928
|
+
let variables;
|
|
929
|
+
try {
|
|
930
|
+
await verifySnapshotHash(workflowSnapshotPath, state.workflowSnapshotHash, "workflow snapshot");
|
|
931
|
+
await verifySnapshotHash(inputsSnapshotPath, state.inputsSnapshotHash, "inputs snapshot");
|
|
932
|
+
definition = await readJsonFile(workflowSnapshotPath);
|
|
933
|
+
variables = await readJsonFile(inputsSnapshotPath);
|
|
934
|
+
}
|
|
935
|
+
catch (error) {
|
|
936
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} snapshot verification failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
937
|
+
}
|
|
938
|
+
const recipe = await readRecipeState(runDir);
|
|
939
|
+
if (recipe.kind === "invalid") {
|
|
940
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} recipe sidecar is unreadable: ${recipe.reason}`);
|
|
941
|
+
}
|
|
942
|
+
if (recipe.kind === "absent" || state.status !== "succeeded" || recipe.state.phase !== "awaiting_decision") {
|
|
943
|
+
throw new RecipeReopenError("not_awaiting_decision", `Run ${options.runId} is not awaiting a decision (status ${state.status}, phase ${recipe.kind === "read" ? recipe.state.phase : "none"})`, recipe.kind === "read"
|
|
944
|
+
? { phase: recipe.state.phase, revision: recipe.state.revision, status: state.status }
|
|
945
|
+
: undefined);
|
|
946
|
+
}
|
|
947
|
+
let resume;
|
|
948
|
+
try {
|
|
949
|
+
resume = await buildResumeState(definition, state, runDir);
|
|
950
|
+
}
|
|
951
|
+
catch (error) {
|
|
952
|
+
throw new RecipeReopenError("integrity", `Run ${options.runId} executed calls cannot be rehydrated: ${error instanceof Error ? error.message : String(error)}`);
|
|
953
|
+
}
|
|
954
|
+
return new WorkflowStateStore({
|
|
955
|
+
runDir,
|
|
956
|
+
definition,
|
|
957
|
+
variables,
|
|
958
|
+
state,
|
|
959
|
+
resume,
|
|
960
|
+
debugKeepArtifacts,
|
|
961
|
+
lockToken: lock.token,
|
|
962
|
+
recipeState: recipe.state,
|
|
963
|
+
fileWriter,
|
|
747
964
|
...(identity.kind === "available"
|
|
748
965
|
? { sessionLogGeneration: identity.sessionLogGeneration }
|
|
749
966
|
: {}),
|
|
@@ -757,6 +974,167 @@ export class WorkflowStateStore {
|
|
|
757
974
|
get runState() {
|
|
758
975
|
return structuredClone(this.state);
|
|
759
976
|
}
|
|
977
|
+
/** The writer every file of this run goes through (durable in production). */
|
|
978
|
+
get fileWriter() {
|
|
979
|
+
return this.writer;
|
|
980
|
+
}
|
|
981
|
+
/** Last committed recipe sidecar held by this store (reopen loads it; commits refresh it). */
|
|
982
|
+
get recipeState() {
|
|
983
|
+
return this.recipeStateCache === undefined ? undefined : structuredClone(this.recipeStateCache);
|
|
984
|
+
}
|
|
985
|
+
/**
|
|
986
|
+
* Turn a reopened terminal envelope into a new active execution: status
|
|
987
|
+
* `running`, terminal fields and the previous execution id cleared. The
|
|
988
|
+
* caller then binds the new id (`bindExecution` via the run owner) before
|
|
989
|
+
* any sidecar commit. A crash in between leaves `running` with no id, which
|
|
990
|
+
* reconcile fails rather than settling the old pin.
|
|
991
|
+
*/
|
|
992
|
+
async beginRecipeExecution() {
|
|
993
|
+
await this.enqueueWrite(async () => {
|
|
994
|
+
if (!isTerminalRunStatus(this.state.status)) {
|
|
995
|
+
throw new RecipeStateInvariantError(`beginRecipeExecution requires a terminal run, found ${this.state.status}`);
|
|
996
|
+
}
|
|
997
|
+
const nextState = structuredClone(this.state);
|
|
998
|
+
nextState.status = "running";
|
|
999
|
+
nextState.completedAt = undefined;
|
|
1000
|
+
nextState.error = undefined;
|
|
1001
|
+
nextState.failureClass = undefined;
|
|
1002
|
+
nextState.executionId = undefined;
|
|
1003
|
+
this.state = nextState;
|
|
1004
|
+
await this.writeState();
|
|
1005
|
+
});
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Commit `recipe-state.json` on this store's write chain under its
|
|
1009
|
+
* `run.lock` (CONTRACTS 0-7). The store stamps `execution_id` (the bound
|
|
1010
|
+
* id; unbound -> error), `revision` (previous + 1), `updated_at` and
|
|
1011
|
+
* `unresolved_operations`, enforces the phase machine and the 0-4 byte
|
|
1012
|
+
* budget, and returns the committed state. Over budget throws
|
|
1013
|
+
* `RecipeStateCapacityError`; a non-terminal write may use 1 MiB minus the
|
|
1014
|
+
* 64 KiB terminal reserve. State.json is not touched.
|
|
1015
|
+
*/
|
|
1016
|
+
async commitRecipeState(draft) {
|
|
1017
|
+
let committed;
|
|
1018
|
+
await this.enqueueWrite(async () => {
|
|
1019
|
+
committed = await this.writeRecipeStateLocked(draft);
|
|
1020
|
+
});
|
|
1021
|
+
return committed;
|
|
1022
|
+
}
|
|
1023
|
+
/**
|
|
1024
|
+
* Terminal recipe commit in ONE critical section: the terminal sidecar first,
|
|
1025
|
+
* then the matching terminal `state.json` (`qualified`/`stopped` ->
|
|
1026
|
+
* succeeded, `failed` -> failed, `cancelled` -> cancelled). A crash between
|
|
1027
|
+
* the two is rolled forward by reconcile. Used by control executions
|
|
1028
|
+
* (decision stop, deadline, paused cancel) and by the runtime for a terminal
|
|
1029
|
+
* stop it publishes itself.
|
|
1030
|
+
*/
|
|
1031
|
+
async commitRecipeTerminal(draft, options = {}) {
|
|
1032
|
+
let committed;
|
|
1033
|
+
await this.enqueueWrite(async () => {
|
|
1034
|
+
if (!isTerminalRecipePhase(draft.phase) || draft.outcome === undefined) {
|
|
1035
|
+
throw new RecipeStateInvariantError("commitRecipeTerminal requires a terminal phase and outcome");
|
|
1036
|
+
}
|
|
1037
|
+
if (!(this.state.status === "running" || this.state.status === "pending")) {
|
|
1038
|
+
throw new RecipeStateInvariantError(`commitRecipeTerminal requires an active execution, found ${this.state.status}`);
|
|
1039
|
+
}
|
|
1040
|
+
committed = await this.writeRecipeStateLocked(draft);
|
|
1041
|
+
const status = runStatusForOutcome(draft.outcome);
|
|
1042
|
+
const nextState = structuredClone(this.state);
|
|
1043
|
+
nextState.status = status;
|
|
1044
|
+
nextState.completedAt = new Date().toISOString();
|
|
1045
|
+
nextState.error = capPersistedError(options.error ?? (status === "cancelled" ? "Workflow cancelled" : status === "failed" ? `Recipe stopped: ${draft.stop_reason ?? "failed"}` : undefined));
|
|
1046
|
+
nextState.failureClass = status === "failed" ? options.failureClass ?? "permanent" : options.failureClass;
|
|
1047
|
+
this.state = nextState;
|
|
1048
|
+
await this.writeState();
|
|
1049
|
+
});
|
|
1050
|
+
return committed;
|
|
1051
|
+
}
|
|
1052
|
+
/**
|
|
1053
|
+
* D1 terminal precedence for a recipe run, inside the write chain: the
|
|
1054
|
+
* terminal fields to publish INSTEAD of `intent`, or `undefined` to publish
|
|
1055
|
+
* `intent` as the caller would. A committed terminal sidecar of this
|
|
1056
|
+
* execution decides; a non-terminal one is terminalized first (cancelled or
|
|
1057
|
+
* failed), so the sidecar is never behind the state that settles it. A
|
|
1058
|
+
* failed sidecar commit throws `RecipeTerminalUnrecordedError` and the
|
|
1059
|
+
* caller must leave state.json active, as after a crash between the two
|
|
1060
|
+
* writes. Non-recipe runs return `undefined` without any I/O.
|
|
1061
|
+
*/
|
|
1062
|
+
async recipeTerminalOverrideLocked(intent) {
|
|
1063
|
+
if (!isSidecarRecipeWorkflow(this.state.workflowName))
|
|
1064
|
+
return undefined;
|
|
1065
|
+
const authority = await this.recipeAuthorityLocked();
|
|
1066
|
+
switch (authority.kind) {
|
|
1067
|
+
case "none":
|
|
1068
|
+
case "unbound":
|
|
1069
|
+
case "foreign":
|
|
1070
|
+
return undefined;
|
|
1071
|
+
case "unreadable":
|
|
1072
|
+
logger.error({ run_id: this.state.runId, reason: authority.reason }, "recipe sidecar is unreadable; publishing the terminal intent without deriving an outcome");
|
|
1073
|
+
return undefined;
|
|
1074
|
+
case "terminal":
|
|
1075
|
+
return authority.status === intent
|
|
1076
|
+
? undefined
|
|
1077
|
+
: recipeOutcomeEnvelope(authority.outcome, authority.state.stop_reason);
|
|
1078
|
+
case "active":
|
|
1079
|
+
if (intent === "succeeded")
|
|
1080
|
+
return undefined;
|
|
1081
|
+
try {
|
|
1082
|
+
await this.writeRecipeStateLocked(recipeSettlementDraft(authority.state, intent, settlementNote(intent)));
|
|
1083
|
+
}
|
|
1084
|
+
catch (error) {
|
|
1085
|
+
// Published bytes count as committed (the cache already holds them).
|
|
1086
|
+
if (!(error instanceof DurablePublishError))
|
|
1087
|
+
throw new RecipeTerminalUnrecordedError(intent, error);
|
|
1088
|
+
}
|
|
1089
|
+
return undefined;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
/** Runs inside the write chain. */
|
|
1093
|
+
async recipeAuthorityLocked() {
|
|
1094
|
+
if (!this.recipeStateLoaded) {
|
|
1095
|
+
const current = await readRecipeState(this.runDir);
|
|
1096
|
+
if (current.kind === "invalid")
|
|
1097
|
+
return classifyRecipeTerminalAuthority(current, this.state.executionId);
|
|
1098
|
+
this.recipeStateCache = current.kind === "read" ? current.state : undefined;
|
|
1099
|
+
this.recipeStateLoaded = true;
|
|
1100
|
+
}
|
|
1101
|
+
return classifyRecipeTerminalAuthority(this.recipeStateCache === undefined ? { kind: "absent" } : { kind: "read", state: this.recipeStateCache }, this.state.executionId);
|
|
1102
|
+
}
|
|
1103
|
+
/** Runs inside the write chain; never enqueue from here. */
|
|
1104
|
+
async writeRecipeStateLocked(draft) {
|
|
1105
|
+
if (!this.lockHeld) {
|
|
1106
|
+
throw new RecipeStateInvariantError("recipe state can only be committed while this store holds run.lock");
|
|
1107
|
+
}
|
|
1108
|
+
const executionId = this.state.executionId;
|
|
1109
|
+
if (executionId === undefined) {
|
|
1110
|
+
throw new RecipeStateInvariantError("recipe state can only be committed by a bound execution");
|
|
1111
|
+
}
|
|
1112
|
+
if (!this.recipeStateLoaded) {
|
|
1113
|
+
const current = await readRecipeState(this.runDir);
|
|
1114
|
+
if (current.kind === "invalid") {
|
|
1115
|
+
throw new RecipeStateInvariantError(`existing recipe sidecar is unreadable: ${current.reason}`);
|
|
1116
|
+
}
|
|
1117
|
+
this.recipeStateCache = current.kind === "read" ? current.state : undefined;
|
|
1118
|
+
this.recipeStateLoaded = true;
|
|
1119
|
+
}
|
|
1120
|
+
const next = stampRecipeState(draft, {
|
|
1121
|
+
runId: this.state.runId,
|
|
1122
|
+
executionId,
|
|
1123
|
+
previous: this.recipeStateCache,
|
|
1124
|
+
});
|
|
1125
|
+
try {
|
|
1126
|
+
await writeRecipeStateAtomic(this.runDir, next, this.writer);
|
|
1127
|
+
}
|
|
1128
|
+
catch (error) {
|
|
1129
|
+
// The bytes are already visible; the next commit must build on them, or
|
|
1130
|
+
// it would reuse this revision.
|
|
1131
|
+
if (error instanceof DurablePublishError)
|
|
1132
|
+
this.recipeStateCache = next;
|
|
1133
|
+
throw error;
|
|
1134
|
+
}
|
|
1135
|
+
this.recipeStateCache = next;
|
|
1136
|
+
return structuredClone(next);
|
|
1137
|
+
}
|
|
760
1138
|
/**
|
|
761
1139
|
* Open a new generic resume attempt before any context preflight or provider
|
|
762
1140
|
* setup. Provider-call attempt counters remain untouched; only the run-level
|
|
@@ -822,10 +1200,13 @@ export class WorkflowStateStore {
|
|
|
822
1200
|
if (this.state.status !== "pending" && this.state.status !== "running") {
|
|
823
1201
|
return;
|
|
824
1202
|
}
|
|
1203
|
+
const recipeOverride = await this.recipeTerminalOverrideLocked("cancelled");
|
|
825
1204
|
const nextState = structuredClone(this.state);
|
|
826
|
-
nextState.status = "cancelled";
|
|
1205
|
+
nextState.status = recipeOverride?.status ?? "cancelled";
|
|
827
1206
|
nextState.completedAt = new Date().toISOString();
|
|
828
|
-
nextState.error = reason;
|
|
1207
|
+
nextState.error = recipeOverride === undefined ? reason : recipeOverride.error;
|
|
1208
|
+
if (recipeOverride !== undefined)
|
|
1209
|
+
nextState.failureClass = recipeOverride.failureClass;
|
|
829
1210
|
this.state = nextState;
|
|
830
1211
|
await this.writeState();
|
|
831
1212
|
});
|
|
@@ -852,11 +1233,15 @@ export class WorkflowStateStore {
|
|
|
852
1233
|
if (this.state.status !== "pending" && this.state.status !== "running") {
|
|
853
1234
|
return;
|
|
854
1235
|
}
|
|
1236
|
+
const recipeOverride = await this.recipeTerminalOverrideLocked("failed");
|
|
855
1237
|
const nextState = structuredClone(this.state);
|
|
856
|
-
nextState.status = "failed";
|
|
1238
|
+
nextState.status = recipeOverride?.status ?? "failed";
|
|
857
1239
|
nextState.completedAt = new Date().toISOString();
|
|
858
|
-
nextState.error = cappedError;
|
|
859
|
-
if (
|
|
1240
|
+
nextState.error = recipeOverride === undefined ? cappedError : recipeOverride.error;
|
|
1241
|
+
if (recipeOverride !== undefined) {
|
|
1242
|
+
nextState.failureClass = recipeOverride.failureClass;
|
|
1243
|
+
}
|
|
1244
|
+
else if (failureClass !== undefined) {
|
|
860
1245
|
nextState.failureClass = failureClass;
|
|
861
1246
|
}
|
|
862
1247
|
this.state = nextState;
|
|
@@ -886,7 +1271,7 @@ export class WorkflowStateStore {
|
|
|
886
1271
|
},
|
|
887
1272
|
};
|
|
888
1273
|
const inputsSnapshotPath = this.state.inputsSnapshotPath ?? INPUTS_SNAPSHOT_FILENAME;
|
|
889
|
-
const inputsSnapshotHash = await writeJsonSnapshot(join(this.runDir, inputsSnapshotPath), variables);
|
|
1274
|
+
const inputsSnapshotHash = await writeJsonSnapshot(this.writer, join(this.runDir, inputsSnapshotPath), variables);
|
|
890
1275
|
const nextState = structuredClone(this.state);
|
|
891
1276
|
nextState.inputsSnapshotPath = inputsSnapshotPath;
|
|
892
1277
|
nextState.inputsSnapshotHash = inputsSnapshotHash;
|
|
@@ -896,7 +1281,7 @@ export class WorkflowStateStore {
|
|
|
896
1281
|
this.state = nextState;
|
|
897
1282
|
await this.writeState();
|
|
898
1283
|
try {
|
|
899
|
-
await writeJsonAtomic(join(getStateDir(), "workflow-contexts", `${this.state.runId}.json`), committedWorkflowContext);
|
|
1284
|
+
await writeJsonAtomic(this.writer, join(getStateDir(), "workflow-contexts", `${this.state.runId}.json`), committedWorkflowContext);
|
|
900
1285
|
}
|
|
901
1286
|
catch (error) {
|
|
902
1287
|
logger.warn({ runId: this.state.runId, err: error }, "failed to write workflow context provenance");
|
|
@@ -1164,6 +1549,7 @@ export class WorkflowStateStore {
|
|
|
1164
1549
|
intent,
|
|
1165
1550
|
chairmanCallId: resolvePlanReviewChairmanCallId(this.definition),
|
|
1166
1551
|
serializedResult: this.planReviewTerminalResult,
|
|
1552
|
+
fileWriter: this.writer,
|
|
1167
1553
|
});
|
|
1168
1554
|
validateStrictStateSubtrees(finalized.state, join(this.runDir, STATE_FILENAME));
|
|
1169
1555
|
this.state = finalized.state;
|
|
@@ -1178,20 +1564,32 @@ export class WorkflowStateStore {
|
|
|
1178
1564
|
await next;
|
|
1179
1565
|
}
|
|
1180
1566
|
async writeState() {
|
|
1181
|
-
await writeRunStateAtomic(join(this.runDir, STATE_FILENAME), this.state);
|
|
1567
|
+
await writeRunStateAtomic(this.writer, join(this.runDir, STATE_FILENAME), this.state);
|
|
1182
1568
|
}
|
|
1183
1569
|
}
|
|
1184
|
-
async function prepareRunDirForCreate(runDir, runId) {
|
|
1185
|
-
await
|
|
1570
|
+
async function prepareRunDirForCreate(runDir, runId, fileWriter) {
|
|
1571
|
+
await mkdirDurable(fileWriter, dirname(runDir));
|
|
1572
|
+
let created = false;
|
|
1186
1573
|
try {
|
|
1187
1574
|
await mkdir(runDir);
|
|
1188
|
-
|
|
1575
|
+
created = true;
|
|
1189
1576
|
}
|
|
1190
1577
|
catch (error) {
|
|
1191
1578
|
if (!(error instanceof Error) || !("code" in error) || error.code !== "EEXIST") {
|
|
1192
1579
|
throw error;
|
|
1193
1580
|
}
|
|
1194
1581
|
}
|
|
1582
|
+
if (created) {
|
|
1583
|
+
try {
|
|
1584
|
+
await fileWriter.syncDirectory(dirname(runDir));
|
|
1585
|
+
}
|
|
1586
|
+
catch (error) {
|
|
1587
|
+
// Nothing is in the directory yet; leave no half-admitted run behind.
|
|
1588
|
+
await rmdir(runDir).catch(() => { });
|
|
1589
|
+
throw error;
|
|
1590
|
+
}
|
|
1591
|
+
return { recycled: false };
|
|
1592
|
+
}
|
|
1195
1593
|
const recycled = await recycleSucceededRunDir(runDir);
|
|
1196
1594
|
if (recycled.recycled) {
|
|
1197
1595
|
const recycledPath = `${runDir}.recycled`;
|
|
@@ -1210,18 +1608,32 @@ async function prepareRunDirForCreate(runDir, runId) {
|
|
|
1210
1608
|
}
|
|
1211
1609
|
throw error;
|
|
1212
1610
|
}
|
|
1611
|
+
// One fsync publishes both the retirement rename and the fresh directory.
|
|
1612
|
+
try {
|
|
1613
|
+
await fileWriter.syncDirectory(dirname(runDir));
|
|
1614
|
+
}
|
|
1615
|
+
catch (error) {
|
|
1616
|
+
await rmdir(runDir).catch(() => { });
|
|
1617
|
+
try {
|
|
1618
|
+
await rename(recycledPath, runDir);
|
|
1619
|
+
}
|
|
1620
|
+
catch {
|
|
1621
|
+
logger.warn({ recycledPath }, "failed to restore recycled directory after runs-root fsync failure");
|
|
1622
|
+
}
|
|
1623
|
+
throw error;
|
|
1624
|
+
}
|
|
1213
1625
|
return recycled;
|
|
1214
1626
|
}
|
|
1215
1627
|
throw duplicateRunDirError(runId);
|
|
1216
1628
|
}
|
|
1217
|
-
async function pruneRunDirectoriesBeforeCreate(runDir) {
|
|
1629
|
+
async function pruneRunDirectoriesBeforeCreate(runDir, fileWriter) {
|
|
1218
1630
|
const runsRoot = getRunsDir();
|
|
1219
1631
|
if (dirname(runDir) !== runsRoot)
|
|
1220
1632
|
return;
|
|
1221
1633
|
// Reconcile before pruning so a stale running run terminalized here becomes
|
|
1222
1634
|
// prunable by a later create once past its retention window.
|
|
1223
1635
|
try {
|
|
1224
|
-
await reconcileStaleRunDirectories(runsRoot);
|
|
1636
|
+
await reconcileStaleRunDirectories(runsRoot, { fileWriter });
|
|
1225
1637
|
}
|
|
1226
1638
|
catch (error) {
|
|
1227
1639
|
logger.warn({ runs_root_dir: runsRoot, err: error instanceof Error ? error.message : String(error) }, "stale workflow run reconcile failed before create");
|
|
@@ -1549,33 +1961,9 @@ export async function readRunStateForStatus(runId, containedRead = false) {
|
|
|
1549
1961
|
...(read.digest !== undefined ? { stateDigest: read.digest } : {}),
|
|
1550
1962
|
};
|
|
1551
1963
|
}
|
|
1552
|
-
|
|
1553
|
-
* Reap a run that has no live worker (orphaned/stale/prior-process) by writing a
|
|
1554
|
-
* terminal "cancelled" state and unlinking its lock. Used by cancel_workflow when
|
|
1555
|
-
* the run is not in this process's in-memory cancel registry.
|
|
1556
|
-
*
|
|
1557
|
-
* The run.lock is the single source of truth for ownership: this acquires the
|
|
1558
|
-
* lock itself (reusing acquireRunLock, which atomically creates the lock and
|
|
1559
|
-
* reclaims a stale same-host/dead-PID lock) BEFORE mutating, closing the TOCTOU
|
|
1560
|
-
* where a concurrent resume_workflow could grab the lock between a caller's
|
|
1561
|
-
* status read and the reap write.
|
|
1562
|
-
*
|
|
1563
|
-
* Returns:
|
|
1564
|
-
* - "locked" — a LIVE lock blocks acquisition (held by a running worker, or a
|
|
1565
|
-
* same-process blocking run); nothing is mutated.
|
|
1566
|
-
* - "ownership_unknown" — the lock is malformed/unreadable; nothing is mutated
|
|
1567
|
-
* or reclaimed.
|
|
1568
|
-
* - "skipped" — lock acquired but the run is already terminal (not
|
|
1569
|
-
* pending/running); the lock is released and nothing is mutated.
|
|
1570
|
-
* The caller re-reads to report already_terminal.
|
|
1571
|
-
* - "cancelled" — lock acquired and run was pending/running; terminal
|
|
1572
|
-
* "cancelled" state is written and the lock removed.
|
|
1573
|
-
*
|
|
1574
|
-
* NOTE: This intentionally does NOT use WorkflowStateStore.resume() (which
|
|
1575
|
-
* recovers + re-locks the run — the wrong intent here).
|
|
1576
|
-
*/
|
|
1577
|
-
export async function reapRunCancelled(runId) {
|
|
1964
|
+
export async function reapRunCancelled(runId, options = {}) {
|
|
1578
1965
|
const runDir = getRunStateDir(runId);
|
|
1966
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
1579
1967
|
try {
|
|
1580
1968
|
await acquireRunLock(runDir);
|
|
1581
1969
|
}
|
|
@@ -1594,6 +1982,19 @@ export async function reapRunCancelled(runId) {
|
|
|
1594
1982
|
if (state.status !== "pending" && state.status !== "running") {
|
|
1595
1983
|
return "skipped";
|
|
1596
1984
|
}
|
|
1985
|
+
let disposition = { kind: "cancel" };
|
|
1986
|
+
if (options.beforeWrite !== undefined) {
|
|
1987
|
+
try {
|
|
1988
|
+
disposition = (await options.beforeWrite({ runDir, state, fileWriter })) ?? { kind: "cancel" };
|
|
1989
|
+
}
|
|
1990
|
+
catch (error) {
|
|
1991
|
+
disposition = { kind: "unavailable", reason: error instanceof Error ? error.message : String(error) };
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1994
|
+
if (disposition.kind === "unavailable") {
|
|
1995
|
+
logger.error({ run_id: runId, reason: disposition.reason }, "orphan cancel refused: the recipe sidecar could not be read or recorded; nothing written");
|
|
1996
|
+
return "recipe_state_unavailable";
|
|
1997
|
+
}
|
|
1597
1998
|
let nextState = structuredClone(state);
|
|
1598
1999
|
if (isPlanReviewRun(state)) {
|
|
1599
2000
|
const intent = {
|
|
@@ -1611,19 +2012,26 @@ export async function reapRunCancelled(runId) {
|
|
|
1611
2012
|
previousState: state,
|
|
1612
2013
|
intendedState: intended,
|
|
1613
2014
|
intent,
|
|
2015
|
+
fileWriter,
|
|
1614
2016
|
});
|
|
1615
2017
|
nextState = finalized.state;
|
|
1616
2018
|
validateStrictStateSubtrees(nextState, join(runDir, STATE_FILENAME));
|
|
1617
2019
|
}
|
|
2020
|
+
else if (disposition.kind === "settled") {
|
|
2021
|
+
nextState.status = disposition.envelope.status;
|
|
2022
|
+
nextState.completedAt = new Date().toISOString();
|
|
2023
|
+
nextState.error = disposition.envelope.error;
|
|
2024
|
+
nextState.failureClass = disposition.envelope.failureClass;
|
|
2025
|
+
}
|
|
1618
2026
|
else {
|
|
1619
2027
|
nextState.status = "cancelled";
|
|
1620
2028
|
nextState.completedAt = new Date().toISOString();
|
|
1621
2029
|
nextState.error = "Reaped: no live worker process";
|
|
1622
2030
|
}
|
|
1623
|
-
await writeRunStateAtomic(join(runDir, STATE_FILENAME), nextState);
|
|
2031
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), nextState);
|
|
1624
2032
|
await safeUnlink(lockPath);
|
|
1625
2033
|
lockHeld = false;
|
|
1626
|
-
return "cancelled";
|
|
2034
|
+
return disposition.kind === "settled" ? "settled" : "cancelled";
|
|
1627
2035
|
}
|
|
1628
2036
|
finally {
|
|
1629
2037
|
if (lockHeld) {
|
|
@@ -1653,7 +2061,8 @@ export async function reapRunCancelled(runId) {
|
|
|
1653
2061
|
* once, then release. Idempotent: reconciled runs are terminal `failed` and
|
|
1654
2062
|
* count as skipped_terminal on re-scan.
|
|
1655
2063
|
*/
|
|
1656
|
-
export async function reconcileStaleRunDirectories(runsRootDir) {
|
|
2064
|
+
export async function reconcileStaleRunDirectories(runsRootDir, options = {}) {
|
|
2065
|
+
const fileWriter = resolveFileWriter(options.fileWriter);
|
|
1657
2066
|
const result = {
|
|
1658
2067
|
scanned: 0,
|
|
1659
2068
|
reconciled: 0,
|
|
@@ -1701,7 +2110,7 @@ export async function reconcileStaleRunDirectories(runsRootDir) {
|
|
|
1701
2110
|
continue;
|
|
1702
2111
|
}
|
|
1703
2112
|
try {
|
|
1704
|
-
const outcome = await reconcileStaleRun(runDirPath);
|
|
2113
|
+
const outcome = await reconcileStaleRun(runDirPath, fileWriter);
|
|
1705
2114
|
result[outcome] += 1;
|
|
1706
2115
|
}
|
|
1707
2116
|
catch (error) {
|
|
@@ -1716,7 +2125,7 @@ export async function reconcileStaleRunDirectories(runsRootDir) {
|
|
|
1716
2125
|
* Terminalize one pre-screened stale run under its own lock. See
|
|
1717
2126
|
* reconcileStaleRunDirectories for the authority and safety model.
|
|
1718
2127
|
*/
|
|
1719
|
-
async function reconcileStaleRun(runDir) {
|
|
2128
|
+
async function reconcileStaleRun(runDir, fileWriter) {
|
|
1720
2129
|
try {
|
|
1721
2130
|
await acquireRunLock(runDir);
|
|
1722
2131
|
}
|
|
@@ -1732,6 +2141,7 @@ async function reconcileStaleRun(runDir) {
|
|
|
1732
2141
|
return "skipped_terminal";
|
|
1733
2142
|
}
|
|
1734
2143
|
const reconciledAt = new Date().toISOString();
|
|
2144
|
+
const recipeRollForward = await recipeReconcileStatus(runDir, state, fileWriter);
|
|
1735
2145
|
let nextState = structuredClone(state);
|
|
1736
2146
|
for (const stage of nextState.stages) {
|
|
1737
2147
|
for (const call of stage.calls) {
|
|
@@ -1752,6 +2162,19 @@ async function reconcileStaleRun(runDir) {
|
|
|
1752
2162
|
stage.state = deriveStageState(stage.calls);
|
|
1753
2163
|
stage.completedAt = deriveStageCompletedAt(stage.calls);
|
|
1754
2164
|
}
|
|
2165
|
+
if (recipeRollForward !== undefined) {
|
|
2166
|
+
// CONTRACTS 0-7: the sidecar already records how this execution ended
|
|
2167
|
+
// (a complete pause checkpoint, or a terminal outcome); publish it.
|
|
2168
|
+
nextState.status = recipeRollForward;
|
|
2169
|
+
nextState.completedAt = reconciledAt;
|
|
2170
|
+
nextState.error = recipeRollForward === "succeeded"
|
|
2171
|
+
? undefined
|
|
2172
|
+
: recipeRollForward === "cancelled" ? "Workflow cancelled" : RECIPE_ROLL_FORWARD_FAILED_ERROR;
|
|
2173
|
+
nextState.failureClass = recipeRollForward === "failed" ? "permanent" : undefined;
|
|
2174
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), nextState);
|
|
2175
|
+
logger.info({ run_dir: runDir, run_id: state.runId, status: recipeRollForward }, "rolled stale recipe run forward from its sidecar");
|
|
2176
|
+
return "reconciled";
|
|
2177
|
+
}
|
|
1755
2178
|
if (isPlanReviewRun(state)) {
|
|
1756
2179
|
const intent = {
|
|
1757
2180
|
status: "failed",
|
|
@@ -1769,6 +2192,7 @@ async function reconcileStaleRun(runDir) {
|
|
|
1769
2192
|
previousState: state,
|
|
1770
2193
|
intendedState: intended,
|
|
1771
2194
|
intent,
|
|
2195
|
+
fileWriter,
|
|
1772
2196
|
});
|
|
1773
2197
|
nextState = finalized.state;
|
|
1774
2198
|
validateStrictStateSubtrees(nextState, join(runDir, STATE_FILENAME));
|
|
@@ -1779,7 +2203,7 @@ async function reconcileStaleRun(runDir) {
|
|
|
1779
2203
|
nextState.error = STALE_RUN_RECONCILE_ERROR;
|
|
1780
2204
|
nextState.failureClass = "transient";
|
|
1781
2205
|
}
|
|
1782
|
-
await writeRunStateAtomic(join(runDir, STATE_FILENAME), nextState);
|
|
2206
|
+
await writeRunStateAtomic(fileWriter, join(runDir, STATE_FILENAME), nextState);
|
|
1783
2207
|
logger.info({ run_dir: runDir, run_id: state.runId }, "reconciled stale running workflow run to terminal failed");
|
|
1784
2208
|
return "reconciled";
|
|
1785
2209
|
}
|
|
@@ -1787,6 +2211,92 @@ async function reconcileStaleRun(runDir) {
|
|
|
1787
2211
|
await safeUnlink(lockPath);
|
|
1788
2212
|
}
|
|
1789
2213
|
}
|
|
2214
|
+
export const RECIPE_ROLL_FORWARD_FAILED_ERROR = "Reconciled: recipe sidecar recorded a failed outcome";
|
|
2215
|
+
/**
|
|
2216
|
+
* Reconcile rule (CONTRACTS 0-7, D1), evaluated under the lock. Only a sidecar
|
|
2217
|
+
* of THIS execution counts: a complete `awaiting_decision` checkpoint with
|
|
2218
|
+
* zero unresolved operations publishes `succeeded`; a terminal sidecar
|
|
2219
|
+
* publishes its outcome's status. Any other active sidecar of this execution
|
|
2220
|
+
* is first terminalized `stopped(recovery_required)` (operations without a
|
|
2221
|
+
* result become uncertain), as `mark*` does, so the sidecar is never behind
|
|
2222
|
+
* the `failed` state that follows; if that record cannot be committed this
|
|
2223
|
+
* throws `RecipeTerminalUnrecordedError` and state.json stays active. No,
|
|
2224
|
+
* unreadable, or another execution's sidecar is left as is, and the run gets
|
|
2225
|
+
* the ordinary stale-run failure.
|
|
2226
|
+
*/
|
|
2227
|
+
async function recipeReconcileStatus(runDir, state, fileWriter) {
|
|
2228
|
+
if (state.workflowName !== PING_PONG_WORKFLOW)
|
|
2229
|
+
return undefined;
|
|
2230
|
+
const authority = classifyRecipeTerminalAuthority(await readRecipeState(runDir), state.executionId);
|
|
2231
|
+
if (authority.kind === "terminal")
|
|
2232
|
+
return authority.status;
|
|
2233
|
+
if (authority.kind !== "active")
|
|
2234
|
+
return undefined;
|
|
2235
|
+
if (isCompleteAwaitingCheckpoint(authority.state, state.executionId))
|
|
2236
|
+
return "succeeded";
|
|
2237
|
+
const next = stampRecipeState(recipeSettlementDraft(authority.state, "failed", settlementNote("failed")), {
|
|
2238
|
+
runId: state.runId,
|
|
2239
|
+
executionId: authority.state.execution_id,
|
|
2240
|
+
previous: authority.state,
|
|
2241
|
+
});
|
|
2242
|
+
try {
|
|
2243
|
+
await writeRecipeStateAtomic(runDir, next, fileWriter);
|
|
2244
|
+
}
|
|
2245
|
+
catch (error) {
|
|
2246
|
+
// Published bytes count as committed.
|
|
2247
|
+
if (!(error instanceof DurablePublishError))
|
|
2248
|
+
throw new RecipeTerminalUnrecordedError("failed", error);
|
|
2249
|
+
}
|
|
2250
|
+
return undefined;
|
|
2251
|
+
}
|
|
2252
|
+
/**
|
|
2253
|
+
* `reapRunCancelled` beforeWrite hook for an orphaned ping-pong run (CONTRACTS
|
|
2254
|
+
* 0-7, D1), under the lock. Only a sidecar of THIS execution is ever written
|
|
2255
|
+
* or decides:
|
|
2256
|
+
* - a terminal one settles the run with its own outcome's status;
|
|
2257
|
+
* - a non-terminal one records terminal `stopped(cancelled)` (every operation
|
|
2258
|
+
* without a result becomes `uncertain`; `effects_stopped` only when none
|
|
2259
|
+
* remain), then the run is cancelled;
|
|
2260
|
+
* - absent, another execution's, or an unbound state: the sidecar bytes stay
|
|
2261
|
+
* as they are and the run is cancelled;
|
|
2262
|
+
* - unreadable, or a record that cannot be committed: `unavailable`, and the
|
|
2263
|
+
* reap writes nothing.
|
|
2264
|
+
*/
|
|
2265
|
+
export async function recordRecipeOrphanCancel(context) {
|
|
2266
|
+
if (context.state.workflowName !== PING_PONG_WORKFLOW)
|
|
2267
|
+
return { kind: "cancel" };
|
|
2268
|
+
const authority = classifyRecipeTerminalAuthority(await readRecipeState(context.runDir), context.state.executionId);
|
|
2269
|
+
switch (authority.kind) {
|
|
2270
|
+
case "none":
|
|
2271
|
+
case "unbound":
|
|
2272
|
+
case "foreign":
|
|
2273
|
+
return { kind: "cancel" };
|
|
2274
|
+
case "unreadable":
|
|
2275
|
+
return { kind: "unavailable", reason: `recipe sidecar is unreadable: ${authority.reason}` };
|
|
2276
|
+
case "terminal":
|
|
2277
|
+
return { kind: "settled", envelope: recipeOutcomeEnvelope(authority.outcome, authority.state.stop_reason) };
|
|
2278
|
+
case "active": {
|
|
2279
|
+
const draft = recipeSettlementDraft(authority.state, "cancelled", "no live worker at cancel; result unknown");
|
|
2280
|
+
try {
|
|
2281
|
+
const next = stampRecipeState(draft, {
|
|
2282
|
+
runId: context.state.runId,
|
|
2283
|
+
executionId: authority.state.execution_id,
|
|
2284
|
+
previous: authority.state,
|
|
2285
|
+
});
|
|
2286
|
+
await writeRecipeStateAtomic(context.runDir, next, resolveFileWriter(context.fileWriter));
|
|
2287
|
+
}
|
|
2288
|
+
catch (error) {
|
|
2289
|
+
if (!(error instanceof DurablePublishError)) {
|
|
2290
|
+
return {
|
|
2291
|
+
kind: "unavailable",
|
|
2292
|
+
reason: `terminal recipe sidecar could not be committed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2293
|
+
};
|
|
2294
|
+
}
|
|
2295
|
+
}
|
|
2296
|
+
return { kind: "cancel" };
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
1790
2300
|
function isRunLockRecord(value) {
|
|
1791
2301
|
const record = asRecord(value);
|
|
1792
2302
|
const pid = record["pid"];
|
|
@@ -1818,6 +2328,11 @@ async function recycleSucceededRunDir(runDir) {
|
|
|
1818
2328
|
if (state.status !== "succeeded") {
|
|
1819
2329
|
return { recycled: false };
|
|
1820
2330
|
}
|
|
2331
|
+
// CONTRACTS 0-7 run-ID recycling: a recipe run (a pause is terminal
|
|
2332
|
+
// succeeded) is never recycled; fail closed on an uncertain sidecar probe.
|
|
2333
|
+
if (isSidecarRecipeWorkflow(state.workflowName) || await recipeSidecarPresence(runDir) === "present") {
|
|
2334
|
+
return { recycled: false };
|
|
2335
|
+
}
|
|
1821
2336
|
const identity = await readSessionLogGeneration(runDir);
|
|
1822
2337
|
let retiredSessionLogGeneration;
|
|
1823
2338
|
if (identity.kind === "available") {
|
|
@@ -1921,7 +2436,7 @@ function capPersistedError(message) {
|
|
|
1921
2436
|
const omitted = encoded.length - end;
|
|
1922
2437
|
return `${encoded.subarray(0, end).toString("utf-8")} …[truncated ${omitted} bytes]`;
|
|
1923
2438
|
}
|
|
1924
|
-
async function persistOutput(runDir, stageId, callId, outputText) {
|
|
2439
|
+
async function persistOutput(fileWriter, runDir, stageId, callId, outputText) {
|
|
1925
2440
|
const sizeBytes = Buffer.byteLength(outputText, "utf-8");
|
|
1926
2441
|
if (sizeBytes <= OUTPUT_INLINE_THRESHOLD_BYTES) {
|
|
1927
2442
|
return {
|
|
@@ -1930,12 +2445,21 @@ async function persistOutput(runDir, stageId, callId, outputText) {
|
|
|
1930
2445
|
};
|
|
1931
2446
|
}
|
|
1932
2447
|
const relativePath = `${stageId}.${callId}.output.txt`;
|
|
1933
|
-
await writeTextAtomic(join(runDir, relativePath), outputText);
|
|
2448
|
+
await writeTextAtomic(fileWriter, join(runDir, relativePath), outputText);
|
|
1934
2449
|
return {
|
|
1935
2450
|
outputInline: undefined,
|
|
1936
2451
|
outputPath: relativePath,
|
|
1937
2452
|
};
|
|
1938
2453
|
}
|
|
2454
|
+
/** Provenance from untrusted persisted state: only exact known literals survive. */
|
|
2455
|
+
function persistedFailureProvenance(entry) {
|
|
2456
|
+
const failureClass = entry.failureClass === "transient" || entry.failureClass === "permanent" ? entry.failureClass : undefined;
|
|
2457
|
+
const failureKind = parseCallFailureKind(entry.failureKind);
|
|
2458
|
+
return {
|
|
2459
|
+
...(failureClass !== undefined ? { failureClass } : {}),
|
|
2460
|
+
...(failureKind !== undefined ? { failureKind } : {}),
|
|
2461
|
+
};
|
|
2462
|
+
}
|
|
1939
2463
|
async function buildResumeState(definition, state, runDir) {
|
|
1940
2464
|
const completedCalls = new Map();
|
|
1941
2465
|
const preparedInputs = new Map();
|
|
@@ -1958,7 +2482,7 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
1958
2482
|
for (const call of stage.calls) {
|
|
1959
2483
|
const entry = persistedStage.calls.find((candidate) => candidate.callId === call.id);
|
|
1960
2484
|
if (entry?.state === "succeeded") {
|
|
1961
|
-
completedCalls.set(`${stage.id}.${call.id}`, await rehydrateExecutedCall(runDir, entry, definition.name
|
|
2485
|
+
completedCalls.set(`${stage.id}.${call.id}`, await rehydrateExecutedCall(runDir, entry, usesContainedArchitectureEvidence(definition.name)));
|
|
1962
2486
|
}
|
|
1963
2487
|
else if (entry?.state === "failed" && entry.errorMessage != null) {
|
|
1964
2488
|
failedCalls.set(`${stage.id}.${call.id}`, {
|
|
@@ -1966,6 +2490,7 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
1966
2490
|
callId: call.id,
|
|
1967
2491
|
error: entry.errorMessage,
|
|
1968
2492
|
terminal: true,
|
|
2493
|
+
...persistedFailureProvenance(entry),
|
|
1969
2494
|
});
|
|
1970
2495
|
}
|
|
1971
2496
|
}
|
|
@@ -1975,7 +2500,7 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
1975
2500
|
for (const call of stage.calls) {
|
|
1976
2501
|
const entry = persistedStage.calls.find((candidate) => candidate.callId === call.id);
|
|
1977
2502
|
if (entry?.state === "succeeded") {
|
|
1978
|
-
completedCalls.set(`${stage.id}.${call.id}`, await rehydrateExecutedCall(runDir, entry, definition.name
|
|
2503
|
+
completedCalls.set(`${stage.id}.${call.id}`, await rehydrateExecutedCall(runDir, entry, usesContainedArchitectureEvidence(definition.name)));
|
|
1979
2504
|
continue;
|
|
1980
2505
|
}
|
|
1981
2506
|
if (entry?.state === "failed" && entry.errorMessage != null) {
|
|
@@ -1987,10 +2512,11 @@ async function buildResumeState(definition, state, runDir) {
|
|
|
1987
2512
|
// intermediate attempts remain `running` with attemptsConsumed.
|
|
1988
2513
|
// Never redispatch a persisted failed call after process restart.
|
|
1989
2514
|
terminal: true,
|
|
2515
|
+
...persistedFailureProvenance(entry),
|
|
1990
2516
|
});
|
|
1991
2517
|
}
|
|
1992
2518
|
if (entry?.materializedInputPath != null) {
|
|
1993
|
-
preparedInputs.set(`${stage.id}.${call.id}`, definition.name
|
|
2519
|
+
preparedInputs.set(`${stage.id}.${call.id}`, usesContainedArchitectureEvidence(definition.name)
|
|
1994
2520
|
? JSON.parse((await readContainedRelativeFile(runDir, entry.materializedInputPath, { wantContent: true, requireRegular: true, requireExactRequestedPath: true, maxBytes: MAX_RECORD_BYTES })).content)
|
|
1995
2521
|
: await readJsonFile(join(runDir, entry.materializedInputPath)));
|
|
1996
2522
|
nextRetryCounts.set(`${stage.id}.${call.id}`, entry.attemptsConsumed ??
|
|
@@ -2281,26 +2807,28 @@ async function verifySnapshotHash(filePath, expectedHash, label) {
|
|
|
2281
2807
|
throw new Error(`Corrupted ${label}: hash mismatch for ${filePath}`);
|
|
2282
2808
|
}
|
|
2283
2809
|
}
|
|
2284
|
-
async function writeJsonSnapshot(filePath, value) {
|
|
2810
|
+
async function writeJsonSnapshot(fileWriter, filePath, value) {
|
|
2285
2811
|
const serialized = serializeJson(value);
|
|
2286
|
-
await writeTextAtomic(filePath, serialized);
|
|
2812
|
+
await writeTextAtomic(fileWriter, filePath, serialized);
|
|
2287
2813
|
return hashContent(serialized);
|
|
2288
2814
|
}
|
|
2289
|
-
async function writeRunStateAtomic(filePath, state) {
|
|
2815
|
+
async function writeRunStateAtomic(fileWriter, filePath, state) {
|
|
2290
2816
|
const { executionCompletedAt: _stale, ...rest } = state;
|
|
2291
2817
|
const witnessed = rest.executionId !== undefined && isTerminalRunStatus(rest.status) && rest.completedAt !== undefined
|
|
2292
2818
|
? { ...rest, executionCompletedAt: rest.completedAt }
|
|
2293
2819
|
: rest;
|
|
2294
|
-
await writeJsonAtomic(filePath, witnessed);
|
|
2820
|
+
await writeJsonAtomic(fileWriter, filePath, witnessed);
|
|
2295
2821
|
}
|
|
2296
|
-
async function writeJsonAtomic(filePath, value) {
|
|
2297
|
-
await writeTextAtomic(filePath, serializeJson(value));
|
|
2822
|
+
async function writeJsonAtomic(fileWriter, filePath, value) {
|
|
2823
|
+
await writeTextAtomic(fileWriter, filePath, serializeJson(value));
|
|
2298
2824
|
}
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2825
|
+
/**
|
|
2826
|
+
* Run files stay 0644 whatever the umask: the dashboard container reads them
|
|
2827
|
+
* as another user.
|
|
2828
|
+
*/
|
|
2829
|
+
async function writeTextAtomic(fileWriter, filePath, content) {
|
|
2830
|
+
await mkdirDurable(fileWriter, dirname(filePath));
|
|
2831
|
+
await fileWriter.writeFile(filePath, content, { mode: 0o644 });
|
|
2304
2832
|
}
|
|
2305
2833
|
function serializeJson(value) {
|
|
2306
2834
|
return `${JSON.stringify(value, null, 2)}\n`;
|
|
@@ -2312,7 +2840,8 @@ async function readJsonFile(filePath) {
|
|
|
2312
2840
|
return JSON.parse(await readFile(filePath, "utf-8"));
|
|
2313
2841
|
}
|
|
2314
2842
|
/**
|
|
2315
|
-
* Atomically create `run.lock`, reclaiming a same-host dead-PID lock
|
|
2843
|
+
* Atomically create `run.lock`, reclaiming a same-host dead-PID lock behind
|
|
2844
|
+
* `run.lock.steal` (see `reclaimDeadRunLock`).
|
|
2316
2845
|
*
|
|
2317
2846
|
* `declareHeartbeat` is deliberately opt-in: only a caller that goes on to hold
|
|
2318
2847
|
* the lock behind a live `WorkflowStateStore` (create/resume) beats it. The
|
|
@@ -2331,7 +2860,9 @@ async function acquireRunLock(runDir, options = {}) {
|
|
|
2331
2860
|
? { heartbeatIntervalMs: RUN_LOCK_HEARTBEAT_MS }
|
|
2332
2861
|
: {}),
|
|
2333
2862
|
};
|
|
2863
|
+
let stealBusyPath;
|
|
2334
2864
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
2865
|
+
stealBusyPath = undefined;
|
|
2335
2866
|
try {
|
|
2336
2867
|
const handle = await open(lockPath, "wx");
|
|
2337
2868
|
try {
|
|
@@ -2361,12 +2892,16 @@ async function acquireRunLock(runDir, options = {}) {
|
|
|
2361
2892
|
}
|
|
2362
2893
|
const existing = existingValue;
|
|
2363
2894
|
if (existing.hostname === hostname() && !isPidLive(existing.pid)) {
|
|
2364
|
-
await
|
|
2895
|
+
const outcome = await reclaimDeadRunLock(lockPath, existing);
|
|
2896
|
+
stealBusyPath = outcome.kind === "steal_lock_busy" ? outcome.stealPath : undefined;
|
|
2365
2897
|
continue;
|
|
2366
2898
|
}
|
|
2367
2899
|
throw new Error(`Run lock is held for ${runDir} by pid ${String(existing.pid)} on ${existing.hostname} since ${existing.startedAt}`);
|
|
2368
2900
|
}
|
|
2369
2901
|
}
|
|
2902
|
+
if (stealBusyPath !== undefined) {
|
|
2903
|
+
throw new Error(`Failed to acquire run lock for ${runDir}: its owner pid is dead but reclaim was declined because ${stealBusyPath} exists; if no other process is acquiring this run, remove ${stealBusyPath}`);
|
|
2904
|
+
}
|
|
2370
2905
|
throw new Error(`Failed to acquire run lock for ${runDir} after 2 attempts; lock ownership was not established`);
|
|
2371
2906
|
}
|
|
2372
2907
|
function isPidLive(pid) {
|