@abelxiaoxing/cadence 1.8.4 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +324 -117
- package/THIRD_PARTY_NOTICES.md +15 -0
- package/agents/design-explorer.md +14 -18
- package/agents/diagnosis-worker.md +14 -14
- package/agents/implementation-worker.md +16 -17
- package/config/workflow-task.example.json +7 -0
- package/docs/remote-access.md +60 -0
- package/docs/reuse-and-distribution.md +44 -0
- package/docs/service-management.md +108 -0
- package/licenses/bundled-web.txt +6245 -0
- package/licenses/pi-web-ui-MIT.txt +21 -0
- package/licenses/pi-web-ui-reuse.md +22 -0
- package/package.json +92 -20
- package/src/index.ts +21 -1802
- package/src/native-build.ts +6 -0
- package/src/native-client.ts +103 -0
- package/src/native-export.ts +58 -0
- package/src/native-history.ts +359 -0
- package/src/operator-cli.mjs +453 -1937
- package/src/plan-proposal.ts +87 -0
- package/src/sdk-worker.mjs +549 -0
- package/src/service/contracts/active.ts +82 -0
- package/src/service/contracts/budget.ts +148 -0
- package/src/service/contracts/build.ts +67 -0
- package/src/service/contracts/capture.ts +48 -0
- package/src/service/contracts/core.ts +33 -0
- package/src/service/contracts/current.ts +261 -0
- package/src/service/contracts/domain.ts +68 -0
- package/src/service/contracts/history.ts +2245 -0
- package/src/service/contracts/models.ts +139 -0
- package/src/service/coordination.ts +862 -0
- package/src/service/private-files.ts +204 -0
- package/src/service-cli.mjs +5592 -0
- package/src/service-client.ts +747 -0
- package/src/service-contract.ts +245 -0
- package/src/service-host.mjs +22382 -0
- package/src/service-http-client.ts +425 -0
- package/src/service-wire.ts +372 -0
- package/src/task-shell.ts +1045 -0
- package/src/task-store.ts +479 -0
- package/src/task-types.ts +184 -0
- package/src/verification-diagnostics.ts +29 -72
- package/src/worker-capture.mjs +1197 -0
- package/src/worker-public.mjs +1753 -0
- package/src/worker-redaction.mjs +608 -0
- package/src/worker-tools.mjs +202 -0
- package/web/dist/assets/KaTeX_AMS-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_AMS-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_AMS-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Caligraphic-Bold.ttf +0 -0
- package/web/dist/assets/KaTeX_Caligraphic-Bold.woff +0 -0
- package/web/dist/assets/KaTeX_Caligraphic-Bold.woff2 +0 -0
- package/web/dist/assets/KaTeX_Caligraphic-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_Caligraphic-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_Caligraphic-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Fraktur-Bold.ttf +0 -0
- package/web/dist/assets/KaTeX_Fraktur-Bold.woff +0 -0
- package/web/dist/assets/KaTeX_Fraktur-Bold.woff2 +0 -0
- package/web/dist/assets/KaTeX_Fraktur-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_Fraktur-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_Fraktur-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Main-Bold.ttf +0 -0
- package/web/dist/assets/KaTeX_Main-Bold.woff +0 -0
- package/web/dist/assets/KaTeX_Main-Bold.woff2 +0 -0
- package/web/dist/assets/KaTeX_Main-BoldItalic.ttf +0 -0
- package/web/dist/assets/KaTeX_Main-BoldItalic.woff +0 -0
- package/web/dist/assets/KaTeX_Main-BoldItalic.woff2 +0 -0
- package/web/dist/assets/KaTeX_Main-Italic.ttf +0 -0
- package/web/dist/assets/KaTeX_Main-Italic.woff +0 -0
- package/web/dist/assets/KaTeX_Main-Italic.woff2 +0 -0
- package/web/dist/assets/KaTeX_Main-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_Main-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_Main-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Math-BoldItalic.ttf +0 -0
- package/web/dist/assets/KaTeX_Math-BoldItalic.woff +0 -0
- package/web/dist/assets/KaTeX_Math-BoldItalic.woff2 +0 -0
- package/web/dist/assets/KaTeX_Math-Italic.ttf +0 -0
- package/web/dist/assets/KaTeX_Math-Italic.woff +0 -0
- package/web/dist/assets/KaTeX_Math-Italic.woff2 +0 -0
- package/web/dist/assets/KaTeX_SansSerif-Bold.ttf +0 -0
- package/web/dist/assets/KaTeX_SansSerif-Bold.woff +0 -0
- package/web/dist/assets/KaTeX_SansSerif-Bold.woff2 +0 -0
- package/web/dist/assets/KaTeX_SansSerif-Italic.ttf +0 -0
- package/web/dist/assets/KaTeX_SansSerif-Italic.woff +0 -0
- package/web/dist/assets/KaTeX_SansSerif-Italic.woff2 +0 -0
- package/web/dist/assets/KaTeX_SansSerif-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_SansSerif-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_SansSerif-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Script-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_Script-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_Script-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Size1-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_Size1-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_Size1-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Size2-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_Size2-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_Size2-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Size3-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_Size3-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_Size3-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Size4-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_Size4-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_Size4-Regular.woff2 +0 -0
- package/web/dist/assets/KaTeX_Typewriter-Regular.ttf +0 -0
- package/web/dist/assets/KaTeX_Typewriter-Regular.woff +0 -0
- package/web/dist/assets/KaTeX_Typewriter-Regular.woff2 +0 -0
- package/web/dist/assets/cadence.css +2 -0
- package/web/dist/assets/cadence.js +7 -0
- package/web/dist/assets/markdown.js +296 -0
- package/web/dist/assets/react.js +9 -0
- package/web/dist/assets/rolldown-runtime.js +1 -0
- package/web/dist/assets/timeline-client.js +41 -0
- package/web/dist/assets/vis-timeline.css +1274 -0
- package/web/dist/assets/vis-timeline.js +32 -0
- package/web/dist/index.html +18 -0
- package/web/dist/manifest.json +505 -0
- package/config/plan-draft.example.json +0 -125
- package/config/plan-draft.multiple-tasks.example.json +0 -375
- package/config/plan-draft.quick.example.json +0 -78
- package/prompts/abel-design.md +0 -382
- package/prompts/abel-diagnose.md +0 -71
- package/prompts/abel-implement.md +0 -284
- package/prompts/abel-init.md +0 -52
- package/src/activation.ts +0 -44
- package/src/activity-contracts.ts +0 -82
- package/src/agent-registry.ts +0 -53
- package/src/apply-transaction.ts +0 -1437
- package/src/artifact-store.ts +0 -213
- package/src/build-windows-job.mjs +0 -95
- package/src/candidate-artifact.ts +0 -143
- package/src/candidate-context.ts +0 -289
- package/src/candidate-patch.ts +0 -271
- package/src/candidate-workspace.ts +0 -80
- package/src/canonical.ts +0 -16
- package/src/change-contract-schema.ts +0 -191
- package/src/change-contract.ts +0 -324
- package/src/change-verification.ts +0 -1617
- package/src/contracts.ts +0 -2466
- package/src/control-contracts.ts +0 -382
- package/src/delivery-compiler.ts +0 -1805
- package/src/delivery-revision.ts +0 -42
- package/src/design-control.ts +0 -1299
- package/src/design-diagnostics.ts +0 -181
- package/src/design-journal.ts +0 -1773
- package/src/directory-sync.ts +0 -36
- package/src/durable-contracts.ts +0 -227
- package/src/durable-workflow.ts +0 -822
- package/src/evidence-draft.ts +0 -122
- package/src/execution-profile.ts +0 -64
- package/src/execution-retention.ts +0 -77
- package/src/file-snapshot.ts +0 -163
- package/src/implement-continuation.ts +0 -559
- package/src/implement-graph.ts +0 -690
- package/src/implement-plan.ts +0 -72
- package/src/install-config.d.mts +0 -6
- package/src/isolation-backend.ts +0 -353
- package/src/model-source.ts +0 -22
- package/src/openspec-cli.ts +0 -494
- package/src/operator-cli-entry.mjs +0 -24
- package/src/operator-tools.ts +0 -309
- package/src/package-candidate-process.ts +0 -206
- package/src/package-candidate.ts +0 -24
- package/src/package-delivery.ts +0 -486
- package/src/package-state.ts +0 -320
- package/src/package-verification.ts +0 -635
- package/src/package-workflow.ts +0 -305
- package/src/packet-runtime.ts +0 -459
- package/src/phase-execution.ts +0 -2163
- package/src/pi-adapter.ts +0 -19
- package/src/plan-draft-summary.ts +0 -527
- package/src/plan-draft.ts +0 -502
- package/src/remove-path.ts +0 -32
- package/src/run-state.ts +0 -420
- package/src/run-store.ts +0 -1179
- package/src/safe-path.ts +0 -77
- package/src/scoped-grep-worker.mjs +0 -23
- package/src/scoped-tools.ts +0 -775
- package/src/single-task-draft.ts +0 -142
- package/src/sqlite-schema.ts +0 -171
- package/src/state-root.ts +0 -215
- package/src/storage-schema.ts +0 -360
- package/src/subagent-activity.ts +0 -1417
- package/src/subagent-process.ts +0 -476
- package/src/task-ledger.ts +0 -1338
- package/src/transport-budget.ts +0 -50
- package/src/verification-capability.ts +0 -1191
- package/src/verification-environment-io.ts +0 -389
- package/src/verification-environment.ts +0 -162
- package/src/verification-identity.ts +0 -77
- package/src/verification-scan.ts +0 -64
- package/src/windows-job-backend.ts +0 -419
- package/src/windows-job.c +0 -149
- package/src/workflow-engine.ts +0 -28
- package/src/workflow-policy.ts +0 -1670
- package/src/workflow-recovery-policy.ts +0 -97
- package/src/workflow-scheduling.ts +0 -105
- package/src/workflow-state-machine.ts +0 -4341
- package/src/workflow-status.ts +0 -576
- package/src/workspace-io-entry.mjs +0 -76
- package/src/workspace-io-worker.mjs +0 -1312
- package/src/workspace-io.ts +0 -89
- package/src/workspace-store.ts +0 -634
|
@@ -1,4341 +0,0 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { DatabaseSync } from "node:sqlite";
|
|
4
|
-
import type { WorkflowActivityUpdate } from "./activity-contracts.ts";
|
|
5
|
-
import { permitsContextRead } from "./candidate-context.ts";
|
|
6
|
-
import {
|
|
7
|
-
assertPlanWithinChangeContract,
|
|
8
|
-
normalizeChangeContract,
|
|
9
|
-
retainedChangeContract,
|
|
10
|
-
} from "./change-contract.ts";
|
|
11
|
-
import { LIMITS, verificationInputPaths } from "./contracts.ts";
|
|
12
|
-
import {
|
|
13
|
-
assertControlCommand,
|
|
14
|
-
type ControlCommand,
|
|
15
|
-
type ControlStage,
|
|
16
|
-
} from "./control-contracts.ts";
|
|
17
|
-
import {
|
|
18
|
-
compileImplementPlan,
|
|
19
|
-
DeliveryValidationError,
|
|
20
|
-
type ImplementPlan,
|
|
21
|
-
type PlanTaskDraft,
|
|
22
|
-
} from "./delivery-compiler.ts";
|
|
23
|
-
import { compareDeliveryRevision } from "./delivery-revision.ts";
|
|
24
|
-
import { snapshotFiles } from "./file-snapshot.ts";
|
|
25
|
-
import {
|
|
26
|
-
canonicalJson,
|
|
27
|
-
type RunProjection,
|
|
28
|
-
type RunState,
|
|
29
|
-
} from "./run-state.ts";
|
|
30
|
-
import { type OperationLease, RunStore } from "./run-store.ts";
|
|
31
|
-
import { isSafeRegularFile } from "./safe-path.ts";
|
|
32
|
-
import { configureSqlite, ensureSqliteSchema } from "./sqlite-schema.ts";
|
|
33
|
-
import type { ResolvedStateRoot } from "./state-root.ts";
|
|
34
|
-
import {
|
|
35
|
-
AMENDMENT_SCHEMA,
|
|
36
|
-
ENGINE_ADDITIONS,
|
|
37
|
-
ENGINE_SCHEMA,
|
|
38
|
-
RECOVERY_ADDITIONS,
|
|
39
|
-
RECOVERY_GRANT_SCHEMA,
|
|
40
|
-
RECOVERY_SCHEMA,
|
|
41
|
-
REJECTED_DELIVERY_SCHEMA,
|
|
42
|
-
} from "./storage-schema.ts";
|
|
43
|
-
import {
|
|
44
|
-
type ActiveOperation,
|
|
45
|
-
assertDelivery,
|
|
46
|
-
type BootstrapRow,
|
|
47
|
-
baselineRecoveryKey,
|
|
48
|
-
bootstrapAcceptanceHash,
|
|
49
|
-
CHANGE_NAME,
|
|
50
|
-
cancellableRead,
|
|
51
|
-
classifyPersistedContextRequest,
|
|
52
|
-
decideRecoveryAction,
|
|
53
|
-
type EngineOperationRow,
|
|
54
|
-
type EngineRunRow,
|
|
55
|
-
type EngineTaskRow,
|
|
56
|
-
emitWorkflowActivity,
|
|
57
|
-
expectedClassification,
|
|
58
|
-
type HeldOperationLease,
|
|
59
|
-
hash,
|
|
60
|
-
IDENTIFIER,
|
|
61
|
-
isCancellationException,
|
|
62
|
-
isRecord,
|
|
63
|
-
isWorkflowApprovalCode,
|
|
64
|
-
normalizeBootstrapAcceptanceFacts,
|
|
65
|
-
normalizeVerificationPrerequisite,
|
|
66
|
-
normalizeWorkflowContextRequest,
|
|
67
|
-
normalizeWorkflowVerificationStatus,
|
|
68
|
-
parseJsonRecord,
|
|
69
|
-
parsePlan,
|
|
70
|
-
phases,
|
|
71
|
-
recoveryKey,
|
|
72
|
-
type SafeAttemptDiagnostic,
|
|
73
|
-
SHA256,
|
|
74
|
-
type WorkflowApplication,
|
|
75
|
-
type WorkflowApplicationContext,
|
|
76
|
-
type WorkflowAttemptOutcome,
|
|
77
|
-
type WorkflowAvailableDelivery,
|
|
78
|
-
type WorkflowChangeVerifier,
|
|
79
|
-
type WorkflowContextRequest,
|
|
80
|
-
type WorkflowDelivery,
|
|
81
|
-
type WorkflowDeliverySource,
|
|
82
|
-
type WorkflowEngineOptions,
|
|
83
|
-
type WorkflowRecoveryFact,
|
|
84
|
-
type WorkflowRunLifecycle,
|
|
85
|
-
type WorkflowVerificationStatus,
|
|
86
|
-
type WorkflowWorker,
|
|
87
|
-
} from "./workflow-policy.ts";
|
|
88
|
-
import {
|
|
89
|
-
assessRecoveryGrant,
|
|
90
|
-
recoveryExhausted,
|
|
91
|
-
} from "./workflow-recovery-policy.ts";
|
|
92
|
-
import { selectRunnableTasks } from "./workflow-scheduling.ts";
|
|
93
|
-
import {
|
|
94
|
-
firstPausedTask,
|
|
95
|
-
projectWorkflowStatus,
|
|
96
|
-
type WorkflowResourceBudget,
|
|
97
|
-
} from "./workflow-status.ts";
|
|
98
|
-
export class WorkflowEngine {
|
|
99
|
-
readonly #consumerRoot: string;
|
|
100
|
-
readonly #stateRoot: ResolvedStateRoot;
|
|
101
|
-
readonly #runStore: RunStore;
|
|
102
|
-
readonly #database: DatabaseSync;
|
|
103
|
-
readonly #deliverySource: WorkflowDeliverySource;
|
|
104
|
-
readonly #worker: WorkflowWorker;
|
|
105
|
-
readonly #changeVerifier: WorkflowChangeVerifier;
|
|
106
|
-
readonly #application?: WorkflowApplication;
|
|
107
|
-
readonly #lifecycle?: WorkflowRunLifecycle;
|
|
108
|
-
readonly #now: () => number;
|
|
109
|
-
readonly #leaseTtlMs: number;
|
|
110
|
-
readonly #workHardLimit: number;
|
|
111
|
-
#activeTasks = 0;
|
|
112
|
-
readonly #capacityWaiters = new Set<() => void>();
|
|
113
|
-
readonly #active = new Map<string, ActiveOperation>();
|
|
114
|
-
readonly #operationLeases = new Map<string, HeldOperationLease>();
|
|
115
|
-
#orphanRecoveryTimer?: ReturnType<typeof setTimeout>;
|
|
116
|
-
readonly #commands = new Map<
|
|
117
|
-
AbortController,
|
|
118
|
-
{ command: ControlCommand; settled: Promise<Record<string, unknown>> }
|
|
119
|
-
>();
|
|
120
|
-
#closing?: Promise<void>;
|
|
121
|
-
readonly #amendments = new Map<string, Promise<Record<string, unknown>>>();
|
|
122
|
-
#closed = false;
|
|
123
|
-
|
|
124
|
-
private constructor(options: WorkflowEngineOptions) {
|
|
125
|
-
this.#consumerRoot = path.resolve(options.consumerRoot);
|
|
126
|
-
this.#stateRoot = options.stateRoot;
|
|
127
|
-
this.#deliverySource = options.deliverySource;
|
|
128
|
-
this.#worker = options.worker;
|
|
129
|
-
this.#changeVerifier = options.changeVerifier;
|
|
130
|
-
this.#application = options.application;
|
|
131
|
-
this.#lifecycle = options.lifecycle;
|
|
132
|
-
this.#now = options.now ?? Date.now;
|
|
133
|
-
this.#leaseTtlMs = options.leaseTtlMs ?? 60_000;
|
|
134
|
-
this.#workHardLimit = options.workHardLimit ?? 512;
|
|
135
|
-
if (!Number.isSafeInteger(this.#workHardLimit) || this.#workHardLimit < 1)
|
|
136
|
-
throw new Error("workflow-work-limit-invalid");
|
|
137
|
-
this.#runStore = RunStore.open(options.stateRoot, { now: this.#now });
|
|
138
|
-
this.#database = new DatabaseSync(options.stateRoot.databasePath);
|
|
139
|
-
try {
|
|
140
|
-
configureSqlite(this.#database);
|
|
141
|
-
ensureSqliteSchema(this.#database, ENGINE_SCHEMA, ENGINE_ADDITIONS);
|
|
142
|
-
ensureSqliteSchema(this.#database, RECOVERY_SCHEMA, RECOVERY_ADDITIONS);
|
|
143
|
-
ensureSqliteSchema(this.#database, RECOVERY_GRANT_SCHEMA);
|
|
144
|
-
ensureSqliteSchema(this.#database, AMENDMENT_SCHEMA);
|
|
145
|
-
ensureSqliteSchema(this.#database, REJECTED_DELIVERY_SCHEMA);
|
|
146
|
-
} catch (error) {
|
|
147
|
-
this.#database.close();
|
|
148
|
-
this.#runStore.close();
|
|
149
|
-
throw error;
|
|
150
|
-
}
|
|
151
|
-
this.#transaction(() => {
|
|
152
|
-
const budgets = this.#database
|
|
153
|
-
.prepare(
|
|
154
|
-
"SELECT run_id FROM workflow_work_budget WHERE recovery_policy = 0",
|
|
155
|
-
)
|
|
156
|
-
.all() as { run_id: string }[];
|
|
157
|
-
for (const { run_id } of budgets) {
|
|
158
|
-
const plans = this.#database
|
|
159
|
-
.prepare(
|
|
160
|
-
"SELECT plan_json FROM workflow_engine_deliveries WHERE run_id = ?",
|
|
161
|
-
)
|
|
162
|
-
.all(run_id) as { plan_json: string }[];
|
|
163
|
-
const high = Math.max(
|
|
164
|
-
0,
|
|
165
|
-
...plans.map((row) => this.#phaseCount(JSON.parse(row.plan_json))),
|
|
166
|
-
);
|
|
167
|
-
this.#database
|
|
168
|
-
.prepare(
|
|
169
|
-
"UPDATE workflow_work_budget SET phase_high_water = ?, hard_limit = MAX(hard_limit, max_work, used), recovery_policy = 1 WHERE run_id = ?",
|
|
170
|
-
)
|
|
171
|
-
.run(high, run_id);
|
|
172
|
-
}
|
|
173
|
-
});
|
|
174
|
-
this.#recoverInterruptedOperations();
|
|
175
|
-
this.#recoverTerminalCleanup();
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
static open(options: WorkflowEngineOptions): WorkflowEngine {
|
|
179
|
-
if (
|
|
180
|
-
path.resolve(options.consumerRoot) !== options.stateRoot.consumerRoot ||
|
|
181
|
-
options.stateRoot.consumerRootHash.length !== 64
|
|
182
|
-
) {
|
|
183
|
-
throw new Error("workflow-engine-root-mismatch");
|
|
184
|
-
}
|
|
185
|
-
if (
|
|
186
|
-
!options.deliverySource ||
|
|
187
|
-
typeof options.deliverySource.load !== "function" ||
|
|
188
|
-
!options.worker ||
|
|
189
|
-
typeof options.worker.runAttempt !== "function" ||
|
|
190
|
-
typeof options.worker.rebind !== "function" ||
|
|
191
|
-
!options.changeVerifier ||
|
|
192
|
-
typeof options.changeVerifier.verify !== "function" ||
|
|
193
|
-
(options.application !== undefined &&
|
|
194
|
-
(typeof options.application.prepare === "function") !==
|
|
195
|
-
(typeof options.application.applyPrepared === "function"))
|
|
196
|
-
) {
|
|
197
|
-
throw new Error("workflow-engine-service-invalid");
|
|
198
|
-
}
|
|
199
|
-
if (
|
|
200
|
-
options.leaseTtlMs !== undefined &&
|
|
201
|
-
(!Number.isSafeInteger(options.leaseTtlMs) || options.leaseTtlMs < 1)
|
|
202
|
-
) {
|
|
203
|
-
throw new Error("workflow-engine-lease-invalid");
|
|
204
|
-
}
|
|
205
|
-
return new WorkflowEngine(options);
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
#assertOpen(): void {
|
|
209
|
-
if (this.#closed) throw new Error("workflow-engine-closed");
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
#transaction<T>(work: () => T): T {
|
|
213
|
-
this.#database.exec("BEGIN IMMEDIATE");
|
|
214
|
-
try {
|
|
215
|
-
const result = work();
|
|
216
|
-
this.#database.exec("COMMIT");
|
|
217
|
-
return result;
|
|
218
|
-
} catch (error) {
|
|
219
|
-
this.#database.exec("ROLLBACK");
|
|
220
|
-
throw error;
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
#operationKey(runId: string, operationId: string): string {
|
|
225
|
-
return `${runId}\0${operationId}`;
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
#leaseOperationId(runId: string, _operationId: string): string {
|
|
229
|
-
return `engine-run-${hash(runId).slice(0, 40)}`;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
#holdOperationLease(lease: OperationLease, commandOperationId: string): void {
|
|
233
|
-
const key = this.#operationKey(lease.runId, commandOperationId);
|
|
234
|
-
const existing = this.#operationLeases.get(key);
|
|
235
|
-
if (existing) clearInterval(existing.timer);
|
|
236
|
-
const intervalMs = Math.max(1, Math.floor(this.#leaseTtlMs / 3));
|
|
237
|
-
const held: HeldOperationLease = {
|
|
238
|
-
lease,
|
|
239
|
-
timer: setInterval(() => {
|
|
240
|
-
if (this.#operationLeases.get(key) !== held) return;
|
|
241
|
-
try {
|
|
242
|
-
held.lease = this.#runStore.renewLease(held.lease, this.#leaseTtlMs);
|
|
243
|
-
this.#database
|
|
244
|
-
.prepare(
|
|
245
|
-
`UPDATE workflow_engine_operations
|
|
246
|
-
SET lease_expires_at = ?
|
|
247
|
-
WHERE run_id = ? AND operation_id = ? AND state = 'running'
|
|
248
|
-
AND lease_token = ?`,
|
|
249
|
-
)
|
|
250
|
-
.run(
|
|
251
|
-
held.lease.expiresAt,
|
|
252
|
-
lease.runId,
|
|
253
|
-
commandOperationId,
|
|
254
|
-
lease.token,
|
|
255
|
-
);
|
|
256
|
-
} catch {
|
|
257
|
-
held.error = new Error("lease-fenced");
|
|
258
|
-
clearInterval(held.timer);
|
|
259
|
-
const active = this.#active.get(lease.runId);
|
|
260
|
-
if (active?.operationId === commandOperationId) {
|
|
261
|
-
active.controller.abort(held.error);
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
}, intervalMs),
|
|
265
|
-
};
|
|
266
|
-
held.timer.unref?.();
|
|
267
|
-
this.#operationLeases.set(key, held);
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
#operationLease(runId: string, operationId: string): OperationLease {
|
|
271
|
-
const held = this.#operationLeases.get(
|
|
272
|
-
this.#operationKey(runId, operationId),
|
|
273
|
-
);
|
|
274
|
-
if (!held || held.error) throw held?.error ?? new Error("lease-fenced");
|
|
275
|
-
this.#runStore.assertLease(held.lease);
|
|
276
|
-
return held.lease;
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
#releaseOperationLease(runId: string, operationId: string): void {
|
|
280
|
-
const key = this.#operationKey(runId, operationId);
|
|
281
|
-
const held = this.#operationLeases.get(key);
|
|
282
|
-
if (!held) return;
|
|
283
|
-
clearInterval(held.timer);
|
|
284
|
-
this.#operationLeases.delete(key);
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
#assertLeaseInTransaction(lease: OperationLease): void {
|
|
288
|
-
const row = this.#database
|
|
289
|
-
.prepare(
|
|
290
|
-
`SELECT state, lease_token, lease_expires_at
|
|
291
|
-
FROM operations WHERE run_id = ? AND operation_id = ?`,
|
|
292
|
-
)
|
|
293
|
-
.get(lease.runId, lease.operationId) as
|
|
294
|
-
| {
|
|
295
|
-
state: string;
|
|
296
|
-
lease_token: string | null;
|
|
297
|
-
lease_expires_at: number | null;
|
|
298
|
-
}
|
|
299
|
-
| undefined;
|
|
300
|
-
if (
|
|
301
|
-
row?.state !== "running" ||
|
|
302
|
-
row.lease_token !== lease.token ||
|
|
303
|
-
row.lease_expires_at === null ||
|
|
304
|
-
row.lease_expires_at < this.#now()
|
|
305
|
-
) {
|
|
306
|
-
throw new Error("lease-fenced");
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
#leasedTransaction<T>(lease: OperationLease, work: () => T): T {
|
|
311
|
-
return this.#transaction(() => {
|
|
312
|
-
this.#assertLeaseInTransaction(lease);
|
|
313
|
-
return work();
|
|
314
|
-
});
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
amend(
|
|
318
|
-
change: string,
|
|
319
|
-
batchId: string,
|
|
320
|
-
request: unknown,
|
|
321
|
-
execute: (assertAuthority: () => void) => Promise<Record<string, unknown>>,
|
|
322
|
-
): Promise<Record<string, unknown>> {
|
|
323
|
-
this.#assertOpen();
|
|
324
|
-
if (this.#closing) throw new Error("workflow-engine-closed");
|
|
325
|
-
const runId = this.#lookupRun("abel-implement", change);
|
|
326
|
-
if (!runId) throw new Error("run-not-found");
|
|
327
|
-
if (this.#amendments.has(runId)) throw new Error("amendment-busy");
|
|
328
|
-
const operationId = `amend-${hash(batchId, canonicalJson(request)).slice(0, 48)}`;
|
|
329
|
-
const pending = Promise.resolve()
|
|
330
|
-
.then(async () => {
|
|
331
|
-
const replay = this.#beginOperation(runId, operationId, "amend");
|
|
332
|
-
if (replay) return replay;
|
|
333
|
-
const lease = this.#operationLease(runId, operationId);
|
|
334
|
-
const assertAuthority = () => {
|
|
335
|
-
this.#assertOpen();
|
|
336
|
-
if (this.#closing) throw new Error("workflow-engine-closed");
|
|
337
|
-
this.#runStore.assertLease(lease);
|
|
338
|
-
const status = this.#statusByRun(runId);
|
|
339
|
-
if (
|
|
340
|
-
!["approval-needed", "paused"].includes(String(status.state)) ||
|
|
341
|
-
(status.decisionBatch as { id?: string } | undefined)?.id !==
|
|
342
|
-
batchId
|
|
343
|
-
)
|
|
344
|
-
throw new Error("amendment-batch-stale");
|
|
345
|
-
};
|
|
346
|
-
try {
|
|
347
|
-
assertAuthority();
|
|
348
|
-
if (
|
|
349
|
-
!isRecord(request) ||
|
|
350
|
-
!["status", "validate-plan-draft"].includes(
|
|
351
|
-
String(request.operation),
|
|
352
|
-
)
|
|
353
|
-
) {
|
|
354
|
-
this.#leasedTransaction(lease, () => {
|
|
355
|
-
this.#database
|
|
356
|
-
.prepare(
|
|
357
|
-
"INSERT INTO workflow_amendment_budget(run_id, used) VALUES (?, 0) ON CONFLICT DO NOTHING",
|
|
358
|
-
)
|
|
359
|
-
.run(runId);
|
|
360
|
-
const reserved = this.#database
|
|
361
|
-
.prepare(
|
|
362
|
-
"UPDATE workflow_amendment_budget SET used = used + 1 WHERE run_id = ? AND used < 64",
|
|
363
|
-
)
|
|
364
|
-
.run(runId);
|
|
365
|
-
if (reserved.changes !== 1)
|
|
366
|
-
throw new Error("amendment-budget-exhausted");
|
|
367
|
-
});
|
|
368
|
-
}
|
|
369
|
-
this.#assertAmendmentContract(runId, change, request);
|
|
370
|
-
const outcome = await execute(assertAuthority);
|
|
371
|
-
assertAuthority();
|
|
372
|
-
return this.#commitOperation(runId, operationId, outcome);
|
|
373
|
-
} catch (error) {
|
|
374
|
-
this.#interruptOperation(runId, operationId);
|
|
375
|
-
throw error;
|
|
376
|
-
}
|
|
377
|
-
})
|
|
378
|
-
.finally(() => {
|
|
379
|
-
if (this.#amendments.get(runId) === pending)
|
|
380
|
-
this.#amendments.delete(runId);
|
|
381
|
-
});
|
|
382
|
-
this.#amendments.set(runId, pending);
|
|
383
|
-
return pending;
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
#assertAmendmentContract(
|
|
387
|
-
runId: string,
|
|
388
|
-
change: string,
|
|
389
|
-
request: unknown,
|
|
390
|
-
): void {
|
|
391
|
-
if (!isRecord(request)) throw new Error("amendment-contract-invalid");
|
|
392
|
-
if (
|
|
393
|
-
(request.operation === "approve-gate" && request.gate === "gate-a") ||
|
|
394
|
-
(request.operation === "record-decision" &&
|
|
395
|
-
request.category === "behavior")
|
|
396
|
-
)
|
|
397
|
-
throw new Error("amendment-changes-accepted-behavior");
|
|
398
|
-
if (
|
|
399
|
-
![
|
|
400
|
-
"write-artifact",
|
|
401
|
-
"delete-artifact",
|
|
402
|
-
"compile-plan",
|
|
403
|
-
"validate-plan-draft",
|
|
404
|
-
"finalize-delivery",
|
|
405
|
-
].includes(String(request.operation))
|
|
406
|
-
)
|
|
407
|
-
return;
|
|
408
|
-
const relative = String(request.path);
|
|
409
|
-
if (relative === "proposal.md" || relative.startsWith("specs/")) {
|
|
410
|
-
const file = `openspec/changes/${change}/${relative}`;
|
|
411
|
-
if (
|
|
412
|
-
request.operation !== "write-artifact" ||
|
|
413
|
-
!isSafeRegularFile(this.#consumerRoot, file) ||
|
|
414
|
-
readFileSync(path.join(this.#consumerRoot, file), "utf8") !==
|
|
415
|
-
request.content
|
|
416
|
-
)
|
|
417
|
-
throw new Error("amendment-changes-accepted-behavior");
|
|
418
|
-
}
|
|
419
|
-
const compile = [
|
|
420
|
-
"compile-plan",
|
|
421
|
-
"validate-plan-draft",
|
|
422
|
-
"finalize-delivery",
|
|
423
|
-
].includes(String(request.operation));
|
|
424
|
-
if (!compile) return;
|
|
425
|
-
const draftPath = `openspec/changes/${change}/plan-draft.json`;
|
|
426
|
-
if (!isSafeRegularFile(this.#consumerRoot, draftPath)) return;
|
|
427
|
-
const run = this.#engineRun(runId);
|
|
428
|
-
if (run.current_revision === null) return;
|
|
429
|
-
const prior = this.#planForRevision(runId, run.current_revision);
|
|
430
|
-
const contract = retainedChangeContract(prior);
|
|
431
|
-
const draft = JSON.parse(
|
|
432
|
-
readFileSync(path.join(this.#consumerRoot, draftPath), "utf8"),
|
|
433
|
-
);
|
|
434
|
-
if (
|
|
435
|
-
draft.changeContract !== undefined &&
|
|
436
|
-
canonicalJson(normalizeChangeContract(draft.changeContract)) !==
|
|
437
|
-
canonicalJson(contract)
|
|
438
|
-
)
|
|
439
|
-
throw new Error("change-contract-mismatch");
|
|
440
|
-
// Match DesignController's inheritance before validating authorized modes.
|
|
441
|
-
if (draft.changeContract === undefined && prior.changeContract)
|
|
442
|
-
draft.changeContract = contract;
|
|
443
|
-
const next = compileImplementPlan(draft, {
|
|
444
|
-
consumerRoot: this.#consumerRoot,
|
|
445
|
-
bindExecutionInputs: true,
|
|
446
|
-
}).plan;
|
|
447
|
-
assertPlanWithinChangeContract(contract, next.tasks, [
|
|
448
|
-
next.verification.change.fullSuite,
|
|
449
|
-
next.verification.change.postApply,
|
|
450
|
-
]);
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
#lookupRun(stage: ControlStage, change: string): string | undefined {
|
|
454
|
-
return (
|
|
455
|
-
this.#database
|
|
456
|
-
.prepare(
|
|
457
|
-
`SELECT run_id FROM runs
|
|
458
|
-
WHERE root_hash = ? AND stage = ? AND lookup_key = ?`,
|
|
459
|
-
)
|
|
460
|
-
.get(this.#stateRoot.consumerRootHash, stage, `change:${change}`) as
|
|
461
|
-
| { run_id: string }
|
|
462
|
-
| undefined
|
|
463
|
-
)?.run_id;
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
#ensureEngineRun(runId: string): void {
|
|
467
|
-
this.#database
|
|
468
|
-
.prepare(
|
|
469
|
-
`INSERT INTO workflow_engine_runs(
|
|
470
|
-
run_id, current_revision, baseline_workspace_revision,
|
|
471
|
-
current_workspace_revision, cleanup_state, next_queue_position
|
|
472
|
-
) VALUES (?, NULL, NULL, NULL, 'none', 1)
|
|
473
|
-
ON CONFLICT(run_id) DO NOTHING`,
|
|
474
|
-
)
|
|
475
|
-
.run(runId);
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
#engineRun(runId: string): EngineRunRow {
|
|
479
|
-
const row = this.#database
|
|
480
|
-
.prepare(
|
|
481
|
-
`SELECT current_revision, baseline_workspace_revision,
|
|
482
|
-
current_workspace_revision, cleanup_state, route_id,
|
|
483
|
-
route_fingerprint,
|
|
484
|
-
transaction_id, verification_json, delivery_diagnostics_json,
|
|
485
|
-
next_queue_position
|
|
486
|
-
FROM workflow_engine_runs WHERE run_id = ?`,
|
|
487
|
-
)
|
|
488
|
-
.get(runId) as EngineRunRow | undefined;
|
|
489
|
-
if (!row) throw new Error("workflow-engine-run-unavailable");
|
|
490
|
-
return row;
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
#operation(
|
|
494
|
-
runId: string,
|
|
495
|
-
operationId: string,
|
|
496
|
-
): EngineOperationRow | undefined {
|
|
497
|
-
return this.#database
|
|
498
|
-
.prepare(
|
|
499
|
-
`SELECT command, state, outcome_json FROM workflow_engine_operations
|
|
500
|
-
WHERE run_id = ? AND operation_id = ?`,
|
|
501
|
-
)
|
|
502
|
-
.get(runId, operationId) as EngineOperationRow | undefined;
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
#operationReplay(
|
|
506
|
-
runId: string,
|
|
507
|
-
operationId: string,
|
|
508
|
-
command: string,
|
|
509
|
-
): Record<string, unknown> | undefined {
|
|
510
|
-
const row = this.#operation(runId, operationId);
|
|
511
|
-
if (row && row.command !== command) {
|
|
512
|
-
throw new Error("operation-command-mismatch");
|
|
513
|
-
}
|
|
514
|
-
return row?.state === "committed" && row.outcome_json
|
|
515
|
-
? parseJsonRecord(row.outcome_json)
|
|
516
|
-
: undefined;
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
#beginOperation(
|
|
520
|
-
runId: string,
|
|
521
|
-
operationId: string,
|
|
522
|
-
command: string,
|
|
523
|
-
): Record<string, unknown> | undefined {
|
|
524
|
-
if (!IDENTIFIER.test(operationId)) throw new Error("invalid-operation-id");
|
|
525
|
-
const replay = this.#operationReplay(runId, operationId, command);
|
|
526
|
-
if (replay) return replay;
|
|
527
|
-
const existing = this.#operation(runId, operationId);
|
|
528
|
-
if (existing?.state === "running") {
|
|
529
|
-
const leaseStatus = this.#runStore.inspectOperation(
|
|
530
|
-
runId,
|
|
531
|
-
this.#leaseOperationId(runId, operationId),
|
|
532
|
-
);
|
|
533
|
-
if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
|
|
534
|
-
this.#database
|
|
535
|
-
.prepare(
|
|
536
|
-
`UPDATE workflow_engine_operations
|
|
537
|
-
SET state = 'committed', lease_token = NULL,
|
|
538
|
-
lease_expires_at = NULL, outcome_json = ?
|
|
539
|
-
WHERE run_id = ? AND operation_id = ?`,
|
|
540
|
-
)
|
|
541
|
-
.run(JSON.stringify(leaseStatus.outcome), runId, operationId);
|
|
542
|
-
return structuredClone(leaseStatus.outcome);
|
|
543
|
-
}
|
|
544
|
-
if (leaseStatus?.state === "running") {
|
|
545
|
-
throw new Error("operation-already-running");
|
|
546
|
-
}
|
|
547
|
-
this.#database
|
|
548
|
-
.prepare(
|
|
549
|
-
`UPDATE workflow_engine_operations
|
|
550
|
-
SET state = 'interrupted', lease_token = NULL,
|
|
551
|
-
lease_expires_at = NULL
|
|
552
|
-
WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
|
|
553
|
-
)
|
|
554
|
-
.run(runId, operationId);
|
|
555
|
-
}
|
|
556
|
-
const leaseOperationId = this.#leaseOperationId(runId, operationId);
|
|
557
|
-
const priorLease = this.#runStore.inspectOperation(runId, leaseOperationId);
|
|
558
|
-
const preempt = command === "cancel" || command === "discard";
|
|
559
|
-
if (priorLease?.state === "running" && !preempt) {
|
|
560
|
-
throw new Error("operation-already-running");
|
|
561
|
-
}
|
|
562
|
-
const lease = this.#runStore.acquireLease({
|
|
563
|
-
runId,
|
|
564
|
-
operationId: leaseOperationId,
|
|
565
|
-
ttlMs: this.#leaseTtlMs,
|
|
566
|
-
exclusive: true,
|
|
567
|
-
...(preempt ? { preempt: true } : {}),
|
|
568
|
-
});
|
|
569
|
-
if (preempt) {
|
|
570
|
-
this.#database
|
|
571
|
-
.prepare(
|
|
572
|
-
`UPDATE workflow_engine_operations
|
|
573
|
-
SET state = 'interrupted', lease_token = NULL,
|
|
574
|
-
lease_expires_at = NULL
|
|
575
|
-
WHERE run_id = ? AND operation_id <> ? AND state = 'running'`,
|
|
576
|
-
)
|
|
577
|
-
.run(runId, operationId);
|
|
578
|
-
}
|
|
579
|
-
this.#database
|
|
580
|
-
.prepare(
|
|
581
|
-
`INSERT INTO workflow_engine_operations(
|
|
582
|
-
run_id, operation_id, command, state, lease_token,
|
|
583
|
-
lease_expires_at, outcome_json
|
|
584
|
-
) VALUES (?, ?, ?, 'running', ?, ?, NULL)
|
|
585
|
-
ON CONFLICT(run_id, operation_id) DO UPDATE SET
|
|
586
|
-
command = excluded.command, state = 'running',
|
|
587
|
-
lease_token = excluded.lease_token,
|
|
588
|
-
lease_expires_at = excluded.lease_expires_at,
|
|
589
|
-
outcome_json = NULL`,
|
|
590
|
-
)
|
|
591
|
-
.run(runId, operationId, command, lease.token, lease.expiresAt);
|
|
592
|
-
this.#holdOperationLease(lease, operationId);
|
|
593
|
-
return undefined;
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
#commitOperation(
|
|
597
|
-
runId: string,
|
|
598
|
-
operationId: string,
|
|
599
|
-
outcome: Record<string, unknown>,
|
|
600
|
-
): Record<string, unknown> {
|
|
601
|
-
const lease = this.#operationLease(runId, operationId);
|
|
602
|
-
try {
|
|
603
|
-
const committed = this.#runStore.commitLease(lease, outcome);
|
|
604
|
-
const result = this.#database
|
|
605
|
-
.prepare(
|
|
606
|
-
`UPDATE workflow_engine_operations
|
|
607
|
-
SET state = 'committed', lease_token = NULL,
|
|
608
|
-
lease_expires_at = NULL, outcome_json = ?
|
|
609
|
-
WHERE run_id = ? AND operation_id = ? AND state = 'running'
|
|
610
|
-
AND lease_token = ?`,
|
|
611
|
-
)
|
|
612
|
-
.run(JSON.stringify(committed), runId, operationId, lease.token);
|
|
613
|
-
if (Number(result.changes) !== 1) {
|
|
614
|
-
throw new Error("operation-journal-fenced");
|
|
615
|
-
}
|
|
616
|
-
return structuredClone(committed);
|
|
617
|
-
} finally {
|
|
618
|
-
this.#releaseOperationLease(runId, operationId);
|
|
619
|
-
}
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
#interruptOperation(runId: string, operationId: string): void {
|
|
623
|
-
let lease: OperationLease;
|
|
624
|
-
try {
|
|
625
|
-
lease = this.#operationLease(runId, operationId);
|
|
626
|
-
if (this.#operation(runId, operationId)?.command !== "amend")
|
|
627
|
-
this.#pauseInterruptedRun(runId, operationId, lease);
|
|
628
|
-
this.#runStore.interruptLease(lease);
|
|
629
|
-
this.#database
|
|
630
|
-
.prepare(
|
|
631
|
-
`UPDATE workflow_engine_operations
|
|
632
|
-
SET state = 'interrupted', lease_token = NULL,
|
|
633
|
-
lease_expires_at = NULL
|
|
634
|
-
WHERE run_id = ? AND operation_id = ? AND state = 'running'
|
|
635
|
-
AND lease_token = ?`,
|
|
636
|
-
)
|
|
637
|
-
.run(runId, operationId, lease.token);
|
|
638
|
-
} catch (error) {
|
|
639
|
-
if (!(error instanceof Error) || error.message !== "lease-fenced") {
|
|
640
|
-
throw error;
|
|
641
|
-
}
|
|
642
|
-
} finally {
|
|
643
|
-
this.#releaseOperationLease(runId, operationId);
|
|
644
|
-
}
|
|
645
|
-
}
|
|
646
|
-
|
|
647
|
-
#transition(
|
|
648
|
-
runId: string,
|
|
649
|
-
to: RunState,
|
|
650
|
-
identity: string,
|
|
651
|
-
code?: string,
|
|
652
|
-
lease?: OperationLease,
|
|
653
|
-
): RunProjection {
|
|
654
|
-
if (lease) this.#runStore.assertLease(lease);
|
|
655
|
-
const current = this.#runStore.status(runId);
|
|
656
|
-
if (
|
|
657
|
-
current.state === to &&
|
|
658
|
-
(to !== "paused" || code === undefined || current.pauseCode === code)
|
|
659
|
-
) {
|
|
660
|
-
return current;
|
|
661
|
-
}
|
|
662
|
-
return this.#runStore.transition({
|
|
663
|
-
runId,
|
|
664
|
-
to,
|
|
665
|
-
operationId: `eng-${hash(runId, identity, String(current.sequence), to).slice(0, 40)}`,
|
|
666
|
-
...(code ? { code } : {}),
|
|
667
|
-
...(lease ? { lease } : {}),
|
|
668
|
-
});
|
|
669
|
-
}
|
|
670
|
-
|
|
671
|
-
#pauseInterruptedRun(
|
|
672
|
-
runId: string,
|
|
673
|
-
operationId: string,
|
|
674
|
-
lease: OperationLease,
|
|
675
|
-
): void {
|
|
676
|
-
const projection = this.#runStore.status(runId);
|
|
677
|
-
if (["completed", "discarded", "rejected"].includes(projection.state))
|
|
678
|
-
return;
|
|
679
|
-
const write = (database: DatabaseSync) => {
|
|
680
|
-
database
|
|
681
|
-
.prepare(`UPDATE workflow_engine_tasks
|
|
682
|
-
SET state = 'paused',
|
|
683
|
-
pause_code = CASE WHEN pause_code = 'red-artifact-constraint'
|
|
684
|
-
THEN pause_code ELSE 'operation-interrupted' END,
|
|
685
|
-
context_request_json = CASE WHEN pause_code = 'red-artifact-constraint'
|
|
686
|
-
THEN context_request_json ELSE NULL END,
|
|
687
|
-
queue_position = NULL
|
|
688
|
-
WHERE run_id = ? AND state IN ('phase-running', 'validating')`)
|
|
689
|
-
.run(runId);
|
|
690
|
-
};
|
|
691
|
-
const active = [
|
|
692
|
-
"created",
|
|
693
|
-
"validating-delivery",
|
|
694
|
-
"ready",
|
|
695
|
-
"queued",
|
|
696
|
-
"connecting",
|
|
697
|
-
"running",
|
|
698
|
-
"validating",
|
|
699
|
-
"verifying",
|
|
700
|
-
"retryable",
|
|
701
|
-
"approval-needed",
|
|
702
|
-
"change-verifying",
|
|
703
|
-
"ready-to-apply",
|
|
704
|
-
];
|
|
705
|
-
const to =
|
|
706
|
-
projection.state === "applying"
|
|
707
|
-
? "recovering"
|
|
708
|
-
: active.includes(projection.state)
|
|
709
|
-
? "paused"
|
|
710
|
-
: undefined;
|
|
711
|
-
if (to) {
|
|
712
|
-
// Keep task and run projections atomic; retained evidence and budgets are
|
|
713
|
-
// deliberately untouched. The original exception still reaches the caller.
|
|
714
|
-
this.#runStore.transitionAtomically(
|
|
715
|
-
{
|
|
716
|
-
runId,
|
|
717
|
-
to,
|
|
718
|
-
lease,
|
|
719
|
-
code: "operation-interrupted",
|
|
720
|
-
operationId: `eng-${hash(runId, operationId, String(projection.sequence), "interrupted").slice(0, 40)}`,
|
|
721
|
-
},
|
|
722
|
-
write,
|
|
723
|
-
);
|
|
724
|
-
} else {
|
|
725
|
-
this.#leasedTransaction(lease, () => write(this.#database));
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
#recoverInterruptedOperations(): void {
|
|
730
|
-
if (this.#orphanRecoveryTimer) {
|
|
731
|
-
clearTimeout(this.#orphanRecoveryTimer);
|
|
732
|
-
this.#orphanRecoveryTimer = undefined;
|
|
733
|
-
}
|
|
734
|
-
let nextExpiry: number | undefined;
|
|
735
|
-
const interrupted = this.#database
|
|
736
|
-
.prepare(
|
|
737
|
-
`SELECT run_id, operation_id FROM workflow_engine_operations
|
|
738
|
-
WHERE state = 'running' OR (state = 'interrupted' AND command <> 'amend'
|
|
739
|
-
AND (run_id IN (SELECT run_id FROM workflow_engine_tasks
|
|
740
|
-
WHERE state IN ('phase-running', 'validating'))
|
|
741
|
-
OR run_id IN (SELECT run_id FROM runs WHERE state IN
|
|
742
|
-
('running', 'validating', 'verifying', 'change-verifying', 'applying'))))`,
|
|
743
|
-
)
|
|
744
|
-
.all() as unknown as Array<{ run_id: string; operation_id: string }>;
|
|
745
|
-
for (const row of interrupted) {
|
|
746
|
-
const leaseStatus = this.#runStore.inspectOperation(
|
|
747
|
-
row.run_id,
|
|
748
|
-
this.#leaseOperationId(row.run_id, row.operation_id),
|
|
749
|
-
);
|
|
750
|
-
if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
|
|
751
|
-
this.#database
|
|
752
|
-
.prepare(
|
|
753
|
-
`UPDATE workflow_engine_operations
|
|
754
|
-
SET state = 'committed', lease_token = NULL,
|
|
755
|
-
lease_expires_at = NULL, outcome_json = ?
|
|
756
|
-
WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
|
|
757
|
-
)
|
|
758
|
-
.run(
|
|
759
|
-
JSON.stringify(leaseStatus.outcome),
|
|
760
|
-
row.run_id,
|
|
761
|
-
row.operation_id,
|
|
762
|
-
);
|
|
763
|
-
continue;
|
|
764
|
-
}
|
|
765
|
-
if (leaseStatus?.state === "running") {
|
|
766
|
-
if (leaseStatus.expiresAt !== undefined) {
|
|
767
|
-
nextExpiry = Math.min(nextExpiry ?? Infinity, leaseStatus.expiresAt);
|
|
768
|
-
}
|
|
769
|
-
continue;
|
|
770
|
-
}
|
|
771
|
-
this.#database
|
|
772
|
-
.prepare(
|
|
773
|
-
`UPDATE workflow_engine_operations
|
|
774
|
-
SET state = 'interrupted', lease_token = NULL,
|
|
775
|
-
lease_expires_at = NULL
|
|
776
|
-
WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
|
|
777
|
-
)
|
|
778
|
-
.run(row.run_id, row.operation_id);
|
|
779
|
-
// Claim the same run-exclusive lease before repairing projections. A live
|
|
780
|
-
// owner (including a different host) must never be mistaken for an orphan.
|
|
781
|
-
let recoveryLease: OperationLease;
|
|
782
|
-
try {
|
|
783
|
-
recoveryLease = this.#runStore.acquireLease({
|
|
784
|
-
runId: row.run_id,
|
|
785
|
-
operationId: this.#leaseOperationId(row.run_id, row.operation_id),
|
|
786
|
-
ttlMs: this.#leaseTtlMs,
|
|
787
|
-
exclusive: true,
|
|
788
|
-
});
|
|
789
|
-
} catch (error) {
|
|
790
|
-
if (
|
|
791
|
-
error instanceof Error &&
|
|
792
|
-
error.message === "operation-already-running"
|
|
793
|
-
)
|
|
794
|
-
continue;
|
|
795
|
-
throw error;
|
|
796
|
-
}
|
|
797
|
-
try {
|
|
798
|
-
this.#pauseInterruptedRun(row.run_id, row.operation_id, recoveryLease);
|
|
799
|
-
} finally {
|
|
800
|
-
this.#runStore.interruptLease(recoveryLease);
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
if (nextExpiry !== undefined && !this.#closed) {
|
|
804
|
-
const delay = Math.max(1, nextExpiry - this.#now() + 1);
|
|
805
|
-
this.#orphanRecoveryTimer = setTimeout(() => {
|
|
806
|
-
this.#orphanRecoveryTimer = undefined;
|
|
807
|
-
if (this.#closed) return;
|
|
808
|
-
try {
|
|
809
|
-
this.#recoverInterruptedOperations();
|
|
810
|
-
} catch {
|
|
811
|
-
// A later status/restart will surface durable journal corruption.
|
|
812
|
-
}
|
|
813
|
-
}, delay);
|
|
814
|
-
this.#orphanRecoveryTimer.unref?.();
|
|
815
|
-
}
|
|
816
|
-
}
|
|
817
|
-
|
|
818
|
-
#recoverTerminalCleanup(): void {
|
|
819
|
-
const rows = this.#database
|
|
820
|
-
.prepare(
|
|
821
|
-
`SELECT workflow_engine_runs.run_id
|
|
822
|
-
FROM workflow_engine_runs
|
|
823
|
-
JOIN runs ON runs.run_id = workflow_engine_runs.run_id
|
|
824
|
-
WHERE runs.state IN ('completed', 'discarded')
|
|
825
|
-
AND workflow_engine_runs.cleanup_state = 'retained'`,
|
|
826
|
-
)
|
|
827
|
-
.all() as unknown as Array<{ run_id: string }>;
|
|
828
|
-
for (const row of rows) {
|
|
829
|
-
try {
|
|
830
|
-
this.#cleanupTerminal(row.run_id, "restart-cleanup");
|
|
831
|
-
} catch {
|
|
832
|
-
// Terminal status remains truthful and a later reload retries cleanup.
|
|
833
|
-
}
|
|
834
|
-
}
|
|
835
|
-
}
|
|
836
|
-
|
|
837
|
-
#tasks(runId: string): EngineTaskRow[] {
|
|
838
|
-
return this.#database
|
|
839
|
-
.prepare(
|
|
840
|
-
`SELECT task_id, task_order, delivery_revision, plan_json, state,
|
|
841
|
-
phase, pause_code, context_request_json, attempt_diagnostic_json,
|
|
842
|
-
route_id, route_fingerprint, queue_position
|
|
843
|
-
FROM workflow_engine_tasks WHERE run_id = ? ORDER BY task_order`,
|
|
844
|
-
)
|
|
845
|
-
.all(runId) as unknown as EngineTaskRow[];
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
#taskPlan(row: EngineTaskRow): PlanTaskDraft {
|
|
849
|
-
return JSON.parse(row.plan_json) as PlanTaskDraft;
|
|
850
|
-
}
|
|
851
|
-
|
|
852
|
-
#currentPlan(runId: string): ImplementPlan {
|
|
853
|
-
const revision = this.#engineRun(runId).current_revision;
|
|
854
|
-
if (revision === null) throw new Error("delivery-not-bound");
|
|
855
|
-
return this.#planForRevision(runId, revision);
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
#planForRevision(runId: string, revision: number): ImplementPlan {
|
|
859
|
-
const row = this.#database
|
|
860
|
-
.prepare(
|
|
861
|
-
`SELECT plan_json FROM workflow_engine_deliveries
|
|
862
|
-
WHERE run_id = ? AND revision = ?`,
|
|
863
|
-
)
|
|
864
|
-
.get(runId, revision) as { plan_json: string } | undefined;
|
|
865
|
-
if (!row) throw new Error("delivery-not-bound");
|
|
866
|
-
return parsePlan(row.plan_json);
|
|
867
|
-
}
|
|
868
|
-
|
|
869
|
-
async #loadDelivery(
|
|
870
|
-
runId: string,
|
|
871
|
-
lease: OperationLease,
|
|
872
|
-
stage: ControlStage,
|
|
873
|
-
change: string,
|
|
874
|
-
deliveryRevision?: number,
|
|
875
|
-
receiptHash?: string,
|
|
876
|
-
signal?: AbortSignal,
|
|
877
|
-
): Promise<WorkflowDelivery> {
|
|
878
|
-
if (deliveryRevision === undefined && this.#deliverySource.discoverLatest) {
|
|
879
|
-
try {
|
|
880
|
-
const discover = this.#deliverySource.discoverLatest;
|
|
881
|
-
const available = await cancellableRead(
|
|
882
|
-
() => discover.call(this.#deliverySource, { stage, change, signal }),
|
|
883
|
-
signal,
|
|
884
|
-
);
|
|
885
|
-
if (
|
|
886
|
-
available &&
|
|
887
|
-
Number.isSafeInteger(available.deliveryRevision) &&
|
|
888
|
-
available.deliveryRevision > 0 &&
|
|
889
|
-
SHA256.test(available.receiptHash)
|
|
890
|
-
) {
|
|
891
|
-
deliveryRevision = available.deliveryRevision;
|
|
892
|
-
receiptHash = available.receiptHash;
|
|
893
|
-
}
|
|
894
|
-
} catch {
|
|
895
|
-
// Discovery is optional; authoritative loading still reports diagnostics.
|
|
896
|
-
signal?.throwIfAborted();
|
|
897
|
-
}
|
|
898
|
-
}
|
|
899
|
-
try {
|
|
900
|
-
const delivery = await cancellableRead(
|
|
901
|
-
() =>
|
|
902
|
-
this.#deliverySource.load({
|
|
903
|
-
stage,
|
|
904
|
-
change,
|
|
905
|
-
...(deliveryRevision === undefined ? {} : { deliveryRevision }),
|
|
906
|
-
...(receiptHash === undefined ? {} : { receiptHash }),
|
|
907
|
-
...(signal ? { signal } : {}),
|
|
908
|
-
}),
|
|
909
|
-
signal,
|
|
910
|
-
);
|
|
911
|
-
signal?.throwIfAborted();
|
|
912
|
-
assertDelivery(delivery, stage, change, deliveryRevision, receiptHash);
|
|
913
|
-
return structuredClone(delivery);
|
|
914
|
-
} catch (error) {
|
|
915
|
-
if (
|
|
916
|
-
error instanceof DeliveryValidationError &&
|
|
917
|
-
deliveryRevision !== undefined &&
|
|
918
|
-
receiptHash !== undefined
|
|
919
|
-
) {
|
|
920
|
-
this.#leasedTransaction(lease, () => {
|
|
921
|
-
this.#database
|
|
922
|
-
.prepare(
|
|
923
|
-
"INSERT INTO workflow_rejected_deliveries(run_id, delivery_revision, receipt_hash) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
|
|
924
|
-
)
|
|
925
|
-
.run(runId, deliveryRevision, receiptHash);
|
|
926
|
-
});
|
|
927
|
-
}
|
|
928
|
-
throw error;
|
|
929
|
-
}
|
|
930
|
-
}
|
|
931
|
-
|
|
932
|
-
#admitDelivery(
|
|
933
|
-
runId: string,
|
|
934
|
-
delivery: WorkflowDelivery,
|
|
935
|
-
lease: OperationLease,
|
|
936
|
-
): void {
|
|
937
|
-
this.#runStore.assertLease(lease);
|
|
938
|
-
const existing = this.#database
|
|
939
|
-
.prepare(
|
|
940
|
-
`SELECT receipt_hash, plan_json FROM workflow_engine_deliveries
|
|
941
|
-
WHERE run_id = ? AND revision = ?`,
|
|
942
|
-
)
|
|
943
|
-
.get(runId, delivery.revision) as
|
|
944
|
-
| { receipt_hash: string; plan_json: string }
|
|
945
|
-
| undefined;
|
|
946
|
-
const capturedBudget = this.#database
|
|
947
|
-
.prepare("SELECT hard_limit FROM workflow_work_budget WHERE run_id = ?")
|
|
948
|
-
.get(runId) as { hard_limit: number } | undefined;
|
|
949
|
-
if (
|
|
950
|
-
this.#phaseCount(delivery.plan) >
|
|
951
|
-
(capturedBudget?.hard_limit ?? this.#workHardLimit) &&
|
|
952
|
-
!existing
|
|
953
|
-
)
|
|
954
|
-
throw new DeliveryValidationError([
|
|
955
|
-
"verification-work-budget-insufficient",
|
|
956
|
-
]);
|
|
957
|
-
const serializedPlan = JSON.stringify(delivery.plan);
|
|
958
|
-
if (
|
|
959
|
-
existing &&
|
|
960
|
-
(existing.receipt_hash !== delivery.receiptHash ||
|
|
961
|
-
existing.plan_json !== serializedPlan)
|
|
962
|
-
) {
|
|
963
|
-
throw new Error("delivery-revision-conflict");
|
|
964
|
-
}
|
|
965
|
-
const engineRun = this.#engineRun(runId);
|
|
966
|
-
if (
|
|
967
|
-
engineRun.current_revision !== null &&
|
|
968
|
-
delivery.revision < engineRun.current_revision
|
|
969
|
-
) {
|
|
970
|
-
throw new Error("delivery-revision-stale");
|
|
971
|
-
}
|
|
972
|
-
const priorRows = this.#tasks(runId);
|
|
973
|
-
const priorById = new Map(priorRows.map((row) => [row.task_id, row]));
|
|
974
|
-
const priorPlan =
|
|
975
|
-
engineRun.current_revision === null
|
|
976
|
-
? undefined
|
|
977
|
-
: this.#planForRevision(runId, engineRun.current_revision);
|
|
978
|
-
const comparison = compareDeliveryRevision(
|
|
979
|
-
priorPlan,
|
|
980
|
-
delivery.plan,
|
|
981
|
-
priorRows,
|
|
982
|
-
);
|
|
983
|
-
const { compatible, boundaryExpanded } = comparison;
|
|
984
|
-
let invalidated = comparison.invalidated;
|
|
985
|
-
let revalidatedWorkspace:
|
|
986
|
-
| {
|
|
987
|
-
baselineRevisionId: string;
|
|
988
|
-
currentWorkspaceRevisionId: string;
|
|
989
|
-
}
|
|
990
|
-
| undefined;
|
|
991
|
-
if (
|
|
992
|
-
priorPlan &&
|
|
993
|
-
(invalidated.size > 0 || boundaryExpanded) &&
|
|
994
|
-
engineRun.baseline_workspace_revision &&
|
|
995
|
-
engineRun.current_workspace_revision
|
|
996
|
-
) {
|
|
997
|
-
if (!this.#worker.revalidateDelivery) {
|
|
998
|
-
if (boundaryExpanded) {
|
|
999
|
-
throw new Error("workflow-delivery-revalidation-unavailable");
|
|
1000
|
-
}
|
|
1001
|
-
if (
|
|
1002
|
-
engineRun.current_workspace_revision !==
|
|
1003
|
-
engineRun.baseline_workspace_revision
|
|
1004
|
-
) {
|
|
1005
|
-
throw new Error("workflow-delivery-revalidation-unavailable");
|
|
1006
|
-
}
|
|
1007
|
-
} else {
|
|
1008
|
-
const result = this.#worker.revalidateDelivery({
|
|
1009
|
-
runId,
|
|
1010
|
-
deliveryRevision: delivery.revision,
|
|
1011
|
-
plan: structuredClone(delivery.plan),
|
|
1012
|
-
tasks: priorRows
|
|
1013
|
-
.filter((row) => compatible.has(row.task_id))
|
|
1014
|
-
.map((row) => ({
|
|
1015
|
-
taskId: row.task_id,
|
|
1016
|
-
deliveryRevision: row.delivery_revision,
|
|
1017
|
-
state: row.state,
|
|
1018
|
-
phase: row.phase,
|
|
1019
|
-
contextReadPaths: this.#contextReads(runId, row.task_id),
|
|
1020
|
-
})),
|
|
1021
|
-
invalidatedTaskIds: [...invalidated].sort(),
|
|
1022
|
-
baselineRevisionId: engineRun.baseline_workspace_revision,
|
|
1023
|
-
currentWorkspaceRevisionId: engineRun.current_workspace_revision,
|
|
1024
|
-
});
|
|
1025
|
-
const resultIds = new Set(result.invalidatedTaskIds);
|
|
1026
|
-
if (
|
|
1027
|
-
!Array.isArray(result.invalidatedTaskIds) ||
|
|
1028
|
-
resultIds.size !== result.invalidatedTaskIds.length ||
|
|
1029
|
-
[...invalidated].some((taskId) => !resultIds.has(taskId)) ||
|
|
1030
|
-
[...resultIds].some((taskId) => !priorById.has(taskId)) ||
|
|
1031
|
-
!SHA256.test(result.baselineRevisionId) ||
|
|
1032
|
-
!SHA256.test(result.currentWorkspaceRevisionId)
|
|
1033
|
-
) {
|
|
1034
|
-
throw new Error("workflow-delivery-revalidation-invalid");
|
|
1035
|
-
}
|
|
1036
|
-
invalidated = resultIds;
|
|
1037
|
-
revalidatedWorkspace = {
|
|
1038
|
-
baselineRevisionId: result.baselineRevisionId,
|
|
1039
|
-
currentWorkspaceRevisionId: result.currentWorkspaceRevisionId,
|
|
1040
|
-
};
|
|
1041
|
-
}
|
|
1042
|
-
}
|
|
1043
|
-
const deliveryBindings = delivery.approvalProofs
|
|
1044
|
-
? ([
|
|
1045
|
-
{
|
|
1046
|
-
runId,
|
|
1047
|
-
gate: "gate-a" as const,
|
|
1048
|
-
revision: delivery.revision,
|
|
1049
|
-
receiptHash: delivery.receiptHash,
|
|
1050
|
-
approvalProof: structuredClone(delivery.approvalProofs.gateA),
|
|
1051
|
-
operationId: `bind-a-${hash(
|
|
1052
|
-
runId,
|
|
1053
|
-
String(delivery.revision),
|
|
1054
|
-
delivery.approvalProofs.gateA.recordHash,
|
|
1055
|
-
).slice(0, 40)}`,
|
|
1056
|
-
lease,
|
|
1057
|
-
},
|
|
1058
|
-
{
|
|
1059
|
-
runId,
|
|
1060
|
-
gate: "gate-b" as const,
|
|
1061
|
-
revision: delivery.revision,
|
|
1062
|
-
receiptHash: delivery.receiptHash,
|
|
1063
|
-
approvalProof: structuredClone(delivery.approvalProofs.gateB),
|
|
1064
|
-
operationId: `bind-b-${hash(
|
|
1065
|
-
runId,
|
|
1066
|
-
String(delivery.revision),
|
|
1067
|
-
delivery.approvalProofs.gateB.recordHash,
|
|
1068
|
-
).slice(0, 40)}`,
|
|
1069
|
-
lease,
|
|
1070
|
-
},
|
|
1071
|
-
] as const)
|
|
1072
|
-
: ([
|
|
1073
|
-
{
|
|
1074
|
-
runId,
|
|
1075
|
-
gate: delivery.gate,
|
|
1076
|
-
revision: delivery.revision,
|
|
1077
|
-
receiptHash: delivery.receiptHash,
|
|
1078
|
-
operationId: `bind-${hash(
|
|
1079
|
-
runId,
|
|
1080
|
-
String(delivery.revision),
|
|
1081
|
-
delivery.receiptHash,
|
|
1082
|
-
).slice(0, 40)}`,
|
|
1083
|
-
lease,
|
|
1084
|
-
},
|
|
1085
|
-
] as const);
|
|
1086
|
-
this.#runStore.bindDeliveriesAtomically(deliveryBindings, (database) => {
|
|
1087
|
-
const engineLease = database
|
|
1088
|
-
.prepare(
|
|
1089
|
-
`SELECT state, lease_expires_at
|
|
1090
|
-
FROM workflow_engine_operations
|
|
1091
|
-
WHERE run_id = ? AND lease_token = ?`,
|
|
1092
|
-
)
|
|
1093
|
-
.get(runId, lease.token) as
|
|
1094
|
-
| { state: string; lease_expires_at: number | null }
|
|
1095
|
-
| undefined;
|
|
1096
|
-
if (
|
|
1097
|
-
engineLease?.state !== "running" ||
|
|
1098
|
-
engineLease.lease_expires_at === null ||
|
|
1099
|
-
engineLease.lease_expires_at < this.#now()
|
|
1100
|
-
) {
|
|
1101
|
-
throw new Error("lease-fenced");
|
|
1102
|
-
}
|
|
1103
|
-
database
|
|
1104
|
-
.prepare(
|
|
1105
|
-
`INSERT INTO workflow_engine_runs(
|
|
1106
|
-
run_id, current_revision, delivery_diagnostics_json,
|
|
1107
|
-
next_queue_position
|
|
1108
|
-
) VALUES (?, ?, NULL, 1)
|
|
1109
|
-
ON CONFLICT(run_id) DO UPDATE SET
|
|
1110
|
-
current_revision = excluded.current_revision,
|
|
1111
|
-
verification_json = NULL,
|
|
1112
|
-
delivery_diagnostics_json = NULL`,
|
|
1113
|
-
)
|
|
1114
|
-
.run(runId, delivery.revision);
|
|
1115
|
-
if (revalidatedWorkspace) {
|
|
1116
|
-
database
|
|
1117
|
-
.prepare(
|
|
1118
|
-
`UPDATE workflow_engine_runs
|
|
1119
|
-
SET baseline_workspace_revision = ?,
|
|
1120
|
-
current_workspace_revision = ?, cleanup_state = 'retained'
|
|
1121
|
-
WHERE run_id = ?`,
|
|
1122
|
-
)
|
|
1123
|
-
.run(
|
|
1124
|
-
revalidatedWorkspace.baselineRevisionId,
|
|
1125
|
-
revalidatedWorkspace.currentWorkspaceRevisionId,
|
|
1126
|
-
runId,
|
|
1127
|
-
);
|
|
1128
|
-
}
|
|
1129
|
-
database
|
|
1130
|
-
.prepare(
|
|
1131
|
-
`INSERT INTO workflow_engine_deliveries(
|
|
1132
|
-
run_id, revision, gate, receipt_hash, plan_json
|
|
1133
|
-
) VALUES (?, ?, ?, ?, ?)
|
|
1134
|
-
ON CONFLICT(run_id, revision) DO NOTHING`,
|
|
1135
|
-
)
|
|
1136
|
-
.run(
|
|
1137
|
-
runId,
|
|
1138
|
-
delivery.revision,
|
|
1139
|
-
delivery.gate,
|
|
1140
|
-
delivery.receiptHash,
|
|
1141
|
-
serializedPlan,
|
|
1142
|
-
);
|
|
1143
|
-
|
|
1144
|
-
this.#growWorkBudget(database, runId, delivery.plan);
|
|
1145
|
-
const admitted = new Set<string>();
|
|
1146
|
-
delivery.plan.tasks.forEach((task, taskOrder) => {
|
|
1147
|
-
admitted.add(task.taskId);
|
|
1148
|
-
const taskJson = JSON.stringify(task);
|
|
1149
|
-
const prior = priorById.get(task.taskId);
|
|
1150
|
-
if (!prior) {
|
|
1151
|
-
database
|
|
1152
|
-
.prepare(
|
|
1153
|
-
`INSERT INTO workflow_engine_tasks(
|
|
1154
|
-
run_id, task_id, task_order, delivery_revision, plan_json,
|
|
1155
|
-
state, phase
|
|
1156
|
-
) VALUES (?, ?, ?, ?, ?, 'pending', ?)`,
|
|
1157
|
-
)
|
|
1158
|
-
.run(
|
|
1159
|
-
runId,
|
|
1160
|
-
task.taskId,
|
|
1161
|
-
taskOrder,
|
|
1162
|
-
delivery.revision,
|
|
1163
|
-
taskJson,
|
|
1164
|
-
phases(task)[0],
|
|
1165
|
-
);
|
|
1166
|
-
return;
|
|
1167
|
-
}
|
|
1168
|
-
if (compatible.has(task.taskId) && !invalidated.has(task.taskId)) {
|
|
1169
|
-
database
|
|
1170
|
-
.prepare(
|
|
1171
|
-
`UPDATE workflow_engine_tasks
|
|
1172
|
-
SET task_order = ?, plan_json = ?
|
|
1173
|
-
WHERE run_id = ? AND task_id = ?`,
|
|
1174
|
-
)
|
|
1175
|
-
.run(taskOrder, taskJson, runId, task.taskId);
|
|
1176
|
-
return;
|
|
1177
|
-
}
|
|
1178
|
-
database
|
|
1179
|
-
.prepare(
|
|
1180
|
-
"DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
|
|
1181
|
-
)
|
|
1182
|
-
.run(runId, task.taskId);
|
|
1183
|
-
database
|
|
1184
|
-
.prepare(
|
|
1185
|
-
`UPDATE workflow_engine_tasks
|
|
1186
|
-
SET task_order = ?, delivery_revision = ?, plan_json = ?,
|
|
1187
|
-
state = 'pending', phase = ?, pause_code = NULL,
|
|
1188
|
-
context_request_json = NULL, attempt_diagnostic_json = NULL,
|
|
1189
|
-
queue_position = NULL
|
|
1190
|
-
WHERE run_id = ? AND task_id = ?`,
|
|
1191
|
-
)
|
|
1192
|
-
.run(
|
|
1193
|
-
taskOrder,
|
|
1194
|
-
delivery.revision,
|
|
1195
|
-
taskJson,
|
|
1196
|
-
phases(task)[0],
|
|
1197
|
-
runId,
|
|
1198
|
-
task.taskId,
|
|
1199
|
-
);
|
|
1200
|
-
});
|
|
1201
|
-
for (const row of priorRows) {
|
|
1202
|
-
if (!admitted.has(row.task_id)) {
|
|
1203
|
-
database
|
|
1204
|
-
.prepare(
|
|
1205
|
-
"DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
|
|
1206
|
-
)
|
|
1207
|
-
.run(runId, row.task_id);
|
|
1208
|
-
database
|
|
1209
|
-
.prepare(
|
|
1210
|
-
`DELETE FROM workflow_engine_tasks
|
|
1211
|
-
WHERE run_id = ? AND task_id = ?`,
|
|
1212
|
-
)
|
|
1213
|
-
.run(runId, row.task_id);
|
|
1214
|
-
}
|
|
1215
|
-
}
|
|
1216
|
-
});
|
|
1217
|
-
}
|
|
1218
|
-
|
|
1219
|
-
#cancelledDelivery(
|
|
1220
|
-
runId: string,
|
|
1221
|
-
operationId: string,
|
|
1222
|
-
error: unknown,
|
|
1223
|
-
signal: AbortSignal | undefined,
|
|
1224
|
-
lease: OperationLease,
|
|
1225
|
-
): Record<string, unknown> | undefined {
|
|
1226
|
-
if (!signal || !isCancellationException(error, signal)) return undefined;
|
|
1227
|
-
try {
|
|
1228
|
-
this.#runStore.assertLease(lease);
|
|
1229
|
-
if (
|
|
1230
|
-
["validating-delivery", "ready"].includes(
|
|
1231
|
-
this.#runStore.status(runId).state,
|
|
1232
|
-
)
|
|
1233
|
-
) {
|
|
1234
|
-
this.#transition(
|
|
1235
|
-
runId,
|
|
1236
|
-
"paused",
|
|
1237
|
-
`${operationId}:delivery-cancelled`,
|
|
1238
|
-
"operation-cancelled",
|
|
1239
|
-
lease,
|
|
1240
|
-
);
|
|
1241
|
-
}
|
|
1242
|
-
} catch (failure) {
|
|
1243
|
-
if (!(failure instanceof Error) || failure.message !== "lease-fenced")
|
|
1244
|
-
throw failure;
|
|
1245
|
-
} finally {
|
|
1246
|
-
this.#interruptOperation(runId, operationId);
|
|
1247
|
-
}
|
|
1248
|
-
return this.#statusByRun(runId);
|
|
1249
|
-
}
|
|
1250
|
-
|
|
1251
|
-
#deliveryFailure(
|
|
1252
|
-
runId: string,
|
|
1253
|
-
operationId: string,
|
|
1254
|
-
error: unknown,
|
|
1255
|
-
lease: OperationLease,
|
|
1256
|
-
): Record<string, unknown> | undefined {
|
|
1257
|
-
if (!(error instanceof DeliveryValidationError)) return undefined;
|
|
1258
|
-
this.#leasedTransaction(lease, () => {
|
|
1259
|
-
this.#database
|
|
1260
|
-
.prepare(
|
|
1261
|
-
`UPDATE workflow_engine_runs
|
|
1262
|
-
SET delivery_diagnostics_json = ? WHERE run_id = ?`,
|
|
1263
|
-
)
|
|
1264
|
-
.run(JSON.stringify(error.diagnostics), runId);
|
|
1265
|
-
});
|
|
1266
|
-
this.#transition(
|
|
1267
|
-
runId,
|
|
1268
|
-
"paused",
|
|
1269
|
-
`${operationId}:delivery-invalid`,
|
|
1270
|
-
"delivery-invalid",
|
|
1271
|
-
lease,
|
|
1272
|
-
);
|
|
1273
|
-
return {
|
|
1274
|
-
...this.#statusByRun(runId),
|
|
1275
|
-
delivery: {
|
|
1276
|
-
code: "delivery-invalid",
|
|
1277
|
-
diagnostics: [...error.diagnostics],
|
|
1278
|
-
},
|
|
1279
|
-
};
|
|
1280
|
-
}
|
|
1281
|
-
|
|
1282
|
-
#setTask(
|
|
1283
|
-
runId: string,
|
|
1284
|
-
taskId: string,
|
|
1285
|
-
values: {
|
|
1286
|
-
state?: string;
|
|
1287
|
-
phase?: "red" | "green" | "refactor";
|
|
1288
|
-
pauseCode?: string | null;
|
|
1289
|
-
contextRequest?: WorkflowContextRequest | null;
|
|
1290
|
-
attemptDiagnostic?: SafeAttemptDiagnostic | null;
|
|
1291
|
-
routeId?: string | null;
|
|
1292
|
-
routeFingerprint?: string | null;
|
|
1293
|
-
queuePosition?: number | null;
|
|
1294
|
-
},
|
|
1295
|
-
lease?: OperationLease,
|
|
1296
|
-
): void {
|
|
1297
|
-
if (values.attemptDiagnostic?.recovery && lease) {
|
|
1298
|
-
const fact = values.attemptDiagnostic.recovery;
|
|
1299
|
-
const prior = this.#recovery(runId, fact.key);
|
|
1300
|
-
if (
|
|
1301
|
-
!prior ||
|
|
1302
|
-
fact.failures > prior.failures ||
|
|
1303
|
-
fact.feedback.maxAttempts < prior.feedback.maxAttempts
|
|
1304
|
-
)
|
|
1305
|
-
this.#recordRecovery(runId, fact, lease);
|
|
1306
|
-
}
|
|
1307
|
-
const assignments: string[] = [];
|
|
1308
|
-
const parameters: Array<string | number | null> = [];
|
|
1309
|
-
if (values.state !== undefined) {
|
|
1310
|
-
assignments.push("state = ?");
|
|
1311
|
-
parameters.push(values.state);
|
|
1312
|
-
}
|
|
1313
|
-
if (values.phase !== undefined) {
|
|
1314
|
-
assignments.push("phase = ?");
|
|
1315
|
-
parameters.push(values.phase);
|
|
1316
|
-
}
|
|
1317
|
-
if (values.pauseCode !== undefined) {
|
|
1318
|
-
assignments.push("pause_code = ?");
|
|
1319
|
-
parameters.push(values.pauseCode);
|
|
1320
|
-
}
|
|
1321
|
-
if (values.contextRequest !== undefined) {
|
|
1322
|
-
assignments.push("context_request_json = ?");
|
|
1323
|
-
parameters.push(
|
|
1324
|
-
values.contextRequest === null
|
|
1325
|
-
? null
|
|
1326
|
-
: JSON.stringify(
|
|
1327
|
-
normalizeWorkflowContextRequest(values.contextRequest),
|
|
1328
|
-
),
|
|
1329
|
-
);
|
|
1330
|
-
}
|
|
1331
|
-
if (values.attemptDiagnostic !== undefined) {
|
|
1332
|
-
assignments.push("attempt_diagnostic_json = ?");
|
|
1333
|
-
parameters.push(
|
|
1334
|
-
values.attemptDiagnostic === null
|
|
1335
|
-
? null
|
|
1336
|
-
: JSON.stringify(
|
|
1337
|
-
Object.fromEntries(
|
|
1338
|
-
Object.entries(values.attemptDiagnostic).filter(
|
|
1339
|
-
([key]) => key !== "recovery",
|
|
1340
|
-
),
|
|
1341
|
-
),
|
|
1342
|
-
),
|
|
1343
|
-
);
|
|
1344
|
-
}
|
|
1345
|
-
if (values.routeId !== undefined) {
|
|
1346
|
-
assignments.push("route_id = ?");
|
|
1347
|
-
parameters.push(values.routeId);
|
|
1348
|
-
}
|
|
1349
|
-
if (values.routeFingerprint !== undefined) {
|
|
1350
|
-
assignments.push("route_fingerprint = ?");
|
|
1351
|
-
parameters.push(values.routeFingerprint);
|
|
1352
|
-
}
|
|
1353
|
-
if (values.queuePosition !== undefined) {
|
|
1354
|
-
assignments.push("queue_position = ?");
|
|
1355
|
-
parameters.push(values.queuePosition);
|
|
1356
|
-
}
|
|
1357
|
-
if (assignments.length === 0) return;
|
|
1358
|
-
const write = () => {
|
|
1359
|
-
this.#database
|
|
1360
|
-
.prepare(
|
|
1361
|
-
`UPDATE workflow_engine_tasks SET ${assignments.join(", ")}
|
|
1362
|
-
WHERE run_id = ? AND task_id = ?`,
|
|
1363
|
-
)
|
|
1364
|
-
.run(...parameters, runId, taskId);
|
|
1365
|
-
};
|
|
1366
|
-
if (lease) this.#leasedTransaction(lease, write);
|
|
1367
|
-
else write();
|
|
1368
|
-
}
|
|
1369
|
-
|
|
1370
|
-
#setVerificationStatus(
|
|
1371
|
-
runId: string,
|
|
1372
|
-
status: WorkflowVerificationStatus | null,
|
|
1373
|
-
lease: OperationLease,
|
|
1374
|
-
): void {
|
|
1375
|
-
const serialized =
|
|
1376
|
-
status === null
|
|
1377
|
-
? null
|
|
1378
|
-
: JSON.stringify(normalizeWorkflowVerificationStatus(status));
|
|
1379
|
-
this.#leasedTransaction(lease, () => {
|
|
1380
|
-
this.#database
|
|
1381
|
-
.prepare(
|
|
1382
|
-
`UPDATE workflow_engine_runs SET verification_json = ?
|
|
1383
|
-
WHERE run_id = ?`,
|
|
1384
|
-
)
|
|
1385
|
-
.run(serialized, runId);
|
|
1386
|
-
});
|
|
1387
|
-
}
|
|
1388
|
-
|
|
1389
|
-
#queueTask(
|
|
1390
|
-
runId: string,
|
|
1391
|
-
row: EngineTaskRow,
|
|
1392
|
-
lease: OperationLease,
|
|
1393
|
-
reason: "conflict" | "capacity" = "conflict",
|
|
1394
|
-
): void {
|
|
1395
|
-
const pauseCode = reason === "capacity" ? "worker-capacity" : null;
|
|
1396
|
-
if (row.state === "queued" && row.queue_position !== null) {
|
|
1397
|
-
if (row.pause_code !== pauseCode)
|
|
1398
|
-
this.#setTask(runId, row.task_id, { pauseCode }, lease);
|
|
1399
|
-
return;
|
|
1400
|
-
}
|
|
1401
|
-
const engineRun = this.#engineRun(runId);
|
|
1402
|
-
this.#leasedTransaction(lease, () => {
|
|
1403
|
-
this.#setTask(runId, row.task_id, {
|
|
1404
|
-
state: "queued",
|
|
1405
|
-
pauseCode,
|
|
1406
|
-
queuePosition: engineRun.next_queue_position,
|
|
1407
|
-
});
|
|
1408
|
-
this.#database
|
|
1409
|
-
.prepare(
|
|
1410
|
-
`UPDATE workflow_engine_runs
|
|
1411
|
-
SET next_queue_position = next_queue_position + 1
|
|
1412
|
-
WHERE run_id = ?`,
|
|
1413
|
-
)
|
|
1414
|
-
.run(runId);
|
|
1415
|
-
});
|
|
1416
|
-
}
|
|
1417
|
-
|
|
1418
|
-
#recordWorkspaceFacts(
|
|
1419
|
-
runId: string,
|
|
1420
|
-
outcome: WorkflowAttemptOutcome,
|
|
1421
|
-
lease: OperationLease,
|
|
1422
|
-
): void {
|
|
1423
|
-
const baselineRevisionId = outcome.baselineRevisionId;
|
|
1424
|
-
const currentWorkspaceRevisionId =
|
|
1425
|
-
outcome.kind === "phase-committed"
|
|
1426
|
-
? outcome.isolatedRevisionId
|
|
1427
|
-
: outcome.currentWorkspaceRevisionId;
|
|
1428
|
-
if (
|
|
1429
|
-
(baselineRevisionId !== undefined && !SHA256.test(baselineRevisionId)) ||
|
|
1430
|
-
(currentWorkspaceRevisionId !== undefined &&
|
|
1431
|
-
!SHA256.test(currentWorkspaceRevisionId)) ||
|
|
1432
|
-
(outcome.kind !== "phase-committed" &&
|
|
1433
|
-
(baselineRevisionId === undefined) !==
|
|
1434
|
-
(currentWorkspaceRevisionId === undefined))
|
|
1435
|
-
) {
|
|
1436
|
-
throw new Error("workflow-attempt-workspace-fact-invalid");
|
|
1437
|
-
}
|
|
1438
|
-
if (currentWorkspaceRevisionId === undefined) return;
|
|
1439
|
-
this.#leasedTransaction(lease, () => {
|
|
1440
|
-
this.#database
|
|
1441
|
-
.prepare(
|
|
1442
|
-
`UPDATE workflow_engine_runs
|
|
1443
|
-
SET baseline_workspace_revision = COALESCE(
|
|
1444
|
-
baseline_workspace_revision, ?
|
|
1445
|
-
),
|
|
1446
|
-
current_workspace_revision = ?,
|
|
1447
|
-
cleanup_state = CASE
|
|
1448
|
-
WHEN ? IS NULL THEN cleanup_state ELSE 'retained'
|
|
1449
|
-
END
|
|
1450
|
-
WHERE run_id = ?`,
|
|
1451
|
-
)
|
|
1452
|
-
.run(
|
|
1453
|
-
baselineRevisionId ?? null,
|
|
1454
|
-
currentWorkspaceRevisionId,
|
|
1455
|
-
baselineRevisionId ?? null,
|
|
1456
|
-
runId,
|
|
1457
|
-
);
|
|
1458
|
-
});
|
|
1459
|
-
}
|
|
1460
|
-
|
|
1461
|
-
#recordRouteFacts(
|
|
1462
|
-
runId: string,
|
|
1463
|
-
taskId: string,
|
|
1464
|
-
outcome: WorkflowAttemptOutcome,
|
|
1465
|
-
lease: OperationLease,
|
|
1466
|
-
): void {
|
|
1467
|
-
if (
|
|
1468
|
-
(outcome.routeId === undefined) !==
|
|
1469
|
-
(outcome.routeFingerprint === undefined) ||
|
|
1470
|
-
(outcome.routeId !== undefined && !IDENTIFIER.test(outcome.routeId)) ||
|
|
1471
|
-
(outcome.routeFingerprint !== undefined &&
|
|
1472
|
-
!SHA256.test(outcome.routeFingerprint))
|
|
1473
|
-
) {
|
|
1474
|
-
throw new Error("workflow-attempt-route-fact-invalid");
|
|
1475
|
-
}
|
|
1476
|
-
if (!outcome.routeId || !outcome.routeFingerprint) return;
|
|
1477
|
-
const routeId = outcome.routeId;
|
|
1478
|
-
const routeFingerprint = outcome.routeFingerprint;
|
|
1479
|
-
this.#leasedTransaction(lease, () => {
|
|
1480
|
-
this.#setTask(runId, taskId, {
|
|
1481
|
-
routeId,
|
|
1482
|
-
routeFingerprint,
|
|
1483
|
-
});
|
|
1484
|
-
this.#database
|
|
1485
|
-
.prepare(
|
|
1486
|
-
`UPDATE workflow_engine_runs
|
|
1487
|
-
SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
|
|
1488
|
-
)
|
|
1489
|
-
.run(routeId, routeFingerprint, runId);
|
|
1490
|
-
});
|
|
1491
|
-
}
|
|
1492
|
-
|
|
1493
|
-
async #runTask(
|
|
1494
|
-
runId: string,
|
|
1495
|
-
row: EngineTaskRow,
|
|
1496
|
-
operationId: string,
|
|
1497
|
-
signal: AbortSignal,
|
|
1498
|
-
lease: OperationLease,
|
|
1499
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
1500
|
-
): Promise<void> {
|
|
1501
|
-
const task = this.#taskPlan(row);
|
|
1502
|
-
const orderedPhases = phases(task);
|
|
1503
|
-
let phaseIndex = orderedPhases.indexOf(row.phase);
|
|
1504
|
-
let artifactAttempts = 0;
|
|
1505
|
-
const persistedContextRequest = row.context_request_json
|
|
1506
|
-
? normalizeWorkflowContextRequest(
|
|
1507
|
-
JSON.parse(row.context_request_json) as unknown,
|
|
1508
|
-
)
|
|
1509
|
-
: undefined;
|
|
1510
|
-
const persistedRedArtifactCorrection =
|
|
1511
|
-
row.phase === "green" &&
|
|
1512
|
-
persistedContextRequest !== undefined &&
|
|
1513
|
-
classifyPersistedContextRequest(persistedContextRequest, task, "green")
|
|
1514
|
-
.code === "red-artifact-constraint";
|
|
1515
|
-
let redArtifactCorrection =
|
|
1516
|
-
row.phase === "green" &&
|
|
1517
|
-
(row.pause_code === "red-artifact-constraint" ||
|
|
1518
|
-
persistedRedArtifactCorrection);
|
|
1519
|
-
let requestedContext = persistedContextRequest;
|
|
1520
|
-
let correctionContextRequest = redArtifactCorrection
|
|
1521
|
-
? persistedContextRequest
|
|
1522
|
-
: undefined;
|
|
1523
|
-
if (phaseIndex < 0) throw new Error("workflow-task-phase-invalid");
|
|
1524
|
-
while (phaseIndex < orderedPhases.length) {
|
|
1525
|
-
const phase = orderedPhases[phaseIndex];
|
|
1526
|
-
row.phase = phase;
|
|
1527
|
-
if (signal.aborted) {
|
|
1528
|
-
this.#setTask(
|
|
1529
|
-
runId,
|
|
1530
|
-
row.task_id,
|
|
1531
|
-
{
|
|
1532
|
-
state: "paused",
|
|
1533
|
-
phase,
|
|
1534
|
-
pauseCode: "operation-cancelled",
|
|
1535
|
-
contextRequest: null,
|
|
1536
|
-
queuePosition: null,
|
|
1537
|
-
},
|
|
1538
|
-
lease,
|
|
1539
|
-
);
|
|
1540
|
-
return;
|
|
1541
|
-
}
|
|
1542
|
-
const current = this.#engineRun(runId);
|
|
1543
|
-
if (current.current_revision === null)
|
|
1544
|
-
throw new Error("delivery-not-bound");
|
|
1545
|
-
const deliveryRevision = current.current_revision;
|
|
1546
|
-
const plan = this.#planForRevision(runId, deliveryRevision);
|
|
1547
|
-
this.#leasedTransaction(lease, () => {
|
|
1548
|
-
for (const ref of requestedContext?.refs ?? []) {
|
|
1549
|
-
if (
|
|
1550
|
-
ref.kind === "requested-path" &&
|
|
1551
|
-
ref.access === "read" &&
|
|
1552
|
-
permitsContextRead(ref.path, task.roots) &&
|
|
1553
|
-
(current.current_workspace_revision
|
|
1554
|
-
? current.baseline_workspace_revision &&
|
|
1555
|
-
this.#worker.isContextReadAvailable?.({
|
|
1556
|
-
runId,
|
|
1557
|
-
deliveryRevision,
|
|
1558
|
-
plan,
|
|
1559
|
-
baselineRevisionId: current.baseline_workspace_revision,
|
|
1560
|
-
currentWorkspaceRevisionId:
|
|
1561
|
-
current.current_workspace_revision,
|
|
1562
|
-
path: ref.path,
|
|
1563
|
-
})
|
|
1564
|
-
: isSafeRegularFile(this.#consumerRoot, ref.path))
|
|
1565
|
-
) {
|
|
1566
|
-
this.#database
|
|
1567
|
-
.prepare(
|
|
1568
|
-
"INSERT INTO workflow_context_reads(run_id, task_id, path) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
|
|
1569
|
-
)
|
|
1570
|
-
.run(runId, row.task_id, ref.path);
|
|
1571
|
-
}
|
|
1572
|
-
}
|
|
1573
|
-
});
|
|
1574
|
-
const verificationStatus = current.verification_json
|
|
1575
|
-
? normalizeWorkflowVerificationStatus(
|
|
1576
|
-
JSON.parse(current.verification_json) as unknown,
|
|
1577
|
-
)
|
|
1578
|
-
: undefined;
|
|
1579
|
-
const priorDiagnostic = this.#safeAttemptDiagnostic(runId, row);
|
|
1580
|
-
const priorRecovery = priorDiagnostic?.recovery;
|
|
1581
|
-
const preparationKey = baselineRecoveryKey(task, phase);
|
|
1582
|
-
const activeRecovery =
|
|
1583
|
-
priorRecovery &&
|
|
1584
|
-
[recoveryKey(task, phase, row, current), preparationKey].includes(
|
|
1585
|
-
priorRecovery.key,
|
|
1586
|
-
)
|
|
1587
|
-
? priorRecovery
|
|
1588
|
-
: undefined;
|
|
1589
|
-
const maxRecoveryAttempts = Math.min(
|
|
1590
|
-
plan.verification.artifactCorrection.maxAttempts,
|
|
1591
|
-
activeRecovery?.feedback.maxAttempts ?? Infinity,
|
|
1592
|
-
);
|
|
1593
|
-
if (
|
|
1594
|
-
activeRecovery &&
|
|
1595
|
-
activeRecovery.feedback.maxAttempts !== maxRecoveryAttempts
|
|
1596
|
-
) {
|
|
1597
|
-
this.#leasedTransaction(lease, () => {
|
|
1598
|
-
this.#database
|
|
1599
|
-
.prepare(
|
|
1600
|
-
"UPDATE workflow_recovery_incidents SET max_attempts = MIN(max_attempts, ?) WHERE run_id = ? AND incident_key = ?",
|
|
1601
|
-
)
|
|
1602
|
-
.run(maxRecoveryAttempts, runId, activeRecovery.key);
|
|
1603
|
-
});
|
|
1604
|
-
activeRecovery.feedback.maxAttempts = maxRecoveryAttempts;
|
|
1605
|
-
}
|
|
1606
|
-
const granted = this.#hasRecoveryGrant(
|
|
1607
|
-
runId,
|
|
1608
|
-
operationId,
|
|
1609
|
-
activeRecovery?.key ?? recoveryKey(task, phase, row, current),
|
|
1610
|
-
);
|
|
1611
|
-
const verificationOnly =
|
|
1612
|
-
activeRecovery?.key !== preparationKey &&
|
|
1613
|
-
recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted) &&
|
|
1614
|
-
(await this.#worker.hasPendingVerification?.({
|
|
1615
|
-
runId,
|
|
1616
|
-
operationId,
|
|
1617
|
-
deliveryRevision: row.delivery_revision,
|
|
1618
|
-
taskId: row.task_id,
|
|
1619
|
-
phase,
|
|
1620
|
-
task,
|
|
1621
|
-
plan,
|
|
1622
|
-
signal,
|
|
1623
|
-
...(current.baseline_workspace_revision
|
|
1624
|
-
? { baselineRevisionId: current.baseline_workspace_revision }
|
|
1625
|
-
: {}),
|
|
1626
|
-
...(current.current_workspace_revision
|
|
1627
|
-
? { currentWorkspaceRevisionId: current.current_workspace_revision }
|
|
1628
|
-
: {}),
|
|
1629
|
-
})) === true;
|
|
1630
|
-
if (
|
|
1631
|
-
!verificationOnly &&
|
|
1632
|
-
activeRecovery &&
|
|
1633
|
-
recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted)
|
|
1634
|
-
) {
|
|
1635
|
-
this.#setTask(
|
|
1636
|
-
runId,
|
|
1637
|
-
row.task_id,
|
|
1638
|
-
{
|
|
1639
|
-
state: "paused",
|
|
1640
|
-
phase,
|
|
1641
|
-
pauseCode: activeRecovery.feedback.code,
|
|
1642
|
-
attemptDiagnostic: priorDiagnostic ?? null,
|
|
1643
|
-
queuePosition: null,
|
|
1644
|
-
},
|
|
1645
|
-
lease,
|
|
1646
|
-
);
|
|
1647
|
-
return;
|
|
1648
|
-
}
|
|
1649
|
-
if (activeRecovery && activeRecovery.key !== preparationKey)
|
|
1650
|
-
artifactAttempts = Math.max(artifactAttempts, activeRecovery.failures);
|
|
1651
|
-
const repair =
|
|
1652
|
-
verificationStatus?.scope === "change-task-affected" &&
|
|
1653
|
-
verificationStatus.attribution === "introduced" &&
|
|
1654
|
-
verificationStatus.taskId === row.task_id &&
|
|
1655
|
-
verificationStatus.failureIdentities &&
|
|
1656
|
-
verificationStatus.failureIdentities.length > 0
|
|
1657
|
-
? {
|
|
1658
|
-
attribution: "introduced" as const,
|
|
1659
|
-
failureIdentities: [...verificationStatus.failureIdentities],
|
|
1660
|
-
}
|
|
1661
|
-
: undefined;
|
|
1662
|
-
const artifactCorrection =
|
|
1663
|
-
redArtifactCorrection && phase === "green"
|
|
1664
|
-
? {
|
|
1665
|
-
code: "red-artifact-constraint" as const,
|
|
1666
|
-
attempt: artifactAttempts + 1,
|
|
1667
|
-
maxAttempts: maxRecoveryAttempts,
|
|
1668
|
-
...(correctionContextRequest
|
|
1669
|
-
? {
|
|
1670
|
-
contextRequest: structuredClone(correctionContextRequest),
|
|
1671
|
-
}
|
|
1672
|
-
: {}),
|
|
1673
|
-
}
|
|
1674
|
-
: undefined;
|
|
1675
|
-
if (this.#worker.prepareTask) {
|
|
1676
|
-
// Reserve this selected task's conflicts while asynchronous baseline
|
|
1677
|
-
// preparation runs, without reserving a Worker attempt.
|
|
1678
|
-
this.#setTask(
|
|
1679
|
-
runId,
|
|
1680
|
-
row.task_id,
|
|
1681
|
-
{ state: "phase-ready", phase, queuePosition: null },
|
|
1682
|
-
lease,
|
|
1683
|
-
);
|
|
1684
|
-
if (granted && activeRecovery?.key === preparationKey) {
|
|
1685
|
-
this.#leasedTransaction(lease, () => {
|
|
1686
|
-
const consumed = this.#database
|
|
1687
|
-
.prepare(
|
|
1688
|
-
"UPDATE workflow_recovery_grants SET consumed = 1 WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
|
|
1689
|
-
)
|
|
1690
|
-
.run(runId, operationId, preparationKey);
|
|
1691
|
-
if (consumed.changes !== 1)
|
|
1692
|
-
throw new Error("recovery-request-stale");
|
|
1693
|
-
this.#database
|
|
1694
|
-
.prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
|
|
1695
|
-
SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'launch' FROM workflow_recovery_events WHERE run_id = ?`)
|
|
1696
|
-
.run(runId, preparationKey, runId);
|
|
1697
|
-
});
|
|
1698
|
-
}
|
|
1699
|
-
const prepared = await this.#worker.prepareTask({
|
|
1700
|
-
runId,
|
|
1701
|
-
operationId,
|
|
1702
|
-
deliveryRevision: row.delivery_revision,
|
|
1703
|
-
taskId: row.task_id,
|
|
1704
|
-
phase,
|
|
1705
|
-
task: structuredClone(task),
|
|
1706
|
-
plan: structuredClone(plan),
|
|
1707
|
-
...(current.baseline_workspace_revision
|
|
1708
|
-
? { baselineRevisionId: current.baseline_workspace_revision }
|
|
1709
|
-
: {}),
|
|
1710
|
-
...(current.current_workspace_revision
|
|
1711
|
-
? { currentWorkspaceRevisionId: current.current_workspace_revision }
|
|
1712
|
-
: {}),
|
|
1713
|
-
signal,
|
|
1714
|
-
});
|
|
1715
|
-
this.#runStore.assertLease(lease);
|
|
1716
|
-
if (prepared.kind === "phase-committed")
|
|
1717
|
-
throw new Error("workflow-preparation-outcome-invalid");
|
|
1718
|
-
this.#recordWorkspaceFacts(
|
|
1719
|
-
runId,
|
|
1720
|
-
{
|
|
1721
|
-
...prepared,
|
|
1722
|
-
kind: "paused",
|
|
1723
|
-
code: "task-prepared",
|
|
1724
|
-
// Preparation has no merge authority over a sibling's newer commit.
|
|
1725
|
-
...(prepared.currentWorkspaceRevisionId
|
|
1726
|
-
? {
|
|
1727
|
-
currentWorkspaceRevisionId:
|
|
1728
|
-
this.#engineRun(runId).current_workspace_revision ??
|
|
1729
|
-
prepared.currentWorkspaceRevisionId,
|
|
1730
|
-
}
|
|
1731
|
-
: {}),
|
|
1732
|
-
},
|
|
1733
|
-
lease,
|
|
1734
|
-
);
|
|
1735
|
-
// Preparation may finish successfully while close is settling it.
|
|
1736
|
-
// Re-enter the cancellation guard before granting any Worker work.
|
|
1737
|
-
if (signal.aborted) continue;
|
|
1738
|
-
if (prepared.kind !== "prepared") {
|
|
1739
|
-
const prerequisite =
|
|
1740
|
-
prepared.kind !== "operation-cancelled"
|
|
1741
|
-
? normalizeVerificationPrerequisite(prepared.prerequisite)
|
|
1742
|
-
: undefined;
|
|
1743
|
-
if (prerequisite) {
|
|
1744
|
-
const verification =
|
|
1745
|
-
prerequisite.scope === "baseline-full-suite"
|
|
1746
|
-
? plan.verification.baseline?.fullSuite
|
|
1747
|
-
: (task.baselineVerification ?? task.affectedVerification);
|
|
1748
|
-
if (
|
|
1749
|
-
!verification ||
|
|
1750
|
-
verification.id !== prerequisite.verificationId ||
|
|
1751
|
-
(prerequisite.scope === "baseline-task-affected" &&
|
|
1752
|
-
prerequisite.taskId !== row.task_id) ||
|
|
1753
|
-
(prerequisite.input &&
|
|
1754
|
-
!verificationInputPaths(verification).includes(
|
|
1755
|
-
prerequisite.input.path,
|
|
1756
|
-
)) ||
|
|
1757
|
-
(prerequisite.producer &&
|
|
1758
|
-
!plan.outputs.some(
|
|
1759
|
-
(output) =>
|
|
1760
|
-
output.path === prerequisite.input?.path &&
|
|
1761
|
-
output.producer.taskId === prerequisite.producer?.taskId &&
|
|
1762
|
-
output.producer.phase === prerequisite.producer?.phase,
|
|
1763
|
-
))
|
|
1764
|
-
)
|
|
1765
|
-
throw new Error("workflow-prerequisite-evidence-invalid");
|
|
1766
|
-
}
|
|
1767
|
-
if (prerequisite && prepared.kind === "paused") {
|
|
1768
|
-
const identity = hash(
|
|
1769
|
-
canonicalJson({
|
|
1770
|
-
...prerequisite,
|
|
1771
|
-
verificationId: undefined,
|
|
1772
|
-
taskId: undefined,
|
|
1773
|
-
producer: prerequisite.producer
|
|
1774
|
-
? { phase: prerequisite.producer.phase }
|
|
1775
|
-
: undefined,
|
|
1776
|
-
}),
|
|
1777
|
-
);
|
|
1778
|
-
const priorPreparation = this.#recovery(runId, preparationKey);
|
|
1779
|
-
if (priorPreparation?.feedback.prerequisiteIdentity !== identity) {
|
|
1780
|
-
const failures = (priorPreparation?.failures ?? 0) + 1;
|
|
1781
|
-
this.#recordRecovery(
|
|
1782
|
-
runId,
|
|
1783
|
-
{
|
|
1784
|
-
key: preparationKey,
|
|
1785
|
-
failures,
|
|
1786
|
-
feedback: {
|
|
1787
|
-
code: prepared.code,
|
|
1788
|
-
attempt: failures + 1,
|
|
1789
|
-
maxAttempts: maxRecoveryAttempts,
|
|
1790
|
-
strategy: "repair-verification",
|
|
1791
|
-
prerequisiteIdentity: identity,
|
|
1792
|
-
},
|
|
1793
|
-
},
|
|
1794
|
-
lease,
|
|
1795
|
-
);
|
|
1796
|
-
}
|
|
1797
|
-
}
|
|
1798
|
-
if (prepared.kind !== "operation-cancelled" && prepared.verification)
|
|
1799
|
-
this.#setVerificationStatus(runId, prepared.verification, lease);
|
|
1800
|
-
this.#setTask(
|
|
1801
|
-
runId,
|
|
1802
|
-
row.task_id,
|
|
1803
|
-
{
|
|
1804
|
-
state:
|
|
1805
|
-
prepared.kind === "approval-needed"
|
|
1806
|
-
? "approval-needed"
|
|
1807
|
-
: "paused",
|
|
1808
|
-
phase,
|
|
1809
|
-
pauseCode:
|
|
1810
|
-
prepared.kind === "operation-cancelled"
|
|
1811
|
-
? "operation-cancelled"
|
|
1812
|
-
: prepared.code,
|
|
1813
|
-
attemptDiagnostic: {
|
|
1814
|
-
...priorDiagnostic,
|
|
1815
|
-
prerequisite,
|
|
1816
|
-
},
|
|
1817
|
-
queuePosition: null,
|
|
1818
|
-
},
|
|
1819
|
-
lease,
|
|
1820
|
-
);
|
|
1821
|
-
return;
|
|
1822
|
-
}
|
|
1823
|
-
if (priorDiagnostic) delete priorDiagnostic.prerequisite;
|
|
1824
|
-
if (activeRecovery?.key === preparationKey) {
|
|
1825
|
-
// A successful original-baseline observation resolves this incident.
|
|
1826
|
-
// Keep its events and immutable failure evidence, then re-enter the
|
|
1827
|
-
// ordinary phase recovery checks before reserving Worker work.
|
|
1828
|
-
this.#leasedTransaction(lease, () => {
|
|
1829
|
-
this.#database
|
|
1830
|
-
.prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
|
|
1831
|
-
SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'resolved' FROM workflow_recovery_events WHERE run_id = ?`)
|
|
1832
|
-
.run(runId, preparationKey, runId);
|
|
1833
|
-
this.#database
|
|
1834
|
-
.prepare(
|
|
1835
|
-
"DELETE FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
|
|
1836
|
-
)
|
|
1837
|
-
.run(runId, preparationKey);
|
|
1838
|
-
});
|
|
1839
|
-
if (priorDiagnostic) delete priorDiagnostic.recovery;
|
|
1840
|
-
const retainedDiagnostic =
|
|
1841
|
-
priorDiagnostic && Object.keys(priorDiagnostic).length
|
|
1842
|
-
? priorDiagnostic
|
|
1843
|
-
: null;
|
|
1844
|
-
this.#setTask(
|
|
1845
|
-
runId,
|
|
1846
|
-
row.task_id,
|
|
1847
|
-
{ attemptDiagnostic: retainedDiagnostic },
|
|
1848
|
-
lease,
|
|
1849
|
-
);
|
|
1850
|
-
row.attempt_diagnostic_json = retainedDiagnostic
|
|
1851
|
-
? JSON.stringify(retainedDiagnostic)
|
|
1852
|
-
: null;
|
|
1853
|
-
continue;
|
|
1854
|
-
}
|
|
1855
|
-
}
|
|
1856
|
-
this.#setTask(
|
|
1857
|
-
runId,
|
|
1858
|
-
row.task_id,
|
|
1859
|
-
{
|
|
1860
|
-
state: "phase-running",
|
|
1861
|
-
phase,
|
|
1862
|
-
pauseCode: artifactCorrection?.code ?? null,
|
|
1863
|
-
contextRequest: requestedContext ?? null,
|
|
1864
|
-
attemptDiagnostic: priorDiagnostic ?? null,
|
|
1865
|
-
queuePosition: null,
|
|
1866
|
-
},
|
|
1867
|
-
lease,
|
|
1868
|
-
);
|
|
1869
|
-
if (
|
|
1870
|
-
!this.#reserveWork(
|
|
1871
|
-
runId,
|
|
1872
|
-
plan,
|
|
1873
|
-
recoveryKey(task, phase, row, current),
|
|
1874
|
-
lease,
|
|
1875
|
-
granted ? operationId : undefined,
|
|
1876
|
-
)
|
|
1877
|
-
) {
|
|
1878
|
-
this.#setTask(
|
|
1879
|
-
runId,
|
|
1880
|
-
row.task_id,
|
|
1881
|
-
{
|
|
1882
|
-
state: "paused",
|
|
1883
|
-
phase,
|
|
1884
|
-
pauseCode: "change-work-budget-exhausted",
|
|
1885
|
-
queuePosition: null,
|
|
1886
|
-
},
|
|
1887
|
-
lease,
|
|
1888
|
-
);
|
|
1889
|
-
return;
|
|
1890
|
-
}
|
|
1891
|
-
let initialReservation = true;
|
|
1892
|
-
let outcome: WorkflowAttemptOutcome;
|
|
1893
|
-
try {
|
|
1894
|
-
outcome = await this.#worker.runAttempt({
|
|
1895
|
-
runId,
|
|
1896
|
-
operationId,
|
|
1897
|
-
deliveryRevision: row.delivery_revision,
|
|
1898
|
-
taskId: row.task_id,
|
|
1899
|
-
phase,
|
|
1900
|
-
task: structuredClone(task),
|
|
1901
|
-
contextReadPaths: this.#contextReads(runId, row.task_id),
|
|
1902
|
-
plan: structuredClone(plan),
|
|
1903
|
-
...(current.baseline_workspace_revision
|
|
1904
|
-
? { baselineRevisionId: current.baseline_workspace_revision }
|
|
1905
|
-
: {}),
|
|
1906
|
-
...(current.current_workspace_revision
|
|
1907
|
-
? {
|
|
1908
|
-
currentWorkspaceRevisionId: current.current_workspace_revision,
|
|
1909
|
-
}
|
|
1910
|
-
: {}),
|
|
1911
|
-
...(row.route_id || current.route_id
|
|
1912
|
-
? { routeId: row.route_id ?? current.route_id ?? undefined }
|
|
1913
|
-
: {}),
|
|
1914
|
-
...(row.route_fingerprint || current.route_fingerprint
|
|
1915
|
-
? {
|
|
1916
|
-
routeFingerprint:
|
|
1917
|
-
row.route_fingerprint ??
|
|
1918
|
-
current.route_fingerprint ??
|
|
1919
|
-
undefined,
|
|
1920
|
-
}
|
|
1921
|
-
: {}),
|
|
1922
|
-
...(repair ? { repair } : {}),
|
|
1923
|
-
...(artifactCorrection ? { artifactCorrection } : {}),
|
|
1924
|
-
...(requestedContext ? { contextRequest: requestedContext } : {}),
|
|
1925
|
-
...(granted ? { additionalAttempt: true } : {}),
|
|
1926
|
-
...(verificationOnly ? { verificationOnly: true } : {}),
|
|
1927
|
-
...(activeRecovery
|
|
1928
|
-
? { recoveryFeedback: activeRecovery.feedback }
|
|
1929
|
-
: {}),
|
|
1930
|
-
signal,
|
|
1931
|
-
recoveryDecision: (request) => {
|
|
1932
|
-
signal.throwIfAborted();
|
|
1933
|
-
this.#runStore.assertLease(lease);
|
|
1934
|
-
return decideRecoveryAction(plan, request, {
|
|
1935
|
-
additionalAttempt: granted,
|
|
1936
|
-
verificationOnly,
|
|
1937
|
-
});
|
|
1938
|
-
},
|
|
1939
|
-
reserveCandidate: () => {
|
|
1940
|
-
signal.throwIfAborted();
|
|
1941
|
-
if (verificationOnly) return false;
|
|
1942
|
-
if (initialReservation) {
|
|
1943
|
-
initialReservation = false;
|
|
1944
|
-
return true;
|
|
1945
|
-
}
|
|
1946
|
-
if (granted) return false;
|
|
1947
|
-
return this.#reserveWork(
|
|
1948
|
-
runId,
|
|
1949
|
-
plan,
|
|
1950
|
-
recoveryKey(task, phase, row, current),
|
|
1951
|
-
lease,
|
|
1952
|
-
);
|
|
1953
|
-
},
|
|
1954
|
-
onActivity: (event) => {
|
|
1955
|
-
const projection = this.#runStore.status(runId);
|
|
1956
|
-
emitWorkflowActivity(onActivity, {
|
|
1957
|
-
...event,
|
|
1958
|
-
stage: projection.stage,
|
|
1959
|
-
runId,
|
|
1960
|
-
...(projection.change ? { change: projection.change } : {}),
|
|
1961
|
-
taskId: row.task_id,
|
|
1962
|
-
phase,
|
|
1963
|
-
objective: task.objective,
|
|
1964
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
1965
|
-
});
|
|
1966
|
-
},
|
|
1967
|
-
});
|
|
1968
|
-
} catch (error) {
|
|
1969
|
-
if (isCancellationException(error, signal)) {
|
|
1970
|
-
outcome = { kind: "operation-cancelled", code: "cancelled" };
|
|
1971
|
-
} else {
|
|
1972
|
-
throw error;
|
|
1973
|
-
}
|
|
1974
|
-
}
|
|
1975
|
-
if (!isRecord(outcome) || typeof outcome.kind !== "string") {
|
|
1976
|
-
throw new Error("workflow-attempt-outcome-invalid");
|
|
1977
|
-
}
|
|
1978
|
-
this.#recordWorkspaceFacts(runId, outcome, lease);
|
|
1979
|
-
this.#recordRouteFacts(runId, row.task_id, outcome, lease);
|
|
1980
|
-
if (outcome.routeId && outcome.routeFingerprint) {
|
|
1981
|
-
row.route_id = outcome.routeId;
|
|
1982
|
-
row.route_fingerprint = outcome.routeFingerprint;
|
|
1983
|
-
}
|
|
1984
|
-
if (
|
|
1985
|
-
outcome.kind !== "phase-committed" &&
|
|
1986
|
-
outcome.kind !== "operation-cancelled" &&
|
|
1987
|
-
outcome.verification
|
|
1988
|
-
) {
|
|
1989
|
-
this.#setVerificationStatus(runId, outcome.verification, lease);
|
|
1990
|
-
}
|
|
1991
|
-
if (
|
|
1992
|
-
outcome.kind !== "phase-committed" &&
|
|
1993
|
-
outcome.kind !== "operation-cancelled" &&
|
|
1994
|
-
outcome.contextRequest
|
|
1995
|
-
) {
|
|
1996
|
-
requestedContext = normalizeWorkflowContextRequest(
|
|
1997
|
-
outcome.contextRequest,
|
|
1998
|
-
);
|
|
1999
|
-
}
|
|
2000
|
-
if (
|
|
2001
|
-
outcome.kind !== "phase-committed" &&
|
|
2002
|
-
outcome.kind !== "operation-cancelled" &&
|
|
2003
|
-
outcome.attemptDiagnostic
|
|
2004
|
-
) {
|
|
2005
|
-
const previous = row.attempt_diagnostic_json
|
|
2006
|
-
? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
|
|
2007
|
-
: undefined;
|
|
2008
|
-
const fingerprint = hash(
|
|
2009
|
-
outcome.code,
|
|
2010
|
-
outcome.attemptDiagnostic.finalCategory ?? "unknown",
|
|
2011
|
-
outcome.attemptDiagnostic.schema ?? "unknown",
|
|
2012
|
-
JSON.stringify(outcome.attemptDiagnostic.identityMismatch ?? []),
|
|
2013
|
-
).slice(0, 16);
|
|
2014
|
-
const sameFailure = previous?.fingerprint === fingerprint;
|
|
2015
|
-
const sameFailureCount = sameFailure
|
|
2016
|
-
? (previous?.sameFailureCount ?? 1) + 1
|
|
2017
|
-
: 1;
|
|
2018
|
-
const attemptDiagnostic = {
|
|
2019
|
-
...outcome.attemptDiagnostic,
|
|
2020
|
-
...(priorDiagnostic?.recovery
|
|
2021
|
-
? { recovery: priorDiagnostic.recovery }
|
|
2022
|
-
: {}),
|
|
2023
|
-
fingerprint,
|
|
2024
|
-
sameFailureCount,
|
|
2025
|
-
...(sameFailureCount > 1
|
|
2026
|
-
? { action: "rebind-or-revise-delivery" as const }
|
|
2027
|
-
: {}),
|
|
2028
|
-
};
|
|
2029
|
-
// Only parent preparation can contribute original-baseline authority.
|
|
2030
|
-
delete attemptDiagnostic.prerequisite;
|
|
2031
|
-
this.#setTask(runId, row.task_id, { attemptDiagnostic }, lease);
|
|
2032
|
-
row.attempt_diagnostic_json = JSON.stringify(attemptDiagnostic);
|
|
2033
|
-
}
|
|
2034
|
-
if (
|
|
2035
|
-
outcome.kind === "paused" &&
|
|
2036
|
-
outcome.code === "repair-attempts-exhausted"
|
|
2037
|
-
) {
|
|
2038
|
-
const maxAttempts = plan.verification.repair.maxAttempts;
|
|
2039
|
-
const failures = Math.max(
|
|
2040
|
-
maxAttempts,
|
|
2041
|
-
(activeRecovery?.failures ?? 0) +
|
|
2042
|
-
(granted || verificationOnly ? 1 : 0),
|
|
2043
|
-
);
|
|
2044
|
-
const diagnostic: SafeAttemptDiagnostic = {
|
|
2045
|
-
recovery: {
|
|
2046
|
-
key: recoveryKey(task, phase, row, this.#engineRun(runId)),
|
|
2047
|
-
failures,
|
|
2048
|
-
feedback: {
|
|
2049
|
-
code: outcome.code,
|
|
2050
|
-
attempt: failures + 1,
|
|
2051
|
-
maxAttempts,
|
|
2052
|
-
strategy: "repair-verification",
|
|
2053
|
-
},
|
|
2054
|
-
},
|
|
2055
|
-
};
|
|
2056
|
-
this.#setTask(
|
|
2057
|
-
runId,
|
|
2058
|
-
row.task_id,
|
|
2059
|
-
{ attemptDiagnostic: diagnostic },
|
|
2060
|
-
lease,
|
|
2061
|
-
);
|
|
2062
|
-
row.attempt_diagnostic_json = JSON.stringify(diagnostic);
|
|
2063
|
-
}
|
|
2064
|
-
if (
|
|
2065
|
-
outcome.kind === "retryable" &&
|
|
2066
|
-
["artifact", "stale", "verification"].includes(
|
|
2067
|
-
outcome.retryPolicy ?? "",
|
|
2068
|
-
)
|
|
2069
|
-
) {
|
|
2070
|
-
const key = recoveryKey(task, phase, row, this.#engineRun(runId));
|
|
2071
|
-
const failures =
|
|
2072
|
-
(activeRecovery?.key === key ? activeRecovery.failures : 0) + 1;
|
|
2073
|
-
const strategy =
|
|
2074
|
-
outcome.code === "needs-task-split" ||
|
|
2075
|
-
outcome.code === "task-split-needed"
|
|
2076
|
-
? ("compact-patch" as const)
|
|
2077
|
-
: outcome.retryPolicy === "stale"
|
|
2078
|
-
? ("refresh-candidate" as const)
|
|
2079
|
-
: outcome.retryPolicy === "verification"
|
|
2080
|
-
? ("repair-verification" as const)
|
|
2081
|
-
: ("revise-candidate" as const);
|
|
2082
|
-
const diagnostic: SafeAttemptDiagnostic = {
|
|
2083
|
-
...this.#safeAttemptDiagnostic(runId, row),
|
|
2084
|
-
recovery: {
|
|
2085
|
-
key,
|
|
2086
|
-
failures,
|
|
2087
|
-
feedback: {
|
|
2088
|
-
code: outcome.code,
|
|
2089
|
-
attempt: failures + 1,
|
|
2090
|
-
maxAttempts: maxRecoveryAttempts,
|
|
2091
|
-
strategy,
|
|
2092
|
-
...(outcome.verification?.failureIdentities
|
|
2093
|
-
? { failureIdentities: outcome.verification.failureIdentities }
|
|
2094
|
-
: {}),
|
|
2095
|
-
},
|
|
2096
|
-
},
|
|
2097
|
-
};
|
|
2098
|
-
this.#setTask(
|
|
2099
|
-
runId,
|
|
2100
|
-
row.task_id,
|
|
2101
|
-
{ attemptDiagnostic: diagnostic },
|
|
2102
|
-
lease,
|
|
2103
|
-
);
|
|
2104
|
-
row.attempt_diagnostic_json = JSON.stringify(diagnostic);
|
|
2105
|
-
if (outcome.code === "red-artifact-constraint") {
|
|
2106
|
-
redArtifactCorrection = true;
|
|
2107
|
-
correctionContextRequest = outcome.contextRequest
|
|
2108
|
-
? normalizeWorkflowContextRequest(outcome.contextRequest)
|
|
2109
|
-
: correctionContextRequest;
|
|
2110
|
-
requestedContext = correctionContextRequest ?? requestedContext;
|
|
2111
|
-
}
|
|
2112
|
-
artifactAttempts = Math.max(artifactAttempts + 1, failures);
|
|
2113
|
-
const maxAttempts = maxRecoveryAttempts;
|
|
2114
|
-
if (artifactAttempts < maxAttempts) {
|
|
2115
|
-
this.#setTask(
|
|
2116
|
-
runId,
|
|
2117
|
-
row.task_id,
|
|
2118
|
-
{
|
|
2119
|
-
state: "retryable",
|
|
2120
|
-
phase,
|
|
2121
|
-
pauseCode: redArtifactCorrection
|
|
2122
|
-
? "red-artifact-constraint"
|
|
2123
|
-
: outcome.code,
|
|
2124
|
-
contextRequest: requestedContext ?? null,
|
|
2125
|
-
queuePosition: null,
|
|
2126
|
-
},
|
|
2127
|
-
lease,
|
|
2128
|
-
);
|
|
2129
|
-
emitWorkflowActivity(onActivity, {
|
|
2130
|
-
state: "retrying",
|
|
2131
|
-
stage: this.#runStore.status(runId).stage,
|
|
2132
|
-
runId,
|
|
2133
|
-
taskId: row.task_id,
|
|
2134
|
-
phase,
|
|
2135
|
-
objective: task.objective,
|
|
2136
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
2137
|
-
code: outcome.code,
|
|
2138
|
-
attempt: artifactAttempts,
|
|
2139
|
-
maxAttempts,
|
|
2140
|
-
wait: "artifact-correction",
|
|
2141
|
-
});
|
|
2142
|
-
continue;
|
|
2143
|
-
}
|
|
2144
|
-
}
|
|
2145
|
-
if (outcome.kind === "phase-committed") {
|
|
2146
|
-
if (
|
|
2147
|
-
!SHA256.test(outcome.artifactHash) ||
|
|
2148
|
-
!SHA256.test(outcome.isolatedRevisionId) ||
|
|
2149
|
-
(outcome.baselineRevisionId !== undefined &&
|
|
2150
|
-
!SHA256.test(outcome.baselineRevisionId)) ||
|
|
2151
|
-
outcome.classification !== expectedClassification(phase)
|
|
2152
|
-
) {
|
|
2153
|
-
throw new Error("workflow-attempt-fact-invalid");
|
|
2154
|
-
}
|
|
2155
|
-
this.#leasedTransaction(lease, () => {
|
|
2156
|
-
const resolvedKey = recoveryKey(task, phase, row, current);
|
|
2157
|
-
if (this.#recovery(runId, resolvedKey)) {
|
|
2158
|
-
this.#database
|
|
2159
|
-
.prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
|
|
2160
|
-
SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'resolved' FROM workflow_recovery_events WHERE run_id = ?`)
|
|
2161
|
-
.run(runId, resolvedKey, runId);
|
|
2162
|
-
}
|
|
2163
|
-
this.#database
|
|
2164
|
-
.prepare(
|
|
2165
|
-
"DELETE FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
|
|
2166
|
-
)
|
|
2167
|
-
.run(runId, recoveryKey(task, phase, row, current));
|
|
2168
|
-
});
|
|
2169
|
-
if (repair) {
|
|
2170
|
-
this.#setVerificationStatus(runId, null, lease);
|
|
2171
|
-
this.#setTask(
|
|
2172
|
-
runId,
|
|
2173
|
-
row.task_id,
|
|
2174
|
-
{
|
|
2175
|
-
state: "verified",
|
|
2176
|
-
phase,
|
|
2177
|
-
pauseCode: null,
|
|
2178
|
-
contextRequest: null,
|
|
2179
|
-
attemptDiagnostic: null,
|
|
2180
|
-
queuePosition: null,
|
|
2181
|
-
},
|
|
2182
|
-
lease,
|
|
2183
|
-
);
|
|
2184
|
-
return;
|
|
2185
|
-
}
|
|
2186
|
-
artifactAttempts = 0;
|
|
2187
|
-
row.attempt_diagnostic_json = null;
|
|
2188
|
-
redArtifactCorrection = false;
|
|
2189
|
-
requestedContext = undefined;
|
|
2190
|
-
correctionContextRequest = undefined;
|
|
2191
|
-
phaseIndex += 1;
|
|
2192
|
-
if (phaseIndex >= orderedPhases.length) {
|
|
2193
|
-
this.#setTask(
|
|
2194
|
-
runId,
|
|
2195
|
-
row.task_id,
|
|
2196
|
-
{
|
|
2197
|
-
state: "verified",
|
|
2198
|
-
phase,
|
|
2199
|
-
pauseCode: null,
|
|
2200
|
-
contextRequest: null,
|
|
2201
|
-
attemptDiagnostic: null,
|
|
2202
|
-
queuePosition: null,
|
|
2203
|
-
},
|
|
2204
|
-
lease,
|
|
2205
|
-
);
|
|
2206
|
-
return;
|
|
2207
|
-
}
|
|
2208
|
-
this.#setTask(
|
|
2209
|
-
runId,
|
|
2210
|
-
row.task_id,
|
|
2211
|
-
{
|
|
2212
|
-
state: "phase-ready",
|
|
2213
|
-
phase: orderedPhases[phaseIndex],
|
|
2214
|
-
pauseCode: null,
|
|
2215
|
-
contextRequest: null,
|
|
2216
|
-
attemptDiagnostic: null,
|
|
2217
|
-
queuePosition: null,
|
|
2218
|
-
},
|
|
2219
|
-
lease,
|
|
2220
|
-
);
|
|
2221
|
-
continue;
|
|
2222
|
-
}
|
|
2223
|
-
if (outcome.kind === "operation-cancelled") {
|
|
2224
|
-
this.#setTask(
|
|
2225
|
-
runId,
|
|
2226
|
-
row.task_id,
|
|
2227
|
-
{
|
|
2228
|
-
state: "paused",
|
|
2229
|
-
phase,
|
|
2230
|
-
pauseCode: "operation-cancelled",
|
|
2231
|
-
contextRequest: null,
|
|
2232
|
-
queuePosition: null,
|
|
2233
|
-
},
|
|
2234
|
-
lease,
|
|
2235
|
-
);
|
|
2236
|
-
return;
|
|
2237
|
-
}
|
|
2238
|
-
if (
|
|
2239
|
-
outcome.kind === "paused" ||
|
|
2240
|
-
outcome.kind === "retryable" ||
|
|
2241
|
-
outcome.kind === "approval-needed"
|
|
2242
|
-
) {
|
|
2243
|
-
if (typeof outcome.code !== "string" || outcome.code.length === 0) {
|
|
2244
|
-
throw new Error("workflow-attempt-outcome-invalid");
|
|
2245
|
-
}
|
|
2246
|
-
const approvalCodeInvalid =
|
|
2247
|
-
outcome.kind === "approval-needed" &&
|
|
2248
|
-
!isWorkflowApprovalCode(outcome.code);
|
|
2249
|
-
this.#setTask(
|
|
2250
|
-
runId,
|
|
2251
|
-
row.task_id,
|
|
2252
|
-
{
|
|
2253
|
-
state: approvalCodeInvalid ? "paused" : outcome.kind,
|
|
2254
|
-
phase,
|
|
2255
|
-
pauseCode: approvalCodeInvalid
|
|
2256
|
-
? "approval-code-invalid"
|
|
2257
|
-
: outcome.code,
|
|
2258
|
-
contextRequest: approvalCodeInvalid
|
|
2259
|
-
? null
|
|
2260
|
-
: (requestedContext ?? null),
|
|
2261
|
-
queuePosition: null,
|
|
2262
|
-
},
|
|
2263
|
-
lease,
|
|
2264
|
-
);
|
|
2265
|
-
return;
|
|
2266
|
-
}
|
|
2267
|
-
throw new Error("workflow-attempt-outcome-invalid");
|
|
2268
|
-
}
|
|
2269
|
-
}
|
|
2270
|
-
|
|
2271
|
-
#firstPaused(rows: EngineTaskRow[]): EngineTaskRow | undefined {
|
|
2272
|
-
return firstPausedTask(rows);
|
|
2273
|
-
}
|
|
2274
|
-
|
|
2275
|
-
#recoverLegacyContextApproval(runId: string): boolean {
|
|
2276
|
-
const projection = this.#runStore.status(runId);
|
|
2277
|
-
if (!["approval-needed", "paused"].includes(projection.state)) return false;
|
|
2278
|
-
const rows = this.#tasks(runId);
|
|
2279
|
-
const approvalRows = rows.filter((row) => row.state === "approval-needed");
|
|
2280
|
-
if (approvalRows.length === 0) return false;
|
|
2281
|
-
const reclassified = rows.flatMap((row) => {
|
|
2282
|
-
if (
|
|
2283
|
-
row.state !== "approval-needed" ||
|
|
2284
|
-
row.pause_code !== "boundary-review-needed" ||
|
|
2285
|
-
row.context_request_json === null
|
|
2286
|
-
) {
|
|
2287
|
-
return [];
|
|
2288
|
-
}
|
|
2289
|
-
try {
|
|
2290
|
-
const request = normalizeWorkflowContextRequest(
|
|
2291
|
-
JSON.parse(row.context_request_json) as unknown,
|
|
2292
|
-
);
|
|
2293
|
-
const classified = classifyPersistedContextRequest(
|
|
2294
|
-
request,
|
|
2295
|
-
this.#taskPlan(row),
|
|
2296
|
-
row.phase,
|
|
2297
|
-
);
|
|
2298
|
-
if (classified.kind === "approval-needed") return [];
|
|
2299
|
-
return [
|
|
2300
|
-
{
|
|
2301
|
-
row,
|
|
2302
|
-
state: classified.kind === "retryable" ? "retryable" : "paused",
|
|
2303
|
-
code: classified.code,
|
|
2304
|
-
contextRequest: JSON.stringify(
|
|
2305
|
-
normalizeWorkflowContextRequest(classified.contextRequest),
|
|
2306
|
-
),
|
|
2307
|
-
},
|
|
2308
|
-
];
|
|
2309
|
-
} catch {
|
|
2310
|
-
return [];
|
|
2311
|
-
}
|
|
2312
|
-
});
|
|
2313
|
-
const reclassifiedTaskIds = new Set(
|
|
2314
|
-
reclassified.map((entry) => entry.row.task_id),
|
|
2315
|
-
);
|
|
2316
|
-
const retainedApprovals = approvalRows.filter(
|
|
2317
|
-
(row) => !reclassifiedTaskIds.has(row.task_id),
|
|
2318
|
-
);
|
|
2319
|
-
const updateTasks = (database: DatabaseSync) => {
|
|
2320
|
-
for (const entry of reclassified) {
|
|
2321
|
-
const updated = database
|
|
2322
|
-
.prepare(
|
|
2323
|
-
`UPDATE workflow_engine_tasks
|
|
2324
|
-
SET state = ?, phase = ?, pause_code = ?, context_request_json = ?,
|
|
2325
|
-
queue_position = NULL
|
|
2326
|
-
WHERE run_id = ? AND task_id = ?
|
|
2327
|
-
AND state = 'approval-needed'
|
|
2328
|
-
AND pause_code = 'boundary-review-needed'`,
|
|
2329
|
-
)
|
|
2330
|
-
.run(
|
|
2331
|
-
entry.state,
|
|
2332
|
-
entry.row.phase,
|
|
2333
|
-
entry.code,
|
|
2334
|
-
entry.contextRequest,
|
|
2335
|
-
runId,
|
|
2336
|
-
entry.row.task_id,
|
|
2337
|
-
);
|
|
2338
|
-
if (Number(updated.changes) !== 1) {
|
|
2339
|
-
throw new Error("workflow-legacy-context-race");
|
|
2340
|
-
}
|
|
2341
|
-
}
|
|
2342
|
-
};
|
|
2343
|
-
const transitionWithUpdates = (
|
|
2344
|
-
to: "paused" | "approval-needed",
|
|
2345
|
-
code: string,
|
|
2346
|
-
) =>
|
|
2347
|
-
this.#runStore.transitionAtomically(
|
|
2348
|
-
{
|
|
2349
|
-
runId,
|
|
2350
|
-
to,
|
|
2351
|
-
operationId: `legacy-context-${hash(
|
|
2352
|
-
runId,
|
|
2353
|
-
String(projection.sequence),
|
|
2354
|
-
to,
|
|
2355
|
-
code,
|
|
2356
|
-
JSON.stringify(
|
|
2357
|
-
reclassified.map((entry) => [entry.row.task_id, entry.code]),
|
|
2358
|
-
),
|
|
2359
|
-
).slice(0, 40)}`,
|
|
2360
|
-
code,
|
|
2361
|
-
},
|
|
2362
|
-
updateTasks,
|
|
2363
|
-
);
|
|
2364
|
-
if (retainedApprovals.length > 0) {
|
|
2365
|
-
if (projection.state === "approval-needed") {
|
|
2366
|
-
if (reclassified.length === 0) return false;
|
|
2367
|
-
this.#transaction(() => updateTasks(this.#database));
|
|
2368
|
-
} else {
|
|
2369
|
-
transitionWithUpdates(
|
|
2370
|
-
"approval-needed",
|
|
2371
|
-
retainedApprovals[0].pause_code ?? "approval-code-invalid",
|
|
2372
|
-
);
|
|
2373
|
-
}
|
|
2374
|
-
return true;
|
|
2375
|
-
}
|
|
2376
|
-
const first = reclassified[0];
|
|
2377
|
-
if (!first) return false;
|
|
2378
|
-
transitionWithUpdates("paused", first.code);
|
|
2379
|
-
return true;
|
|
2380
|
-
}
|
|
2381
|
-
|
|
2382
|
-
async #finishChange(
|
|
2383
|
-
runId: string,
|
|
2384
|
-
operationId: string,
|
|
2385
|
-
signal: AbortSignal,
|
|
2386
|
-
lease: OperationLease,
|
|
2387
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
2388
|
-
): Promise<Record<string, unknown>> {
|
|
2389
|
-
const plan = this.#currentPlan(runId);
|
|
2390
|
-
let engineRun = this.#engineRun(runId);
|
|
2391
|
-
if (engineRun.current_revision === null)
|
|
2392
|
-
throw new Error("delivery-not-bound");
|
|
2393
|
-
const deliveryRevision = engineRun.current_revision;
|
|
2394
|
-
this.#transition(
|
|
2395
|
-
runId,
|
|
2396
|
-
"change-verifying",
|
|
2397
|
-
`${operationId}:change-verifying`,
|
|
2398
|
-
undefined,
|
|
2399
|
-
lease,
|
|
2400
|
-
);
|
|
2401
|
-
const projection = this.#runStore.status(runId);
|
|
2402
|
-
emitWorkflowActivity(onActivity, {
|
|
2403
|
-
state: "verifying",
|
|
2404
|
-
stage: projection.stage,
|
|
2405
|
-
runId,
|
|
2406
|
-
...(projection.change ? { change: projection.change } : {}),
|
|
2407
|
-
objective: projection.change
|
|
2408
|
-
? `verify change ${projection.change}`
|
|
2409
|
-
: "verify workflow change",
|
|
2410
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
2411
|
-
});
|
|
2412
|
-
const verification = await this.#changeVerifier.verify({
|
|
2413
|
-
taskEvidence: this.#tasks(runId).map((row) => ({
|
|
2414
|
-
taskId: row.task_id,
|
|
2415
|
-
deliveryRevision: row.delivery_revision,
|
|
2416
|
-
})),
|
|
2417
|
-
runId,
|
|
2418
|
-
deliveryRevision,
|
|
2419
|
-
plan: structuredClone(plan),
|
|
2420
|
-
...(engineRun.baseline_workspace_revision
|
|
2421
|
-
? { baselineRevisionId: engineRun.baseline_workspace_revision }
|
|
2422
|
-
: {}),
|
|
2423
|
-
...(engineRun.current_workspace_revision
|
|
2424
|
-
? {
|
|
2425
|
-
currentWorkspaceRevisionId: engineRun.current_workspace_revision,
|
|
2426
|
-
}
|
|
2427
|
-
: {}),
|
|
2428
|
-
signal,
|
|
2429
|
-
});
|
|
2430
|
-
this.#runStore.assertLease(lease);
|
|
2431
|
-
if (verification.kind !== "verified") {
|
|
2432
|
-
if (verification.verification) {
|
|
2433
|
-
this.#setVerificationStatus(runId, verification.verification, lease);
|
|
2434
|
-
}
|
|
2435
|
-
if (
|
|
2436
|
-
verification.code === "verification-repair-required" &&
|
|
2437
|
-
verification.verification?.scope === "change-task-affected" &&
|
|
2438
|
-
verification.verification.attribution === "introduced" &&
|
|
2439
|
-
verification.verification.taskId
|
|
2440
|
-
) {
|
|
2441
|
-
const task = this.#tasks(runId).find(
|
|
2442
|
-
(candidate) =>
|
|
2443
|
-
candidate.task_id === verification.verification?.taskId,
|
|
2444
|
-
);
|
|
2445
|
-
if (task) {
|
|
2446
|
-
const repairKey = recoveryKey(
|
|
2447
|
-
this.#taskPlan(task),
|
|
2448
|
-
task.phase,
|
|
2449
|
-
task,
|
|
2450
|
-
this.#engineRun(runId),
|
|
2451
|
-
);
|
|
2452
|
-
const used = (
|
|
2453
|
-
this.#database
|
|
2454
|
-
.prepare(
|
|
2455
|
-
"SELECT COUNT(*) AS count FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ? AND kind = 'repair'",
|
|
2456
|
-
)
|
|
2457
|
-
.get(runId, repairKey) as { count: number }
|
|
2458
|
-
).count;
|
|
2459
|
-
if (used >= plan.verification.repair.maxAttempts) {
|
|
2460
|
-
this.#setTask(
|
|
2461
|
-
runId,
|
|
2462
|
-
task.task_id,
|
|
2463
|
-
{
|
|
2464
|
-
state: "paused",
|
|
2465
|
-
phase: task.phase,
|
|
2466
|
-
pauseCode: "repair-attempts-exhausted",
|
|
2467
|
-
attemptDiagnostic: {
|
|
2468
|
-
recovery: {
|
|
2469
|
-
key: recoveryKey(
|
|
2470
|
-
this.#taskPlan(task),
|
|
2471
|
-
task.phase,
|
|
2472
|
-
task,
|
|
2473
|
-
this.#engineRun(runId),
|
|
2474
|
-
),
|
|
2475
|
-
failures: used,
|
|
2476
|
-
feedback: {
|
|
2477
|
-
code: "repair-attempts-exhausted",
|
|
2478
|
-
attempt: used + 1,
|
|
2479
|
-
maxAttempts: plan.verification.repair.maxAttempts,
|
|
2480
|
-
strategy: "repair-verification",
|
|
2481
|
-
failureIdentities:
|
|
2482
|
-
verification.verification.failureIdentities,
|
|
2483
|
-
},
|
|
2484
|
-
},
|
|
2485
|
-
},
|
|
2486
|
-
queuePosition: null,
|
|
2487
|
-
},
|
|
2488
|
-
lease,
|
|
2489
|
-
);
|
|
2490
|
-
this.#transition(
|
|
2491
|
-
runId,
|
|
2492
|
-
"paused",
|
|
2493
|
-
`${operationId}:repair-attempts-exhausted:${task.task_id}`,
|
|
2494
|
-
"repair-attempts-exhausted",
|
|
2495
|
-
lease,
|
|
2496
|
-
);
|
|
2497
|
-
return this.#statusByRun(runId);
|
|
2498
|
-
}
|
|
2499
|
-
this.#leasedTransaction(lease, () => {
|
|
2500
|
-
this.#database
|
|
2501
|
-
.prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
|
|
2502
|
-
SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'repair' FROM workflow_recovery_events WHERE run_id = ?`)
|
|
2503
|
-
.run(runId, repairKey, runId);
|
|
2504
|
-
});
|
|
2505
|
-
const taskPlan = this.#taskPlan(task);
|
|
2506
|
-
this.#setTask(
|
|
2507
|
-
runId,
|
|
2508
|
-
task.task_id,
|
|
2509
|
-
{
|
|
2510
|
-
state: "repairable",
|
|
2511
|
-
phase: taskPlan.phases.refactor ? "refactor" : "green",
|
|
2512
|
-
pauseCode: verification.code,
|
|
2513
|
-
queuePosition: null,
|
|
2514
|
-
},
|
|
2515
|
-
lease,
|
|
2516
|
-
);
|
|
2517
|
-
this.#transition(
|
|
2518
|
-
runId,
|
|
2519
|
-
"running",
|
|
2520
|
-
`${operationId}:repairable:${task.task_id}`,
|
|
2521
|
-
undefined,
|
|
2522
|
-
lease,
|
|
2523
|
-
);
|
|
2524
|
-
return this.#advance(runId, operationId, signal, lease, onActivity);
|
|
2525
|
-
}
|
|
2526
|
-
}
|
|
2527
|
-
const approvalCodeInvalid =
|
|
2528
|
-
verification.kind === "approval-needed" &&
|
|
2529
|
-
!isWorkflowApprovalCode(verification.code);
|
|
2530
|
-
const state =
|
|
2531
|
-
verification.kind === "approval-needed" && !approvalCodeInvalid
|
|
2532
|
-
? "approval-needed"
|
|
2533
|
-
: "paused";
|
|
2534
|
-
const code = approvalCodeInvalid
|
|
2535
|
-
? "approval-code-invalid"
|
|
2536
|
-
: verification.code;
|
|
2537
|
-
this.#transition(
|
|
2538
|
-
runId,
|
|
2539
|
-
state,
|
|
2540
|
-
`${operationId}:change-verification-${state}`,
|
|
2541
|
-
code,
|
|
2542
|
-
lease,
|
|
2543
|
-
);
|
|
2544
|
-
return this.#statusByRun(runId);
|
|
2545
|
-
}
|
|
2546
|
-
if (verification.currentWorkspaceRevisionId !== undefined) {
|
|
2547
|
-
const currentWorkspaceRevisionId =
|
|
2548
|
-
verification.currentWorkspaceRevisionId;
|
|
2549
|
-
if (!SHA256.test(currentWorkspaceRevisionId)) {
|
|
2550
|
-
throw new Error("workflow-verification-workspace-fact-invalid");
|
|
2551
|
-
}
|
|
2552
|
-
this.#leasedTransaction(lease, () => {
|
|
2553
|
-
this.#database
|
|
2554
|
-
.prepare(
|
|
2555
|
-
`UPDATE workflow_engine_runs
|
|
2556
|
-
SET current_workspace_revision = ?, cleanup_state = 'retained'
|
|
2557
|
-
WHERE run_id = ?`,
|
|
2558
|
-
)
|
|
2559
|
-
.run(currentWorkspaceRevisionId, runId);
|
|
2560
|
-
});
|
|
2561
|
-
engineRun = this.#engineRun(runId);
|
|
2562
|
-
}
|
|
2563
|
-
this.#transition(
|
|
2564
|
-
runId,
|
|
2565
|
-
"ready-to-apply",
|
|
2566
|
-
`${operationId}:ready-to-apply`,
|
|
2567
|
-
undefined,
|
|
2568
|
-
lease,
|
|
2569
|
-
);
|
|
2570
|
-
if (!this.#application) {
|
|
2571
|
-
this.#transition(
|
|
2572
|
-
runId,
|
|
2573
|
-
"paused",
|
|
2574
|
-
`${operationId}:application-unavailable`,
|
|
2575
|
-
"application-unavailable",
|
|
2576
|
-
lease,
|
|
2577
|
-
);
|
|
2578
|
-
return this.#statusByRun(runId);
|
|
2579
|
-
}
|
|
2580
|
-
const transactionId = `apply-${hash(
|
|
2581
|
-
runId,
|
|
2582
|
-
String(deliveryRevision),
|
|
2583
|
-
operationId,
|
|
2584
|
-
).slice(0, 40)}`;
|
|
2585
|
-
this.#leasedTransaction(lease, () => {
|
|
2586
|
-
this.#database
|
|
2587
|
-
.prepare(
|
|
2588
|
-
`UPDATE workflow_engine_runs SET transaction_id = ? WHERE run_id = ?`,
|
|
2589
|
-
)
|
|
2590
|
-
.run(transactionId, runId);
|
|
2591
|
-
});
|
|
2592
|
-
const applicationInput = {
|
|
2593
|
-
transactionId,
|
|
2594
|
-
runId,
|
|
2595
|
-
consumerRoot: this.#consumerRoot,
|
|
2596
|
-
deliveryRevision,
|
|
2597
|
-
plan: structuredClone(plan),
|
|
2598
|
-
contextReadPaths: this.#contextReads(runId),
|
|
2599
|
-
...(engineRun.baseline_workspace_revision
|
|
2600
|
-
? { baselineRevisionId: engineRun.baseline_workspace_revision }
|
|
2601
|
-
: {}),
|
|
2602
|
-
...(engineRun.current_workspace_revision
|
|
2603
|
-
? {
|
|
2604
|
-
currentWorkspaceRevisionId: engineRun.current_workspace_revision,
|
|
2605
|
-
}
|
|
2606
|
-
: {}),
|
|
2607
|
-
signal,
|
|
2608
|
-
};
|
|
2609
|
-
let application: Record<string, unknown>;
|
|
2610
|
-
if (this.#application.prepare && this.#application.applyPrepared) {
|
|
2611
|
-
const prepared = await this.#application.prepare(applicationInput);
|
|
2612
|
-
this.#runStore.assertLease(lease);
|
|
2613
|
-
if (prepared.state !== "prepared") {
|
|
2614
|
-
return this.#settleApplication(
|
|
2615
|
-
runId,
|
|
2616
|
-
operationId,
|
|
2617
|
-
prepared,
|
|
2618
|
-
lease,
|
|
2619
|
-
onActivity,
|
|
2620
|
-
);
|
|
2621
|
-
}
|
|
2622
|
-
this.#transition(
|
|
2623
|
-
runId,
|
|
2624
|
-
"applying",
|
|
2625
|
-
`${operationId}:applying`,
|
|
2626
|
-
undefined,
|
|
2627
|
-
lease,
|
|
2628
|
-
);
|
|
2629
|
-
emitWorkflowActivity(onActivity, {
|
|
2630
|
-
state: "applying",
|
|
2631
|
-
stage: projection.stage,
|
|
2632
|
-
runId,
|
|
2633
|
-
...(projection.change ? { change: projection.change } : {}),
|
|
2634
|
-
objective: projection.change
|
|
2635
|
-
? `apply change ${projection.change}`
|
|
2636
|
-
: "apply workflow change",
|
|
2637
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
2638
|
-
});
|
|
2639
|
-
const active = this.#active.get(runId);
|
|
2640
|
-
if (active) active.kind = "applying";
|
|
2641
|
-
try {
|
|
2642
|
-
application = await this.#application.applyPrepared(applicationInput);
|
|
2643
|
-
} catch (error) {
|
|
2644
|
-
this.#runStore.assertLease(lease);
|
|
2645
|
-
if (this.#runStore.status(runId).state === "applying")
|
|
2646
|
-
this.#transition(
|
|
2647
|
-
runId,
|
|
2648
|
-
"recovering",
|
|
2649
|
-
`${operationId}:application-interrupted`,
|
|
2650
|
-
"operation-interrupted",
|
|
2651
|
-
lease,
|
|
2652
|
-
);
|
|
2653
|
-
throw error;
|
|
2654
|
-
}
|
|
2655
|
-
} else {
|
|
2656
|
-
this.#transition(
|
|
2657
|
-
runId,
|
|
2658
|
-
"applying",
|
|
2659
|
-
`${operationId}:applying`,
|
|
2660
|
-
undefined,
|
|
2661
|
-
lease,
|
|
2662
|
-
);
|
|
2663
|
-
emitWorkflowActivity(onActivity, {
|
|
2664
|
-
state: "applying",
|
|
2665
|
-
stage: projection.stage,
|
|
2666
|
-
runId,
|
|
2667
|
-
...(projection.change ? { change: projection.change } : {}),
|
|
2668
|
-
objective: projection.change
|
|
2669
|
-
? `apply change ${projection.change}`
|
|
2670
|
-
: "apply workflow change",
|
|
2671
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
2672
|
-
});
|
|
2673
|
-
const active = this.#active.get(runId);
|
|
2674
|
-
if (active) active.kind = "applying";
|
|
2675
|
-
application = await this.#application.begin(applicationInput);
|
|
2676
|
-
}
|
|
2677
|
-
this.#runStore.assertLease(lease);
|
|
2678
|
-
return this.#settleApplication(
|
|
2679
|
-
runId,
|
|
2680
|
-
operationId,
|
|
2681
|
-
application,
|
|
2682
|
-
lease,
|
|
2683
|
-
onActivity,
|
|
2684
|
-
);
|
|
2685
|
-
}
|
|
2686
|
-
|
|
2687
|
-
#settleApplication(
|
|
2688
|
-
runId: string,
|
|
2689
|
-
operationId: string,
|
|
2690
|
-
outcome: Record<string, unknown>,
|
|
2691
|
-
lease: OperationLease,
|
|
2692
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
2693
|
-
): Record<string, unknown> {
|
|
2694
|
-
const state = outcome.state;
|
|
2695
|
-
if (state === "completed") {
|
|
2696
|
-
const current = this.#runStore.status(runId);
|
|
2697
|
-
if (current.state === "recovering") {
|
|
2698
|
-
this.#transition(
|
|
2699
|
-
runId,
|
|
2700
|
-
"completed",
|
|
2701
|
-
`${operationId}:recovered-complete`,
|
|
2702
|
-
undefined,
|
|
2703
|
-
lease,
|
|
2704
|
-
);
|
|
2705
|
-
} else if (
|
|
2706
|
-
current.state === "applying" ||
|
|
2707
|
-
current.state === "ready-to-apply"
|
|
2708
|
-
) {
|
|
2709
|
-
this.#transition(
|
|
2710
|
-
runId,
|
|
2711
|
-
"completed",
|
|
2712
|
-
`${operationId}:apply-complete`,
|
|
2713
|
-
undefined,
|
|
2714
|
-
lease,
|
|
2715
|
-
);
|
|
2716
|
-
}
|
|
2717
|
-
this.#cleanupTerminal(runId, operationId, lease);
|
|
2718
|
-
} else if (state === "paused") {
|
|
2719
|
-
if (this.#runStore.status(runId).state === "applying") {
|
|
2720
|
-
this.#transition(
|
|
2721
|
-
runId,
|
|
2722
|
-
"recovering",
|
|
2723
|
-
`${operationId}:apply-recovering`,
|
|
2724
|
-
typeof outcome.code === "string" ? outcome.code : "recovery-required",
|
|
2725
|
-
lease,
|
|
2726
|
-
);
|
|
2727
|
-
const recovering = this.#runStore.status(runId);
|
|
2728
|
-
emitWorkflowActivity(onActivity, {
|
|
2729
|
-
state: "recovering",
|
|
2730
|
-
stage: recovering.stage,
|
|
2731
|
-
runId,
|
|
2732
|
-
...(recovering.change ? { change: recovering.change } : {}),
|
|
2733
|
-
objective: recovering.change
|
|
2734
|
-
? `recover change ${recovering.change}`
|
|
2735
|
-
: "recover workflow application",
|
|
2736
|
-
code:
|
|
2737
|
-
typeof outcome.code === "string"
|
|
2738
|
-
? outcome.code
|
|
2739
|
-
: "recovery-required",
|
|
2740
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
2741
|
-
});
|
|
2742
|
-
}
|
|
2743
|
-
this.#transition(
|
|
2744
|
-
runId,
|
|
2745
|
-
"paused",
|
|
2746
|
-
`${operationId}:apply-paused`,
|
|
2747
|
-
typeof outcome.code === "string" ? outcome.code : "recovery-required",
|
|
2748
|
-
lease,
|
|
2749
|
-
);
|
|
2750
|
-
} else if (state === "discarded") {
|
|
2751
|
-
if (this.#runStore.status(runId).state === "applying") {
|
|
2752
|
-
this.#transition(
|
|
2753
|
-
runId,
|
|
2754
|
-
"recovering",
|
|
2755
|
-
`${operationId}:discard-recovering`,
|
|
2756
|
-
"run-discarded",
|
|
2757
|
-
lease,
|
|
2758
|
-
);
|
|
2759
|
-
const recovering = this.#runStore.status(runId);
|
|
2760
|
-
emitWorkflowActivity(onActivity, {
|
|
2761
|
-
state: "recovering",
|
|
2762
|
-
stage: recovering.stage,
|
|
2763
|
-
runId,
|
|
2764
|
-
...(recovering.change ? { change: recovering.change } : {}),
|
|
2765
|
-
objective: recovering.change
|
|
2766
|
-
? `recover change ${recovering.change}`
|
|
2767
|
-
: "recover workflow application",
|
|
2768
|
-
code: "run-discarded",
|
|
2769
|
-
legalCommands: ["status"],
|
|
2770
|
-
});
|
|
2771
|
-
}
|
|
2772
|
-
this.#transition(
|
|
2773
|
-
runId,
|
|
2774
|
-
"discarded",
|
|
2775
|
-
`${operationId}:apply-discarded`,
|
|
2776
|
-
undefined,
|
|
2777
|
-
lease,
|
|
2778
|
-
);
|
|
2779
|
-
this.#cleanupTerminal(runId, operationId, lease);
|
|
2780
|
-
} else if (state === "recovering") {
|
|
2781
|
-
this.#transition(
|
|
2782
|
-
runId,
|
|
2783
|
-
"recovering",
|
|
2784
|
-
`${operationId}:application-recovering`,
|
|
2785
|
-
typeof outcome.code === "string" ? outcome.code : "recovery-required",
|
|
2786
|
-
lease,
|
|
2787
|
-
);
|
|
2788
|
-
const recovering = this.#runStore.status(runId);
|
|
2789
|
-
emitWorkflowActivity(onActivity, {
|
|
2790
|
-
state: "recovering",
|
|
2791
|
-
stage: recovering.stage,
|
|
2792
|
-
runId,
|
|
2793
|
-
...(recovering.change ? { change: recovering.change } : {}),
|
|
2794
|
-
objective: recovering.change
|
|
2795
|
-
? `recover change ${recovering.change}`
|
|
2796
|
-
: "recover workflow application",
|
|
2797
|
-
code:
|
|
2798
|
-
typeof outcome.code === "string" ? outcome.code : "recovery-required",
|
|
2799
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
2800
|
-
});
|
|
2801
|
-
} else {
|
|
2802
|
-
throw new Error("workflow-application-outcome-invalid");
|
|
2803
|
-
}
|
|
2804
|
-
return this.#statusByRun(runId);
|
|
2805
|
-
}
|
|
2806
|
-
|
|
2807
|
-
#applicationContext(runId: string): WorkflowApplicationContext {
|
|
2808
|
-
const current = this.#engineRun(runId);
|
|
2809
|
-
if (current.current_revision === null)
|
|
2810
|
-
throw new Error("delivery-not-bound");
|
|
2811
|
-
return {
|
|
2812
|
-
runId,
|
|
2813
|
-
deliveryRevision: current.current_revision,
|
|
2814
|
-
plan: structuredClone(this.#currentPlan(runId)),
|
|
2815
|
-
contextReadPaths: this.#contextReads(runId),
|
|
2816
|
-
...(current.baseline_workspace_revision
|
|
2817
|
-
? { baselineRevisionId: current.baseline_workspace_revision }
|
|
2818
|
-
: {}),
|
|
2819
|
-
...(current.current_workspace_revision
|
|
2820
|
-
? { currentWorkspaceRevisionId: current.current_workspace_revision }
|
|
2821
|
-
: {}),
|
|
2822
|
-
};
|
|
2823
|
-
}
|
|
2824
|
-
|
|
2825
|
-
#cleanupTerminal(
|
|
2826
|
-
runId: string,
|
|
2827
|
-
_currentOperationId: string,
|
|
2828
|
-
lease?: OperationLease,
|
|
2829
|
-
): void {
|
|
2830
|
-
if (lease) this.#runStore.assertLease(lease);
|
|
2831
|
-
this.#lifecycle?.cleanup(runId);
|
|
2832
|
-
const cleanup = () => {
|
|
2833
|
-
this.#database
|
|
2834
|
-
.prepare("DELETE FROM workflow_engine_tasks WHERE run_id = ?")
|
|
2835
|
-
.run(runId);
|
|
2836
|
-
this.#database
|
|
2837
|
-
.prepare("DELETE FROM workflow_engine_deliveries WHERE run_id = ?")
|
|
2838
|
-
.run(runId);
|
|
2839
|
-
this.#database
|
|
2840
|
-
.prepare(`DELETE FROM workspace_revisions WHERE owner_run_id = ?`)
|
|
2841
|
-
.run(runId);
|
|
2842
|
-
this.#database
|
|
2843
|
-
.prepare(`DELETE FROM apply_transactions WHERE owner_run_id = ?`)
|
|
2844
|
-
.run(runId);
|
|
2845
|
-
this.#database
|
|
2846
|
-
.prepare(
|
|
2847
|
-
`UPDATE workflow_engine_runs
|
|
2848
|
-
SET baseline_workspace_revision = NULL,
|
|
2849
|
-
current_workspace_revision = NULL,
|
|
2850
|
-
transaction_id = NULL, verification_json = NULL,
|
|
2851
|
-
cleanup_state = 'complete'
|
|
2852
|
-
WHERE run_id = ?`,
|
|
2853
|
-
)
|
|
2854
|
-
.run(runId);
|
|
2855
|
-
};
|
|
2856
|
-
if (lease) this.#leasedTransaction(lease, cleanup);
|
|
2857
|
-
else this.#transaction(cleanup);
|
|
2858
|
-
}
|
|
2859
|
-
|
|
2860
|
-
#waitForCapacity(signal: AbortSignal): Promise<void> {
|
|
2861
|
-
if (signal.aborted || this.#activeTasks < LIMITS.maxActiveChildSessions)
|
|
2862
|
-
return Promise.resolve();
|
|
2863
|
-
return new Promise((resolve) => {
|
|
2864
|
-
const wake = () => {
|
|
2865
|
-
this.#capacityWaiters.delete(wake);
|
|
2866
|
-
signal.removeEventListener("abort", wake);
|
|
2867
|
-
resolve();
|
|
2868
|
-
};
|
|
2869
|
-
this.#capacityWaiters.add(wake);
|
|
2870
|
-
signal.addEventListener("abort", wake, { once: true });
|
|
2871
|
-
});
|
|
2872
|
-
}
|
|
2873
|
-
|
|
2874
|
-
async #advance(
|
|
2875
|
-
runId: string,
|
|
2876
|
-
operationId: string,
|
|
2877
|
-
signal: AbortSignal,
|
|
2878
|
-
lease: OperationLease,
|
|
2879
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
2880
|
-
): Promise<Record<string, unknown>> {
|
|
2881
|
-
this.#transition(
|
|
2882
|
-
runId,
|
|
2883
|
-
"running",
|
|
2884
|
-
`${operationId}:running`,
|
|
2885
|
-
undefined,
|
|
2886
|
-
lease,
|
|
2887
|
-
);
|
|
2888
|
-
const run = this.#runStore.status(runId);
|
|
2889
|
-
emitWorkflowActivity(onActivity, {
|
|
2890
|
-
state: "running",
|
|
2891
|
-
stage: run.stage,
|
|
2892
|
-
runId,
|
|
2893
|
-
...(run.change ? { change: run.change } : {}),
|
|
2894
|
-
objective: run.change ? `change ${run.change}` : "workflow run",
|
|
2895
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
2896
|
-
});
|
|
2897
|
-
const existingVerification = this.#engineRun(runId).verification_json;
|
|
2898
|
-
const repairStatus = existingVerification
|
|
2899
|
-
? normalizeWorkflowVerificationStatus(
|
|
2900
|
-
JSON.parse(existingVerification) as unknown,
|
|
2901
|
-
)
|
|
2902
|
-
: undefined;
|
|
2903
|
-
const retainsRepairContext =
|
|
2904
|
-
repairStatus?.scope === "change-task-affected" &&
|
|
2905
|
-
repairStatus.attribution === "introduced" &&
|
|
2906
|
-
repairStatus.taskId !== undefined &&
|
|
2907
|
-
this.#tasks(runId).some(
|
|
2908
|
-
(task) =>
|
|
2909
|
-
task.task_id === repairStatus.taskId && task.state !== "verified",
|
|
2910
|
-
);
|
|
2911
|
-
if (!retainsRepairContext) {
|
|
2912
|
-
this.#setVerificationStatus(runId, null, lease);
|
|
2913
|
-
}
|
|
2914
|
-
const attempted = new Set<string>();
|
|
2915
|
-
// Keep every launched operation owned until it settles, including failures.
|
|
2916
|
-
// A fulfilled envelope avoids unhandled rejections while another task wins
|
|
2917
|
-
// the race; the finally barrier also protects cancellation and storage close.
|
|
2918
|
-
type Settlement =
|
|
2919
|
-
| { taskId: string; status: "fulfilled" }
|
|
2920
|
-
| { taskId: string; status: "rejected"; reason: unknown };
|
|
2921
|
-
const pending = new Map<string, Promise<Settlement>>();
|
|
2922
|
-
let firstFailure: { reason: unknown } | undefined;
|
|
2923
|
-
try {
|
|
2924
|
-
while (!signal.aborted) {
|
|
2925
|
-
if (firstFailure) throw firstFailure.reason;
|
|
2926
|
-
const rows = this.#tasks(runId);
|
|
2927
|
-
const selected = selectRunnableTasks({
|
|
2928
|
-
rows,
|
|
2929
|
-
attempted,
|
|
2930
|
-
activeTasks: this.#activeTasks,
|
|
2931
|
-
capacity: LIMITS.maxActiveChildSessions,
|
|
2932
|
-
nextQueuePosition: this.#engineRun(runId).next_queue_position,
|
|
2933
|
-
});
|
|
2934
|
-
const runnable = selected.runnable.map((taskId) => {
|
|
2935
|
-
const row = rows.find((candidate) => candidate.task_id === taskId);
|
|
2936
|
-
if (!row) throw new Error("scheduler-task-unavailable");
|
|
2937
|
-
attempted.add(taskId);
|
|
2938
|
-
return row;
|
|
2939
|
-
});
|
|
2940
|
-
for (const update of selected.updates) {
|
|
2941
|
-
const row = rows.find(
|
|
2942
|
-
(candidate) => candidate.task_id === update.taskId,
|
|
2943
|
-
);
|
|
2944
|
-
if (!row) throw new Error("scheduler-task-unavailable");
|
|
2945
|
-
if (update.kind === "pending")
|
|
2946
|
-
this.#setTask(
|
|
2947
|
-
runId,
|
|
2948
|
-
row.task_id,
|
|
2949
|
-
{ state: "pending", pauseCode: null, queuePosition: null },
|
|
2950
|
-
lease,
|
|
2951
|
-
);
|
|
2952
|
-
else this.#queueTask(runId, row, lease, update.reason);
|
|
2953
|
-
}
|
|
2954
|
-
this.#activeTasks += runnable.length;
|
|
2955
|
-
for (const row of runnable) {
|
|
2956
|
-
const operation = this.#runTask(
|
|
2957
|
-
runId,
|
|
2958
|
-
row,
|
|
2959
|
-
operationId,
|
|
2960
|
-
signal,
|
|
2961
|
-
lease,
|
|
2962
|
-
onActivity,
|
|
2963
|
-
)
|
|
2964
|
-
.then<Settlement, Settlement>(
|
|
2965
|
-
() => ({ taskId: row.task_id, status: "fulfilled" }),
|
|
2966
|
-
(reason: unknown) => {
|
|
2967
|
-
firstFailure ??= { reason };
|
|
2968
|
-
return { taskId: row.task_id, status: "rejected", reason };
|
|
2969
|
-
},
|
|
2970
|
-
)
|
|
2971
|
-
.finally(() => {
|
|
2972
|
-
this.#activeTasks--;
|
|
2973
|
-
for (const wake of this.#capacityWaiters) wake();
|
|
2974
|
-
});
|
|
2975
|
-
pending.set(row.task_id, operation);
|
|
2976
|
-
}
|
|
2977
|
-
if (pending.size > 0) {
|
|
2978
|
-
const settled = await Promise.race([
|
|
2979
|
-
...pending.values(),
|
|
2980
|
-
...(selected.capacityBlocked
|
|
2981
|
-
? [this.#waitForCapacity(signal).then(() => null)]
|
|
2982
|
-
: []),
|
|
2983
|
-
]);
|
|
2984
|
-
if (settled === null) continue;
|
|
2985
|
-
pending.delete(settled.taskId);
|
|
2986
|
-
if (settled.status === "rejected") throw settled.reason;
|
|
2987
|
-
continue;
|
|
2988
|
-
}
|
|
2989
|
-
if (selected.capacityBlocked) {
|
|
2990
|
-
await this.#waitForCapacity(signal);
|
|
2991
|
-
continue;
|
|
2992
|
-
}
|
|
2993
|
-
break;
|
|
2994
|
-
}
|
|
2995
|
-
} finally {
|
|
2996
|
-
await Promise.all(pending.values());
|
|
2997
|
-
}
|
|
2998
|
-
if (firstFailure) throw firstFailure.reason;
|
|
2999
|
-
|
|
3000
|
-
if (signal.aborted) {
|
|
3001
|
-
const current = this.#runStore.status(runId);
|
|
3002
|
-
if (!["paused", "discarded", "completed"].includes(current.state)) {
|
|
3003
|
-
this.#transition(
|
|
3004
|
-
runId,
|
|
3005
|
-
"paused",
|
|
3006
|
-
`${operationId}:cancelled`,
|
|
3007
|
-
"operation-cancelled",
|
|
3008
|
-
lease,
|
|
3009
|
-
);
|
|
3010
|
-
}
|
|
3011
|
-
return this.#statusByRun(runId);
|
|
3012
|
-
}
|
|
3013
|
-
const rows = this.#tasks(runId);
|
|
3014
|
-
const paused = this.#firstPaused(rows);
|
|
3015
|
-
if (paused) {
|
|
3016
|
-
this.#transition(
|
|
3017
|
-
runId,
|
|
3018
|
-
paused.state === "approval-needed" ? "approval-needed" : "paused",
|
|
3019
|
-
`${operationId}:task-paused:${paused.task_id}`,
|
|
3020
|
-
paused.pause_code ?? "task-paused",
|
|
3021
|
-
lease,
|
|
3022
|
-
);
|
|
3023
|
-
return this.#statusByRun(runId);
|
|
3024
|
-
}
|
|
3025
|
-
if (rows.every((row) => row.state === "verified")) {
|
|
3026
|
-
return this.#finishChange(runId, operationId, signal, lease, onActivity);
|
|
3027
|
-
}
|
|
3028
|
-
this.#transition(
|
|
3029
|
-
runId,
|
|
3030
|
-
"paused",
|
|
3031
|
-
`${operationId}:scheduler-stalled`,
|
|
3032
|
-
"scheduler-integrity-paused",
|
|
3033
|
-
lease,
|
|
3034
|
-
);
|
|
3035
|
-
return this.#statusByRun(runId);
|
|
3036
|
-
}
|
|
3037
|
-
|
|
3038
|
-
#contextReads(runId: string, taskId?: string): string[] {
|
|
3039
|
-
const rows =
|
|
3040
|
-
taskId === undefined
|
|
3041
|
-
? this.#database
|
|
3042
|
-
.prepare(
|
|
3043
|
-
"SELECT DISTINCT path FROM workflow_context_reads WHERE run_id = ? ORDER BY path",
|
|
3044
|
-
)
|
|
3045
|
-
.all(runId)
|
|
3046
|
-
: this.#database
|
|
3047
|
-
.prepare(
|
|
3048
|
-
"SELECT path FROM workflow_context_reads WHERE run_id = ? AND task_id = ? ORDER BY path",
|
|
3049
|
-
)
|
|
3050
|
-
.all(runId, taskId);
|
|
3051
|
-
return rows.map((row) => String(row.path));
|
|
3052
|
-
}
|
|
3053
|
-
|
|
3054
|
-
#phaseCount(plan: ImplementPlan): number {
|
|
3055
|
-
return plan.tasks.reduce((sum, task) => sum + phases(task).length, 0);
|
|
3056
|
-
}
|
|
3057
|
-
|
|
3058
|
-
#growWorkBudget(
|
|
3059
|
-
database: DatabaseSync,
|
|
3060
|
-
runId: string,
|
|
3061
|
-
plan: ImplementPlan,
|
|
3062
|
-
): void {
|
|
3063
|
-
const count = this.#phaseCount(plan);
|
|
3064
|
-
database
|
|
3065
|
-
.prepare(`INSERT INTO workflow_work_budget(run_id, used, max_work, phase_high_water, hard_limit, recovery_policy)
|
|
3066
|
-
VALUES (?, 0, ?, ?, ?, 1) ON CONFLICT(run_id) DO UPDATE SET
|
|
3067
|
-
phase_high_water = MAX(phase_high_water, excluded.phase_high_water),
|
|
3068
|
-
max_work = MIN(hard_limit, MAX(max_work, 24 + 3 * MAX(phase_high_water, excluded.phase_high_water)))`)
|
|
3069
|
-
.run(
|
|
3070
|
-
runId,
|
|
3071
|
-
Math.min(this.#workHardLimit, 24 + 3 * count),
|
|
3072
|
-
count,
|
|
3073
|
-
this.#workHardLimit,
|
|
3074
|
-
);
|
|
3075
|
-
}
|
|
3076
|
-
|
|
3077
|
-
#hasRecoveryGrant(runId: string, operationId: string, key: string): boolean {
|
|
3078
|
-
return Boolean(
|
|
3079
|
-
this.#database
|
|
3080
|
-
.prepare(
|
|
3081
|
-
"SELECT 1 FROM workflow_recovery_grants WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
|
|
3082
|
-
)
|
|
3083
|
-
.get(runId, operationId, key),
|
|
3084
|
-
);
|
|
3085
|
-
}
|
|
3086
|
-
|
|
3087
|
-
#recoveryConditions(runId: string, key: string) {
|
|
3088
|
-
const run = this.#engineRun(runId);
|
|
3089
|
-
const task = this.#tasks(runId).find((row) =>
|
|
3090
|
-
[
|
|
3091
|
-
recoveryKey(this.#taskPlan(row), row.phase, row, run),
|
|
3092
|
-
baselineRecoveryKey(this.#taskPlan(row), row.phase),
|
|
3093
|
-
].includes(key),
|
|
3094
|
-
);
|
|
3095
|
-
return {
|
|
3096
|
-
route: task?.route_fingerprint ?? run.route_fingerprint,
|
|
3097
|
-
context: task
|
|
3098
|
-
? hash(
|
|
3099
|
-
canonicalJson(
|
|
3100
|
-
snapshotFiles(
|
|
3101
|
-
this.#consumerRoot,
|
|
3102
|
-
this.#contextReads(runId, task.task_id),
|
|
3103
|
-
),
|
|
3104
|
-
),
|
|
3105
|
-
)
|
|
3106
|
-
: null,
|
|
3107
|
-
};
|
|
3108
|
-
}
|
|
3109
|
-
|
|
3110
|
-
#failureSequence(runId: string, key: string): number {
|
|
3111
|
-
return (
|
|
3112
|
-
this.#database
|
|
3113
|
-
.prepare(
|
|
3114
|
-
"SELECT COALESCE(MAX(sequence), 0) AS sequence FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ?",
|
|
3115
|
-
)
|
|
3116
|
-
.get(runId, key) as { sequence: number }
|
|
3117
|
-
).sequence;
|
|
3118
|
-
}
|
|
3119
|
-
|
|
3120
|
-
#grantRecovery(
|
|
3121
|
-
runId: string,
|
|
3122
|
-
operationId: string,
|
|
3123
|
-
request: NonNullable<
|
|
3124
|
-
Extract<ControlCommand, { command: "resume" }>["recovery"]
|
|
3125
|
-
>,
|
|
3126
|
-
lease: OperationLease,
|
|
3127
|
-
): void {
|
|
3128
|
-
this.#leasedTransaction(lease, () => {
|
|
3129
|
-
const incident = this.#recovery(runId, request.incidentKey);
|
|
3130
|
-
const current = this.#tasks(runId).some(
|
|
3131
|
-
(row) =>
|
|
3132
|
-
["paused", "retryable"].includes(row.state) &&
|
|
3133
|
-
[
|
|
3134
|
-
recoveryKey(
|
|
3135
|
-
this.#taskPlan(row),
|
|
3136
|
-
row.phase,
|
|
3137
|
-
row,
|
|
3138
|
-
this.#engineRun(runId),
|
|
3139
|
-
),
|
|
3140
|
-
baselineRecoveryKey(this.#taskPlan(row), row.phase),
|
|
3141
|
-
].includes(request.incidentKey),
|
|
3142
|
-
);
|
|
3143
|
-
if (
|
|
3144
|
-
!current ||
|
|
3145
|
-
!incident ||
|
|
3146
|
-
incident.failures < incident.feedback.maxAttempts ||
|
|
3147
|
-
this.#failureSequence(runId, request.incidentKey) !==
|
|
3148
|
-
request.failureSequence
|
|
3149
|
-
)
|
|
3150
|
-
throw new Error("recovery-request-stale");
|
|
3151
|
-
const conditions = this.#database
|
|
3152
|
-
.prepare(
|
|
3153
|
-
"SELECT conditions_json FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
|
|
3154
|
-
)
|
|
3155
|
-
.get(runId, request.incidentKey) as { conditions_json: string };
|
|
3156
|
-
const previous = JSON.parse(conditions.conditions_json);
|
|
3157
|
-
const observed = this.#recoveryConditions(runId, request.incidentKey);
|
|
3158
|
-
const budget = this.#database
|
|
3159
|
-
.prepare(
|
|
3160
|
-
"SELECT used, max_work FROM workflow_work_budget WHERE run_id = ?",
|
|
3161
|
-
)
|
|
3162
|
-
.get(runId) as { used: number; max_work: number };
|
|
3163
|
-
const rejected = assessRecoveryGrant({
|
|
3164
|
-
current,
|
|
3165
|
-
incident,
|
|
3166
|
-
failureSequence: this.#failureSequence(runId, request.incidentKey),
|
|
3167
|
-
requestedSequence: request.failureSequence,
|
|
3168
|
-
reason: request.reason,
|
|
3169
|
-
previous,
|
|
3170
|
-
observed,
|
|
3171
|
-
budget,
|
|
3172
|
-
});
|
|
3173
|
-
if (rejected) throw new Error(rejected);
|
|
3174
|
-
this.#database
|
|
3175
|
-
.prepare(
|
|
3176
|
-
"INSERT INTO workflow_recovery_grants(run_id, operation_id, incident_key, failure_sequence, reason) VALUES (?, ?, ?, ?, ?)",
|
|
3177
|
-
)
|
|
3178
|
-
.run(
|
|
3179
|
-
runId,
|
|
3180
|
-
operationId,
|
|
3181
|
-
request.incidentKey,
|
|
3182
|
-
request.failureSequence,
|
|
3183
|
-
request.reason,
|
|
3184
|
-
);
|
|
3185
|
-
});
|
|
3186
|
-
}
|
|
3187
|
-
|
|
3188
|
-
#reserveWork(
|
|
3189
|
-
runId: string,
|
|
3190
|
-
plan: ImplementPlan,
|
|
3191
|
-
key: string,
|
|
3192
|
-
lease: OperationLease,
|
|
3193
|
-
grantOperationId?: string,
|
|
3194
|
-
): boolean {
|
|
3195
|
-
return this.#leasedTransaction(lease, () => {
|
|
3196
|
-
this.#growWorkBudget(this.#database, runId, plan);
|
|
3197
|
-
const reserved = this.#database
|
|
3198
|
-
.prepare(`UPDATE workflow_work_budget SET used = used + 1
|
|
3199
|
-
WHERE run_id = ? AND used < max_work`)
|
|
3200
|
-
.run(runId);
|
|
3201
|
-
if (reserved.changes !== 1) return false;
|
|
3202
|
-
if (grantOperationId) {
|
|
3203
|
-
const consumed = this.#database
|
|
3204
|
-
.prepare(
|
|
3205
|
-
"UPDATE workflow_recovery_grants SET consumed = 1 WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
|
|
3206
|
-
)
|
|
3207
|
-
.run(runId, grantOperationId, key);
|
|
3208
|
-
if (consumed.changes !== 1) throw new Error("recovery-request-stale");
|
|
3209
|
-
}
|
|
3210
|
-
this.#database
|
|
3211
|
-
.prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
|
|
3212
|
-
SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'launch' FROM workflow_recovery_events WHERE run_id = ?`)
|
|
3213
|
-
.run(runId, key, runId);
|
|
3214
|
-
return true;
|
|
3215
|
-
});
|
|
3216
|
-
}
|
|
3217
|
-
|
|
3218
|
-
#recovery(runId: string, key: string): WorkflowRecoveryFact | undefined {
|
|
3219
|
-
const row = this.#database
|
|
3220
|
-
.prepare(
|
|
3221
|
-
`SELECT failures, max_attempts, feedback_json FROM workflow_recovery_incidents
|
|
3222
|
-
WHERE run_id = ? AND incident_key = ?`,
|
|
3223
|
-
)
|
|
3224
|
-
.get(runId, key) as
|
|
3225
|
-
| { failures: number; max_attempts: number; feedback_json: string }
|
|
3226
|
-
| undefined;
|
|
3227
|
-
return row
|
|
3228
|
-
? {
|
|
3229
|
-
key,
|
|
3230
|
-
failures: row.failures,
|
|
3231
|
-
feedback: {
|
|
3232
|
-
...JSON.parse(row.feedback_json),
|
|
3233
|
-
maxAttempts: row.max_attempts,
|
|
3234
|
-
},
|
|
3235
|
-
}
|
|
3236
|
-
: undefined;
|
|
3237
|
-
}
|
|
3238
|
-
|
|
3239
|
-
#recordRecovery(
|
|
3240
|
-
runId: string,
|
|
3241
|
-
fact: WorkflowRecoveryFact,
|
|
3242
|
-
lease: OperationLease,
|
|
3243
|
-
): void {
|
|
3244
|
-
this.#leasedTransaction(lease, () => {
|
|
3245
|
-
this.#database
|
|
3246
|
-
.prepare(
|
|
3247
|
-
`INSERT INTO workflow_recovery_incidents(run_id, incident_key, failures, max_attempts, feedback_json, conditions_json)
|
|
3248
|
-
VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(run_id, incident_key) DO UPDATE SET
|
|
3249
|
-
failures = MAX(failures, excluded.failures), max_attempts = MIN(max_attempts, excluded.max_attempts), feedback_json = excluded.feedback_json, conditions_json = excluded.conditions_json`,
|
|
3250
|
-
)
|
|
3251
|
-
.run(
|
|
3252
|
-
runId,
|
|
3253
|
-
fact.key,
|
|
3254
|
-
fact.failures,
|
|
3255
|
-
fact.feedback.maxAttempts,
|
|
3256
|
-
JSON.stringify(fact.feedback),
|
|
3257
|
-
JSON.stringify(this.#recoveryConditions(runId, fact.key)),
|
|
3258
|
-
);
|
|
3259
|
-
this.#database
|
|
3260
|
-
.prepare(
|
|
3261
|
-
`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
|
|
3262
|
-
SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'failure' FROM workflow_recovery_events WHERE run_id = ?`,
|
|
3263
|
-
)
|
|
3264
|
-
.run(runId, fact.key, runId);
|
|
3265
|
-
});
|
|
3266
|
-
}
|
|
3267
|
-
|
|
3268
|
-
#safeAttemptDiagnostic(
|
|
3269
|
-
runId: string,
|
|
3270
|
-
row: EngineTaskRow | undefined,
|
|
3271
|
-
): SafeAttemptDiagnostic | undefined {
|
|
3272
|
-
if (!row) return undefined;
|
|
3273
|
-
const diagnostic: SafeAttemptDiagnostic = row.attempt_diagnostic_json
|
|
3274
|
-
? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
|
|
3275
|
-
: {};
|
|
3276
|
-
delete diagnostic.recovery;
|
|
3277
|
-
const recovery =
|
|
3278
|
-
this.#recovery(
|
|
3279
|
-
runId,
|
|
3280
|
-
baselineRecoveryKey(this.#taskPlan(row), row.phase),
|
|
3281
|
-
) ??
|
|
3282
|
-
this.#recovery(
|
|
3283
|
-
runId,
|
|
3284
|
-
recoveryKey(
|
|
3285
|
-
this.#taskPlan(row),
|
|
3286
|
-
row.phase,
|
|
3287
|
-
row,
|
|
3288
|
-
this.#engineRun(runId),
|
|
3289
|
-
),
|
|
3290
|
-
);
|
|
3291
|
-
if (recovery) diagnostic.recovery = recovery;
|
|
3292
|
-
return Object.keys(diagnostic).length ? diagnostic : undefined;
|
|
3293
|
-
}
|
|
3294
|
-
|
|
3295
|
-
#statusByRun(runId: string): Record<string, unknown> {
|
|
3296
|
-
const projection = this.#runStore.status(runId);
|
|
3297
|
-
const rows = this.#tasks(runId);
|
|
3298
|
-
let engineRun: EngineRunRow | undefined;
|
|
3299
|
-
try {
|
|
3300
|
-
engineRun = this.#engineRun(runId);
|
|
3301
|
-
} catch {
|
|
3302
|
-
engineRun = undefined;
|
|
3303
|
-
}
|
|
3304
|
-
const attemptDiagnostics = new Map<string, SafeAttemptDiagnostic>();
|
|
3305
|
-
const failureSequences = new Map<string, number>();
|
|
3306
|
-
for (const row of rows) {
|
|
3307
|
-
const diagnostic = this.#safeAttemptDiagnostic(runId, row);
|
|
3308
|
-
if (!diagnostic) continue;
|
|
3309
|
-
attemptDiagnostics.set(row.task_id, diagnostic);
|
|
3310
|
-
if (diagnostic.recovery) {
|
|
3311
|
-
const key = diagnostic.recovery.key;
|
|
3312
|
-
failureSequences.set(key, this.#failureSequence(runId, key));
|
|
3313
|
-
}
|
|
3314
|
-
}
|
|
3315
|
-
const amendmentUsed =
|
|
3316
|
-
(
|
|
3317
|
-
this.#database
|
|
3318
|
-
.prepare(
|
|
3319
|
-
"SELECT used FROM workflow_amendment_budget WHERE run_id = ?",
|
|
3320
|
-
)
|
|
3321
|
-
.get(runId) as { used: number } | undefined
|
|
3322
|
-
)?.used ?? 0;
|
|
3323
|
-
const resourceBudget = this.#database
|
|
3324
|
-
.prepare(
|
|
3325
|
-
"SELECT used, max_work AS maximum, phase_high_water AS phaseHighWater, hard_limit AS hardLimit FROM workflow_work_budget WHERE run_id = ?",
|
|
3326
|
-
)
|
|
3327
|
-
.get(runId) as WorkflowResourceBudget | undefined;
|
|
3328
|
-
return projectWorkflowStatus({
|
|
3329
|
-
projection,
|
|
3330
|
-
rows,
|
|
3331
|
-
engineRun,
|
|
3332
|
-
attemptDiagnostics,
|
|
3333
|
-
failureSequences,
|
|
3334
|
-
amendmentUsed,
|
|
3335
|
-
resourceBudget,
|
|
3336
|
-
});
|
|
3337
|
-
}
|
|
3338
|
-
|
|
3339
|
-
async #withAvailableDelivery(
|
|
3340
|
-
outcome: Record<string, unknown>,
|
|
3341
|
-
signal?: AbortSignal,
|
|
3342
|
-
): Promise<Record<string, unknown>> {
|
|
3343
|
-
if (
|
|
3344
|
-
outcome.stage !== "abel-implement" ||
|
|
3345
|
-
(isRecord(outcome.pause) &&
|
|
3346
|
-
outcome.pause.code === "operation-cancelled") ||
|
|
3347
|
-
(outcome.state !== "approval-needed" &&
|
|
3348
|
-
!(
|
|
3349
|
-
outcome.state === "paused" &&
|
|
3350
|
-
(isRecord(outcome.decisionBatch) ||
|
|
3351
|
-
(isRecord(outcome.recovery) && outcome.recovery.exhausted === true))
|
|
3352
|
-
)) ||
|
|
3353
|
-
typeof outcome.change !== "string" ||
|
|
3354
|
-
!this.#deliverySource.discoverLatest
|
|
3355
|
-
) {
|
|
3356
|
-
return outcome;
|
|
3357
|
-
}
|
|
3358
|
-
let available: WorkflowAvailableDelivery | undefined;
|
|
3359
|
-
try {
|
|
3360
|
-
const discover = this.#deliverySource.discoverLatest;
|
|
3361
|
-
const change = outcome.change;
|
|
3362
|
-
available = await cancellableRead(
|
|
3363
|
-
() =>
|
|
3364
|
-
discover.call(this.#deliverySource, {
|
|
3365
|
-
stage: "abel-implement",
|
|
3366
|
-
change,
|
|
3367
|
-
...(signal ? { signal } : {}),
|
|
3368
|
-
}),
|
|
3369
|
-
signal,
|
|
3370
|
-
);
|
|
3371
|
-
} catch {
|
|
3372
|
-
return outcome;
|
|
3373
|
-
}
|
|
3374
|
-
const currentRevision =
|
|
3375
|
-
typeof outcome.deliveryRevision === "number"
|
|
3376
|
-
? outcome.deliveryRevision
|
|
3377
|
-
: 0;
|
|
3378
|
-
if (
|
|
3379
|
-
!available ||
|
|
3380
|
-
!Number.isSafeInteger(available.deliveryRevision) ||
|
|
3381
|
-
available.deliveryRevision <= currentRevision ||
|
|
3382
|
-
!SHA256.test(available.receiptHash)
|
|
3383
|
-
) {
|
|
3384
|
-
return outcome;
|
|
3385
|
-
}
|
|
3386
|
-
if (
|
|
3387
|
-
this.#database
|
|
3388
|
-
.prepare(
|
|
3389
|
-
"SELECT 1 FROM workflow_rejected_deliveries WHERE run_id = ? AND delivery_revision = ? AND receipt_hash = ?",
|
|
3390
|
-
)
|
|
3391
|
-
.get(
|
|
3392
|
-
String(outcome.runId),
|
|
3393
|
-
available.deliveryRevision,
|
|
3394
|
-
available.receiptHash,
|
|
3395
|
-
)
|
|
3396
|
-
) {
|
|
3397
|
-
return outcome;
|
|
3398
|
-
}
|
|
3399
|
-
const conditionalCommands = (
|
|
3400
|
-
Array.isArray(outcome.conditionalCommands)
|
|
3401
|
-
? outcome.conditionalCommands
|
|
3402
|
-
: [{ command: "resume", stage: outcome.stage, change: outcome.change }]
|
|
3403
|
-
).map((entry) =>
|
|
3404
|
-
isRecord(entry) && entry.command === "resume"
|
|
3405
|
-
? {
|
|
3406
|
-
...entry,
|
|
3407
|
-
satisfiedBy: {
|
|
3408
|
-
deliveryRevision: available.deliveryRevision,
|
|
3409
|
-
receiptHash: available.receiptHash,
|
|
3410
|
-
},
|
|
3411
|
-
}
|
|
3412
|
-
: entry,
|
|
3413
|
-
);
|
|
3414
|
-
return {
|
|
3415
|
-
...outcome,
|
|
3416
|
-
legalCommands:
|
|
3417
|
-
outcome.state === "approval-needed"
|
|
3418
|
-
? ["status", "resume", "discard"]
|
|
3419
|
-
: ["status", "resume", "rebind", "discard"],
|
|
3420
|
-
availableDelivery: structuredClone(available),
|
|
3421
|
-
continuation:
|
|
3422
|
-
(isRecord(outcome.resourceBudget) &&
|
|
3423
|
-
outcome.resourceBudget.remaining === 0) ||
|
|
3424
|
-
(isRecord(outcome.recovery) &&
|
|
3425
|
-
outcome.recovery.code === "change-recovery-budget-exhausted")
|
|
3426
|
-
? undefined
|
|
3427
|
-
: {
|
|
3428
|
-
owner: "parent",
|
|
3429
|
-
automatic: true,
|
|
3430
|
-
command: "resume",
|
|
3431
|
-
stage: "abel-implement",
|
|
3432
|
-
change: outcome.change,
|
|
3433
|
-
},
|
|
3434
|
-
...(conditionalCommands === undefined ? {} : { conditionalCommands }),
|
|
3435
|
-
};
|
|
3436
|
-
}
|
|
3437
|
-
|
|
3438
|
-
#launchAdvance(
|
|
3439
|
-
runId: string,
|
|
3440
|
-
operationId: string,
|
|
3441
|
-
externalSignal?: AbortSignal,
|
|
3442
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
3443
|
-
): Promise<Record<string, unknown>> {
|
|
3444
|
-
const lease = this.#operationLease(runId, operationId);
|
|
3445
|
-
const controller = new AbortController();
|
|
3446
|
-
const abortFromExternal = () => {
|
|
3447
|
-
controller.abort(
|
|
3448
|
-
externalSignal?.reason ?? new Error("operation-cancelled"),
|
|
3449
|
-
);
|
|
3450
|
-
};
|
|
3451
|
-
if (externalSignal?.aborted) abortFromExternal();
|
|
3452
|
-
else
|
|
3453
|
-
externalSignal?.addEventListener("abort", abortFromExternal, {
|
|
3454
|
-
once: true,
|
|
3455
|
-
});
|
|
3456
|
-
const active: ActiveOperation = {
|
|
3457
|
-
operationId,
|
|
3458
|
-
kind: "work",
|
|
3459
|
-
controller,
|
|
3460
|
-
settled: Promise.resolve({}),
|
|
3461
|
-
};
|
|
3462
|
-
this.#active.set(runId, active);
|
|
3463
|
-
const settled = (async () => {
|
|
3464
|
-
try {
|
|
3465
|
-
const outcome = await this.#advance(
|
|
3466
|
-
runId,
|
|
3467
|
-
operationId,
|
|
3468
|
-
controller.signal,
|
|
3469
|
-
lease,
|
|
3470
|
-
onActivity,
|
|
3471
|
-
);
|
|
3472
|
-
return this.#commitOperation(runId, operationId, outcome);
|
|
3473
|
-
} catch (error) {
|
|
3474
|
-
this.#interruptOperation(runId, operationId);
|
|
3475
|
-
const controlSettled =
|
|
3476
|
-
controller.signal.aborted &&
|
|
3477
|
-
controller.signal.reason instanceof Error &&
|
|
3478
|
-
["operation-cancelled", "run-discarded"].includes(
|
|
3479
|
-
controller.signal.reason.message,
|
|
3480
|
-
) &&
|
|
3481
|
-
(isCancellationException(error, controller.signal) ||
|
|
3482
|
-
(error instanceof Error &&
|
|
3483
|
-
["lease-fenced", "operation-journal-fenced"].includes(
|
|
3484
|
-
error.message,
|
|
3485
|
-
)));
|
|
3486
|
-
if (controlSettled) return this.#statusByRun(runId);
|
|
3487
|
-
throw error;
|
|
3488
|
-
} finally {
|
|
3489
|
-
externalSignal?.removeEventListener("abort", abortFromExternal);
|
|
3490
|
-
if (this.#active.get(runId) === active) this.#active.delete(runId);
|
|
3491
|
-
}
|
|
3492
|
-
})();
|
|
3493
|
-
active.settled = settled;
|
|
3494
|
-
return settled;
|
|
3495
|
-
}
|
|
3496
|
-
|
|
3497
|
-
async #start(
|
|
3498
|
-
command: Extract<ControlCommand, { command: "start" }>,
|
|
3499
|
-
signal?: AbortSignal,
|
|
3500
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
3501
|
-
) {
|
|
3502
|
-
const existingRun = this.#lookupRun(command.stage, command.change);
|
|
3503
|
-
if (existingRun) {
|
|
3504
|
-
this.#ensureEngineRun(existingRun);
|
|
3505
|
-
const replay = this.#operationReplay(
|
|
3506
|
-
existingRun,
|
|
3507
|
-
command.operationId,
|
|
3508
|
-
command.command,
|
|
3509
|
-
);
|
|
3510
|
-
if (replay) return replay;
|
|
3511
|
-
const restarted = this.#runStore.startRun({
|
|
3512
|
-
stage: command.stage,
|
|
3513
|
-
change: command.change,
|
|
3514
|
-
operationId: command.operationId,
|
|
3515
|
-
});
|
|
3516
|
-
if (restarted.state !== "created") {
|
|
3517
|
-
const begun = this.#beginOperation(
|
|
3518
|
-
existingRun,
|
|
3519
|
-
command.operationId,
|
|
3520
|
-
command.command,
|
|
3521
|
-
);
|
|
3522
|
-
if (begun) return begun;
|
|
3523
|
-
return this.#commitOperation(
|
|
3524
|
-
existingRun,
|
|
3525
|
-
command.operationId,
|
|
3526
|
-
this.#statusByRun(existingRun),
|
|
3527
|
-
);
|
|
3528
|
-
}
|
|
3529
|
-
}
|
|
3530
|
-
|
|
3531
|
-
const runId =
|
|
3532
|
-
existingRun ??
|
|
3533
|
-
this.#runStore.startRun({
|
|
3534
|
-
stage: command.stage,
|
|
3535
|
-
change: command.change,
|
|
3536
|
-
operationId: command.operationId,
|
|
3537
|
-
}).runId;
|
|
3538
|
-
this.#ensureEngineRun(runId);
|
|
3539
|
-
const begun = this.#beginOperation(
|
|
3540
|
-
runId,
|
|
3541
|
-
command.operationId,
|
|
3542
|
-
command.command,
|
|
3543
|
-
);
|
|
3544
|
-
if (begun) return begun;
|
|
3545
|
-
const lease = this.#operationLease(runId, command.operationId);
|
|
3546
|
-
this.#transition(
|
|
3547
|
-
runId,
|
|
3548
|
-
"validating-delivery",
|
|
3549
|
-
`${command.operationId}:validating-delivery`,
|
|
3550
|
-
undefined,
|
|
3551
|
-
lease,
|
|
3552
|
-
);
|
|
3553
|
-
emitWorkflowActivity(onActivity, {
|
|
3554
|
-
state: "validating",
|
|
3555
|
-
stage: command.stage,
|
|
3556
|
-
runId,
|
|
3557
|
-
change: command.change,
|
|
3558
|
-
objective: `validate delivery ${command.change}`,
|
|
3559
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
3560
|
-
});
|
|
3561
|
-
try {
|
|
3562
|
-
const delivery = await this.#loadDelivery(
|
|
3563
|
-
runId,
|
|
3564
|
-
lease,
|
|
3565
|
-
command.stage,
|
|
3566
|
-
command.change,
|
|
3567
|
-
undefined,
|
|
3568
|
-
undefined,
|
|
3569
|
-
signal,
|
|
3570
|
-
);
|
|
3571
|
-
this.#admitDelivery(runId, delivery, lease);
|
|
3572
|
-
this.#transition(
|
|
3573
|
-
runId,
|
|
3574
|
-
"ready",
|
|
3575
|
-
`${command.operationId}:delivery-ready`,
|
|
3576
|
-
undefined,
|
|
3577
|
-
lease,
|
|
3578
|
-
);
|
|
3579
|
-
return this.#launchAdvance(
|
|
3580
|
-
runId,
|
|
3581
|
-
command.operationId,
|
|
3582
|
-
signal,
|
|
3583
|
-
onActivity,
|
|
3584
|
-
);
|
|
3585
|
-
} catch (error) {
|
|
3586
|
-
const cancelled = this.#cancelledDelivery(
|
|
3587
|
-
runId,
|
|
3588
|
-
command.operationId,
|
|
3589
|
-
error,
|
|
3590
|
-
signal,
|
|
3591
|
-
lease,
|
|
3592
|
-
);
|
|
3593
|
-
if (cancelled) return cancelled;
|
|
3594
|
-
const failure = this.#deliveryFailure(
|
|
3595
|
-
runId,
|
|
3596
|
-
command.operationId,
|
|
3597
|
-
error,
|
|
3598
|
-
lease,
|
|
3599
|
-
);
|
|
3600
|
-
if (failure) {
|
|
3601
|
-
return this.#commitOperation(runId, command.operationId, failure);
|
|
3602
|
-
}
|
|
3603
|
-
this.#interruptOperation(runId, command.operationId);
|
|
3604
|
-
throw error;
|
|
3605
|
-
}
|
|
3606
|
-
}
|
|
3607
|
-
|
|
3608
|
-
async #resume(
|
|
3609
|
-
command: Extract<ControlCommand, { command: "resume" }>,
|
|
3610
|
-
signal?: AbortSignal,
|
|
3611
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
3612
|
-
) {
|
|
3613
|
-
const runId = this.#lookupRun(command.stage, command.change);
|
|
3614
|
-
if (!runId) throw new Error("run-not-found");
|
|
3615
|
-
this.#recoverLegacyContextApproval(runId);
|
|
3616
|
-
const replay = this.#beginOperation(
|
|
3617
|
-
runId,
|
|
3618
|
-
command.operationId,
|
|
3619
|
-
command.command,
|
|
3620
|
-
);
|
|
3621
|
-
if (replay) return replay;
|
|
3622
|
-
const lease = this.#operationLease(runId, command.operationId);
|
|
3623
|
-
const current = this.#runStore.status(runId);
|
|
3624
|
-
if (current.state === "recovering") {
|
|
3625
|
-
emitWorkflowActivity(onActivity, {
|
|
3626
|
-
state: "recovering",
|
|
3627
|
-
stage: current.stage,
|
|
3628
|
-
runId,
|
|
3629
|
-
...(current.change ? { change: current.change } : {}),
|
|
3630
|
-
objective: current.change
|
|
3631
|
-
? `recover change ${current.change}`
|
|
3632
|
-
: "recover workflow application",
|
|
3633
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
3634
|
-
});
|
|
3635
|
-
const transactionId = this.#engineRun(runId).transaction_id;
|
|
3636
|
-
if (!transactionId || !this.#application) {
|
|
3637
|
-
this.#interruptOperation(runId, command.operationId);
|
|
3638
|
-
throw new Error("apply-recovery-unavailable");
|
|
3639
|
-
}
|
|
3640
|
-
const recovered = await this.#application.recover(
|
|
3641
|
-
transactionId,
|
|
3642
|
-
this.#applicationContext(runId),
|
|
3643
|
-
);
|
|
3644
|
-
this.#runStore.assertLease(lease);
|
|
3645
|
-
const outcome = this.#settleApplication(
|
|
3646
|
-
runId,
|
|
3647
|
-
command.operationId,
|
|
3648
|
-
recovered,
|
|
3649
|
-
lease,
|
|
3650
|
-
onActivity,
|
|
3651
|
-
);
|
|
3652
|
-
return this.#commitOperation(runId, command.operationId, outcome);
|
|
3653
|
-
}
|
|
3654
|
-
if (
|
|
3655
|
-
!["paused", "retryable", "approval-needed", "ready"].includes(
|
|
3656
|
-
current.state,
|
|
3657
|
-
)
|
|
3658
|
-
) {
|
|
3659
|
-
this.#interruptOperation(runId, command.operationId);
|
|
3660
|
-
throw new Error("resume-not-allowed");
|
|
3661
|
-
}
|
|
3662
|
-
try {
|
|
3663
|
-
const engineRevision = this.#engineRun(runId).current_revision;
|
|
3664
|
-
const projectedRevision = current.deliveryRevision;
|
|
3665
|
-
if (
|
|
3666
|
-
command.deliveryRevision === undefined &&
|
|
3667
|
-
(current.state === "approval-needed" ||
|
|
3668
|
-
this.#statusByRun(runId).decisionBatch !== undefined ||
|
|
3669
|
-
this.#statusByRun(runId).recovery !== undefined) &&
|
|
3670
|
-
this.#deliverySource.discoverLatest
|
|
3671
|
-
) {
|
|
3672
|
-
const available = await this.#deliverySource.discoverLatest({
|
|
3673
|
-
stage: command.stage,
|
|
3674
|
-
change: command.change,
|
|
3675
|
-
signal,
|
|
3676
|
-
});
|
|
3677
|
-
this.#runStore.assertLease(lease);
|
|
3678
|
-
if (available && available.deliveryRevision > (engineRevision ?? 0)) {
|
|
3679
|
-
command = {
|
|
3680
|
-
...command,
|
|
3681
|
-
deliveryRevision: available.deliveryRevision,
|
|
3682
|
-
receiptHash: available.receiptHash,
|
|
3683
|
-
};
|
|
3684
|
-
}
|
|
3685
|
-
}
|
|
3686
|
-
const deliveryOutOfSync =
|
|
3687
|
-
projectedRevision !== undefined && projectedRevision !== engineRevision;
|
|
3688
|
-
if (
|
|
3689
|
-
current.state === "approval-needed" &&
|
|
3690
|
-
(command.deliveryRevision === undefined ||
|
|
3691
|
-
command.receiptHash === undefined ||
|
|
3692
|
-
command.deliveryRevision <=
|
|
3693
|
-
(engineRevision ?? projectedRevision ?? 0))
|
|
3694
|
-
) {
|
|
3695
|
-
this.#interruptOperation(runId, command.operationId);
|
|
3696
|
-
throw new Error("approval-receipt-required");
|
|
3697
|
-
}
|
|
3698
|
-
if (
|
|
3699
|
-
command.deliveryRevision !== undefined ||
|
|
3700
|
-
engineRevision === null ||
|
|
3701
|
-
deliveryOutOfSync
|
|
3702
|
-
) {
|
|
3703
|
-
this.#transition(
|
|
3704
|
-
runId,
|
|
3705
|
-
"validating-delivery",
|
|
3706
|
-
`${command.operationId}:validating-delivery`,
|
|
3707
|
-
undefined,
|
|
3708
|
-
lease,
|
|
3709
|
-
);
|
|
3710
|
-
emitWorkflowActivity(onActivity, {
|
|
3711
|
-
state: "validating",
|
|
3712
|
-
stage: command.stage,
|
|
3713
|
-
runId,
|
|
3714
|
-
change: command.change,
|
|
3715
|
-
objective: `validate delivery ${command.change}`,
|
|
3716
|
-
legalCommands: ["status", "cancel", "discard"],
|
|
3717
|
-
});
|
|
3718
|
-
const requestedRevision =
|
|
3719
|
-
command.deliveryRevision ??
|
|
3720
|
-
(deliveryOutOfSync ? projectedRevision : undefined);
|
|
3721
|
-
const projectedBinding = current.deliveryBindings.find(
|
|
3722
|
-
(binding) =>
|
|
3723
|
-
binding.gate === "gate-b" && binding.revision === requestedRevision,
|
|
3724
|
-
);
|
|
3725
|
-
const delivery = await this.#loadDelivery(
|
|
3726
|
-
runId,
|
|
3727
|
-
lease,
|
|
3728
|
-
command.stage,
|
|
3729
|
-
command.change,
|
|
3730
|
-
requestedRevision,
|
|
3731
|
-
command.receiptHash ?? projectedBinding?.receiptHash,
|
|
3732
|
-
signal,
|
|
3733
|
-
);
|
|
3734
|
-
this.#admitDelivery(runId, delivery, lease);
|
|
3735
|
-
}
|
|
3736
|
-
if (command.recovery)
|
|
3737
|
-
this.#grantRecovery(
|
|
3738
|
-
runId,
|
|
3739
|
-
command.operationId,
|
|
3740
|
-
command.recovery,
|
|
3741
|
-
lease,
|
|
3742
|
-
);
|
|
3743
|
-
this.#transition(
|
|
3744
|
-
runId,
|
|
3745
|
-
"ready",
|
|
3746
|
-
`${command.operationId}:resume-ready`,
|
|
3747
|
-
undefined,
|
|
3748
|
-
lease,
|
|
3749
|
-
);
|
|
3750
|
-
return this.#launchAdvance(
|
|
3751
|
-
runId,
|
|
3752
|
-
command.operationId,
|
|
3753
|
-
signal,
|
|
3754
|
-
onActivity,
|
|
3755
|
-
);
|
|
3756
|
-
} catch (error) {
|
|
3757
|
-
const cancelled = this.#cancelledDelivery(
|
|
3758
|
-
runId,
|
|
3759
|
-
command.operationId,
|
|
3760
|
-
error,
|
|
3761
|
-
signal,
|
|
3762
|
-
lease,
|
|
3763
|
-
);
|
|
3764
|
-
if (cancelled) return cancelled;
|
|
3765
|
-
const failure = this.#deliveryFailure(
|
|
3766
|
-
runId,
|
|
3767
|
-
command.operationId,
|
|
3768
|
-
error,
|
|
3769
|
-
lease,
|
|
3770
|
-
);
|
|
3771
|
-
if (failure) {
|
|
3772
|
-
return this.#commitOperation(runId, command.operationId, failure);
|
|
3773
|
-
}
|
|
3774
|
-
// A revised delivery may invalidate a grant. Retain its admitted facts,
|
|
3775
|
-
// but never leave a rejected resume looking ready or still validating.
|
|
3776
|
-
if (
|
|
3777
|
-
error instanceof Error &&
|
|
3778
|
-
[
|
|
3779
|
-
"recovery-request-stale",
|
|
3780
|
-
"recovery-evidence-unavailable",
|
|
3781
|
-
"change-work-budget-exhausted",
|
|
3782
|
-
].includes(error.message) &&
|
|
3783
|
-
this.#runStore.status(runId).state === "validating-delivery"
|
|
3784
|
-
) {
|
|
3785
|
-
this.#transition(
|
|
3786
|
-
runId,
|
|
3787
|
-
"paused",
|
|
3788
|
-
`${command.operationId}:recovery-rejected`,
|
|
3789
|
-
error.message,
|
|
3790
|
-
lease,
|
|
3791
|
-
);
|
|
3792
|
-
}
|
|
3793
|
-
this.#interruptOperation(runId, command.operationId);
|
|
3794
|
-
throw error;
|
|
3795
|
-
}
|
|
3796
|
-
}
|
|
3797
|
-
|
|
3798
|
-
#rebind(command: Extract<ControlCommand, { command: "rebind" }>) {
|
|
3799
|
-
const runId = this.#lookupRun(command.stage, command.change);
|
|
3800
|
-
if (!runId) throw new Error("run-not-found");
|
|
3801
|
-
const replay = this.#beginOperation(
|
|
3802
|
-
runId,
|
|
3803
|
-
command.operationId,
|
|
3804
|
-
command.command,
|
|
3805
|
-
);
|
|
3806
|
-
if (replay) return replay;
|
|
3807
|
-
const lease = this.#operationLease(runId, command.operationId);
|
|
3808
|
-
const state = this.#runStore.status(runId).state;
|
|
3809
|
-
if (!["paused", "retryable"].includes(state)) {
|
|
3810
|
-
this.#interruptOperation(runId, command.operationId);
|
|
3811
|
-
throw new Error("rebind-not-allowed");
|
|
3812
|
-
}
|
|
3813
|
-
const task = this.#tasks(runId).find((row) =>
|
|
3814
|
-
["paused", "retryable"].includes(row.state),
|
|
3815
|
-
);
|
|
3816
|
-
if (!task) {
|
|
3817
|
-
this.#interruptOperation(runId, command.operationId);
|
|
3818
|
-
throw new Error("rebind-task-unavailable");
|
|
3819
|
-
}
|
|
3820
|
-
const rebound = this.#worker.rebind({
|
|
3821
|
-
runId,
|
|
3822
|
-
taskId: task.task_id,
|
|
3823
|
-
task: structuredClone(this.#taskPlan(task)),
|
|
3824
|
-
role: "implementation-worker",
|
|
3825
|
-
routeId: command.routeId,
|
|
3826
|
-
});
|
|
3827
|
-
if (!rebound.ok) {
|
|
3828
|
-
this.#interruptOperation(runId, command.operationId);
|
|
3829
|
-
throw new Error(rebound.code);
|
|
3830
|
-
}
|
|
3831
|
-
const routeId = rebound.routeId ?? rebound.route?.id ?? command.routeId;
|
|
3832
|
-
const routeFingerprint =
|
|
3833
|
-
rebound.routeFingerprint ?? rebound.route?.fingerprint;
|
|
3834
|
-
if (
|
|
3835
|
-
!IDENTIFIER.test(routeId) ||
|
|
3836
|
-
(routeFingerprint !== undefined && !SHA256.test(routeFingerprint))
|
|
3837
|
-
) {
|
|
3838
|
-
this.#interruptOperation(runId, command.operationId);
|
|
3839
|
-
throw new Error("route-rebind-result-invalid");
|
|
3840
|
-
}
|
|
3841
|
-
const priorRouteId = task.route_id ?? this.#engineRun(runId).route_id;
|
|
3842
|
-
const priorRouteFingerprint =
|
|
3843
|
-
task.route_fingerprint ?? this.#engineRun(runId).route_fingerprint;
|
|
3844
|
-
this.#leasedTransaction(lease, () => {
|
|
3845
|
-
this.#setTask(runId, task.task_id, {
|
|
3846
|
-
routeId,
|
|
3847
|
-
routeFingerprint: routeFingerprint ?? null,
|
|
3848
|
-
});
|
|
3849
|
-
this.#database
|
|
3850
|
-
.prepare(
|
|
3851
|
-
`UPDATE workflow_engine_runs
|
|
3852
|
-
SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
|
|
3853
|
-
)
|
|
3854
|
-
.run(routeId, routeFingerprint ?? null, runId);
|
|
3855
|
-
});
|
|
3856
|
-
const outcome = {
|
|
3857
|
-
...this.#statusByRun(runId),
|
|
3858
|
-
routeBinding: {
|
|
3859
|
-
routeId,
|
|
3860
|
-
...(routeFingerprint ? { routeFingerprint } : {}),
|
|
3861
|
-
},
|
|
3862
|
-
operation: {
|
|
3863
|
-
kind: "route-rebound",
|
|
3864
|
-
...(priorRouteId ? { priorRouteId } : {}),
|
|
3865
|
-
...(priorRouteFingerprint ? { priorRouteFingerprint } : {}),
|
|
3866
|
-
routeId,
|
|
3867
|
-
...(routeFingerprint ? { routeFingerprint } : {}),
|
|
3868
|
-
},
|
|
3869
|
-
};
|
|
3870
|
-
return this.#commitOperation(runId, command.operationId, outcome);
|
|
3871
|
-
}
|
|
3872
|
-
|
|
3873
|
-
async #controlApply(
|
|
3874
|
-
runId: string,
|
|
3875
|
-
operationId: string,
|
|
3876
|
-
intent: "cancel" | "discard",
|
|
3877
|
-
lease: OperationLease,
|
|
3878
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
3879
|
-
): Promise<Record<string, unknown>> {
|
|
3880
|
-
const transactionId = this.#engineRun(runId).transaction_id;
|
|
3881
|
-
if (!transactionId || !this.#application) {
|
|
3882
|
-
throw new Error("apply-recovery-unavailable");
|
|
3883
|
-
}
|
|
3884
|
-
const context = this.#applicationContext(runId);
|
|
3885
|
-
this.#application.requestControl(transactionId, intent, context);
|
|
3886
|
-
const active = this.#active.get(runId);
|
|
3887
|
-
if (active && active.operationId !== operationId) {
|
|
3888
|
-
active.controller.abort(
|
|
3889
|
-
new Error(
|
|
3890
|
-
intent === "cancel" ? "operation-cancelled" : "run-discarded",
|
|
3891
|
-
),
|
|
3892
|
-
);
|
|
3893
|
-
}
|
|
3894
|
-
if (this.#runStore.status(runId).state === "applying") {
|
|
3895
|
-
this.#transition(
|
|
3896
|
-
runId,
|
|
3897
|
-
"recovering",
|
|
3898
|
-
`${operationId}:${intent}-recovering`,
|
|
3899
|
-
intent === "cancel" ? "operation-cancelled" : "run-discarded",
|
|
3900
|
-
lease,
|
|
3901
|
-
);
|
|
3902
|
-
const recovering = this.#runStore.status(runId);
|
|
3903
|
-
emitWorkflowActivity(onActivity, {
|
|
3904
|
-
state: "recovering",
|
|
3905
|
-
stage: recovering.stage,
|
|
3906
|
-
runId,
|
|
3907
|
-
...(recovering.change ? { change: recovering.change } : {}),
|
|
3908
|
-
objective: recovering.change
|
|
3909
|
-
? `recover change ${recovering.change}`
|
|
3910
|
-
: "recover workflow application",
|
|
3911
|
-
code: intent === "cancel" ? "operation-cancelled" : "run-discarded",
|
|
3912
|
-
legalCommands: ["status"],
|
|
3913
|
-
});
|
|
3914
|
-
}
|
|
3915
|
-
const recovered = await this.#application.recover(transactionId, context);
|
|
3916
|
-
this.#runStore.assertLease(lease);
|
|
3917
|
-
return this.#settleApplication(
|
|
3918
|
-
runId,
|
|
3919
|
-
operationId,
|
|
3920
|
-
recovered,
|
|
3921
|
-
lease,
|
|
3922
|
-
onActivity,
|
|
3923
|
-
);
|
|
3924
|
-
}
|
|
3925
|
-
|
|
3926
|
-
async #abortActiveOperation(runId: string, reason: Error): Promise<void> {
|
|
3927
|
-
const pending: Promise<unknown>[] = [];
|
|
3928
|
-
const amendment = this.#amendments.get(runId);
|
|
3929
|
-
if (amendment) pending.push(amendment);
|
|
3930
|
-
for (const [controller, entry] of this.#commands) {
|
|
3931
|
-
if (
|
|
3932
|
-
!["start", "resume"].includes(entry.command.command) ||
|
|
3933
|
-
this.#lookupRun(entry.command.stage, entry.command.change) !== runId
|
|
3934
|
-
)
|
|
3935
|
-
continue;
|
|
3936
|
-
controller.abort(reason);
|
|
3937
|
-
pending.push(entry.settled);
|
|
3938
|
-
}
|
|
3939
|
-
const active = this.#active.get(runId);
|
|
3940
|
-
if (active) {
|
|
3941
|
-
active.controller.abort(reason);
|
|
3942
|
-
pending.push(active.settled);
|
|
3943
|
-
}
|
|
3944
|
-
const results = await Promise.allSettled(pending);
|
|
3945
|
-
for (const result of results) {
|
|
3946
|
-
if (result.status !== "rejected") continue;
|
|
3947
|
-
const error = result.reason;
|
|
3948
|
-
if (
|
|
3949
|
-
error !== reason &&
|
|
3950
|
-
!(
|
|
3951
|
-
error instanceof Error &&
|
|
3952
|
-
["lease-fenced", "operation-journal-fenced"].includes(error.message)
|
|
3953
|
-
)
|
|
3954
|
-
)
|
|
3955
|
-
throw error;
|
|
3956
|
-
}
|
|
3957
|
-
}
|
|
3958
|
-
|
|
3959
|
-
async #cancel(
|
|
3960
|
-
command: Extract<ControlCommand, { command: "cancel" }>,
|
|
3961
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
3962
|
-
) {
|
|
3963
|
-
const runId = this.#lookupRun(command.stage, command.change);
|
|
3964
|
-
if (!runId) throw new Error("run-not-found");
|
|
3965
|
-
const replay = this.#beginOperation(
|
|
3966
|
-
runId,
|
|
3967
|
-
command.operationId,
|
|
3968
|
-
command.command,
|
|
3969
|
-
);
|
|
3970
|
-
if (replay) return replay;
|
|
3971
|
-
const lease = this.#operationLease(runId, command.operationId);
|
|
3972
|
-
const state = this.#runStore.status(runId).state;
|
|
3973
|
-
let outcome: Record<string, unknown>;
|
|
3974
|
-
if (state === "applying" || state === "recovering") {
|
|
3975
|
-
outcome = await this.#controlApply(
|
|
3976
|
-
runId,
|
|
3977
|
-
command.operationId,
|
|
3978
|
-
"cancel",
|
|
3979
|
-
lease,
|
|
3980
|
-
onActivity,
|
|
3981
|
-
);
|
|
3982
|
-
} else {
|
|
3983
|
-
const activeSettlement = this.#abortActiveOperation(
|
|
3984
|
-
runId,
|
|
3985
|
-
new Error("operation-cancelled"),
|
|
3986
|
-
);
|
|
3987
|
-
const current = this.#runStore.status(runId);
|
|
3988
|
-
if (
|
|
3989
|
-
!["paused", "discarded", "completed", "rejected"].includes(
|
|
3990
|
-
current.state,
|
|
3991
|
-
)
|
|
3992
|
-
) {
|
|
3993
|
-
this.#transition(
|
|
3994
|
-
runId,
|
|
3995
|
-
"paused",
|
|
3996
|
-
`${command.operationId}:cancelled`,
|
|
3997
|
-
"operation-cancelled",
|
|
3998
|
-
lease,
|
|
3999
|
-
);
|
|
4000
|
-
}
|
|
4001
|
-
await activeSettlement;
|
|
4002
|
-
for (const task of this.#tasks(runId)) {
|
|
4003
|
-
if (
|
|
4004
|
-
["phase-ready", "phase-running", "validating"].includes(task.state)
|
|
4005
|
-
) {
|
|
4006
|
-
this.#setTask(
|
|
4007
|
-
runId,
|
|
4008
|
-
task.task_id,
|
|
4009
|
-
{
|
|
4010
|
-
state: "paused",
|
|
4011
|
-
pauseCode: "operation-cancelled",
|
|
4012
|
-
queuePosition: null,
|
|
4013
|
-
},
|
|
4014
|
-
lease,
|
|
4015
|
-
);
|
|
4016
|
-
}
|
|
4017
|
-
}
|
|
4018
|
-
outcome = this.#statusByRun(runId);
|
|
4019
|
-
}
|
|
4020
|
-
outcome = {
|
|
4021
|
-
...outcome,
|
|
4022
|
-
operation: { kind: "operation-cancelled" },
|
|
4023
|
-
};
|
|
4024
|
-
return this.#commitOperation(runId, command.operationId, outcome);
|
|
4025
|
-
}
|
|
4026
|
-
|
|
4027
|
-
async #discard(
|
|
4028
|
-
command: Extract<ControlCommand, { command: "discard" }>,
|
|
4029
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
4030
|
-
) {
|
|
4031
|
-
const runId = this.#lookupRun(command.stage, command.change);
|
|
4032
|
-
if (!runId) throw new Error("run-not-found");
|
|
4033
|
-
const replay = this.#beginOperation(
|
|
4034
|
-
runId,
|
|
4035
|
-
command.operationId,
|
|
4036
|
-
command.command,
|
|
4037
|
-
);
|
|
4038
|
-
if (replay) return replay;
|
|
4039
|
-
const lease = this.#operationLease(runId, command.operationId);
|
|
4040
|
-
const state = this.#runStore.status(runId).state;
|
|
4041
|
-
let outcome: Record<string, unknown>;
|
|
4042
|
-
if (state === "applying" || state === "recovering") {
|
|
4043
|
-
outcome = await this.#controlApply(
|
|
4044
|
-
runId,
|
|
4045
|
-
command.operationId,
|
|
4046
|
-
"discard",
|
|
4047
|
-
lease,
|
|
4048
|
-
onActivity,
|
|
4049
|
-
);
|
|
4050
|
-
} else {
|
|
4051
|
-
const activeSettlement = this.#abortActiveOperation(
|
|
4052
|
-
runId,
|
|
4053
|
-
new Error("run-discarded"),
|
|
4054
|
-
);
|
|
4055
|
-
const current = this.#runStore.status(runId);
|
|
4056
|
-
if (current.state !== "discarded") {
|
|
4057
|
-
if (["completed", "rejected"].includes(current.state)) {
|
|
4058
|
-
this.#interruptOperation(runId, command.operationId);
|
|
4059
|
-
throw new Error("discard-not-allowed");
|
|
4060
|
-
}
|
|
4061
|
-
this.#transition(
|
|
4062
|
-
runId,
|
|
4063
|
-
"discarded",
|
|
4064
|
-
`${command.operationId}:discarded`,
|
|
4065
|
-
undefined,
|
|
4066
|
-
lease,
|
|
4067
|
-
);
|
|
4068
|
-
await activeSettlement;
|
|
4069
|
-
this.#cleanupTerminal(runId, command.operationId, lease);
|
|
4070
|
-
} else {
|
|
4071
|
-
await activeSettlement;
|
|
4072
|
-
}
|
|
4073
|
-
outcome = this.#statusByRun(runId);
|
|
4074
|
-
}
|
|
4075
|
-
return this.#commitOperation(runId, command.operationId, outcome);
|
|
4076
|
-
}
|
|
4077
|
-
|
|
4078
|
-
execute(
|
|
4079
|
-
value: unknown,
|
|
4080
|
-
signal?: AbortSignal,
|
|
4081
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
4082
|
-
): Promise<Record<string, unknown>> {
|
|
4083
|
-
let command: ControlCommand;
|
|
4084
|
-
try {
|
|
4085
|
-
this.#assertOpen();
|
|
4086
|
-
if (this.#closing) throw new Error("workflow-engine-closed");
|
|
4087
|
-
command = assertControlCommand(value);
|
|
4088
|
-
} catch (error) {
|
|
4089
|
-
return Promise.reject(error);
|
|
4090
|
-
}
|
|
4091
|
-
const controller = new AbortController();
|
|
4092
|
-
const onAbort = () => controller.abort(signal?.reason);
|
|
4093
|
-
if (signal?.aborted) onAbort();
|
|
4094
|
-
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
4095
|
-
// Register before execution can yield or invoke external callbacks.
|
|
4096
|
-
const settled = Promise.resolve()
|
|
4097
|
-
.then(() => this.#executeCommand(command, controller.signal, onActivity))
|
|
4098
|
-
.finally(() => {
|
|
4099
|
-
signal?.removeEventListener("abort", onAbort);
|
|
4100
|
-
this.#commands.delete(controller);
|
|
4101
|
-
});
|
|
4102
|
-
this.#commands.set(controller, { command, settled });
|
|
4103
|
-
return settled;
|
|
4104
|
-
}
|
|
4105
|
-
|
|
4106
|
-
async #executeCommand(
|
|
4107
|
-
command: ControlCommand,
|
|
4108
|
-
signal?: AbortSignal,
|
|
4109
|
-
onActivity?: (event: WorkflowActivityUpdate) => void,
|
|
4110
|
-
): Promise<Record<string, unknown>> {
|
|
4111
|
-
switch (command.command) {
|
|
4112
|
-
case "start":
|
|
4113
|
-
return this.#withAvailableDelivery(
|
|
4114
|
-
await this.#start(command, signal, onActivity),
|
|
4115
|
-
signal,
|
|
4116
|
-
);
|
|
4117
|
-
case "status": {
|
|
4118
|
-
const runId = this.#lookupRun(command.stage, command.change);
|
|
4119
|
-
if (!runId) {
|
|
4120
|
-
return {
|
|
4121
|
-
stage: command.stage,
|
|
4122
|
-
change: command.change,
|
|
4123
|
-
state: "not-started",
|
|
4124
|
-
durable: true,
|
|
4125
|
-
completed: false,
|
|
4126
|
-
legalCommands: ["start"],
|
|
4127
|
-
tasks: [],
|
|
4128
|
-
queue: [],
|
|
4129
|
-
};
|
|
4130
|
-
}
|
|
4131
|
-
this.#recoverLegacyContextApproval(runId);
|
|
4132
|
-
return this.#withAvailableDelivery(this.#statusByRun(runId), signal);
|
|
4133
|
-
}
|
|
4134
|
-
case "resume":
|
|
4135
|
-
return this.#withAvailableDelivery(
|
|
4136
|
-
await this.#resume(command, signal, onActivity),
|
|
4137
|
-
signal,
|
|
4138
|
-
);
|
|
4139
|
-
case "rebind":
|
|
4140
|
-
return this.#rebind(command);
|
|
4141
|
-
case "cancel":
|
|
4142
|
-
return this.#cancel(command, onActivity);
|
|
4143
|
-
case "discard":
|
|
4144
|
-
return this.#discard(command, onActivity);
|
|
4145
|
-
}
|
|
4146
|
-
}
|
|
4147
|
-
|
|
4148
|
-
prepareBootstrapHandoff(
|
|
4149
|
-
input: Record<string, unknown>,
|
|
4150
|
-
): Record<string, unknown> {
|
|
4151
|
-
this.#assertOpen();
|
|
4152
|
-
const acceptanceFacts = normalizeBootstrapAcceptanceFacts(
|
|
4153
|
-
input.acceptanceFacts,
|
|
4154
|
-
);
|
|
4155
|
-
const workspaceManifestHash =
|
|
4156
|
-
typeof input.workspaceManifestHash === "string" &&
|
|
4157
|
-
SHA256.test(input.workspaceManifestHash)
|
|
4158
|
-
? input.workspaceManifestHash
|
|
4159
|
-
: undefined;
|
|
4160
|
-
if (
|
|
4161
|
-
typeof input.operationId !== "string" ||
|
|
4162
|
-
!IDENTIFIER.test(input.operationId) ||
|
|
4163
|
-
typeof input.runId !== "string" ||
|
|
4164
|
-
typeof input.change !== "string" ||
|
|
4165
|
-
!CHANGE_NAME.test(input.change) ||
|
|
4166
|
-
typeof input.receiptHash !== "string" ||
|
|
4167
|
-
!SHA256.test(input.receiptHash) ||
|
|
4168
|
-
typeof input.graphHash !== "string" ||
|
|
4169
|
-
!SHA256.test(input.graphHash) ||
|
|
4170
|
-
typeof input.acceptanceHash !== "string" ||
|
|
4171
|
-
!SHA256.test(input.acceptanceHash) ||
|
|
4172
|
-
(acceptanceFacts !== undefined) !==
|
|
4173
|
-
(workspaceManifestHash !== undefined) ||
|
|
4174
|
-
(acceptanceFacts !== undefined &&
|
|
4175
|
-
workspaceManifestHash !== undefined &&
|
|
4176
|
-
bootstrapAcceptanceHash(acceptanceFacts, workspaceManifestHash) !==
|
|
4177
|
-
input.acceptanceHash)
|
|
4178
|
-
) {
|
|
4179
|
-
throw new Error("bootstrap-handoff-invalid");
|
|
4180
|
-
}
|
|
4181
|
-
this.#runStore.status(input.runId);
|
|
4182
|
-
const existing = this.#database
|
|
4183
|
-
.prepare(
|
|
4184
|
-
`SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
|
|
4185
|
-
FROM bootstrap_handoffs WHERE receipt_hash = ?`,
|
|
4186
|
-
)
|
|
4187
|
-
.get(input.receiptHash) as BootstrapRow | undefined;
|
|
4188
|
-
if (existing) {
|
|
4189
|
-
const outcome = parseJsonRecord(existing.facts_json);
|
|
4190
|
-
const existingAcceptanceFacts = normalizeBootstrapAcceptanceFacts(
|
|
4191
|
-
outcome.acceptanceFacts,
|
|
4192
|
-
);
|
|
4193
|
-
const storedBindingValid =
|
|
4194
|
-
outcome.handoffId === existing.handoff_id &&
|
|
4195
|
-
outcome.runId === existing.owner_run_id &&
|
|
4196
|
-
outcome.receiptHash === existing.receipt_hash &&
|
|
4197
|
-
outcome.selectorBeforeCutover === "bootstrap" &&
|
|
4198
|
-
existing.state === "prepared" &&
|
|
4199
|
-
outcome.state === "prepared" &&
|
|
4200
|
-
outcome.selectorCasPending === true;
|
|
4201
|
-
if (!storedBindingValid) throw new Error("bootstrap-handoff-invalid");
|
|
4202
|
-
const stableBindingMatches =
|
|
4203
|
-
existing.owner_run_id === input.runId &&
|
|
4204
|
-
outcome.change === input.change &&
|
|
4205
|
-
outcome.graphHash === input.graphHash;
|
|
4206
|
-
if (!stableBindingMatches) {
|
|
4207
|
-
throw new Error("bootstrap-handoff-conflict");
|
|
4208
|
-
}
|
|
4209
|
-
const acceptanceMatches =
|
|
4210
|
-
outcome.acceptanceHash === input.acceptanceHash &&
|
|
4211
|
-
canonicalJson(existingAcceptanceFacts ?? null) ===
|
|
4212
|
-
canonicalJson(acceptanceFacts ?? null) &&
|
|
4213
|
-
outcome.workspaceManifestHash === workspaceManifestHash;
|
|
4214
|
-
if (acceptanceMatches) return structuredClone(outcome);
|
|
4215
|
-
if (
|
|
4216
|
-
existing.state !== "prepared" ||
|
|
4217
|
-
outcome.state !== "prepared" ||
|
|
4218
|
-
outcome.selectorBeforeCutover !== "bootstrap" ||
|
|
4219
|
-
outcome.selectorCasPending !== true ||
|
|
4220
|
-
acceptanceFacts === undefined ||
|
|
4221
|
-
workspaceManifestHash === undefined
|
|
4222
|
-
) {
|
|
4223
|
-
throw new Error("bootstrap-handoff-conflict");
|
|
4224
|
-
}
|
|
4225
|
-
const refreshedHandoffId = `handoff-${hash(
|
|
4226
|
-
input.runId,
|
|
4227
|
-
input.receiptHash,
|
|
4228
|
-
input.graphHash,
|
|
4229
|
-
input.acceptanceHash,
|
|
4230
|
-
workspaceManifestHash,
|
|
4231
|
-
).slice(0, 40)}`;
|
|
4232
|
-
const refreshed = {
|
|
4233
|
-
...outcome,
|
|
4234
|
-
handoffId: refreshedHandoffId,
|
|
4235
|
-
acceptanceHash: input.acceptanceHash,
|
|
4236
|
-
acceptanceFacts,
|
|
4237
|
-
workspaceManifestHash,
|
|
4238
|
-
};
|
|
4239
|
-
this.#transaction(() => {
|
|
4240
|
-
const result = this.#database
|
|
4241
|
-
.prepare(
|
|
4242
|
-
`UPDATE bootstrap_handoffs
|
|
4243
|
-
SET handoff_id = ?, facts_json = ?
|
|
4244
|
-
WHERE handoff_id = ? AND state = 'prepared' AND facts_json = ?`,
|
|
4245
|
-
)
|
|
4246
|
-
.run(
|
|
4247
|
-
refreshedHandoffId,
|
|
4248
|
-
JSON.stringify(refreshed),
|
|
4249
|
-
existing.handoff_id,
|
|
4250
|
-
existing.facts_json,
|
|
4251
|
-
);
|
|
4252
|
-
if (Number(result.changes) !== 1) {
|
|
4253
|
-
throw new Error("bootstrap-handoff-conflict");
|
|
4254
|
-
}
|
|
4255
|
-
});
|
|
4256
|
-
return structuredClone(refreshed);
|
|
4257
|
-
}
|
|
4258
|
-
const handoffId = `handoff-${hash(
|
|
4259
|
-
input.runId,
|
|
4260
|
-
input.receiptHash,
|
|
4261
|
-
input.graphHash,
|
|
4262
|
-
input.acceptanceHash,
|
|
4263
|
-
workspaceManifestHash ?? "acceptance-pending",
|
|
4264
|
-
).slice(0, 40)}`;
|
|
4265
|
-
const outcome = {
|
|
4266
|
-
handoffId,
|
|
4267
|
-
state: "prepared",
|
|
4268
|
-
runId: input.runId,
|
|
4269
|
-
change: input.change,
|
|
4270
|
-
receiptHash: input.receiptHash,
|
|
4271
|
-
graphHash: input.graphHash,
|
|
4272
|
-
acceptanceHash: input.acceptanceHash,
|
|
4273
|
-
...(acceptanceFacts ? { acceptanceFacts } : {}),
|
|
4274
|
-
...(workspaceManifestHash ? { workspaceManifestHash } : {}),
|
|
4275
|
-
selectorBeforeCutover: "bootstrap",
|
|
4276
|
-
selectorCasPending: true,
|
|
4277
|
-
};
|
|
4278
|
-
const preparedRows = this.#database
|
|
4279
|
-
.prepare(
|
|
4280
|
-
`SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
|
|
4281
|
-
FROM bootstrap_handoffs WHERE state = 'prepared'
|
|
4282
|
-
ORDER BY handoff_id`,
|
|
4283
|
-
)
|
|
4284
|
-
.all() as unknown as BootstrapRow[];
|
|
4285
|
-
if (preparedRows.length > 0) throw new Error("bootstrap-handoff-conflict");
|
|
4286
|
-
this.#transaction(() => {
|
|
4287
|
-
this.#database
|
|
4288
|
-
.prepare(
|
|
4289
|
-
`INSERT INTO bootstrap_handoffs(
|
|
4290
|
-
handoff_id, owner_run_id, receipt_hash, state, facts_json
|
|
4291
|
-
) VALUES (?, ?, ?, 'prepared', ?)`,
|
|
4292
|
-
)
|
|
4293
|
-
.run(
|
|
4294
|
-
handoffId,
|
|
4295
|
-
String(input.runId),
|
|
4296
|
-
String(input.receiptHash),
|
|
4297
|
-
JSON.stringify(outcome),
|
|
4298
|
-
);
|
|
4299
|
-
});
|
|
4300
|
-
return structuredClone(outcome);
|
|
4301
|
-
}
|
|
4302
|
-
|
|
4303
|
-
inspectBootstrapHandoff(receiptHash: string): Record<string, unknown> {
|
|
4304
|
-
this.#assertOpen();
|
|
4305
|
-
if (!SHA256.test(receiptHash)) throw new Error("bootstrap-handoff-invalid");
|
|
4306
|
-
const row = this.#database
|
|
4307
|
-
.prepare(
|
|
4308
|
-
`SELECT facts_json FROM bootstrap_handoffs WHERE receipt_hash = ?`,
|
|
4309
|
-
)
|
|
4310
|
-
.get(receiptHash) as { facts_json: string } | undefined;
|
|
4311
|
-
if (!row) throw new Error("bootstrap-handoff-not-found");
|
|
4312
|
-
return parseJsonRecord(row.facts_json);
|
|
4313
|
-
}
|
|
4314
|
-
|
|
4315
|
-
close(): Promise<void> {
|
|
4316
|
-
if (this.#closing) return this.#closing;
|
|
4317
|
-
if (this.#closed) return Promise.resolve();
|
|
4318
|
-
this.#closing = Promise.resolve().then(async () => {
|
|
4319
|
-
if (this.#orphanRecoveryTimer) {
|
|
4320
|
-
clearTimeout(this.#orphanRecoveryTimer);
|
|
4321
|
-
this.#orphanRecoveryTimer = undefined;
|
|
4322
|
-
}
|
|
4323
|
-
const reason = new Error("workflow-engine-closed");
|
|
4324
|
-
const commands = [...this.#commands];
|
|
4325
|
-
for (const [controller] of commands) controller.abort(reason);
|
|
4326
|
-
const active = [...this.#active.values()];
|
|
4327
|
-
for (const operation of active) operation.controller.abort(reason);
|
|
4328
|
-
await Promise.allSettled(commands.map(([, entry]) => entry.settled));
|
|
4329
|
-
await Promise.allSettled(active.map((operation) => operation.settled));
|
|
4330
|
-
await Promise.allSettled([...this.#amendments.values()]);
|
|
4331
|
-
for (const held of this.#operationLeases.values())
|
|
4332
|
-
clearInterval(held.timer);
|
|
4333
|
-
this.#operationLeases.clear();
|
|
4334
|
-
this.#lifecycle?.close?.();
|
|
4335
|
-
this.#database.close();
|
|
4336
|
-
this.#runStore.close();
|
|
4337
|
-
this.#closed = true;
|
|
4338
|
-
});
|
|
4339
|
-
return this.#closing;
|
|
4340
|
-
}
|
|
4341
|
-
}
|