@igorjpimenta/manifold 0.0.0-stage → 2026.11.22-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/.migrations/ops/0000_futuristic_karma.sql +111 -0
  2. package/.migrations/ops/0001_low_maximus.sql +7 -0
  3. package/.migrations/ops/meta/0000_snapshot.json +763 -0
  4. package/.migrations/ops/meta/0001_snapshot.json +173 -0
  5. package/.migrations/ops/meta/_journal.json +20 -0
  6. package/.migrations/runtime/0000_real_boomer.sql +180 -0
  7. package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
  8. package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
  9. package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
  10. package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
  11. package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
  12. package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
  13. package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
  14. package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
  15. package/.migrations/runtime/0009_lying_shocker.sql +96 -0
  16. package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
  17. package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
  18. package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
  19. package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
  20. package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
  21. package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
  22. package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
  23. package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
  24. package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
  25. package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
  26. package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
  27. package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
  28. package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
  29. package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
  30. package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
  31. package/.migrations/runtime/meta/_journal.json +97 -0
  32. package/README.md +1583 -2
  33. package/RELEASING.md +89 -0
  34. package/agent/fleet.js +100 -0
  35. package/bin/cli.js +854 -0
  36. package/bin/manifold.js +4 -0
  37. package/boot/flow-revival.js +536 -0
  38. package/budget/index.js +495 -0
  39. package/budget/lib.js +59 -0
  40. package/dashboard/cost.js +528 -0
  41. package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
  42. package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
  43. package/dashboard/dist/index.html +13 -0
  44. package/dashboard/event-log.js +267 -0
  45. package/dashboard/index.html +12 -0
  46. package/dashboard/package-lock.json +2877 -0
  47. package/dashboard/package.json +37 -0
  48. package/dashboard/pipeline-graph.js +131 -0
  49. package/dashboard/postcss.config.js +6 -0
  50. package/dashboard/search.js +183 -0
  51. package/dashboard/server.js +1015 -0
  52. package/dashboard/src/App.tsx +42 -0
  53. package/dashboard/src/api/client.ts +118 -0
  54. package/dashboard/src/api/types.ts +510 -0
  55. package/dashboard/src/api/ws.tsx +112 -0
  56. package/dashboard/src/components/CancelButton.tsx +63 -0
  57. package/dashboard/src/components/ClickToCopy.tsx +68 -0
  58. package/dashboard/src/components/CopyButton.tsx +61 -0
  59. package/dashboard/src/components/DailyCostTrend.tsx +85 -0
  60. package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
  61. package/dashboard/src/components/Funnel.tsx +149 -0
  62. package/dashboard/src/components/GateForm.tsx +158 -0
  63. package/dashboard/src/components/GraphView.tsx +302 -0
  64. package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
  65. package/dashboard/src/components/Histogram.tsx +292 -0
  66. package/dashboard/src/components/Layout.tsx +28 -0
  67. package/dashboard/src/components/NeedsAttention.tsx +98 -0
  68. package/dashboard/src/components/NodeContextCard.tsx +132 -0
  69. package/dashboard/src/components/PathTrail.tsx +35 -0
  70. package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
  71. package/dashboard/src/components/RerunButton.tsx +46 -0
  72. package/dashboard/src/components/RunComparison.tsx +53 -0
  73. package/dashboard/src/components/Sparkline.tsx +94 -0
  74. package/dashboard/src/components/StatStrip.tsx +147 -0
  75. package/dashboard/src/components/StateTree.tsx +84 -0
  76. package/dashboard/src/components/StatusBadge.tsx +18 -0
  77. package/dashboard/src/components/SubflowPathPills.tsx +38 -0
  78. package/dashboard/src/components/SubflowSummary.tsx +24 -0
  79. package/dashboard/src/components/SubflowsCard.tsx +327 -0
  80. package/dashboard/src/components/TodayStrip.tsx +86 -0
  81. package/dashboard/src/components/TranscriptPane.tsx +99 -0
  82. package/dashboard/src/components/TranscriptRow.tsx +69 -0
  83. package/dashboard/src/components/ui/badge.tsx +38 -0
  84. package/dashboard/src/components/ui/button.tsx +28 -0
  85. package/dashboard/src/components/ui/card.tsx +42 -0
  86. package/dashboard/src/components/ui/skeleton.tsx +9 -0
  87. package/dashboard/src/components/ui/table.tsx +62 -0
  88. package/dashboard/src/index.css +26 -0
  89. package/dashboard/src/lib/anomaly.ts +42 -0
  90. package/dashboard/src/lib/format.ts +30 -0
  91. package/dashboard/src/lib/step-history.ts +13 -0
  92. package/dashboard/src/lib/subflow.ts +154 -0
  93. package/dashboard/src/lib/use-click-outside.ts +28 -0
  94. package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
  95. package/dashboard/src/lib/utils.ts +6 -0
  96. package/dashboard/src/main.tsx +10 -0
  97. package/dashboard/src/types/shims.d.ts +4 -0
  98. package/dashboard/src/views/Home.tsx +478 -0
  99. package/dashboard/src/views/PipelineDetail.tsx +1185 -0
  100. package/dashboard/src/views/PipelinesList.tsx +111 -0
  101. package/dashboard/src/views/RunDetail.tsx +248 -0
  102. package/dashboard/src/views/Search.tsx +196 -0
  103. package/dashboard/stats.js +668 -0
  104. package/dashboard/tailwind.config.ts +43 -0
  105. package/dashboard/tree-stats.js +12 -0
  106. package/dashboard/tsconfig.json +25 -0
  107. package/dashboard/views.js +231 -0
  108. package/dashboard/vite.config.ts +34 -0
  109. package/drizzle.config.js +12 -0
  110. package/drizzle.config.ops.js +12 -0
  111. package/elevation.js +116 -0
  112. package/engine/admission-promotion-resume.js +127 -0
  113. package/engine/agent-dispatch.js +64 -0
  114. package/engine/child-resume-trigger.js +315 -0
  115. package/engine/executor-budget.js +87 -0
  116. package/engine/executor-cancel.js +83 -0
  117. package/engine/executor-errors.js +122 -0
  118. package/engine/executor-helpers.js +254 -0
  119. package/engine/executor-nodes.js +292 -0
  120. package/engine/executor-persist.js +212 -0
  121. package/engine/executor-subflow-children.js +275 -0
  122. package/engine/executor.js +1013 -0
  123. package/engine/global-concurrency.js +33 -0
  124. package/engine/loader.js +528 -0
  125. package/engine/needs-resolver.js +229 -0
  126. package/engine/pipelines-watcher.js +133 -0
  127. package/engine/result-replay.js +34 -0
  128. package/engine/revival-messages.js +11 -0
  129. package/engine/spawn-child.js +55 -0
  130. package/engine/wait-for-flow-settlement.js +109 -0
  131. package/events/catalog.js +26 -0
  132. package/events/emit.js +44 -0
  133. package/events/kinds.js +118 -0
  134. package/events/serve-ws.js +74 -0
  135. package/events/stream.js +35 -0
  136. package/events/ws-dialog.js +257 -0
  137. package/events/ws-frame.js +194 -0
  138. package/events/ws-run-events.js +70 -0
  139. package/events.js +32 -0
  140. package/gates/gate-resolver.js +576 -0
  141. package/gates/paused-flows.js +55 -0
  142. package/graphql/columns.js +67 -0
  143. package/graphql/schema.js +333 -0
  144. package/index.js +49 -0
  145. package/mcp/http.js +159 -0
  146. package/mcp/identity.js +94 -0
  147. package/mcp/stdio.js +80 -0
  148. package/mcp/tools.js +245 -0
  149. package/mcp/wire-ops.js +178 -0
  150. package/package.json +49 -4
  151. package/routes/bearer-auth.js +57 -0
  152. package/routes/discuss-audio.js +87 -0
  153. package/routes/discuss-component.js +39 -0
  154. package/routes/discuss-draft.js +159 -0
  155. package/routes/discuss-helpers.js +20 -0
  156. package/routes/discuss.js +848 -0
  157. package/routes/failures.js +104 -0
  158. package/routes/graphql.js +61 -0
  159. package/routes/health.js +54 -0
  160. package/routes/operator-discuss.js +193 -0
  161. package/routes/operator-resume.js +32 -0
  162. package/routes/operator-retry.js +301 -0
  163. package/routes/operator.js +539 -0
  164. package/routes/pipelines.js +118 -0
  165. package/routes/runs.js +1129 -0
  166. package/scripts/ci/import-check.mjs +50 -0
  167. package/scripts/link-core-self.mjs +31 -0
  168. package/scripts/with-guard.sh +24 -0
  169. package/server/app.js +503 -0
  170. package/server/registry.js +53 -0
  171. package/server/serve.js +163 -0
  172. package/state/admission-queue-store.js +325 -0
  173. package/state/attribution-registries.js +102 -0
  174. package/state/batch-writer.js +162 -0
  175. package/state/db-guard.js +63 -0
  176. package/state/db-health.js +27 -0
  177. package/state/db.js +122 -0
  178. package/state/discuss-store.js +299 -0
  179. package/state/failure-store.js +427 -0
  180. package/state/migrate.js +104 -0
  181. package/state/node-cost.js +67 -0
  182. package/state/node-executions.js +775 -0
  183. package/state/node-health-store.js +124 -0
  184. package/state/ops-merge.js +93 -0
  185. package/state/paths.js +146 -0
  186. package/state/pipeline-dirs.js +99 -0
  187. package/state/pipeline-registry.js +449 -0
  188. package/state/run-store.js +697 -0
  189. package/state/schema.js +438 -0
  190. package/state/schema.ops.js +44 -0
  191. package/state/session-index.js +177 -0
  192. package/state/session-keys.js +39 -0
  193. package/state/state-store.js +311 -0
  194. package/state/subflow-state.js +126 -0
  195. package/state/tool-handoff.js +169 -0
  196. package/telemetry/budget-alert.js +353 -0
  197. package/telemetry/cli.js +331 -0
  198. package/telemetry/grammar.js +119 -0
  199. package/telemetry/http-budget.js +132 -0
  200. package/telemetry/http-cost.js +670 -0
  201. package/telemetry/http-helpers.js +134 -0
  202. package/telemetry/http-tree.js +349 -0
  203. package/telemetry/http.js +44 -0
  204. package/telemetry/index.js +307 -0
  205. package/telemetry/ingest.js +134 -0
  206. package/telemetry/labels.js +115 -0
  207. package/telemetry/pricer.js +77 -0
  208. package/telemetry/resolver.js +314 -0
  209. package/telemetry/store.js +147 -0
  210. package/utils/run-patch.js +43 -0
@@ -0,0 +1,1013 @@
1
+ // executor.js — runs a manifold pipeline against a managed flow_run.
2
+ //
3
+ // Lifecycle of one flow:
4
+ // 1. Caller invokes start(pipelineId)
5
+ // 2. We create_flow_runs row at status='running', lastStep='__start__'.
6
+ // 3. We enter the run loop:
7
+ // a. Read flow row.
8
+ // b. If terminal — return.
9
+ // c. If blocked — return (resume() wakes us back up later).
10
+ // d. Resolve edge from lastStep → nextNodeId.
11
+ // e. If nextNodeId === '__end__': persist finish() with current state; return.
12
+ // f. Execute the node; collect stateUpdate (and waitJson if gate).
13
+ // g. Apply update onto state.
14
+ // h. Persist: setRunning() to advance lastStep, or setWaiting() to block.
15
+ // i. Loop.
16
+
17
+ import { randomUUID } from "node:crypto";
18
+ import { mkdir, writeFile } from "node:fs/promises";
19
+ import path from "node:path";
20
+ import os from "node:os";
21
+ import { getNodeExecutionsStore } from "../state/node-executions.js";
22
+ import { nodeEmitter } from "../events.js";
23
+ import { resolveStorePaths } from "../state/paths.js";
24
+
25
+ import {
26
+ isTerminal, isBlocked, isQueued, shortSummary, deepClone, resolveEdge,
27
+ validateGateLayout, isFreshNodeEntry, describeWaitFallback, assembleState,
28
+ } from "./executor-helpers.js";
29
+ import { NodeBlocked, GatewayHandoffNeeded } from "./executor-errors.js";
30
+ import { REVIVAL_MSG } from "./revival-messages.js";
31
+ import { createBudgetGate } from "./executor-budget.js";
32
+ import { createPersist } from "./executor-persist.js";
33
+ import { createAgentDispatch } from "./agent-dispatch.js";
34
+ import { dispatchSubflowChildren } from "./executor-subflow-children.js";
35
+ import { createNodeRunner } from "./executor-nodes.js";
36
+ import { completeCancellation } from "./executor-cancel.js";
37
+
38
+ export function createExecutor({ api, config, approvalQueue, pipelinesDir, store, pipelinesRef, ensurePipelinesLoaded, logger, runTurn, failureStore, nodeHealthStore, notifyFlowEnteredWaiting, sessionIndex, nodeExecutionsStore, runStore, admissionQueueStore, emit }) {
39
+ // Explicit params win; api is the fallback the adapter still
40
+ // supplies (tests construct with api only).
41
+ const _config = config ?? api?.config ?? null;
42
+ const _lineage = api?.runtime?.lineage ?? null;
43
+ const _pipelinesDir = pipelinesDir ?? _config?.pipelinesDir ?? resolveStorePaths().pipelinesDir;
44
+ // Injectable for tests (defaults to the real singleton in production).
45
+ // A test constructing a real createExecutor() without this override would
46
+ // write its fixture rows into the real store's runtime.sqlite; store/
47
+ // branchService/failureStore etc. above already follow this DI pattern.
48
+ const nodeExecutions = nodeExecutionsStore ?? getNodeExecutionsStore();
49
+
50
+ // Wraps nodeExecutions.write() for the write-then-fail call sites below —
51
+ // write() can throw (its own self-validating kind-invariant check), and
52
+ // unlike its normal position (after persistFailRetrying, so the fail
53
+ // status is already durably persisted before any crash), these sites
54
+ // call it FIRST so the failure can be linked to the invocation/execution
55
+ // it just created. A local catch here preserves "the fail status always
56
+ // gets persisted" even if write() throws — persistFailRetrying still
57
+ // runs, just without a linked id, instead of the throw propagating out
58
+ // of runFlow() before the fail status is ever written.
59
+ async function safeWriteExecution(args) {
60
+ try {
61
+ return (await nodeExecutions.write(args)) ?? {};
62
+ } catch (err) {
63
+ logger.warn(`manifold [${args.flowId.slice(0, 8)}] nodeExecutions.write threw: ${err?.message ?? err}`);
64
+ return {};
65
+ }
66
+ }
67
+
68
+ const { maybeBudgetGate } = createBudgetGate({
69
+ getApprovalQueue: () => approvalQueue ?? api?.runtime?.approvalQueue,
70
+ logger,
71
+ });
72
+
73
+ const { persistRunningRetrying, persistWaitingRetrying, persistSkipRetrying, persistFinishRetrying, persistFailRetrying } = createPersist({
74
+ store, failureStore, notifyFlowEnteredWaiting, logger,
75
+ });
76
+
77
+ // The harness injects runTurn — core never speaks the runtime's call
78
+ // shape, so a node dispatch is one function call from the engine's point
79
+ // of view (see engine/agent-dispatch.js and the harness's run-turn.js).
80
+ const { dispatchAgentWithRetry } = createAgentDispatch({
81
+ runTurn,
82
+ sessionIndex,
83
+ logger,
84
+ emit,
85
+ });
86
+
87
+ // nodeCtx is assigned after start/runFlow are defined (see below).
88
+ // Subflow/node runners read it via getter at call time, not at factory-construction
89
+ // time, so the undefined-at-construction gap is safe.
90
+ let nodeCtx;
91
+
92
+ // Every pipeline's subflow node now runs under childFlows:true — this is a
93
+ // direct call to the only dispatch path that remains. `runFlow` is a
94
+ // hoisted function declaration defined below — safe to reference here.
95
+ // `kind:"subflow"` covers both the many-children-via-selector shape and
96
+ // single-child use.
97
+ async function executeSubflowNode({ node, nodeId, flowId, state }) {
98
+ // A subflow node's own selector(state) can legitimately need a PRIOR
99
+ // subflow node's aggregated per-subflow results (e.g. PROPOSE's selector
100
+ // reading state._subflowGroups.SCORE to resolve each shortlisted
101
+ // listing's score data) — same reason dispatchCodeEntry
102
+ // (executor-nodes.js) enriches state before calling a "code" node's
103
+ // run(). Subflow dispatch never does this enrichment, so any such
104
+ // selector silently sees undefined subflow groups and resolves to zero
105
+ // items regardless of how many upstream branches actually succeeded.
106
+ //
107
+ // assembleState(baseState: state) computes _subflowGroups here (the
108
+ // same getAllSubflowGroups call, folded in) and additionally overlays
109
+ // each node's resultJson on top of the real stateJson passed as
110
+ // baseState. The overlay is a no-op against real flows (resultJson-derived
111
+ // values are byte-identical to state_json's), so the enrichment stays
112
+ // scoped to the single narrowest read site (node.selector) rather than
113
+ // all 5 root call sites at once.
114
+ const enrichedState = await assembleState(flowId, { runStore, nodeExecutionsStore: nodeExecutions, baseState: state });
115
+ return dispatchSubflowChildren({
116
+ node, nodeId, flowId, state: enrichedState, store, runStore, nodeExecutionsStore: nodeExecutions,
117
+ maybeBudgetGate, runFlow, logger, lineage: _lineage, pipelinesRef, admissionQueueStore,
118
+ });
119
+ }
120
+
121
+ const { executeNode } = createNodeRunner({
122
+ maybeBudgetGate, dispatchAgentWithRetry, executeSubflowNode,
123
+ api, config: _config, lineage: _lineage, pipelinesDir: _pipelinesDir, logger,
124
+ getNodeCtx: () => nodeCtx,
125
+ runStore, nodeExecutionsStore: nodeExecutions,
126
+ nodeHealthStore, pipelinesRef,
127
+ });
128
+
129
+ /**
130
+ * Create the flow_run row for a new pipeline run and return the record.
131
+ * Does NOT start the execution loop — call `runFlow(record.flowId)` to drive it.
132
+ */
133
+ async function start(pipelineId, args = {}, triggeredBy = null) {
134
+ const pipeline = pipelinesRef.value.get(pipelineId);
135
+ if (!pipeline) throw new Error(`unknown pipeline: ${pipelineId}`);
136
+
137
+ const runId = randomUUID();
138
+ const initialState = {
139
+ runId,
140
+ startedAt: Date.now(),
141
+ args,
142
+ triggeredBy: triggeredBy ?? null,
143
+ ...deepClone(pipeline.state),
144
+ };
145
+
146
+ const record = await store.create({ pipelineId, initialState });
147
+ logger.info(`manifold [${record.flowId.slice(0, 8)}] start: pipeline=${pipelineId} runId=${runId.slice(0, 8)}`);
148
+
149
+ // Per-pipeline singleton gate. Default (unset) is unlimited — the
150
+ // shared admission pool below is the only gate almost every pipeline
151
+ // needs. `allowConcurrent: false` opts a pipeline out of overlap
152
+ // entirely (e.g. a browser-automation poller that would race itself
153
+ // over shared session/gate state): reject this run before it ever
154
+ // competes for an admission slot if another non-terminal run of the
155
+ // same pipelineId already exists.
156
+ if (pipeline.allowConcurrent === false) {
157
+ const others = await runStore.listRunsPage({ pipelineId });
158
+ const active = others.find((r) => r.flowId !== record.flowId && !isTerminal(r.status));
159
+ if (active) {
160
+ logger.info(`manifold [${record.flowId.slice(0, 8)}] start: skipped — ${pipelineId} already has an active run (flowId=${active.flowId.slice(0, 8)}, status=${active.status})`);
161
+ await store.skip({
162
+ flowId: record.flowId,
163
+ expectedRevision: record.revision,
164
+ stateJson: record.stateJson,
165
+ reason: `another run of ${pipelineId} is already active (flowId=${active.flowId}, status=${active.status})`,
166
+ });
167
+ return await store.get(record.flowId);
168
+ }
169
+ }
170
+
171
+ // Whole-flow-run admission — the shared pool (state/admission-queue-
172
+ // store.js), same one subflow child-dispatch draws from. Every new flow
173
+ // competes for a real slot the instant it's created, and queues (rather
174
+ // than being rejected) if the pool is full — the caller gets back a
175
+ // record with status:"queued", exactly like an unadmitted subflow
176
+ // child. state-store.js's
177
+ // release/promotion wiring wakes it the moment a slot frees, no restart
178
+ // or caller retry needed.
179
+ const admission = await admissionQueueStore.admit({ kind: "flow-start", flowId: record.flowId });
180
+ if (admission.status === "queued") {
181
+ logger.info(`manifold [${record.flowId.slice(0, 8)}] start: queued — no admission slot available yet`);
182
+ await store.setQueued({
183
+ flowId: record.flowId,
184
+ expectedRevision: record.revision,
185
+ lastStep: record.lastStep,
186
+ stateJson: record.stateJson,
187
+ waitJson: { kind: "flow-start" },
188
+ statusReason: "queued: waiting for a shared admission slot",
189
+ });
190
+ return await store.get(record.flowId);
191
+ }
192
+ return record;
193
+ }
194
+
195
+ /**
196
+ * Drive the run loop until the flow reaches a terminal status or blocks.
197
+ * Safe to call multiple times — re-enters the loop wherever the flow is.
198
+ *
199
+ * `kind` describes why THIS runFlow() call was made — "fresh" | "retry" |
200
+ * "revival" | "recheck" — and is only actually consulted for the specific
201
+ * node the flow was already sitting at (a non-fresh re-entry); every
202
+ * subsequent genuinely-new node this loop advances to resolves to "fresh"
203
+ * regardless of what was passed here (see the execKind computation below).
204
+ * Required, not defaulted — see state/node-executions.js's write() for why
205
+ * a caller must always be explicit about this.
206
+ */
207
+ async function runFlow(flowId, kind) {
208
+ while (true) {
209
+ const flow = await store.get(flowId);
210
+ if (!flow) {
211
+ logger.warn(`manifold [${flowId.slice(0, 8)}] vanished mid-execution`);
212
+ return;
213
+ }
214
+ if (isTerminal(flow.status)) {
215
+ logger.info(`manifold [${flowId.slice(0, 8)}] terminal (${flow.status})`);
216
+ return;
217
+ }
218
+ if (isBlocked(flow.status)) {
219
+ logger.info(`manifold [${flowId.slice(0, 8)}] ${flow.status} at ${flow.lastStep} on ${flow.waitJson?.kind ?? "?"}`);
220
+ return;
221
+ }
222
+ // A flow queued for its OWN flow-start or child-dispatch admission
223
+ // (kind:"flow-start" / "child-dispatch") is not yet cleared to run at
224
+ // all — its admission check lives OUTSIDE this loop (in start()'s
225
+ // caller for flow-start, in the parent's dispatch loop for
226
+ // child-dispatch), so a fresh runFlow() call here has no code path
227
+ // that would naturally re-request admission; it would just dispatch
228
+ // lastStep's outbound edge unconditionally, bypassing the pool
229
+ // entirely. Return immediately — admission-queue-store's own
230
+ // release()/promotion wakes it later.
231
+ if (isQueued(flow.status) && (flow.waitJson?.kind === "flow-start" || flow.waitJson?.kind === "child-dispatch")) {
232
+ logger.info(`manifold [${flowId.slice(0, 8)}] queued at ${flow.lastStep} on ${flow.waitJson?.kind}`);
233
+ return;
234
+ }
235
+
236
+ const state = flow.stateJson ?? {};
237
+ if (state._paused) {
238
+ logger.info(`manifold [${flowId.slice(0, 8)}] paused at ${flow.lastStep}`);
239
+ return;
240
+ }
241
+ const pipelineId = state.pipelineId;
242
+ if (pipelineId && !pipelinesRef.value.get(pipelineId) && typeof ensurePipelinesLoaded === "function") {
243
+ try { await ensurePipelinesLoaded(); } catch (err) {
244
+ logger.warn(`manifold [${flowId.slice(0, 8)}] ensurePipelinesLoaded failed: ${err?.message ?? err}`);
245
+ }
246
+ }
247
+ const pipeline = pipelineId ? pipelinesRef.value.get(pipelineId) : null;
248
+ if (!pipeline) {
249
+ await persistFailRetrying({
250
+ flowId, revision: flow.revision, state,
251
+ message: `pipeline not loaded: ${pipelineId}`,
252
+ failureNode: "<engine:pipeline-load>",
253
+ });
254
+ return;
255
+ }
256
+
257
+ const lastStep = flow.lastStep ?? "__start__";
258
+ // A cascade leg's own edge function (2-arg mode, see resolveEdge) can
259
+ // read the error the leg just threw — carried here via a transient
260
+ // `_cascadeError` state key set right before the retry-continue below,
261
+ // guarded on nodeId===lastStep so a stale marker from an earlier
262
+ // loop-back is never read once lastStep has moved past it. Survives a
263
+ // gateway restart for free — it's just persisted state_json, same as
264
+ // every other field the executor writes.
265
+ const cascadeError = state?._cascadeError?.nodeId === lastStep ? state._cascadeError : null;
266
+ // Edge functions read via assembleState(baseState: state), not raw
267
+ // state — the single call site every edge decision in the engine goes
268
+ // through (root and child flows alike, since they share this one
269
+ // runFlow loop). A separate local var, not a reassignment of `state`
270
+ // itself — `state` is still what gets persisted below; only the value
271
+ // handed to the edge function changes.
272
+ const edgeState = await assembleState(flowId, { runStore, nodeExecutionsStore: nodeExecutions, baseState: state });
273
+ const nextNodeId = resolveEdge(pipeline.edges, lastStep, edgeState, {
274
+ logger, flowId,
275
+ error: cascadeError ? { message: cascadeError.message, errorType: cascadeError.errorType } : null,
276
+ });
277
+ if (!nextNodeId) {
278
+ // If this exhaustion traces back to a throw already recorded
279
+ // immediately at throw-time (the cascade catch branch below),
280
+ // thread that same invocationId/executionId through so
281
+ // upsertFailureRow's execution_id-based upsert collapses this into
282
+ // the SAME failure_events row instead of creating a second,
283
+ // less-detailed one for the identical logical moment. A genuine
284
+ // non-throw dead-end (unknown node, business-logic exhaustion with
285
+ // no antecedent throw) has no cascadeError and gets the plain
286
+ // generic message, same as always.
287
+ await persistFailRetrying({
288
+ flowId, revision: flow.revision, state,
289
+ message: cascadeError?.message ?? `edge ${lastStep} returned no next node`,
290
+ failureNode: lastStep,
291
+ invocationId: cascadeError?.invocationId ?? null,
292
+ executionId: cascadeError?.executionId ?? null,
293
+ errorType: cascadeError?.errorType ?? null,
294
+ });
295
+ return;
296
+ }
297
+ if (nextNodeId === "__end__") {
298
+ // This root read site carries the same resultJson-replay need as
299
+ // every other: pipeline.display() and successCriteria() both read
300
+ // node output through here. assembleState() adds _subflowGroups AND
301
+ // replays resultJson onto raw `state` — both are needed, because
302
+ // state_json is entry-only and carries no node output of its own.
303
+ const enrichedState = await assembleState(flowId, { runStore, nodeExecutionsStore: nodeExecutions, baseState: state });
304
+ await runAuditIfConfigured({ flowId, pipeline, state: enrichedState, logger });
305
+ // failureStore.recordFromTerminalState's soft-marker scan was
306
+ // deleted — zero soft-error producers remain anywhere in
307
+ // `.pipelines`/`.shared`; every failure now persists directly from
308
+ // this file's own throw-catch paths via recordEngineFailure.
309
+ // finalState is enrichedState-based, not state-based — a node's own
310
+ // output (e.g. a runAudit-only sentinel field like
311
+ // `_successCriteriaReason`) only exists via resultJson replay now,
312
+ // so finalState._successCriteriaReason below would silently read
313
+ // undefined off raw `state` otherwise. Safe to carry _subflowGroups
314
+ // through here too: the persist-layer filter
315
+ // (executor-persist.js's toPersistableState) already narrows
316
+ // whatever's passed to persistFailRetrying/persistSkipRetrying/
317
+ // persistFinishRetrying down to entry-only fields regardless of
318
+ // what finalState itself contains.
319
+ let finalState = enrichedState;
320
+ if (typeof pipeline.display === "function") {
321
+ try {
322
+ const out = await Promise.resolve(pipeline.display(enrichedState));
323
+ if (out && typeof out === "object") {
324
+ finalState = {
325
+ ...enrichedState,
326
+ _display: {
327
+ title: typeof out.title === "string" ? out.title : null,
328
+ summary: typeof out.summary === "string" ? out.summary : null,
329
+ funnels: Array.isArray(out.funnels) ? out.funnels : null,
330
+ metrics: Array.isArray(out.metrics) ? out.metrics : null,
331
+ },
332
+ };
333
+ }
334
+ } catch (err) {
335
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] pipeline.display() threw: ${err?.message ?? err}`);
336
+ }
337
+ }
338
+ const successCriteria = pipeline.runAudit?.successCriteria;
339
+ if (typeof successCriteria === "function") {
340
+ let verdict;
341
+ try {
342
+ verdict = await Promise.resolve(successCriteria({ ...enrichedState, ...finalState }));
343
+ } catch (err) {
344
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] successCriteria threw: ${err?.message ?? err}`);
345
+ }
346
+ if (verdict === "failed") {
347
+ await persistFailRetrying({
348
+ flowId, revision: flow.revision,
349
+ state: finalState,
350
+ message: finalState._successCriteriaReason ?? "successCriteria: failed",
351
+ failureNode: "__end__",
352
+ });
353
+ logger.info(`manifold [${flowId.slice(0, 8)}] __end__ → failed (successCriteria)`);
354
+ return;
355
+ }
356
+ if (verdict === "skipped") {
357
+ await persistSkipRetrying({
358
+ flowId, revision: flow.revision,
359
+ state: finalState,
360
+ reason: finalState._successCriteriaReason ?? "successCriteria: skipped",
361
+ });
362
+ logger.info(`manifold [${flowId.slice(0, 8)}] __end__ → skipped (successCriteria)`);
363
+ return;
364
+ }
365
+ }
366
+ await persistFinishRetrying({ flowId, revision: flow.revision, state: finalState });
367
+ logger.info(`manifold [${flowId.slice(0, 8)}] __end__`);
368
+ return;
369
+ }
370
+
371
+ const node = pipeline.nodes[nextNodeId];
372
+ if (!node) {
373
+ await persistFailRetrying({
374
+ flowId, revision: flow.revision, state,
375
+ message: `unknown node: ${nextNodeId}`,
376
+ failureNode: nextNodeId,
377
+ });
378
+ return;
379
+ }
380
+
381
+ logger.info(`manifold [${flowId.slice(0, 8)}] → ${nextNodeId} (${node.kind})`);
382
+
383
+ const enteredAt = Date.now();
384
+ // Computed once per dispatch, from this flow's
385
+ // own last-dispatched node — see executor-helpers.js's isFreshNodeEntry
386
+ // for the full reasoning. execKind is the actual value threaded to every write()
387
+ // call below in this iteration (only one ever fires per iteration) —
388
+ // a genuinely fresh node entry is always "fresh" regardless of why
389
+ // this runFlow() call happened; only a non-fresh re-entry (the node
390
+ // the flow was already at) "spends" the caller-supplied kind.
391
+ const lastNodeId = await nodeExecutions.findCurrentStep(flowId);
392
+ const forceNew = isFreshNodeEntry(lastNodeId, nextNodeId);
393
+ const execKind = forceNew ? "fresh" : kind;
394
+ // Create the invocation row before execution: "last invocation for
395
+ // this flow" tracks the in-flight node even during execution, not
396
+ // just after completion — this is how the currently-running node is
397
+ // identified.
398
+ const enterResult = await nodeExecutions.enter({
399
+ flowId, pipelineId: state.pipelineId ?? null,
400
+ nodeId: nextNodeId, subflowKey: null, kind: execKind,
401
+ });
402
+ const invocationId = enterResult?.invocationId ?? null;
403
+ const isNew = enterResult?.isNew ?? (execKind === "fresh");
404
+ process.nextTick(() => nodeEmitter.emit("node.executing", {
405
+ flowId, nodeId: nextNodeId,
406
+ pipelineId: state.pipelineId ?? null,
407
+ parentInvocationId: flow.parentInvocationId ?? null,
408
+ }));
409
+ let stateUpdate = {};
410
+ let waitJson = null;
411
+ let blockedInfo = null;
412
+ let skippedInfo = null;
413
+ let reentrant = false;
414
+ let exitedAt = null;
415
+ try {
416
+ // "needs:" resolution is lazy, via ctx.needs(varName) called
417
+ // from inside the node's own run()/promptBuilder()/selector(), NOT
418
+ // checked here ahead of the dispatch. executor-nodes.js/executor-subflow-children.js
419
+ // build that accessor (needs-resolver.js's makeNeedsCtx) and pass
420
+ // it through as part of ctx — nothing to do at this call site
421
+ // beyond letting a NodeBlocked(_kind:"needs-not-ready") thrown from
422
+ // inside executeNode() propagate to the catch block below exactly
423
+ // like any other NodeBlocked (the budget-gate pause already relies
424
+ // on this same path). Known gap, deliberately deferred (needs: is
425
+ // dormant, zero pipelines declare it yet): reviveAgentKindOrphan's
426
+ // own dispatch path does not pass a needs ctx at all, so an agent-
427
+ // kind node that crashes mid-flight can't call ctx.needs() on
428
+ // revival — worth fixing before any pipeline adopts needs: on an
429
+ // agent-kind node, not before.
430
+ const result = await executeNode({ node, nodeId: nextNodeId, state, flowId, invocationId });
431
+ exitedAt = Date.now();
432
+ stateUpdate = result.stateUpdate ?? {};
433
+ waitJson = result.waitJson ?? null;
434
+ blockedInfo = result.blockedInfo ?? null;
435
+ skippedInfo = result.skippedInfo ?? null;
436
+ reentrant = result.reentrant === true;
437
+ } catch (err) {
438
+ exitedAt ??= Date.now();
439
+ if (err instanceof NodeBlocked) {
440
+ blockedInfo = err._blocked;
441
+ } else if (err instanceof GatewayHandoffNeeded) {
442
+ blockedInfo = {
443
+ reason: `node ${nextNodeId} needs gateway context: ${err.message}`,
444
+ needed: [],
445
+ hint: "No automatic resume — requires a scope-gap fix at the call site or manual task-revival.",
446
+ _kind: "needs-gateway-handoff",
447
+ };
448
+ logger.info(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} → needs-gateway-handoff: ${err.message}`);
449
+ } else {
450
+ const msg = err?.message ?? String(err);
451
+ // Duck-typed, not `instanceof NodeFailure` — pipeline/shared code
452
+ // and the engine itself can load executor-errors.js through
453
+ // different module registries (the gateway daemon's data:-URL
454
+ // pipeline loader vs. jiti-transpiled CJS for CLI tooling), which
455
+ // gives structurally-identical but reference-distinct NodeFailure
456
+ // classes; instanceof silently returns false across that
457
+ // boundary even for a genuine NodeFailure. A plain Error never
458
+ // has .errorType set, so this is equivalent and robust either way.
459
+ const errorType = err?.errorType ?? null;
460
+ logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
461
+ // No error/errorType here — flow_node_executions is purely an
462
+ // execution-window/identity ledger; every throw's actual content
463
+ // now goes to failure_events instead, via the recordEngineFailure
464
+ // call(s) below.
465
+ const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
466
+ if (pipeline.kind === "cascade") {
467
+ // A cascade-kind flow treats ANY throw like a completed dispatch,
468
+ // not a failure
469
+ // — lastStep advances to the node that just threw (same persist
470
+ // shape as the ordinary success path further below), and it's
471
+ // THAT node's own edge function that decides what happens next,
472
+ // the same way it would after a success. A cascade leg throwing
473
+ // NodeFailure (see executor-errors.js) carries WHY it failed via
474
+ // `errorType` — threaded here into `_cascadeError`, a transient
475
+ // state key the next loop iteration reads (guarded on
476
+ // nodeId===lastStep, see the resolveEdge call above) and passes
477
+ // to a 2-arg edge function as its error argument. This is what
478
+ // lets a cascade's own edge decide retry-vs-fallback-vs-give-up
479
+ // from the REAL thrown error, not a soft-returned field.
480
+ //
481
+ // Every throw persists a permanent failure_events row
482
+ // IMMEDIATELY, right here, regardless of what the edge
483
+ // decides next — not just on eventual give-up. This is the one
484
+ // gap the persistFailRetrying->recordEngineFailure path never
485
+ // covers (a cascade-continuing throw never reaches
486
+ // persistFailRetrying at all). If this attempt turns out to be
487
+ // the cascade's final one, the `!nextNodeId` branch at the top
488
+ // of this loop reuses this SAME invocationId/executionId when
489
+ // it fails the flow, so upsertFailureRow's execution_id upsert
490
+ // collapses onto this row instead of creating a second one.
491
+ try {
492
+ await failureStore?.recordEngineFailure?.({
493
+ flowId, pipelineId: state.pipelineId ?? null, failureNode: nextNodeId,
494
+ message: msg, state, invocationId: written.invocationId ?? null, executionId: written.executionId ?? null, errorType,
495
+ });
496
+ } catch (recErr) {
497
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] recordEngineFailure (cascade) threw: ${recErr?.message ?? recErr}`);
498
+ }
499
+ logger.info(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} → cascading (its own edge decides what happens next)`);
500
+ await persistRunningRetrying({
501
+ flowId, revision: flow.revision, lastStep: nextNodeId,
502
+ state: {
503
+ ...state,
504
+ _cascadeError: {
505
+ nodeId: nextNodeId, message: msg, errorType,
506
+ invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
507
+ },
508
+ },
509
+ });
510
+ continue;
511
+ }
512
+ await persistFailRetrying({
513
+ flowId, revision: flow.revision, state,
514
+ message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
515
+ invocationId: written.invocationId ?? null, executionId: written.executionId ?? null, errorType,
516
+ });
517
+ return;
518
+ }
519
+ }
520
+
521
+ if (blockedInfo) {
522
+ if (!blockedInfo._kind) {
523
+ const msg = blockedInfo.reason ?? `${nextNodeId} blocked without declaring a wait kind`;
524
+ logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
525
+ const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
526
+ await persistFailRetrying({
527
+ flowId, revision: flow.revision, state,
528
+ message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
529
+ invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
530
+ });
531
+ return;
532
+ }
533
+ const waitJsonForBlock = {
534
+ kind: blockedInfo._kind,
535
+ nodeId: nextNodeId,
536
+ reason: blockedInfo.reason ?? "node blocked",
537
+ needed: blockedInfo.needed ?? [],
538
+ hint: blockedInfo.hint ?? "",
539
+ ...(blockedInfo.responder ? { responder: blockedInfo.responder } : {}),
540
+ ...(Array.isArray(blockedInfo.choices) && blockedInfo.choices.length > 0
541
+ ? { choices: blockedInfo.choices.map((c) => String(c)) }
542
+ : {}),
543
+ ...(Array.isArray(blockedInfo.items) ? { items: blockedInfo.items } : {}),
544
+ renderedAt: Date.now(),
545
+ retry: true,
546
+ };
547
+ try {
548
+ validateGateLayout(waitJsonForBlock.items);
549
+ } catch (err) {
550
+ const msg = err?.message ?? String(err);
551
+ logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
552
+ const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
553
+ await persistFailRetrying({
554
+ flowId, revision: flow.revision, state,
555
+ message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
556
+ invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
557
+ });
558
+ return;
559
+ }
560
+ const summary = `${nextNodeId}: ${waitJsonForBlock.reason}`;
561
+ logger.info(`manifold [${flowId.slice(0, 8)}] ${waitJsonForBlock.kind} → waiting (will re-run ${nextNodeId} on resume)\n reason: ${waitJsonForBlock.reason}\n hint: ${waitJsonForBlock.hint}`);
562
+ await persistWaitingRetrying({
563
+ flowId, revision: flow.revision, lastStep,
564
+ blockingNodeId: nextNodeId,
565
+ state,
566
+ waitJson: waitJsonForBlock,
567
+ statusReason: shortSummary(summary),
568
+ });
569
+ nodeExecutions.write({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, error: null, kind: execKind, invocationId, isNewInvocation: isNew });
570
+ return;
571
+ }
572
+
573
+ if (skippedInfo) {
574
+ logger.info(`manifold [${flowId.slice(0, 8)}] ${nextNodeId}: _skipped — ${skippedInfo.reason}`);
575
+ await persistSkipRetrying({
576
+ flowId, revision: flow.revision,
577
+ state,
578
+ reason: skippedInfo.reason ?? "skipped",
579
+ });
580
+ nodeExecutions.write({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, error: null, kind: execKind, invocationId, isNewInvocation: isNew });
581
+ return;
582
+ }
583
+
584
+ const freshAfterNode = (await store.get(flowId)) ?? { stateJson: state, revision: flow.revision };
585
+ // entryState is taken directly from freshAfterNode.stateJson, not
586
+ // recomputed as a merge of this node's own output. Every persist call
587
+ // below filters its `state:` argument down to entry/bookkeeping fields
588
+ // only (executor-persist.js's toPersistableState), so a wide merge of
589
+ // the node's output here would never reach the database. A node's own
590
+ // output reaches result_json separately, via writeResult(invocationId,
591
+ // stateUpdate) further down — entirely independent of this variable.
592
+ // freshAfterNode.stateJson is already entry-only (it was itself
593
+ // persisted through the same filter), so it's used directly.
594
+ let entryState = freshAfterNode.stateJson ?? state;
595
+ // This node just completed successfully (no throw, no blockedInfo, no
596
+ // skippedInfo — we're past all of those branches above) — any
597
+ // _cascadeError left over from an earlier failed attempt AT THIS SAME
598
+ // node is now stale. Only ever lives on this entry-only state (a
599
+ // success's own stateUpdate never mentions _cascadeError), so a node
600
+ // retried after a cascade throw and then succeeding would otherwise
601
+ // carry a stale error into its own next edge-function call if that
602
+ // edge checks `error` before `state`. Clear it explicitly.
603
+ if (entryState._cascadeError?.nodeId === nextNodeId) {
604
+ const { _cascadeError, ...rest } = entryState;
605
+ entryState = rest;
606
+ }
607
+ const persistRevision = freshAfterNode.revision ?? flow.revision;
608
+
609
+ if (waitJson) {
610
+ try {
611
+ validateGateLayout(waitJson.items);
612
+ } catch (err) {
613
+ const msg = err?.message ?? String(err);
614
+ logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
615
+ const written = await safeWriteExecution({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
616
+ await persistFailRetrying({
617
+ flowId, revision: persistRevision, state: entryState,
618
+ message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
619
+ invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
620
+ });
621
+ return;
622
+ }
623
+ // The wait summary is synthesized by describeWaitFallback(waitJson,
624
+ // nextNodeId) — no pipeline-visible render hook feeds this log line.
625
+ const summary = describeWaitFallback(waitJson, nextNodeId);
626
+ logger.info(`manifold [${flowId.slice(0, 8)}] ${reentrant ? "reentrant gate" : "gate"} → blocked\n${summary}`);
627
+ await persistWaitingRetrying({
628
+ flowId, revision: persistRevision,
629
+ lastStep: reentrant ? lastStep : nextNodeId,
630
+ blockingNodeId: nextNodeId,
631
+ state: entryState,
632
+ waitJson,
633
+ statusReason: shortSummary(summary),
634
+ });
635
+ nodeExecutions.write({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, error: null, kind: execKind, invocationId, isNewInvocation: isNew });
636
+ return;
637
+ }
638
+
639
+ // Write this node's own result FIRST and durably, before advancing
640
+ // flow_runs.last_step to nextNodeId via persistRunningRetrying below.
641
+ // last_step is what boot-time orphan revival and the pipeline-
642
+ // concurrency dequeue path both trust as "this node completed" — they
643
+ // jump straight to resolving its outbound edge without re-running
644
+ // apply(). Writing the result first makes that invariant crash-safe:
645
+ // a restart landing before last_step advances leaves last_step
646
+ // pointing at the previous node, so revival re-runs this node's
647
+ // apply() again — a duplicate call, not a silent failure.
648
+ //
649
+ // Also still AWAITED, not fire-and-forget: state_json is entry-only,
650
+ // so the very next loop iteration's assembleState() call (line ~288,
651
+ // feeding the edge function) is the ONLY way this node's own output
652
+ // becomes visible again — if
653
+ // that read raced ahead of this write, the edge function would see
654
+ // this node's output as still-missing, some edges then re-route back
655
+ // to the same node (or worse), producing a tight busy-loop.
656
+ // Awaiting costs one DB round-trip per dispatch, already priced into
657
+ // every other persist call this loop makes.
658
+ const resultWrite = await nodeExecutions.writeResult(invocationId, stateUpdate).catch(() => null);
659
+ if (!resultWrite || !resultWrite.ok) {
660
+ const reason = resultWrite?.reason ?? "threw";
661
+ // "race" means a CONCURRENT writer already committed real content
662
+ // for this exact invocation — advancing is correct, the data
663
+ // genuinely is there, just not from this call. Every other reason
664
+ // ("no-invocation-id"/"no-such-invocation"/"unserializable"/
665
+ // "db-error"/a genuine throw, normalized via the catch above) means
666
+ // NO ONE wrote anything. Advancing last_step in that case would
667
+ // move the flow past this node with its output never durably
668
+ // written: the next runFlow() iteration's assembleState() reads
669
+ // nothing for this node, and resolveEdge() dead-ends with "no next
670
+ // node" — the SCORE_LLM "edge returned no next node" shape, and the
671
+ // boot-revival gap it left uncovered (see reviveAgentKindOrphan
672
+ // below). Fail the flow outright for any non-race reason instead of
673
+ // silently continuing, so the operator sees the true cause.
674
+ if (reason !== "race") {
675
+ const msg = `resultJson write failed (${reason})`;
676
+ logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} ${msg}`);
677
+ const written = await safeWriteExecution({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
678
+ await persistFailRetrying({
679
+ flowId, revision: persistRevision, state: entryState,
680
+ message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
681
+ invocationId: written.invocationId ?? invocationId, executionId: written.executionId ?? null,
682
+ });
683
+ return;
684
+ }
685
+ logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} resultJson write failed (race) — another writer already completed this invocation`);
686
+ }
687
+
688
+ const runResult = await persistRunningRetrying({
689
+ flowId, revision: persistRevision, lastStep: nextNodeId,
690
+ state: entryState,
691
+ });
692
+ if (!runResult.applied) {
693
+ const msg = `state persist failed: ${runResult.code}`;
694
+ logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
695
+ const written = await safeWriteExecution({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
696
+ await persistFailRetrying({
697
+ flowId, revision: persistRevision, state: entryState,
698
+ message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
699
+ invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
700
+ });
701
+ return;
702
+ }
703
+ nodeExecutions.write({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, error: null, kind: execKind, invocationId, isNewInvocation: isNew });
704
+ }
705
+ }
706
+
707
+ // Object passed to `code`-kind node bodies as ctx. `invokePipeline` fires a
708
+ // background flow start and returns immediately. Assigned after start/runFlow
709
+ // are defined; subflow/node runners read it via getter so the gap is safe.
710
+ nodeCtx = Object.freeze({
711
+ async invokePipeline(targetPipelineId, args = {}) {
712
+ const record = await start(targetPipelineId, args);
713
+ // start() does not throw when the shared admission pool is full —
714
+ // it queues instead (see start()'s own comment). Preserves the
715
+ // "singleton-busy" contract at this call site specifically: a caller
716
+ // handling invoked:false is expected to retry invokePipeline itself
717
+ // later, not rely on this record being silently promoted and run out
718
+ // from under it with nobody tracking the outcome — so the just-created
719
+ // queued record is cancelled rather than left to run unobserved.
720
+ if (record.status === "queued") {
721
+ await cancel(record.flowId, "invokePipeline: shared admission pool was full").catch(() => {});
722
+ return {
723
+ invoked: false, reason: "singleton-busy", target: targetPipelineId,
724
+ message: `pipeline "${targetPipelineId}" is at its concurrency limit — queued instead of started`,
725
+ };
726
+ }
727
+ setImmediate(() => {
728
+ runFlow(record.flowId, "fresh").catch((err) => {
729
+ logger?.warn?.(`manifold invokePipeline(${targetPipelineId}) [${record.flowId.slice(0, 8)}]: runFlow failed: ${err?.message ?? err}`);
730
+ });
731
+ });
732
+ return { invoked: true, target: targetPipelineId, flowId: record.flowId };
733
+ },
734
+ });
735
+
736
+ // resume() is the generic "continue a waiting flow" entry point, called
737
+ // both after a gate decision resolves (gate-resolver.js, routes/
738
+ // operator.js) and by boot-time revival's children-pending check
739
+ // (boot/flow-revival.js) — both are the same underlying mechanism (a
740
+ // node the flow was already sitting at gets re-evaluated), just
741
+ // triggered by different external events, so both resolve to "recheck".
742
+ async function resume(flowId) {
743
+ return runFlow(flowId, "recheck");
744
+ }
745
+
746
+ async function rerun(sourceFlowId, argsOverride) {
747
+ const source = await store.get(sourceFlowId);
748
+ if (!source) throw new Error(`source flow not found: ${sourceFlowId}`);
749
+ return rerunFromSource(source, argsOverride);
750
+ }
751
+
752
+ async function rerunFromSource(sourceFlow, argsOverride) {
753
+ if (!sourceFlow) throw new Error(`source flow not found`);
754
+ const sourceFlowId = sourceFlow.flowId;
755
+ const pipelineId = sourceFlow.stateJson?.pipelineId;
756
+ if (!pipelineId) throw new Error(`source flow has no pipelineId in state; cannot determine which pipeline to rerun`);
757
+ const args = argsOverride !== undefined ? argsOverride : (sourceFlow.stateJson?.args ?? {});
758
+ const triggeredBy = {
759
+ kind: "rerun",
760
+ label: `Rerun of ${sourceFlowId.slice(0, 8)}`,
761
+ parentFlowId: sourceFlowId,
762
+ ts: Date.now(),
763
+ };
764
+ const record = await start(pipelineId, args, triggeredBy);
765
+ try {
766
+ const fresh = await store.get(record.flowId);
767
+ if (fresh) {
768
+ await store.setRunning({
769
+ flowId: record.flowId,
770
+ expectedRevision: fresh.revision,
771
+ lastStep: fresh.lastStep,
772
+ stateJson: { ...fresh.stateJson, rerunOf: sourceFlowId },
773
+ });
774
+ }
775
+ } catch (err) {
776
+ logger?.warn?.(`manifold rerun: failed to stamp rerunOf on [${record.flowId.slice(0, 8)}]: ${err?.message ?? err}`);
777
+ }
778
+ return record;
779
+ }
780
+
781
+ async function cancel(flowId, reason = null) {
782
+ const flow = await store.get(flowId);
783
+ if (!flow) return { cancelled: false, reason: "Flow not found." };
784
+ // completeCancellation stamps in-flight branches with cancelledAt, kills any
785
+ // active LLM sessions, then writes status='cancelled' + stamped stateJson to DB.
786
+ return completeCancellation({
787
+ store,
788
+ flowId,
789
+ stateJson: flow.stateJson ?? {},
790
+ lastStep: flow.lastStep,
791
+ revision: flow.revision,
792
+ cfg: _config,
793
+ logger,
794
+ runStore,
795
+ nodeExecutionsStore: nodeExecutions,
796
+ reason,
797
+ });
798
+ }
799
+
800
+ /**
801
+ * Revive an "agent"-kind orphan left mid-flight by a crashed process.
802
+ *
803
+ * Unlike a code-kind orphan (safe to just re-enter via runFlow — the
804
+ * node hadn't produced any side effect yet), an agent-kind node may have
805
+ * already dispatched an agent turn that completed (or partially acted)
806
+ * before the crash. Re-running node.promptBuilder(state) fresh would give
807
+ * the agent no way to know it already tried this once, risking a
808
+ * duplicate side effect. Instead this sends REVIVAL_MSG to the SAME
809
+ * deterministic session key the original dispatch used, so the agent can
810
+ * check its own prior turns and decide whether to finish or resume.
811
+ */
812
+ // nodeId is the crashed/in-flight node — the caller (boot/flow-revival.js)
813
+ // resolves it from f.currentStep/waitJson.nodeId, since flow.lastStep is
814
+ // the last COMPLETED node, not this one. Falls back to flow.lastStep only
815
+ // for callers that don't pass it explicitly (defensive, not the expected path).
816
+ async function reviveAgentKindOrphan(flow, nodeId = flow.lastStep) {
817
+ const flowId = flow.flowId;
818
+ const flowId8 = flowId.slice(0, 8);
819
+ const state = flow.stateJson ?? {};
820
+ const pipeline = pipelinesRef.value.get(state.pipelineId);
821
+ const node = pipeline?.nodes?.[nodeId];
822
+ if (!node || node.kind !== "agent") {
823
+ logger.warn(`manifold revival [${flowId8}]: reviveAgentKindOrphan called for non-agent node "${nodeId}"`);
824
+ return;
825
+ }
826
+
827
+ logger.info(`manifold revival [${flowId8}]: reviving interrupted agent-node "${nodeId}"`);
828
+ const enteredAt = Date.now();
829
+ // Resolved up front (not after executeNode returns) so a tool call made
830
+ // during this revived dispatch can register itself under the SAME
831
+ // invocationId the original, interrupted attempt already created — see
832
+ // tool-handoff.js. No fresh nodeExecutions.enter() call here; the
833
+ // invocation row already exists from the original attempt.
834
+ const invocation = await nodeExecutions.getLatestInvocationForNode(flowId, nodeId, null);
835
+ const invocationId = invocation?.id ?? null;
836
+ let result;
837
+ try {
838
+ result = await executeNode({ node, nodeId, state, flowId, reviveMessage: REVIVAL_MSG, invocationId });
839
+ } catch (err) {
840
+ if (err instanceof GatewayHandoffNeeded) {
841
+ // A request-scope gap in the revived dispatch: block rather than
842
+ // failing the flow outright. No automatic resume exists for this
843
+ // wait kind (see executor-errors.js's GatewayHandoffNeeded
844
+ // docstring) — recovery
845
+ // is either a scope-gap fix at the actual call site or manual
846
+ // task-revival.
847
+ await persistWaitingRetrying({
848
+ flowId, revision: flow.revision, lastStep: nodeId, blockingNodeId: nodeId,
849
+ state,
850
+ waitJson: {
851
+ kind: "needs-gateway-handoff", nodeId, reason: err.message, needed: [],
852
+ hint: "No automatic resume — requires a scope-gap fix at the call site or manual task-revival.",
853
+ renderedAt: Date.now(), retry: true,
854
+ },
855
+ statusReason: shortSummary(`${nodeId}: needs gateway context: ${err.message}`),
856
+ });
857
+ return;
858
+ }
859
+ const msg = err?.message ?? String(err);
860
+ // Duck-typed — see the main loop's identical check above for why
861
+ // `instanceof NodeFailure` isn't reliable across this codebase's
862
+ // module-loading boundaries.
863
+ const errorType = err?.errorType ?? null;
864
+ logger.warn(`manifold revival [${flowId8}]: agent-node "${nodeId}" revival failed: ${msg}`);
865
+ // Revival always continues the exact node that crashed — there's no
866
+ // fresh dispatch decision here (unlike runFlow's loop,
867
+ // which resolves an edge to decide nextNodeId), so kind is hardcoded
868
+ // "revival" rather than derived from isFreshNodeEntry.
869
+ const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId, enteredAt, exitedAt: Date.now(), kind: "revival" });
870
+ await persistFailRetrying({
871
+ flowId, revision: flow.revision, state,
872
+ message: `${nodeId}: revival failed: ${msg}`, failureNode: nodeId,
873
+ invocationId: written.invocationId ?? null, executionId: written.executionId ?? null, errorType,
874
+ });
875
+ return;
876
+ }
877
+
878
+ const { stateUpdate = {}, blockedInfo, skippedInfo } = result;
879
+
880
+ if (blockedInfo) {
881
+ if (!blockedInfo._kind) {
882
+ const msg = blockedInfo.reason ?? `${nodeId} blocked without declaring a wait kind`;
883
+ logger.warn(`manifold revival [${flowId8}]: agent-node "${nodeId}" ${msg}`);
884
+ // Same reasoning as above — revival is always a continuation.
885
+ const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId, enteredAt, exitedAt: Date.now(), kind: "revival" });
886
+ await persistFailRetrying({
887
+ flowId, revision: flow.revision, state,
888
+ message: `${nodeId}: ${msg}`, failureNode: nodeId,
889
+ invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
890
+ });
891
+ return;
892
+ }
893
+ const waitJsonForBlock = {
894
+ kind: blockedInfo._kind, nodeId,
895
+ reason: blockedInfo.reason ?? "node blocked", needed: blockedInfo.needed ?? [],
896
+ hint: blockedInfo.hint ?? "", renderedAt: Date.now(), retry: true,
897
+ };
898
+ await persistWaitingRetrying({
899
+ flowId, revision: flow.revision, lastStep: nodeId, blockingNodeId: nodeId,
900
+ state,
901
+ waitJson: waitJsonForBlock,
902
+ statusReason: shortSummary(`${nodeId}: ${waitJsonForBlock.reason}`),
903
+ });
904
+ return;
905
+ }
906
+
907
+ if (skippedInfo) {
908
+ await persistSkipRetrying({
909
+ flowId, revision: flow.revision,
910
+ state,
911
+ reason: skippedInfo.reason ?? "skipped",
912
+ });
913
+ return;
914
+ }
915
+
916
+ // No wide stateUpdate merge before persisting — `state` here is
917
+ // already entry-only (freshly read via store.get() upstream), and
918
+ // stateUpdate's own node-output content reaches
919
+ // result_json separately via writeResult(), not through this. See the
920
+ // main dispatch loop's identical cleanup above for the full reasoning.
921
+ let entryState = state;
922
+ // Same staleness clear as the main loop's success path above — an
923
+ // agent-kind cascade leg (e.g. SUBMIT_AGENT) revived and now succeeding
924
+ // shouldn't carry a leftover _cascadeError from its earlier crashed
925
+ // attempt into its own next edge-function call.
926
+ if (entryState._cascadeError?.nodeId === nodeId) {
927
+ const { _cascadeError, ...rest } = entryState;
928
+ entryState = rest;
929
+ }
930
+ // The main loop writes a node's result BEFORE advancing last_step, so a
931
+ // crash-then-revival never reads a stale/empty result for the node it
932
+ // just resumed. This revival path resumes the SAME crashed dispatch (no
933
+ // fresh nodeExecutions.enter() call here — the invocation row already
934
+ // exists from the original, interrupted attempt) and carries no
935
+ // writeResult() call of its own, so it writes the result here: without
936
+ // it, last_step has already advanced to `nodeId` while its result_json
937
+ // stayed empty from the interrupted attempt. The very next runFlow() iteration's
938
+ // assembleState() then read nothing for this node, and resolveEdge()
939
+ // dead-ended with "no next node" — deterministic on every revived
940
+ // agent-node whose outbound edge depends on its own output, not a race.
941
+ const resultWrite = await nodeExecutions.writeResult(invocationId, stateUpdate).catch(() => null);
942
+ if (!resultWrite || !resultWrite.ok) {
943
+ const reason = resultWrite?.reason ?? "threw";
944
+ // "race" means a CONCURRENT writer already committed real content for
945
+ // this exact invocation — advancing is correct, the data genuinely is
946
+ // there, just not from this call. Every other reason (db-error,
947
+ // no-such-invocation, no-invocation-id, unserializable, threw) means
948
+ // NO ONE wrote anything: advancing here would move last_step past
949
+ // this node with its output never durably written. The next runFlow()
950
+ // iteration's assembleState() then reads nothing for this node, and
951
+ // resolveEdge() dead-ends with "no next node" — deterministic given
952
+ // the failure, not a race, and completely swallowing the REAL reason
953
+ // behind a generic edge-resolution message. Fail the flow outright
954
+ // instead — same shape as persistRunningRetrying's own CAS-failure
955
+ // branch just below — so the operator sees the true cause and can
956
+ // retry this one node, rather than a downstream "no next node" that
957
+ // hides it.
958
+ if (reason !== "race") {
959
+ const msg = `resultJson write failed (${reason})`;
960
+ logger.warn(`manifold revival [${flowId8}]: agent-node "${nodeId}" ${msg}`);
961
+ const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId, enteredAt, exitedAt: Date.now(), kind: "revival" });
962
+ await persistFailRetrying({
963
+ flowId, revision: flow.revision, state,
964
+ message: `${nodeId}: ${msg}`, failureNode: nodeId,
965
+ invocationId: written.invocationId ?? invocationId, executionId: written.executionId ?? null,
966
+ });
967
+ return;
968
+ }
969
+ logger.warn(`manifold revival [${flowId8}]: agent-node "${nodeId}" resultJson write failed (race) — another writer already completed this invocation`);
970
+ }
971
+ await persistRunningRetrying({ flowId, revision: flow.revision, lastStep: nodeId, state: entryState });
972
+ logger.info(`manifold revival [${flowId8}]: agent-node "${nodeId}" revived — continuing`);
973
+ return runFlow(flowId, "revival");
974
+ }
975
+
976
+ return { start, runFlow, resume, cancel, rerun, rerunFromSource, reviveAgentKindOrphan };
977
+ }
978
+
979
+ // ── audit ─────────────────────────────────────────────────────────────────────
980
+
981
+ async function runAuditIfConfigured({ flowId, pipeline, state, logger }) {
982
+ const audit = pipeline?.runAudit;
983
+ if (!audit) return;
984
+ if (typeof audit.runsDir !== "string" || typeof audit.summarize !== "function") {
985
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] runAudit malformed (need runsDir:string + summarize:function); skipping`);
986
+ return;
987
+ }
988
+ const runId = state?.runId;
989
+ if (!runId) {
990
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] runAudit: state.runId missing; skipping`);
991
+ return;
992
+ }
993
+ try {
994
+ const runDir = path.join(audit.runsDir, runId);
995
+ await mkdir(runDir, { recursive: true });
996
+ await writeFile(path.join(runDir, "final-state.json"), JSON.stringify(state, null, 2), "utf-8");
997
+ let summary;
998
+ try { summary = await Promise.resolve(audit.summarize(state)); }
999
+ catch (err) {
1000
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] runAudit.summarize threw: ${err?.message ?? err}`);
1001
+ return;
1002
+ }
1003
+ if (summary && typeof summary === "object") {
1004
+ await writeFile(
1005
+ path.join(audit.runsDir, "log.jsonl"),
1006
+ JSON.stringify(summary) + "\n",
1007
+ { flag: "a", encoding: "utf-8" },
1008
+ );
1009
+ }
1010
+ } catch (err) {
1011
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] runAudit failed: ${err?.message ?? err}`);
1012
+ }
1013
+ }