@igorjpimenta/manifold 0.0.0-bootstrap.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/.migrations/ops/0000_futuristic_karma.sql +111 -0
  2. package/.migrations/ops/0001_low_maximus.sql +7 -0
  3. package/.migrations/ops/meta/0000_snapshot.json +763 -0
  4. package/.migrations/ops/meta/0001_snapshot.json +173 -0
  5. package/.migrations/ops/meta/_journal.json +20 -0
  6. package/.migrations/runtime/0000_real_boomer.sql +180 -0
  7. package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
  8. package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
  9. package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
  10. package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
  11. package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
  12. package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
  13. package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
  14. package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
  15. package/.migrations/runtime/0009_lying_shocker.sql +96 -0
  16. package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
  17. package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
  18. package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
  19. package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
  20. package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
  21. package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
  22. package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
  23. package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
  24. package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
  25. package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
  26. package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
  27. package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
  28. package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
  29. package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
  30. package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
  31. package/.migrations/runtime/meta/_journal.json +97 -0
  32. package/README.md +1584 -0
  33. package/RELEASING.md +89 -0
  34. package/agent/fleet.js +100 -0
  35. package/bin/cli.js +854 -0
  36. package/bin/manifold.js +4 -0
  37. package/boot/flow-revival.js +536 -0
  38. package/budget/index.js +495 -0
  39. package/budget/lib.js +59 -0
  40. package/dashboard/cost.js +528 -0
  41. package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
  42. package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
  43. package/dashboard/dist/index.html +13 -0
  44. package/dashboard/event-log.js +267 -0
  45. package/dashboard/index.html +12 -0
  46. package/dashboard/package-lock.json +2877 -0
  47. package/dashboard/package.json +37 -0
  48. package/dashboard/pipeline-graph.js +131 -0
  49. package/dashboard/postcss.config.js +6 -0
  50. package/dashboard/search.js +183 -0
  51. package/dashboard/server.js +1015 -0
  52. package/dashboard/src/App.tsx +42 -0
  53. package/dashboard/src/api/client.ts +118 -0
  54. package/dashboard/src/api/types.ts +510 -0
  55. package/dashboard/src/api/ws.tsx +112 -0
  56. package/dashboard/src/components/CancelButton.tsx +63 -0
  57. package/dashboard/src/components/ClickToCopy.tsx +68 -0
  58. package/dashboard/src/components/CopyButton.tsx +61 -0
  59. package/dashboard/src/components/DailyCostTrend.tsx +85 -0
  60. package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
  61. package/dashboard/src/components/Funnel.tsx +149 -0
  62. package/dashboard/src/components/GateForm.tsx +158 -0
  63. package/dashboard/src/components/GraphView.tsx +302 -0
  64. package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
  65. package/dashboard/src/components/Histogram.tsx +292 -0
  66. package/dashboard/src/components/Layout.tsx +28 -0
  67. package/dashboard/src/components/NeedsAttention.tsx +98 -0
  68. package/dashboard/src/components/NodeContextCard.tsx +132 -0
  69. package/dashboard/src/components/PathTrail.tsx +35 -0
  70. package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
  71. package/dashboard/src/components/RerunButton.tsx +46 -0
  72. package/dashboard/src/components/RunComparison.tsx +53 -0
  73. package/dashboard/src/components/Sparkline.tsx +94 -0
  74. package/dashboard/src/components/StatStrip.tsx +147 -0
  75. package/dashboard/src/components/StateTree.tsx +84 -0
  76. package/dashboard/src/components/StatusBadge.tsx +18 -0
  77. package/dashboard/src/components/SubflowPathPills.tsx +38 -0
  78. package/dashboard/src/components/SubflowSummary.tsx +24 -0
  79. package/dashboard/src/components/SubflowsCard.tsx +327 -0
  80. package/dashboard/src/components/TodayStrip.tsx +86 -0
  81. package/dashboard/src/components/TranscriptPane.tsx +99 -0
  82. package/dashboard/src/components/TranscriptRow.tsx +69 -0
  83. package/dashboard/src/components/ui/badge.tsx +38 -0
  84. package/dashboard/src/components/ui/button.tsx +28 -0
  85. package/dashboard/src/components/ui/card.tsx +42 -0
  86. package/dashboard/src/components/ui/skeleton.tsx +9 -0
  87. package/dashboard/src/components/ui/table.tsx +62 -0
  88. package/dashboard/src/index.css +26 -0
  89. package/dashboard/src/lib/anomaly.ts +42 -0
  90. package/dashboard/src/lib/format.ts +30 -0
  91. package/dashboard/src/lib/step-history.ts +13 -0
  92. package/dashboard/src/lib/subflow.ts +154 -0
  93. package/dashboard/src/lib/use-click-outside.ts +28 -0
  94. package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
  95. package/dashboard/src/lib/utils.ts +6 -0
  96. package/dashboard/src/main.tsx +10 -0
  97. package/dashboard/src/types/shims.d.ts +4 -0
  98. package/dashboard/src/views/Home.tsx +478 -0
  99. package/dashboard/src/views/PipelineDetail.tsx +1185 -0
  100. package/dashboard/src/views/PipelinesList.tsx +111 -0
  101. package/dashboard/src/views/RunDetail.tsx +248 -0
  102. package/dashboard/src/views/Search.tsx +196 -0
  103. package/dashboard/stats.js +668 -0
  104. package/dashboard/tailwind.config.ts +43 -0
  105. package/dashboard/tree-stats.js +12 -0
  106. package/dashboard/tsconfig.json +25 -0
  107. package/dashboard/views.js +231 -0
  108. package/dashboard/vite.config.ts +34 -0
  109. package/drizzle.config.js +12 -0
  110. package/drizzle.config.ops.js +12 -0
  111. package/elevation.js +116 -0
  112. package/engine/admission-promotion-resume.js +127 -0
  113. package/engine/agent-dispatch.js +64 -0
  114. package/engine/child-resume-trigger.js +315 -0
  115. package/engine/executor-budget.js +87 -0
  116. package/engine/executor-cancel.js +83 -0
  117. package/engine/executor-errors.js +122 -0
  118. package/engine/executor-helpers.js +254 -0
  119. package/engine/executor-nodes.js +292 -0
  120. package/engine/executor-persist.js +212 -0
  121. package/engine/executor-subflow-children.js +275 -0
  122. package/engine/executor.js +1013 -0
  123. package/engine/global-concurrency.js +33 -0
  124. package/engine/loader.js +528 -0
  125. package/engine/needs-resolver.js +229 -0
  126. package/engine/pipelines-watcher.js +133 -0
  127. package/engine/result-replay.js +34 -0
  128. package/engine/revival-messages.js +11 -0
  129. package/engine/spawn-child.js +55 -0
  130. package/engine/wait-for-flow-settlement.js +109 -0
  131. package/events/catalog.js +26 -0
  132. package/events/emit.js +44 -0
  133. package/events/kinds.js +118 -0
  134. package/events/serve-ws.js +74 -0
  135. package/events/stream.js +35 -0
  136. package/events/ws-dialog.js +257 -0
  137. package/events/ws-frame.js +194 -0
  138. package/events/ws-run-events.js +70 -0
  139. package/events.js +32 -0
  140. package/gates/gate-resolver.js +576 -0
  141. package/gates/paused-flows.js +55 -0
  142. package/graphql/columns.js +67 -0
  143. package/graphql/schema.js +333 -0
  144. package/index.js +49 -0
  145. package/mcp/http.js +159 -0
  146. package/mcp/identity.js +94 -0
  147. package/mcp/stdio.js +80 -0
  148. package/mcp/tools.js +245 -0
  149. package/mcp/wire-ops.js +178 -0
  150. package/package.json +51 -0
  151. package/routes/bearer-auth.js +57 -0
  152. package/routes/discuss-audio.js +87 -0
  153. package/routes/discuss-component.js +39 -0
  154. package/routes/discuss-draft.js +159 -0
  155. package/routes/discuss-helpers.js +20 -0
  156. package/routes/discuss.js +848 -0
  157. package/routes/failures.js +104 -0
  158. package/routes/graphql.js +61 -0
  159. package/routes/health.js +54 -0
  160. package/routes/operator-discuss.js +193 -0
  161. package/routes/operator-resume.js +32 -0
  162. package/routes/operator-retry.js +301 -0
  163. package/routes/operator.js +539 -0
  164. package/routes/pipelines.js +118 -0
  165. package/routes/runs.js +1129 -0
  166. package/scripts/ci/import-check.mjs +50 -0
  167. package/scripts/link-core-self.mjs +31 -0
  168. package/scripts/with-guard.sh +24 -0
  169. package/server/app.js +503 -0
  170. package/server/registry.js +53 -0
  171. package/server/serve.js +163 -0
  172. package/state/admission-queue-store.js +325 -0
  173. package/state/attribution-registries.js +102 -0
  174. package/state/batch-writer.js +162 -0
  175. package/state/db-guard.js +63 -0
  176. package/state/db-health.js +27 -0
  177. package/state/db.js +122 -0
  178. package/state/discuss-store.js +299 -0
  179. package/state/failure-store.js +427 -0
  180. package/state/migrate.js +104 -0
  181. package/state/node-cost.js +67 -0
  182. package/state/node-executions.js +775 -0
  183. package/state/node-health-store.js +124 -0
  184. package/state/ops-merge.js +93 -0
  185. package/state/paths.js +146 -0
  186. package/state/pipeline-dirs.js +99 -0
  187. package/state/pipeline-registry.js +449 -0
  188. package/state/run-store.js +697 -0
  189. package/state/schema.js +438 -0
  190. package/state/schema.ops.js +44 -0
  191. package/state/session-index.js +177 -0
  192. package/state/session-keys.js +39 -0
  193. package/state/state-store.js +311 -0
  194. package/state/subflow-state.js +126 -0
  195. package/state/tool-handoff.js +169 -0
  196. package/telemetry/budget-alert.js +353 -0
  197. package/telemetry/cli.js +331 -0
  198. package/telemetry/grammar.js +119 -0
  199. package/telemetry/http-budget.js +132 -0
  200. package/telemetry/http-cost.js +670 -0
  201. package/telemetry/http-helpers.js +134 -0
  202. package/telemetry/http-tree.js +349 -0
  203. package/telemetry/http.js +44 -0
  204. package/telemetry/index.js +307 -0
  205. package/telemetry/ingest.js +134 -0
  206. package/telemetry/labels.js +115 -0
  207. package/telemetry/pricer.js +77 -0
  208. package/telemetry/resolver.js +314 -0
  209. package/telemetry/store.js +147 -0
  210. package/utils/run-patch.js +43 -0
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env node
2
+ import { runCli } from "./cli.js";
3
+
4
+ await runCli(process.argv.slice(2));
@@ -0,0 +1,536 @@
1
+ import { completeCancellation } from "../engine/executor-cancel.js";
2
+ import { areSiblingsAllTerminal, flipToRunningAndResume, subflowOutcomeSatisfied } from "../engine/child-resume-trigger.js";
3
+ import { resolveEdge } from "../engine/executor-helpers.js";
4
+ import { wakePromotedRelease, isEngineResumableKind, scheduleDetachedResume } from "../engine/admission-promotion-resume.js";
5
+ import { admissionKindForFlow } from "../engine/child-resume-trigger.js";
6
+ import { TERMINAL_STATUS_SET } from "../state/run-store.js";
7
+
8
+ // A promotion (release()'s side effect of flipping a queued row straight to
9
+ // 'admitted') is supposed to be immediately followed by wakePromotedRelease
10
+ // driving that flow to 'running' — see admission-promotion-resume.js and
11
+ // every release()/releaseByFlow() call site in this codebase. But that
12
+ // follow-through is still two separate steps (a durable DB write, then an
13
+ // async continuation that isn't part of the same transaction): if the
14
+ // continuation never runs (a crash in that exact window) or throws in a way
15
+ // nothing catches, the row stays 'admitted' forever while its flow sits
16
+ // non-terminal and non-running — holding a live slot in the shared pool
17
+ // with nothing left to drive it. The terminal-leak sweep above can't catch
18
+ // this: it only reconciles rows whose owning flow is already terminal, and
19
+ // this flow never gets there — it's stuck exactly where it was queued.
20
+ //
21
+ // minAgeMs guards against a false positive on a promotion that's
22
+ // genuinely still in flight (the continuation hasn't had a chance to run
23
+ // yet) — skip anything admitted more recently than that window. Exported
24
+ // (not folded into reviveOrphanFlows) so index.js can also run it on a
25
+ // periodic timer: reviveOrphanFlows only runs once, at boot, so a leak
26
+ // occurring hours into uptime would otherwise go undetected until the next
27
+ // restart.
28
+ //
29
+ // Also reaps admitted rows of kinds this engine does not admit — see the
30
+ // unresumable-kind pass below for why skipping those is what wedged the whole
31
+ // pool shut, not a safe no-op.
32
+ export async function reconcileStuckAdmissions({ admissionQueueStore, runStore, executor, api, logger, minAgeMs = 30_000 }) {
33
+ if (!admissionQueueStore) return { stuckAdmissionsResumed: 0, stuckAdmissionsFailed: 0, retiredKindReaped: 0 };
34
+ const now = Date.now();
35
+ const admittedRows = await admissionQueueStore.listByStatus("admitted");
36
+
37
+ // Reap admitted rows whose kind this engine does not admit. Such a kind
38
+ // can never be driven: resumePromotedAdmission refuses non-resumable
39
+ // kinds, and every flow transition releases by the flow's CURRENT
40
+ // admission kind, so releaseByFlow never matches such a row either.
41
+ // "Unresumable" therefore makes the row a permanent pool leak, not a safe
42
+ // skip — admit()'s and fillFreeSlots()'s admitted-count queries have no
43
+ // kind filter, so zombie rows count against the ceiling forever and wedge
44
+ // both shut. release() is the reclaim: it frees the slot and, as a side
45
+ // effect, promotes the globally-oldest queued row, which
46
+ // wakePromotedRelease then drives — the freed capacity goes back to work
47
+ // in the same pass instead of idling until some unrelated release.
48
+ // A QUEUED row of such a kind is left here (it holds no slot), but
49
+ // fillFreeSlots() can promote one into 'admitted'; this pass is what
50
+ // reclaims it after that, past the same grace window.
51
+ let retiredKindReaped = 0;
52
+ const resumableRows = [];
53
+ for (const row of admittedRows) {
54
+ if (isEngineResumableKind(row.kind)) { resumableRows.push(row); continue; }
55
+ if (row.admittedAt && now - row.admittedAt < minAgeMs) continue;
56
+ const flow = await runStore.get(row.flowId);
57
+ // A flow still 'running' owns the slot legitimately and releases it
58
+ // through its own transition path; reclaiming the row underneath it
59
+ // would transiently over-admit the pool. Missing or terminal flows are
60
+ // reaped: no live process can release them (the terminal-leak sweep
61
+ // above only runs at boot, and release() is a safe no-op if it already
62
+ // did).
63
+ if (flow?.status === "running") continue;
64
+ try {
65
+ const result = await admissionQueueStore.release(row.id);
66
+ await wakePromotedRelease(result, {
67
+ runStore, resume: executor.resume, logger, api,
68
+ context: `manifold revival: retired-kind reap [${row.id.slice(0, 8)}]`,
69
+ });
70
+ retiredKindReaped++;
71
+ logger.warn(
72
+ `manifold revival: reaped retired-kind '${row.kind}' admission [${row.id.slice(0, 8)}] for flow ` +
73
+ `[${row.flowId.slice(0, 8)}] (${flow ? `status=${flow.status}` : "flow_runs row gone"}) — ` +
74
+ `no code path can drive this kind, slot reclaimed`,
75
+ );
76
+ } catch (err) {
77
+ logger.warn(`manifold revival: retired-kind reap [${row.id.slice(0, 8)}] failed: ${err?.message ?? err}`);
78
+ }
79
+ }
80
+
81
+ // Filter down to genuinely stuck candidates first — cheap, fast DB
82
+ // reads, fine to do one at a time. flipToRunningAndResume itself is the
83
+ // slow, unbounded part: it can resolve a flow all the way to a fresh
84
+ // admission request of its own, which — under a congested pool — waits
85
+ // on yet another slot. Driving that sequentially, one candidate at a
86
+ // time, meant a single candidate stuck behind real contention blocked
87
+ // every other candidate behind it in the same pass indefinitely: this
88
+ // sweep exists specifically to recover flows stuck exactly like that, so
89
+ // serializing on one of them defeated its own purpose under load.
90
+ const candidates = [];
91
+ for (const row of resumableRows) {
92
+ if (row.admittedAt && now - row.admittedAt < minAgeMs) continue;
93
+ const flow = await runStore.get(row.flowId);
94
+ // No flow_runs row at all, or already terminal: not this sweep's job —
95
+ // the terminal-leak reconciliation above (boot-only) or a future run of
96
+ // it handles those. This sweep only exists for "admitted but genuinely
97
+ // stuck non-terminal", so a flow already 'running' is also skipped —
98
+ // that's the healthy, expected state.
99
+ if (!flow || flow.status === "running" || TERMINAL_STATUS_SET.has(flow.status)) continue;
100
+ // The row being 'admitted' only means THIS flow's row for THIS kind
101
+ // holds a live slot — it says nothing about what the flow is currently
102
+ // doing: a flow can be 'queued' on a different resource (a cross-process
103
+ // lock) while its slot is legitimately still counted. Force-driving that
104
+ // would resume it on top of its own in-flight wait. The real, narrow
105
+ // signature of the leak this sweep exists for is a flow queued waiting
106
+ // specifically to be admitted for the SAME kind it already holds — a
107
+ // state that can only exist if a promotion's flip-to-running
108
+ // follow-through was dropped, since nothing else produces it.
109
+ if (flow.waitJson?.kind !== row.kind) continue;
110
+ candidates.push(row);
111
+ }
112
+
113
+ let stuckAdmissionsResumed = 0;
114
+ let stuckAdmissionsFailed = 0;
115
+ await Promise.all(candidates.map(async (row) => {
116
+ logger.warn(
117
+ `manifold revival: admitted ${row.kind} slot [${row.id.slice(0, 8)}] for flow [${row.flowId.slice(0, 8)}] ` +
118
+ `never resumed (status=queued, admitted ${Math.round((now - (row.admittedAt ?? now)) / 60000)}min ago) — force-driving flipToRunningAndResume`,
119
+ );
120
+ try {
121
+ // No admissionQueueStore passed through: this flow already holds the
122
+ // live admission (that's how it got into `admittedRows` above) —
123
+ // passing it would have flipToRunningAndResume call admit() again,
124
+ // which the resume-admitted CLI command's own header comment already
125
+ // flags as unsafe to do for a flow that already holds the slot.
126
+ const result = await flipToRunningAndResume({ parentFlowId: row.flowId, runStore, resume: executor.resume, logger, api });
127
+ if (result.resumed) {
128
+ stuckAdmissionsResumed++;
129
+ } else {
130
+ stuckAdmissionsFailed++;
131
+ logger.warn(`manifold revival: stuck-admission force-resume for [${row.flowId.slice(0, 8)}] did not apply: ${result.reason ?? "unknown"}`);
132
+ }
133
+ } catch (err) {
134
+ stuckAdmissionsFailed++;
135
+ logger.warn(`manifold revival: stuck-admission force-resume for [${row.flowId.slice(0, 8)}] threw: ${err?.message ?? err}`);
136
+ }
137
+ }));
138
+ return { stuckAdmissionsResumed, stuckAdmissionsFailed, retiredKindReaped };
139
+ }
140
+
141
+ /**
142
+ * Boot-time orphan recovery for flow_runs left in status='running' (crashed
143
+ * mid-flight) or status='queued' (waiting on a shared
144
+ * admission-pool slot — flow-start or child-dispatch — when the
145
+ * process died) after a gateway crash. Called by the manifold-flow-revival
146
+ * service in index.js.
147
+ *
148
+ * Returns a summary object for observability and testing:
149
+ * { revived: number, revivedAgent: number, queuedForAdmission: number, readmitted: number, recancelled: number, childrenReconciled: number, gatesReconciled: number, admissionLeaksReleased: number, freeSlotsFilled: number }
150
+ */
151
+
152
+ export async function reviveOrphanFlows({
153
+ all, pipelines, store, logger, executor, createCancelStore,
154
+ runStore, nodeExecutionsStore, maybeResolveFlow, sessionIndex, admissionQueueStore, api,
155
+ // Threaded through to the stuck-promotion reconciliation call below —
156
+ // exists mainly so tests can exercise that pass without waiting out its
157
+ // real-world grace window (a genuine boot's admission rows are all older
158
+ // than this process's own start, so the default is trivially satisfied
159
+ // there;
160
+ // a unit test that admits a row moments before calling this function is
161
+ // not so lucky).
162
+ stuckAdmissionMinAgeMs,
163
+ }) {
164
+ // ── Complete interrupted cancellations ──────────────────────────────────────
165
+ // store.cancel() writes cancel_requested_at first, then status='cancelled',
166
+ // in two separate SQL statements. A crash between them leaves a flow with
167
+ // cancelRequestedAt set but status still 'running'/'blocked'/'waiting'.
168
+ // Re-cancel those before reviving anything else so they don't land in toRevive.
169
+ // Uses the same completeCancellation path as executor.cancel() — step 1 is
170
+ // already done, so f.revision is already the post-requestCancel value and is
171
+ // passed directly to completeCancellation as the expectedRevision for setRunning.
172
+ const cancelledByRevival = new Set();
173
+ const terminalStatuses = new Set(["cancelled", "failed", "succeeded"]);
174
+ const cancelStores = new Map();
175
+ function cancelStoreFor(ownerKey) {
176
+ if (!ownerKey || !createCancelStore) return store;
177
+ if (!cancelStores.has(ownerKey)) cancelStores.set(ownerKey, createCancelStore(ownerKey));
178
+ return cancelStores.get(ownerKey);
179
+ }
180
+
181
+ for (const f of all) {
182
+ if (f.cancelRequestedAt == null) continue;
183
+ if (terminalStatuses.has(f.status)) continue;
184
+ logger.info(
185
+ `manifold revival: completing interrupted cancel [${f.flowId.slice(0, 8)}] status=${f.status}`,
186
+ );
187
+ await completeCancellation({
188
+ store: cancelStoreFor(f.ownerKey),
189
+ flowId: f.flowId,
190
+ stateJson: f.stateJson,
191
+ lastStep: f.lastStep,
192
+ revision: f.revision,
193
+ logger,
194
+ runStore,
195
+ nodeExecutionsStore,
196
+ reason: "cancel requested before restart, completed on revival",
197
+ });
198
+ cancelledByRevival.add(f.flowId);
199
+ }
200
+
201
+ // ── Classify orphans (running AND queued, any wait kind) ───────────────────
202
+ // 'running' orphans: process died mid-flight. 'queued' orphans: process
203
+ // died (or restarted) while the flow was waiting on an internal resource —
204
+ // the shared flow-start/child-dispatch admission pool.
205
+ // All land here uniformly; the skip-direct-dispatch step
206
+ // below and the revive-survivors step's dispatch don't need to know *why*
207
+ // a row was queued, only whether it wants to run.
208
+ //
209
+ // isAgent decides HOW to revive an agent-kind node: reviveAgentKindOrphan()
210
+ // resumes the SAME deterministic session with REVIVAL_MSG, a continuation
211
+ // prompt, instead of re-running promptBuilder fresh (see engine/
212
+ // revival-messages.js) — and it sends that continuation prompt
213
+ // UNCONDITIONALLY, even to a session with zero prior turns (executor-
214
+ // nodes.js's dispatch: `reviveMessage ?? promptBuilder(...)`, with no
215
+ // branch on whether a prior invocation/session actually exists). So
216
+ // isAgent is only correct when a dispatch attempt genuinely started —
217
+ // sending REVIVAL_MSG as literally the first message to a node that was
218
+ // only ever queued, never dispatched, would be a "continue where you left
219
+ // off" prompt with nothing to continue.
220
+ //
221
+ // For a 'running' orphan, targetNodeId comes from
222
+ // nodeExecutionsStore.findCurrentStep() below, which IS race-free proof a
223
+ // dispatch attempt genuinely entered this node: the nodeExecutions.enter()
224
+ // call creates the invocation row BEFORE executeNode() runs, well before
225
+ // dispatchAgentWithRetry gets to recording a flow_sessions entry
226
+ // (session-index.js) or calling subagent.run() — so currentStep alone is
227
+ // sufficient there, without consulting sessionIndex.hasSessionForNode().
228
+ //
229
+ // The 'running' case must not REQUIRE a session record. The flow_sessions
230
+ // record is written well after enter() (past several awaits — budget gate,
231
+ // assembleState, promptBuilder, dispatch-semaphore acquisition), so a crash
232
+ // in that narrow window leaves a genuinely in-flight agent dispatch with a
233
+ // correct targetNodeId but hasSessionForNode()===false, and requiring it
234
+ // misroutes that dispatch into the generic resolveEdge()/runFlow() path
235
+ // below, which has no node output to route on and fails with "edge <NODE>
236
+ // returned no next node". A running flow with no invocation row yet
237
+ // (findCurrentStep() -> null) is the other case, and there falling through
238
+ // to resolveEdge() IS correct; hasSessionForNode() is what still decides
239
+ // isAgent for it, via the !dispatchGenuinelyEntered branch below.
240
+ //
241
+ // For a 'queued' orphan, targetNodeId instead comes from waitJson.nodeId
242
+ // — set by the admission wait before it acquires a slot, which only
243
+ // proves the node was ABOUT to be dispatched, not that dispatch began.
244
+ // hasSessionForNode() still gates isAgent there.
245
+ //
246
+ // f.lastStep is the last COMPLETED node, not the one in flight — using it
247
+ // directly as the crashed node's id is wrong. The real target node is:
248
+ // - queued: f.waitJson.nodeId — the admission wait sets it directly
249
+ // (executor.js) since a queued flow never got far enough to advance
250
+ // lastStep at all.
251
+ // - running: nodeExecutionsStore.findCurrentStep(flowId) — the invocation
252
+ // row is created (enter()) before executeNode() runs, so
253
+ // the last invocation's node_id correctly identifies the in-flight node
254
+ // even across a crash. Falls back to resolveEdge(lastStep) if the query
255
+ // returns null (first iteration of a brand-new flow with no invocations).
256
+ const orphans = [];
257
+ for (const f of all) {
258
+ if (f.status !== "running" && f.status !== "queued") continue;
259
+ if (cancelledByRevival.has(f.flowId)) continue;
260
+ const pipeline = pipelines.get(f.stateJson?.pipelineId);
261
+ const currentStep = f.status === "running"
262
+ ? await nodeExecutionsStore?.findCurrentStep(f.flowId)
263
+ : null;
264
+ const targetNodeId = f.waitJson?.nodeId
265
+ ?? currentStep
266
+ ?? resolveEdge(pipeline?.edges ?? {}, f.lastStep, f.stateJson, { logger, flowId: f.flowId });
267
+ const nodeKind = pipeline?.nodes?.[targetNodeId]?.kind;
268
+ const dispatchGenuinelyEntered = currentStep != null && targetNodeId === currentStep;
269
+ const hasSession = nodeKind === "agent" && !dispatchGenuinelyEntered && sessionIndex != null
270
+ ? !!(await sessionIndex.hasSessionForNode(f.flowId, targetNodeId))
271
+ : false;
272
+ const isAgent = nodeKind === "agent" && (dispatchGenuinelyEntered || hasSession);
273
+ orphans.push({ flow: f, isAgent, targetNodeId, nodeKind });
274
+ }
275
+
276
+ // ── Skip direct dispatch for orphans queued on the shared admission
277
+ // pool ──────────────────────────────────────────────────────────────────
278
+ // The orphan-classification step above classified every non-terminal
279
+ // orphan as 'running' or 'queued'.
280
+ // A 'queued' orphan waiting on kind:"flow-start" or "child-dispatch" is
281
+ // tracked in the DURABLE combined admission pool (state/admission-queue-
282
+ // store.js — the same runtime.sqlite every other table here survives a
283
+ // crash in), which already reflects who was genuinely admitted vs still
284
+ // queued the moment the gateway died. Nothing needs to be re-derived: a
285
+ // live queued flow-start/child-dispatch flow is never dispatched
286
+ // directly either (see engine/executor.js's runFlow() isQueued guard) —
287
+ // the same admission-queue-store release()/promotion cycle that wakes a
288
+ // live queued flow wakes a boot-revived one too, once some other admitted
289
+ // flow elsewhere goes blocked or terminal, no special boot-time action
290
+ // needed. Throttling happens atomically at the moment each flow admits,
291
+ // not as a bulk post-hoc displacement correction here.
292
+ //
293
+ // A 'queued' orphan on waitJson.kind:"agent-dispatch" — a kind this engine
294
+ // does not admit (state/admission-queue-store.js's migration deletes its
295
+ // rows) — gave its own run slot back when it queued, so it holds nothing.
296
+ // Re-entering it directly would run it outside the pool; instead it
297
+ // re-requests its own run slot (flipToRunningAndResume with the store:
298
+ // admitted → flipped to running and resumed; pool full → queued on its
299
+ // own kind for the normal promotion cycle).
300
+ //
301
+ const toRevive = [];
302
+ let queuedForAdmission = 0;
303
+ let readmitted = 0;
304
+ for (const o of orphans) {
305
+ const kind = o.flow.waitJson?.kind;
306
+ if (o.flow.status === "queued" && (kind === "flow-start" || kind === "child-dispatch")) {
307
+ logger.info(`manifold revival: [${o.flow.flowId.slice(0, 8)}] left queued for admission (${kind}) — will resume via the normal release/promotion cycle`);
308
+ queuedForAdmission++;
309
+ continue;
310
+ }
311
+ if (o.flow.status === "queued" && kind === "agent-dispatch" && admissionQueueStore && runStore) {
312
+ const result = await flipToRunningAndResume({
313
+ parentFlowId: o.flow.flowId, runStore, admissionQueueStore, logger, api,
314
+ resume: (flowId) => { scheduleDetachedResume(executor.resume, flowId, { logger, api, context: "manifold revival: re-admit" }); },
315
+ });
316
+ logger.info(
317
+ `manifold revival: [${o.flow.flowId.slice(0, 8)}] re-requested its ${admissionKindForFlow(o.flow)} slot — ` +
318
+ `${result.resumed ? "admitted, resuming" : result.reason}`,
319
+ );
320
+ readmitted++;
321
+ continue;
322
+ }
323
+ toRevive.push(o);
324
+ }
325
+
326
+ // ── Revive survivors ────────────────────────────────────────────────────────
327
+ let revivedAgent = 0;
328
+ if (toRevive.length === 0) {
329
+ logger.info("manifold revival: no revivable orphan flows at boot");
330
+ } else {
331
+ logger.info(`manifold revival: reviving ${toRevive.length} orphan flow(s) from lastStep`);
332
+ for (const o of toRevive) {
333
+ const flow = o.flow;
334
+ const pipelineId = flow.stateJson?.pipelineId ?? "?";
335
+ logger.info(`manifold revival: [${flow.flowId.slice(0, 8)}] pipeline=${pipelineId} step=${o.targetNodeId}`);
336
+ if (o.isAgent) {
337
+ revivedAgent++;
338
+ // Resumes the SAME session with a continuation prompt. No admission
339
+ // work: the flow's own run slot (flow-start/child-dispatch) is
340
+ // durable across the crash and still held, and the agent call runs
341
+ // inside it like any other node — it's released through the flow's
342
+ // normal lifecycle once the run blocks or finishes.
343
+ executor.reviveAgentKindOrphan(flow, o.targetNodeId)
344
+ .catch((err) => {
345
+ logger.warn(`manifold revival: reviveAgentKindOrphan [${flow.flowId.slice(0, 8)}] error: ${err?.message ?? err}`);
346
+ });
347
+ } else {
348
+ executor.runFlow(flow.flowId, "revival").catch((err) => {
349
+ logger.warn(`manifold revival: runFlow [${flow.flowId.slice(0, 8)}] error: ${err?.message ?? err}`);
350
+ });
351
+ }
352
+ }
353
+ }
354
+
355
+ // ── Children-pending reconciliation ─────────────────────────────────────────
356
+ // The resume-trigger (state-store.js's finish/skip/fail/cancel) fires
357
+ // on the LAST sibling's terminal write — but if the gateway crashes at
358
+ // exactly that moment, the fire can be lost. Re-check every blocked parent
359
+ // directly against its children's current terminal state and resume if
360
+ // already satisfied, the same idempotent-recheck shape gate-resolver.js's
361
+ // maybeResolveFlow already uses for the analogous gate-side problem. No-op
362
+ // when nodeExecutionsStore/runStore aren't wired in (older callers, tests).
363
+ let childrenReconciled = 0;
364
+ if (runStore && nodeExecutionsStore) {
365
+ for (const f of all) {
366
+ if (f.status !== "blocked") continue;
367
+ if (f.waitJson?.kind !== "children-pending") continue;
368
+ const nodeId = f.waitJson?.nodeId;
369
+ if (!nodeId) continue;
370
+ const { allTerminal, children } = await areSiblingsAllTerminal({
371
+ flowId: f.flowId, nodeId, runStore, nodeExecutionsStore,
372
+ });
373
+ if (!allTerminal) continue;
374
+ childrenReconciled++;
375
+ // Same mode check as the live-process path
376
+ // (child-resume-trigger.js's maybeResumeParent) — a parent orphaned
377
+ // mid-resume by a gateway crash must fail (not resume) if its
378
+ // subflow node's mode isn't satisfied, same as it would have if the
379
+ // crash hadn't happened.
380
+ const mode = pipelines?.get?.(f.pipelineId)?.nodes?.[nodeId]?.mode;
381
+ if (mode && !subflowOutcomeSatisfied(mode, children)) {
382
+ logger.info(`manifold revival: failing [${f.flowId.slice(0, 8)}] — subflow "${nodeId}" (mode:"${mode}") not satisfied`);
383
+ await store.fail({
384
+ flowId: f.flowId, expectedRevision: f.revision, stateJson: f.stateJson,
385
+ statusReason: `subflow "${nodeId}" (mode:"${mode}") not satisfied — ${mode === "any" ? "every child failed" : "at least one child did not succeed"}`,
386
+ });
387
+ continue;
388
+ }
389
+ logger.info(`manifold revival: resuming [${f.flowId.slice(0, 8)}] — all children of ${nodeId} terminal`);
390
+ // Must flip status back to 'running' before calling resume() —
391
+ // executor.resume() is just runFlow(), whose own first check returns
392
+ // immediately for a flow still in status='blocked'. See
393
+ // child-resume-trigger.js's flipToRunningAndResume for why.
394
+ await flipToRunningAndResume({ parentFlowId: f.flowId, runStore, resume: executor.resume, logger, admissionQueueStore, api }).catch((err) => {
395
+ logger.warn(`manifold revival: children-pending resume [${f.flowId.slice(0, 8)}] error: ${err?.message ?? err}`);
396
+ });
397
+ }
398
+ }
399
+
400
+ // ── Stuck-gate-decision reconciliation ──────────────────────────────────────
401
+ // A gate decision is recorded in two steps: (1) write approvalPending[itemId]
402
+ // durably, (2) call maybeResolveFlow → applyResolutionsAndResume, which
403
+ // moves the decision into approvedIds/rejectedIds and resumes the flow.
404
+ // If the process ever dies between those two steps, approvalPending
405
+ // permanently shows a real decision that was never actually applied —
406
+ // the flow stays blocked forever, since nothing else ever re-drives step
407
+ // (2). Same idempotent-recheck shape as the children-pending
408
+ // reconciliation above, applied to the analogous gate-side gap. No-op
409
+ // when maybeResolveFlow isn't wired in (older callers, tests).
410
+ let gatesReconciled = 0;
411
+ if (maybeResolveFlow) {
412
+ for (const f of all) {
413
+ if (f.status !== "blocked") continue;
414
+ const waitKind = f.waitJson?.kind;
415
+ if (waitKind !== "human-gate" && waitKind !== "gate" && waitKind !== "verify-required") continue;
416
+ const items = Array.isArray(f.waitJson?.items) ? f.waitJson.items : [];
417
+ if (items.length === 0) continue;
418
+ const approvalPending = f.stateJson?.approvalPending ?? {};
419
+ const hasUnappliedDecision = items.some(({ itemId }) => {
420
+ const entry = approvalPending[itemId];
421
+ return entry && (entry.decision || entry.sendFailed);
422
+ });
423
+ if (!hasUnappliedDecision) continue;
424
+ logger.info(`manifold revival: [${f.flowId.slice(0, 8)}] has a recorded gate decision never applied — reconciling`);
425
+ gatesReconciled++;
426
+ maybeResolveFlow(f.flowId).catch((err) => {
427
+ logger.warn(`manifold revival: gate reconciliation [${f.flowId.slice(0, 8)}] error: ${err?.message ?? err}`);
428
+ });
429
+ }
430
+ }
431
+
432
+ // ── Admission-pool leak reconciliation ──────────────────────────────────────
433
+ // A flow_admission_queue row (state/admission-queue-store.js) is only ever
434
+ // supposed to exist while its owning flow is genuinely non-terminal —
435
+ // every terminal transition (finish/fail/cancel) releases it via
436
+ // state-store.js's triggerAdmissionRelease. That release silently no-ops
437
+ // if the caller's state-store was constructed without admissionQueueStore
438
+ // wired in, leaving the flow's row "admitted" forever and shrinking the
439
+ // one shared pool (a single combined ceiling across every
440
+ // flow-start/child-dispatch, gateway-wide) for the rest of the process's
441
+ // uptime. Rather than trust every future release call site
442
+ // to stay correctly wired, sweep for the
443
+ // invariant violation itself on every boot: a terminal flow can never
444
+ // legitimately hold a live admission entry, so any row whose owning flow
445
+ // is already terminal is definitionally a leak, regardless of which bug
446
+ // produced it. statusByFlowId is built from `all` (already loaded above)
447
+ // plus cancelledByRevival (flows this very call just finished cancelling,
448
+ // whose stale status in `all` is older than that) — no extra flow_runs reads.
449
+ // Own terminal-status set, not the narrower `terminalStatuses` above
450
+ // (cancelled/failed/succeeded only, scoped to that phase's specific
451
+ // re-cancel check) — this sweep must also catch "skipped" (FILTER's
452
+ // dedup path, run-store.js's own TERMINAL_STATUSES), any of which can
453
+ // equally hold a leaked admission row.
454
+ const admissionTerminalStatuses = new Set(["succeeded", "failed", "cancelled", "skipped"]);
455
+ let admissionLeaksReleased = 0;
456
+ if (admissionQueueStore) {
457
+ const statusByFlowId = new Map(all.map((f) => [f.flowId, f.status]));
458
+ const admissionRows = [
459
+ ...(await admissionQueueStore.listByStatus("admitted")),
460
+ ...(await admissionQueueStore.listByStatus("queued")),
461
+ ];
462
+ for (const row of admissionRows) {
463
+ const owningStatus = cancelledByRevival.has(row.flowId) ? "cancelled" : statusByFlowId.get(row.flowId);
464
+ if (!owningStatus || !admissionTerminalStatuses.has(owningStatus)) continue;
465
+ logger.info(
466
+ `manifold revival: releasing leaked ${row.kind} admission [${row.id.slice(0, 8)}] ` +
467
+ `for already-terminal flow [${row.flowId.slice(0, 8)}] (status=${owningStatus})`,
468
+ );
469
+ try {
470
+ const result = await admissionQueueStore.release(row.id);
471
+ // release() frees this leaked slot and, as a side effect, may
472
+ // promote the globally-oldest queued row into 'admitted' — a
473
+ // DIFFERENT flow than the one this iteration is cleaning up.
474
+ // Every other caller of release()/releaseByFlow() in this codebase
475
+ // drives wakePromotedRelease() on that release, and so must this
476
+ // sweep: calling release() directly and discarding its result would
477
+ // flip the row to 'admitted' with nothing to ever flip the promoted
478
+ // flow's own flow_runs status to 'running' or call runFlow() on it
479
+ // — its admission would look live forever while the flow itself sat
480
+ // frozen exactly where it was queued, permanently shrinking the
481
+ // shared pool by one slot per promotion this sweep caused.
482
+ await wakePromotedRelease(result, {
483
+ runStore, resume: executor.resume, logger, api,
484
+ context: `manifold revival: leak-release promotion [${row.id.slice(0, 8)}]`,
485
+ });
486
+ } catch (err) {
487
+ logger.warn(`manifold revival: admission release [${row.id.slice(0, 8)}] failed: ${err?.message ?? err}`);
488
+ }
489
+ admissionLeaksReleased++;
490
+ }
491
+ }
492
+
493
+ // ── Fill free capacity ──────────────────────────────────────────────────────
494
+ // Every release promotes exactly one row into the slot it frees, so the
495
+ // pool never drops below its limit with work queued — except when
496
+ // admitted rows vanish without a release (a crash between a delete and
497
+ // its promotion, or a migration dropping a whole kind). Promote queued
498
+ // rows into whatever capacity is free now, so the pool doesn't idle below
499
+ // its limit until enough unrelated releases happen.
500
+ let freeSlotsFilled = 0;
501
+ if (admissionQueueStore) {
502
+ try {
503
+ const promoted = await admissionQueueStore.fillFreeSlots();
504
+ for (const p of promoted) {
505
+ await wakePromotedRelease({ released: false, promoted: p }, {
506
+ runStore, resume: executor.resume, logger, api,
507
+ context: `manifold revival: free-slot fill [${p.id.slice(0, 8)}]`,
508
+ });
509
+ }
510
+ freeSlotsFilled = promoted.length;
511
+ if (freeSlotsFilled > 0) logger.info(`manifold revival: filled ${freeSlotsFilled} free admission slot(s) from the queue`);
512
+ } catch (err) {
513
+ logger.warn(`manifold revival: free-slot fill failed: ${err?.message ?? err}`);
514
+ }
515
+ }
516
+
517
+ // ── Stuck-promotion reconciliation ──────────────────────────────────────────
518
+ // The sweep above only catches leaks on already-terminal flows. Run the
519
+ // non-terminal counterpart too (see reconcileStuckAdmissions's own header)
520
+ // so a boot doesn't just inherit the same "left queued for admission —
521
+ // will resume via the normal release/promotion cycle" hope the process
522
+ // that crashed was already relying on.
523
+ const { stuckAdmissionsResumed, stuckAdmissionsFailed, retiredKindReaped } = await reconcileStuckAdmissions({
524
+ admissionQueueStore, runStore, executor, api, logger,
525
+ ...(stuckAdmissionMinAgeMs != null ? { minAgeMs: stuckAdmissionMinAgeMs } : {}),
526
+ });
527
+ if (retiredKindReaped > 0) {
528
+ logger.warn(`manifold revival: reaped ${retiredKindReaped} retired-kind admission slot(s) no code path could drive`);
529
+ }
530
+
531
+ return {
532
+ revived: toRevive.length, revivedAgent, queuedForAdmission, readmitted, recancelled: cancelledByRevival.size,
533
+ childrenReconciled, gatesReconciled, admissionLeaksReleased, freeSlotsFilled,
534
+ stuckAdmissionsResumed, stuckAdmissionsFailed, retiredKindReaped,
535
+ };
536
+ }