@igorjpimenta/manifold 0.0.0-bootstrap.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/.migrations/ops/0000_futuristic_karma.sql +111 -0
  2. package/.migrations/ops/0001_low_maximus.sql +7 -0
  3. package/.migrations/ops/meta/0000_snapshot.json +763 -0
  4. package/.migrations/ops/meta/0001_snapshot.json +173 -0
  5. package/.migrations/ops/meta/_journal.json +20 -0
  6. package/.migrations/runtime/0000_real_boomer.sql +180 -0
  7. package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
  8. package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
  9. package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
  10. package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
  11. package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
  12. package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
  13. package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
  14. package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
  15. package/.migrations/runtime/0009_lying_shocker.sql +96 -0
  16. package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
  17. package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
  18. package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
  19. package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
  20. package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
  21. package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
  22. package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
  23. package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
  24. package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
  25. package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
  26. package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
  27. package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
  28. package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
  29. package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
  30. package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
  31. package/.migrations/runtime/meta/_journal.json +97 -0
  32. package/README.md +1584 -0
  33. package/RELEASING.md +89 -0
  34. package/agent/fleet.js +100 -0
  35. package/bin/cli.js +854 -0
  36. package/bin/manifold.js +4 -0
  37. package/boot/flow-revival.js +536 -0
  38. package/budget/index.js +495 -0
  39. package/budget/lib.js +59 -0
  40. package/dashboard/cost.js +528 -0
  41. package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
  42. package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
  43. package/dashboard/dist/index.html +13 -0
  44. package/dashboard/event-log.js +267 -0
  45. package/dashboard/index.html +12 -0
  46. package/dashboard/package-lock.json +2877 -0
  47. package/dashboard/package.json +37 -0
  48. package/dashboard/pipeline-graph.js +131 -0
  49. package/dashboard/postcss.config.js +6 -0
  50. package/dashboard/search.js +183 -0
  51. package/dashboard/server.js +1015 -0
  52. package/dashboard/src/App.tsx +42 -0
  53. package/dashboard/src/api/client.ts +118 -0
  54. package/dashboard/src/api/types.ts +510 -0
  55. package/dashboard/src/api/ws.tsx +112 -0
  56. package/dashboard/src/components/CancelButton.tsx +63 -0
  57. package/dashboard/src/components/ClickToCopy.tsx +68 -0
  58. package/dashboard/src/components/CopyButton.tsx +61 -0
  59. package/dashboard/src/components/DailyCostTrend.tsx +85 -0
  60. package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
  61. package/dashboard/src/components/Funnel.tsx +149 -0
  62. package/dashboard/src/components/GateForm.tsx +158 -0
  63. package/dashboard/src/components/GraphView.tsx +302 -0
  64. package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
  65. package/dashboard/src/components/Histogram.tsx +292 -0
  66. package/dashboard/src/components/Layout.tsx +28 -0
  67. package/dashboard/src/components/NeedsAttention.tsx +98 -0
  68. package/dashboard/src/components/NodeContextCard.tsx +132 -0
  69. package/dashboard/src/components/PathTrail.tsx +35 -0
  70. package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
  71. package/dashboard/src/components/RerunButton.tsx +46 -0
  72. package/dashboard/src/components/RunComparison.tsx +53 -0
  73. package/dashboard/src/components/Sparkline.tsx +94 -0
  74. package/dashboard/src/components/StatStrip.tsx +147 -0
  75. package/dashboard/src/components/StateTree.tsx +84 -0
  76. package/dashboard/src/components/StatusBadge.tsx +18 -0
  77. package/dashboard/src/components/SubflowPathPills.tsx +38 -0
  78. package/dashboard/src/components/SubflowSummary.tsx +24 -0
  79. package/dashboard/src/components/SubflowsCard.tsx +327 -0
  80. package/dashboard/src/components/TodayStrip.tsx +86 -0
  81. package/dashboard/src/components/TranscriptPane.tsx +99 -0
  82. package/dashboard/src/components/TranscriptRow.tsx +69 -0
  83. package/dashboard/src/components/ui/badge.tsx +38 -0
  84. package/dashboard/src/components/ui/button.tsx +28 -0
  85. package/dashboard/src/components/ui/card.tsx +42 -0
  86. package/dashboard/src/components/ui/skeleton.tsx +9 -0
  87. package/dashboard/src/components/ui/table.tsx +62 -0
  88. package/dashboard/src/index.css +26 -0
  89. package/dashboard/src/lib/anomaly.ts +42 -0
  90. package/dashboard/src/lib/format.ts +30 -0
  91. package/dashboard/src/lib/step-history.ts +13 -0
  92. package/dashboard/src/lib/subflow.ts +154 -0
  93. package/dashboard/src/lib/use-click-outside.ts +28 -0
  94. package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
  95. package/dashboard/src/lib/utils.ts +6 -0
  96. package/dashboard/src/main.tsx +10 -0
  97. package/dashboard/src/types/shims.d.ts +4 -0
  98. package/dashboard/src/views/Home.tsx +478 -0
  99. package/dashboard/src/views/PipelineDetail.tsx +1185 -0
  100. package/dashboard/src/views/PipelinesList.tsx +111 -0
  101. package/dashboard/src/views/RunDetail.tsx +248 -0
  102. package/dashboard/src/views/Search.tsx +196 -0
  103. package/dashboard/stats.js +668 -0
  104. package/dashboard/tailwind.config.ts +43 -0
  105. package/dashboard/tree-stats.js +12 -0
  106. package/dashboard/tsconfig.json +25 -0
  107. package/dashboard/views.js +231 -0
  108. package/dashboard/vite.config.ts +34 -0
  109. package/drizzle.config.js +12 -0
  110. package/drizzle.config.ops.js +12 -0
  111. package/elevation.js +116 -0
  112. package/engine/admission-promotion-resume.js +127 -0
  113. package/engine/agent-dispatch.js +64 -0
  114. package/engine/child-resume-trigger.js +315 -0
  115. package/engine/executor-budget.js +87 -0
  116. package/engine/executor-cancel.js +83 -0
  117. package/engine/executor-errors.js +122 -0
  118. package/engine/executor-helpers.js +254 -0
  119. package/engine/executor-nodes.js +292 -0
  120. package/engine/executor-persist.js +212 -0
  121. package/engine/executor-subflow-children.js +275 -0
  122. package/engine/executor.js +1013 -0
  123. package/engine/global-concurrency.js +33 -0
  124. package/engine/loader.js +528 -0
  125. package/engine/needs-resolver.js +229 -0
  126. package/engine/pipelines-watcher.js +133 -0
  127. package/engine/result-replay.js +34 -0
  128. package/engine/revival-messages.js +11 -0
  129. package/engine/spawn-child.js +55 -0
  130. package/engine/wait-for-flow-settlement.js +109 -0
  131. package/events/catalog.js +26 -0
  132. package/events/emit.js +44 -0
  133. package/events/kinds.js +118 -0
  134. package/events/serve-ws.js +74 -0
  135. package/events/stream.js +35 -0
  136. package/events/ws-dialog.js +257 -0
  137. package/events/ws-frame.js +194 -0
  138. package/events/ws-run-events.js +70 -0
  139. package/events.js +32 -0
  140. package/gates/gate-resolver.js +576 -0
  141. package/gates/paused-flows.js +55 -0
  142. package/graphql/columns.js +67 -0
  143. package/graphql/schema.js +333 -0
  144. package/index.js +49 -0
  145. package/mcp/http.js +159 -0
  146. package/mcp/identity.js +94 -0
  147. package/mcp/stdio.js +80 -0
  148. package/mcp/tools.js +245 -0
  149. package/mcp/wire-ops.js +178 -0
  150. package/package.json +51 -0
  151. package/routes/bearer-auth.js +57 -0
  152. package/routes/discuss-audio.js +87 -0
  153. package/routes/discuss-component.js +39 -0
  154. package/routes/discuss-draft.js +159 -0
  155. package/routes/discuss-helpers.js +20 -0
  156. package/routes/discuss.js +848 -0
  157. package/routes/failures.js +104 -0
  158. package/routes/graphql.js +61 -0
  159. package/routes/health.js +54 -0
  160. package/routes/operator-discuss.js +193 -0
  161. package/routes/operator-resume.js +32 -0
  162. package/routes/operator-retry.js +301 -0
  163. package/routes/operator.js +539 -0
  164. package/routes/pipelines.js +118 -0
  165. package/routes/runs.js +1129 -0
  166. package/scripts/ci/import-check.mjs +50 -0
  167. package/scripts/link-core-self.mjs +31 -0
  168. package/scripts/with-guard.sh +24 -0
  169. package/server/app.js +503 -0
  170. package/server/registry.js +53 -0
  171. package/server/serve.js +163 -0
  172. package/state/admission-queue-store.js +325 -0
  173. package/state/attribution-registries.js +102 -0
  174. package/state/batch-writer.js +162 -0
  175. package/state/db-guard.js +63 -0
  176. package/state/db-health.js +27 -0
  177. package/state/db.js +122 -0
  178. package/state/discuss-store.js +299 -0
  179. package/state/failure-store.js +427 -0
  180. package/state/migrate.js +104 -0
  181. package/state/node-cost.js +67 -0
  182. package/state/node-executions.js +775 -0
  183. package/state/node-health-store.js +124 -0
  184. package/state/ops-merge.js +93 -0
  185. package/state/paths.js +146 -0
  186. package/state/pipeline-dirs.js +99 -0
  187. package/state/pipeline-registry.js +449 -0
  188. package/state/run-store.js +697 -0
  189. package/state/schema.js +438 -0
  190. package/state/schema.ops.js +44 -0
  191. package/state/session-index.js +177 -0
  192. package/state/session-keys.js +39 -0
  193. package/state/state-store.js +311 -0
  194. package/state/subflow-state.js +126 -0
  195. package/state/tool-handoff.js +169 -0
  196. package/telemetry/budget-alert.js +353 -0
  197. package/telemetry/cli.js +331 -0
  198. package/telemetry/grammar.js +119 -0
  199. package/telemetry/http-budget.js +132 -0
  200. package/telemetry/http-cost.js +670 -0
  201. package/telemetry/http-helpers.js +134 -0
  202. package/telemetry/http-tree.js +349 -0
  203. package/telemetry/http.js +44 -0
  204. package/telemetry/index.js +307 -0
  205. package/telemetry/ingest.js +134 -0
  206. package/telemetry/labels.js +115 -0
  207. package/telemetry/pricer.js +77 -0
  208. package/telemetry/resolver.js +314 -0
  209. package/telemetry/store.js +147 -0
  210. package/utils/run-patch.js +43 -0
@@ -0,0 +1,212 @@
1
+ import { mergeForConflict, shortSummary } from "./executor-helpers.js";
2
+ import { REVISION_RETRY_LIMIT } from "./executor-errors.js";
3
+ import { ENTRY_FIELDS } from "./needs-resolver.js";
4
+
5
+ export { REVISION_RETRY_LIMIT };
6
+
7
+ // The write cutover: state_json stops accumulating a key per dispatched
8
+ // node — a node's own output lives exclusively in
9
+ // flow_node_invocations.result_json and is reconstructed on read via
10
+ // assembleState() (already wired into every dispatch-loop read site). What's
11
+ // persisted here narrows to entry/bookkeeping data only: pipelineId,
12
+ // everything in needs-resolver.js's ENTRY_FIELDS (the same allowlist
13
+ // ctx.needs({entry}) already validates against — this is its exact intended
14
+ // meaning), and the 3 fields buildChildInitialState
15
+ // (executor-subflow-children.js) sets on a freshly spawned child. `gates` is
16
+ // deliberately NOT in this list — it's
17
+ // sourced exclusively from gate_records (executor-nodes.js's human-gate
18
+ // branch stopped writing state.gates in the same change that added this
19
+ // filter) and reading it back through state_json would reopen the staleness
20
+ // bug gate_records exists to prevent.
21
+ //
22
+ // Applied once, centrally, inside these 6 functions rather than at each of
23
+ // executor.js's ~20 call sites — every persist path funnels through here,
24
+ // so this is the one place that needs to change for the cutover, and it's
25
+ // a single choke point to audit instead of twenty scattered edits.
26
+ //
27
+ // A global, unconditional filter, not a per-flow gate: the alternative
28
+ // ("check whether THIS flow's current state_json has 100% resultJson
29
+ // coverage before dropping its blob") was considered and rejected once the
30
+ // backfill closed every genuine gap across every non-test, non-terminal
31
+ // flow (verified via a full-history audit, not sampling) — carrying that
32
+ // runtime check forever would guard against a scenario the backfill + the
33
+ // live write path already rule out structurally, not just today.
34
+ const PERSISTED_FIELDS = new Set([...ENTRY_FIELDS, "pipelineId", "item", "itemIndex", "parentItem"]);
35
+
36
+ function toPersistableState(state) {
37
+ if (!state || typeof state !== "object") return state;
38
+ const out = {};
39
+ for (const key of PERSISTED_FIELDS) {
40
+ if (key in state) out[key] = state[key];
41
+ }
42
+ return out;
43
+ }
44
+
45
+ export function createPersist({ store, failureStore, notifyFlowEnteredWaiting, logger }) {
46
+ async function persistRunningRetrying({ flowId, revision, lastStep, state }) {
47
+ let rev = revision;
48
+ let curState = state;
49
+ for (let i = 0; i < REVISION_RETRY_LIMIT; i++) {
50
+ const result = await store.setRunning({ flowId, expectedRevision: rev, lastStep, stateJson: toPersistableState(curState) });
51
+ if (result.applied) return result;
52
+ if (result.code !== "revision_conflict") {
53
+ logger.warn(`manifold [${flowId.slice(0, 8)}] setRunning failed: ${result.code}`);
54
+ return result;
55
+ }
56
+ const fresh = await store.get(flowId);
57
+ if (!fresh) return result;
58
+ rev = fresh.revision;
59
+ curState = mergeForConflict(fresh.stateJson, state);
60
+ }
61
+ return { applied: false, code: "retry_exhausted" };
62
+ }
63
+
64
+ async function persistWaitingRetrying(args) {
65
+ let rev = args.revision;
66
+ let curState = args.state;
67
+ for (let i = 0; i < REVISION_RETRY_LIMIT; i++) {
68
+ const result = await store.setWaiting({
69
+ flowId: args.flowId,
70
+ expectedRevision: rev,
71
+ lastStep: args.lastStep,
72
+ blockingNodeId: args.blockingNodeId ?? null,
73
+ stateJson: toPersistableState(curState),
74
+ waitJson: args.waitJson,
75
+ statusReason: args.statusReason,
76
+ });
77
+ if (result.applied) {
78
+ // Push-from-node: tell whoever's listening (typically the
79
+ // telegram-block-notifier) that this flow just transitioned to waiting.
80
+ if (typeof notifyFlowEnteredWaiting === "function") {
81
+ try {
82
+ const fresh = await store.get(args.flowId);
83
+ if (fresh) {
84
+ const ret = notifyFlowEnteredWaiting(fresh);
85
+ if (ret && typeof ret.then === "function") {
86
+ ret.catch((err) => logger?.warn?.(`manifold [${args.flowId.slice(0, 8)}] notifyFlowEnteredWaiting rejected: ${err?.message ?? err}`));
87
+ }
88
+ }
89
+ } catch (err) {
90
+ logger?.warn?.(`manifold [${args.flowId.slice(0, 8)}] notifyFlowEnteredWaiting threw: ${err?.message ?? err}`);
91
+ }
92
+ }
93
+ return result;
94
+ }
95
+ if (result.code !== "revision_conflict") return result;
96
+ const fresh = await store.get(args.flowId);
97
+ if (!fresh) return result;
98
+ rev = fresh.revision;
99
+ curState = mergeForConflict(fresh.stateJson, args.state);
100
+ }
101
+ return { applied: false, code: "retry_exhausted" };
102
+ }
103
+
104
+ // Same shape as persistWaitingRetrying, but for status='queued' (waiting
105
+ // on an internal resource — a shared admission-pool slot, cross-process
106
+ // lock — rather than an operator decision). No notifyFlowEnteredWaiting push:
107
+ // that hook exists specifically for external-channel delivery (e.g. the
108
+ // telegram-block-notifier) on human-gate waits, which queued transitions
109
+ // are not.
110
+ async function persistQueuedRetrying(args) {
111
+ let rev = args.revision;
112
+ let curState = args.state;
113
+ for (let i = 0; i < REVISION_RETRY_LIMIT; i++) {
114
+ const result = await store.setQueued({
115
+ flowId: args.flowId,
116
+ expectedRevision: rev,
117
+ lastStep: args.lastStep,
118
+ blockingNodeId: args.blockingNodeId ?? null,
119
+ stateJson: toPersistableState(curState),
120
+ waitJson: args.waitJson,
121
+ statusReason: args.statusReason,
122
+ });
123
+ if (result.applied) return result;
124
+ if (result.code !== "revision_conflict") return result;
125
+ const fresh = await store.get(args.flowId);
126
+ if (!fresh) return result;
127
+ rev = fresh.revision;
128
+ curState = mergeForConflict(fresh.stateJson, args.state);
129
+ }
130
+ return { applied: false, code: "retry_exhausted" };
131
+ }
132
+
133
+ async function persistSkipRetrying({ flowId, revision, state, reason }) {
134
+ let rev = revision;
135
+ let curState = state;
136
+ for (let i = 0; i < REVISION_RETRY_LIMIT; i++) {
137
+ const result = await store.skip({ flowId, expectedRevision: rev, stateJson: toPersistableState(curState), reason });
138
+ if (result.applied) return result;
139
+ if (result.code === "terminal") return result;
140
+ if (result.code !== "revision_conflict") return result;
141
+ const fresh = await store.get(flowId);
142
+ if (!fresh) return result;
143
+ rev = fresh.revision;
144
+ curState = mergeForConflict(fresh.stateJson, state);
145
+ }
146
+ return { applied: false, code: "retry_exhausted" };
147
+ }
148
+
149
+ async function persistFinishRetrying({ flowId, revision, state }) {
150
+ let rev = revision;
151
+ let curState = state;
152
+ for (let i = 0; i < REVISION_RETRY_LIMIT; i++) {
153
+ const result = await store.finish({ flowId, expectedRevision: rev, stateJson: toPersistableState(curState) });
154
+ if (result.applied) return result;
155
+ if (result.code !== "revision_conflict") return result;
156
+ const fresh = await store.get(flowId);
157
+ if (!fresh) return result;
158
+ rev = fresh.revision;
159
+ curState = mergeForConflict(fresh.stateJson, state);
160
+ }
161
+ return { applied: false, code: "retry_exhausted" };
162
+ }
163
+
164
+ async function persistFailRetrying({ flowId, revision, state, message, failureNode, invocationId = null, executionId = null, errorType = null }) {
165
+ let rev = revision;
166
+ let curState = state;
167
+ for (let i = 0; i < REVISION_RETRY_LIMIT; i++) {
168
+ const result = await store.fail({
169
+ flowId,
170
+ expectedRevision: rev,
171
+ stateJson: toPersistableState(curState),
172
+ statusReason: shortSummary(message),
173
+ });
174
+ if (result.applied) {
175
+ try {
176
+ await failureStore?.recordEngineFailure?.({
177
+ flowId,
178
+ pipelineId: curState?.pipelineId,
179
+ failureNode,
180
+ message,
181
+ // Deliberately unfiltered — recordEngineFailure's own `state`
182
+ // param is currently unused in its body (see state/failure-store.js),
183
+ // separate from flow_runs.state_json entirely, so there's nothing
184
+ // here that needs entry-only narrowing.
185
+ state: curState,
186
+ invocationId,
187
+ executionId,
188
+ errorType,
189
+ });
190
+ } catch (err) {
191
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] recordEngineFailure threw: ${err?.message ?? err}`);
192
+ }
193
+ return result;
194
+ }
195
+ if (result.code !== "revision_conflict") return result;
196
+ const fresh = await store.get(flowId);
197
+ if (!fresh) return result;
198
+ rev = fresh.revision;
199
+ curState = mergeForConflict(fresh.stateJson, state);
200
+ }
201
+ return { applied: false, code: "retry_exhausted" };
202
+ }
203
+
204
+ return {
205
+ persistRunningRetrying,
206
+ persistWaitingRetrying,
207
+ persistQueuedRetrying,
208
+ persistSkipRetrying,
209
+ persistFinishRetrying,
210
+ persistFailRetrying,
211
+ };
212
+ }
@@ -0,0 +1,275 @@
1
+ // executor-subflow-children.js — real child-flow dispatch for a
2
+ // `kind:"subflow"` node.
3
+ //
4
+ // Deliberately small: unlike the old branch-worker model (subCtx/item
5
+ // calling convention, auto-nested `data`, its own worker-pool concurrency
6
+ // scheduler, gate-batching), a childFlows
7
+ // subflow's items are ordinary flow_runs children driven by the exact same
8
+ // executor.js/runFlow() every other flow uses — this module's only job is
9
+ // resolving items → spawning/finding each child → checking whether the
10
+ // whole fan-out is done. No data-shape adapter, no calling-convention
11
+ // translation: a childFlows subflow's `.flow.nodes`/`.flow.edges` are
12
+ // validated (engine/loader.js) as plain root-level nodes, so by the time
13
+ // this runs they already speak the same convention as the rest of the
14
+ // engine.
15
+ //
16
+ // node.concurrency: a boolean PARTICIPATION switch, not a per-node number.
17
+ // true (default, including when unset) means this
18
+ // subflow's children draw from the single shared combined admission pool
19
+ // (state/admission-queue-store.js, kind:"child-dispatch" — the same pool
20
+ // flow-start draws from); false opts a node out entirely (unbounded —
21
+ // including any agent nodes its children run, since those execute inside
22
+ // the child's own run with no admission of their own), for a fan-out cheap
23
+ // enough that throttling it would only add latency for no protective
24
+ // benefit. The shared pool is deliberately not
25
+ // scoped to any one pipeline, node, or kind, since the resource actually
26
+ // being protected (host CPU/memory, a shared account session, agent worker
27
+ // processes) is shared by literally everything.
28
+
29
+ import { resolveOrCreateChild } from "./spawn-child.js";
30
+ import { areSiblingsAllTerminal } from "./child-resume-trigger.js";
31
+ import { makeNeedsCtx } from "./needs-resolver.js";
32
+
33
+ // Above the shared admission cap (engine/global-concurrency.js's default of
34
+ // 8) since this batch size only bounds cheap
35
+ // row-bookkeeping (resolveOrCreateChild, store.get/setQueued), not the
36
+ // actual resource-consuming work that cap protects; low enough to avoid a
37
+ // thundering-herd write burst against the one shared WAL file.
38
+ const DISPATCH_BATCH_SIZE = 10;
39
+
40
+ // Bounded-concurrency map, error-isolated per item — Promise.allSettled per
41
+ // batch, not Promise.all, so one item's rejection can never affect its
42
+ // batch-mates or later batches (matches this file's own former per-item
43
+ // try/catch granularity). No general-purpose equivalent exists elsewhere in
44
+ // this codebase; small enough to keep local to this one caller rather than
45
+ // add a dependency or a new shared module.
46
+ async function runInBatches(items, batchSize, fn) {
47
+ const results = new Array(items.length);
48
+ for (let start = 0; start < items.length; start += batchSize) {
49
+ const batch = items.slice(start, start + batchSize);
50
+ const settled = await Promise.allSettled(batch.map((item, i) => fn(item, start + i)));
51
+ for (let i = 0; i < settled.length; i++) results[start + i] = settled[i];
52
+ }
53
+ return results;
54
+ }
55
+
56
+ // subflowKey is node.itemKey when the pipeline declares one (e.g. CONNECT),
57
+ // else the item's own id/listingId.
58
+ function deriveSubflowKey(node, item) {
59
+ if (node.itemKey) return item?.[node.itemKey];
60
+ return item?.id ?? item?.listingId ?? String(item);
61
+ }
62
+
63
+ // A child's initialState is deliberately NOT a full clone of the parent's
64
+ // state. Any node inside the subflow that genuinely needs a value from an
65
+ // ancestor flow declares it explicitly via `needs:`
66
+ // (engine/needs-resolver.js), resolved lazily at read time against that
67
+ // ancestor's own resultJson/entry fields — not inherited wholesale at
68
+ // spawn time.
69
+ function buildChildInitialState(state, item, itemIndex) {
70
+ // Captured before spawn — so a single-item retry-cascade (selector: () =>
71
+ // [{id:"only"}], structuring a code -> infra-recovery -> agent-fallback
72
+ // chain, not a real per-item fan-out) can still reach the real
73
+ // parent-branch item its inner nodes need — without this, `item` below
74
+ // silently replaces the real listing/candidate/conversation object with
75
+ // the cascade's own placeholder, and any inner node reading
76
+ // `state.item.id`/`.jobId` for the real target id gets the literal string
77
+ // "only" instead. A real fan-out's own item is unaffected either way —
78
+ // parentItem is a pure addition.
79
+ const parentItem = state?.item;
80
+ return { item, itemIndex, parentItem };
81
+ }
82
+
83
+ export async function dispatchSubflowChildren({ node, nodeId, flowId, state, store, runStore, nodeExecutionsStore, maybeBudgetGate, runFlow, logger, api, lineage, pipelinesRef, admissionQueueStore }) {
84
+ const _lineage = lineage ?? api?.runtime?.lineage ?? null;
85
+ // "needs:" resolution — a subflow node's own selector can declare .needs and call
86
+ // ctx.needs(varName) to reach an ancestor flow, same lazy/opt-in
87
+ // mechanism code and agent nodes get. A rejected/not-ready need throws
88
+ // NodeBlocked here, propagating straight through (no local try/catch in
89
+ // this function) to executor.js's own NodeBlocked handling, unchanged.
90
+ const items = await Promise.resolve(node.selector(state, { needs: makeNeedsCtx({ node, flowId, runStore, nodeExecutionsStore, pipelinesRef }) }));
91
+ if (!Array.isArray(items) || items.length === 0) {
92
+ logger?.info?.(`manifold [${flowId.slice(0, 8)}] ${nodeId}: 0 items, skipping (childFlows)`);
93
+ return { stateUpdate: {} };
94
+ }
95
+
96
+ // Skip the subflow-entry gate for a cascade-shape node (flow.kind:
97
+ // "cascade" — the code-then-agent-fallback migration, e.g.
98
+ // ENSURE_LOGIN/SUBMIT/SEND_REPLY). A cascade's own code leg frequently
99
+ // resolves the item for free (edge routes straight to "__end__" without ever touching the
100
+ // agent leg — e.g. a cached login), so gating here, before that code leg
101
+ // even runs, produces a false-positive hold on the common case. Real
102
+ // spend stays fully covered: the agent leg (e.g. ENSURE_LOGIN_AGENT), if
103
+ // and when the cascade actually reaches it, is independently gated by
104
+ // dispatchAgentEntry's own maybeBudgetGate call. A real multi-item
105
+ // childFlows fan-out (e.g. SCORE, buildScoreLoopNode) has no flow.kind at
106
+ // all, so it's unaffected by this guard and keeps its existing
107
+ // subflow-entry gate.
108
+ if (node.flow?.kind !== "cascade") {
109
+ const gate = await maybeBudgetGate({
110
+ flowId, nodeId, node,
111
+ pipelineId: state?.pipelineId ?? null,
112
+ pipelineName: state?.pipelineId ?? null,
113
+ itemCount: items.length,
114
+ items,
115
+ });
116
+ if (gate.skip) return { stateUpdate: {} };
117
+ if (gate.pause) return { blockedInfo: gate.blockedInfo };
118
+ }
119
+
120
+ const childPipelineId = `${state?.pipelineId ?? ""}::${nodeId}`;
121
+
122
+ // Pre-filter valid items up front (same empty-subflowKey skip as before)
123
+ // so the batched phases below work off a clean list.
124
+ const validItems = [];
125
+ for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
126
+ const item = items[itemIndex];
127
+ const subflowKey = deriveSubflowKey(node, item);
128
+ if (subflowKey == null || subflowKey === "") {
129
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] ${nodeId}: item ${itemIndex} produced an empty subflowKey, skipping`);
130
+ continue;
131
+ }
132
+ validItems.push({ item, itemIndex, subflowKey: String(subflowKey) });
133
+ }
134
+
135
+ // Race-safe row resolution, batched with bounded concurrency.
136
+ // resolveOrCreateChild's race-safety (spawn-child.js) is per-identity
137
+ // (flowId, nodeId, subflowKey), backed by a unique index — it was never
138
+ // dependent on THIS loop processing items one at a time; concurrent
139
+ // calls for DIFFERENT identities have nothing to race over.
140
+ //
141
+ // node.dispatchGuard: optional async (item, subflowKey, { flowId }) => boolean,
142
+ // declared by a pipeline that needs cross-flow dedup beyond what
143
+ // node.selector's own upstream filtering can guarantee — the selector's
144
+ // list is decided once per parent-flow visit, so two parent flows racing
145
+ // through the same selector logic can both decide to dispatch the same
146
+ // subflowKey before either commits. Threaded through to
147
+ // findOrCreateInvocation (state/node-executions.js), which evaluates it
148
+ // inside the same write-queue turn as the actual insert, so the second
149
+ // caller's guard always sees the first caller's already-committed row.
150
+ const resolved = await runInBatches(validItems, DISPATCH_BATCH_SIZE, (v) =>
151
+ resolveOrCreateChild({
152
+ flowId,
153
+ pipelineId: state?.pipelineId ?? null,
154
+ nodeId,
155
+ subflowKey: v.subflowKey,
156
+ childPipelineId,
157
+ initialState: buildChildInitialState(state, v.item, v.itemIndex),
158
+ store,
159
+ runStore,
160
+ nodeExecutionsStore,
161
+ guard: node.dispatchGuard ? () => node.dispatchGuard(v.item, v.subflowKey, { flowId }) : undefined,
162
+ }).then((r) => ({ ...v, ...r }))
163
+ );
164
+
165
+ const childFlowIds = [];
166
+ const created = [];
167
+ for (let i = 0; i < resolved.length; i++) {
168
+ const settled = resolved[i];
169
+ const v = validItems[i];
170
+ if (settled.status === "rejected") {
171
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] ${nodeId}: failed to spawn child for item ${v.itemIndex} (subflowKey=${v.subflowKey}): ${settled.reason?.message ?? settled.reason}`);
172
+ continue;
173
+ }
174
+ const { childFlowId, created: isNew, guarded } = settled.value;
175
+ if (guarded) {
176
+ logger?.info?.(`manifold [${flowId.slice(0, 8)}] ${nodeId}: item ${v.itemIndex} (subflowKey=${v.subflowKey}) rejected by dispatchGuard, skipping`);
177
+ continue;
178
+ }
179
+ childFlowIds.push(childFlowId);
180
+ // Eager creation dispatches once; an already-existing child was already
181
+ // dispatched on a prior visit to this subflow node (or is legitimately
182
+ // mid-flight/blocked on its own gate) — re-triggering it here would be
183
+ // redundant, not incorrect (runFlow is documented idempotent/
184
+ // re-entrant), so it's simply skipped.
185
+ if (isNew) created.push({ ...v, childFlowId });
186
+ }
187
+
188
+ // Real atomic admission per new child, via the shared combined
189
+ // pool (state/admission-queue-store.js). Each admit() call is itself one
190
+ // atomic DB transaction — count-then-claim happens inside a single
191
+ // BEGIN IMMEDIATE, so concurrent callers (another subflow node, another
192
+ // pipeline, another process) admit or queue against the TRUE live count
193
+ // every time.
194
+ //
195
+ // node.concurrency (see this file's header): true (default)
196
+ // participates in the shared pool; false opts out entirely (unadmitted,
197
+ // always dispatched).
198
+ //
199
+ // Only "running" counts as occupying a slot — a child "blocked" on its
200
+ // own human gate consumes nothing while it waits, so it must NOT occupy
201
+ // one; state-store.js's setWaiting releases this admission the moment
202
+ // that happens (see triggerAdmissionRelease there), or a gated child
203
+ // could starve the pool for hours. The exact same rule applies to a ROOT
204
+ // flow's own admission too (kind:"flow-start") — the shared admission
205
+ // pool treats every kind identically.
206
+ const toQueue = [];
207
+ const toDispatch = [];
208
+ for (const c of created) {
209
+ // Register the child's own trigger edge (flow:<childFlowId> ->
210
+ // flow:<flowId>) BEFORE scheduling its run, so the resolver can walk
211
+ // straight through to the parent's already-resolved root (e.g. the cron
212
+ // that triggered the top-level fan-out) instead of dead-ending —
213
+ // without this, cost attributed to the child's nodes collapses into an
214
+ // unlinked "unknown" bucket. recordTrigger is async (it resolves the
215
+ // root, then upserts session_lineage), so it is awaited: the runFlow
216
+ // scheduling further down is setImmediate, which fires on the next tick
217
+ // and would otherwise let the child's first node resolve lineage before
218
+ // this edge lands.
219
+ await _lineage?.recordTrigger?.({
220
+ flowId: c.childFlowId,
221
+ pipelineId: childPipelineId,
222
+ parentSessionKey: `flow:${flowId}`,
223
+ });
224
+
225
+ if (node.concurrency === false) {
226
+ toDispatch.push(c);
227
+ continue;
228
+ }
229
+ const admission = await admissionQueueStore.admit({ kind: "child-dispatch", flowId: c.childFlowId, nodeId });
230
+ (admission.status === "queued" ? toQueue : toDispatch).push(c);
231
+ }
232
+
233
+ // Dispatch: setImmediate is already non-blocking fire-and-forget — no
234
+ // batching needed, this was never part of the serialization problem.
235
+ for (const c of toDispatch) {
236
+ setImmediate(() => {
237
+ runFlow(c.childFlowId, "fresh").catch((err) => {
238
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] ${nodeId}: child ${c.childFlowId.slice(0, 8)} runFlow failed: ${err?.message ?? err}`);
239
+ });
240
+ });
241
+ }
242
+
243
+ // Queue: store.get() + store.setQueued() are genuinely awaited DB calls —
244
+ // batch these too, same reason as the resolution step above.
245
+ await runInBatches(toQueue, DISPATCH_BATCH_SIZE, async (c) => {
246
+ const row = await store.get(c.childFlowId);
247
+ if (row) {
248
+ await store.setQueued({
249
+ flowId: c.childFlowId,
250
+ expectedRevision: row.revision,
251
+ lastStep: row.lastStep,
252
+ stateJson: row.stateJson,
253
+ waitJson: { kind: "child-dispatch" },
254
+ statusReason: `queued: waiting for a shared admission slot (child of node ${nodeId})`,
255
+ });
256
+ }
257
+ }).then((settled) => {
258
+ for (let i = 0; i < settled.length; i++) {
259
+ if (settled[i].status === "rejected") {
260
+ const c = toQueue[i];
261
+ logger?.warn?.(`manifold [${flowId.slice(0, 8)}] ${nodeId}: failed to queue child ${c.childFlowId.slice(0, 8)} (item ${c.itemIndex}): ${settled[i].reason?.message ?? settled[i].reason}`);
262
+ }
263
+ }
264
+ });
265
+
266
+ const { allTerminal } = await areSiblingsAllTerminal({ flowId, nodeId, runStore, nodeExecutionsStore });
267
+ if (allTerminal) {
268
+ return { stateUpdate: {} };
269
+ }
270
+ return {
271
+ stateUpdate: {},
272
+ waitJson: { kind: "children-pending", nodeId, childFlowIds },
273
+ reentrant: true,
274
+ };
275
+ }