@igorjpimenta/manifold 0.0.0-bootstrap.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/.migrations/ops/0000_futuristic_karma.sql +111 -0
  2. package/.migrations/ops/0001_low_maximus.sql +7 -0
  3. package/.migrations/ops/meta/0000_snapshot.json +763 -0
  4. package/.migrations/ops/meta/0001_snapshot.json +173 -0
  5. package/.migrations/ops/meta/_journal.json +20 -0
  6. package/.migrations/runtime/0000_real_boomer.sql +180 -0
  7. package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
  8. package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
  9. package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
  10. package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
  11. package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
  12. package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
  13. package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
  14. package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
  15. package/.migrations/runtime/0009_lying_shocker.sql +96 -0
  16. package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
  17. package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
  18. package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
  19. package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
  20. package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
  21. package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
  22. package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
  23. package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
  24. package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
  25. package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
  26. package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
  27. package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
  28. package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
  29. package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
  30. package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
  31. package/.migrations/runtime/meta/_journal.json +97 -0
  32. package/README.md +1584 -0
  33. package/RELEASING.md +89 -0
  34. package/agent/fleet.js +100 -0
  35. package/bin/cli.js +854 -0
  36. package/bin/manifold.js +4 -0
  37. package/boot/flow-revival.js +536 -0
  38. package/budget/index.js +495 -0
  39. package/budget/lib.js +59 -0
  40. package/dashboard/cost.js +528 -0
  41. package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
  42. package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
  43. package/dashboard/dist/index.html +13 -0
  44. package/dashboard/event-log.js +267 -0
  45. package/dashboard/index.html +12 -0
  46. package/dashboard/package-lock.json +2877 -0
  47. package/dashboard/package.json +37 -0
  48. package/dashboard/pipeline-graph.js +131 -0
  49. package/dashboard/postcss.config.js +6 -0
  50. package/dashboard/search.js +183 -0
  51. package/dashboard/server.js +1015 -0
  52. package/dashboard/src/App.tsx +42 -0
  53. package/dashboard/src/api/client.ts +118 -0
  54. package/dashboard/src/api/types.ts +510 -0
  55. package/dashboard/src/api/ws.tsx +112 -0
  56. package/dashboard/src/components/CancelButton.tsx +63 -0
  57. package/dashboard/src/components/ClickToCopy.tsx +68 -0
  58. package/dashboard/src/components/CopyButton.tsx +61 -0
  59. package/dashboard/src/components/DailyCostTrend.tsx +85 -0
  60. package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
  61. package/dashboard/src/components/Funnel.tsx +149 -0
  62. package/dashboard/src/components/GateForm.tsx +158 -0
  63. package/dashboard/src/components/GraphView.tsx +302 -0
  64. package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
  65. package/dashboard/src/components/Histogram.tsx +292 -0
  66. package/dashboard/src/components/Layout.tsx +28 -0
  67. package/dashboard/src/components/NeedsAttention.tsx +98 -0
  68. package/dashboard/src/components/NodeContextCard.tsx +132 -0
  69. package/dashboard/src/components/PathTrail.tsx +35 -0
  70. package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
  71. package/dashboard/src/components/RerunButton.tsx +46 -0
  72. package/dashboard/src/components/RunComparison.tsx +53 -0
  73. package/dashboard/src/components/Sparkline.tsx +94 -0
  74. package/dashboard/src/components/StatStrip.tsx +147 -0
  75. package/dashboard/src/components/StateTree.tsx +84 -0
  76. package/dashboard/src/components/StatusBadge.tsx +18 -0
  77. package/dashboard/src/components/SubflowPathPills.tsx +38 -0
  78. package/dashboard/src/components/SubflowSummary.tsx +24 -0
  79. package/dashboard/src/components/SubflowsCard.tsx +327 -0
  80. package/dashboard/src/components/TodayStrip.tsx +86 -0
  81. package/dashboard/src/components/TranscriptPane.tsx +99 -0
  82. package/dashboard/src/components/TranscriptRow.tsx +69 -0
  83. package/dashboard/src/components/ui/badge.tsx +38 -0
  84. package/dashboard/src/components/ui/button.tsx +28 -0
  85. package/dashboard/src/components/ui/card.tsx +42 -0
  86. package/dashboard/src/components/ui/skeleton.tsx +9 -0
  87. package/dashboard/src/components/ui/table.tsx +62 -0
  88. package/dashboard/src/index.css +26 -0
  89. package/dashboard/src/lib/anomaly.ts +42 -0
  90. package/dashboard/src/lib/format.ts +30 -0
  91. package/dashboard/src/lib/step-history.ts +13 -0
  92. package/dashboard/src/lib/subflow.ts +154 -0
  93. package/dashboard/src/lib/use-click-outside.ts +28 -0
  94. package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
  95. package/dashboard/src/lib/utils.ts +6 -0
  96. package/dashboard/src/main.tsx +10 -0
  97. package/dashboard/src/types/shims.d.ts +4 -0
  98. package/dashboard/src/views/Home.tsx +478 -0
  99. package/dashboard/src/views/PipelineDetail.tsx +1185 -0
  100. package/dashboard/src/views/PipelinesList.tsx +111 -0
  101. package/dashboard/src/views/RunDetail.tsx +248 -0
  102. package/dashboard/src/views/Search.tsx +196 -0
  103. package/dashboard/stats.js +668 -0
  104. package/dashboard/tailwind.config.ts +43 -0
  105. package/dashboard/tree-stats.js +12 -0
  106. package/dashboard/tsconfig.json +25 -0
  107. package/dashboard/views.js +231 -0
  108. package/dashboard/vite.config.ts +34 -0
  109. package/drizzle.config.js +12 -0
  110. package/drizzle.config.ops.js +12 -0
  111. package/elevation.js +116 -0
  112. package/engine/admission-promotion-resume.js +127 -0
  113. package/engine/agent-dispatch.js +64 -0
  114. package/engine/child-resume-trigger.js +315 -0
  115. package/engine/executor-budget.js +87 -0
  116. package/engine/executor-cancel.js +83 -0
  117. package/engine/executor-errors.js +122 -0
  118. package/engine/executor-helpers.js +254 -0
  119. package/engine/executor-nodes.js +292 -0
  120. package/engine/executor-persist.js +212 -0
  121. package/engine/executor-subflow-children.js +275 -0
  122. package/engine/executor.js +1013 -0
  123. package/engine/global-concurrency.js +33 -0
  124. package/engine/loader.js +528 -0
  125. package/engine/needs-resolver.js +229 -0
  126. package/engine/pipelines-watcher.js +133 -0
  127. package/engine/result-replay.js +34 -0
  128. package/engine/revival-messages.js +11 -0
  129. package/engine/spawn-child.js +55 -0
  130. package/engine/wait-for-flow-settlement.js +109 -0
  131. package/events/catalog.js +26 -0
  132. package/events/emit.js +44 -0
  133. package/events/kinds.js +118 -0
  134. package/events/serve-ws.js +74 -0
  135. package/events/stream.js +35 -0
  136. package/events/ws-dialog.js +257 -0
  137. package/events/ws-frame.js +194 -0
  138. package/events/ws-run-events.js +70 -0
  139. package/events.js +32 -0
  140. package/gates/gate-resolver.js +576 -0
  141. package/gates/paused-flows.js +55 -0
  142. package/graphql/columns.js +67 -0
  143. package/graphql/schema.js +333 -0
  144. package/index.js +49 -0
  145. package/mcp/http.js +159 -0
  146. package/mcp/identity.js +94 -0
  147. package/mcp/stdio.js +80 -0
  148. package/mcp/tools.js +245 -0
  149. package/mcp/wire-ops.js +178 -0
  150. package/package.json +51 -0
  151. package/routes/bearer-auth.js +57 -0
  152. package/routes/discuss-audio.js +87 -0
  153. package/routes/discuss-component.js +39 -0
  154. package/routes/discuss-draft.js +159 -0
  155. package/routes/discuss-helpers.js +20 -0
  156. package/routes/discuss.js +848 -0
  157. package/routes/failures.js +104 -0
  158. package/routes/graphql.js +61 -0
  159. package/routes/health.js +54 -0
  160. package/routes/operator-discuss.js +193 -0
  161. package/routes/operator-resume.js +32 -0
  162. package/routes/operator-retry.js +301 -0
  163. package/routes/operator.js +539 -0
  164. package/routes/pipelines.js +118 -0
  165. package/routes/runs.js +1129 -0
  166. package/scripts/ci/import-check.mjs +50 -0
  167. package/scripts/link-core-self.mjs +31 -0
  168. package/scripts/with-guard.sh +24 -0
  169. package/server/app.js +503 -0
  170. package/server/registry.js +53 -0
  171. package/server/serve.js +163 -0
  172. package/state/admission-queue-store.js +325 -0
  173. package/state/attribution-registries.js +102 -0
  174. package/state/batch-writer.js +162 -0
  175. package/state/db-guard.js +63 -0
  176. package/state/db-health.js +27 -0
  177. package/state/db.js +122 -0
  178. package/state/discuss-store.js +299 -0
  179. package/state/failure-store.js +427 -0
  180. package/state/migrate.js +104 -0
  181. package/state/node-cost.js +67 -0
  182. package/state/node-executions.js +775 -0
  183. package/state/node-health-store.js +124 -0
  184. package/state/ops-merge.js +93 -0
  185. package/state/paths.js +146 -0
  186. package/state/pipeline-dirs.js +99 -0
  187. package/state/pipeline-registry.js +449 -0
  188. package/state/run-store.js +697 -0
  189. package/state/schema.js +438 -0
  190. package/state/schema.ops.js +44 -0
  191. package/state/session-index.js +177 -0
  192. package/state/session-keys.js +39 -0
  193. package/state/state-store.js +311 -0
  194. package/state/subflow-state.js +126 -0
  195. package/state/tool-handoff.js +169 -0
  196. package/telemetry/budget-alert.js +353 -0
  197. package/telemetry/cli.js +331 -0
  198. package/telemetry/grammar.js +119 -0
  199. package/telemetry/http-budget.js +132 -0
  200. package/telemetry/http-cost.js +670 -0
  201. package/telemetry/http-helpers.js +134 -0
  202. package/telemetry/http-tree.js +349 -0
  203. package/telemetry/http.js +44 -0
  204. package/telemetry/index.js +307 -0
  205. package/telemetry/ingest.js +134 -0
  206. package/telemetry/labels.js +115 -0
  207. package/telemetry/pricer.js +77 -0
  208. package/telemetry/resolver.js +314 -0
  209. package/telemetry/store.js +147 -0
  210. package/utils/run-patch.js +43 -0
@@ -0,0 +1,229 @@
1
+ // needs-resolver.js — Option B, per-node "needs:": a node declares
2
+ // `needs: { varName: {hops, nodeId} }`, keyed by the local variable name
3
+ // it'll be read as (an ORM-style relation declaration, not an array) —
4
+ // and resolves each one lazily, only when `ctx.needs(varName)` is actually
5
+ // awaited inside that node's own run()/promptBuilder()/selector(), not
6
+ // fetched in bulk before dispatch. A branch of code that never calls
7
+ // ctx.needs("x") never pays for or blocks on that declared need.
8
+ //
9
+ // True per-property-access laziness (a bare `state._needs.x` read
10
+ // transparently blocking) was considered and rejected: this codebase's
11
+ // only SQLite driver (@libsql/client) is fully async, and the only way to
12
+ // make an async read look synchronous from inside a plain property getter
13
+ // is Atomics.wait blocking the ENTIRE process's event loop — which would
14
+ // stall every other concurrently-dispatching flow and every HTTP request
15
+ // on this gateway for the duration of the read. `ctx.needs(varName)` as an
16
+ // async function call gets the same lazy-only-if-touched property safely,
17
+ // using async/await machinery this engine already has (node.run() is
18
+ // already awaited today) instead of inventing dangerous new plumbing.
19
+ //
20
+ // Deliberately its own module, not folded into executor-helpers.js:
21
+ // resolveOneNeed needs areSiblingsAllTerminal (child-resume-trigger.js),
22
+ // which itself imports isTerminal from executor-helpers.js — importing it
23
+ // back the other way would be a cycle.
24
+ //
25
+ // hops:1 = immediate parent flow, hops:2 = grandparent, etc. Walked via
26
+ // flow_runs.parent_invocation_id ->
27
+ // flow_node_invocations.getInvocation().flowId -> repeat — bounded by the
28
+ // caller's own declared (small, explicit) hop count, so no new denormalized
29
+ // parent_invocation_id column on flow_node_invocations is needed even for
30
+ // hops > 1: a bounded loop of existing point lookups is exactly as cheap.
31
+ //
32
+ // hops:0 = same-flow — the declaring node's own flow, no walk at all
33
+ // (walkUpHops runs zero loop iterations and returns flowId unchanged).
34
+ // A node can opt into the same explicit, lazy, distinguishes-"not-ready"-
35
+ // from-"never" accessor for same-flow reads too, instead of two different
36
+ // idioms depending on whether the source is same-flow or cross-flow.
37
+ //
38
+ // A second declaration shape, `{hops, entry}` (mutually exclusive with
39
+ // `nodeId`/`config`): reaches an ancestor's own ENTRY/bookkeeping field —
40
+ // runId, args, pipelineId, cost, etc. — set once at that flow's own
41
+ // start() (or updated directly by gate-resolver.js), never via any node's
42
+ // own resultJson. This is a different read path entirely from
43
+ // `nodeId`-based needs: no resultJson replay, no "not ready yet" state (an
44
+ // entry field exists the instant its flow does), just a direct read of
45
+ // that ancestor flow's own persisted stateJson at the target hop. `entry`
46
+ // is validated at load time (engine/loader.js) against ENTRY_FIELDS below
47
+ // — `gates` is deliberately excluded even though it's a real stateJson
48
+ // key, since it has its own dedicated, staleness-safe accessor
49
+ // (gate_records) and reading it through here would reopen exactly the bug
50
+ // that mechanism was built to close.
51
+ //
52
+ // A third shape, `{hops, config}` (mutually exclusive with the other two):
53
+ // reaches a pipeline-DECLARED static config field (index.js's own `state:`
54
+ // block — e.g. sourceFilter, minClusterSize) at the target hop. Config is
55
+ // code, not data: it's never written to stateJson (toPersistableState
56
+ // never carries it — see executor-persist.js), never subject to that
57
+ // narrowing, and resolved live off the currently-loaded pipeline
58
+ // definition via pipelinesRef, same source start() itself reads from.
59
+ // Always "ready" the instant the target flow exists, same as `entry` — it
60
+ // doesn't depend on any node having run. Unlike `entry`, not validated
61
+ // against a fixed key set at load time (pipelines declare arbitrary
62
+ // `state:` shapes, no schema) — an unknown key just resolves `undefined`.
63
+ import { getSubflowGroup } from "../state/subflow-state.js";
64
+ import { areSiblingsAllTerminal } from "./child-resume-trigger.js";
65
+ import { NodeBlocked } from "./executor-errors.js";
66
+
67
+ export const ENTRY_FIELDS = new Set([
68
+ "runId", "startedAt", "args", "triggeredBy", "pipelineId",
69
+ "cost", "_display",
70
+ "approvedIds", "rejectedIds", "editedTexts", "approvalPending", "approvalResults",
71
+ "_cascadeError",
72
+ ]);
73
+
74
+ async function walkUpHops(flowId, hops, runStore, nodeExecutionsStore) {
75
+ let currentFlowId = flowId;
76
+ for (let i = 0; i < hops; i++) {
77
+ const flow = await runStore.get(currentFlowId);
78
+ const parentInvocationId = flow?.parentInvocationId ?? null;
79
+ if (!parentInvocationId) return null; // walked off the top — fewer real ancestors than declared
80
+ const inv = await nodeExecutionsStore.getInvocation(parentInvocationId);
81
+ if (!inv?.flowId) return null;
82
+ currentFlowId = inv.flowId;
83
+ }
84
+ return currentFlowId;
85
+ }
86
+
87
+ // Three-state result per declared need:
88
+ // "not-found" — walked off the top, OR the target flow has no
89
+ // invocation/subflow-fan-out for nodeId at all (never dispatched
90
+ // there) — a needs: entry naming the wrong hop count or a nodeId
91
+ // that hasn't run yet both land here; the caller treats both as
92
+ // "not ready, recheck later" the same way it treats an ordinary
93
+ // not-ready result, since a not-yet-dispatched ancestor node is not
94
+ // distinguishable from "give it more time" without knowing that
95
+ // ancestor's own future dispatch order, which this accessor has no
96
+ // way to see.
97
+ // "not-ready" — invocation row (or subflow fan-out) exists but isn't
98
+ // complete yet: resultWrittenAt still null for an ordinary node, or
99
+ // not every sibling is terminal yet for a subflow-sourced one.
100
+ // "ready" — value is the resolved data. A subflow-sourced need's value
101
+ // is the same { [subflowKey]: row } aggregate shape _subflowGroups
102
+ // already exposes elsewhere, not a different mechanism. An over-cap
103
+ // ordinary result carries the same { _resultTooLarge, bytes } marker
104
+ // assembleState() already uses.
105
+ export async function resolveOneNeed({ flowId, hops, nodeId, runStore, nodeExecutionsStore }) {
106
+ const targetFlowId = await walkUpHops(flowId, hops, runStore, nodeExecutionsStore);
107
+ if (!targetFlowId) return { status: "not-found" };
108
+
109
+ // Subflow-sourced check first: a real subflow node's own dispatch never
110
+ // writes an ordinary resultJson row for itself (dispatchSubflowChildren
111
+ // always returns stateUpdate:{}, per this doc's own corrected "SCORE is
112
+ // itself a subflow" resolution), so an ordinary getResultForNode lookup
113
+ // for a subflow node's id would always read as "not-found" even once its
114
+ // fan-out is real and complete. getSubflowGroup only ever returns real
115
+ // rows for a node that has actually spawned childFlows children, so
116
+ // checking it first (rather than guessing from a pipeline definition
117
+ // this accessor doesn't have on hand) is unambiguous.
118
+ const group = await getSubflowGroup({ flowId: targetFlowId, groupId: nodeId, runStore, nodeExecutionsStore });
119
+ if (group && Object.keys(group).length > 0) {
120
+ const { allTerminal, reason } = await areSiblingsAllTerminal({ flowId: targetFlowId, nodeId, runStore, nodeExecutionsStore });
121
+ if (reason === "no_siblings") return { status: "not-found" };
122
+ if (!allTerminal) return { status: "not-ready" };
123
+ return { status: "ready", value: group };
124
+ }
125
+
126
+ const row = await nodeExecutionsStore.getResultForNode(targetFlowId, nodeId);
127
+ if (!row) return { status: "not-found" };
128
+ if (row.resultWrittenAt == null) return { status: "not-ready" };
129
+ if (row.resultTooLargeBytes != null && row.resultJson == null) {
130
+ // writeResult() (state/node-executions.js) does not cap results, so this
131
+ // can only fire for a row that kept nothing at all — the real
132
+ // value isn't recoverable.
133
+ return { status: "ready", value: { _resultTooLarge: true, bytes: row.resultTooLargeBytes } };
134
+ }
135
+ try {
136
+ // A node's resultJson is its own raw stateUpdate — by convention almost
137
+ // always `{[nodeId]: value}` (assembleState's own doc comment notes
138
+ // this isn't a GUARANTEED shape, just the near-universal one), so
139
+ // unwrap it here: a needs: author writing needs:{x:{hops,nodeId:"SCAN"}}
140
+ // expects ctx.needs("x") to resolve to what state.SCAN means in SCAN's
141
+ // own flow, not the wrapper object around it. assembleState() itself
142
+ // doesn't unwrap because it's replaying MANY nodes' updates onto one
143
+ // shared object via spread, where the wrapper IS the update; this
144
+ // accessor fetches exactly one node's row, so unwrapping is correct
145
+ // here and would be wrong there.
146
+ const parsed = JSON.parse(row.resultJson ?? "null");
147
+ return { status: "ready", value: parsed && typeof parsed === "object" ? parsed[nodeId] : undefined };
148
+ } catch {
149
+ return { status: "not-found" }; // corrupted row — same treatment assembleState() gives a bad row
150
+ }
151
+ }
152
+
153
+ // Resolves a {hops, entry} need — always "ready" once the target ancestor
154
+ // flow itself exists (an entry field is set the instant its flow is
155
+ // created; there is no "dispatched but not complete" state the way a
156
+ // node's own resultJson has). "not-found" only when the walk itself fails
157
+ // (fewer real ancestors than declared, or the target flow vanished).
158
+ export async function resolveOneEntry({ flowId, hops, entry, runStore, nodeExecutionsStore }) {
159
+ const targetFlowId = await walkUpHops(flowId, hops, runStore, nodeExecutionsStore);
160
+ if (!targetFlowId) return { status: "not-found" };
161
+ const flow = await runStore.get(targetFlowId);
162
+ if (!flow) return { status: "not-found" };
163
+ return { status: "ready", value: flow.stateJson?.[entry] };
164
+ }
165
+
166
+ // Resolves a {hops, config} need — reads a pipeline-declared static config
167
+ // field off the target flow's own pipeline DEFINITION, not its stateJson.
168
+ // "not-found" when the walk fails, the target flow vanished, or its
169
+ // pipeline isn't currently loaded in pipelinesRef (mirrors runFlow()'s own
170
+ // "pipeline not loaded" handling in executor.js — a transient loader race,
171
+ // not a real absence). Otherwise always "ready", same as `entry` — an
172
+ // unknown config key simply resolves to `undefined`.
173
+ export async function resolveOneConfig({ flowId, hops, config, runStore, nodeExecutionsStore, pipelinesRef }) {
174
+ const targetFlowId = await walkUpHops(flowId, hops, runStore, nodeExecutionsStore);
175
+ if (!targetFlowId) return { status: "not-found" };
176
+ const flow = await runStore.get(targetFlowId);
177
+ if (!flow) return { status: "not-found" };
178
+ const pipeline = pipelinesRef?.value?.get(flow.stateJson?.pipelineId);
179
+ if (!pipeline) return { status: "not-found" };
180
+ return { status: "ready", value: pipeline.state?.[config] };
181
+ }
182
+
183
+ // Builds the `needs(varName)` async function passed to a declaring node's
184
+ // own run()/promptBuilder()/selector() as part of its ctx. Each call is an
185
+ // independent, fresh resolveOneNeed — no bulk fetch, no caching across
186
+ // calls within the same dispatch (consistent with this engine's "no cache
187
+ // layer, direct read" convention every other store accessor already
188
+ // follows). A not-ready/not-found result throws NodeBlocked with `_kind:
189
+ // "needs-not-ready"` explicitly stamped onto `_blocked` — mirroring the
190
+ // established pattern the budget-gate pause already uses
191
+ // (executor-nodes.js's dispatchAgentEntry) — a bare NodeBlocked with no
192
+ // `_kind` is treated as a hard FAILURE by executor.js's blockedInfo
193
+ // handling, not a wait, so this must never be omitted.
194
+ export function makeNeedsCtx({ node, flowId, runStore, nodeExecutionsStore, pipelinesRef }) {
195
+ return async function needs(varName) {
196
+ const spec = node?.needs?.[varName];
197
+ if (!spec) {
198
+ throw new Error(`ctx.needs("${varName}"): not declared in this node's .needs — check node.needs`);
199
+ }
200
+ const result = spec.entry !== undefined
201
+ ? await resolveOneEntry({ flowId, hops: spec.hops, entry: spec.entry, runStore, nodeExecutionsStore })
202
+ : spec.config !== undefined
203
+ ? await resolveOneConfig({ flowId, hops: spec.hops, config: spec.config, runStore, nodeExecutionsStore, pipelinesRef })
204
+ : await resolveOneNeed({ flowId, hops: spec.hops, nodeId: spec.nodeId, runStore, nodeExecutionsStore });
205
+ if (result.status !== "ready") {
206
+ const target = spec.entry !== undefined ? `entry:"${spec.entry}"` : spec.config !== undefined ? `config:"${spec.config}"` : `nodeId:"${spec.nodeId}"`;
207
+ const err = new NodeBlocked({
208
+ reason: `waiting on needs.${varName} (hops:${spec.hops}, ${target}, ${result.status})`,
209
+ needed: [spec.entry ?? spec.config ?? spec.nodeId],
210
+ hint: "declared needs: not ready yet — will recheck on next dispatch",
211
+ });
212
+ err._blocked = { ...err._blocked, _kind: "needs-not-ready" };
213
+ throw err;
214
+ }
215
+ // Optional field-pick, so an author can ask for exactly the sub-key(s)
216
+ // it actually reads instead of the whole ancestor node's output — the
217
+ // same "scoped read, not the whole blob" reasoning this engine follows
218
+ // elsewhere, just applied at the single-node
219
+ // granularity ctx.needs() operates at. `fields` is always an array,
220
+ // even for a single key — no separate "unwrap to a bare value" mode,
221
+ // so ctx.needs()'s return shape is always a dict, never sometimes-a-
222
+ // dict-sometimes-a-scalar depending on how many fields were picked.
223
+ // Not declared -> today's default, the whole resolved value, unchanged.
224
+ if (Array.isArray(spec.fields)) {
225
+ return Object.fromEntries(spec.fields.map((f) => [f, result.value?.[f]]));
226
+ }
227
+ return result.value;
228
+ };
229
+ }
@@ -0,0 +1,133 @@
1
+ // pipelines-watcher.js — keep `pipelinesRef` in sync with the on-disk
2
+ // pipelines directory handed in as `pipelinesDir`.
3
+ //
4
+ // Why: without this, `loadPipelines` only runs once at gateway boot (via the
5
+ // `manifold-loader` service or `ensurePipelinesLoaded`'s "load once if
6
+ // empty" guard). New pipeline folders dropped after boot, or edits to existing
7
+ // `index.js` files, are invisible until the next gateway restart.
8
+ //
9
+ // How: poll every `pollMs` (default 1000). Each tick, compute a fingerprint
10
+ // of `<dir>/<entry>/index.js` files keyed by name and mtime. If the
11
+ // fingerprint changed since last tick, call `loadPipelines` and swap into
12
+ // `pipelinesRef.value`. The loader already cache-busts imports by reading
13
+ // each index.js itself and importing it as a base64-encoded `data:` URL
14
+ // (each payload is a unique URL, so Node's ESM cache can't return a stale
15
+ // module), so the swap picks up edited code.
16
+ //
17
+ // Note on directory mtime: the top-level `.pipelines/` mtime only changes on
18
+ // add/remove at depth 1 — it does NOT change when an `index.js` is edited.
19
+ // That's why we fingerprint each `index.js`'s mtime explicitly.
20
+
21
+ import { readdir, stat } from "node:fs/promises";
22
+ import path from "node:path";
23
+ import { loadPipelines } from "./loader.js";
24
+
25
+ const DEFAULT_POLL_MS = 1000;
26
+
27
+ export function createPipelinesWatcher({ pipelinesDir, pipelinesRef, logger, pollMs = DEFAULT_POLL_MS }) {
28
+ let lastFingerprint = null;
29
+ let timer = null;
30
+ let inFlight = false;
31
+ const listeners = new Set();
32
+
33
+ async function fingerprint() {
34
+ let entries;
35
+ try {
36
+ entries = await readdir(pipelinesDir, { withFileTypes: true });
37
+ } catch (err) {
38
+ // Directory missing — treat as empty fingerprint. If it later appears,
39
+ // the next tick reloads.
40
+ if (err.code === "ENOENT") return "";
41
+ throw err;
42
+ }
43
+ const parts = [];
44
+ for (const ent of entries.sort((a, b) => a.name.localeCompare(b.name))) {
45
+ if (ent.name.startsWith(".") || !ent.isDirectory()) continue;
46
+ const idx = path.join(pipelinesDir, ent.name, "index.js");
47
+ try {
48
+ const s = await stat(idx);
49
+ parts.push(`${ent.name}:${s.mtimeMs}`);
50
+ } catch {
51
+ // index.js missing — skip; loader would skip it too.
52
+ }
53
+ const fleet = path.join(pipelinesDir, ent.name, "fleet.yaml");
54
+ try {
55
+ const sf = await stat(fleet);
56
+ parts.push(`${ent.name}:fleet:${sf.mtimeMs}`);
57
+ } catch { /* fleet.yaml optional */ }
58
+ }
59
+ return parts.join("|");
60
+ }
61
+
62
+ async function tick() {
63
+ if (inFlight) return;
64
+ inFlight = true;
65
+ try {
66
+ const fp = await fingerprint();
67
+ if (lastFingerprint === null) {
68
+ // First tick: prime the fingerprint without reloading — the loader
69
+ // service has already loaded pipelines at gateway boot.
70
+ lastFingerprint = fp;
71
+ return;
72
+ }
73
+ if (fp === lastFingerprint) return;
74
+ const loaded = await loadPipelines(pipelinesDir, logger);
75
+ // Merge rather than full-replace: if a pipeline that was previously live
76
+ // fails to reload (transient ESM cache error, file mid-write during a git
77
+ // op) its entry would silently vanish from the live Map and stay missing
78
+ // until the next mtime change. Instead, keep stale entries whose index.js
79
+ // is still on disk — they stay live until the disk copy is actually gone.
80
+ const merged = new Map(loaded);
81
+ for (const [id, existing] of pipelinesRef.value) {
82
+ if (merged.has(id)) continue;
83
+ // Synthetic subflow-child pipelines (id = `${parentPipelineId}::${nodeId}`,
84
+ // registered in-memory by loader.js's registerSubflowChildren) are never
85
+ // disk-backed at <pipelinesDir>/<id>/ — stat-ing their own id always
86
+ // throws, so they'd be dropped every time their real parent transiently
87
+ // fails to reload, even though the parent itself survives via this same
88
+ // loop. Check the parent's directory instead (loader.js tags every
89
+ // synthetic entry with _parentPipelineId for exactly this).
90
+ const diskCheckId = existing._parentPipelineId ?? id;
91
+ try {
92
+ await stat(path.join(pipelinesDir, diskCheckId, "index.js"));
93
+ merged.set(id, existing);
94
+ logger?.warn?.(`manifold pipelines-watcher: kept stale entry for "${id}" (failed to reload)`);
95
+ } catch {
96
+ // parent's index.js gone — intentional removal, drop it
97
+ }
98
+ }
99
+ pipelinesRef.value = merged;
100
+ lastFingerprint = fp;
101
+ logger?.info?.(`manifold pipelines-watcher: reloaded ${loaded.size} pipeline(s) after disk change`);
102
+ for (const cb of listeners) {
103
+ try { cb({ size: loaded.size, ids: [...loaded.keys()] }); }
104
+ catch (err) { logger?.warn?.(`manifold pipelines-watcher: listener error: ${err?.message ?? err}`); }
105
+ }
106
+ } catch (err) {
107
+ // Swallow — next tick retries.
108
+ logger?.warn?.(`manifold pipelines-watcher: tick error: ${err?.message ?? err}`);
109
+ } finally {
110
+ inFlight = false;
111
+ }
112
+ }
113
+
114
+ return {
115
+ start() {
116
+ if (timer) return;
117
+ timer = setInterval(tick, pollMs);
118
+ if (typeof timer?.unref === "function") timer.unref();
119
+ },
120
+ stop() {
121
+ if (timer) { clearInterval(timer); timer = null; }
122
+ lastFingerprint = null;
123
+ listeners.clear();
124
+ },
125
+ onReload(cb) {
126
+ listeners.add(cb);
127
+ return () => listeners.delete(cb);
128
+ },
129
+ // Exposed for tests / probes.
130
+ _tick: tick,
131
+ _fingerprint: fingerprint,
132
+ };
133
+ }
@@ -0,0 +1,34 @@
1
+ // result-replay.js — the pure merge step shared by assembleState()
2
+ // (executor-helpers.js) and subflow-state.js's per-child `data` field.
3
+ //
4
+ // Split out on its own specifically to avoid a circular import:
5
+ // executor-helpers.js's assembleState() already imports getAllSubflowGroups
6
+ // from subflow-state.js, so subflow-state.js can't import assembleState()
7
+ // back — but it DOES need this exact same "replay a flow's own completed
8
+ // resultJson rows onto a base object" logic for its own `data` field
9
+ // (state/subflow-state.js's normalizeChildAsSubflowRow).
10
+ // No _subflowGroups nesting happens here — a child's OWN nested subflow
11
+ // groups aren't part of "this child's own top-level output"; a caller
12
+ // that needs those calls assembleState() on that child's flowId directly.
13
+ export function replayResults(baseState, results) {
14
+ let state = { ...baseState };
15
+ for (const r of results ?? []) {
16
+ if (r.resultWrittenAt == null) continue;
17
+ if (r.resultTooLargeBytes != null && r.resultJson == null) {
18
+ // writeResult() (state/node-executions.js) does not cap results, so
19
+ // this can only fire for a row that kept nothing at all — the
20
+ // real value was never stored and isn't recoverable. A row that DID
21
+ // keep some fields has a non-null resultJson and falls through to the
22
+ // normal merge below: no reason to throw away real partial data just
23
+ // because resultTooLargeBytes also happens to be set on that row.
24
+ state = { ...state, [r.nodeId]: { _resultTooLarge: true, bytes: r.resultTooLargeBytes } };
25
+ continue;
26
+ }
27
+ let update;
28
+ try { update = JSON.parse(r.resultJson ?? "null"); } catch { continue; }
29
+ if (update && typeof update === "object" && !Array.isArray(update)) {
30
+ state = { ...state, ...update };
31
+ }
32
+ }
33
+ return state;
34
+ }
@@ -0,0 +1,11 @@
1
+ // revival-messages.js — shared boot-revival continuation prompt.
2
+ //
3
+ // Sent to an EXISTING session key after a gateway restart interrupts an
4
+ // in-flight agent turn. Never re-dispatches the original task with a fresh
5
+ // prompt — that would give the agent no way to know it already tried this
6
+ // once, risking a duplicate side effect (e.g. a duplicate submission) if the
7
+ // prior attempt actually completed before the crash. Sending this to the
8
+ // same session lets the agent check its own history first.
9
+
10
+ export const REVIVAL_MSG =
11
+ "Continue from where you left off — your prior session was interrupted by a gateway restart or process death. Read your conversation history, identify the in-flight task, and resume or finalize it.";
@@ -0,0 +1,55 @@
1
+ // spawn-child.js — Phase 2 branch/flow unification: race-safe identity +
2
+ // creation for a subflow-item child flow.
3
+
4
+ // resolveOrCreateChild finds or creates the flow_node_invocations row
5
+ // identifying "nodeId processing item subflowKey", then finds or creates
6
+ // the flow_runs child that invocation should own.
7
+ //
8
+ // Race safety comes entirely from findOrCreateInvocation's insert-then-
9
+ // reselect pattern (state/node-executions.js) backed by the invocation
10
+ // table's unique index — two concurrent callers for the same identity tuple
11
+ // always agree on one invocation id, so at most one of them ever wins the
12
+ // "does a child exist yet" check and creates one. That identity tuple
13
+ // includes flowId, though — it says nothing about two DIFFERENT parent
14
+ // flows independently deciding to dispatch the same subflowKey (e.g. two
15
+ // freelance_freelancer-proposal scans both fanning out SCORE for the same
16
+ // listing). `guard`, when supplied, is threaded straight through to
17
+ // findOrCreateInvocation to close exactly that gap — see its own comment
18
+ // for why running it inside that call's write-queue turn makes it
19
+ // race-safe across flowIds, not just within one.
20
+ export async function resolveOrCreateChild({
21
+ flowId,
22
+ pipelineId = null,
23
+ nodeId,
24
+ subflowKey,
25
+ childPipelineId,
26
+ initialState,
27
+ store,
28
+ runStore,
29
+ nodeExecutionsStore,
30
+ guard,
31
+ }) {
32
+ const invocationId = await nodeExecutionsStore.findOrCreateInvocation({
33
+ flowId, pipelineId, nodeId, subflowKey, guard,
34
+ });
35
+ if (invocationId == null) {
36
+ return { invocationId: null, childFlowId: null, created: false, guarded: true };
37
+ }
38
+
39
+ const existing = await runStore.getByParentInvocationId(invocationId);
40
+ if (existing) {
41
+ return { invocationId, childFlowId: existing.flowId, created: false };
42
+ }
43
+
44
+ const child = await store.create({
45
+ pipelineId: childPipelineId,
46
+ initialState,
47
+ parentInvocationId: invocationId,
48
+ });
49
+ // Re-query to find the actual winner of the concurrent-insert race.
50
+ // With async clients, multiple callers can pass the existence check
51
+ // before any of them persists, so only the one that won the UNIQUE
52
+ // constraint has a real row. The winner is always findable via the index.
53
+ const winner = await runStore.getByParentInvocationId(invocationId);
54
+ return { invocationId, childFlowId: winner?.flowId ?? child.flowId, created: winner?.flowId === child.flowId };
55
+ }
@@ -0,0 +1,109 @@
1
+ // wait-for-flow-settlement.js — recursively waits for a flow AND every
2
+ // descendant it spawns via any childFlows:true subflow node, at any depth,
3
+ // to reach a stopping point (genuinely terminal, or a real human-gate
4
+ // wait). Exists specifically for the one-shot CLI (bin/cli.js) — the
5
+ // live gateway process never needs this: its own event loop keeps
6
+ // draining pending setImmediate/timer callbacks for as long as it runs,
7
+ // so children spawned by dispatchSubflowChildren settle naturally over
8
+ // time. A one-shot CLI invocation has no such guarantee: `withExit`'s
9
+ // `process.exit(0)` kills any pending setImmediate outright.
10
+ //
11
+ // `bin/cli.js`'s start/rerun/retry/approve commands each `await` exactly
12
+ // one `executor.runFlow()`/`resume()` call for the ONE flow they were
13
+ // given, documented as "blocks until the flow reaches terminal or
14
+ // human-gate" — but that await resolves the moment THAT flow itself goes
15
+ // blocked/terminal, without ever waiting on any children it spawned along
16
+ // the way (dispatchSubflowChildren fires each child via a bare
17
+ // `setImmediate(() => runFlow(childFlowId, "fresh"))` and returns
18
+ // immediately).
19
+ //
20
+ // Deliberately NOT a fix inside dispatchSubflowChildren/executor.js
21
+ // itself — that fire-and-forget dispatch is correct and load-bearing for
22
+ // the live gateway (a subflow node's own step must return promptly so the
23
+ // executor isn't stuck waiting on one flow's children before it can
24
+ // process anything else). This module only changes what the CLI does
25
+ // with its own already-live process before letting it exit.
26
+
27
+ const TERMINAL_STATUSES = new Set(["succeeded", "failed", "cancelled", "skipped"]);
28
+
29
+ // waitJson.kind values that will NEVER resolve on their own no matter how
30
+ // long this process keeps polling — each needs a genuinely different
31
+ // resumption path than "wait":
32
+ // - "human-gate": resumed later via the gate-resolve API, against
33
+ // whatever process an operator/dashboard/Telegram calls it from —
34
+ // not by this one-shot CLI process staying alive.
35
+ // - "needs-gateway-handoff": thrown by executor-errors.js's
36
+ // GatewayHandoffNeeded when a kind:"agent" node dispatches outside a
37
+ // live gateway request scope — which the one-shot CLI process always
38
+ // is. That file's own docstring confirms there is no automatic
39
+ // resume for this wait kind anywhere in the codebase; it needs either
40
+ // the scope gap fixed at the dispatch's own call site or manual
41
+ // task-revival. Without excluding it here, any childFlows:true
42
+ // descendant containing a kind:"agent" node would make this module's
43
+ // unbounded poll loop hang forever instead of surfacing the (already
44
+ // pre-existing, already-expected) blocked status promptly.
45
+ // Anything else non-terminal (running, queued on the shared admission pool,
46
+ // blocked on its own children-pending) needs this process to keep running
47
+ // blocked on its own children-pending) needs this process to keep running
48
+ // for it to ever resolve, since nothing else will wake it.
49
+ const NEVER_AUTO_RESOLVES = new Set(["human-gate", "needs-gateway-handoff"]);
50
+
51
+ function isRestPoint(flow) {
52
+ if (!flow) return true; // gone/not found — nothing left to wait on
53
+ if (TERMINAL_STATUSES.has(flow.status)) return true;
54
+ return flow.status === "blocked" && NEVER_AUTO_RESOLVES.has(flow.waitJson?.kind);
55
+ }
56
+
57
+ async function collectDirectChildren(flowId, { runStore, nodeExecutionsStore }) {
58
+ const subflowNodeIds = await nodeExecutionsStore.listSubflowNodeIdsForFlow(flowId);
59
+ if (subflowNodeIds.length === 0) return [];
60
+ const byNode = await nodeExecutionsStore.listInvocationIdsForNodes(flowId, subflowNodeIds);
61
+ const invocationIds = [...byNode.values()].flat();
62
+ if (invocationIds.length === 0) return [];
63
+ return runStore.listForParentInvocations(invocationIds);
64
+ }
65
+
66
+ // One pass over the whole current tree (re-discovered fresh every call,
67
+ // since a resumed flow can route to nodes/children that didn't exist on
68
+ // the previous pass). Returns the list of flowIds still needing this
69
+ // process to keep running — empty means safe to exit.
70
+ async function findPending(rootFlowId, ctx) {
71
+ const visited = new Set();
72
+ const pending = [];
73
+ const queue = [rootFlowId];
74
+ while (queue.length > 0) {
75
+ const flowId = queue.shift();
76
+ if (visited.has(flowId)) continue;
77
+ visited.add(flowId);
78
+ const flow = await ctx.store.get(flowId);
79
+ if (!isRestPoint(flow)) pending.push(flowId);
80
+ const children = await collectDirectChildren(flowId, ctx);
81
+ for (const child of children) queue.push(child.flowId);
82
+ }
83
+ return pending;
84
+ }
85
+
86
+ /**
87
+ * @param {string} rootFlowId
88
+ * @param {object} ctx — { store, runStore, nodeExecutionsStore, logger }
89
+ * @param {object} [opts]
90
+ * @param {number} [opts.pollMs] — default 300
91
+ * @param {number} [opts.logEveryMs] — progress log cadence while waiting, default 5000
92
+ * @returns {Promise<{settled: true}>} — resolves once every flow in the
93
+ * tree is terminal or genuinely gate-blocked. No timeout: unbounded,
94
+ * same philosophy the CLI's pre-existing single-flow await already had
95
+ * ("ctrl-c to stop" is the documented escape hatch, not a silent give-up).
96
+ */
97
+ export async function waitForFlowSettlement(rootFlowId, ctx, { pollMs = 300, logEveryMs = 5000 } = {}) {
98
+ let lastLogAt = 0;
99
+ while (true) {
100
+ const pending = await findPending(rootFlowId, ctx);
101
+ if (pending.length === 0) return { settled: true };
102
+ const now = Date.now();
103
+ if (now - lastLogAt >= logEveryMs) {
104
+ lastLogAt = now;
105
+ ctx.logger?.info?.(`manifold: waiting on ${pending.length} still-active flow(s) beneath ${rootFlowId.slice(0, 8)}: ${pending.map((id) => id.slice(0, 8)).join(", ")}`);
106
+ }
107
+ await new Promise((resolve) => setTimeout(resolve, pollMs));
108
+ }
109
+ }
@@ -0,0 +1,26 @@
1
+ // events/catalog.js — single producer for pipeline-catalog changes.
2
+ //
3
+ // One watcher reload feeds both sinks with identical content: the gateway
4
+ // WS stream (canonical pipeline.catalog.updated) and the dashboard bus
5
+ // (legacy pipelines_changed type, canonical kind stamped). Separate sinks
6
+ // stay separate; only the content decisions merge here.
7
+ export function createCatalogPublisher({ emit, getDashboardBroadcast }) {
8
+ return async function publishCatalogChanged(info) {
9
+ const now = Date.now();
10
+ const payload = { count: info.size, ids: info.ids, updatedAt: now };
11
+ if (emit) {
12
+ try { await emit("pipeline.catalog.updated", payload); } catch {}
13
+ }
14
+ try {
15
+ const broadcastDashboard = getDashboardBroadcast?.();
16
+ if (broadcastDashboard) {
17
+ await broadcastDashboard({
18
+ kind: "pipeline.catalog.updated",
19
+ at: now,
20
+ size: payload.count,
21
+ ids: payload.ids,
22
+ });
23
+ }
24
+ } catch {}
25
+ };
26
+ }
package/events/emit.js ADDED
@@ -0,0 +1,44 @@
1
+ // events/emit.js — single fan-out for canonical kinds.
2
+ //
3
+ // emit(kind, payload) validates, stamps the envelope, and delivers to the
4
+ // gateway WS broadcast. Transport faults are reported in the delivery
5
+ // receipt, never thrown (a dead gateway must not fail the dispatch that
6
+ // produced the event); validation faults reject (programmer errors).
7
+ // The `plugin.` prefix is applied here and only here — producers name
8
+ // the kind, never the wire.
9
+ import {
10
+ assertPayload,
11
+ isDashboardOnly,
12
+ } from "./kinds.js";
13
+
14
+ export function createEmitter({ getBroadcast, logger } = {}) {
15
+ async function emit(kind, payload = {}) {
16
+ // Throws (rejects) on unknown/reserved kinds and missing fields.
17
+ assertPayload(kind, payload);
18
+ // A `type` field here is a producer bug — the kind is the only name.
19
+ const { type: _strayType, ...fields } = payload;
20
+ const envelope = { ...fields, kind };
21
+ if (envelope.emittedAtMs == null) envelope.emittedAtMs = Date.now();
22
+
23
+ // Gateway WS boundary — the only delivery path. Dashboard-only kinds
24
+ // skip it.
25
+ try {
26
+ const broadcast = typeof getBroadcast === "function" ? getBroadcast() : null;
27
+ if (broadcast && !isDashboardOnly(kind)) {
28
+ try {
29
+ broadcast(`plugin.${kind}`, envelope);
30
+ return { ok: true, kind, deliveries: { ws: "sent" } };
31
+ } catch (err) {
32
+ try { logger?.warn?.(`events/emit: ws broadcast failed (${kind}): ${err?.message ?? err}`); } catch {}
33
+ return { ok: false, kind, deliveries: { ws: "error" } };
34
+ }
35
+ }
36
+ } catch {
37
+ return { ok: false, kind, deliveries: { ws: "error" } };
38
+ }
39
+
40
+ return { ok: true, kind, deliveries: { ws: "skipped" } };
41
+ };
42
+
43
+ return { emit };
44
+ }