@igorjpimenta/manifold 0.0.0-stage → 2026.11.22-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/.migrations/ops/0000_futuristic_karma.sql +111 -0
  2. package/.migrations/ops/0001_low_maximus.sql +7 -0
  3. package/.migrations/ops/meta/0000_snapshot.json +763 -0
  4. package/.migrations/ops/meta/0001_snapshot.json +173 -0
  5. package/.migrations/ops/meta/_journal.json +20 -0
  6. package/.migrations/runtime/0000_real_boomer.sql +180 -0
  7. package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
  8. package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
  9. package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
  10. package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
  11. package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
  12. package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
  13. package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
  14. package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
  15. package/.migrations/runtime/0009_lying_shocker.sql +96 -0
  16. package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
  17. package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
  18. package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
  19. package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
  20. package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
  21. package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
  22. package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
  23. package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
  24. package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
  25. package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
  26. package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
  27. package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
  28. package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
  29. package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
  30. package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
  31. package/.migrations/runtime/meta/_journal.json +97 -0
  32. package/README.md +1583 -2
  33. package/RELEASING.md +89 -0
  34. package/agent/fleet.js +100 -0
  35. package/bin/cli.js +854 -0
  36. package/bin/manifold.js +4 -0
  37. package/boot/flow-revival.js +536 -0
  38. package/budget/index.js +495 -0
  39. package/budget/lib.js +59 -0
  40. package/dashboard/cost.js +528 -0
  41. package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
  42. package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
  43. package/dashboard/dist/index.html +13 -0
  44. package/dashboard/event-log.js +267 -0
  45. package/dashboard/index.html +12 -0
  46. package/dashboard/package-lock.json +2877 -0
  47. package/dashboard/package.json +37 -0
  48. package/dashboard/pipeline-graph.js +131 -0
  49. package/dashboard/postcss.config.js +6 -0
  50. package/dashboard/search.js +183 -0
  51. package/dashboard/server.js +1015 -0
  52. package/dashboard/src/App.tsx +42 -0
  53. package/dashboard/src/api/client.ts +118 -0
  54. package/dashboard/src/api/types.ts +510 -0
  55. package/dashboard/src/api/ws.tsx +112 -0
  56. package/dashboard/src/components/CancelButton.tsx +63 -0
  57. package/dashboard/src/components/ClickToCopy.tsx +68 -0
  58. package/dashboard/src/components/CopyButton.tsx +61 -0
  59. package/dashboard/src/components/DailyCostTrend.tsx +85 -0
  60. package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
  61. package/dashboard/src/components/Funnel.tsx +149 -0
  62. package/dashboard/src/components/GateForm.tsx +158 -0
  63. package/dashboard/src/components/GraphView.tsx +302 -0
  64. package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
  65. package/dashboard/src/components/Histogram.tsx +292 -0
  66. package/dashboard/src/components/Layout.tsx +28 -0
  67. package/dashboard/src/components/NeedsAttention.tsx +98 -0
  68. package/dashboard/src/components/NodeContextCard.tsx +132 -0
  69. package/dashboard/src/components/PathTrail.tsx +35 -0
  70. package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
  71. package/dashboard/src/components/RerunButton.tsx +46 -0
  72. package/dashboard/src/components/RunComparison.tsx +53 -0
  73. package/dashboard/src/components/Sparkline.tsx +94 -0
  74. package/dashboard/src/components/StatStrip.tsx +147 -0
  75. package/dashboard/src/components/StateTree.tsx +84 -0
  76. package/dashboard/src/components/StatusBadge.tsx +18 -0
  77. package/dashboard/src/components/SubflowPathPills.tsx +38 -0
  78. package/dashboard/src/components/SubflowSummary.tsx +24 -0
  79. package/dashboard/src/components/SubflowsCard.tsx +327 -0
  80. package/dashboard/src/components/TodayStrip.tsx +86 -0
  81. package/dashboard/src/components/TranscriptPane.tsx +99 -0
  82. package/dashboard/src/components/TranscriptRow.tsx +69 -0
  83. package/dashboard/src/components/ui/badge.tsx +38 -0
  84. package/dashboard/src/components/ui/button.tsx +28 -0
  85. package/dashboard/src/components/ui/card.tsx +42 -0
  86. package/dashboard/src/components/ui/skeleton.tsx +9 -0
  87. package/dashboard/src/components/ui/table.tsx +62 -0
  88. package/dashboard/src/index.css +26 -0
  89. package/dashboard/src/lib/anomaly.ts +42 -0
  90. package/dashboard/src/lib/format.ts +30 -0
  91. package/dashboard/src/lib/step-history.ts +13 -0
  92. package/dashboard/src/lib/subflow.ts +154 -0
  93. package/dashboard/src/lib/use-click-outside.ts +28 -0
  94. package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
  95. package/dashboard/src/lib/utils.ts +6 -0
  96. package/dashboard/src/main.tsx +10 -0
  97. package/dashboard/src/types/shims.d.ts +4 -0
  98. package/dashboard/src/views/Home.tsx +478 -0
  99. package/dashboard/src/views/PipelineDetail.tsx +1185 -0
  100. package/dashboard/src/views/PipelinesList.tsx +111 -0
  101. package/dashboard/src/views/RunDetail.tsx +248 -0
  102. package/dashboard/src/views/Search.tsx +196 -0
  103. package/dashboard/stats.js +668 -0
  104. package/dashboard/tailwind.config.ts +43 -0
  105. package/dashboard/tree-stats.js +12 -0
  106. package/dashboard/tsconfig.json +25 -0
  107. package/dashboard/views.js +231 -0
  108. package/dashboard/vite.config.ts +34 -0
  109. package/drizzle.config.js +12 -0
  110. package/drizzle.config.ops.js +12 -0
  111. package/elevation.js +116 -0
  112. package/engine/admission-promotion-resume.js +127 -0
  113. package/engine/agent-dispatch.js +64 -0
  114. package/engine/child-resume-trigger.js +315 -0
  115. package/engine/executor-budget.js +87 -0
  116. package/engine/executor-cancel.js +83 -0
  117. package/engine/executor-errors.js +122 -0
  118. package/engine/executor-helpers.js +254 -0
  119. package/engine/executor-nodes.js +292 -0
  120. package/engine/executor-persist.js +212 -0
  121. package/engine/executor-subflow-children.js +275 -0
  122. package/engine/executor.js +1013 -0
  123. package/engine/global-concurrency.js +33 -0
  124. package/engine/loader.js +528 -0
  125. package/engine/needs-resolver.js +229 -0
  126. package/engine/pipelines-watcher.js +133 -0
  127. package/engine/result-replay.js +34 -0
  128. package/engine/revival-messages.js +11 -0
  129. package/engine/spawn-child.js +55 -0
  130. package/engine/wait-for-flow-settlement.js +109 -0
  131. package/events/catalog.js +26 -0
  132. package/events/emit.js +44 -0
  133. package/events/kinds.js +118 -0
  134. package/events/serve-ws.js +74 -0
  135. package/events/stream.js +35 -0
  136. package/events/ws-dialog.js +257 -0
  137. package/events/ws-frame.js +194 -0
  138. package/events/ws-run-events.js +70 -0
  139. package/events.js +32 -0
  140. package/gates/gate-resolver.js +576 -0
  141. package/gates/paused-flows.js +55 -0
  142. package/graphql/columns.js +67 -0
  143. package/graphql/schema.js +333 -0
  144. package/index.js +49 -0
  145. package/mcp/http.js +159 -0
  146. package/mcp/identity.js +94 -0
  147. package/mcp/stdio.js +80 -0
  148. package/mcp/tools.js +245 -0
  149. package/mcp/wire-ops.js +178 -0
  150. package/package.json +49 -4
  151. package/routes/bearer-auth.js +57 -0
  152. package/routes/discuss-audio.js +87 -0
  153. package/routes/discuss-component.js +39 -0
  154. package/routes/discuss-draft.js +159 -0
  155. package/routes/discuss-helpers.js +20 -0
  156. package/routes/discuss.js +848 -0
  157. package/routes/failures.js +104 -0
  158. package/routes/graphql.js +61 -0
  159. package/routes/health.js +54 -0
  160. package/routes/operator-discuss.js +193 -0
  161. package/routes/operator-resume.js +32 -0
  162. package/routes/operator-retry.js +301 -0
  163. package/routes/operator.js +539 -0
  164. package/routes/pipelines.js +118 -0
  165. package/routes/runs.js +1129 -0
  166. package/scripts/ci/import-check.mjs +50 -0
  167. package/scripts/link-core-self.mjs +31 -0
  168. package/scripts/with-guard.sh +24 -0
  169. package/server/app.js +503 -0
  170. package/server/registry.js +53 -0
  171. package/server/serve.js +163 -0
  172. package/state/admission-queue-store.js +325 -0
  173. package/state/attribution-registries.js +102 -0
  174. package/state/batch-writer.js +162 -0
  175. package/state/db-guard.js +63 -0
  176. package/state/db-health.js +27 -0
  177. package/state/db.js +122 -0
  178. package/state/discuss-store.js +299 -0
  179. package/state/failure-store.js +427 -0
  180. package/state/migrate.js +104 -0
  181. package/state/node-cost.js +67 -0
  182. package/state/node-executions.js +775 -0
  183. package/state/node-health-store.js +124 -0
  184. package/state/ops-merge.js +93 -0
  185. package/state/paths.js +146 -0
  186. package/state/pipeline-dirs.js +99 -0
  187. package/state/pipeline-registry.js +449 -0
  188. package/state/run-store.js +697 -0
  189. package/state/schema.js +438 -0
  190. package/state/schema.ops.js +44 -0
  191. package/state/session-index.js +177 -0
  192. package/state/session-keys.js +39 -0
  193. package/state/state-store.js +311 -0
  194. package/state/subflow-state.js +126 -0
  195. package/state/tool-handoff.js +169 -0
  196. package/telemetry/budget-alert.js +353 -0
  197. package/telemetry/cli.js +331 -0
  198. package/telemetry/grammar.js +119 -0
  199. package/telemetry/http-budget.js +132 -0
  200. package/telemetry/http-cost.js +670 -0
  201. package/telemetry/http-helpers.js +134 -0
  202. package/telemetry/http-tree.js +349 -0
  203. package/telemetry/http.js +44 -0
  204. package/telemetry/index.js +307 -0
  205. package/telemetry/ingest.js +134 -0
  206. package/telemetry/labels.js +115 -0
  207. package/telemetry/pricer.js +77 -0
  208. package/telemetry/resolver.js +314 -0
  209. package/telemetry/store.js +147 -0
  210. package/utils/run-patch.js +43 -0
@@ -0,0 +1,39 @@
1
+ // The sessions registry is the FK target for every session-key pointer.
2
+ // Keys are assigned by the harness, so a row has to exist before the first
3
+ // pointer to it is written; writers insert OR IGNORE the key they are about
4
+ // to reference (telemetry/store.js, state/session-index.js,
5
+ // state/tool-handoff.js, budget/index.js). This module backfills the keys
6
+ // already referenced by rows that predate the registry — it is the only
7
+ // place that reads across the child tables, and it runs once per file open
8
+ // after migrations (and after the ops→runtime copy, which brings its own
9
+ // queue/lineage/cost keys).
10
+
11
+ export const SESSION_KEYS_FROM_ROWS_SQL = `
12
+ INSERT OR IGNORE INTO sessions (session_key, first_seen_at)
13
+ SELECT k, ? FROM (
14
+ SELECT child_session_key AS k FROM session_lineage
15
+ UNION SELECT parent_session_key FROM session_lineage WHERE parent_session_key IS NOT NULL
16
+ UNION SELECT root_session_key FROM session_lineage
17
+ UNION SELECT session_key FROM flow_sessions
18
+ UNION SELECT session_key FROM discuss_threads
19
+ UNION SELECT session_key FROM tool_results
20
+ UNION SELECT enqueuer_session_key FROM queue_items WHERE enqueuer_session_key IS NOT NULL
21
+ UNION SELECT session_key FROM cost_by_call WHERE session_key IS NOT NULL
22
+ UNION SELECT root_session_key FROM cost_by_call WHERE root_session_key IS NOT NULL
23
+ )
24
+ `;
25
+
26
+ export async function backfillSessionKeys(client, now = Date.now()) {
27
+ await client.execute({ sql: SESSION_KEYS_FROM_ROWS_SQL, args: [now] });
28
+ }
29
+
30
+ export const ENSURE_SESSION_SQL = "INSERT OR IGNORE INTO sessions (session_key, first_seen_at) VALUES (?, ?)";
31
+
32
+ // Writer-side guard: the rows an insert is about to reference must exist
33
+ // before the insert, or the FK turns a best-effort write into a dropped
34
+ // one. Batch, so N keys cost one round-trip.
35
+ export async function ensureSessionKeys(client, keys, now = Date.now()) {
36
+ const unique = [...new Set(keys.filter((k) => typeof k === "string" && k.length > 0))];
37
+ if (unique.length === 0) return;
38
+ await client.batch(unique.map((k) => ({ sql: ENSURE_SESSION_SQL, args: [k, now] })), "write");
39
+ }
@@ -0,0 +1,311 @@
1
+ // state-store.js — thin wrapper over run-store for manifold flow state.
2
+ //
3
+ // All reads and CAS writes go directly through runStore (runtime.sqlite).
4
+ // No dual-write; registry.sqlite is never touched.
5
+ //
6
+ // After each successful write, flowStateEmitter fires so trigger-based
7
+ // consumers (event-log, live views) wake immediately instead of polling.
8
+
9
+ import { randomUUID } from "node:crypto";
10
+ import { flowStateEmitter, gateEmitter, flowEmitter } from "../events.js";
11
+ import { maybeResumeParent } from "../engine/child-resume-trigger.js";
12
+ import { wakePromotedRelease } from "../engine/admission-promotion-resume.js";
13
+
14
+ function notifyChange(flowId) {
15
+ if (!flowId) return;
16
+ process.nextTick(() => flowStateEmitter.emit("change", flowId));
17
+ }
18
+
19
+ // After a terminal CAS write, check whether this flow was a subflow-spawned
20
+ // child and, if every sibling is now also terminal, wake the parent (or
21
+ // fail it, if its subflow node's `mode` isn't satisfied — see child-resume-
22
+ // trigger.js's maybeResumeParent). Fire-and-forget — never blocks the
23
+ // caller's terminal-transition return. No-ops cleanly until
24
+ // resumeParentRef.resume is set (see index.js wiring) — currently dormant,
25
+ // since no real flow sets parentInvocationId yet. `failFlow` is `store.fail`
26
+ // itself — passed
27
+ // in by the caller (this module's own store methods, below) rather than
28
+ // referenced directly, since `triggerParentResume` is a free function
29
+ // outside the `store` object literal's closure. `admissionQueueStore` is
30
+ // threaded through to maybeResumeParent's own flipToRunningAndResume call —
31
+ // the parent's own admission was released when it entered its
32
+ // children-pending wait (this file's own setWaiting hook, below), so
33
+ // resuming it must re-request one, not assume it's still held.
34
+ function triggerParentResume(flowId, { nodeExecutionsStore, runStore, resumeParentRef, pipelinesRef, failFlow, failureStore, admissionQueueStore, api, logger }) {
35
+ if (!nodeExecutionsStore || !resumeParentRef?.resume) return;
36
+ maybeResumeParent({ flowId, runStore, nodeExecutionsStore, pipelinesRef, resume: resumeParentRef.resume, failFlow, failureStore, admissionQueueStore, api, logger })
37
+ .catch((err) => logger?.warn?.(`state-store: triggerParentResume(${flowId}) failed: ${err?.message ?? err}`));
38
+ }
39
+
40
+ // The OTHER thing that needs to run on every setWaiting (blocked) and every
41
+ // terminal transition below: releasing whatever live admission this flow
42
+ // currently holds in the shared combined pool (state/admission-queue-
43
+ // store.js) — kind:"flow-start" for a root flow, kind:"child-dispatch" for
44
+ // a subflow child (a flow can only ever hold one of the two, permanently,
45
+ // so trying both kinds is cheap and safe: releaseByFlow is a one-SELECT
46
+ // no-op for whichever kind doesn't apply). Per the pool's own design (see
47
+ // admission-queue-store.js's header), a flow only occupies a slot while
48
+ // genuinely "running"; leaving running for ANY other reason (blocked or
49
+ // terminal) must give it back — and if that release promotes a queued
50
+ // flow/child, wake it via wakePromotedRelease (engine/admission-promotion-
51
+ // resume.js).
52
+ //
53
+ // Awaited by every call site below (async, not fire-and-forget): firing it
54
+ // detached lets a caller that returns/exits right after
55
+ // setWaiting/finish/skip/fail/cancel (a short-lived CLI command, or
56
+ // the gateway process dying in that exact window) race ahead of the
57
+ // release and the promoted flow's flip to 'running', leaving it permanently
58
+ // 'admitted' with nothing having ever driven it to 'running'. Only that
59
+ // durable part is awaited — the promoted flow's run itself starts on its
60
+ // own tick (see engine/admission-promotion-resume.js's header for why).
61
+ // Awaiting here doesn't make the DB promotion and the
62
+ // resume literally one transaction (still two separate writes; a genuine
63
+ // process crash between them is what boot/flow-revival.js's reconciliation
64
+ // sweep exists to catch), but it does close the far more common
65
+ // short-lived-caller race at its source, for every one of the 5 terminal/
66
+ // blocked transitions below.
67
+ async function triggerAdmissionRelease(flowId, { admissionQueueStore, runStore, resumeParentRef, api, logger }) {
68
+ if (!admissionQueueStore || !runStore) return;
69
+ try {
70
+ const results = await Promise.all([
71
+ admissionQueueStore.releaseByFlow({ flowId, kind: "flow-start" }),
72
+ admissionQueueStore.releaseByFlow({ flowId, kind: "child-dispatch" }),
73
+ ]);
74
+ if (!resumeParentRef?.resume) return;
75
+ await wakePromotedRelease(results, {
76
+ runStore, resume: resumeParentRef.resume, logger, api,
77
+ context: `state-store: triggerAdmissionRelease(${flowId})`,
78
+ });
79
+ } catch (err) {
80
+ logger?.warn?.(`state-store: triggerAdmissionRelease(${flowId}) failed: ${err?.message ?? err}`);
81
+ }
82
+ }
83
+
84
+ // _session kept in signature for call-site compatibility, used as
85
+ // ownerKey on new flows.
86
+ //
87
+ // nodeExecutionsStore + resumeParentRef support subflow-parent resumption
88
+ // (see child-resume-trigger.js) — resumeParentRef is a mutable box because
89
+ // createStateStore runs before createExecutor exists (index.js's
90
+ // composition order), so `resume` is set onto the box after the fact.
91
+ // pipelinesRef is read-only here, used only to look up a subflow node's
92
+ // `mode` when deciding whether to resume or fail a parent (see child-
93
+ // resume-trigger.js's maybeResumeParent). failureStore is optional,
94
+ // threaded through to maybeResumeParent so a mode-not-satisfied failure
95
+ // gets a real failure_events row instead of a bare status write.
96
+ export function createStateStore(api, _session, { runStore, nodeExecutionsStore, resumeParentRef, pipelinesRef, failureStore, admissionQueueStore, logger } = {}) {
97
+ const store = {
98
+ /**
99
+ * Create a managed flow_run for a fresh pipeline execution.
100
+ * Returns the new flow record (status='running', lastStep='__start__').
101
+ *
102
+ * parentInvocationId links a subflow-spawned child to the
103
+ * flow_node_invocations row that spawned it (branch/flow
104
+ * unification) — undefined/null for ordinary top-level flows.
105
+ */
106
+ async create({ pipelineId, initialState, parentInvocationId }) {
107
+ const flowId = randomUUID();
108
+ const now = Date.now();
109
+ // pipelineId spread LAST — deliberately: initialState is caller-
110
+ // supplied and must never be able to override which pipeline this
111
+ // flow runs as.
112
+ const stateJson = { ...initialState, pipelineId };
113
+ await runStore.insert({
114
+ flowId,
115
+ ownerKey: _session ?? "",
116
+ pipelineId,
117
+ revision: 0,
118
+ status: "running",
119
+ currentStep: null,
120
+ lastStep: "__start__",
121
+ stateJson,
122
+ createdAt: now,
123
+ updatedAt: now,
124
+ parentInvocationId: parentInvocationId ?? null,
125
+ });
126
+ notifyChange(flowId);
127
+ return {
128
+ flowId,
129
+ pipelineId,
130
+ status: "running",
131
+ revision: 0,
132
+ lastStep: "__start__",
133
+ currentStep: null,
134
+ stateJson,
135
+ createdAt: now,
136
+ updatedAt: now,
137
+ parentInvocationId: parentInvocationId ?? null,
138
+ };
139
+ },
140
+
141
+ async get(flowId) {
142
+ return runStore.get(flowId);
143
+ },
144
+
145
+ async list() {
146
+ return runStore.list();
147
+ },
148
+
149
+ async listFor(pipelineId) {
150
+ return runStore.listFor(pipelineId);
151
+ },
152
+
153
+ /**
154
+ * Persist a "still running, advance to next step" transition.
155
+ */
156
+ async setRunning({ flowId, expectedRevision, lastStep, stateJson }) {
157
+ const result = await runStore.casSetRunning(flowId, { expectedRevision, lastStep, stateJson });
158
+ if (result.applied) {
159
+ notifyChange(flowId);
160
+ process.nextTick(() => flowEmitter.emit("flow.step.changed", { flowId, lastStep, stateJson, status: "running" }));
161
+ }
162
+ return result;
163
+ },
164
+
165
+ /**
166
+ * Persist a "block on external signal" transition (human-gate).
167
+ */
168
+ async setWaiting({ flowId, expectedRevision, lastStep, blockingNodeId, stateJson, waitJson, statusReason }) {
169
+ const updatedAt = Date.now();
170
+ const result = await runStore.casSetWaiting(flowId, {
171
+ expectedRevision,
172
+ lastStep,
173
+ blockingNodeId: blockingNodeId ?? null,
174
+ stateJson,
175
+ waitJson,
176
+ statusReason,
177
+ });
178
+ if (result.applied) {
179
+ notifyChange(flowId);
180
+ if (waitJson?.kind === "human-gate") {
181
+ process.nextTick(() => gateEmitter.emit("gate.opened", {
182
+ flowId, nodeId: blockingNodeId,
183
+ items: waitJson.items ?? [], waitJson, stateJson, lastStep,
184
+ updatedAt,
185
+ }));
186
+ }
187
+ await triggerAdmissionRelease(flowId, { admissionQueueStore, runStore, resumeParentRef, api, logger });
188
+ }
189
+ return result;
190
+ },
191
+
192
+ /**
193
+ * Persist a "block on an internal resource" transition (an admission
194
+ * slot, a cross-process lock) — same shape as setWaiting.
195
+ */
196
+ async setQueued({ flowId, expectedRevision, lastStep, blockingNodeId, stateJson, waitJson, statusReason }) {
197
+ const result = await runStore.casSetQueued(flowId, {
198
+ expectedRevision,
199
+ lastStep,
200
+ blockingNodeId: blockingNodeId ?? null,
201
+ stateJson,
202
+ waitJson,
203
+ statusReason,
204
+ });
205
+ if (result.applied) {
206
+ notifyChange(flowId);
207
+ // Release this flow's own admission (flow-start/child-dispatch),
208
+ // same as setWaiting above — but ONLY when this queuing is for a
209
+ // DIFFERENT internal resource (a cross-process lock) than the pool
210
+ // slot itself. setQueued is also called when a
211
+ // flow's OWN flow-start/child-dispatch admission request is what's
212
+ // queuing (executor.js's start(), executor-subflow-children.js's
213
+ // per-child dispatch) — at that point the flow holds nothing yet,
214
+ // so releaseByFlow would delete the very row admit() just created
215
+ // moments ago (releaseRowById deletes unconditionally on status,
216
+ // not just 'admitted' ones), permanently orphaning it. waitJson.kind
217
+ // === "flow-start" or "child-dispatch" is exactly that
218
+ // self-referential case; anything else (e.g. "resource-lock")
219
+ // means the flow is already running and giving back a slot it
220
+ // separately holds.
221
+ if (waitJson?.kind !== "flow-start" && waitJson?.kind !== "child-dispatch") {
222
+ await triggerAdmissionRelease(flowId, { admissionQueueStore, runStore, resumeParentRef, api, logger });
223
+ }
224
+ }
225
+ return result;
226
+ },
227
+
228
+ /**
229
+ * Terminal success.
230
+ */
231
+ async finish({ flowId, expectedRevision, stateJson }) {
232
+ const result = await runStore.casFinish(flowId, { expectedRevision, stateJson });
233
+ if (result.applied) {
234
+ notifyChange(flowId);
235
+ process.nextTick(() => flowEmitter.emit("flow.terminal", { flowId, status: "succeeded", stateJson }));
236
+ triggerParentResume(flowId, { nodeExecutionsStore, runStore, resumeParentRef, pipelinesRef, failFlow: store.fail, failureStore, admissionQueueStore, api, logger });
237
+ await triggerAdmissionRelease(flowId, { admissionQueueStore, runStore, resumeParentRef, api, logger });
238
+ }
239
+ return result;
240
+ },
241
+
242
+ /**
243
+ * Terminal skip — pipeline ran but produced no output (nothing to do).
244
+ * runtime.sqlite gets status='skipped'. `reason` is the node's own
245
+ * _skipped.reason (see engine/executor-helpers.js's detectSkippedSentinel).
246
+ */
247
+ async skip({ flowId, expectedRevision, stateJson, reason }) {
248
+ const result = await runStore.casSkip(flowId, { expectedRevision, stateJson, reason });
249
+ if (result.applied) {
250
+ notifyChange(flowId);
251
+ process.nextTick(() => flowEmitter.emit("flow.terminal", { flowId, status: "skipped", stateJson }));
252
+ triggerParentResume(flowId, { nodeExecutionsStore, runStore, resumeParentRef, pipelinesRef, failFlow: store.fail, failureStore, admissionQueueStore, api, logger });
253
+ await triggerAdmissionRelease(flowId, { admissionQueueStore, runStore, resumeParentRef, api, logger });
254
+ }
255
+ return result;
256
+ },
257
+
258
+ /**
259
+ * Terminal failure.
260
+ */
261
+ async fail({ flowId, expectedRevision, stateJson, statusReason }) {
262
+ const result = await runStore.casFail(flowId, { expectedRevision, stateJson, statusReason });
263
+ if (result.applied) {
264
+ notifyChange(flowId);
265
+ process.nextTick(() => flowEmitter.emit("flow.terminal", { flowId, status: "failed", stateJson }));
266
+ triggerParentResume(flowId, { nodeExecutionsStore, runStore, resumeParentRef, pipelinesRef, failFlow: store.fail, failureStore, admissionQueueStore, api, logger });
267
+ await triggerAdmissionRelease(flowId, { admissionQueueStore, runStore, resumeParentRef, api, logger });
268
+ }
269
+ return result;
270
+ },
271
+
272
+ /**
273
+ * Force a terminal flow back to 'blocked' so a subsequent setRunning
274
+ * (whose CAS excludes terminal statuses) will accept it. Used by
275
+ * operator-retry.js's flow-level retry path — goes through this DI
276
+ * seam (not a raw DatabaseSync write) so tests can observe/control it
277
+ * via the same store double every other transition uses.
278
+ */
279
+ async forceBlock({ flowId, expectedRevision, stateJson, statusReason, ownerKey }) {
280
+ const result = await runStore.forceBlock(flowId, { expectedRevision, stateJson, statusReason, ownerKey });
281
+ if (result.applied) notifyChange(flowId);
282
+ return result;
283
+ },
284
+
285
+ /**
286
+ * Terminal cancellation. Stamps stateJson and flips status atomically
287
+ * (run-store.js's cancelWithStateJson — a single db.batch(), not two
288
+ * sequential writes, so a crash between them can't leave state_json
289
+ * updated with status still not 'cancelled'). Returns { cancelled: boolean }.
290
+ */
291
+ async cancel({ flowId, stateJson, reason = null }) {
292
+ const result = await runStore.cancelWithStateJson(flowId, stateJson, reason);
293
+ if (result.cancelled) {
294
+ notifyChange(flowId);
295
+ process.nextTick(() => flowEmitter.emit("flow.terminal", { flowId, status: "cancelled", stateJson }));
296
+ triggerParentResume(flowId, { nodeExecutionsStore, runStore, resumeParentRef, pipelinesRef, failFlow: store.fail, failureStore, admissionQueueStore, api, logger });
297
+ await triggerAdmissionRelease(flowId, { admissionQueueStore, runStore, resumeParentRef, api, logger });
298
+ }
299
+ return result;
300
+ },
301
+
302
+ async requestCancel({ flowId }) {
303
+ const current = await runStore.get(flowId);
304
+ if (!current) return { applied: false, code: "not_found" };
305
+ await runStore.stampCancelRequested(flowId);
306
+ return { applied: true };
307
+ },
308
+ };
309
+
310
+ return store;
311
+ }
@@ -0,0 +1,126 @@
1
+ // subflow-state.js — read branch/subflow data for a flow, grouped by groupId.
2
+ //
3
+ // Dual-mode (Phase 2):
4
+ // a subflow node's items may live as `flow_branches` rows (old model) OR
5
+ // as real `flow_runs` children (new model, one child per item, linked via
6
+ // parent_invocation_id — see engine/spawn-child.js). Per group, exactly one
7
+ // model applies — a cutover takes a whole subflow node at once, never
8
+ // mixing sources within one group. If
9
+ // `nodeExecutionsStore.listInvocationIdsForNode` finds ANY child-spawn
10
+ // invocation for a group, that group is treated as fully migrated (even
11
+ // if a child row is momentarily missing mid-creation) — old and new data
12
+ // are never blended for the same group, which would be more confusing
13
+ // than a narrow race window.
14
+ //
15
+ // Both functions still return the same `{ [subflowKey]: <row> } `/
16
+ // `{ [groupId]: { [subflowKey]: <row> } }` nested shape pre-Phase-2 callers
17
+ // were already written against (field renamed branchKey -> subflowKey —
18
+ // the old-model path below is normalized to the same field name so every
19
+ // caller sees one consistent shape regardless of which model actually
20
+ // answered) — see normalizeChildAsSubflowRow below for the new-model
21
+ // row's shape, kept as close to branchService's mapBranchRow as the two
22
+ // models allow.
23
+
24
+ import { replayResults } from "../engine/result-replay.js";
25
+
26
+ // Maps one flow_runs child row (+ its invocation) into the old
27
+ // flow_branches row shape. Fields with no clean new-model equivalent yet
28
+ // (_display, activeSessionKey, per-subflow errors) default to empty/null —
29
+ // real values arrive per-pipeline as each one's cutover lands and a real
30
+ // caller demonstrates it needs them.
31
+ //
32
+ // `data`: replays that child's own completed resultJson rows onto its
33
+ // entry-only stateJson, same merge engine-helpers.js's assembleState()
34
+ // uses for the top-level case — imported from result-replay.js
35
+ // specifically to avoid a circular import (assembleState() itself calls
36
+ // getAllSubflowGroups, defined in this file). Async; callers below already
37
+ // run inside an async loop.
38
+ async function normalizeChildAsSubflowRow(child, invocation, nodeExecutionsStore) {
39
+ const isBlockedOnGate = child.status === "blocked" && child.waitJson != null;
40
+ const results = (await nodeExecutionsStore?.listResultsForFlow?.(child.flowId)) ?? [];
41
+ return {
42
+ // invocation.nodeId, not a distinct groupId field (dropped from
43
+ // flow_node_invocations once every subflow node became childFlows-
44
+ // implicit). Kept as the "groupId" key here to match the old-model
45
+ // row shape every caller of getAllSubflowGroups/getSubflowGroup already
46
+ // expects — group_id and node_id were always identical for a real
47
+ // subflow-child invocation, so this is not a behavior change.
48
+ groupId: invocation.nodeId,
49
+ subflowKey: invocation.subflowKey,
50
+ // 'blocked' folds into "running" to match the old model, where status
51
+ // stayed 'running' for the whole branch lifetime and a pending gate
52
+ // was signaled via `awaiting` alone, not a distinct status value.
53
+ // 'queued' is passed through unchanged (Phase 4) — it has no old-model
54
+ // analogue to preserve, and clients already render it distinctly
55
+ // (self-resolving internal-resource wait, not a gate).
56
+ status: child.status === "blocked" ? "running" : child.status,
57
+ step: child.lastStep,
58
+ startedAt: child.createdAt,
59
+ updatedAt: child.updatedAt,
60
+ doneAt: (child.status === "succeeded" || child.status === "skipped") ? child.endedAt : null,
61
+ failedAt: child.status === "failed" ? child.endedAt : null,
62
+ cancelledAt: child.status === "cancelled" ? child.endedAt : null,
63
+ data: replayResults(child.stateJson ?? {}, results),
64
+ errors: [],
65
+ awaiting: isBlockedOnGate
66
+ ? { requestPayload: child.waitJson, requestedAt: child.updatedAt, subNodeId: child.currentStep ?? null }
67
+ : null,
68
+ // Additive — the run-level equivalent of this field (routes/runs.js's
69
+ // RunSummary/RunDetail.statusReason) already reaches clients;
70
+ // per-branch rows never carried it, which is why a queued child's
71
+ // reason (e.g. "queued for admission")
72
+ // had nowhere to surface once `awaiting` was reserved for gate waits.
73
+ statusReason: child.statusReason ?? null,
74
+ _display: null,
75
+ activeSessionKey: null,
76
+ // Additive — not on the old shape. The new-model row's own flow_runs
77
+ // id, for callers that want to navigate to it directly rather than
78
+ // treat it as an inline data blob.
79
+ flowId: child.flowId,
80
+ };
81
+ }
82
+
83
+
84
+ // Returns the child-model group as `{ [subflowKey]: row }`, or null if this
85
+ // group has no real children yet.
86
+ async function getMigratedLoopGroup({ flowId, groupId, runStore, nodeExecutionsStore }) {
87
+ // groupId here is always a subflow node's own id (see the field comment in
88
+ // normalizeChildAsSubflowRow above) — listInvocationIdsForNode takes a
89
+ // nodeId param; passing groupId straight through is correct.
90
+ const invocationIds = (await nodeExecutionsStore?.listInvocationIdsForNode?.(flowId, groupId)) ?? [];
91
+ if (invocationIds.length === 0) return null;
92
+
93
+ const children = (await runStore?.listForParentInvocations?.(invocationIds)) ?? [];
94
+ // A genuine childFlows subflow-child invocation always has a matching
95
+ // flow_runs child — dispatchLoopChildren never creates an invocation
96
+ // without also spawning one (engine/executor-subflow-children.js). If
97
+ // invocationIds is non-empty but children comes back empty, these
98
+ // invocations aren't real subflow children at all — they're historical
99
+ // pre-Milestone-9 sub-node timing rows that happen to share a node_id
100
+ // with this "group" (a real risk since flow_node_invocations.group_id
101
+ // was dropped — subflow_key IS NOT NULL alone can't distinguish this
102
+ // specific case). Return null so the caller sees an empty group.
103
+ if (children.length === 0) return null;
104
+ const group = {};
105
+ for (const child of children) {
106
+ const invocation = await nodeExecutionsStore.getInvocation(child.parentInvocationId);
107
+ if (!invocation?.subflowKey) continue;
108
+ group[invocation.subflowKey] = await normalizeChildAsSubflowRow(child, invocation, nodeExecutionsStore);
109
+ }
110
+ return group;
111
+ }
112
+
113
+ export async function getAllSubflowGroups({ flowId, runStore, nodeExecutionsStore } = {}) {
114
+ const groups = {};
115
+ const groupIds = (await nodeExecutionsStore?.listSubflowNodeIdsForFlow?.(flowId)) ?? [];
116
+ for (const groupId of groupIds) {
117
+ const migrated = await getMigratedLoopGroup({ flowId, groupId, runStore, nodeExecutionsStore });
118
+ groups[groupId] = migrated ?? {};
119
+ }
120
+ return groups;
121
+ }
122
+
123
+ export async function getSubflowGroup({ flowId, groupId, runStore, nodeExecutionsStore } = {}) {
124
+ const migrated = await getMigratedLoopGroup({ flowId, groupId, runStore, nodeExecutionsStore });
125
+ return migrated ?? {};
126
+ }
@@ -0,0 +1,169 @@
1
+ // tool-handoff.js — sqlite-backed bridge between a dispatched agent's
2
+ // tool call and the dispatching node's apply(). Core owns only this registry
3
+ // and the engine side that reads it. It is also published for harness use
4
+ // (package.json "./tool-handoff").
5
+ //
6
+ // The harness turn (the plugin's run-turn.js) only ever returns
7
+ // {text, runId, toolResult?} to apply() — a turn's own reply, never a tool
8
+ // call's structured payload — so a tool has no other channel to hand
9
+ // structured data back to the node that spawned it.
10
+ //
11
+ // This is deliberately durable, not in-memory: the delivering side (an MCP
12
+ // tool call) and the consuming side (the engine's executor loop) may live
13
+ // in different processes, and an in-process Map would silently drop results
14
+ // across that boundary. A `tool_results` row keyed by the harness-stamped
15
+ // session is the rendezvous both sides can reach: the agent knows its
16
+ // session, the dispatch knows it too, neither needs the other's
17
+ // invocationId.
18
+ //
19
+ // Writes upsert (latest wins per session); the dispatch consumes its row
20
+ // and clears it, so a retry never reads a stale result from a prior
21
+ // attempt. Rows are keyed by session, not invocation: one row per session
22
+ // at most, so the table cannot grow unbounded over gateway uptime.
23
+ //
24
+ // Lifecycle (tool side: MCP deliver_result; engine side: the executor loop):
25
+ // writePendingResult({ session, runId?, agentId?, kind?, payload })
26
+ // readPendingResult(session) — the row, or null
27
+ // clearPendingResult(session) — the dispatch, once apply() has
28
+ // consumed the payload (also covers the failure path — a failed turn
29
+ // never retries into a stale row)
30
+ // waitForToolResult(session, { pollMs? }) — { promise, cancel }; the
31
+ // promise resolves with the row the moment one exists (checked once up
32
+ // front, then polled), so a result written before the waiter starts is
33
+ // never missed
34
+
35
+ import { eq } from "drizzle-orm";
36
+ import { getSharedDb, withSharedWrite } from "./db.js";
37
+ import { migrateDatabase, RUNTIME_MIGRATIONS_FOLDER } from "./migrate.js";
38
+ import { toolResults, flowRuns } from "./schema.js";
39
+ import { resolveStorePaths } from "./paths.js";
40
+ import { ensureSessionKeys } from "./session-keys.js";
41
+
42
+ // One shared client per DB path, migrated once. Concurrent first callers
43
+ // share the same promise, so the migration never races itself. Cross-
44
+ // process races resolve the same way every other store's do: each process
45
+ // migrates idempotently, then shares its own connection.
46
+ const _clients = new Map();
47
+
48
+ async function toolDb(dbPath) {
49
+ let pending = _clients.get(dbPath);
50
+ if (!pending) {
51
+ pending = (async () => {
52
+ await migrateDatabase({ dbPath, migrationsFolder: RUNTIME_MIGRATIONS_FOLDER });
53
+ return getSharedDb(dbPath);
54
+ })().catch((err) => {
55
+ _clients.delete(dbPath);
56
+ throw err;
57
+ });
58
+ _clients.set(dbPath, pending);
59
+ }
60
+ return pending;
61
+ }
62
+
63
+ function storeDbPath() {
64
+ return resolveStorePaths().runtimeDb;
65
+ }
66
+
67
+ function toRow(r) {
68
+ if (!r) return null;
69
+ let payload = null;
70
+ try { payload = JSON.parse(r.payloadJson); } catch { payload = null; }
71
+ return {
72
+ session: r.session,
73
+ runId: r.runId ?? null,
74
+ agentId: r.agentId ?? null,
75
+ kind: r.kind ?? null,
76
+ payload,
77
+ };
78
+ }
79
+
80
+ /** Latest-wins upsert for one session. No session key, no write. */
81
+ export async function writePendingResult({ session, runId = null, agentId = null, kind = null, payload, dbPath = null } = {}) {
82
+ if (!session || payload === undefined) return;
83
+ const path = dbPath ?? storeDbPath();
84
+ const { db, client } = await toolDb(path);
85
+ let payloadJson;
86
+ try { payloadJson = JSON.stringify(payload); } catch { return; }
87
+ await withSharedWrite(path, async () => {
88
+ await ensureSessionKeys(client, [session]);
89
+ // run_id carries an FK; a result may legitimately arrive for a run this
90
+ // process never saw (late delivery, resumed session, different worker).
91
+ // Keep the result and the session rendezvous, drop only the dangling
92
+ // pointer — the FK must not turn a late write into a constraint error.
93
+ let safeRunId = runId ?? null;
94
+ if (safeRunId !== null) {
95
+ const known = await db.select({ flowId: flowRuns.flowId }).from(flowRuns)
96
+ .where(eq(flowRuns.flowId, safeRunId)).limit(1);
97
+ if (known.length === 0) safeRunId = null;
98
+ }
99
+ return db.insert(toolResults).values({
100
+ session, runId: safeRunId, agentId: agentId ?? null,
101
+ kind: kind ?? null, payloadJson, updatedAt: Date.now(),
102
+ }).onConflictDoUpdate({
103
+ target: toolResults.session,
104
+ set: {
105
+ runId: safeRunId, agentId: agentId ?? null, kind: kind ?? null,
106
+ payloadJson, updatedAt: Date.now(),
107
+ },
108
+ });
109
+ });
110
+ }
111
+
112
+ /** The unconsumed result for one session, or null. */
113
+ export async function readPendingResult(session, { dbPath = null } = {}) {
114
+ if (!session) return null;
115
+ const path = dbPath ?? storeDbPath();
116
+ const { db, client } = await toolDb(path);
117
+ const rows = await db.select().from(toolResults).where(eq(toolResults.session, session)).limit(1);
118
+ return toRow(rows[0] ?? null);
119
+ }
120
+
121
+ /** Forget one session's result. Idempotent — unknown keys are a no-op. */
122
+ export async function clearPendingResult(session, { dbPath = null } = {}) {
123
+ if (!session) return;
124
+ const path = dbPath ?? storeDbPath();
125
+ const { db, client } = await toolDb(path);
126
+ await withSharedWrite(path, () =>
127
+ db.delete(toolResults).where(eq(toolResults.session, session))
128
+ );
129
+ }
130
+
131
+ // Poll-based waiter: resolves with the row the moment one exists for the
132
+ // session (checked once up front, so a result written before the waiter
133
+ // starts is never missed). Returns { promise, cancel } — the caller MUST
134
+ // call cancel() once the race is decided, or the interval keeps the event
135
+ // loop alive for nothing.
136
+ export function waitForToolResult(session, { pollMs = 250, dbPath = null } = {}) {
137
+ let timer = null;
138
+ let settled = false;
139
+ let resolveFn;
140
+ const promise = new Promise((resolve) => { resolveFn = resolve; });
141
+ const finish = (row) => {
142
+ if (settled) return;
143
+ settled = true;
144
+ if (timer) clearInterval(timer);
145
+ resolveFn(row);
146
+ };
147
+ const cancel = () => {
148
+ if (settled) return;
149
+ settled = true;
150
+ if (timer) clearInterval(timer);
151
+ };
152
+ (async () => {
153
+ try {
154
+ const first = await readPendingResult(session, { dbPath });
155
+ if (first) { finish(first); return; }
156
+ timer = setInterval(async () => {
157
+ try {
158
+ const row = await readPendingResult(session, { dbPath });
159
+ if (row) finish(row);
160
+ } catch { /* transient read failure: keep polling */ }
161
+ }, Math.max(10, pollMs));
162
+ if (timer.unref) timer.unref();
163
+ } catch {
164
+ // Store unavailable: leave the promise pending; the waiter's own
165
+ // race (waitForRun) still settles the dispatch either way.
166
+ }
167
+ })();
168
+ return { promise, cancel };
169
+ }