@igorjpimenta/manifold 0.0.0-stage → 2026.11.22-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/.migrations/ops/0000_futuristic_karma.sql +111 -0
  2. package/.migrations/ops/0001_low_maximus.sql +7 -0
  3. package/.migrations/ops/meta/0000_snapshot.json +763 -0
  4. package/.migrations/ops/meta/0001_snapshot.json +173 -0
  5. package/.migrations/ops/meta/_journal.json +20 -0
  6. package/.migrations/runtime/0000_real_boomer.sql +180 -0
  7. package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
  8. package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
  9. package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
  10. package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
  11. package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
  12. package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
  13. package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
  14. package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
  15. package/.migrations/runtime/0009_lying_shocker.sql +96 -0
  16. package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
  17. package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
  18. package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
  19. package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
  20. package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
  21. package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
  22. package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
  23. package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
  24. package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
  25. package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
  26. package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
  27. package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
  28. package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
  29. package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
  30. package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
  31. package/.migrations/runtime/meta/_journal.json +97 -0
  32. package/README.md +1583 -2
  33. package/RELEASING.md +89 -0
  34. package/agent/fleet.js +100 -0
  35. package/bin/cli.js +854 -0
  36. package/bin/manifold.js +4 -0
  37. package/boot/flow-revival.js +536 -0
  38. package/budget/index.js +495 -0
  39. package/budget/lib.js +59 -0
  40. package/dashboard/cost.js +528 -0
  41. package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
  42. package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
  43. package/dashboard/dist/index.html +13 -0
  44. package/dashboard/event-log.js +267 -0
  45. package/dashboard/index.html +12 -0
  46. package/dashboard/package-lock.json +2877 -0
  47. package/dashboard/package.json +37 -0
  48. package/dashboard/pipeline-graph.js +131 -0
  49. package/dashboard/postcss.config.js +6 -0
  50. package/dashboard/search.js +183 -0
  51. package/dashboard/server.js +1015 -0
  52. package/dashboard/src/App.tsx +42 -0
  53. package/dashboard/src/api/client.ts +118 -0
  54. package/dashboard/src/api/types.ts +510 -0
  55. package/dashboard/src/api/ws.tsx +112 -0
  56. package/dashboard/src/components/CancelButton.tsx +63 -0
  57. package/dashboard/src/components/ClickToCopy.tsx +68 -0
  58. package/dashboard/src/components/CopyButton.tsx +61 -0
  59. package/dashboard/src/components/DailyCostTrend.tsx +85 -0
  60. package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
  61. package/dashboard/src/components/Funnel.tsx +149 -0
  62. package/dashboard/src/components/GateForm.tsx +158 -0
  63. package/dashboard/src/components/GraphView.tsx +302 -0
  64. package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
  65. package/dashboard/src/components/Histogram.tsx +292 -0
  66. package/dashboard/src/components/Layout.tsx +28 -0
  67. package/dashboard/src/components/NeedsAttention.tsx +98 -0
  68. package/dashboard/src/components/NodeContextCard.tsx +132 -0
  69. package/dashboard/src/components/PathTrail.tsx +35 -0
  70. package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
  71. package/dashboard/src/components/RerunButton.tsx +46 -0
  72. package/dashboard/src/components/RunComparison.tsx +53 -0
  73. package/dashboard/src/components/Sparkline.tsx +94 -0
  74. package/dashboard/src/components/StatStrip.tsx +147 -0
  75. package/dashboard/src/components/StateTree.tsx +84 -0
  76. package/dashboard/src/components/StatusBadge.tsx +18 -0
  77. package/dashboard/src/components/SubflowPathPills.tsx +38 -0
  78. package/dashboard/src/components/SubflowSummary.tsx +24 -0
  79. package/dashboard/src/components/SubflowsCard.tsx +327 -0
  80. package/dashboard/src/components/TodayStrip.tsx +86 -0
  81. package/dashboard/src/components/TranscriptPane.tsx +99 -0
  82. package/dashboard/src/components/TranscriptRow.tsx +69 -0
  83. package/dashboard/src/components/ui/badge.tsx +38 -0
  84. package/dashboard/src/components/ui/button.tsx +28 -0
  85. package/dashboard/src/components/ui/card.tsx +42 -0
  86. package/dashboard/src/components/ui/skeleton.tsx +9 -0
  87. package/dashboard/src/components/ui/table.tsx +62 -0
  88. package/dashboard/src/index.css +26 -0
  89. package/dashboard/src/lib/anomaly.ts +42 -0
  90. package/dashboard/src/lib/format.ts +30 -0
  91. package/dashboard/src/lib/step-history.ts +13 -0
  92. package/dashboard/src/lib/subflow.ts +154 -0
  93. package/dashboard/src/lib/use-click-outside.ts +28 -0
  94. package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
  95. package/dashboard/src/lib/utils.ts +6 -0
  96. package/dashboard/src/main.tsx +10 -0
  97. package/dashboard/src/types/shims.d.ts +4 -0
  98. package/dashboard/src/views/Home.tsx +478 -0
  99. package/dashboard/src/views/PipelineDetail.tsx +1185 -0
  100. package/dashboard/src/views/PipelinesList.tsx +111 -0
  101. package/dashboard/src/views/RunDetail.tsx +248 -0
  102. package/dashboard/src/views/Search.tsx +196 -0
  103. package/dashboard/stats.js +668 -0
  104. package/dashboard/tailwind.config.ts +43 -0
  105. package/dashboard/tree-stats.js +12 -0
  106. package/dashboard/tsconfig.json +25 -0
  107. package/dashboard/views.js +231 -0
  108. package/dashboard/vite.config.ts +34 -0
  109. package/drizzle.config.js +12 -0
  110. package/drizzle.config.ops.js +12 -0
  111. package/elevation.js +116 -0
  112. package/engine/admission-promotion-resume.js +127 -0
  113. package/engine/agent-dispatch.js +64 -0
  114. package/engine/child-resume-trigger.js +315 -0
  115. package/engine/executor-budget.js +87 -0
  116. package/engine/executor-cancel.js +83 -0
  117. package/engine/executor-errors.js +122 -0
  118. package/engine/executor-helpers.js +254 -0
  119. package/engine/executor-nodes.js +292 -0
  120. package/engine/executor-persist.js +212 -0
  121. package/engine/executor-subflow-children.js +275 -0
  122. package/engine/executor.js +1013 -0
  123. package/engine/global-concurrency.js +33 -0
  124. package/engine/loader.js +528 -0
  125. package/engine/needs-resolver.js +229 -0
  126. package/engine/pipelines-watcher.js +133 -0
  127. package/engine/result-replay.js +34 -0
  128. package/engine/revival-messages.js +11 -0
  129. package/engine/spawn-child.js +55 -0
  130. package/engine/wait-for-flow-settlement.js +109 -0
  131. package/events/catalog.js +26 -0
  132. package/events/emit.js +44 -0
  133. package/events/kinds.js +118 -0
  134. package/events/serve-ws.js +74 -0
  135. package/events/stream.js +35 -0
  136. package/events/ws-dialog.js +257 -0
  137. package/events/ws-frame.js +194 -0
  138. package/events/ws-run-events.js +70 -0
  139. package/events.js +32 -0
  140. package/gates/gate-resolver.js +576 -0
  141. package/gates/paused-flows.js +55 -0
  142. package/graphql/columns.js +67 -0
  143. package/graphql/schema.js +333 -0
  144. package/index.js +49 -0
  145. package/mcp/http.js +159 -0
  146. package/mcp/identity.js +94 -0
  147. package/mcp/stdio.js +80 -0
  148. package/mcp/tools.js +245 -0
  149. package/mcp/wire-ops.js +178 -0
  150. package/package.json +49 -4
  151. package/routes/bearer-auth.js +57 -0
  152. package/routes/discuss-audio.js +87 -0
  153. package/routes/discuss-component.js +39 -0
  154. package/routes/discuss-draft.js +159 -0
  155. package/routes/discuss-helpers.js +20 -0
  156. package/routes/discuss.js +848 -0
  157. package/routes/failures.js +104 -0
  158. package/routes/graphql.js +61 -0
  159. package/routes/health.js +54 -0
  160. package/routes/operator-discuss.js +193 -0
  161. package/routes/operator-resume.js +32 -0
  162. package/routes/operator-retry.js +301 -0
  163. package/routes/operator.js +539 -0
  164. package/routes/pipelines.js +118 -0
  165. package/routes/runs.js +1129 -0
  166. package/scripts/ci/import-check.mjs +50 -0
  167. package/scripts/link-core-self.mjs +31 -0
  168. package/scripts/with-guard.sh +24 -0
  169. package/server/app.js +503 -0
  170. package/server/registry.js +53 -0
  171. package/server/serve.js +163 -0
  172. package/state/admission-queue-store.js +325 -0
  173. package/state/attribution-registries.js +102 -0
  174. package/state/batch-writer.js +162 -0
  175. package/state/db-guard.js +63 -0
  176. package/state/db-health.js +27 -0
  177. package/state/db.js +122 -0
  178. package/state/discuss-store.js +299 -0
  179. package/state/failure-store.js +427 -0
  180. package/state/migrate.js +104 -0
  181. package/state/node-cost.js +67 -0
  182. package/state/node-executions.js +775 -0
  183. package/state/node-health-store.js +124 -0
  184. package/state/ops-merge.js +93 -0
  185. package/state/paths.js +146 -0
  186. package/state/pipeline-dirs.js +99 -0
  187. package/state/pipeline-registry.js +449 -0
  188. package/state/run-store.js +697 -0
  189. package/state/schema.js +438 -0
  190. package/state/schema.ops.js +44 -0
  191. package/state/session-index.js +177 -0
  192. package/state/session-keys.js +39 -0
  193. package/state/state-store.js +311 -0
  194. package/state/subflow-state.js +126 -0
  195. package/state/tool-handoff.js +169 -0
  196. package/telemetry/budget-alert.js +353 -0
  197. package/telemetry/cli.js +331 -0
  198. package/telemetry/grammar.js +119 -0
  199. package/telemetry/http-budget.js +132 -0
  200. package/telemetry/http-cost.js +670 -0
  201. package/telemetry/http-helpers.js +134 -0
  202. package/telemetry/http-tree.js +349 -0
  203. package/telemetry/http.js +44 -0
  204. package/telemetry/index.js +307 -0
  205. package/telemetry/ingest.js +134 -0
  206. package/telemetry/labels.js +115 -0
  207. package/telemetry/pricer.js +77 -0
  208. package/telemetry/resolver.js +314 -0
  209. package/telemetry/store.js +147 -0
  210. package/utils/run-patch.js +43 -0
@@ -0,0 +1,427 @@
1
+ // failure-store.js — durable record of flow / branch failures, for
2
+ // operator triage via a consumer's Failures surface + deep links.
3
+ //
4
+ // Storage: the store home's runtime.sqlite (see defaultDbPath()).
5
+ //
6
+ // Why a separate db file rather than another table in registry.sqlite:
7
+ // schema isolation. failure_events is queried independently from
8
+ // flow_runs (the dashboard's /api/failures route doesn't need flow state,
9
+ // just the snapshot we stamp here). One file = one writer story; no
10
+ // cross-table transactions to coordinate.
11
+ //
12
+ // Schema: this table is deliberately thin — id, flow_id, failed_at,
13
+ // failure_summary, invocation_id, execution_id, error_type. Everything else
14
+ // a caller might want (pipeline_id, branch_item_id, failure_node,
15
+ // recovery_hint, whether it's currently resolved) is reconstructed at READ
16
+ // time via deriveFailureContext, joining through invocation_id/execution_id
17
+ // to flow_node_invocations/flow_node_executions — the engine's own
18
+ // node-dispatch identity system, which this table deliberately does NOT
19
+ // duplicate in parallel.
20
+ //
21
+ // Identity: execution_id is the real conflict target now (partial UNIQUE,
22
+ // WHERE execution_id IS NOT NULL) — a flow whose retry path re-enters and
23
+ // re-fails the same node updates the existing row instead of piling up
24
+ // duplicates. The handful of structurally-unknown-dispatch failures (no
25
+ // execution_id at all — pipeline never loaded, dead edge, unknown next-node
26
+ // id) fall back to a manual dedup-by-flow_id (see upsertFailureRow) —
27
+ // SQLite's NULL != NULL unique-index semantics mean the DB-level conflict
28
+ // target can't cover this case on its own.
29
+ //
30
+ // error_type — the sole classification axis now. A node's own thrown
31
+ // NodeFailure carries one of: never_attempted | confirmed_absent |
32
+ // inconclusive | infra_unavailable | blocked. A subflow node whose exit
33
+ // mode wasn't satisfied gets "subflow-mode-not-satisfied" instead — not a
34
+ // thrown error at all, a policy verdict computed by
35
+ // engine/child-resume-trigger.js once every spawned child has gone
36
+ // terminal. null means an ordinary engine-caught throw with no further
37
+ // node-level classification (the common case — still safe to retry).
38
+
39
+ import { randomUUID } from "node:crypto";
40
+ import { eq } from "drizzle-orm";
41
+ import { getSharedDb, withSharedWrite, resolveStorePaths } from "./db.js";
42
+ import { flowRuns } from "./schema.js";
43
+ import path from "node:path";
44
+ import os from "node:os";
45
+ import { mkdirSync } from "node:fs";
46
+ import { migrateDatabase, RUNTIME_MIGRATIONS_FOLDER } from "./migrate.js";
47
+ import { failureEmitter } from "../events.js";
48
+ import { getNodeExecutionsStore } from "./node-executions.js";
49
+
50
+ function defaultDbPath() {
51
+ return resolveStorePaths().runtimeDb;
52
+ }
53
+
54
+ // Maps a failure's (error_type, summary) to the operator action offered for it
55
+ // Inbox should surface for this row:
56
+ // retry-step — system-recoverable; one-tap retry the failed step
57
+ // skip-only — work is moot (already done, listing closed); skip removes
58
+ // investigate — unknown / engineer-level, or a policy-level verdict with
59
+ // no single thrown error to point at; operator looks
60
+ // before acting
61
+ //
62
+ // error_type is null for the common case — an ordinary engine-caught throw
63
+ // with no further node-level classification — which is still a plain
64
+ // "just retry it" situation. "blocked" is the one error_type whose summary
65
+ // text actually distinguishes a genuinely different case (a listing
66
+ // already closed/awarded — "workana-forbidden" — vs. an unrecognized
67
+ // block, which stays investigate rather than guessing). "redraft"/"wait"
68
+ // aren't reachable today: nothing currently sets an error_type/summary
69
+ // combination that would justify either — see git history if a producer
70
+ // for either ever comes back.
71
+ //
72
+ // Computed fresh at read time (see deriveFailureContext) — not stored.
73
+ function classifyRecoveryHint(errorType, summary) {
74
+ if (errorType == null) return "retry-step";
75
+ if (errorType === "infra_unavailable") return "retry-step";
76
+ if (errorType === "blocked") {
77
+ const s = String(summary ?? "").toLowerCase();
78
+ return s.includes("workana-forbidden") ? "skip-only" : "investigate";
79
+ }
80
+ // confirmed_absent, inconclusive, never_attempted, subflow-mode-not-
81
+ // satisfied, and any future error_type this hasn't been taught yet —
82
+ // safer to over-surface as "needs a look" than to silently misroute.
83
+ return "investigate";
84
+ }
85
+
86
+ function summarize(s, maxLen = 280) {
87
+ if (typeof s !== "string") s = String(s ?? "");
88
+ s = s.replace(/\s+/g, " ").trim();
89
+ return s.length > maxLen ? s.slice(0, maxLen - 1) + "…" : s;
90
+ }
91
+
92
+ // ─── Public API ─────────────────────────────────────────────────────────
93
+
94
+ export function createFailureStore(opts = {}) {
95
+ const _p = _createFailureStoreImpl(opts);
96
+ return new Proxy({}, { get(_, k) {
97
+ if (k === "then" || typeof k === "symbol") return undefined;
98
+ if (k === "_db") return new Proxy({}, { get(__, method) {
99
+ if (method === "then" || typeof method === "symbol") return undefined;
100
+ return (...a) => _p.then(s => s._db[method](...a));
101
+ } });
102
+ return (...a) => _p.then(s => { if (typeof s[k] !== "function") throw new TypeError(`failure-store: ${String(k)} is not a function`); return s[k](...a); });
103
+ } });
104
+ }
105
+
106
+ async function _createFailureStoreImpl({ dbPath = defaultDbPath(), logger, nodeExecutionsStore, emit } = {}) {
107
+ // Injectable for tests (defaults to the real singleton in production) —
108
+ // same DI pattern as engine/executor.js.
109
+ const nodeExecutions = nodeExecutionsStore ?? await getNodeExecutionsStore();
110
+ mkdirSync(path.dirname(dbPath), { recursive: true });
111
+ await migrateDatabase({ dbPath, migrationsFolder: RUNTIME_MIGRATIONS_FOLDER });
112
+ const { db, client } = await getSharedDb(dbPath);
113
+
114
+ // ── Write-side identity ────────────────────────────────────────────
115
+ // execution_id is the real conflict target (partial UNIQUE index,
116
+ // v12). When there's no execution_id to anchor to (a handful of
117
+ // structurally-unknown-dispatch engine failures — pipeline never
118
+ // loaded, dead edge, unknown next-node id — or a soft-marker whose
119
+ // best-effort execution lookup came up empty), fall back to a manual
120
+ // dedup by flow_id: SQLite's NULL != NULL unique-index semantics mean
121
+ // the DB-level conflict target can't cover a null execution_id on its
122
+ // own.
123
+ // Returns the row's id on a successful write, null on no-op — the id lets
124
+ // callers re-fetch the fully-derived row (pipeline_id/branch_item_id/
125
+ // failure_node/recovery_hint) before emitting "failure.created", instead
126
+ // of emitting the bare write-time fields the push/WS consumers don't
127
+ // understand.
128
+ // Wrapped in withSharedWrite (state/db.js) — every other writer sharing
129
+ // this same runtime.sqlite connection (run-store.js, admission-queue-
130
+ // store.js) already routes through it, and a bare client.execute() here
131
+ // can throw SQLITE_BUSY into a CONCURRENT writer's own transaction rather
132
+ // than just itself.
133
+ async function upsertFailureRow({ flowId, failedAt, failureSummary, invocationId, executionId, errorType = null }) {
134
+ return withSharedWrite(dbPath, async () => {
135
+ if (executionId) {
136
+ const r = await client.execute({
137
+ sql: `
138
+ INSERT INTO failure_events (id, flow_id, failed_at, failure_summary, invocation_id, execution_id, error_type)
139
+ VALUES (?, ?, ?, ?, ?, ?, ?)
140
+ ON CONFLICT(execution_id) WHERE execution_id IS NOT NULL DO UPDATE SET
141
+ failed_at = excluded.failed_at,
142
+ failure_summary = excluded.failure_summary,
143
+ invocation_id = excluded.invocation_id,
144
+ error_type = excluded.error_type
145
+ `,
146
+ args: [randomUUID(), flowId, failedAt, failureSummary, invocationId ?? null, executionId, errorType ?? null],
147
+ });
148
+ if (r.rowsAffected === 0) return null;
149
+ const { rows } = await client.execute({ sql: "SELECT id FROM failure_events WHERE execution_id = ?", args: [executionId] });
150
+ return rows[0]?.id ?? null;
151
+ }
152
+ const existing = await client.execute({
153
+ sql: "SELECT id FROM failure_events WHERE flow_id = ? AND execution_id IS NULL LIMIT 1",
154
+ args: [flowId],
155
+ });
156
+ if (existing.rows.length > 0) {
157
+ const r = await client.execute({
158
+ sql: "UPDATE failure_events SET failed_at = ?, failure_summary = ?, invocation_id = ?, error_type = ? WHERE id = ?",
159
+ args: [failedAt, failureSummary, invocationId ?? null, errorType ?? null, existing.rows[0].id],
160
+ });
161
+ return r.rowsAffected > 0 ? existing.rows[0].id : null;
162
+ }
163
+ const newId = randomUUID();
164
+ const r = await client.execute({
165
+ sql: `INSERT INTO failure_events (id, flow_id, failed_at, failure_summary, invocation_id, execution_id, error_type)
166
+ VALUES (?, ?, ?, ?, ?, NULL, ?)`,
167
+ args: [newId, flowId, failedAt, failureSummary, invocationId ?? null, errorType ?? null],
168
+ });
169
+ return r.rowsAffected > 0 ? newId : null;
170
+ });
171
+ }
172
+
173
+ // ── Read-side derivation ────────────────────────────────────────────
174
+ // Reconstructs everything NOT stored directly on the row:
175
+ // pipeline_id (via flow_id -> flow_runs, always available), branch_item_id
176
+ // / failure_node (via invocation_id -> flow_node_invocations, only when a
177
+ // real dispatch happened), recovery_hint (pure function, computed fresh),
178
+ // and whether this failure currently reads as resolved.
179
+ async function deriveFailureContext(row) {
180
+ let pipelineId = null;
181
+ try {
182
+ const flowRows = await db.select({ pipelineId: flowRuns.pipelineId }).from(flowRuns).where(eq(flowRuns.flowId, row.flow_id)).limit(1);
183
+ pipelineId = flowRows[0]?.pipelineId ?? null;
184
+ } catch { /* best-effort */ }
185
+
186
+ let failureNode = null, inv = null;
187
+ if (row.invocation_id) {
188
+ inv = await nodeExecutions.getInvocation(row.invocation_id);
189
+ if (inv) failureNode = inv.nodeId ?? null;
190
+ }
191
+
192
+ // branchItemId ("which item, if this failure happened inside a
193
+ // subflow's spawned child") almost never lives on `inv` directly —
194
+ // see resolveItemInvocation's own comment for why — so it's derived
195
+ // via the same parent-invocation walk, purely for display now
196
+ // (computeResolvedAt no longer uses item identity for resolution).
197
+ const itemInv = await resolveItemInvocation(row, inv);
198
+ const branchItemId = itemInv?.subflowKey ?? null;
199
+
200
+ const recoveryHint = classifyRecoveryHint(row.error_type, row.failure_summary);
201
+ const resolvedAt = await computeResolvedAt(row);
202
+
203
+ return {
204
+ id: row.id,
205
+ flow_id: row.flow_id,
206
+ pipeline_id: pipelineId,
207
+ branch_item_id: branchItemId,
208
+ failed_at: Number(row.failed_at),
209
+ failure_node: failureNode,
210
+ failure_summary: row.failure_summary,
211
+ recovery_hint: recoveryHint,
212
+ resolved_at: resolvedAt,
213
+ invocation_id: row.invocation_id,
214
+ execution_id: row.execution_id,
215
+ error_type: row.error_type ?? null,
216
+ };
217
+ }
218
+
219
+ // Resolves the "item identity" (pipelineId/nodeId/subflowKey of the
220
+ // subflow dispatch this failure's flow was spawned FOR) used by both
221
+ // deriveFailureContext's branch_item_id display field and
222
+ // computeResolvedAt's cross-flow supersession check below.
223
+ //
224
+ // Almost never on the failing node's OWN invocation: engine/executor.js's
225
+ // ordinary dispatch loop always writes subflowKey:null (every real
226
+ // failure site — SUBMIT_CODE, SCORE_LLM, a subflow's own inner nodes —
227
+ // runs INSIDE a subflow's spawned child flow, not as the subflow node
228
+ // itself). The tag lives on the PARENT's own invocation of the subflow
229
+ // node that spawned this failure's flow, reachable via this flow's own
230
+ // flow_runs.parent_invocation_id — the exact lookup child-resume-
231
+ // trigger.js's maybeResumeParent already does to find its own parent.
232
+ // `inv` (the failing node's own invocation) is checked first only for
233
+ // the (currently theoretical) case of a node whose own invocation
234
+ // somehow already carries a subflowKey directly.
235
+ async function resolveItemInvocation(row, inv) {
236
+ if (inv?.subflowKey) return inv;
237
+ try {
238
+ const childRows = await db.select({ parentInvocationId: flowRuns.parentInvocationId }).from(flowRuns).where(eq(flowRuns.flowId, row.flow_id)).limit(1);
239
+ const parentInvocationId = childRows[0]?.parentInvocationId ?? null;
240
+ if (parentInvocationId) {
241
+ const parentInv = await nodeExecutions.getInvocation(parentInvocationId);
242
+ if (parentInv?.subflowKey) return parentInv;
243
+ }
244
+ } catch { /* best-effort */ }
245
+ return null;
246
+ }
247
+
248
+ // A failure resolves only via same-flow completion: this exact flow_id
249
+ // reached a clean terminal status (succeeded/skipped) despite this
250
+ // failure. Applies identically to root and subflow-child flows — no
251
+ // special-cased mechanism for children. Returns a timestamp (ms) when
252
+ // resolved, null when still open — not just a boolean, since the Done
253
+ // section sorts by "most recently resolved" and windows it to the
254
+ // last 7 days.
255
+ //
256
+ // Do not extend resolution to cross-flow supersession — matching a
257
+ // LATER invocation of the same (pipeline_id, node_id, subflow_key)
258
+ // tuple in ANY flow:
259
+ // subflow_key isn't always a real per-item identity — single-item
260
+ // retry-cascade nodes use a constant literal key for every dispatch —
261
+ // so tuple matching collapses distinct items and can falsely mark one
262
+ // job's failure resolved because a different job's dispatch of the same
263
+ // node succeeded later. Every flow's own failure stays open until that
264
+ // exact flow_id itself resolves.
265
+ async function computeResolvedAt(row) {
266
+ try {
267
+ const flowRows = await db.select({ status: flowRuns.status, updatedAt: flowRuns.updatedAt }).from(flowRuns).where(eq(flowRuns.flowId, row.flow_id)).limit(1);
268
+ const flow = flowRows[0];
269
+ if (flow && (flow.status === "succeeded" || flow.status === "skipped")) return Number(flow.updatedAt);
270
+ } catch { /* best-effort */ }
271
+ return null;
272
+ }
273
+
274
+ // Failures persist to failure_events directly from the engine's
275
+ // throw-catch paths (recordEngineFailure); no terminal-state soft-marker
276
+ // scan feeds this table.
277
+
278
+ // ── Reads ──────────────────────────────────────────────────────────
279
+ // `resolved` is computed at read time now (see deriveFailureContext),
280
+ // not a stored column — filtering happens in JS after derivation, not
281
+ // in the SQL WHERE clause. At current table scale (low thousands, no
282
+ // pagination anywhere else on this table either) this is an acceptable
283
+ // tradeoff for the correctness win of not duplicating derivable state;
284
+ // revisit if this table's row count grows enough for per-row derivation
285
+ // cost to matter.
286
+ // undefined / false → only open failures (the default "Recover" view)
287
+ // true → only resolved failures (the "Done" view)
288
+ // null → both
289
+ // `flowId` — the cascade-retry walk (routes/operator-retry.js) needs
290
+ // "this specific flow's own open failure, with its real id/error_type" —
291
+ // queryErrorsByFlow returns neither (ErrorEntry shape, for display/
292
+ // errorNodes membership only), so this is the reuse point rather than a
293
+ // bespoke third read method.
294
+ async function list({ resolved, limit = 50, since, flowId } = {}) {
295
+ const where = [];
296
+ const args = [];
297
+ if (since != null) { where.push("failed_at >= ?"); args.push(since); }
298
+ if (flowId != null) { where.push("flow_id = ?"); args.push(flowId); }
299
+ const sqlText = `
300
+ SELECT * FROM failure_events
301
+ ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
302
+ ORDER BY failed_at DESC
303
+ `;
304
+ const { rows } = await client.execute({ sql: sqlText, args });
305
+ const withContext = await Promise.all(rows.map((r) => deriveFailureContext(r)));
306
+ let filtered = withContext;
307
+ if (resolved === undefined || resolved === false) filtered = withContext.filter((r) => r.resolved_at == null);
308
+ else if (resolved === true) filtered = withContext.filter((r) => r.resolved_at != null);
309
+ return filtered.slice(0, limit);
310
+ }
311
+
312
+ async function get(id) {
313
+ const { rows } = await client.execute({ sql: "SELECT * FROM failure_events WHERE id = ?", args: [id] });
314
+ const row = rows[0] ?? null;
315
+ return row ? await deriveFailureContext(row) : null;
316
+ }
317
+
318
+ // Maps a joined failure_events+flow_node_invocations row to the
319
+ // canonical ErrorEntry shape shared across every consumer.
320
+ function _toErrorEntry(row) {
321
+ return {
322
+ nodeId: row.node_id ?? null,
323
+ subflowKey: row.subflow_key ?? null,
324
+ message: row.failure_summary,
325
+ errorType: row.error_type ?? null,
326
+ at: Number(row.failed_at),
327
+ executionId: row.execution_id ?? null,
328
+ invocationId: row.invocation_id ?? null,
329
+ };
330
+ }
331
+
332
+ // flow_node_executions no longer stores error content at all, so "which
333
+ // errors happened on this flow" can only be answered from failure_events
334
+ // now. Same ErrorEntry shape, same "every error for the flow, caller
335
+ // filters by subflowKey if it wants root-level only" contract.
336
+ async function queryErrorsByFlow(flowId) {
337
+ const { rows } = await client.execute({
338
+ sql: `SELECT f.failed_at, f.failure_summary, f.error_type, f.execution_id, f.invocation_id, i.node_id, i.subflow_key
339
+ FROM failure_events f LEFT JOIN flow_node_invocations i ON f.invocation_id = i.id
340
+ WHERE f.flow_id = ?
341
+ ORDER BY f.failed_at ASC`,
342
+ args: [flowId],
343
+ });
344
+ return rows.map(_toErrorEntry);
345
+ }
346
+
347
+ async function queryErrorsByFlowIds(flowIds) {
348
+ if (!Array.isArray(flowIds) || flowIds.length === 0) return new Map();
349
+ const placeholders = flowIds.map(() => "?").join(", ");
350
+ const { rows } = await client.execute({
351
+ sql: `SELECT f.flow_id, f.failed_at, f.failure_summary, f.error_type, f.execution_id, f.invocation_id, i.node_id, i.subflow_key
352
+ FROM failure_events f LEFT JOIN flow_node_invocations i ON f.invocation_id = i.id
353
+ WHERE f.flow_id IN (${placeholders})
354
+ ORDER BY f.failed_at ASC`,
355
+ args: flowIds,
356
+ });
357
+ const out = new Map();
358
+ for (const r of rows) {
359
+ if (!out.has(r.flow_id)) out.set(r.flow_id, []);
360
+ out.get(r.flow_id).push(_toErrorEntry(r));
361
+ }
362
+ return out;
363
+ }
364
+
365
+ // ── Mutations ──────────────────────────────────────────────────────
366
+ // recordRetry/markIgnored/markUnignored/resolveOpenForBranch and the
367
+ // Skip action they supported are gone. retry_count/
368
+ // last_retried_at are derivable (COUNT/MAX over flow_node_executions
369
+ // WHERE kind='retry' for the invocation) but confirmed not wanted in
370
+ // the UI at all, so no replacement is computed here either. Ignore
371
+ // state moved to client-side device-local storage entirely.
372
+
373
+ // Engine-failed recorder — called both when a flow definitively fails
374
+ // (node threw with no cascade left to try, edge resolved to nothing,
375
+ // unknown next-node id, pipeline-not-loaded) AND, as of Phase 1.5,
376
+ // immediately on every cascade-leg throw regardless of whether the
377
+ // cascade goes on to retry/escalate/succeed — every throw gets a
378
+ // permanent row here, no exceptions. errorType, when known, is the
379
+ // node's own domain-specific reason (never_attempted|confirmed_absent|
380
+ // inconclusive|infra_unavailable|blocked), or "subflow-mode-not-
381
+ // satisfied" for the one non-throw producer (engine/child-resume-
382
+ // trigger.js). null means an ordinary engine-caught throw with no
383
+ // further classification — still safe to retry. computeResolvedAt
384
+ // answers "did this ultimately resolve" separately, at read time.
385
+ async function recordEngineFailure({ flowId, pipelineId, failureNode, message, state, invocationId = null, executionId = null, errorType = null }) {
386
+ if (!flowId || !pipelineId) return { recorded: 0 };
387
+ const now = Date.now();
388
+ const summary = summarize(message ?? "engine-failed (no message)");
389
+ try {
390
+ const insertedId = await upsertFailureRow({
391
+ flowId, failedAt: now, failureSummary: summary,
392
+ invocationId, executionId, errorType,
393
+ });
394
+ if (insertedId) {
395
+ logger?.info?.(`manifold failure-store: ${errorType ?? "engine-failed"} ${flowId.slice(0, 8)} (${pipelineId}) at ${failureNode || "<engine>"}`);
396
+ const context = await get(insertedId);
397
+ if (context) failureEmitter.emit("failure.created", context);
398
+ // Fire-and-forget: delivery must never fail the recording.
399
+ try {
400
+ await emit?.("failure.recorded", {
401
+ failureId: insertedId,
402
+ flowId,
403
+ pipelineId: pipelineId ?? null,
404
+ failureNode: failureNode ?? null,
405
+ failureSummary: summary,
406
+ errorType: errorType ?? null,
407
+ });
408
+ } catch {}
409
+ return { recorded: 1 };
410
+ }
411
+ return { recorded: 0 };
412
+ } catch (err) {
413
+ logger?.warn?.(`manifold failure-store: recordEngineFailure failed: ${err?.message ?? err}`);
414
+ return { recorded: 0 };
415
+ }
416
+ }
417
+
418
+ return {
419
+ recordEngineFailure,
420
+ list,
421
+ get,
422
+ queryErrorsByFlow,
423
+ queryErrorsByFlowIds,
424
+ close() { try { client.close(); } catch { /* ignore */ } },
425
+ _db: client, // for tests
426
+ };
427
+ }
@@ -0,0 +1,104 @@
1
+ // state/migrate.js — better-sqlite3 migrate-on-open, exactly winch's src/db.js
2
+ // shape: schema DDL lives in state/schema.js (runtime.sqlite) and
3
+ // state/schema.ops.js (ops.sqlite); `npm run db:generate` snapshots them
4
+ // into .migrations/*/<n>.sql; drizzle's migrate() applies pending snapshots.
5
+ //
6
+ // Call sites run this BEFORE opening any libsql handle for the file, and it
7
+ // runs at most once per file per process (migratedPaths): a second driver
8
+ // writing under live handles is the one ordering that was observed to
9
+ // poison them, so the first opener applies pending snapshots while no
10
+ // handle exists yet and later opens find the journal current. Cross-process
11
+ // safety comes from the drizzle journal transaction + busy_timeout, same
12
+ // posture as winch.
13
+ //
14
+ // Snapshots are the sole schema path. A snapshot that fails throws here,
15
+ // failing the open fast with the file named, instead of surfacing as
16
+ // missing-column errors on first query later.
17
+ //
18
+ // The 0000 baseline files carry hand-added IF NOT EXISTS on every CREATE
19
+ // (one-time anchor edit — drizzle-kit emits plain CREATE). Future
20
+ // snapshots must keep that property; the snapshot hygiene tests fail the
21
+ // build otherwise. Never hand-edit snapshots to add DROP/RENAME.
22
+ import path from "node:path";
23
+ import { existsSync, mkdirSync } from "node:fs";
24
+ import { fileURLToPath } from "node:url";
25
+ import Database from "better-sqlite3";
26
+ import { drizzle } from "drizzle-orm/better-sqlite3";
27
+ import { migrate } from "drizzle-orm/better-sqlite3/migrator";
28
+ import * as runtimeSchema from "./schema.js";
29
+ import * as opsSchema from "./schema.ops.js";
30
+ import { mergeOpsTables } from "./ops-merge.js";
31
+ import { SESSION_KEYS_FROM_ROWS_SQL } from "./session-keys.js";
32
+ import { BACKFILL_ATTRIBUTION_SQL, BACKFILL_LEGACY_BRANCH_ITEMS_SQL } from "./attribution-registries.js";
33
+
34
+ const HERE = path.dirname(fileURLToPath(import.meta.url));
35
+ const REPO_ROOT = path.join(HERE, "..");
36
+
37
+ export const RUNTIME_MIGRATIONS_FOLDER = path.join(REPO_ROOT, ".migrations", "runtime");
38
+ export const OPS_MIGRATIONS_FOLDER = path.join(REPO_ROOT, ".migrations", "ops");
39
+
40
+ const SCHEMAS = new Map([
41
+ [RUNTIME_MIGRATIONS_FOLDER, runtimeSchema],
42
+ [OPS_MIGRATIONS_FOLDER, opsSchema],
43
+ ]);
44
+
45
+ const migratedPaths = new Set();
46
+
47
+ export function migrateDatabase({ dbPath, migrationsFolder, logger } = {}) {
48
+ if (migratedPaths.has(dbPath)) return { applied: false };
49
+ // ops/0001 drops the tables that moved to runtime.sqlite, so the copy has
50
+ // to exist first. The runtime sibling is migrated here — including on an
51
+ // install where only ops exists — because that run is what performs the
52
+ // merge and the registry backfill. Non-canonical paths (tests, one-off
53
+ // stores) are left alone: they have no sibling to copy from.
54
+ if (migrationsFolder === OPS_MIGRATIONS_FOLDER && path.basename(dbPath) === "ops.sqlite") {
55
+ const runtimeDbPath = path.join(path.dirname(dbPath), "runtime.sqlite");
56
+ migrateDatabase({ dbPath: runtimeDbPath, migrationsFolder: RUNTIME_MIGRATIONS_FOLDER, logger });
57
+ }
58
+ const dir = path.dirname(dbPath);
59
+ if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
60
+ const sqlite = new Database(dbPath);
61
+ try {
62
+ sqlite.exec("PRAGMA journal_mode = WAL;");
63
+ // Snapshot rebuilds copy rows that predate the constraints (SQLite cannot
64
+ // add an FK in place), and drizzle wraps every snapshot in one
65
+ // transaction — a PRAGMA inside it is a no-op, so the posture has to be
66
+ // set here, around the whole run. Enforcement resumes before the handle
67
+ // closes; opening a migrated file never re-validates existing rows.
68
+ sqlite.exec("PRAGMA foreign_keys = OFF;");
69
+ // 30000 to match the house busy posture (state/db.js), not winch's
70
+ // 5000 — the contended moment is a second process mid-migration.
71
+ sqlite.exec("PRAGMA busy_timeout = 30000;");
72
+ migrate(drizzle(sqlite, { schema: SCHEMAS.get(migrationsFolder) ?? {} }), { migrationsFolder });
73
+ sqlite.exec("PRAGMA foreign_keys = ON;");
74
+ } catch (err) {
75
+ throw new Error(
76
+ `migrate: ${migrationsFolder} snapshot failed on ${dbPath}: ${err?.message ?? err}`,
77
+ );
78
+ } finally {
79
+ sqlite.close();
80
+ }
81
+ if (migrationsFolder === RUNTIME_MIGRATIONS_FOLDER) {
82
+ mergeOpsTables({ runtimeDbPath: dbPath, logger });
83
+ // The registry must hold every key already referenced before the FKs
84
+ // are consulted; rows copied by mergeOpsTables carry keys of their own,
85
+ // so this runs after it rather than inside the migration snapshot.
86
+ const backfill = new Database(dbPath);
87
+ try {
88
+ const now = Date.now();
89
+ backfill.prepare(SESSION_KEYS_FROM_ROWS_SQL).run(now);
90
+ for (const sql of BACKFILL_ATTRIBUTION_SQL) backfill.prepare(sql).run(now);
91
+ const hasBranches = backfill
92
+ .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'flow_branches'")
93
+ .get();
94
+ if (hasBranches) backfill.prepare(BACKFILL_LEGACY_BRANCH_ITEMS_SQL).run(now);
95
+ } catch (err) {
96
+ throw new Error(`migrate: sessions backfill failed on ${dbPath}: ${err?.message ?? err}`);
97
+ } finally {
98
+ backfill.close();
99
+ }
100
+ }
101
+ migratedPaths.add(dbPath);
102
+ logger?.info?.(`migrate: snapshots current at ${dbPath}`);
103
+ return { applied: true };
104
+ }
@@ -0,0 +1,67 @@
1
+ // node-cost.js — lazy-singleton reader over cost_by_call (runtime.sqlite).
2
+ // No new storage: lineage already tags every LLM call with pipeline_id +
3
+ // node_id. This just exposes a percentile query so estimateNodeCost can
4
+ // return historically-grounded low/high bounds instead of a static range.
5
+
6
+ import path from "node:path";
7
+ import { getSharedDb, resolveStorePaths } from "./db.js";
8
+ import os from "node:os";
9
+
10
+ function defaultDbPath() {
11
+ return resolveStorePaths().runtimeDb;
12
+ }
13
+
14
+ const COST_SQL = `
15
+ SELECT cost_total FROM cost_by_call
16
+ WHERE pipeline_id = ? AND node_id = ? AND ts >= ?
17
+ ORDER BY cost_total ASC
18
+ `;
19
+
20
+ function percentile(sorted, p) {
21
+ const idx = p * (sorted.length - 1);
22
+ const lo = Math.floor(idx);
23
+ const hi = Math.ceil(idx);
24
+ return lo === hi ? sorted[lo] : sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo);
25
+ }
26
+
27
+ export function createNodeCostReader({ dbPath = defaultDbPath() } = {}) {
28
+ let _client = null;
29
+
30
+ async function getClient() {
31
+ if (_client) return _client;
32
+ try {
33
+ ({ client: _client } = await getSharedDb(dbPath, { readonly: true }));
34
+ } catch {
35
+ _client = null;
36
+ }
37
+ return _client;
38
+ }
39
+
40
+ async function nodeCallStats(pipelineId, nodeId, { sinceDays = 30, minSamples = 3 } = {}) {
41
+ if (!pipelineId || !nodeId) return null;
42
+ const client = await getClient();
43
+ if (!client) return null;
44
+ const sinceTs = Date.now() - sinceDays * 86_400_000;
45
+ let rows;
46
+ try {
47
+ const result = await client.execute({ sql: COST_SQL, args: [pipelineId, nodeId, sinceTs] });
48
+ rows = result.rows;
49
+ } catch {
50
+ return null;
51
+ }
52
+ if (rows.length < minSamples) return null;
53
+ const costs = rows.map((r) => Number(r.cost_total)).sort((a, b) => a - b);
54
+ return {
55
+ p25: percentile(costs, 0.25),
56
+ p50: percentile(costs, 0.50),
57
+ p75: percentile(costs, 0.75),
58
+ sampleSize: costs.length,
59
+ };
60
+ }
61
+
62
+ return { nodeCallStats };
63
+ }
64
+
65
+ // Default singleton backed by the store home's ops.sqlite
66
+ const _default = createNodeCostReader();
67
+ export const { nodeCallStats } = _default;