@igorjpimenta/manifold 0.0.0-bootstrap.1 → 0.0.0-stage

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/README.md +2 -1583
  2. package/package.json +4 -49
  3. package/.migrations/ops/0000_futuristic_karma.sql +0 -111
  4. package/.migrations/ops/0001_low_maximus.sql +0 -7
  5. package/.migrations/ops/meta/0000_snapshot.json +0 -763
  6. package/.migrations/ops/meta/0001_snapshot.json +0 -173
  7. package/.migrations/ops/meta/_journal.json +0 -20
  8. package/.migrations/runtime/0000_real_boomer.sql +0 -180
  9. package/.migrations/runtime/0001_hot_dakota_north.sql +0 -2
  10. package/.migrations/runtime/0002_breezy_vertigo.sql +0 -8
  11. package/.migrations/runtime/0003_dashing_jack_flag.sql +0 -5
  12. package/.migrations/runtime/0004_panoramic_ego.sql +0 -1
  13. package/.migrations/runtime/0005_tidy_ben_grimm.sql +0 -10
  14. package/.migrations/runtime/0006_wet_nightmare.sql +0 -57
  15. package/.migrations/runtime/0007_ambiguous_chronomancer.sql +0 -8
  16. package/.migrations/runtime/0008_wealthy_thundra.sql +0 -301
  17. package/.migrations/runtime/0009_lying_shocker.sql +0 -96
  18. package/.migrations/runtime/0010_solid_phantom_reporter.sql +0 -160
  19. package/.migrations/runtime/0011_windy_demogoblin.sql +0 -175
  20. package/.migrations/runtime/0012_restore_harness_fks.sql +0 -103
  21. package/.migrations/runtime/meta/0000_snapshot.json +0 -1181
  22. package/.migrations/runtime/meta/0001_snapshot.json +0 -1209
  23. package/.migrations/runtime/meta/0002_snapshot.json +0 -1261
  24. package/.migrations/runtime/meta/0003_snapshot.json +0 -1292
  25. package/.migrations/runtime/meta/0004_snapshot.json +0 -1240
  26. package/.migrations/runtime/meta/0006_snapshot.json +0 -1310
  27. package/.migrations/runtime/meta/0007_snapshot.json +0 -1362
  28. package/.migrations/runtime/meta/0008_snapshot.json +0 -1742
  29. package/.migrations/runtime/meta/0009_snapshot.json +0 -2413
  30. package/.migrations/runtime/meta/0010_snapshot.json +0 -2562
  31. package/.migrations/runtime/meta/0011_snapshot.json +0 -2812
  32. package/.migrations/runtime/meta/0012_snapshot.json +0 -2812
  33. package/.migrations/runtime/meta/_journal.json +0 -97
  34. package/RELEASING.md +0 -89
  35. package/agent/fleet.js +0 -100
  36. package/bin/cli.js +0 -854
  37. package/bin/manifold.js +0 -4
  38. package/boot/flow-revival.js +0 -536
  39. package/budget/index.js +0 -495
  40. package/budget/lib.js +0 -59
  41. package/dashboard/cost.js +0 -528
  42. package/dashboard/dist/assets/index-CUfp63sj.css +0 -1
  43. package/dashboard/dist/assets/index-Qy9JMpma.js +0 -371
  44. package/dashboard/dist/index.html +0 -13
  45. package/dashboard/event-log.js +0 -267
  46. package/dashboard/index.html +0 -12
  47. package/dashboard/package-lock.json +0 -2877
  48. package/dashboard/package.json +0 -37
  49. package/dashboard/pipeline-graph.js +0 -131
  50. package/dashboard/postcss.config.js +0 -6
  51. package/dashboard/search.js +0 -183
  52. package/dashboard/server.js +0 -1015
  53. package/dashboard/src/App.tsx +0 -42
  54. package/dashboard/src/api/client.ts +0 -118
  55. package/dashboard/src/api/types.ts +0 -510
  56. package/dashboard/src/api/ws.tsx +0 -112
  57. package/dashboard/src/components/CancelButton.tsx +0 -63
  58. package/dashboard/src/components/ClickToCopy.tsx +0 -68
  59. package/dashboard/src/components/CopyButton.tsx +0 -61
  60. package/dashboard/src/components/DailyCostTrend.tsx +0 -85
  61. package/dashboard/src/components/DailyTimeTrend.tsx +0 -94
  62. package/dashboard/src/components/Funnel.tsx +0 -149
  63. package/dashboard/src/components/GateForm.tsx +0 -158
  64. package/dashboard/src/components/GraphView.tsx +0 -302
  65. package/dashboard/src/components/GroupPathsSummary.tsx +0 -47
  66. package/dashboard/src/components/Histogram.tsx +0 -292
  67. package/dashboard/src/components/Layout.tsx +0 -28
  68. package/dashboard/src/components/NeedsAttention.tsx +0 -98
  69. package/dashboard/src/components/NodeContextCard.tsx +0 -132
  70. package/dashboard/src/components/PathTrail.tsx +0 -35
  71. package/dashboard/src/components/PipelineCostBreakdown.tsx +0 -277
  72. package/dashboard/src/components/RerunButton.tsx +0 -46
  73. package/dashboard/src/components/RunComparison.tsx +0 -53
  74. package/dashboard/src/components/Sparkline.tsx +0 -94
  75. package/dashboard/src/components/StatStrip.tsx +0 -147
  76. package/dashboard/src/components/StateTree.tsx +0 -84
  77. package/dashboard/src/components/StatusBadge.tsx +0 -18
  78. package/dashboard/src/components/SubflowPathPills.tsx +0 -38
  79. package/dashboard/src/components/SubflowSummary.tsx +0 -24
  80. package/dashboard/src/components/SubflowsCard.tsx +0 -327
  81. package/dashboard/src/components/TodayStrip.tsx +0 -86
  82. package/dashboard/src/components/TranscriptPane.tsx +0 -99
  83. package/dashboard/src/components/TranscriptRow.tsx +0 -69
  84. package/dashboard/src/components/ui/badge.tsx +0 -38
  85. package/dashboard/src/components/ui/button.tsx +0 -28
  86. package/dashboard/src/components/ui/card.tsx +0 -42
  87. package/dashboard/src/components/ui/skeleton.tsx +0 -9
  88. package/dashboard/src/components/ui/table.tsx +0 -62
  89. package/dashboard/src/index.css +0 -26
  90. package/dashboard/src/lib/anomaly.ts +0 -42
  91. package/dashboard/src/lib/format.ts +0 -30
  92. package/dashboard/src/lib/step-history.ts +0 -13
  93. package/dashboard/src/lib/subflow.ts +0 -154
  94. package/dashboard/src/lib/use-click-outside.ts +0 -28
  95. package/dashboard/src/lib/use-ignored-clusters.ts +0 -67
  96. package/dashboard/src/lib/utils.ts +0 -6
  97. package/dashboard/src/main.tsx +0 -10
  98. package/dashboard/src/types/shims.d.ts +0 -4
  99. package/dashboard/src/views/Home.tsx +0 -478
  100. package/dashboard/src/views/PipelineDetail.tsx +0 -1185
  101. package/dashboard/src/views/PipelinesList.tsx +0 -111
  102. package/dashboard/src/views/RunDetail.tsx +0 -248
  103. package/dashboard/src/views/Search.tsx +0 -196
  104. package/dashboard/stats.js +0 -668
  105. package/dashboard/tailwind.config.ts +0 -43
  106. package/dashboard/tree-stats.js +0 -12
  107. package/dashboard/tsconfig.json +0 -25
  108. package/dashboard/views.js +0 -231
  109. package/dashboard/vite.config.ts +0 -34
  110. package/drizzle.config.js +0 -12
  111. package/drizzle.config.ops.js +0 -12
  112. package/elevation.js +0 -116
  113. package/engine/admission-promotion-resume.js +0 -127
  114. package/engine/agent-dispatch.js +0 -64
  115. package/engine/child-resume-trigger.js +0 -315
  116. package/engine/executor-budget.js +0 -87
  117. package/engine/executor-cancel.js +0 -83
  118. package/engine/executor-errors.js +0 -122
  119. package/engine/executor-helpers.js +0 -254
  120. package/engine/executor-nodes.js +0 -292
  121. package/engine/executor-persist.js +0 -212
  122. package/engine/executor-subflow-children.js +0 -275
  123. package/engine/executor.js +0 -1013
  124. package/engine/global-concurrency.js +0 -33
  125. package/engine/loader.js +0 -528
  126. package/engine/needs-resolver.js +0 -229
  127. package/engine/pipelines-watcher.js +0 -133
  128. package/engine/result-replay.js +0 -34
  129. package/engine/revival-messages.js +0 -11
  130. package/engine/spawn-child.js +0 -55
  131. package/engine/wait-for-flow-settlement.js +0 -109
  132. package/events/catalog.js +0 -26
  133. package/events/emit.js +0 -44
  134. package/events/kinds.js +0 -118
  135. package/events/serve-ws.js +0 -74
  136. package/events/stream.js +0 -35
  137. package/events/ws-dialog.js +0 -257
  138. package/events/ws-frame.js +0 -194
  139. package/events/ws-run-events.js +0 -70
  140. package/events.js +0 -32
  141. package/gates/gate-resolver.js +0 -576
  142. package/gates/paused-flows.js +0 -55
  143. package/graphql/columns.js +0 -67
  144. package/graphql/schema.js +0 -333
  145. package/index.js +0 -49
  146. package/mcp/http.js +0 -159
  147. package/mcp/identity.js +0 -94
  148. package/mcp/stdio.js +0 -80
  149. package/mcp/tools.js +0 -245
  150. package/mcp/wire-ops.js +0 -178
  151. package/routes/bearer-auth.js +0 -57
  152. package/routes/discuss-audio.js +0 -87
  153. package/routes/discuss-component.js +0 -39
  154. package/routes/discuss-draft.js +0 -159
  155. package/routes/discuss-helpers.js +0 -20
  156. package/routes/discuss.js +0 -848
  157. package/routes/failures.js +0 -104
  158. package/routes/graphql.js +0 -61
  159. package/routes/health.js +0 -54
  160. package/routes/operator-discuss.js +0 -193
  161. package/routes/operator-resume.js +0 -32
  162. package/routes/operator-retry.js +0 -301
  163. package/routes/operator.js +0 -539
  164. package/routes/pipelines.js +0 -118
  165. package/routes/runs.js +0 -1129
  166. package/scripts/ci/import-check.mjs +0 -50
  167. package/scripts/link-core-self.mjs +0 -31
  168. package/scripts/with-guard.sh +0 -24
  169. package/server/app.js +0 -503
  170. package/server/registry.js +0 -53
  171. package/server/serve.js +0 -163
  172. package/state/admission-queue-store.js +0 -325
  173. package/state/attribution-registries.js +0 -102
  174. package/state/batch-writer.js +0 -162
  175. package/state/db-guard.js +0 -63
  176. package/state/db-health.js +0 -27
  177. package/state/db.js +0 -122
  178. package/state/discuss-store.js +0 -299
  179. package/state/failure-store.js +0 -427
  180. package/state/migrate.js +0 -104
  181. package/state/node-cost.js +0 -67
  182. package/state/node-executions.js +0 -775
  183. package/state/node-health-store.js +0 -124
  184. package/state/ops-merge.js +0 -93
  185. package/state/paths.js +0 -146
  186. package/state/pipeline-dirs.js +0 -99
  187. package/state/pipeline-registry.js +0 -449
  188. package/state/run-store.js +0 -697
  189. package/state/schema.js +0 -438
  190. package/state/schema.ops.js +0 -44
  191. package/state/session-index.js +0 -177
  192. package/state/session-keys.js +0 -39
  193. package/state/state-store.js +0 -311
  194. package/state/subflow-state.js +0 -126
  195. package/state/tool-handoff.js +0 -169
  196. package/telemetry/budget-alert.js +0 -353
  197. package/telemetry/cli.js +0 -331
  198. package/telemetry/grammar.js +0 -119
  199. package/telemetry/http-budget.js +0 -132
  200. package/telemetry/http-cost.js +0 -670
  201. package/telemetry/http-helpers.js +0 -134
  202. package/telemetry/http-tree.js +0 -349
  203. package/telemetry/http.js +0 -44
  204. package/telemetry/index.js +0 -307
  205. package/telemetry/ingest.js +0 -134
  206. package/telemetry/labels.js +0 -115
  207. package/telemetry/pricer.js +0 -77
  208. package/telemetry/resolver.js +0 -314
  209. package/telemetry/store.js +0 -147
  210. package/utils/run-patch.js +0 -43
package/server/serve.js DELETED
@@ -1,163 +0,0 @@
1
- // serve.js — `manifold serve`: the core application as its own process.
2
- //
3
- // Everything harness-specific arrives through the host this file builds:
4
- // - agent turns go to MANIFOLD_TURN_URL (the harness adapter's endpoint)
5
- // - discuss session ids are minted locally (no harness to ask)
6
- // - elevation loops back to this process's own API port
7
- //
8
- // Loopback only by default; the routes authenticate with MANIFOLD_BEARER.
9
-
10
- import http from "node:http";
11
- import { createRouteRegistry } from "./registry.js";
12
- import { createManifoldApp, DEFAULT_DASHBOARD_HOST, DEFAULT_DASHBOARD_PORT } from "./app.js";
13
- import { createPipelineDirsStore } from "../state/pipeline-dirs.js";
14
- import { resolveStorePaths } from "../state/paths.js";
15
-
16
- export const DEFAULT_API_PORT = 7431;
17
- export const DEFAULT_API_HOST = "127.0.0.1";
18
-
19
- function createLogger(prefix) {
20
- const line = (level, msg) => {
21
- const text = `[${prefix}] ${msg}`;
22
- if (level === "info") console.log(text);
23
- else console.error(text);
24
- };
25
- return {
26
- info: (msg) => line("info", msg),
27
- warn: (msg) => line("warn", msg),
28
- error: (msg) => line("error", msg),
29
- debug: (msg) => line("debug", msg),
30
- };
31
- }
32
-
33
- // Deterministic discuss thread ids — core names the thread, the harness would
34
- // have minted it. Same flow/gate/item always maps to the same thread.
35
- export function localSessions() {
36
- return {
37
- newDiscussSessionId({ agentId, flowId, gateNodeId, itemId }) {
38
- return `manifold:discuss:${flowId}:${gateNodeId ?? "flow"}:${itemId ?? "0"}:${agentId ?? "agent"}`;
39
- },
40
- // No harness model table: the caller's modelOverride wins, else default.
41
- async resolveAgentEffectiveModelPrimary() { return null; },
42
- isProviderCooldown() { return false; },
43
- };
44
- }
45
-
46
- export function createHttpRunTurn({ url, token, timeoutMs = 600_000, logger, fetchImpl = globalThis.fetch }) {
47
- return async (args) => {
48
- if (!url) {
49
- throw new Error("manifold: agent turn requested but MANIFOLD_TURN_URL is unset (agent nodes need a harness)");
50
- }
51
- const res = await fetchImpl(url, {
52
- method: "POST",
53
- headers: {
54
- "content-type": "application/json",
55
- ...(token ? { authorization: `Bearer ${token}` } : {}),
56
- },
57
- body: JSON.stringify(args),
58
- signal: AbortSignal.timeout(timeoutMs),
59
- });
60
- const payload = await res.json().catch(() => ({}));
61
- if (!res.ok || payload?.ok === false) {
62
- const detail = payload?.error ?? `HTTP ${res.status}`;
63
- logger?.warn?.(`manifold: agent turn failed: ${detail}`);
64
- throw new Error(`manifold: agent turn failed: ${detail}`);
65
- }
66
- return { text: payload.text ?? "", runId: payload.runId ?? null, usage: payload.usage ?? null };
67
- };
68
- }
69
-
70
- // Elevation loopback against this process: the `_elev` handlers are mounted on
71
- // the same server, so the operator-scope crossing the gateway needed collapses
72
- // to a localhost call carrying the shared bearer.
73
- export function createLocalElevation({ port, host, token, fetchImpl = globalThis.fetch }) {
74
- return async function invokeElevated(path, payload, opts = {}) {
75
- const timeoutMs = opts.timeoutMs ?? 30_000;
76
- if (!token) {
77
- return { ok: false, status: 503, body: JSON.stringify({ ok: false, code: "no_bearer", error: "MANIFOLD_BEARER unset" }), error: "MANIFOLD_BEARER unset" };
78
- }
79
- try {
80
- const res = await fetchImpl(`http://${host}:${port}${path}`, {
81
- method: "POST",
82
- headers: { "content-type": "application/json", authorization: `Bearer ${token}` },
83
- body: JSON.stringify(payload ?? {}),
84
- signal: AbortSignal.timeout(timeoutMs),
85
- });
86
- return { ok: res.ok, status: res.status, body: await res.text() };
87
- } catch (err) {
88
- return { ok: false, status: 0, body: "", error: err?.message ?? String(err) };
89
- }
90
- };
91
- }
92
-
93
- export function resolveServeConfig({ env = process.env } = {}) {
94
- const storePaths = resolveStorePaths({ env });
95
- const pipelineDirsStore = createPipelineDirsStore({ dbPath: storePaths.runtimeDb, logger: createLogger("manifold") });
96
- let pipelinesDir = env.MANIFOLD_PIPELINES_DIR ?? null;
97
- if (!pipelinesDir) {
98
- pipelinesDir = pipelineDirsStore.resolveDir({
99
- harness: env.MANIFOLD_HARNESS ?? null,
100
- env,
101
- dataDir: storePaths.dataDir,
102
- configDir: null,
103
- }).dir;
104
- }
105
- return {
106
- storePaths,
107
- config: {
108
- pipelinesDir,
109
- ownerSessionKey: env.MANIFOLD_OWNER_SESSION_KEY || undefined,
110
- discussDefaultAgent: env.MANIFOLD_DISCUSS_DEFAULT_AGENT || null,
111
- dashboardEnabled: env.MANIFOLD_DASHBOARD_ENABLED !== "false",
112
- dashboardPort: Number(env.MANIFOLD_DASHBOARD_PORT) || DEFAULT_DASHBOARD_PORT,
113
- dashboardHost: env.MANIFOLD_DASHBOARD_HOST || DEFAULT_DASHBOARD_HOST,
114
- globalConcurrency: Number(env.MANIFOLD_GLOBAL_CONCURRENCY) || undefined,
115
- },
116
- apiPort: Number(env.MANIFOLD_API_PORT) || DEFAULT_API_PORT,
117
- apiHost: env.MANIFOLD_API_HOST || DEFAULT_API_HOST,
118
- turnUrl: env.MANIFOLD_TURN_URL || null,
119
- turnToken: env.MANIFOLD_TURN_TOKEN || env.MANIFOLD_BEARER || null,
120
- bearer: env.MANIFOLD_BEARER || null,
121
- };
122
- }
123
-
124
- export async function serve({ env = process.env, logger: injectedLogger = null } = {}) {
125
- const cfg = resolveServeConfig({ env });
126
- const logger = injectedLogger ?? createLogger("manifold");
127
-
128
- const registry = createRouteRegistry({ logger });
129
- const host = {
130
- logger,
131
- registerHttpRoute: registry.registerHttpRoute,
132
- runTurn: createHttpRunTurn({ url: cfg.turnUrl, token: cfg.turnToken, logger }),
133
- invokeElevated: createLocalElevation({ port: cfg.apiPort, host: cfg.apiHost, token: cfg.bearer }),
134
- sessions: localSessions(),
135
- getBroadcast: () => null,
136
- };
137
-
138
- const app = createManifoldApp({ host, config: cfg.config });
139
- const server = http.createServer((req, res) => registry.handle(req, res));
140
-
141
- await new Promise((resolve, reject) => {
142
- server.once("error", reject);
143
- server.listen(cfg.apiPort, cfg.apiHost, resolve);
144
- });
145
- logger.info(`manifold serve: api on http://${cfg.apiHost}:${cfg.apiPort} (${registry.routes.length} route(s))`);
146
-
147
- await app.start({ logger });
148
- logger.info("manifold serve: ready");
149
-
150
- let stopping = false;
151
- const shutdown = async (signal) => {
152
- if (stopping) return;
153
- stopping = true;
154
- logger.info(`manifold serve: ${signal}, stopping`);
155
- await app.stop({ logger }).catch(() => {});
156
- await new Promise((resolve) => server.close(resolve));
157
- process.exit(0);
158
- };
159
- process.once("SIGINT", () => { shutdown("SIGINT"); });
160
- process.once("SIGTERM", () => { shutdown("SIGTERM"); });
161
-
162
- return { app, registry, server, cfg, shutdown };
163
- }
@@ -1,325 +0,0 @@
1
- // admission-queue-store.js — the single durable admission table for flow
2
- // RUNS: starting a flow (kind:"flow-start") and dispatching a subflow child
3
- // (kind:"child-dispatch"). One row per run; nothing inside a running flow
4
- // (agent nodes included) takes a slot of its own — a run executes one node
5
- // at a time, so the run cap already bounds everything it does.
6
- //
7
- // ONE combined pool, not one per kind — deliberately. Both kinds compete
8
- // for the same shared count (no `kind` filter on the admitted-count query
9
- // or the promotion query below). This only works because admission tracks
10
- // flow_runs.status, not just start/end: a flow-run's own admission entry
11
- // (whichever kind spawned it) is held only while it's genuinely "running" —
12
- // the moment it goes "blocked" (a human-gate wait, or its own
13
- // children-pending wait while a subflow fans out), state-store.js's
14
- // setWaiting releases that entry via releaseByFlow(), so it stops occupying
15
- // a shared slot for however long it's actually idle. Without that, a parent
16
- // would hold its slot for its ENTIRE lifetime while its children ALSO need
17
- // slots — one fan-out could alone exhaust the whole pool.
18
- //
19
- // child-dispatch releases on blocked/terminal but deliberately
20
- // does NOT re-request admission when a blocked child later resumes — the
21
- // mechanism it replaced never re-checked the cap at resume time either
22
- // (the cap only ever governed fan-out DISPATCH, not a continuously-
23
- // enforced running ceiling), so this is exact behavioral parity, not a new
24
- // gap. Whether flow-start (a whole run's own admission, not yet wired)
25
- // needs real re-admission-on-resume is still open — later stage's call.
26
- //
27
- // Atomicity: every decision (count admitted, then claim admitted-or-queued)
28
- // runs inside one client.transaction("write") — BEGIN IMMEDIATE, acquiring
29
- // SQLite's writer lock before the SELECT even runs. A bare
30
- // `await count-query` then, much later, a separate write is two non-atomic
31
- // steps, racy across concurrent callers; this is the primitive that closes
32
- // the window.
33
-
34
- import path from "node:path";
35
- import os from "node:os";
36
- import { mkdirSync } from "node:fs";
37
- import { randomUUID } from "node:crypto";
38
- import { getSharedDb, withSharedWrite, resolveStorePaths } from "./db.js";
39
- import { migrateDatabase, RUNTIME_MIGRATIONS_FOLDER } from "./migrate.js";
40
-
41
- function defaultDbPath() {
42
- return resolveStorePaths().runtimeDb;
43
- }
44
-
45
- // Same Proxy-wrapped async-ready factory idiom as run-store.js/state-
46
- // store.js — every method call transparently awaits store construction
47
- // (migration included) before running.
48
- export function createAdmissionQueueStore(opts = {}) {
49
- const _p = _createImpl(opts);
50
- return new Proxy({}, {
51
- get(_, k) {
52
- if (k === "then" || typeof k === "symbol") return undefined;
53
- if (k === "ready") return () => _p.then(() => {});
54
- return (...args) => _p.then((s) => {
55
- if (typeof s[k] !== "function") throw new TypeError(`admission-queue-store: ${String(k)} is not a function`);
56
- return s[k](...args);
57
- });
58
- },
59
- });
60
- }
61
-
62
- async function _createImpl({ dbPath = defaultDbPath(), getLimit } = {}) {
63
- if (typeof getLimit !== "function") {
64
- throw new TypeError("admission-queue-store: getLimit() function is required (e.g. global-concurrency.js's getGlobalConcurrencyLimit)");
65
- }
66
- mkdirSync(path.dirname(dbPath), { recursive: true });
67
- await migrateDatabase({ dbPath, migrationsFolder: RUNTIME_MIGRATIONS_FOLDER });
68
- const { client } = await getSharedDb(dbPath);
69
-
70
- // Runs `fn(tx)` inside one BEGIN IMMEDIATE transaction, committing on
71
- // success and always closing (which rolls back if not committed) — the
72
- // exact try/finally shape @libsql/client's own docs specify as required.
73
- // Serialized and retried on SQLITE_BUSY via state/db.js's
74
- // withSharedWrite(), keyed by dbPath: @libsql/client's local sqlite3
75
- // backend does NOT safely support multiple Transaction objects in flight
76
- // concurrently on one shared connection — commit() itself fails with
77
- // "cannot commit transaction - SQL statements in
78
- // progress", consistent with concurrent Transaction objects interleaving
79
- // execute() calls over the same underlying native connection and
80
- // corrupting each other's in-flight statement state; a client-library
81
- // limitation, not a SQLite-level one — BEGIN IMMEDIATE itself would have
82
- // serialized this correctly at the file-lock level. Corroborated by this
83
- // store's own test suite, which drives 30 genuinely concurrent admit()
84
- // callers through it.
85
- //
86
- // Because withSharedWrite's queue is keyed by dbPath rather than private
87
- // to this store, every OTHER writer sharing this same connection via
88
- // db.js's getSharedDb (run-store.js's CAS writes, batch-writer.js's
89
- // flushes for node-executions.js/node-health-store.js) serializes against
90
- // this store's transactions too, once it also calls withSharedWrite for
91
- // the same dbPath — closing the corruption/stall path this store's own
92
- // private serializer could not (see test/unit/
93
- // state/admission-queue-cross-module-contention.test.js): a plain
94
- // client.execute()/db.update()/db.batch() issued on the same connection
95
- // while a transaction() is open can fail instantly, bypassing the
96
- // busy_timeout PRAGMA entirely, and can corrupt the transaction holder's
97
- // own commit. Until every writer on this connection routes through the
98
- // same primitive, that gap remains open for whichever writers haven't
99
- // adopted it yet.
100
- //
101
- // Safe to retry the whole fn(tx) on a fresh transaction: an uncommitted
102
- // transaction has zero durable effect (SQLite's own atomicity), so
103
- // re-running every one of fn's statements against a new tx is not a
104
- // partial-redo risk.
105
- async function withWriteTx(fn) {
106
- return withSharedWrite(dbPath, async () => {
107
- const tx = await client.transaction("write");
108
- try {
109
- const result = await fn(tx);
110
- await tx.commit();
111
- return result;
112
- } finally {
113
- tx.close();
114
- }
115
- });
116
- }
117
-
118
- // Shared by release(id) and releaseByFlow() — deletes the row and, if it
119
- // was occupying an admitted slot, promotes the oldest queued entry across
120
- // the whole combined pool. Must run inside the caller's own withWriteTx
121
- // callback (not its own transaction) so both release paths stay a single
122
- // atomic decision.
123
- async function releaseRowById(tx, id) {
124
- const { rows: selfRows } = await tx.execute({
125
- sql: `SELECT status FROM flow_admission_queue WHERE id = ?`,
126
- args: [id],
127
- });
128
- const self = selfRows[0] ?? null;
129
- if (!self) return { released: false, promoted: null };
130
-
131
- await tx.execute({ sql: `DELETE FROM flow_admission_queue WHERE id = ?`, args: [id] });
132
-
133
- let promoted = null;
134
- if (self.status === "admitted") {
135
- const { rows: nextRows } = await tx.execute({
136
- sql: `SELECT id, kind, flow_id, node_id FROM flow_admission_queue
137
- WHERE status = 'queued' ORDER BY created_at ASC LIMIT 1`,
138
- args: [],
139
- });
140
- const next = nextRows[0] ?? null;
141
- if (next) {
142
- const now = Date.now();
143
- await tx.execute({
144
- sql: `UPDATE flow_admission_queue SET status = 'admitted', admitted_at = ? WHERE id = ?`,
145
- args: [now, next.id],
146
- });
147
- promoted = { id: next.id, kind: next.kind, flowId: next.flow_id, nodeId: next.node_id };
148
- }
149
- }
150
- return { released: true, promoted };
151
- }
152
-
153
- return {
154
- // Decides admitted-vs-queued for one new entry, atomically against the
155
- // CURRENT combined count of admitted rows (all kinds pooled — see this
156
- // file's header). Returns { id, status }. Never throws on contention —
157
- // "queued" is a normal, expected outcome, not an error.
158
- //
159
- // Idempotent per (kind, flowId, nodeId): if a live (admitted or queued)
160
- // row already exists for this exact tuple, returns THAT row instead of
161
- // inserting a second one. Without this, any caller that re-enters the
162
- // same logical request (e.g. a resume racing a revival for the same
163
- // flow) creates a genuine duplicate row, permanently shrinking the
164
- // shared pool by one slot per occurrence.
165
- //
166
- // A queued caller is never woken in-process: when a slot frees,
167
- // release() promotes the row and the releasing call site drives the
168
- // flow forward through wakePromotedRelease (engine/admission-promotion-
169
- // resume.js) — flip to 'running', then resume.
170
- async admit({ kind, flowId, nodeId = null }) {
171
- if (!kind) throw new TypeError("admission-queue-store.admit: kind is required");
172
- if (!flowId) throw new TypeError("admission-queue-store.admit: flowId is required");
173
- const limit = getLimit();
174
- return withWriteTx(async (tx) => {
175
- const { rows: existingRows } = await tx.execute({
176
- sql: `SELECT id, status FROM flow_admission_queue
177
- WHERE kind = ? AND flow_id = ? AND (node_id = ? OR (? IS NULL AND node_id IS NULL))
178
- AND status IN ('admitted', 'queued')
179
- LIMIT 1`,
180
- args: [kind, flowId, nodeId, nodeId],
181
- });
182
- if (existingRows[0]) {
183
- const { id, status } = existingRows[0];
184
- return { id, status };
185
- }
186
- const { rows } = await tx.execute({
187
- sql: `SELECT COUNT(*) AS n FROM flow_admission_queue WHERE status = 'admitted'`,
188
- args: [],
189
- });
190
- const admittedCount = Number(rows[0]?.n ?? 0);
191
- const id = randomUUID();
192
- const now = Date.now();
193
- const status = admittedCount < limit ? "admitted" : "queued";
194
- await tx.execute({
195
- sql: `INSERT INTO flow_admission_queue (id, kind, flow_id, node_id, status, created_at, admitted_at)
196
- VALUES (?, ?, ?, ?, ?, ?, ?)`,
197
- args: [id, kind, flowId, nodeId, status, now, status === "admitted" ? now : null],
198
- });
199
- return { id, status };
200
- });
201
- },
202
-
203
- // Releases one entry by id. If it was occupying an admitted slot,
204
- // atomically promotes the oldest queued entry — FIFO across the WHOLE
205
- // combined pool, not scoped to the releasing entry's own kind, matching
206
- // the one-pool design. Returns { released, promoted }; promoted is
207
- // either null or { id, flowId, nodeId, kind } for the caller to wake
208
- // via wakePromotedRelease. A release() for an id that no longer exists
209
- // (already released, or never existed) is a safe no-op —
210
- // { released: false, promoted: null } — never throws, since a caller
211
- // racing its own cleanup against a process restart shouldn't have to
212
- // special-case "already gone".
213
- async release(id) {
214
- if (!id) throw new TypeError("admission-queue-store.release: id is required");
215
- return withWriteTx(async (tx) => releaseRowById(tx, id));
216
- },
217
-
218
- // Releases the CURRENT (admitted or queued) entry for a given
219
- // (flowId, kind) pair, without the caller needing to carry its own
220
- // admission id around — the shape a status-store hook wants: state-
221
- // store.js's setWaiting/finish/skip/fail/cancel know the flowId that
222
- // just left "running" (or went terminal), not any admission id, since
223
- // the flow could have crashed and been revived by a different process
224
- // between admit() and now. Same promotion/notify semantics as
225
- // release(id); a no-op ({ released: false, promoted: null }) when this
226
- // flow never held a live entry for this kind (true for every ordinary
227
- // flow that never participated in this kind's admission at all — the
228
- // common case, deliberately cheap: one indexed SELECT).
229
- async releaseByFlow({ flowId, kind }) {
230
- if (!flowId) throw new TypeError("admission-queue-store.releaseByFlow: flowId is required");
231
- if (!kind) throw new TypeError("admission-queue-store.releaseByFlow: kind is required");
232
- return withWriteTx(async (tx) => {
233
- const { rows } = await tx.execute({
234
- sql: `SELECT id FROM flow_admission_queue
235
- WHERE flow_id = ? AND kind = ? AND status IN ('admitted', 'queued')
236
- ORDER BY created_at DESC LIMIT 1`,
237
- args: [flowId, kind],
238
- });
239
- const self = rows[0] ?? null;
240
- if (!self) return { released: false, promoted: null };
241
- return releaseRowById(tx, self.id);
242
- });
243
- },
244
-
245
- // Promotes oldest queued rows into any free capacity, until the pool is
246
- // full or nothing is queued. Returns the promoted entries ({ id, kind,
247
- // flowId, nodeId }) for the caller to wake via wakePromotedRelease.
248
- // Normal operation never needs this — every release promotes exactly
249
- // one row into the slot it frees — but a pool that loses admitted rows
250
- // without a release (a migration deleting a whole kind, a crash between
251
- // a delete and its promotion) would otherwise sit below its limit with
252
- // work still queued, until enough unrelated releases happened.
253
- async fillFreeSlots() {
254
- const limit = getLimit();
255
- return withWriteTx(async (tx) => {
256
- const { rows } = await tx.execute({
257
- sql: `SELECT COUNT(*) AS n FROM flow_admission_queue WHERE status = 'admitted'`,
258
- args: [],
259
- });
260
- const free = limit - Number(rows[0]?.n ?? 0);
261
- if (free <= 0) return [];
262
- const { rows: nextRows } = await tx.execute({
263
- sql: `SELECT id, kind, flow_id, node_id FROM flow_admission_queue
264
- WHERE status = 'queued' ORDER BY created_at ASC LIMIT ?`,
265
- args: [free],
266
- });
267
- const now = Date.now();
268
- const promoted = [];
269
- for (const next of nextRows) {
270
- await tx.execute({
271
- sql: `UPDATE flow_admission_queue SET status = 'admitted', admitted_at = ? WHERE id = ?`,
272
- args: [now, next.id],
273
- });
274
- promoted.push({ id: next.id, kind: next.kind, flowId: next.flow_id, nodeId: next.node_id });
275
- }
276
- return promoted;
277
- });
278
- },
279
-
280
- // Observability / test hook — counts across the whole combined pool,
281
- // not scoped to one kind (matching admit()/release()'s own scope).
282
- async _stats() {
283
- const { rows } = await client.execute(
284
- `SELECT status, COUNT(*) AS n FROM flow_admission_queue GROUP BY status`
285
- );
286
- const out = { admitted: 0, queued: 0 };
287
- for (const r of rows) out[r.status] = Number(r.n);
288
- return out;
289
- },
290
-
291
- // Lists queued entries oldest-first — general observability / test hook.
292
- // Not currently called from any boot-revival or dispatch path (see
293
- // boot/flow-revival.js's own skip-direct-dispatch comment for how
294
- // queued orphans are actually handled today).
295
- async listByStatus(status) {
296
- const { rows } = await client.execute({
297
- sql: `SELECT id, kind, flow_id, node_id, created_at, admitted_at
298
- FROM flow_admission_queue WHERE status = ? ORDER BY created_at ASC`,
299
- args: [status],
300
- });
301
- return rows.map((r) => ({
302
- id: r.id, kind: r.kind, flowId: r.flow_id, nodeId: r.node_id,
303
- createdAt: Number(r.created_at),
304
- admittedAt: r.admitted_at != null ? Number(r.admitted_at) : null,
305
- }));
306
- },
307
-
308
- // Fetches one entry by id — callers releasing/inspecting their own
309
- // admission handle without going through listByStatus's full scan.
310
- async get(id) {
311
- const { rows } = await client.execute({
312
- sql: `SELECT id, kind, flow_id, node_id, status, created_at, admitted_at
313
- FROM flow_admission_queue WHERE id = ?`,
314
- args: [id],
315
- });
316
- const r = rows[0];
317
- if (!r) return null;
318
- return {
319
- id: r.id, kind: r.kind, flowId: r.flow_id, nodeId: r.node_id, status: r.status,
320
- createdAt: Number(r.created_at),
321
- admittedAt: r.admitted_at != null ? Number(r.admitted_at) : null,
322
- };
323
- },
324
- };
325
- }
@@ -1,102 +0,0 @@
1
- // Attribution registries: item keys, node names and turn-run ids are written
2
- // by engine and telemetry frames, so no table of ours owns the key space.
3
- // These helpers materialize the key set — writers register what they are
4
- // about to attribute, in the same write; the migration backfills the union
5
- // so historical rows keep their attribution and the FKs are satisfiable.
6
- //
7
- // A writer that skips the guard fails its insert instead of writing an
8
- // untracked pointer. Rows are append-only: pruning a flow or a branch must
9
- // not erase the attribution that names it.
10
-
11
- export const ENSURE_FLOW_ITEM_SQL =
12
- "INSERT OR IGNORE INTO flow_items (flow_id, item_key, first_seen_at) VALUES (?, ?, ?)";
13
- export const ENSURE_TURN_RUN_SQL =
14
- "INSERT OR IGNORE INTO turn_runs (run_id, first_seen_at) VALUES (?, ?)";
15
- export const ENSURE_PIPELINE_NODE_SQL =
16
- "INSERT OR IGNORE INTO pipeline_nodes (pipeline_id, node_name, first_seen_at) VALUES (?, ?, ?)";
17
-
18
- // Composite keys are skipped whole when any part is null: a partial pointer
19
- // cannot be checked, and SQLite would treat the row as satisfying the FK
20
- // anyway — registering half of it would only create noise.
21
- function pairs(rows) {
22
- const seen = new Map();
23
- for (const [a, b] of rows) {
24
- if (typeof a !== "string" || !a || typeof b !== "string" || !b) continue;
25
- seen.set(`${a}\u0000${b}`, [a, b]);
26
- }
27
- return [...seen.values()];
28
- }
29
-
30
- export async function ensureFlowItems(client, rows, now = Date.now()) {
31
- const keys = pairs(rows);
32
- if (keys.length === 0) return;
33
- await client.batch(keys.map(([flowId, itemKey]) => ({
34
- sql: ENSURE_FLOW_ITEM_SQL, args: [flowId, itemKey, now],
35
- })), "write");
36
- }
37
-
38
- export async function ensureTurnRuns(client, runIds, now = Date.now()) {
39
- const unique = [...new Set(runIds.filter((r) => typeof r === "string" && r.length > 0))];
40
- if (unique.length === 0) return;
41
- await client.batch(unique.map((runId) => ({
42
- sql: ENSURE_TURN_RUN_SQL, args: [runId, now],
43
- })), "write");
44
- }
45
-
46
- export async function ensurePipelineNodes(client, rows, now = Date.now()) {
47
- const keys = pairs(rows);
48
- if (keys.length === 0) return;
49
- await client.batch(keys.map(([pipelineId, nodeName]) => ({
50
- sql: ENSURE_PIPELINE_NODE_SQL, args: [pipelineId, nodeName, now],
51
- })), "write");
52
- }
53
-
54
- // Union backfill, one pass. Node names also come from the version registry
55
- // so a pipeline's declared nodes are known before their first turn. Each
56
- // batch item is one statement; the legacy flow_branches table is read only
57
- // when it exists, because files created before v7 still carry the branch
58
- // keys that item attribution was written against.
59
- export const BACKFILL_FLOW_ITEMS_SQL = `
60
- INSERT OR IGNORE INTO flow_items (flow_id, item_key, first_seen_at)
61
- SELECT flow_id, item_key, ? FROM (
62
- SELECT flow_id, item_key FROM session_lineage WHERE item_key IS NOT NULL AND flow_id IS NOT NULL
63
- UNION SELECT flow_id, item_key FROM cost_by_call WHERE item_key IS NOT NULL AND flow_id IS NOT NULL
64
- UNION SELECT flow_id, branch_item_id FROM flow_sessions WHERE branch_item_id IS NOT NULL
65
- )
66
- `;
67
-
68
- export const BACKFILL_LEGACY_BRANCH_ITEMS_SQL = `
69
- INSERT OR IGNORE INTO flow_items (flow_id, item_key, first_seen_at)
70
- SELECT DISTINCT flow_id, branch_key, ? FROM flow_branches
71
- `;
72
-
73
- export const BACKFILL_TURN_RUNS_SQL = `
74
- INSERT OR IGNORE INTO turn_runs (run_id, first_seen_at)
75
- SELECT run_id, ? FROM session_lineage WHERE run_id IS NOT NULL
76
- `;
77
-
78
- export const BACKFILL_PIPELINE_NODES_SQL = `
79
- INSERT OR IGNORE INTO pipeline_nodes (pipeline_id, node_name, first_seen_at)
80
- SELECT pipeline_id, node_name, ? FROM (
81
- SELECT pipeline_id, node_name FROM pipeline_node_versions WHERE node_name IS NOT NULL
82
- UNION SELECT pipeline_id, node_id FROM session_lineage WHERE node_id IS NOT NULL AND pipeline_id IS NOT NULL
83
- UNION SELECT pipeline_id, node_id FROM cost_by_call WHERE node_id IS NOT NULL AND pipeline_id IS NOT NULL
84
- UNION SELECT pipeline_id, node FROM queue_items WHERE node IS NOT NULL AND pipeline_id IS NOT NULL
85
- UNION SELECT pipeline_id, node_id FROM node_health WHERE node_id IS NOT NULL
86
- )
87
- `;
88
-
89
- export const BACKFILL_ATTRIBUTION_SQL = [BACKFILL_FLOW_ITEMS_SQL, BACKFILL_TURN_RUNS_SQL, BACKFILL_PIPELINE_NODES_SQL];
90
-
91
- export async function hasLegacyBranches(client) {
92
- const { rows } = await client.execute(
93
- "SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = 'flow_branches'",
94
- );
95
- return rows.length > 0;
96
- }
97
-
98
- export async function backfillAttribution(client, now = Date.now()) {
99
- const statements = [...BACKFILL_ATTRIBUTION_SQL];
100
- if (await hasLegacyBranches(client)) statements.push(BACKFILL_LEGACY_BRANCH_ITEMS_SQL);
101
- await client.batch(statements.map((sql) => ({ sql, args: [now] })), "write");
102
- }