@igorjpimenta/manifold 0.0.0-stage → 2026.11.22-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.migrations/ops/0000_futuristic_karma.sql +111 -0
- package/.migrations/ops/0001_low_maximus.sql +7 -0
- package/.migrations/ops/meta/0000_snapshot.json +763 -0
- package/.migrations/ops/meta/0001_snapshot.json +173 -0
- package/.migrations/ops/meta/_journal.json +20 -0
- package/.migrations/runtime/0000_real_boomer.sql +180 -0
- package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
- package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
- package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
- package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
- package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
- package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
- package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
- package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
- package/.migrations/runtime/0009_lying_shocker.sql +96 -0
- package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
- package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
- package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
- package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
- package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
- package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
- package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
- package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
- package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
- package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
- package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
- package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
- package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
- package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
- package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
- package/.migrations/runtime/meta/_journal.json +97 -0
- package/README.md +1583 -2
- package/RELEASING.md +89 -0
- package/agent/fleet.js +100 -0
- package/bin/cli.js +854 -0
- package/bin/manifold.js +4 -0
- package/boot/flow-revival.js +536 -0
- package/budget/index.js +495 -0
- package/budget/lib.js +59 -0
- package/dashboard/cost.js +528 -0
- package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
- package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
- package/dashboard/dist/index.html +13 -0
- package/dashboard/event-log.js +267 -0
- package/dashboard/index.html +12 -0
- package/dashboard/package-lock.json +2877 -0
- package/dashboard/package.json +37 -0
- package/dashboard/pipeline-graph.js +131 -0
- package/dashboard/postcss.config.js +6 -0
- package/dashboard/search.js +183 -0
- package/dashboard/server.js +1015 -0
- package/dashboard/src/App.tsx +42 -0
- package/dashboard/src/api/client.ts +118 -0
- package/dashboard/src/api/types.ts +510 -0
- package/dashboard/src/api/ws.tsx +112 -0
- package/dashboard/src/components/CancelButton.tsx +63 -0
- package/dashboard/src/components/ClickToCopy.tsx +68 -0
- package/dashboard/src/components/CopyButton.tsx +61 -0
- package/dashboard/src/components/DailyCostTrend.tsx +85 -0
- package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
- package/dashboard/src/components/Funnel.tsx +149 -0
- package/dashboard/src/components/GateForm.tsx +158 -0
- package/dashboard/src/components/GraphView.tsx +302 -0
- package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
- package/dashboard/src/components/Histogram.tsx +292 -0
- package/dashboard/src/components/Layout.tsx +28 -0
- package/dashboard/src/components/NeedsAttention.tsx +98 -0
- package/dashboard/src/components/NodeContextCard.tsx +132 -0
- package/dashboard/src/components/PathTrail.tsx +35 -0
- package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
- package/dashboard/src/components/RerunButton.tsx +46 -0
- package/dashboard/src/components/RunComparison.tsx +53 -0
- package/dashboard/src/components/Sparkline.tsx +94 -0
- package/dashboard/src/components/StatStrip.tsx +147 -0
- package/dashboard/src/components/StateTree.tsx +84 -0
- package/dashboard/src/components/StatusBadge.tsx +18 -0
- package/dashboard/src/components/SubflowPathPills.tsx +38 -0
- package/dashboard/src/components/SubflowSummary.tsx +24 -0
- package/dashboard/src/components/SubflowsCard.tsx +327 -0
- package/dashboard/src/components/TodayStrip.tsx +86 -0
- package/dashboard/src/components/TranscriptPane.tsx +99 -0
- package/dashboard/src/components/TranscriptRow.tsx +69 -0
- package/dashboard/src/components/ui/badge.tsx +38 -0
- package/dashboard/src/components/ui/button.tsx +28 -0
- package/dashboard/src/components/ui/card.tsx +42 -0
- package/dashboard/src/components/ui/skeleton.tsx +9 -0
- package/dashboard/src/components/ui/table.tsx +62 -0
- package/dashboard/src/index.css +26 -0
- package/dashboard/src/lib/anomaly.ts +42 -0
- package/dashboard/src/lib/format.ts +30 -0
- package/dashboard/src/lib/step-history.ts +13 -0
- package/dashboard/src/lib/subflow.ts +154 -0
- package/dashboard/src/lib/use-click-outside.ts +28 -0
- package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
- package/dashboard/src/lib/utils.ts +6 -0
- package/dashboard/src/main.tsx +10 -0
- package/dashboard/src/types/shims.d.ts +4 -0
- package/dashboard/src/views/Home.tsx +478 -0
- package/dashboard/src/views/PipelineDetail.tsx +1185 -0
- package/dashboard/src/views/PipelinesList.tsx +111 -0
- package/dashboard/src/views/RunDetail.tsx +248 -0
- package/dashboard/src/views/Search.tsx +196 -0
- package/dashboard/stats.js +668 -0
- package/dashboard/tailwind.config.ts +43 -0
- package/dashboard/tree-stats.js +12 -0
- package/dashboard/tsconfig.json +25 -0
- package/dashboard/views.js +231 -0
- package/dashboard/vite.config.ts +34 -0
- package/drizzle.config.js +12 -0
- package/drizzle.config.ops.js +12 -0
- package/elevation.js +116 -0
- package/engine/admission-promotion-resume.js +127 -0
- package/engine/agent-dispatch.js +64 -0
- package/engine/child-resume-trigger.js +315 -0
- package/engine/executor-budget.js +87 -0
- package/engine/executor-cancel.js +83 -0
- package/engine/executor-errors.js +122 -0
- package/engine/executor-helpers.js +254 -0
- package/engine/executor-nodes.js +292 -0
- package/engine/executor-persist.js +212 -0
- package/engine/executor-subflow-children.js +275 -0
- package/engine/executor.js +1013 -0
- package/engine/global-concurrency.js +33 -0
- package/engine/loader.js +528 -0
- package/engine/needs-resolver.js +229 -0
- package/engine/pipelines-watcher.js +133 -0
- package/engine/result-replay.js +34 -0
- package/engine/revival-messages.js +11 -0
- package/engine/spawn-child.js +55 -0
- package/engine/wait-for-flow-settlement.js +109 -0
- package/events/catalog.js +26 -0
- package/events/emit.js +44 -0
- package/events/kinds.js +118 -0
- package/events/serve-ws.js +74 -0
- package/events/stream.js +35 -0
- package/events/ws-dialog.js +257 -0
- package/events/ws-frame.js +194 -0
- package/events/ws-run-events.js +70 -0
- package/events.js +32 -0
- package/gates/gate-resolver.js +576 -0
- package/gates/paused-flows.js +55 -0
- package/graphql/columns.js +67 -0
- package/graphql/schema.js +333 -0
- package/index.js +49 -0
- package/mcp/http.js +159 -0
- package/mcp/identity.js +94 -0
- package/mcp/stdio.js +80 -0
- package/mcp/tools.js +245 -0
- package/mcp/wire-ops.js +178 -0
- package/package.json +49 -4
- package/routes/bearer-auth.js +57 -0
- package/routes/discuss-audio.js +87 -0
- package/routes/discuss-component.js +39 -0
- package/routes/discuss-draft.js +159 -0
- package/routes/discuss-helpers.js +20 -0
- package/routes/discuss.js +848 -0
- package/routes/failures.js +104 -0
- package/routes/graphql.js +61 -0
- package/routes/health.js +54 -0
- package/routes/operator-discuss.js +193 -0
- package/routes/operator-resume.js +32 -0
- package/routes/operator-retry.js +301 -0
- package/routes/operator.js +539 -0
- package/routes/pipelines.js +118 -0
- package/routes/runs.js +1129 -0
- package/scripts/ci/import-check.mjs +50 -0
- package/scripts/link-core-self.mjs +31 -0
- package/scripts/with-guard.sh +24 -0
- package/server/app.js +503 -0
- package/server/registry.js +53 -0
- package/server/serve.js +163 -0
- package/state/admission-queue-store.js +325 -0
- package/state/attribution-registries.js +102 -0
- package/state/batch-writer.js +162 -0
- package/state/db-guard.js +63 -0
- package/state/db-health.js +27 -0
- package/state/db.js +122 -0
- package/state/discuss-store.js +299 -0
- package/state/failure-store.js +427 -0
- package/state/migrate.js +104 -0
- package/state/node-cost.js +67 -0
- package/state/node-executions.js +775 -0
- package/state/node-health-store.js +124 -0
- package/state/ops-merge.js +93 -0
- package/state/paths.js +146 -0
- package/state/pipeline-dirs.js +99 -0
- package/state/pipeline-registry.js +449 -0
- package/state/run-store.js +697 -0
- package/state/schema.js +438 -0
- package/state/schema.ops.js +44 -0
- package/state/session-index.js +177 -0
- package/state/session-keys.js +39 -0
- package/state/state-store.js +311 -0
- package/state/subflow-state.js +126 -0
- package/state/tool-handoff.js +169 -0
- package/telemetry/budget-alert.js +353 -0
- package/telemetry/cli.js +331 -0
- package/telemetry/grammar.js +119 -0
- package/telemetry/http-budget.js +132 -0
- package/telemetry/http-cost.js +670 -0
- package/telemetry/http-helpers.js +134 -0
- package/telemetry/http-tree.js +349 -0
- package/telemetry/http.js +44 -0
- package/telemetry/index.js +307 -0
- package/telemetry/ingest.js +134 -0
- package/telemetry/labels.js +115 -0
- package/telemetry/pricer.js +77 -0
- package/telemetry/resolver.js +314 -0
- package/telemetry/store.js +147 -0
- package/utils/run-patch.js +43 -0
package/server/serve.js
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// serve.js — `manifold serve`: the core application as its own process.
|
|
2
|
+
//
|
|
3
|
+
// Everything harness-specific arrives through the host this file builds:
|
|
4
|
+
// - agent turns go to MANIFOLD_TURN_URL (the harness adapter's endpoint)
|
|
5
|
+
// - discuss session ids are minted locally (no harness to ask)
|
|
6
|
+
// - elevation loops back to this process's own API port
|
|
7
|
+
//
|
|
8
|
+
// Loopback only by default; the routes authenticate with MANIFOLD_BEARER.
|
|
9
|
+
|
|
10
|
+
import http from "node:http";
|
|
11
|
+
import { createRouteRegistry } from "./registry.js";
|
|
12
|
+
import { createManifoldApp, DEFAULT_DASHBOARD_HOST, DEFAULT_DASHBOARD_PORT } from "./app.js";
|
|
13
|
+
import { createPipelineDirsStore } from "../state/pipeline-dirs.js";
|
|
14
|
+
import { resolveStorePaths } from "../state/paths.js";
|
|
15
|
+
|
|
16
|
+
export const DEFAULT_API_PORT = 7431;
|
|
17
|
+
export const DEFAULT_API_HOST = "127.0.0.1";
|
|
18
|
+
|
|
19
|
+
function createLogger(prefix) {
|
|
20
|
+
const line = (level, msg) => {
|
|
21
|
+
const text = `[${prefix}] ${msg}`;
|
|
22
|
+
if (level === "info") console.log(text);
|
|
23
|
+
else console.error(text);
|
|
24
|
+
};
|
|
25
|
+
return {
|
|
26
|
+
info: (msg) => line("info", msg),
|
|
27
|
+
warn: (msg) => line("warn", msg),
|
|
28
|
+
error: (msg) => line("error", msg),
|
|
29
|
+
debug: (msg) => line("debug", msg),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Deterministic discuss thread ids — core names the thread, the harness would
|
|
34
|
+
// have minted it. Same flow/gate/item always maps to the same thread.
|
|
35
|
+
export function localSessions() {
|
|
36
|
+
return {
|
|
37
|
+
newDiscussSessionId({ agentId, flowId, gateNodeId, itemId }) {
|
|
38
|
+
return `manifold:discuss:${flowId}:${gateNodeId ?? "flow"}:${itemId ?? "0"}:${agentId ?? "agent"}`;
|
|
39
|
+
},
|
|
40
|
+
// No harness model table: the caller's modelOverride wins, else default.
|
|
41
|
+
async resolveAgentEffectiveModelPrimary() { return null; },
|
|
42
|
+
isProviderCooldown() { return false; },
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function createHttpRunTurn({ url, token, timeoutMs = 600_000, logger, fetchImpl = globalThis.fetch }) {
|
|
47
|
+
return async (args) => {
|
|
48
|
+
if (!url) {
|
|
49
|
+
throw new Error("manifold: agent turn requested but MANIFOLD_TURN_URL is unset (agent nodes need a harness)");
|
|
50
|
+
}
|
|
51
|
+
const res = await fetchImpl(url, {
|
|
52
|
+
method: "POST",
|
|
53
|
+
headers: {
|
|
54
|
+
"content-type": "application/json",
|
|
55
|
+
...(token ? { authorization: `Bearer ${token}` } : {}),
|
|
56
|
+
},
|
|
57
|
+
body: JSON.stringify(args),
|
|
58
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
59
|
+
});
|
|
60
|
+
const payload = await res.json().catch(() => ({}));
|
|
61
|
+
if (!res.ok || payload?.ok === false) {
|
|
62
|
+
const detail = payload?.error ?? `HTTP ${res.status}`;
|
|
63
|
+
logger?.warn?.(`manifold: agent turn failed: ${detail}`);
|
|
64
|
+
throw new Error(`manifold: agent turn failed: ${detail}`);
|
|
65
|
+
}
|
|
66
|
+
return { text: payload.text ?? "", runId: payload.runId ?? null, usage: payload.usage ?? null };
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Elevation loopback against this process: the `_elev` handlers are mounted on
|
|
71
|
+
// the same server, so the operator-scope crossing the gateway needed collapses
|
|
72
|
+
// to a localhost call carrying the shared bearer.
|
|
73
|
+
export function createLocalElevation({ port, host, token, fetchImpl = globalThis.fetch }) {
|
|
74
|
+
return async function invokeElevated(path, payload, opts = {}) {
|
|
75
|
+
const timeoutMs = opts.timeoutMs ?? 30_000;
|
|
76
|
+
if (!token) {
|
|
77
|
+
return { ok: false, status: 503, body: JSON.stringify({ ok: false, code: "no_bearer", error: "MANIFOLD_BEARER unset" }), error: "MANIFOLD_BEARER unset" };
|
|
78
|
+
}
|
|
79
|
+
try {
|
|
80
|
+
const res = await fetchImpl(`http://${host}:${port}${path}`, {
|
|
81
|
+
method: "POST",
|
|
82
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${token}` },
|
|
83
|
+
body: JSON.stringify(payload ?? {}),
|
|
84
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
85
|
+
});
|
|
86
|
+
return { ok: res.ok, status: res.status, body: await res.text() };
|
|
87
|
+
} catch (err) {
|
|
88
|
+
return { ok: false, status: 0, body: "", error: err?.message ?? String(err) };
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function resolveServeConfig({ env = process.env } = {}) {
|
|
94
|
+
const storePaths = resolveStorePaths({ env });
|
|
95
|
+
const pipelineDirsStore = createPipelineDirsStore({ dbPath: storePaths.runtimeDb, logger: createLogger("manifold") });
|
|
96
|
+
let pipelinesDir = env.MANIFOLD_PIPELINES_DIR ?? null;
|
|
97
|
+
if (!pipelinesDir) {
|
|
98
|
+
pipelinesDir = pipelineDirsStore.resolveDir({
|
|
99
|
+
harness: env.MANIFOLD_HARNESS ?? null,
|
|
100
|
+
env,
|
|
101
|
+
dataDir: storePaths.dataDir,
|
|
102
|
+
configDir: null,
|
|
103
|
+
}).dir;
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
storePaths,
|
|
107
|
+
config: {
|
|
108
|
+
pipelinesDir,
|
|
109
|
+
ownerSessionKey: env.MANIFOLD_OWNER_SESSION_KEY || undefined,
|
|
110
|
+
discussDefaultAgent: env.MANIFOLD_DISCUSS_DEFAULT_AGENT || null,
|
|
111
|
+
dashboardEnabled: env.MANIFOLD_DASHBOARD_ENABLED !== "false",
|
|
112
|
+
dashboardPort: Number(env.MANIFOLD_DASHBOARD_PORT) || DEFAULT_DASHBOARD_PORT,
|
|
113
|
+
dashboardHost: env.MANIFOLD_DASHBOARD_HOST || DEFAULT_DASHBOARD_HOST,
|
|
114
|
+
globalConcurrency: Number(env.MANIFOLD_GLOBAL_CONCURRENCY) || undefined,
|
|
115
|
+
},
|
|
116
|
+
apiPort: Number(env.MANIFOLD_API_PORT) || DEFAULT_API_PORT,
|
|
117
|
+
apiHost: env.MANIFOLD_API_HOST || DEFAULT_API_HOST,
|
|
118
|
+
turnUrl: env.MANIFOLD_TURN_URL || null,
|
|
119
|
+
turnToken: env.MANIFOLD_TURN_TOKEN || env.MANIFOLD_BEARER || null,
|
|
120
|
+
bearer: env.MANIFOLD_BEARER || null,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export async function serve({ env = process.env, logger: injectedLogger = null } = {}) {
|
|
125
|
+
const cfg = resolveServeConfig({ env });
|
|
126
|
+
const logger = injectedLogger ?? createLogger("manifold");
|
|
127
|
+
|
|
128
|
+
const registry = createRouteRegistry({ logger });
|
|
129
|
+
const host = {
|
|
130
|
+
logger,
|
|
131
|
+
registerHttpRoute: registry.registerHttpRoute,
|
|
132
|
+
runTurn: createHttpRunTurn({ url: cfg.turnUrl, token: cfg.turnToken, logger }),
|
|
133
|
+
invokeElevated: createLocalElevation({ port: cfg.apiPort, host: cfg.apiHost, token: cfg.bearer }),
|
|
134
|
+
sessions: localSessions(),
|
|
135
|
+
getBroadcast: () => null,
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
const app = createManifoldApp({ host, config: cfg.config });
|
|
139
|
+
const server = http.createServer((req, res) => registry.handle(req, res));
|
|
140
|
+
|
|
141
|
+
await new Promise((resolve, reject) => {
|
|
142
|
+
server.once("error", reject);
|
|
143
|
+
server.listen(cfg.apiPort, cfg.apiHost, resolve);
|
|
144
|
+
});
|
|
145
|
+
logger.info(`manifold serve: api on http://${cfg.apiHost}:${cfg.apiPort} (${registry.routes.length} route(s))`);
|
|
146
|
+
|
|
147
|
+
await app.start({ logger });
|
|
148
|
+
logger.info("manifold serve: ready");
|
|
149
|
+
|
|
150
|
+
let stopping = false;
|
|
151
|
+
const shutdown = async (signal) => {
|
|
152
|
+
if (stopping) return;
|
|
153
|
+
stopping = true;
|
|
154
|
+
logger.info(`manifold serve: ${signal}, stopping`);
|
|
155
|
+
await app.stop({ logger }).catch(() => {});
|
|
156
|
+
await new Promise((resolve) => server.close(resolve));
|
|
157
|
+
process.exit(0);
|
|
158
|
+
};
|
|
159
|
+
process.once("SIGINT", () => { shutdown("SIGINT"); });
|
|
160
|
+
process.once("SIGTERM", () => { shutdown("SIGTERM"); });
|
|
161
|
+
|
|
162
|
+
return { app, registry, server, cfg, shutdown };
|
|
163
|
+
}
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
// admission-queue-store.js — the single durable admission table for flow
|
|
2
|
+
// RUNS: starting a flow (kind:"flow-start") and dispatching a subflow child
|
|
3
|
+
// (kind:"child-dispatch"). One row per run; nothing inside a running flow
|
|
4
|
+
// (agent nodes included) takes a slot of its own — a run executes one node
|
|
5
|
+
// at a time, so the run cap already bounds everything it does.
|
|
6
|
+
//
|
|
7
|
+
// ONE combined pool, not one per kind — deliberately. Both kinds compete
|
|
8
|
+
// for the same shared count (no `kind` filter on the admitted-count query
|
|
9
|
+
// or the promotion query below). This only works because admission tracks
|
|
10
|
+
// flow_runs.status, not just start/end: a flow-run's own admission entry
|
|
11
|
+
// (whichever kind spawned it) is held only while it's genuinely "running" —
|
|
12
|
+
// the moment it goes "blocked" (a human-gate wait, or its own
|
|
13
|
+
// children-pending wait while a subflow fans out), state-store.js's
|
|
14
|
+
// setWaiting releases that entry via releaseByFlow(), so it stops occupying
|
|
15
|
+
// a shared slot for however long it's actually idle. Without that, a parent
|
|
16
|
+
// would hold its slot for its ENTIRE lifetime while its children ALSO need
|
|
17
|
+
// slots — one fan-out could alone exhaust the whole pool.
|
|
18
|
+
//
|
|
19
|
+
// child-dispatch releases on blocked/terminal but deliberately
|
|
20
|
+
// does NOT re-request admission when a blocked child later resumes — the
|
|
21
|
+
// mechanism it replaced never re-checked the cap at resume time either
|
|
22
|
+
// (the cap only ever governed fan-out DISPATCH, not a continuously-
|
|
23
|
+
// enforced running ceiling), so this is exact behavioral parity, not a new
|
|
24
|
+
// gap. Whether flow-start (a whole run's own admission, not yet wired)
|
|
25
|
+
// needs real re-admission-on-resume is still open — later stage's call.
|
|
26
|
+
//
|
|
27
|
+
// Atomicity: every decision (count admitted, then claim admitted-or-queued)
|
|
28
|
+
// runs inside one client.transaction("write") — BEGIN IMMEDIATE, acquiring
|
|
29
|
+
// SQLite's writer lock before the SELECT even runs. A bare
|
|
30
|
+
// `await count-query` then, much later, a separate write is two non-atomic
|
|
31
|
+
// steps, racy across concurrent callers; this is the primitive that closes
|
|
32
|
+
// the window.
|
|
33
|
+
|
|
34
|
+
import path from "node:path";
|
|
35
|
+
import os from "node:os";
|
|
36
|
+
import { mkdirSync } from "node:fs";
|
|
37
|
+
import { randomUUID } from "node:crypto";
|
|
38
|
+
import { getSharedDb, withSharedWrite, resolveStorePaths } from "./db.js";
|
|
39
|
+
import { migrateDatabase, RUNTIME_MIGRATIONS_FOLDER } from "./migrate.js";
|
|
40
|
+
|
|
41
|
+
function defaultDbPath() {
|
|
42
|
+
return resolveStorePaths().runtimeDb;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Same Proxy-wrapped async-ready factory idiom as run-store.js/state-
|
|
46
|
+
// store.js — every method call transparently awaits store construction
|
|
47
|
+
// (migration included) before running.
|
|
48
|
+
export function createAdmissionQueueStore(opts = {}) {
|
|
49
|
+
const _p = _createImpl(opts);
|
|
50
|
+
return new Proxy({}, {
|
|
51
|
+
get(_, k) {
|
|
52
|
+
if (k === "then" || typeof k === "symbol") return undefined;
|
|
53
|
+
if (k === "ready") return () => _p.then(() => {});
|
|
54
|
+
return (...args) => _p.then((s) => {
|
|
55
|
+
if (typeof s[k] !== "function") throw new TypeError(`admission-queue-store: ${String(k)} is not a function`);
|
|
56
|
+
return s[k](...args);
|
|
57
|
+
});
|
|
58
|
+
},
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function _createImpl({ dbPath = defaultDbPath(), getLimit } = {}) {
|
|
63
|
+
if (typeof getLimit !== "function") {
|
|
64
|
+
throw new TypeError("admission-queue-store: getLimit() function is required (e.g. global-concurrency.js's getGlobalConcurrencyLimit)");
|
|
65
|
+
}
|
|
66
|
+
mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
67
|
+
await migrateDatabase({ dbPath, migrationsFolder: RUNTIME_MIGRATIONS_FOLDER });
|
|
68
|
+
const { client } = await getSharedDb(dbPath);
|
|
69
|
+
|
|
70
|
+
// Runs `fn(tx)` inside one BEGIN IMMEDIATE transaction, committing on
|
|
71
|
+
// success and always closing (which rolls back if not committed) — the
|
|
72
|
+
// exact try/finally shape @libsql/client's own docs specify as required.
|
|
73
|
+
// Serialized and retried on SQLITE_BUSY via state/db.js's
|
|
74
|
+
// withSharedWrite(), keyed by dbPath: @libsql/client's local sqlite3
|
|
75
|
+
// backend does NOT safely support multiple Transaction objects in flight
|
|
76
|
+
// concurrently on one shared connection — commit() itself fails with
|
|
77
|
+
// "cannot commit transaction - SQL statements in
|
|
78
|
+
// progress", consistent with concurrent Transaction objects interleaving
|
|
79
|
+
// execute() calls over the same underlying native connection and
|
|
80
|
+
// corrupting each other's in-flight statement state; a client-library
|
|
81
|
+
// limitation, not a SQLite-level one — BEGIN IMMEDIATE itself would have
|
|
82
|
+
// serialized this correctly at the file-lock level. Corroborated by this
|
|
83
|
+
// store's own test suite, which drives 30 genuinely concurrent admit()
|
|
84
|
+
// callers through it.
|
|
85
|
+
//
|
|
86
|
+
// Because withSharedWrite's queue is keyed by dbPath rather than private
|
|
87
|
+
// to this store, every OTHER writer sharing this same connection via
|
|
88
|
+
// db.js's getSharedDb (run-store.js's CAS writes, batch-writer.js's
|
|
89
|
+
// flushes for node-executions.js/node-health-store.js) serializes against
|
|
90
|
+
// this store's transactions too, once it also calls withSharedWrite for
|
|
91
|
+
// the same dbPath — closing the corruption/stall path this store's own
|
|
92
|
+
// private serializer could not (see test/unit/
|
|
93
|
+
// state/admission-queue-cross-module-contention.test.js): a plain
|
|
94
|
+
// client.execute()/db.update()/db.batch() issued on the same connection
|
|
95
|
+
// while a transaction() is open can fail instantly, bypassing the
|
|
96
|
+
// busy_timeout PRAGMA entirely, and can corrupt the transaction holder's
|
|
97
|
+
// own commit. Until every writer on this connection routes through the
|
|
98
|
+
// same primitive, that gap remains open for whichever writers haven't
|
|
99
|
+
// adopted it yet.
|
|
100
|
+
//
|
|
101
|
+
// Safe to retry the whole fn(tx) on a fresh transaction: an uncommitted
|
|
102
|
+
// transaction has zero durable effect (SQLite's own atomicity), so
|
|
103
|
+
// re-running every one of fn's statements against a new tx is not a
|
|
104
|
+
// partial-redo risk.
|
|
105
|
+
async function withWriteTx(fn) {
|
|
106
|
+
return withSharedWrite(dbPath, async () => {
|
|
107
|
+
const tx = await client.transaction("write");
|
|
108
|
+
try {
|
|
109
|
+
const result = await fn(tx);
|
|
110
|
+
await tx.commit();
|
|
111
|
+
return result;
|
|
112
|
+
} finally {
|
|
113
|
+
tx.close();
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Shared by release(id) and releaseByFlow() — deletes the row and, if it
|
|
119
|
+
// was occupying an admitted slot, promotes the oldest queued entry across
|
|
120
|
+
// the whole combined pool. Must run inside the caller's own withWriteTx
|
|
121
|
+
// callback (not its own transaction) so both release paths stay a single
|
|
122
|
+
// atomic decision.
|
|
123
|
+
async function releaseRowById(tx, id) {
|
|
124
|
+
const { rows: selfRows } = await tx.execute({
|
|
125
|
+
sql: `SELECT status FROM flow_admission_queue WHERE id = ?`,
|
|
126
|
+
args: [id],
|
|
127
|
+
});
|
|
128
|
+
const self = selfRows[0] ?? null;
|
|
129
|
+
if (!self) return { released: false, promoted: null };
|
|
130
|
+
|
|
131
|
+
await tx.execute({ sql: `DELETE FROM flow_admission_queue WHERE id = ?`, args: [id] });
|
|
132
|
+
|
|
133
|
+
let promoted = null;
|
|
134
|
+
if (self.status === "admitted") {
|
|
135
|
+
const { rows: nextRows } = await tx.execute({
|
|
136
|
+
sql: `SELECT id, kind, flow_id, node_id FROM flow_admission_queue
|
|
137
|
+
WHERE status = 'queued' ORDER BY created_at ASC LIMIT 1`,
|
|
138
|
+
args: [],
|
|
139
|
+
});
|
|
140
|
+
const next = nextRows[0] ?? null;
|
|
141
|
+
if (next) {
|
|
142
|
+
const now = Date.now();
|
|
143
|
+
await tx.execute({
|
|
144
|
+
sql: `UPDATE flow_admission_queue SET status = 'admitted', admitted_at = ? WHERE id = ?`,
|
|
145
|
+
args: [now, next.id],
|
|
146
|
+
});
|
|
147
|
+
promoted = { id: next.id, kind: next.kind, flowId: next.flow_id, nodeId: next.node_id };
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return { released: true, promoted };
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return {
|
|
154
|
+
// Decides admitted-vs-queued for one new entry, atomically against the
|
|
155
|
+
// CURRENT combined count of admitted rows (all kinds pooled — see this
|
|
156
|
+
// file's header). Returns { id, status }. Never throws on contention —
|
|
157
|
+
// "queued" is a normal, expected outcome, not an error.
|
|
158
|
+
//
|
|
159
|
+
// Idempotent per (kind, flowId, nodeId): if a live (admitted or queued)
|
|
160
|
+
// row already exists for this exact tuple, returns THAT row instead of
|
|
161
|
+
// inserting a second one. Without this, any caller that re-enters the
|
|
162
|
+
// same logical request (e.g. a resume racing a revival for the same
|
|
163
|
+
// flow) creates a genuine duplicate row, permanently shrinking the
|
|
164
|
+
// shared pool by one slot per occurrence.
|
|
165
|
+
//
|
|
166
|
+
// A queued caller is never woken in-process: when a slot frees,
|
|
167
|
+
// release() promotes the row and the releasing call site drives the
|
|
168
|
+
// flow forward through wakePromotedRelease (engine/admission-promotion-
|
|
169
|
+
// resume.js) — flip to 'running', then resume.
|
|
170
|
+
async admit({ kind, flowId, nodeId = null }) {
|
|
171
|
+
if (!kind) throw new TypeError("admission-queue-store.admit: kind is required");
|
|
172
|
+
if (!flowId) throw new TypeError("admission-queue-store.admit: flowId is required");
|
|
173
|
+
const limit = getLimit();
|
|
174
|
+
return withWriteTx(async (tx) => {
|
|
175
|
+
const { rows: existingRows } = await tx.execute({
|
|
176
|
+
sql: `SELECT id, status FROM flow_admission_queue
|
|
177
|
+
WHERE kind = ? AND flow_id = ? AND (node_id = ? OR (? IS NULL AND node_id IS NULL))
|
|
178
|
+
AND status IN ('admitted', 'queued')
|
|
179
|
+
LIMIT 1`,
|
|
180
|
+
args: [kind, flowId, nodeId, nodeId],
|
|
181
|
+
});
|
|
182
|
+
if (existingRows[0]) {
|
|
183
|
+
const { id, status } = existingRows[0];
|
|
184
|
+
return { id, status };
|
|
185
|
+
}
|
|
186
|
+
const { rows } = await tx.execute({
|
|
187
|
+
sql: `SELECT COUNT(*) AS n FROM flow_admission_queue WHERE status = 'admitted'`,
|
|
188
|
+
args: [],
|
|
189
|
+
});
|
|
190
|
+
const admittedCount = Number(rows[0]?.n ?? 0);
|
|
191
|
+
const id = randomUUID();
|
|
192
|
+
const now = Date.now();
|
|
193
|
+
const status = admittedCount < limit ? "admitted" : "queued";
|
|
194
|
+
await tx.execute({
|
|
195
|
+
sql: `INSERT INTO flow_admission_queue (id, kind, flow_id, node_id, status, created_at, admitted_at)
|
|
196
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
197
|
+
args: [id, kind, flowId, nodeId, status, now, status === "admitted" ? now : null],
|
|
198
|
+
});
|
|
199
|
+
return { id, status };
|
|
200
|
+
});
|
|
201
|
+
},
|
|
202
|
+
|
|
203
|
+
// Releases one entry by id. If it was occupying an admitted slot,
|
|
204
|
+
// atomically promotes the oldest queued entry — FIFO across the WHOLE
|
|
205
|
+
// combined pool, not scoped to the releasing entry's own kind, matching
|
|
206
|
+
// the one-pool design. Returns { released, promoted }; promoted is
|
|
207
|
+
// either null or { id, flowId, nodeId, kind } for the caller to wake
|
|
208
|
+
// via wakePromotedRelease. A release() for an id that no longer exists
|
|
209
|
+
// (already released, or never existed) is a safe no-op —
|
|
210
|
+
// { released: false, promoted: null } — never throws, since a caller
|
|
211
|
+
// racing its own cleanup against a process restart shouldn't have to
|
|
212
|
+
// special-case "already gone".
|
|
213
|
+
async release(id) {
|
|
214
|
+
if (!id) throw new TypeError("admission-queue-store.release: id is required");
|
|
215
|
+
return withWriteTx(async (tx) => releaseRowById(tx, id));
|
|
216
|
+
},
|
|
217
|
+
|
|
218
|
+
// Releases the CURRENT (admitted or queued) entry for a given
|
|
219
|
+
// (flowId, kind) pair, without the caller needing to carry its own
|
|
220
|
+
// admission id around — the shape a status-store hook wants: state-
|
|
221
|
+
// store.js's setWaiting/finish/skip/fail/cancel know the flowId that
|
|
222
|
+
// just left "running" (or went terminal), not any admission id, since
|
|
223
|
+
// the flow could have crashed and been revived by a different process
|
|
224
|
+
// between admit() and now. Same promotion/notify semantics as
|
|
225
|
+
// release(id); a no-op ({ released: false, promoted: null }) when this
|
|
226
|
+
// flow never held a live entry for this kind (true for every ordinary
|
|
227
|
+
// flow that never participated in this kind's admission at all — the
|
|
228
|
+
// common case, deliberately cheap: one indexed SELECT).
|
|
229
|
+
async releaseByFlow({ flowId, kind }) {
|
|
230
|
+
if (!flowId) throw new TypeError("admission-queue-store.releaseByFlow: flowId is required");
|
|
231
|
+
if (!kind) throw new TypeError("admission-queue-store.releaseByFlow: kind is required");
|
|
232
|
+
return withWriteTx(async (tx) => {
|
|
233
|
+
const { rows } = await tx.execute({
|
|
234
|
+
sql: `SELECT id FROM flow_admission_queue
|
|
235
|
+
WHERE flow_id = ? AND kind = ? AND status IN ('admitted', 'queued')
|
|
236
|
+
ORDER BY created_at DESC LIMIT 1`,
|
|
237
|
+
args: [flowId, kind],
|
|
238
|
+
});
|
|
239
|
+
const self = rows[0] ?? null;
|
|
240
|
+
if (!self) return { released: false, promoted: null };
|
|
241
|
+
return releaseRowById(tx, self.id);
|
|
242
|
+
});
|
|
243
|
+
},
|
|
244
|
+
|
|
245
|
+
// Promotes oldest queued rows into any free capacity, until the pool is
|
|
246
|
+
// full or nothing is queued. Returns the promoted entries ({ id, kind,
|
|
247
|
+
// flowId, nodeId }) for the caller to wake via wakePromotedRelease.
|
|
248
|
+
// Normal operation never needs this — every release promotes exactly
|
|
249
|
+
// one row into the slot it frees — but a pool that loses admitted rows
|
|
250
|
+
// without a release (a migration deleting a whole kind, a crash between
|
|
251
|
+
// a delete and its promotion) would otherwise sit below its limit with
|
|
252
|
+
// work still queued, until enough unrelated releases happened.
|
|
253
|
+
async fillFreeSlots() {
|
|
254
|
+
const limit = getLimit();
|
|
255
|
+
return withWriteTx(async (tx) => {
|
|
256
|
+
const { rows } = await tx.execute({
|
|
257
|
+
sql: `SELECT COUNT(*) AS n FROM flow_admission_queue WHERE status = 'admitted'`,
|
|
258
|
+
args: [],
|
|
259
|
+
});
|
|
260
|
+
const free = limit - Number(rows[0]?.n ?? 0);
|
|
261
|
+
if (free <= 0) return [];
|
|
262
|
+
const { rows: nextRows } = await tx.execute({
|
|
263
|
+
sql: `SELECT id, kind, flow_id, node_id FROM flow_admission_queue
|
|
264
|
+
WHERE status = 'queued' ORDER BY created_at ASC LIMIT ?`,
|
|
265
|
+
args: [free],
|
|
266
|
+
});
|
|
267
|
+
const now = Date.now();
|
|
268
|
+
const promoted = [];
|
|
269
|
+
for (const next of nextRows) {
|
|
270
|
+
await tx.execute({
|
|
271
|
+
sql: `UPDATE flow_admission_queue SET status = 'admitted', admitted_at = ? WHERE id = ?`,
|
|
272
|
+
args: [now, next.id],
|
|
273
|
+
});
|
|
274
|
+
promoted.push({ id: next.id, kind: next.kind, flowId: next.flow_id, nodeId: next.node_id });
|
|
275
|
+
}
|
|
276
|
+
return promoted;
|
|
277
|
+
});
|
|
278
|
+
},
|
|
279
|
+
|
|
280
|
+
// Observability / test hook — counts across the whole combined pool,
|
|
281
|
+
// not scoped to one kind (matching admit()/release()'s own scope).
|
|
282
|
+
async _stats() {
|
|
283
|
+
const { rows } = await client.execute(
|
|
284
|
+
`SELECT status, COUNT(*) AS n FROM flow_admission_queue GROUP BY status`
|
|
285
|
+
);
|
|
286
|
+
const out = { admitted: 0, queued: 0 };
|
|
287
|
+
for (const r of rows) out[r.status] = Number(r.n);
|
|
288
|
+
return out;
|
|
289
|
+
},
|
|
290
|
+
|
|
291
|
+
// Lists queued entries oldest-first — general observability / test hook.
|
|
292
|
+
// Not currently called from any boot-revival or dispatch path (see
|
|
293
|
+
// boot/flow-revival.js's own skip-direct-dispatch comment for how
|
|
294
|
+
// queued orphans are actually handled today).
|
|
295
|
+
async listByStatus(status) {
|
|
296
|
+
const { rows } = await client.execute({
|
|
297
|
+
sql: `SELECT id, kind, flow_id, node_id, created_at, admitted_at
|
|
298
|
+
FROM flow_admission_queue WHERE status = ? ORDER BY created_at ASC`,
|
|
299
|
+
args: [status],
|
|
300
|
+
});
|
|
301
|
+
return rows.map((r) => ({
|
|
302
|
+
id: r.id, kind: r.kind, flowId: r.flow_id, nodeId: r.node_id,
|
|
303
|
+
createdAt: Number(r.created_at),
|
|
304
|
+
admittedAt: r.admitted_at != null ? Number(r.admitted_at) : null,
|
|
305
|
+
}));
|
|
306
|
+
},
|
|
307
|
+
|
|
308
|
+
// Fetches one entry by id — callers releasing/inspecting their own
|
|
309
|
+
// admission handle without going through listByStatus's full scan.
|
|
310
|
+
async get(id) {
|
|
311
|
+
const { rows } = await client.execute({
|
|
312
|
+
sql: `SELECT id, kind, flow_id, node_id, status, created_at, admitted_at
|
|
313
|
+
FROM flow_admission_queue WHERE id = ?`,
|
|
314
|
+
args: [id],
|
|
315
|
+
});
|
|
316
|
+
const r = rows[0];
|
|
317
|
+
if (!r) return null;
|
|
318
|
+
return {
|
|
319
|
+
id: r.id, kind: r.kind, flowId: r.flow_id, nodeId: r.node_id, status: r.status,
|
|
320
|
+
createdAt: Number(r.created_at),
|
|
321
|
+
admittedAt: r.admitted_at != null ? Number(r.admitted_at) : null,
|
|
322
|
+
};
|
|
323
|
+
},
|
|
324
|
+
};
|
|
325
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// Attribution registries: item keys, node names and turn-run ids are written
|
|
2
|
+
// by engine and telemetry frames, so no table of ours owns the key space.
|
|
3
|
+
// These helpers materialize the key set — writers register what they are
|
|
4
|
+
// about to attribute, in the same write; the migration backfills the union
|
|
5
|
+
// so historical rows keep their attribution and the FKs are satisfiable.
|
|
6
|
+
//
|
|
7
|
+
// A writer that skips the guard fails its insert instead of writing an
|
|
8
|
+
// untracked pointer. Rows are append-only: pruning a flow or a branch must
|
|
9
|
+
// not erase the attribution that names it.
|
|
10
|
+
|
|
11
|
+
export const ENSURE_FLOW_ITEM_SQL =
|
|
12
|
+
"INSERT OR IGNORE INTO flow_items (flow_id, item_key, first_seen_at) VALUES (?, ?, ?)";
|
|
13
|
+
export const ENSURE_TURN_RUN_SQL =
|
|
14
|
+
"INSERT OR IGNORE INTO turn_runs (run_id, first_seen_at) VALUES (?, ?)";
|
|
15
|
+
export const ENSURE_PIPELINE_NODE_SQL =
|
|
16
|
+
"INSERT OR IGNORE INTO pipeline_nodes (pipeline_id, node_name, first_seen_at) VALUES (?, ?, ?)";
|
|
17
|
+
|
|
18
|
+
// Composite keys are skipped whole when any part is null: a partial pointer
|
|
19
|
+
// cannot be checked, and SQLite would treat the row as satisfying the FK
|
|
20
|
+
// anyway — registering half of it would only create noise.
|
|
21
|
+
function pairs(rows) {
|
|
22
|
+
const seen = new Map();
|
|
23
|
+
for (const [a, b] of rows) {
|
|
24
|
+
if (typeof a !== "string" || !a || typeof b !== "string" || !b) continue;
|
|
25
|
+
seen.set(`${a}\u0000${b}`, [a, b]);
|
|
26
|
+
}
|
|
27
|
+
return [...seen.values()];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function ensureFlowItems(client, rows, now = Date.now()) {
|
|
31
|
+
const keys = pairs(rows);
|
|
32
|
+
if (keys.length === 0) return;
|
|
33
|
+
await client.batch(keys.map(([flowId, itemKey]) => ({
|
|
34
|
+
sql: ENSURE_FLOW_ITEM_SQL, args: [flowId, itemKey, now],
|
|
35
|
+
})), "write");
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export async function ensureTurnRuns(client, runIds, now = Date.now()) {
|
|
39
|
+
const unique = [...new Set(runIds.filter((r) => typeof r === "string" && r.length > 0))];
|
|
40
|
+
if (unique.length === 0) return;
|
|
41
|
+
await client.batch(unique.map((runId) => ({
|
|
42
|
+
sql: ENSURE_TURN_RUN_SQL, args: [runId, now],
|
|
43
|
+
})), "write");
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export async function ensurePipelineNodes(client, rows, now = Date.now()) {
|
|
47
|
+
const keys = pairs(rows);
|
|
48
|
+
if (keys.length === 0) return;
|
|
49
|
+
await client.batch(keys.map(([pipelineId, nodeName]) => ({
|
|
50
|
+
sql: ENSURE_PIPELINE_NODE_SQL, args: [pipelineId, nodeName, now],
|
|
51
|
+
})), "write");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Union backfill, one pass. Node names also come from the version registry
|
|
55
|
+
// so a pipeline's declared nodes are known before their first turn. Each
|
|
56
|
+
// batch item is one statement; the legacy flow_branches table is read only
|
|
57
|
+
// when it exists, because files created before v7 still carry the branch
|
|
58
|
+
// keys that item attribution was written against.
|
|
59
|
+
export const BACKFILL_FLOW_ITEMS_SQL = `
|
|
60
|
+
INSERT OR IGNORE INTO flow_items (flow_id, item_key, first_seen_at)
|
|
61
|
+
SELECT flow_id, item_key, ? FROM (
|
|
62
|
+
SELECT flow_id, item_key FROM session_lineage WHERE item_key IS NOT NULL AND flow_id IS NOT NULL
|
|
63
|
+
UNION SELECT flow_id, item_key FROM cost_by_call WHERE item_key IS NOT NULL AND flow_id IS NOT NULL
|
|
64
|
+
UNION SELECT flow_id, branch_item_id FROM flow_sessions WHERE branch_item_id IS NOT NULL
|
|
65
|
+
)
|
|
66
|
+
`;
|
|
67
|
+
|
|
68
|
+
export const BACKFILL_LEGACY_BRANCH_ITEMS_SQL = `
|
|
69
|
+
INSERT OR IGNORE INTO flow_items (flow_id, item_key, first_seen_at)
|
|
70
|
+
SELECT DISTINCT flow_id, branch_key, ? FROM flow_branches
|
|
71
|
+
`;
|
|
72
|
+
|
|
73
|
+
export const BACKFILL_TURN_RUNS_SQL = `
|
|
74
|
+
INSERT OR IGNORE INTO turn_runs (run_id, first_seen_at)
|
|
75
|
+
SELECT run_id, ? FROM session_lineage WHERE run_id IS NOT NULL
|
|
76
|
+
`;
|
|
77
|
+
|
|
78
|
+
export const BACKFILL_PIPELINE_NODES_SQL = `
|
|
79
|
+
INSERT OR IGNORE INTO pipeline_nodes (pipeline_id, node_name, first_seen_at)
|
|
80
|
+
SELECT pipeline_id, node_name, ? FROM (
|
|
81
|
+
SELECT pipeline_id, node_name FROM pipeline_node_versions WHERE node_name IS NOT NULL
|
|
82
|
+
UNION SELECT pipeline_id, node_id FROM session_lineage WHERE node_id IS NOT NULL AND pipeline_id IS NOT NULL
|
|
83
|
+
UNION SELECT pipeline_id, node_id FROM cost_by_call WHERE node_id IS NOT NULL AND pipeline_id IS NOT NULL
|
|
84
|
+
UNION SELECT pipeline_id, node FROM queue_items WHERE node IS NOT NULL AND pipeline_id IS NOT NULL
|
|
85
|
+
UNION SELECT pipeline_id, node_id FROM node_health WHERE node_id IS NOT NULL
|
|
86
|
+
)
|
|
87
|
+
`;
|
|
88
|
+
|
|
89
|
+
export const BACKFILL_ATTRIBUTION_SQL = [BACKFILL_FLOW_ITEMS_SQL, BACKFILL_TURN_RUNS_SQL, BACKFILL_PIPELINE_NODES_SQL];
|
|
90
|
+
|
|
91
|
+
export async function hasLegacyBranches(client) {
|
|
92
|
+
const { rows } = await client.execute(
|
|
93
|
+
"SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = 'flow_branches'",
|
|
94
|
+
);
|
|
95
|
+
return rows.length > 0;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export async function backfillAttribution(client, now = Date.now()) {
|
|
99
|
+
const statements = [...BACKFILL_ATTRIBUTION_SQL];
|
|
100
|
+
if (await hasLegacyBranches(client)) statements.push(BACKFILL_LEGACY_BRANCH_ITEMS_SQL);
|
|
101
|
+
await client.batch(statements.map((sql) => ({ sql, args: [now] })), "write");
|
|
102
|
+
}
|