@igorjpimenta/manifold 0.0.0-bootstrap.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.migrations/ops/0000_futuristic_karma.sql +111 -0
- package/.migrations/ops/0001_low_maximus.sql +7 -0
- package/.migrations/ops/meta/0000_snapshot.json +763 -0
- package/.migrations/ops/meta/0001_snapshot.json +173 -0
- package/.migrations/ops/meta/_journal.json +20 -0
- package/.migrations/runtime/0000_real_boomer.sql +180 -0
- package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
- package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
- package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
- package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
- package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
- package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
- package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
- package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
- package/.migrations/runtime/0009_lying_shocker.sql +96 -0
- package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
- package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
- package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
- package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
- package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
- package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
- package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
- package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
- package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
- package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
- package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
- package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
- package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
- package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
- package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
- package/.migrations/runtime/meta/_journal.json +97 -0
- package/README.md +1584 -0
- package/RELEASING.md +89 -0
- package/agent/fleet.js +100 -0
- package/bin/cli.js +854 -0
- package/bin/manifold.js +4 -0
- package/boot/flow-revival.js +536 -0
- package/budget/index.js +495 -0
- package/budget/lib.js +59 -0
- package/dashboard/cost.js +528 -0
- package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
- package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
- package/dashboard/dist/index.html +13 -0
- package/dashboard/event-log.js +267 -0
- package/dashboard/index.html +12 -0
- package/dashboard/package-lock.json +2877 -0
- package/dashboard/package.json +37 -0
- package/dashboard/pipeline-graph.js +131 -0
- package/dashboard/postcss.config.js +6 -0
- package/dashboard/search.js +183 -0
- package/dashboard/server.js +1015 -0
- package/dashboard/src/App.tsx +42 -0
- package/dashboard/src/api/client.ts +118 -0
- package/dashboard/src/api/types.ts +510 -0
- package/dashboard/src/api/ws.tsx +112 -0
- package/dashboard/src/components/CancelButton.tsx +63 -0
- package/dashboard/src/components/ClickToCopy.tsx +68 -0
- package/dashboard/src/components/CopyButton.tsx +61 -0
- package/dashboard/src/components/DailyCostTrend.tsx +85 -0
- package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
- package/dashboard/src/components/Funnel.tsx +149 -0
- package/dashboard/src/components/GateForm.tsx +158 -0
- package/dashboard/src/components/GraphView.tsx +302 -0
- package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
- package/dashboard/src/components/Histogram.tsx +292 -0
- package/dashboard/src/components/Layout.tsx +28 -0
- package/dashboard/src/components/NeedsAttention.tsx +98 -0
- package/dashboard/src/components/NodeContextCard.tsx +132 -0
- package/dashboard/src/components/PathTrail.tsx +35 -0
- package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
- package/dashboard/src/components/RerunButton.tsx +46 -0
- package/dashboard/src/components/RunComparison.tsx +53 -0
- package/dashboard/src/components/Sparkline.tsx +94 -0
- package/dashboard/src/components/StatStrip.tsx +147 -0
- package/dashboard/src/components/StateTree.tsx +84 -0
- package/dashboard/src/components/StatusBadge.tsx +18 -0
- package/dashboard/src/components/SubflowPathPills.tsx +38 -0
- package/dashboard/src/components/SubflowSummary.tsx +24 -0
- package/dashboard/src/components/SubflowsCard.tsx +327 -0
- package/dashboard/src/components/TodayStrip.tsx +86 -0
- package/dashboard/src/components/TranscriptPane.tsx +99 -0
- package/dashboard/src/components/TranscriptRow.tsx +69 -0
- package/dashboard/src/components/ui/badge.tsx +38 -0
- package/dashboard/src/components/ui/button.tsx +28 -0
- package/dashboard/src/components/ui/card.tsx +42 -0
- package/dashboard/src/components/ui/skeleton.tsx +9 -0
- package/dashboard/src/components/ui/table.tsx +62 -0
- package/dashboard/src/index.css +26 -0
- package/dashboard/src/lib/anomaly.ts +42 -0
- package/dashboard/src/lib/format.ts +30 -0
- package/dashboard/src/lib/step-history.ts +13 -0
- package/dashboard/src/lib/subflow.ts +154 -0
- package/dashboard/src/lib/use-click-outside.ts +28 -0
- package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
- package/dashboard/src/lib/utils.ts +6 -0
- package/dashboard/src/main.tsx +10 -0
- package/dashboard/src/types/shims.d.ts +4 -0
- package/dashboard/src/views/Home.tsx +478 -0
- package/dashboard/src/views/PipelineDetail.tsx +1185 -0
- package/dashboard/src/views/PipelinesList.tsx +111 -0
- package/dashboard/src/views/RunDetail.tsx +248 -0
- package/dashboard/src/views/Search.tsx +196 -0
- package/dashboard/stats.js +668 -0
- package/dashboard/tailwind.config.ts +43 -0
- package/dashboard/tree-stats.js +12 -0
- package/dashboard/tsconfig.json +25 -0
- package/dashboard/views.js +231 -0
- package/dashboard/vite.config.ts +34 -0
- package/drizzle.config.js +12 -0
- package/drizzle.config.ops.js +12 -0
- package/elevation.js +116 -0
- package/engine/admission-promotion-resume.js +127 -0
- package/engine/agent-dispatch.js +64 -0
- package/engine/child-resume-trigger.js +315 -0
- package/engine/executor-budget.js +87 -0
- package/engine/executor-cancel.js +83 -0
- package/engine/executor-errors.js +122 -0
- package/engine/executor-helpers.js +254 -0
- package/engine/executor-nodes.js +292 -0
- package/engine/executor-persist.js +212 -0
- package/engine/executor-subflow-children.js +275 -0
- package/engine/executor.js +1013 -0
- package/engine/global-concurrency.js +33 -0
- package/engine/loader.js +528 -0
- package/engine/needs-resolver.js +229 -0
- package/engine/pipelines-watcher.js +133 -0
- package/engine/result-replay.js +34 -0
- package/engine/revival-messages.js +11 -0
- package/engine/spawn-child.js +55 -0
- package/engine/wait-for-flow-settlement.js +109 -0
- package/events/catalog.js +26 -0
- package/events/emit.js +44 -0
- package/events/kinds.js +118 -0
- package/events/serve-ws.js +74 -0
- package/events/stream.js +35 -0
- package/events/ws-dialog.js +257 -0
- package/events/ws-frame.js +194 -0
- package/events/ws-run-events.js +70 -0
- package/events.js +32 -0
- package/gates/gate-resolver.js +576 -0
- package/gates/paused-flows.js +55 -0
- package/graphql/columns.js +67 -0
- package/graphql/schema.js +333 -0
- package/index.js +49 -0
- package/mcp/http.js +159 -0
- package/mcp/identity.js +94 -0
- package/mcp/stdio.js +80 -0
- package/mcp/tools.js +245 -0
- package/mcp/wire-ops.js +178 -0
- package/package.json +51 -0
- package/routes/bearer-auth.js +57 -0
- package/routes/discuss-audio.js +87 -0
- package/routes/discuss-component.js +39 -0
- package/routes/discuss-draft.js +159 -0
- package/routes/discuss-helpers.js +20 -0
- package/routes/discuss.js +848 -0
- package/routes/failures.js +104 -0
- package/routes/graphql.js +61 -0
- package/routes/health.js +54 -0
- package/routes/operator-discuss.js +193 -0
- package/routes/operator-resume.js +32 -0
- package/routes/operator-retry.js +301 -0
- package/routes/operator.js +539 -0
- package/routes/pipelines.js +118 -0
- package/routes/runs.js +1129 -0
- package/scripts/ci/import-check.mjs +50 -0
- package/scripts/link-core-self.mjs +31 -0
- package/scripts/with-guard.sh +24 -0
- package/server/app.js +503 -0
- package/server/registry.js +53 -0
- package/server/serve.js +163 -0
- package/state/admission-queue-store.js +325 -0
- package/state/attribution-registries.js +102 -0
- package/state/batch-writer.js +162 -0
- package/state/db-guard.js +63 -0
- package/state/db-health.js +27 -0
- package/state/db.js +122 -0
- package/state/discuss-store.js +299 -0
- package/state/failure-store.js +427 -0
- package/state/migrate.js +104 -0
- package/state/node-cost.js +67 -0
- package/state/node-executions.js +775 -0
- package/state/node-health-store.js +124 -0
- package/state/ops-merge.js +93 -0
- package/state/paths.js +146 -0
- package/state/pipeline-dirs.js +99 -0
- package/state/pipeline-registry.js +449 -0
- package/state/run-store.js +697 -0
- package/state/schema.js +438 -0
- package/state/schema.ops.js +44 -0
- package/state/session-index.js +177 -0
- package/state/session-keys.js +39 -0
- package/state/state-store.js +311 -0
- package/state/subflow-state.js +126 -0
- package/state/tool-handoff.js +169 -0
- package/telemetry/budget-alert.js +353 -0
- package/telemetry/cli.js +331 -0
- package/telemetry/grammar.js +119 -0
- package/telemetry/http-budget.js +132 -0
- package/telemetry/http-cost.js +670 -0
- package/telemetry/http-helpers.js +134 -0
- package/telemetry/http-tree.js +349 -0
- package/telemetry/http.js +44 -0
- package/telemetry/index.js +307 -0
- package/telemetry/ingest.js +134 -0
- package/telemetry/labels.js +115 -0
- package/telemetry/pricer.js +77 -0
- package/telemetry/resolver.js +314 -0
- package/telemetry/store.js +147 -0
- package/utils/run-patch.js +43 -0
|
@@ -0,0 +1,449 @@
|
|
|
1
|
+
// pipeline-registry.js — SCD Type 2 history of pipeline/node DEFINITIONS
|
|
2
|
+
// (kind/agent/mode), not their execution history (node-executions.js
|
|
3
|
+
// already covers that). Answers "what did node X look like at time T" —
|
|
4
|
+
// pipelinesRef.value is a live, mutable, single-copy in-memory Map with
|
|
5
|
+
// no history of its own, so once a node is renamed/removed or its whole
|
|
6
|
+
// pipeline file is deleted, every past flow that dispatched under the
|
|
7
|
+
// old shape loses the ability to render correctly, forever, with no
|
|
8
|
+
// error — just a silent fallback to kind:null.
|
|
9
|
+
//
|
|
10
|
+
// Write path: reconcilePipelineVersions(pipelinesMap, {logger}) — called
|
|
11
|
+
// once right after the boot-time pipeline load resolves, then again on
|
|
12
|
+
// every pipelines-watcher reload tick. Diffs the given Map against the
|
|
13
|
+
// currently-active (retired_at IS NULL) row per (pipeline_id[, node_name]):
|
|
14
|
+
// content unchanged → no-op; changed or previously absent → retire the
|
|
15
|
+
// old active row (if any) and insert a fresh one; a pipeline_id/node_name
|
|
16
|
+
// that no longer appears in the Map at all → retire without inserting
|
|
17
|
+
// (its last version stays queryable, just marked retired).
|
|
18
|
+
//
|
|
19
|
+
// Read path: getNodeVersionsBatch([{pipelineId, nodeName, atMs}, ...]) —
|
|
20
|
+
// point-in-time lookup, batched to avoid one query per row: fetches every
|
|
21
|
+
// version for each distinct (pipeline_id, node_name) pair in one query,
|
|
22
|
+
// then matches each request's own atMs against those in memory. (The
|
|
23
|
+
// `nodeId` request param is the node's CURRENT name, same convention as
|
|
24
|
+
// every other caller in the codebase — see the node-identity note below
|
|
25
|
+
// for why it's matched against the `node_name` column internally.)
|
|
26
|
+
//
|
|
27
|
+
// Explicit scope boundary: this store answers "what was true," never
|
|
28
|
+
// "what should happen now." Anything deciding CURRENT behavior (mode
|
|
29
|
+
// enforcement in engine/child-resume-trigger.js, dispatch/edge-routing in
|
|
30
|
+
// engine/executor.js) keeps reading pipelinesRef directly, unchanged —
|
|
31
|
+
// conflating "what did happen" with "what should happen now" would be a
|
|
32
|
+
// real correctness bug (a long-running fan-out reacting to a stale mode
|
|
33
|
+
// from whenever it first dispatched instead of the pipeline's current
|
|
34
|
+
// logic), not a simplification.
|
|
35
|
+
//
|
|
36
|
+
// content_hash is sha256 over a canonicalized subset of fields only —
|
|
37
|
+
// `run`/`edges`/`selector`/etc. are functions, not meaningfully hashable,
|
|
38
|
+
// and nothing reads them historically. A reload tick where nothing
|
|
39
|
+
// structurally meaningful changed is a no-op, not a new row.
|
|
40
|
+
//
|
|
41
|
+
// Node identity across a rename: `node_name` is a point-in-time DISPLAY
|
|
42
|
+
// LABEL (e.g. "SUBMIT_SUMMARY" — same string as the wider codebase's
|
|
43
|
+
// `nodeId`, flow_node_invocations.nodeId, pipeline.nodes' object keys;
|
|
44
|
+
// nothing outside this file changes). `node_id` is the node's PERSISTENT
|
|
45
|
+
// identity, independent of its current name. A pipeline author can
|
|
46
|
+
// optionally declare it directly on a node: `nodes: { SUBMIT_SUMMARY: {
|
|
47
|
+
// kind: "code", id: "<any stable string, author's choice>" } }` — when
|
|
48
|
+
// present, that declared value IS the node_id, verbatim, every time.
|
|
49
|
+
// Renaming a node is then just "change the object key, leave `id` alone"
|
|
50
|
+
// — the identity travels with the node's own declaration, no separate
|
|
51
|
+
// rename-link step. A node that never declares `id` still gets one
|
|
52
|
+
// (minted the first time it's ever reconciled, reused as long as its name
|
|
53
|
+
// doesn't change) so getNodeLineageBatch works uniformly either way — an
|
|
54
|
+
// undeclared node just can't survive a FUTURE rename without the author
|
|
55
|
+
// adding `id` at that point, same trade-off as any opt-in metadata.
|
|
56
|
+
// Two (or more) node_names can legitimately share one declared `id` at
|
|
57
|
+
// the same time — a genuine N:1 merge falls out for free, no special
|
|
58
|
+
// case needed. See dashboard/stats.js's autoFunnelFromNodes for the
|
|
59
|
+
// consumer.
|
|
60
|
+
|
|
61
|
+
import { randomUUID, createHash } from "node:crypto";
|
|
62
|
+
import { mkdirSync } from "node:fs";
|
|
63
|
+
import path from "node:path";
|
|
64
|
+
import os from "node:os";
|
|
65
|
+
import { getSharedDb, withSharedWrite, resolveStorePaths } from "./db.js";
|
|
66
|
+
import { migrateDatabase, RUNTIME_MIGRATIONS_FOLDER } from "./migrate.js";
|
|
67
|
+
|
|
68
|
+
function defaultDbPath() {
|
|
69
|
+
return resolveStorePaths().runtimeDb;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function hashOf(obj) {
|
|
73
|
+
return createHash("sha256").update(JSON.stringify(obj)).digest("hex");
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function pipelineContentHash(pipeline) {
|
|
77
|
+
return hashOf({ kind: pipeline.kind ?? null, allowConcurrent: pipeline.allowConcurrent ?? null });
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function nodeContentHash(node) {
|
|
81
|
+
return hashOf({
|
|
82
|
+
kind: node.kind,
|
|
83
|
+
agent: node.kind === "agent" ? (node.agent ?? null) : null,
|
|
84
|
+
mode: node.kind === "subflow" ? (node.mode ?? null) : null,
|
|
85
|
+
// Declaring (or changing) `id` on an otherwise-unchanged node must
|
|
86
|
+
// still trigger a retire+reinsert — without this, _reconcileOneNode's
|
|
87
|
+
// hash-match short-circuit would return before ever reading node.id,
|
|
88
|
+
// silently ignoring a newly-added declaration on any node whose
|
|
89
|
+
// kind/agent/mode didn't also change (the exact retroactive-backfill
|
|
90
|
+
// case: adding `id` to nodes that already exist and aren't otherwise
|
|
91
|
+
// being edited).
|
|
92
|
+
id: node.id ?? null,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
let _singleton = null;
|
|
97
|
+
|
|
98
|
+
function _makeStoreProxy(_p) {
|
|
99
|
+
return new Proxy({}, { get(_, k) {
|
|
100
|
+
if (k === "then" || typeof k === "symbol") return undefined;
|
|
101
|
+
if (k === "_db") return new Proxy({}, { get(__, method) {
|
|
102
|
+
if (method === "then" || typeof method === "symbol") return undefined;
|
|
103
|
+
return (...a) => _p.then(s => s._db[method](...a));
|
|
104
|
+
} });
|
|
105
|
+
return (...a) => _p.then(s => { if (typeof s[k] !== "function") throw new TypeError(`pipeline-registry: ${String(k)} is not a function`); return s[k](...a); });
|
|
106
|
+
} });
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function getPipelineRegistryStore(dbPath = defaultDbPath()) {
|
|
110
|
+
if (!_singleton) {
|
|
111
|
+
_singleton = _makeStoreProxy(_createStore(dbPath));
|
|
112
|
+
}
|
|
113
|
+
return _singleton;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Exported for tests: creates an independent store (bypasses the singleton).
|
|
117
|
+
export function createPipelineRegistryStore(dbPath) {
|
|
118
|
+
return _makeStoreProxy(_createStore(dbPath));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function _createStore(dbPath) {
|
|
122
|
+
let client;
|
|
123
|
+
try {
|
|
124
|
+
mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
125
|
+
await migrateDatabase({ dbPath, migrationsFolder: RUNTIME_MIGRATIONS_FOLDER });
|
|
126
|
+
({ client } = await getSharedDb(dbPath));
|
|
127
|
+
} catch {
|
|
128
|
+
// Purely additive/historical-rendering feature (see this file's own
|
|
129
|
+
// top comment) — if the DB can't open, degrade to a no-op store
|
|
130
|
+
// rather than take pipeline loading down with it.
|
|
131
|
+
const noop = () => {};
|
|
132
|
+
return {
|
|
133
|
+
_db: null,
|
|
134
|
+
reconcilePipelineVersions: async () => {},
|
|
135
|
+
getNodeVersionAt: async () => null,
|
|
136
|
+
getNodeVersionsBatch: async (requests) => (requests ?? []).map(() => null),
|
|
137
|
+
getPipelineVersionAt: async () => null,
|
|
138
|
+
getNodeLineageBatch: async () => new Map(),
|
|
139
|
+
close: noop,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function _activePipelineRow(pipelineId) {
|
|
144
|
+
const { rows } = await client.execute({
|
|
145
|
+
sql: `SELECT id, content_hash FROM pipeline_versions WHERE pipeline_id = ? AND retired_at IS NULL`,
|
|
146
|
+
args: [pipelineId],
|
|
147
|
+
});
|
|
148
|
+
return rows[0] ?? null;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function _activeNodeRow(pipelineId, nodeName) {
|
|
152
|
+
const { rows } = await client.execute({
|
|
153
|
+
sql: `SELECT id, content_hash, node_id FROM pipeline_node_versions WHERE pipeline_id = ? AND node_name = ? AND retired_at IS NULL`,
|
|
154
|
+
args: [pipelineId, nodeName],
|
|
155
|
+
});
|
|
156
|
+
return rows[0] ?? null;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Reads still happen immediately (against the pre-reconcile committed
|
|
160
|
+
// state — see reconcilePipelineVersions' own comment on why that's safe
|
|
161
|
+
// even with writes deferred). The retire+insert pair itself is pushed
|
|
162
|
+
// onto `queries` instead of executed here — reconcilePipelineVersions
|
|
163
|
+
// commits every pipeline's and every node's pair, plus the orphan-
|
|
164
|
+
// retirement pass, in ONE client.batch() call at the end. That makes the
|
|
165
|
+
// whole SCD2 rotation atomic — otherwise a crash mid-reconcile could
|
|
166
|
+
// leave a pipeline with zero active version rows (retired-with-no-
|
|
167
|
+
// replacement) — and collapses what was N individual round-trips into one.
|
|
168
|
+
async function _reconcileOnePipeline(pipelineId, pipeline, now, queries) {
|
|
169
|
+
const hash = pipelineContentHash(pipeline);
|
|
170
|
+
const active = await _activePipelineRow(pipelineId);
|
|
171
|
+
if (active && active.content_hash === hash) return active.id;
|
|
172
|
+
if (active) {
|
|
173
|
+
queries.push({ sql: `UPDATE pipeline_versions SET retired_at = ? WHERE id = ?`, args: [now, active.id] });
|
|
174
|
+
}
|
|
175
|
+
const id = randomUUID();
|
|
176
|
+
queries.push({
|
|
177
|
+
sql: `INSERT INTO pipelines (id, name, created_at, updated_at)
|
|
178
|
+
VALUES (?, ?, ?, ?) ON CONFLICT(id) DO NOTHING`,
|
|
179
|
+
args: [pipelineId, pipelineId, now, now],
|
|
180
|
+
});
|
|
181
|
+
if (pipeline._parentPipelineId) {
|
|
182
|
+
queries.push({
|
|
183
|
+
sql: `INSERT INTO pipelines (id, name, created_at, updated_at)
|
|
184
|
+
VALUES (?, ?, ?, ?) ON CONFLICT(id) DO NOTHING`,
|
|
185
|
+
args: [pipeline._parentPipelineId, pipeline._parentPipelineId, now, now],
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
queries.push({
|
|
189
|
+
sql: `INSERT INTO pipeline_versions
|
|
190
|
+
(id, pipeline_id, kind, allow_concurrent, is_synthetic, parent_pipeline_id, source_path, content_hash, registered_at, retired_at)
|
|
191
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`,
|
|
192
|
+
args: [
|
|
193
|
+
id, pipelineId, pipeline.kind ?? null,
|
|
194
|
+
pipeline.allowConcurrent !== undefined ? JSON.stringify(pipeline.allowConcurrent) : null,
|
|
195
|
+
pipeline._synthetic ? 1 : 0,
|
|
196
|
+
pipeline._parentPipelineId ?? null,
|
|
197
|
+
pipeline._filepath ?? null,
|
|
198
|
+
hash, now,
|
|
199
|
+
],
|
|
200
|
+
});
|
|
201
|
+
return id;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Resolves the persistent node_id for a row about to be written.
|
|
205
|
+
// `activeRow` is this exact node_name's own currently-active row, if
|
|
206
|
+
// any (already fetched by the caller). `declaredId` is `node.id` read
|
|
207
|
+
// straight off the pipeline definition — author-declared, always wins
|
|
208
|
+
// when present, no lookup needed.
|
|
209
|
+
async function _resolveNodeIdentity(pipelineId, nodeName, activeRow, declaredId) {
|
|
210
|
+
if (declaredId) return declaredId;
|
|
211
|
+
// Name unchanged (content just drifted) — same identity as before.
|
|
212
|
+
if (activeRow) return activeRow.node_id;
|
|
213
|
+
// A node reappearing after removal (no active row, no declared id,
|
|
214
|
+
// but it HAS existed before under this exact name) keeps its old
|
|
215
|
+
// identity; a genuinely brand new, undeclared node mints a fresh one.
|
|
216
|
+
const { rows } = await client.execute({
|
|
217
|
+
sql: `SELECT node_id FROM pipeline_node_versions
|
|
218
|
+
WHERE pipeline_id = ? AND node_name = ? ORDER BY registered_at DESC LIMIT 1`,
|
|
219
|
+
args: [pipelineId, nodeName],
|
|
220
|
+
});
|
|
221
|
+
return rows[0]?.node_id ?? randomUUID();
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// See _reconcileOnePipeline's comment — same deferred-write pattern.
|
|
225
|
+
async function _reconcileOneNode(pipelineVersionId, pipelineId, nodeName, node, now, queries) {
|
|
226
|
+
// The runtime node name is the attribution key the engine writes
|
|
227
|
+
// (flow_node_invocations, node_health, lineage, telemetry), so it must
|
|
228
|
+
// exist before the first turn runs — register it on every reconcile,
|
|
229
|
+
// including a content-green one.
|
|
230
|
+
queries.push({
|
|
231
|
+
sql: `INSERT OR IGNORE INTO pipeline_nodes (pipeline_id, node_name, first_seen_at) VALUES (?, ?, ?)`,
|
|
232
|
+
args: [pipelineId, nodeName, now],
|
|
233
|
+
});
|
|
234
|
+
const hash = nodeContentHash(node);
|
|
235
|
+
const active = await _activeNodeRow(pipelineId, nodeName);
|
|
236
|
+
if (active && active.content_hash === hash) return;
|
|
237
|
+
const nodeId = await _resolveNodeIdentity(pipelineId, nodeName, active, node.id ?? null);
|
|
238
|
+
if (active) {
|
|
239
|
+
queries.push({ sql: `UPDATE pipeline_node_versions SET retired_at = ? WHERE id = ?`, args: [now, active.id] });
|
|
240
|
+
}
|
|
241
|
+
queries.push({
|
|
242
|
+
sql: `INSERT INTO pipeline_node_versions
|
|
243
|
+
(id, pipeline_version_id, pipeline_id, node_name, kind, agent, mode, content_hash, registered_at, retired_at, node_id)
|
|
244
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)`,
|
|
245
|
+
args: [
|
|
246
|
+
randomUUID(), pipelineVersionId, pipelineId, nodeName,
|
|
247
|
+
node.kind, node.kind === "agent" ? (node.agent ?? null) : null,
|
|
248
|
+
node.kind === "subflow" ? (node.mode ?? null) : null,
|
|
249
|
+
hash, now, nodeId,
|
|
250
|
+
],
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// Reads throughout this pass (_activePipelineRow/_activeNodeRow inside
|
|
255
|
+
// the per-pipeline/per-node loop, plus the two orphan-detection SELECTs
|
|
256
|
+
// below) all see the same pre-reconcile committed state, since every
|
|
257
|
+
// write is deferred into `queries` and only actually committed in the
|
|
258
|
+
// single client.batch() call at the very end — including the two
|
|
259
|
+
// orphan-detection SELECTs, which correctly still see an about-to-be-
|
|
260
|
+
// retired active row as "active" (retired_at IS NULL) and skip it via
|
|
261
|
+
// the seen-set check, same outcome as before, just simpler to reason
|
|
262
|
+
// about since nothing this pass writes is visible to its own reads.
|
|
263
|
+
async function reconcilePipelineVersions(pipelinesMap, { logger } = {}) {
|
|
264
|
+
if (!pipelinesMap || pipelinesMap.size === 0) return;
|
|
265
|
+
const now = Date.now();
|
|
266
|
+
const seenPipelineIds = [];
|
|
267
|
+
const seenNodeKeys = [];
|
|
268
|
+
const queries = [];
|
|
269
|
+
try {
|
|
270
|
+
for (const [pipelineId, pipeline] of pipelinesMap) {
|
|
271
|
+
seenPipelineIds.push(pipelineId);
|
|
272
|
+
const pipelineVersionId = await _reconcileOnePipeline(pipelineId, pipeline, now, queries);
|
|
273
|
+
for (const [nodeName, node] of Object.entries(pipeline.nodes ?? {})) {
|
|
274
|
+
seenNodeKeys.push([pipelineId, nodeName]);
|
|
275
|
+
await _reconcileOneNode(pipelineVersionId, pipelineId, nodeName, node, now, queries);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// Retire any active row whose pipeline/node no longer appears in
|
|
280
|
+
// the Map at all — a pipeline or node that's been removed. Its
|
|
281
|
+
// last version stays queryable, just marked retired.
|
|
282
|
+
const { rows: activePipelines } = await client.execute(
|
|
283
|
+
`SELECT id, pipeline_id FROM pipeline_versions WHERE retired_at IS NULL`,
|
|
284
|
+
);
|
|
285
|
+
const seenPipelineSet = new Set(seenPipelineIds);
|
|
286
|
+
for (const row of activePipelines) {
|
|
287
|
+
if (!seenPipelineSet.has(row.pipeline_id)) {
|
|
288
|
+
queries.push({ sql: `UPDATE pipeline_versions SET retired_at = ? WHERE id = ?`, args: [now, row.id] });
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
const { rows: activeNodes } = await client.execute(
|
|
293
|
+
`SELECT id, pipeline_id, node_name FROM pipeline_node_versions WHERE retired_at IS NULL`,
|
|
294
|
+
);
|
|
295
|
+
const seenNodeSet = new Set(seenNodeKeys.map(([p, n]) => `${p} ${n}`));
|
|
296
|
+
for (const row of activeNodes) {
|
|
297
|
+
if (!seenNodeSet.has(`${row.pipeline_id} ${row.node_name}`)) {
|
|
298
|
+
queries.push({ sql: `UPDATE pipeline_node_versions SET retired_at = ? WHERE id = ?`, args: [now, row.id] });
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
if (queries.length > 0) {
|
|
303
|
+
await withSharedWrite(dbPath, () => client.batch(queries));
|
|
304
|
+
}
|
|
305
|
+
} catch (err) {
|
|
306
|
+
logger?.warn?.(`manifold pipeline-registry: reconcile failed: ${err?.message ?? err}`);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
async function getNodeVersionAt({ pipelineId, nodeName, atMs }) {
|
|
311
|
+
const results = await getNodeVersionsBatch([{ pipelineId, nodeName, atMs }]);
|
|
312
|
+
return results[0] ?? null;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// Batched point-in-time lookup — one query per distinct (pipeline_id,
|
|
316
|
+
// node_name) pair, not one per request. Returns results in the SAME
|
|
317
|
+
// ORDER as `requests`; a pair with NO recorded
|
|
318
|
+
// version at all resolves to null — callers keep their existing
|
|
319
|
+
// kind:null fallback for that case. See resolveVersionAt below for how
|
|
320
|
+
// a timestamp older than any recorded version is handled (NOT null).
|
|
321
|
+
async function getNodeVersionsBatch(requests) {
|
|
322
|
+
if (!Array.isArray(requests) || requests.length === 0) return [];
|
|
323
|
+
const pairKey = (p, n) => `${p} ${n}`;
|
|
324
|
+
const uniquePairs = new Map(); // pairKey -> {pipelineId, nodeName}
|
|
325
|
+
for (const r of requests) uniquePairs.set(pairKey(r.pipelineId, r.nodeName), r);
|
|
326
|
+
|
|
327
|
+
const versionsByPair = new Map(); // pairKey -> [{kind, agent, registeredAt, retiredAt}]
|
|
328
|
+
for (const { pipelineId, nodeName } of uniquePairs.values()) {
|
|
329
|
+
const { rows } = await client.execute({
|
|
330
|
+
sql: `SELECT kind, agent, registered_at, retired_at FROM pipeline_node_versions
|
|
331
|
+
WHERE pipeline_id = ? AND node_name = ? ORDER BY registered_at ASC`,
|
|
332
|
+
args: [pipelineId, nodeName],
|
|
333
|
+
});
|
|
334
|
+
versionsByPair.set(pairKey(pipelineId, nodeName), rows.map((r) => ({
|
|
335
|
+
kind: r.kind, agent: r.agent ?? null,
|
|
336
|
+
registeredAt: Number(r.registered_at),
|
|
337
|
+
retiredAt: r.retired_at == null ? null : Number(r.retired_at),
|
|
338
|
+
})));
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
return requests.map(({ pipelineId, nodeName, atMs }) => resolveVersionAt(versionsByPair.get(pairKey(pipelineId, nodeName)), atMs));
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// Shared by getNodeVersionsBatch/getPipelineVersionAt. `versions` must
|
|
345
|
+
// already be sorted ascending by registeredAt.
|
|
346
|
+
//
|
|
347
|
+
// atMs predating the EARLIEST recorded version is expected, not an edge
|
|
348
|
+
// case: reconcile only starts recording from whenever this feature first
|
|
349
|
+
// shipped/booted -- every flow that dispatched before that point has
|
|
350
|
+
// real invocations with no real historical record to match. Retire-
|
|
351
|
+
// then-insert always uses one shared `now` for both timestamps (see
|
|
352
|
+
// reconcilePipelineVersions), so once the earliest version exists there
|
|
353
|
+
// are never gaps between versions -- only ever a gap BEFORE the first
|
|
354
|
+
// one. Falling back to that earliest version there (rather than null)
|
|
355
|
+
// is the best available answer, not a guess invented from nothing: the
|
|
356
|
+
// vast majority of nodes are never touched between "this flow ran" and
|
|
357
|
+
// "this feature started tracking pipelines," so the earliest known
|
|
358
|
+
// shape is usually the exact right one. Returning null here -- the
|
|
359
|
+
// naive SCD2 read -- would silently regress correct rendering for
|
|
360
|
+
// every already-existing flow the moment this feature ships, which is
|
|
361
|
+
// a worse outcome than an occasional wrong guess for the rare node
|
|
362
|
+
// that changed in that exact gap.
|
|
363
|
+
function findVersionRowAt(versions, atMs) {
|
|
364
|
+
if (!versions?.length) return null;
|
|
365
|
+
const exact = versions.find((v) => v.registeredAt <= atMs && (v.retiredAt == null || v.retiredAt > atMs));
|
|
366
|
+
return exact ?? (atMs < versions[0].registeredAt ? versions[0] : null);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
function resolveVersionAt(versions, atMs) {
|
|
370
|
+
const match = findVersionRowAt(versions, atMs);
|
|
371
|
+
return match ? { kind: match.kind, agent: match.agent } : null;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
// Pipeline-level (not node-level) point-in-time lookup — e.g. "was this
|
|
375
|
+
// pipeline cascade-kind at time T" (routes/runs.js's cascadeAttemptFor).
|
|
376
|
+
// Single-row lookup, not batched: only one call site needs this today,
|
|
377
|
+
// one per child flow rather than per-node, so an N+1 concern doesn't
|
|
378
|
+
// apply the way it does for getNodeVersionsBatch. Same
|
|
379
|
+
// older-than-earliest-known-version fallback as findVersionRowAt above.
|
|
380
|
+
async function getPipelineVersionAt({ pipelineId, atMs }) {
|
|
381
|
+
const { rows } = await client.execute({
|
|
382
|
+
sql: `SELECT kind, allow_concurrent, registered_at, retired_at FROM pipeline_versions
|
|
383
|
+
WHERE pipeline_id = ? ORDER BY registered_at ASC`,
|
|
384
|
+
args: [pipelineId],
|
|
385
|
+
});
|
|
386
|
+
const versions = rows.map((r) => ({
|
|
387
|
+
kind: r.kind, allow_concurrent: r.allow_concurrent,
|
|
388
|
+
registeredAt: Number(r.registered_at),
|
|
389
|
+
retiredAt: r.retired_at == null ? null : Number(r.retired_at),
|
|
390
|
+
}));
|
|
391
|
+
const match = findVersionRowAt(versions, atMs);
|
|
392
|
+
if (!match) return null;
|
|
393
|
+
let allowConcurrent = null;
|
|
394
|
+
try { allowConcurrent = match.allow_concurrent == null ? null : JSON.parse(match.allow_concurrent); } catch { /* leave null */ }
|
|
395
|
+
return { kind: match.kind, allowConcurrent };
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// Resolves, for each of a pipeline's CURRENT node names, the full set of
|
|
399
|
+
// node_name strings that share its persistent node_id — i.e. every name
|
|
400
|
+
// this identity has ever been called, whether from an author-declared
|
|
401
|
+
// `id` or an auto-minted one. A node with no rename history resolves to
|
|
402
|
+
// a singleton set containing just itself. Two queries total (not one
|
|
403
|
+
// per requested name): first the active rows for the requested names to
|
|
404
|
+
// learn their node_ids, then one query for every row that shares any of
|
|
405
|
+
// those node_ids. Consumed by dashboard/stats.js's autoFunnelFromNodes
|
|
406
|
+
// so an older run's flow_node_invocations.nodeId (recorded under
|
|
407
|
+
// whatever name was live at the time) still counts toward the CURRENT
|
|
408
|
+
// node's funnel stage.
|
|
409
|
+
async function getNodeLineageBatch(pipelineId, nodeIds) {
|
|
410
|
+
if (!Array.isArray(nodeIds) || nodeIds.length === 0) return new Map();
|
|
411
|
+
const { rows: active } = await client.execute({
|
|
412
|
+
sql: `SELECT node_name, node_id FROM pipeline_node_versions
|
|
413
|
+
WHERE pipeline_id = ? AND node_name IN (${nodeIds.map(() => "?").join(",")}) AND retired_at IS NULL`,
|
|
414
|
+
args: [pipelineId, ...nodeIds],
|
|
415
|
+
});
|
|
416
|
+
const identityByNodeName = new Map(active.map((r) => [r.node_name, r.node_id]));
|
|
417
|
+
const identities = [...new Set(identityByNodeName.values())].filter(Boolean);
|
|
418
|
+
|
|
419
|
+
const namesByIdentity = new Map();
|
|
420
|
+
if (identities.length > 0) {
|
|
421
|
+
const { rows: all } = await client.execute({
|
|
422
|
+
sql: `SELECT DISTINCT node_name, node_id FROM pipeline_node_versions
|
|
423
|
+
WHERE node_id IN (${identities.map(() => "?").join(",")})`,
|
|
424
|
+
args: identities,
|
|
425
|
+
});
|
|
426
|
+
for (const r of all) {
|
|
427
|
+
if (!namesByIdentity.has(r.node_id)) namesByIdentity.set(r.node_id, new Set());
|
|
428
|
+
namesByIdentity.get(r.node_id).add(r.node_name);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
const result = new Map();
|
|
433
|
+
for (const nodeId of nodeIds) {
|
|
434
|
+
const identity = identityByNodeName.get(nodeId);
|
|
435
|
+
result.set(nodeId, (identity && namesByIdentity.get(identity)) || new Set([nodeId]));
|
|
436
|
+
}
|
|
437
|
+
return result;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
return {
|
|
441
|
+
_db: client,
|
|
442
|
+
reconcilePipelineVersions,
|
|
443
|
+
getNodeVersionAt,
|
|
444
|
+
getNodeVersionsBatch,
|
|
445
|
+
getPipelineVersionAt,
|
|
446
|
+
getNodeLineageBatch,
|
|
447
|
+
close() { try { client.close(); } catch { /* ignore */ } },
|
|
448
|
+
};
|
|
449
|
+
}
|