@igorjpimenta/manifold 0.0.0-stage → 2026.11.22-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.migrations/ops/0000_futuristic_karma.sql +111 -0
- package/.migrations/ops/0001_low_maximus.sql +7 -0
- package/.migrations/ops/meta/0000_snapshot.json +763 -0
- package/.migrations/ops/meta/0001_snapshot.json +173 -0
- package/.migrations/ops/meta/_journal.json +20 -0
- package/.migrations/runtime/0000_real_boomer.sql +180 -0
- package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
- package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
- package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
- package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
- package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
- package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
- package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
- package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
- package/.migrations/runtime/0009_lying_shocker.sql +96 -0
- package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
- package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
- package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
- package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
- package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
- package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
- package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
- package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
- package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
- package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
- package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
- package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
- package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
- package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
- package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
- package/.migrations/runtime/meta/_journal.json +97 -0
- package/README.md +1583 -2
- package/RELEASING.md +89 -0
- package/agent/fleet.js +100 -0
- package/bin/cli.js +854 -0
- package/bin/manifold.js +4 -0
- package/boot/flow-revival.js +536 -0
- package/budget/index.js +495 -0
- package/budget/lib.js +59 -0
- package/dashboard/cost.js +528 -0
- package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
- package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
- package/dashboard/dist/index.html +13 -0
- package/dashboard/event-log.js +267 -0
- package/dashboard/index.html +12 -0
- package/dashboard/package-lock.json +2877 -0
- package/dashboard/package.json +37 -0
- package/dashboard/pipeline-graph.js +131 -0
- package/dashboard/postcss.config.js +6 -0
- package/dashboard/search.js +183 -0
- package/dashboard/server.js +1015 -0
- package/dashboard/src/App.tsx +42 -0
- package/dashboard/src/api/client.ts +118 -0
- package/dashboard/src/api/types.ts +510 -0
- package/dashboard/src/api/ws.tsx +112 -0
- package/dashboard/src/components/CancelButton.tsx +63 -0
- package/dashboard/src/components/ClickToCopy.tsx +68 -0
- package/dashboard/src/components/CopyButton.tsx +61 -0
- package/dashboard/src/components/DailyCostTrend.tsx +85 -0
- package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
- package/dashboard/src/components/Funnel.tsx +149 -0
- package/dashboard/src/components/GateForm.tsx +158 -0
- package/dashboard/src/components/GraphView.tsx +302 -0
- package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
- package/dashboard/src/components/Histogram.tsx +292 -0
- package/dashboard/src/components/Layout.tsx +28 -0
- package/dashboard/src/components/NeedsAttention.tsx +98 -0
- package/dashboard/src/components/NodeContextCard.tsx +132 -0
- package/dashboard/src/components/PathTrail.tsx +35 -0
- package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
- package/dashboard/src/components/RerunButton.tsx +46 -0
- package/dashboard/src/components/RunComparison.tsx +53 -0
- package/dashboard/src/components/Sparkline.tsx +94 -0
- package/dashboard/src/components/StatStrip.tsx +147 -0
- package/dashboard/src/components/StateTree.tsx +84 -0
- package/dashboard/src/components/StatusBadge.tsx +18 -0
- package/dashboard/src/components/SubflowPathPills.tsx +38 -0
- package/dashboard/src/components/SubflowSummary.tsx +24 -0
- package/dashboard/src/components/SubflowsCard.tsx +327 -0
- package/dashboard/src/components/TodayStrip.tsx +86 -0
- package/dashboard/src/components/TranscriptPane.tsx +99 -0
- package/dashboard/src/components/TranscriptRow.tsx +69 -0
- package/dashboard/src/components/ui/badge.tsx +38 -0
- package/dashboard/src/components/ui/button.tsx +28 -0
- package/dashboard/src/components/ui/card.tsx +42 -0
- package/dashboard/src/components/ui/skeleton.tsx +9 -0
- package/dashboard/src/components/ui/table.tsx +62 -0
- package/dashboard/src/index.css +26 -0
- package/dashboard/src/lib/anomaly.ts +42 -0
- package/dashboard/src/lib/format.ts +30 -0
- package/dashboard/src/lib/step-history.ts +13 -0
- package/dashboard/src/lib/subflow.ts +154 -0
- package/dashboard/src/lib/use-click-outside.ts +28 -0
- package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
- package/dashboard/src/lib/utils.ts +6 -0
- package/dashboard/src/main.tsx +10 -0
- package/dashboard/src/types/shims.d.ts +4 -0
- package/dashboard/src/views/Home.tsx +478 -0
- package/dashboard/src/views/PipelineDetail.tsx +1185 -0
- package/dashboard/src/views/PipelinesList.tsx +111 -0
- package/dashboard/src/views/RunDetail.tsx +248 -0
- package/dashboard/src/views/Search.tsx +196 -0
- package/dashboard/stats.js +668 -0
- package/dashboard/tailwind.config.ts +43 -0
- package/dashboard/tree-stats.js +12 -0
- package/dashboard/tsconfig.json +25 -0
- package/dashboard/views.js +231 -0
- package/dashboard/vite.config.ts +34 -0
- package/drizzle.config.js +12 -0
- package/drizzle.config.ops.js +12 -0
- package/elevation.js +116 -0
- package/engine/admission-promotion-resume.js +127 -0
- package/engine/agent-dispatch.js +64 -0
- package/engine/child-resume-trigger.js +315 -0
- package/engine/executor-budget.js +87 -0
- package/engine/executor-cancel.js +83 -0
- package/engine/executor-errors.js +122 -0
- package/engine/executor-helpers.js +254 -0
- package/engine/executor-nodes.js +292 -0
- package/engine/executor-persist.js +212 -0
- package/engine/executor-subflow-children.js +275 -0
- package/engine/executor.js +1013 -0
- package/engine/global-concurrency.js +33 -0
- package/engine/loader.js +528 -0
- package/engine/needs-resolver.js +229 -0
- package/engine/pipelines-watcher.js +133 -0
- package/engine/result-replay.js +34 -0
- package/engine/revival-messages.js +11 -0
- package/engine/spawn-child.js +55 -0
- package/engine/wait-for-flow-settlement.js +109 -0
- package/events/catalog.js +26 -0
- package/events/emit.js +44 -0
- package/events/kinds.js +118 -0
- package/events/serve-ws.js +74 -0
- package/events/stream.js +35 -0
- package/events/ws-dialog.js +257 -0
- package/events/ws-frame.js +194 -0
- package/events/ws-run-events.js +70 -0
- package/events.js +32 -0
- package/gates/gate-resolver.js +576 -0
- package/gates/paused-flows.js +55 -0
- package/graphql/columns.js +67 -0
- package/graphql/schema.js +333 -0
- package/index.js +49 -0
- package/mcp/http.js +159 -0
- package/mcp/identity.js +94 -0
- package/mcp/stdio.js +80 -0
- package/mcp/tools.js +245 -0
- package/mcp/wire-ops.js +178 -0
- package/package.json +49 -4
- package/routes/bearer-auth.js +57 -0
- package/routes/discuss-audio.js +87 -0
- package/routes/discuss-component.js +39 -0
- package/routes/discuss-draft.js +159 -0
- package/routes/discuss-helpers.js +20 -0
- package/routes/discuss.js +848 -0
- package/routes/failures.js +104 -0
- package/routes/graphql.js +61 -0
- package/routes/health.js +54 -0
- package/routes/operator-discuss.js +193 -0
- package/routes/operator-resume.js +32 -0
- package/routes/operator-retry.js +301 -0
- package/routes/operator.js +539 -0
- package/routes/pipelines.js +118 -0
- package/routes/runs.js +1129 -0
- package/scripts/ci/import-check.mjs +50 -0
- package/scripts/link-core-self.mjs +31 -0
- package/scripts/with-guard.sh +24 -0
- package/server/app.js +503 -0
- package/server/registry.js +53 -0
- package/server/serve.js +163 -0
- package/state/admission-queue-store.js +325 -0
- package/state/attribution-registries.js +102 -0
- package/state/batch-writer.js +162 -0
- package/state/db-guard.js +63 -0
- package/state/db-health.js +27 -0
- package/state/db.js +122 -0
- package/state/discuss-store.js +299 -0
- package/state/failure-store.js +427 -0
- package/state/migrate.js +104 -0
- package/state/node-cost.js +67 -0
- package/state/node-executions.js +775 -0
- package/state/node-health-store.js +124 -0
- package/state/ops-merge.js +93 -0
- package/state/paths.js +146 -0
- package/state/pipeline-dirs.js +99 -0
- package/state/pipeline-registry.js +449 -0
- package/state/run-store.js +697 -0
- package/state/schema.js +438 -0
- package/state/schema.ops.js +44 -0
- package/state/session-index.js +177 -0
- package/state/session-keys.js +39 -0
- package/state/state-store.js +311 -0
- package/state/subflow-state.js +126 -0
- package/state/tool-handoff.js +169 -0
- package/telemetry/budget-alert.js +353 -0
- package/telemetry/cli.js +331 -0
- package/telemetry/grammar.js +119 -0
- package/telemetry/http-budget.js +132 -0
- package/telemetry/http-cost.js +670 -0
- package/telemetry/http-helpers.js +134 -0
- package/telemetry/http-tree.js +349 -0
- package/telemetry/http.js +44 -0
- package/telemetry/index.js +307 -0
- package/telemetry/ingest.js +134 -0
- package/telemetry/labels.js +115 -0
- package/telemetry/pricer.js +77 -0
- package/telemetry/resolver.js +314 -0
- package/telemetry/store.js +147 -0
- package/utils/run-patch.js +43 -0
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
// needs-resolver.js — Option B, per-node "needs:": a node declares
|
|
2
|
+
// `needs: { varName: {hops, nodeId} }`, keyed by the local variable name
|
|
3
|
+
// it'll be read as (an ORM-style relation declaration, not an array) —
|
|
4
|
+
// and resolves each one lazily, only when `ctx.needs(varName)` is actually
|
|
5
|
+
// awaited inside that node's own run()/promptBuilder()/selector(), not
|
|
6
|
+
// fetched in bulk before dispatch. A branch of code that never calls
|
|
7
|
+
// ctx.needs("x") never pays for or blocks on that declared need.
|
|
8
|
+
//
|
|
9
|
+
// True per-property-access laziness (a bare `state._needs.x` read
|
|
10
|
+
// transparently blocking) was considered and rejected: this codebase's
|
|
11
|
+
// only SQLite driver (@libsql/client) is fully async, and the only way to
|
|
12
|
+
// make an async read look synchronous from inside a plain property getter
|
|
13
|
+
// is Atomics.wait blocking the ENTIRE process's event loop — which would
|
|
14
|
+
// stall every other concurrently-dispatching flow and every HTTP request
|
|
15
|
+
// on this gateway for the duration of the read. `ctx.needs(varName)` as an
|
|
16
|
+
// async function call gets the same lazy-only-if-touched property safely,
|
|
17
|
+
// using async/await machinery this engine already has (node.run() is
|
|
18
|
+
// already awaited today) instead of inventing dangerous new plumbing.
|
|
19
|
+
//
|
|
20
|
+
// Deliberately its own module, not folded into executor-helpers.js:
|
|
21
|
+
// resolveOneNeed needs areSiblingsAllTerminal (child-resume-trigger.js),
|
|
22
|
+
// which itself imports isTerminal from executor-helpers.js — importing it
|
|
23
|
+
// back the other way would be a cycle.
|
|
24
|
+
//
|
|
25
|
+
// hops:1 = immediate parent flow, hops:2 = grandparent, etc. Walked via
|
|
26
|
+
// flow_runs.parent_invocation_id ->
|
|
27
|
+
// flow_node_invocations.getInvocation().flowId -> repeat — bounded by the
|
|
28
|
+
// caller's own declared (small, explicit) hop count, so no new denormalized
|
|
29
|
+
// parent_invocation_id column on flow_node_invocations is needed even for
|
|
30
|
+
// hops > 1: a bounded loop of existing point lookups is exactly as cheap.
|
|
31
|
+
//
|
|
32
|
+
// hops:0 = same-flow — the declaring node's own flow, no walk at all
|
|
33
|
+
// (walkUpHops runs zero loop iterations and returns flowId unchanged).
|
|
34
|
+
// A node can opt into the same explicit, lazy, distinguishes-"not-ready"-
|
|
35
|
+
// from-"never" accessor for same-flow reads too, instead of two different
|
|
36
|
+
// idioms depending on whether the source is same-flow or cross-flow.
|
|
37
|
+
//
|
|
38
|
+
// A second declaration shape, `{hops, entry}` (mutually exclusive with
|
|
39
|
+
// `nodeId`/`config`): reaches an ancestor's own ENTRY/bookkeeping field —
|
|
40
|
+
// runId, args, pipelineId, cost, etc. — set once at that flow's own
|
|
41
|
+
// start() (or updated directly by gate-resolver.js), never via any node's
|
|
42
|
+
// own resultJson. This is a different read path entirely from
|
|
43
|
+
// `nodeId`-based needs: no resultJson replay, no "not ready yet" state (an
|
|
44
|
+
// entry field exists the instant its flow does), just a direct read of
|
|
45
|
+
// that ancestor flow's own persisted stateJson at the target hop. `entry`
|
|
46
|
+
// is validated at load time (engine/loader.js) against ENTRY_FIELDS below
|
|
47
|
+
// — `gates` is deliberately excluded even though it's a real stateJson
|
|
48
|
+
// key, since it has its own dedicated, staleness-safe accessor
|
|
49
|
+
// (gate_records) and reading it through here would reopen exactly the bug
|
|
50
|
+
// that mechanism was built to close.
|
|
51
|
+
//
|
|
52
|
+
// A third shape, `{hops, config}` (mutually exclusive with the other two):
|
|
53
|
+
// reaches a pipeline-DECLARED static config field (index.js's own `state:`
|
|
54
|
+
// block — e.g. sourceFilter, minClusterSize) at the target hop. Config is
|
|
55
|
+
// code, not data: it's never written to stateJson (toPersistableState
|
|
56
|
+
// never carries it — see executor-persist.js), never subject to that
|
|
57
|
+
// narrowing, and resolved live off the currently-loaded pipeline
|
|
58
|
+
// definition via pipelinesRef, same source start() itself reads from.
|
|
59
|
+
// Always "ready" the instant the target flow exists, same as `entry` — it
|
|
60
|
+
// doesn't depend on any node having run. Unlike `entry`, not validated
|
|
61
|
+
// against a fixed key set at load time (pipelines declare arbitrary
|
|
62
|
+
// `state:` shapes, no schema) — an unknown key just resolves `undefined`.
|
|
63
|
+
import { getSubflowGroup } from "../state/subflow-state.js";
|
|
64
|
+
import { areSiblingsAllTerminal } from "./child-resume-trigger.js";
|
|
65
|
+
import { NodeBlocked } from "./executor-errors.js";
|
|
66
|
+
|
|
67
|
+
export const ENTRY_FIELDS = new Set([
|
|
68
|
+
"runId", "startedAt", "args", "triggeredBy", "pipelineId",
|
|
69
|
+
"cost", "_display",
|
|
70
|
+
"approvedIds", "rejectedIds", "editedTexts", "approvalPending", "approvalResults",
|
|
71
|
+
"_cascadeError",
|
|
72
|
+
]);
|
|
73
|
+
|
|
74
|
+
async function walkUpHops(flowId, hops, runStore, nodeExecutionsStore) {
|
|
75
|
+
let currentFlowId = flowId;
|
|
76
|
+
for (let i = 0; i < hops; i++) {
|
|
77
|
+
const flow = await runStore.get(currentFlowId);
|
|
78
|
+
const parentInvocationId = flow?.parentInvocationId ?? null;
|
|
79
|
+
if (!parentInvocationId) return null; // walked off the top — fewer real ancestors than declared
|
|
80
|
+
const inv = await nodeExecutionsStore.getInvocation(parentInvocationId);
|
|
81
|
+
if (!inv?.flowId) return null;
|
|
82
|
+
currentFlowId = inv.flowId;
|
|
83
|
+
}
|
|
84
|
+
return currentFlowId;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Three-state result per declared need:
|
|
88
|
+
// "not-found" — walked off the top, OR the target flow has no
|
|
89
|
+
// invocation/subflow-fan-out for nodeId at all (never dispatched
|
|
90
|
+
// there) — a needs: entry naming the wrong hop count or a nodeId
|
|
91
|
+
// that hasn't run yet both land here; the caller treats both as
|
|
92
|
+
// "not ready, recheck later" the same way it treats an ordinary
|
|
93
|
+
// not-ready result, since a not-yet-dispatched ancestor node is not
|
|
94
|
+
// distinguishable from "give it more time" without knowing that
|
|
95
|
+
// ancestor's own future dispatch order, which this accessor has no
|
|
96
|
+
// way to see.
|
|
97
|
+
// "not-ready" — invocation row (or subflow fan-out) exists but isn't
|
|
98
|
+
// complete yet: resultWrittenAt still null for an ordinary node, or
|
|
99
|
+
// not every sibling is terminal yet for a subflow-sourced one.
|
|
100
|
+
// "ready" — value is the resolved data. A subflow-sourced need's value
|
|
101
|
+
// is the same { [subflowKey]: row } aggregate shape _subflowGroups
|
|
102
|
+
// already exposes elsewhere, not a different mechanism. An over-cap
|
|
103
|
+
// ordinary result carries the same { _resultTooLarge, bytes } marker
|
|
104
|
+
// assembleState() already uses.
|
|
105
|
+
export async function resolveOneNeed({ flowId, hops, nodeId, runStore, nodeExecutionsStore }) {
|
|
106
|
+
const targetFlowId = await walkUpHops(flowId, hops, runStore, nodeExecutionsStore);
|
|
107
|
+
if (!targetFlowId) return { status: "not-found" };
|
|
108
|
+
|
|
109
|
+
// Subflow-sourced check first: a real subflow node's own dispatch never
|
|
110
|
+
// writes an ordinary resultJson row for itself (dispatchSubflowChildren
|
|
111
|
+
// always returns stateUpdate:{}, per this doc's own corrected "SCORE is
|
|
112
|
+
// itself a subflow" resolution), so an ordinary getResultForNode lookup
|
|
113
|
+
// for a subflow node's id would always read as "not-found" even once its
|
|
114
|
+
// fan-out is real and complete. getSubflowGroup only ever returns real
|
|
115
|
+
// rows for a node that has actually spawned childFlows children, so
|
|
116
|
+
// checking it first (rather than guessing from a pipeline definition
|
|
117
|
+
// this accessor doesn't have on hand) is unambiguous.
|
|
118
|
+
const group = await getSubflowGroup({ flowId: targetFlowId, groupId: nodeId, runStore, nodeExecutionsStore });
|
|
119
|
+
if (group && Object.keys(group).length > 0) {
|
|
120
|
+
const { allTerminal, reason } = await areSiblingsAllTerminal({ flowId: targetFlowId, nodeId, runStore, nodeExecutionsStore });
|
|
121
|
+
if (reason === "no_siblings") return { status: "not-found" };
|
|
122
|
+
if (!allTerminal) return { status: "not-ready" };
|
|
123
|
+
return { status: "ready", value: group };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const row = await nodeExecutionsStore.getResultForNode(targetFlowId, nodeId);
|
|
127
|
+
if (!row) return { status: "not-found" };
|
|
128
|
+
if (row.resultWrittenAt == null) return { status: "not-ready" };
|
|
129
|
+
if (row.resultTooLargeBytes != null && row.resultJson == null) {
|
|
130
|
+
// writeResult() (state/node-executions.js) does not cap results, so this
|
|
131
|
+
// can only fire for a row that kept nothing at all — the real
|
|
132
|
+
// value isn't recoverable.
|
|
133
|
+
return { status: "ready", value: { _resultTooLarge: true, bytes: row.resultTooLargeBytes } };
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
// A node's resultJson is its own raw stateUpdate — by convention almost
|
|
137
|
+
// always `{[nodeId]: value}` (assembleState's own doc comment notes
|
|
138
|
+
// this isn't a GUARANTEED shape, just the near-universal one), so
|
|
139
|
+
// unwrap it here: a needs: author writing needs:{x:{hops,nodeId:"SCAN"}}
|
|
140
|
+
// expects ctx.needs("x") to resolve to what state.SCAN means in SCAN's
|
|
141
|
+
// own flow, not the wrapper object around it. assembleState() itself
|
|
142
|
+
// doesn't unwrap because it's replaying MANY nodes' updates onto one
|
|
143
|
+
// shared object via spread, where the wrapper IS the update; this
|
|
144
|
+
// accessor fetches exactly one node's row, so unwrapping is correct
|
|
145
|
+
// here and would be wrong there.
|
|
146
|
+
const parsed = JSON.parse(row.resultJson ?? "null");
|
|
147
|
+
return { status: "ready", value: parsed && typeof parsed === "object" ? parsed[nodeId] : undefined };
|
|
148
|
+
} catch {
|
|
149
|
+
return { status: "not-found" }; // corrupted row — same treatment assembleState() gives a bad row
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Resolves a {hops, entry} need — always "ready" once the target ancestor
|
|
154
|
+
// flow itself exists (an entry field is set the instant its flow is
|
|
155
|
+
// created; there is no "dispatched but not complete" state the way a
|
|
156
|
+
// node's own resultJson has). "not-found" only when the walk itself fails
|
|
157
|
+
// (fewer real ancestors than declared, or the target flow vanished).
|
|
158
|
+
export async function resolveOneEntry({ flowId, hops, entry, runStore, nodeExecutionsStore }) {
|
|
159
|
+
const targetFlowId = await walkUpHops(flowId, hops, runStore, nodeExecutionsStore);
|
|
160
|
+
if (!targetFlowId) return { status: "not-found" };
|
|
161
|
+
const flow = await runStore.get(targetFlowId);
|
|
162
|
+
if (!flow) return { status: "not-found" };
|
|
163
|
+
return { status: "ready", value: flow.stateJson?.[entry] };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Resolves a {hops, config} need — reads a pipeline-declared static config
|
|
167
|
+
// field off the target flow's own pipeline DEFINITION, not its stateJson.
|
|
168
|
+
// "not-found" when the walk fails, the target flow vanished, or its
|
|
169
|
+
// pipeline isn't currently loaded in pipelinesRef (mirrors runFlow()'s own
|
|
170
|
+
// "pipeline not loaded" handling in executor.js — a transient loader race,
|
|
171
|
+
// not a real absence). Otherwise always "ready", same as `entry` — an
|
|
172
|
+
// unknown config key simply resolves to `undefined`.
|
|
173
|
+
export async function resolveOneConfig({ flowId, hops, config, runStore, nodeExecutionsStore, pipelinesRef }) {
|
|
174
|
+
const targetFlowId = await walkUpHops(flowId, hops, runStore, nodeExecutionsStore);
|
|
175
|
+
if (!targetFlowId) return { status: "not-found" };
|
|
176
|
+
const flow = await runStore.get(targetFlowId);
|
|
177
|
+
if (!flow) return { status: "not-found" };
|
|
178
|
+
const pipeline = pipelinesRef?.value?.get(flow.stateJson?.pipelineId);
|
|
179
|
+
if (!pipeline) return { status: "not-found" };
|
|
180
|
+
return { status: "ready", value: pipeline.state?.[config] };
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Builds the `needs(varName)` async function passed to a declaring node's
|
|
184
|
+
// own run()/promptBuilder()/selector() as part of its ctx. Each call is an
|
|
185
|
+
// independent, fresh resolveOneNeed — no bulk fetch, no caching across
|
|
186
|
+
// calls within the same dispatch (consistent with this engine's "no cache
|
|
187
|
+
// layer, direct read" convention every other store accessor already
|
|
188
|
+
// follows). A not-ready/not-found result throws NodeBlocked with `_kind:
|
|
189
|
+
// "needs-not-ready"` explicitly stamped onto `_blocked` — mirroring the
|
|
190
|
+
// established pattern the budget-gate pause already uses
|
|
191
|
+
// (executor-nodes.js's dispatchAgentEntry) — a bare NodeBlocked with no
|
|
192
|
+
// `_kind` is treated as a hard FAILURE by executor.js's blockedInfo
|
|
193
|
+
// handling, not a wait, so this must never be omitted.
|
|
194
|
+
export function makeNeedsCtx({ node, flowId, runStore, nodeExecutionsStore, pipelinesRef }) {
|
|
195
|
+
return async function needs(varName) {
|
|
196
|
+
const spec = node?.needs?.[varName];
|
|
197
|
+
if (!spec) {
|
|
198
|
+
throw new Error(`ctx.needs("${varName}"): not declared in this node's .needs — check node.needs`);
|
|
199
|
+
}
|
|
200
|
+
const result = spec.entry !== undefined
|
|
201
|
+
? await resolveOneEntry({ flowId, hops: spec.hops, entry: spec.entry, runStore, nodeExecutionsStore })
|
|
202
|
+
: spec.config !== undefined
|
|
203
|
+
? await resolveOneConfig({ flowId, hops: spec.hops, config: spec.config, runStore, nodeExecutionsStore, pipelinesRef })
|
|
204
|
+
: await resolveOneNeed({ flowId, hops: spec.hops, nodeId: spec.nodeId, runStore, nodeExecutionsStore });
|
|
205
|
+
if (result.status !== "ready") {
|
|
206
|
+
const target = spec.entry !== undefined ? `entry:"${spec.entry}"` : spec.config !== undefined ? `config:"${spec.config}"` : `nodeId:"${spec.nodeId}"`;
|
|
207
|
+
const err = new NodeBlocked({
|
|
208
|
+
reason: `waiting on needs.${varName} (hops:${spec.hops}, ${target}, ${result.status})`,
|
|
209
|
+
needed: [spec.entry ?? spec.config ?? spec.nodeId],
|
|
210
|
+
hint: "declared needs: not ready yet — will recheck on next dispatch",
|
|
211
|
+
});
|
|
212
|
+
err._blocked = { ...err._blocked, _kind: "needs-not-ready" };
|
|
213
|
+
throw err;
|
|
214
|
+
}
|
|
215
|
+
// Optional field-pick, so an author can ask for exactly the sub-key(s)
|
|
216
|
+
// it actually reads instead of the whole ancestor node's output — the
|
|
217
|
+
// same "scoped read, not the whole blob" reasoning this engine follows
|
|
218
|
+
// elsewhere, just applied at the single-node
|
|
219
|
+
// granularity ctx.needs() operates at. `fields` is always an array,
|
|
220
|
+
// even for a single key — no separate "unwrap to a bare value" mode,
|
|
221
|
+
// so ctx.needs()'s return shape is always a dict, never sometimes-a-
|
|
222
|
+
// dict-sometimes-a-scalar depending on how many fields were picked.
|
|
223
|
+
// Not declared -> today's default, the whole resolved value, unchanged.
|
|
224
|
+
if (Array.isArray(spec.fields)) {
|
|
225
|
+
return Object.fromEntries(spec.fields.map((f) => [f, result.value?.[f]]));
|
|
226
|
+
}
|
|
227
|
+
return result.value;
|
|
228
|
+
};
|
|
229
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// pipelines-watcher.js — keep `pipelinesRef` in sync with the on-disk
|
|
2
|
+
// pipelines directory handed in as `pipelinesDir`.
|
|
3
|
+
//
|
|
4
|
+
// Why: without this, `loadPipelines` only runs once at gateway boot (via the
|
|
5
|
+
// `manifold-loader` service or `ensurePipelinesLoaded`'s "load once if
|
|
6
|
+
// empty" guard). New pipeline folders dropped after boot, or edits to existing
|
|
7
|
+
// `index.js` files, are invisible until the next gateway restart.
|
|
8
|
+
//
|
|
9
|
+
// How: poll every `pollMs` (default 1000). Each tick, compute a fingerprint
|
|
10
|
+
// of `<dir>/<entry>/index.js` files keyed by name and mtime. If the
|
|
11
|
+
// fingerprint changed since last tick, call `loadPipelines` and swap into
|
|
12
|
+
// `pipelinesRef.value`. The loader already cache-busts imports by reading
|
|
13
|
+
// each index.js itself and importing it as a base64-encoded `data:` URL
|
|
14
|
+
// (each payload is a unique URL, so Node's ESM cache can't return a stale
|
|
15
|
+
// module), so the swap picks up edited code.
|
|
16
|
+
//
|
|
17
|
+
// Note on directory mtime: the top-level `.pipelines/` mtime only changes on
|
|
18
|
+
// add/remove at depth 1 — it does NOT change when an `index.js` is edited.
|
|
19
|
+
// That's why we fingerprint each `index.js`'s mtime explicitly.
|
|
20
|
+
|
|
21
|
+
import { readdir, stat } from "node:fs/promises";
|
|
22
|
+
import path from "node:path";
|
|
23
|
+
import { loadPipelines } from "./loader.js";
|
|
24
|
+
|
|
25
|
+
const DEFAULT_POLL_MS = 1000;
|
|
26
|
+
|
|
27
|
+
export function createPipelinesWatcher({ pipelinesDir, pipelinesRef, logger, pollMs = DEFAULT_POLL_MS }) {
|
|
28
|
+
let lastFingerprint = null;
|
|
29
|
+
let timer = null;
|
|
30
|
+
let inFlight = false;
|
|
31
|
+
const listeners = new Set();
|
|
32
|
+
|
|
33
|
+
async function fingerprint() {
|
|
34
|
+
let entries;
|
|
35
|
+
try {
|
|
36
|
+
entries = await readdir(pipelinesDir, { withFileTypes: true });
|
|
37
|
+
} catch (err) {
|
|
38
|
+
// Directory missing — treat as empty fingerprint. If it later appears,
|
|
39
|
+
// the next tick reloads.
|
|
40
|
+
if (err.code === "ENOENT") return "";
|
|
41
|
+
throw err;
|
|
42
|
+
}
|
|
43
|
+
const parts = [];
|
|
44
|
+
for (const ent of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
45
|
+
if (ent.name.startsWith(".") || !ent.isDirectory()) continue;
|
|
46
|
+
const idx = path.join(pipelinesDir, ent.name, "index.js");
|
|
47
|
+
try {
|
|
48
|
+
const s = await stat(idx);
|
|
49
|
+
parts.push(`${ent.name}:${s.mtimeMs}`);
|
|
50
|
+
} catch {
|
|
51
|
+
// index.js missing — skip; loader would skip it too.
|
|
52
|
+
}
|
|
53
|
+
const fleet = path.join(pipelinesDir, ent.name, "fleet.yaml");
|
|
54
|
+
try {
|
|
55
|
+
const sf = await stat(fleet);
|
|
56
|
+
parts.push(`${ent.name}:fleet:${sf.mtimeMs}`);
|
|
57
|
+
} catch { /* fleet.yaml optional */ }
|
|
58
|
+
}
|
|
59
|
+
return parts.join("|");
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function tick() {
|
|
63
|
+
if (inFlight) return;
|
|
64
|
+
inFlight = true;
|
|
65
|
+
try {
|
|
66
|
+
const fp = await fingerprint();
|
|
67
|
+
if (lastFingerprint === null) {
|
|
68
|
+
// First tick: prime the fingerprint without reloading — the loader
|
|
69
|
+
// service has already loaded pipelines at gateway boot.
|
|
70
|
+
lastFingerprint = fp;
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
if (fp === lastFingerprint) return;
|
|
74
|
+
const loaded = await loadPipelines(pipelinesDir, logger);
|
|
75
|
+
// Merge rather than full-replace: if a pipeline that was previously live
|
|
76
|
+
// fails to reload (transient ESM cache error, file mid-write during a git
|
|
77
|
+
// op) its entry would silently vanish from the live Map and stay missing
|
|
78
|
+
// until the next mtime change. Instead, keep stale entries whose index.js
|
|
79
|
+
// is still on disk — they stay live until the disk copy is actually gone.
|
|
80
|
+
const merged = new Map(loaded);
|
|
81
|
+
for (const [id, existing] of pipelinesRef.value) {
|
|
82
|
+
if (merged.has(id)) continue;
|
|
83
|
+
// Synthetic subflow-child pipelines (id = `${parentPipelineId}::${nodeId}`,
|
|
84
|
+
// registered in-memory by loader.js's registerSubflowChildren) are never
|
|
85
|
+
// disk-backed at <pipelinesDir>/<id>/ — stat-ing their own id always
|
|
86
|
+
// throws, so they'd be dropped every time their real parent transiently
|
|
87
|
+
// fails to reload, even though the parent itself survives via this same
|
|
88
|
+
// loop. Check the parent's directory instead (loader.js tags every
|
|
89
|
+
// synthetic entry with _parentPipelineId for exactly this).
|
|
90
|
+
const diskCheckId = existing._parentPipelineId ?? id;
|
|
91
|
+
try {
|
|
92
|
+
await stat(path.join(pipelinesDir, diskCheckId, "index.js"));
|
|
93
|
+
merged.set(id, existing);
|
|
94
|
+
logger?.warn?.(`manifold pipelines-watcher: kept stale entry for "${id}" (failed to reload)`);
|
|
95
|
+
} catch {
|
|
96
|
+
// parent's index.js gone — intentional removal, drop it
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
pipelinesRef.value = merged;
|
|
100
|
+
lastFingerprint = fp;
|
|
101
|
+
logger?.info?.(`manifold pipelines-watcher: reloaded ${loaded.size} pipeline(s) after disk change`);
|
|
102
|
+
for (const cb of listeners) {
|
|
103
|
+
try { cb({ size: loaded.size, ids: [...loaded.keys()] }); }
|
|
104
|
+
catch (err) { logger?.warn?.(`manifold pipelines-watcher: listener error: ${err?.message ?? err}`); }
|
|
105
|
+
}
|
|
106
|
+
} catch (err) {
|
|
107
|
+
// Swallow — next tick retries.
|
|
108
|
+
logger?.warn?.(`manifold pipelines-watcher: tick error: ${err?.message ?? err}`);
|
|
109
|
+
} finally {
|
|
110
|
+
inFlight = false;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return {
|
|
115
|
+
start() {
|
|
116
|
+
if (timer) return;
|
|
117
|
+
timer = setInterval(tick, pollMs);
|
|
118
|
+
if (typeof timer?.unref === "function") timer.unref();
|
|
119
|
+
},
|
|
120
|
+
stop() {
|
|
121
|
+
if (timer) { clearInterval(timer); timer = null; }
|
|
122
|
+
lastFingerprint = null;
|
|
123
|
+
listeners.clear();
|
|
124
|
+
},
|
|
125
|
+
onReload(cb) {
|
|
126
|
+
listeners.add(cb);
|
|
127
|
+
return () => listeners.delete(cb);
|
|
128
|
+
},
|
|
129
|
+
// Exposed for tests / probes.
|
|
130
|
+
_tick: tick,
|
|
131
|
+
_fingerprint: fingerprint,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// result-replay.js — the pure merge step shared by assembleState()
|
|
2
|
+
// (executor-helpers.js) and subflow-state.js's per-child `data` field.
|
|
3
|
+
//
|
|
4
|
+
// Split out on its own specifically to avoid a circular import:
|
|
5
|
+
// executor-helpers.js's assembleState() already imports getAllSubflowGroups
|
|
6
|
+
// from subflow-state.js, so subflow-state.js can't import assembleState()
|
|
7
|
+
// back — but it DOES need this exact same "replay a flow's own completed
|
|
8
|
+
// resultJson rows onto a base object" logic for its own `data` field
|
|
9
|
+
// (state/subflow-state.js's normalizeChildAsSubflowRow).
|
|
10
|
+
// No _subflowGroups nesting happens here — a child's OWN nested subflow
|
|
11
|
+
// groups aren't part of "this child's own top-level output"; a caller
|
|
12
|
+
// that needs those calls assembleState() on that child's flowId directly.
|
|
13
|
+
export function replayResults(baseState, results) {
|
|
14
|
+
let state = { ...baseState };
|
|
15
|
+
for (const r of results ?? []) {
|
|
16
|
+
if (r.resultWrittenAt == null) continue;
|
|
17
|
+
if (r.resultTooLargeBytes != null && r.resultJson == null) {
|
|
18
|
+
// writeResult() (state/node-executions.js) does not cap results, so
|
|
19
|
+
// this can only fire for a row that kept nothing at all — the
|
|
20
|
+
// real value was never stored and isn't recoverable. A row that DID
|
|
21
|
+
// keep some fields has a non-null resultJson and falls through to the
|
|
22
|
+
// normal merge below: no reason to throw away real partial data just
|
|
23
|
+
// because resultTooLargeBytes also happens to be set on that row.
|
|
24
|
+
state = { ...state, [r.nodeId]: { _resultTooLarge: true, bytes: r.resultTooLargeBytes } };
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
let update;
|
|
28
|
+
try { update = JSON.parse(r.resultJson ?? "null"); } catch { continue; }
|
|
29
|
+
if (update && typeof update === "object" && !Array.isArray(update)) {
|
|
30
|
+
state = { ...state, ...update };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return state;
|
|
34
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// revival-messages.js — shared boot-revival continuation prompt.
|
|
2
|
+
//
|
|
3
|
+
// Sent to an EXISTING session key after a gateway restart interrupts an
|
|
4
|
+
// in-flight agent turn. Never re-dispatches the original task with a fresh
|
|
5
|
+
// prompt — that would give the agent no way to know it already tried this
|
|
6
|
+
// once, risking a duplicate side effect (e.g. a duplicate submission) if the
|
|
7
|
+
// prior attempt actually completed before the crash. Sending this to the
|
|
8
|
+
// same session lets the agent check its own history first.
|
|
9
|
+
|
|
10
|
+
export const REVIVAL_MSG =
|
|
11
|
+
"Continue from where you left off — your prior session was interrupted by a gateway restart or process death. Read your conversation history, identify the in-flight task, and resume or finalize it.";
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// spawn-child.js — Phase 2 branch/flow unification: race-safe identity +
|
|
2
|
+
// creation for a subflow-item child flow.
|
|
3
|
+
|
|
4
|
+
// resolveOrCreateChild finds or creates the flow_node_invocations row
|
|
5
|
+
// identifying "nodeId processing item subflowKey", then finds or creates
|
|
6
|
+
// the flow_runs child that invocation should own.
|
|
7
|
+
//
|
|
8
|
+
// Race safety comes entirely from findOrCreateInvocation's insert-then-
|
|
9
|
+
// reselect pattern (state/node-executions.js) backed by the invocation
|
|
10
|
+
// table's unique index — two concurrent callers for the same identity tuple
|
|
11
|
+
// always agree on one invocation id, so at most one of them ever wins the
|
|
12
|
+
// "does a child exist yet" check and creates one. That identity tuple
|
|
13
|
+
// includes flowId, though — it says nothing about two DIFFERENT parent
|
|
14
|
+
// flows independently deciding to dispatch the same subflowKey (e.g. two
|
|
15
|
+
// freelance_freelancer-proposal scans both fanning out SCORE for the same
|
|
16
|
+
// listing). `guard`, when supplied, is threaded straight through to
|
|
17
|
+
// findOrCreateInvocation to close exactly that gap — see its own comment
|
|
18
|
+
// for why running it inside that call's write-queue turn makes it
|
|
19
|
+
// race-safe across flowIds, not just within one.
|
|
20
|
+
export async function resolveOrCreateChild({
|
|
21
|
+
flowId,
|
|
22
|
+
pipelineId = null,
|
|
23
|
+
nodeId,
|
|
24
|
+
subflowKey,
|
|
25
|
+
childPipelineId,
|
|
26
|
+
initialState,
|
|
27
|
+
store,
|
|
28
|
+
runStore,
|
|
29
|
+
nodeExecutionsStore,
|
|
30
|
+
guard,
|
|
31
|
+
}) {
|
|
32
|
+
const invocationId = await nodeExecutionsStore.findOrCreateInvocation({
|
|
33
|
+
flowId, pipelineId, nodeId, subflowKey, guard,
|
|
34
|
+
});
|
|
35
|
+
if (invocationId == null) {
|
|
36
|
+
return { invocationId: null, childFlowId: null, created: false, guarded: true };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const existing = await runStore.getByParentInvocationId(invocationId);
|
|
40
|
+
if (existing) {
|
|
41
|
+
return { invocationId, childFlowId: existing.flowId, created: false };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const child = await store.create({
|
|
45
|
+
pipelineId: childPipelineId,
|
|
46
|
+
initialState,
|
|
47
|
+
parentInvocationId: invocationId,
|
|
48
|
+
});
|
|
49
|
+
// Re-query to find the actual winner of the concurrent-insert race.
|
|
50
|
+
// With async clients, multiple callers can pass the existence check
|
|
51
|
+
// before any of them persists, so only the one that won the UNIQUE
|
|
52
|
+
// constraint has a real row. The winner is always findable via the index.
|
|
53
|
+
const winner = await runStore.getByParentInvocationId(invocationId);
|
|
54
|
+
return { invocationId, childFlowId: winner?.flowId ?? child.flowId, created: winner?.flowId === child.flowId };
|
|
55
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// wait-for-flow-settlement.js — recursively waits for a flow AND every
|
|
2
|
+
// descendant it spawns via any childFlows:true subflow node, at any depth,
|
|
3
|
+
// to reach a stopping point (genuinely terminal, or a real human-gate
|
|
4
|
+
// wait). Exists specifically for the one-shot CLI (bin/cli.js) — the
|
|
5
|
+
// live gateway process never needs this: its own event loop keeps
|
|
6
|
+
// draining pending setImmediate/timer callbacks for as long as it runs,
|
|
7
|
+
// so children spawned by dispatchSubflowChildren settle naturally over
|
|
8
|
+
// time. A one-shot CLI invocation has no such guarantee: `withExit`'s
|
|
9
|
+
// `process.exit(0)` kills any pending setImmediate outright.
|
|
10
|
+
//
|
|
11
|
+
// `bin/cli.js`'s start/rerun/retry/approve commands each `await` exactly
|
|
12
|
+
// one `executor.runFlow()`/`resume()` call for the ONE flow they were
|
|
13
|
+
// given, documented as "blocks until the flow reaches terminal or
|
|
14
|
+
// human-gate" — but that await resolves the moment THAT flow itself goes
|
|
15
|
+
// blocked/terminal, without ever waiting on any children it spawned along
|
|
16
|
+
// the way (dispatchSubflowChildren fires each child via a bare
|
|
17
|
+
// `setImmediate(() => runFlow(childFlowId, "fresh"))` and returns
|
|
18
|
+
// immediately).
|
|
19
|
+
//
|
|
20
|
+
// Deliberately NOT a fix inside dispatchSubflowChildren/executor.js
|
|
21
|
+
// itself — that fire-and-forget dispatch is correct and load-bearing for
|
|
22
|
+
// the live gateway (a subflow node's own step must return promptly so the
|
|
23
|
+
// executor isn't stuck waiting on one flow's children before it can
|
|
24
|
+
// process anything else). This module only changes what the CLI does
|
|
25
|
+
// with its own already-live process before letting it exit.
|
|
26
|
+
|
|
27
|
+
const TERMINAL_STATUSES = new Set(["succeeded", "failed", "cancelled", "skipped"]);
|
|
28
|
+
|
|
29
|
+
// waitJson.kind values that will NEVER resolve on their own no matter how
|
|
30
|
+
// long this process keeps polling — each needs a genuinely different
|
|
31
|
+
// resumption path than "wait":
|
|
32
|
+
// - "human-gate": resumed later via the gate-resolve API, against
|
|
33
|
+
// whatever process an operator/dashboard/Telegram calls it from —
|
|
34
|
+
// not by this one-shot CLI process staying alive.
|
|
35
|
+
// - "needs-gateway-handoff": thrown by executor-errors.js's
|
|
36
|
+
// GatewayHandoffNeeded when a kind:"agent" node dispatches outside a
|
|
37
|
+
// live gateway request scope — which the one-shot CLI process always
|
|
38
|
+
// is. That file's own docstring confirms there is no automatic
|
|
39
|
+
// resume for this wait kind anywhere in the codebase; it needs either
|
|
40
|
+
// the scope gap fixed at the dispatch's own call site or manual
|
|
41
|
+
// task-revival. Without excluding it here, any childFlows:true
|
|
42
|
+
// descendant containing a kind:"agent" node would make this module's
|
|
43
|
+
// unbounded poll loop hang forever instead of surfacing the (already
|
|
44
|
+
// pre-existing, already-expected) blocked status promptly.
|
|
45
|
+
// Anything else non-terminal (running, queued on the shared admission pool,
|
|
46
|
+
// blocked on its own children-pending) needs this process to keep running
|
|
47
|
+
// blocked on its own children-pending) needs this process to keep running
|
|
48
|
+
// for it to ever resolve, since nothing else will wake it.
|
|
49
|
+
const NEVER_AUTO_RESOLVES = new Set(["human-gate", "needs-gateway-handoff"]);
|
|
50
|
+
|
|
51
|
+
function isRestPoint(flow) {
|
|
52
|
+
if (!flow) return true; // gone/not found — nothing left to wait on
|
|
53
|
+
if (TERMINAL_STATUSES.has(flow.status)) return true;
|
|
54
|
+
return flow.status === "blocked" && NEVER_AUTO_RESOLVES.has(flow.waitJson?.kind);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function collectDirectChildren(flowId, { runStore, nodeExecutionsStore }) {
|
|
58
|
+
const subflowNodeIds = await nodeExecutionsStore.listSubflowNodeIdsForFlow(flowId);
|
|
59
|
+
if (subflowNodeIds.length === 0) return [];
|
|
60
|
+
const byNode = await nodeExecutionsStore.listInvocationIdsForNodes(flowId, subflowNodeIds);
|
|
61
|
+
const invocationIds = [...byNode.values()].flat();
|
|
62
|
+
if (invocationIds.length === 0) return [];
|
|
63
|
+
return runStore.listForParentInvocations(invocationIds);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// One pass over the whole current tree (re-discovered fresh every call,
|
|
67
|
+
// since a resumed flow can route to nodes/children that didn't exist on
|
|
68
|
+
// the previous pass). Returns the list of flowIds still needing this
|
|
69
|
+
// process to keep running — empty means safe to exit.
|
|
70
|
+
async function findPending(rootFlowId, ctx) {
|
|
71
|
+
const visited = new Set();
|
|
72
|
+
const pending = [];
|
|
73
|
+
const queue = [rootFlowId];
|
|
74
|
+
while (queue.length > 0) {
|
|
75
|
+
const flowId = queue.shift();
|
|
76
|
+
if (visited.has(flowId)) continue;
|
|
77
|
+
visited.add(flowId);
|
|
78
|
+
const flow = await ctx.store.get(flowId);
|
|
79
|
+
if (!isRestPoint(flow)) pending.push(flowId);
|
|
80
|
+
const children = await collectDirectChildren(flowId, ctx);
|
|
81
|
+
for (const child of children) queue.push(child.flowId);
|
|
82
|
+
}
|
|
83
|
+
return pending;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* @param {string} rootFlowId
|
|
88
|
+
* @param {object} ctx — { store, runStore, nodeExecutionsStore, logger }
|
|
89
|
+
* @param {object} [opts]
|
|
90
|
+
* @param {number} [opts.pollMs] — default 300
|
|
91
|
+
* @param {number} [opts.logEveryMs] — progress log cadence while waiting, default 5000
|
|
92
|
+
* @returns {Promise<{settled: true}>} — resolves once every flow in the
|
|
93
|
+
* tree is terminal or genuinely gate-blocked. No timeout: unbounded,
|
|
94
|
+
* same philosophy the CLI's pre-existing single-flow await already had
|
|
95
|
+
* ("ctrl-c to stop" is the documented escape hatch, not a silent give-up).
|
|
96
|
+
*/
|
|
97
|
+
export async function waitForFlowSettlement(rootFlowId, ctx, { pollMs = 300, logEveryMs = 5000 } = {}) {
|
|
98
|
+
let lastLogAt = 0;
|
|
99
|
+
while (true) {
|
|
100
|
+
const pending = await findPending(rootFlowId, ctx);
|
|
101
|
+
if (pending.length === 0) return { settled: true };
|
|
102
|
+
const now = Date.now();
|
|
103
|
+
if (now - lastLogAt >= logEveryMs) {
|
|
104
|
+
lastLogAt = now;
|
|
105
|
+
ctx.logger?.info?.(`manifold: waiting on ${pending.length} still-active flow(s) beneath ${rootFlowId.slice(0, 8)}: ${pending.map((id) => id.slice(0, 8)).join(", ")}`);
|
|
106
|
+
}
|
|
107
|
+
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// events/catalog.js — single producer for pipeline-catalog changes.
|
|
2
|
+
//
|
|
3
|
+
// One watcher reload feeds both sinks with identical content: the gateway
|
|
4
|
+
// WS stream (canonical pipeline.catalog.updated) and the dashboard bus
|
|
5
|
+
// (legacy pipelines_changed type, canonical kind stamped). Separate sinks
|
|
6
|
+
// stay separate; only the content decisions merge here.
|
|
7
|
+
export function createCatalogPublisher({ emit, getDashboardBroadcast }) {
|
|
8
|
+
return async function publishCatalogChanged(info) {
|
|
9
|
+
const now = Date.now();
|
|
10
|
+
const payload = { count: info.size, ids: info.ids, updatedAt: now };
|
|
11
|
+
if (emit) {
|
|
12
|
+
try { await emit("pipeline.catalog.updated", payload); } catch {}
|
|
13
|
+
}
|
|
14
|
+
try {
|
|
15
|
+
const broadcastDashboard = getDashboardBroadcast?.();
|
|
16
|
+
if (broadcastDashboard) {
|
|
17
|
+
await broadcastDashboard({
|
|
18
|
+
kind: "pipeline.catalog.updated",
|
|
19
|
+
at: now,
|
|
20
|
+
size: payload.count,
|
|
21
|
+
ids: payload.ids,
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
} catch {}
|
|
25
|
+
};
|
|
26
|
+
}
|
package/events/emit.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// events/emit.js — single fan-out for canonical kinds.
|
|
2
|
+
//
|
|
3
|
+
// emit(kind, payload) validates, stamps the envelope, and delivers to the
|
|
4
|
+
// gateway WS broadcast. Transport faults are reported in the delivery
|
|
5
|
+
// receipt, never thrown (a dead gateway must not fail the dispatch that
|
|
6
|
+
// produced the event); validation faults reject (programmer errors).
|
|
7
|
+
// The `plugin.` prefix is applied here and only here — producers name
|
|
8
|
+
// the kind, never the wire.
|
|
9
|
+
import {
|
|
10
|
+
assertPayload,
|
|
11
|
+
isDashboardOnly,
|
|
12
|
+
} from "./kinds.js";
|
|
13
|
+
|
|
14
|
+
export function createEmitter({ getBroadcast, logger } = {}) {
|
|
15
|
+
async function emit(kind, payload = {}) {
|
|
16
|
+
// Throws (rejects) on unknown/reserved kinds and missing fields.
|
|
17
|
+
assertPayload(kind, payload);
|
|
18
|
+
// A `type` field here is a producer bug — the kind is the only name.
|
|
19
|
+
const { type: _strayType, ...fields } = payload;
|
|
20
|
+
const envelope = { ...fields, kind };
|
|
21
|
+
if (envelope.emittedAtMs == null) envelope.emittedAtMs = Date.now();
|
|
22
|
+
|
|
23
|
+
// Gateway WS boundary — the only delivery path. Dashboard-only kinds
|
|
24
|
+
// skip it.
|
|
25
|
+
try {
|
|
26
|
+
const broadcast = typeof getBroadcast === "function" ? getBroadcast() : null;
|
|
27
|
+
if (broadcast && !isDashboardOnly(kind)) {
|
|
28
|
+
try {
|
|
29
|
+
broadcast(`plugin.${kind}`, envelope);
|
|
30
|
+
return { ok: true, kind, deliveries: { ws: "sent" } };
|
|
31
|
+
} catch (err) {
|
|
32
|
+
try { logger?.warn?.(`events/emit: ws broadcast failed (${kind}): ${err?.message ?? err}`); } catch {}
|
|
33
|
+
return { ok: false, kind, deliveries: { ws: "error" } };
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
} catch {
|
|
37
|
+
return { ok: false, kind, deliveries: { ws: "error" } };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return { ok: true, kind, deliveries: { ws: "skipped" } };
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
return { emit };
|
|
44
|
+
}
|