@igorjpimenta/manifold 0.0.0-stage → 2026.11.22-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.migrations/ops/0000_futuristic_karma.sql +111 -0
- package/.migrations/ops/0001_low_maximus.sql +7 -0
- package/.migrations/ops/meta/0000_snapshot.json +763 -0
- package/.migrations/ops/meta/0001_snapshot.json +173 -0
- package/.migrations/ops/meta/_journal.json +20 -0
- package/.migrations/runtime/0000_real_boomer.sql +180 -0
- package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
- package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
- package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
- package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
- package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
- package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
- package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
- package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
- package/.migrations/runtime/0009_lying_shocker.sql +96 -0
- package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
- package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
- package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
- package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
- package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
- package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
- package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
- package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
- package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
- package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
- package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
- package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
- package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
- package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
- package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
- package/.migrations/runtime/meta/_journal.json +97 -0
- package/README.md +1583 -2
- package/RELEASING.md +89 -0
- package/agent/fleet.js +100 -0
- package/bin/cli.js +854 -0
- package/bin/manifold.js +4 -0
- package/boot/flow-revival.js +536 -0
- package/budget/index.js +495 -0
- package/budget/lib.js +59 -0
- package/dashboard/cost.js +528 -0
- package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
- package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
- package/dashboard/dist/index.html +13 -0
- package/dashboard/event-log.js +267 -0
- package/dashboard/index.html +12 -0
- package/dashboard/package-lock.json +2877 -0
- package/dashboard/package.json +37 -0
- package/dashboard/pipeline-graph.js +131 -0
- package/dashboard/postcss.config.js +6 -0
- package/dashboard/search.js +183 -0
- package/dashboard/server.js +1015 -0
- package/dashboard/src/App.tsx +42 -0
- package/dashboard/src/api/client.ts +118 -0
- package/dashboard/src/api/types.ts +510 -0
- package/dashboard/src/api/ws.tsx +112 -0
- package/dashboard/src/components/CancelButton.tsx +63 -0
- package/dashboard/src/components/ClickToCopy.tsx +68 -0
- package/dashboard/src/components/CopyButton.tsx +61 -0
- package/dashboard/src/components/DailyCostTrend.tsx +85 -0
- package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
- package/dashboard/src/components/Funnel.tsx +149 -0
- package/dashboard/src/components/GateForm.tsx +158 -0
- package/dashboard/src/components/GraphView.tsx +302 -0
- package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
- package/dashboard/src/components/Histogram.tsx +292 -0
- package/dashboard/src/components/Layout.tsx +28 -0
- package/dashboard/src/components/NeedsAttention.tsx +98 -0
- package/dashboard/src/components/NodeContextCard.tsx +132 -0
- package/dashboard/src/components/PathTrail.tsx +35 -0
- package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
- package/dashboard/src/components/RerunButton.tsx +46 -0
- package/dashboard/src/components/RunComparison.tsx +53 -0
- package/dashboard/src/components/Sparkline.tsx +94 -0
- package/dashboard/src/components/StatStrip.tsx +147 -0
- package/dashboard/src/components/StateTree.tsx +84 -0
- package/dashboard/src/components/StatusBadge.tsx +18 -0
- package/dashboard/src/components/SubflowPathPills.tsx +38 -0
- package/dashboard/src/components/SubflowSummary.tsx +24 -0
- package/dashboard/src/components/SubflowsCard.tsx +327 -0
- package/dashboard/src/components/TodayStrip.tsx +86 -0
- package/dashboard/src/components/TranscriptPane.tsx +99 -0
- package/dashboard/src/components/TranscriptRow.tsx +69 -0
- package/dashboard/src/components/ui/badge.tsx +38 -0
- package/dashboard/src/components/ui/button.tsx +28 -0
- package/dashboard/src/components/ui/card.tsx +42 -0
- package/dashboard/src/components/ui/skeleton.tsx +9 -0
- package/dashboard/src/components/ui/table.tsx +62 -0
- package/dashboard/src/index.css +26 -0
- package/dashboard/src/lib/anomaly.ts +42 -0
- package/dashboard/src/lib/format.ts +30 -0
- package/dashboard/src/lib/step-history.ts +13 -0
- package/dashboard/src/lib/subflow.ts +154 -0
- package/dashboard/src/lib/use-click-outside.ts +28 -0
- package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
- package/dashboard/src/lib/utils.ts +6 -0
- package/dashboard/src/main.tsx +10 -0
- package/dashboard/src/types/shims.d.ts +4 -0
- package/dashboard/src/views/Home.tsx +478 -0
- package/dashboard/src/views/PipelineDetail.tsx +1185 -0
- package/dashboard/src/views/PipelinesList.tsx +111 -0
- package/dashboard/src/views/RunDetail.tsx +248 -0
- package/dashboard/src/views/Search.tsx +196 -0
- package/dashboard/stats.js +668 -0
- package/dashboard/tailwind.config.ts +43 -0
- package/dashboard/tree-stats.js +12 -0
- package/dashboard/tsconfig.json +25 -0
- package/dashboard/views.js +231 -0
- package/dashboard/vite.config.ts +34 -0
- package/drizzle.config.js +12 -0
- package/drizzle.config.ops.js +12 -0
- package/elevation.js +116 -0
- package/engine/admission-promotion-resume.js +127 -0
- package/engine/agent-dispatch.js +64 -0
- package/engine/child-resume-trigger.js +315 -0
- package/engine/executor-budget.js +87 -0
- package/engine/executor-cancel.js +83 -0
- package/engine/executor-errors.js +122 -0
- package/engine/executor-helpers.js +254 -0
- package/engine/executor-nodes.js +292 -0
- package/engine/executor-persist.js +212 -0
- package/engine/executor-subflow-children.js +275 -0
- package/engine/executor.js +1013 -0
- package/engine/global-concurrency.js +33 -0
- package/engine/loader.js +528 -0
- package/engine/needs-resolver.js +229 -0
- package/engine/pipelines-watcher.js +133 -0
- package/engine/result-replay.js +34 -0
- package/engine/revival-messages.js +11 -0
- package/engine/spawn-child.js +55 -0
- package/engine/wait-for-flow-settlement.js +109 -0
- package/events/catalog.js +26 -0
- package/events/emit.js +44 -0
- package/events/kinds.js +118 -0
- package/events/serve-ws.js +74 -0
- package/events/stream.js +35 -0
- package/events/ws-dialog.js +257 -0
- package/events/ws-frame.js +194 -0
- package/events/ws-run-events.js +70 -0
- package/events.js +32 -0
- package/gates/gate-resolver.js +576 -0
- package/gates/paused-flows.js +55 -0
- package/graphql/columns.js +67 -0
- package/graphql/schema.js +333 -0
- package/index.js +49 -0
- package/mcp/http.js +159 -0
- package/mcp/identity.js +94 -0
- package/mcp/stdio.js +80 -0
- package/mcp/tools.js +245 -0
- package/mcp/wire-ops.js +178 -0
- package/package.json +49 -4
- package/routes/bearer-auth.js +57 -0
- package/routes/discuss-audio.js +87 -0
- package/routes/discuss-component.js +39 -0
- package/routes/discuss-draft.js +159 -0
- package/routes/discuss-helpers.js +20 -0
- package/routes/discuss.js +848 -0
- package/routes/failures.js +104 -0
- package/routes/graphql.js +61 -0
- package/routes/health.js +54 -0
- package/routes/operator-discuss.js +193 -0
- package/routes/operator-resume.js +32 -0
- package/routes/operator-retry.js +301 -0
- package/routes/operator.js +539 -0
- package/routes/pipelines.js +118 -0
- package/routes/runs.js +1129 -0
- package/scripts/ci/import-check.mjs +50 -0
- package/scripts/link-core-self.mjs +31 -0
- package/scripts/with-guard.sh +24 -0
- package/server/app.js +503 -0
- package/server/registry.js +53 -0
- package/server/serve.js +163 -0
- package/state/admission-queue-store.js +325 -0
- package/state/attribution-registries.js +102 -0
- package/state/batch-writer.js +162 -0
- package/state/db-guard.js +63 -0
- package/state/db-health.js +27 -0
- package/state/db.js +122 -0
- package/state/discuss-store.js +299 -0
- package/state/failure-store.js +427 -0
- package/state/migrate.js +104 -0
- package/state/node-cost.js +67 -0
- package/state/node-executions.js +775 -0
- package/state/node-health-store.js +124 -0
- package/state/ops-merge.js +93 -0
- package/state/paths.js +146 -0
- package/state/pipeline-dirs.js +99 -0
- package/state/pipeline-registry.js +449 -0
- package/state/run-store.js +697 -0
- package/state/schema.js +438 -0
- package/state/schema.ops.js +44 -0
- package/state/session-index.js +177 -0
- package/state/session-keys.js +39 -0
- package/state/state-store.js +311 -0
- package/state/subflow-state.js +126 -0
- package/state/tool-handoff.js +169 -0
- package/telemetry/budget-alert.js +353 -0
- package/telemetry/cli.js +331 -0
- package/telemetry/grammar.js +119 -0
- package/telemetry/http-budget.js +132 -0
- package/telemetry/http-cost.js +670 -0
- package/telemetry/http-helpers.js +134 -0
- package/telemetry/http-tree.js +349 -0
- package/telemetry/http.js +44 -0
- package/telemetry/index.js +307 -0
- package/telemetry/ingest.js +134 -0
- package/telemetry/labels.js +115 -0
- package/telemetry/pricer.js +77 -0
- package/telemetry/resolver.js +314 -0
- package/telemetry/store.js +147 -0
- package/utils/run-patch.js +43 -0
|
@@ -0,0 +1,1013 @@
|
|
|
1
|
+
// executor.js — runs a manifold pipeline against a managed flow_run.
|
|
2
|
+
//
|
|
3
|
+
// Lifecycle of one flow:
|
|
4
|
+
// 1. Caller invokes start(pipelineId)
|
|
5
|
+
// 2. We create_flow_runs row at status='running', lastStep='__start__'.
|
|
6
|
+
// 3. We enter the run loop:
|
|
7
|
+
// a. Read flow row.
|
|
8
|
+
// b. If terminal — return.
|
|
9
|
+
// c. If blocked — return (resume() wakes us back up later).
|
|
10
|
+
// d. Resolve edge from lastStep → nextNodeId.
|
|
11
|
+
// e. If nextNodeId === '__end__': persist finish() with current state; return.
|
|
12
|
+
// f. Execute the node; collect stateUpdate (and waitJson if gate).
|
|
13
|
+
// g. Apply update onto state.
|
|
14
|
+
// h. Persist: setRunning() to advance lastStep, or setWaiting() to block.
|
|
15
|
+
// i. Loop.
|
|
16
|
+
|
|
17
|
+
import { randomUUID } from "node:crypto";
|
|
18
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
19
|
+
import path from "node:path";
|
|
20
|
+
import os from "node:os";
|
|
21
|
+
import { getNodeExecutionsStore } from "../state/node-executions.js";
|
|
22
|
+
import { nodeEmitter } from "../events.js";
|
|
23
|
+
import { resolveStorePaths } from "../state/paths.js";
|
|
24
|
+
|
|
25
|
+
import {
|
|
26
|
+
isTerminal, isBlocked, isQueued, shortSummary, deepClone, resolveEdge,
|
|
27
|
+
validateGateLayout, isFreshNodeEntry, describeWaitFallback, assembleState,
|
|
28
|
+
} from "./executor-helpers.js";
|
|
29
|
+
import { NodeBlocked, GatewayHandoffNeeded } from "./executor-errors.js";
|
|
30
|
+
import { REVIVAL_MSG } from "./revival-messages.js";
|
|
31
|
+
import { createBudgetGate } from "./executor-budget.js";
|
|
32
|
+
import { createPersist } from "./executor-persist.js";
|
|
33
|
+
import { createAgentDispatch } from "./agent-dispatch.js";
|
|
34
|
+
import { dispatchSubflowChildren } from "./executor-subflow-children.js";
|
|
35
|
+
import { createNodeRunner } from "./executor-nodes.js";
|
|
36
|
+
import { completeCancellation } from "./executor-cancel.js";
|
|
37
|
+
|
|
38
|
+
export function createExecutor({ api, config, approvalQueue, pipelinesDir, store, pipelinesRef, ensurePipelinesLoaded, logger, runTurn, failureStore, nodeHealthStore, notifyFlowEnteredWaiting, sessionIndex, nodeExecutionsStore, runStore, admissionQueueStore, emit }) {
|
|
39
|
+
// Explicit params win; api is the fallback the adapter still
|
|
40
|
+
// supplies (tests construct with api only).
|
|
41
|
+
const _config = config ?? api?.config ?? null;
|
|
42
|
+
const _lineage = api?.runtime?.lineage ?? null;
|
|
43
|
+
const _pipelinesDir = pipelinesDir ?? _config?.pipelinesDir ?? resolveStorePaths().pipelinesDir;
|
|
44
|
+
// Injectable for tests (defaults to the real singleton in production).
|
|
45
|
+
// A test constructing a real createExecutor() without this override would
|
|
46
|
+
// write its fixture rows into the real store's runtime.sqlite; store/
|
|
47
|
+
// branchService/failureStore etc. above already follow this DI pattern.
|
|
48
|
+
const nodeExecutions = nodeExecutionsStore ?? getNodeExecutionsStore();
|
|
49
|
+
|
|
50
|
+
// Wraps nodeExecutions.write() for the write-then-fail call sites below —
|
|
51
|
+
// write() can throw (its own self-validating kind-invariant check), and
|
|
52
|
+
// unlike its normal position (after persistFailRetrying, so the fail
|
|
53
|
+
// status is already durably persisted before any crash), these sites
|
|
54
|
+
// call it FIRST so the failure can be linked to the invocation/execution
|
|
55
|
+
// it just created. A local catch here preserves "the fail status always
|
|
56
|
+
// gets persisted" even if write() throws — persistFailRetrying still
|
|
57
|
+
// runs, just without a linked id, instead of the throw propagating out
|
|
58
|
+
// of runFlow() before the fail status is ever written.
|
|
59
|
+
async function safeWriteExecution(args) {
|
|
60
|
+
try {
|
|
61
|
+
return (await nodeExecutions.write(args)) ?? {};
|
|
62
|
+
} catch (err) {
|
|
63
|
+
logger.warn(`manifold [${args.flowId.slice(0, 8)}] nodeExecutions.write threw: ${err?.message ?? err}`);
|
|
64
|
+
return {};
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const { maybeBudgetGate } = createBudgetGate({
|
|
69
|
+
getApprovalQueue: () => approvalQueue ?? api?.runtime?.approvalQueue,
|
|
70
|
+
logger,
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
const { persistRunningRetrying, persistWaitingRetrying, persistSkipRetrying, persistFinishRetrying, persistFailRetrying } = createPersist({
|
|
74
|
+
store, failureStore, notifyFlowEnteredWaiting, logger,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
// The harness injects runTurn — core never speaks the runtime's call
|
|
78
|
+
// shape, so a node dispatch is one function call from the engine's point
|
|
79
|
+
// of view (see engine/agent-dispatch.js and the harness's run-turn.js).
|
|
80
|
+
const { dispatchAgentWithRetry } = createAgentDispatch({
|
|
81
|
+
runTurn,
|
|
82
|
+
sessionIndex,
|
|
83
|
+
logger,
|
|
84
|
+
emit,
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
// nodeCtx is assigned after start/runFlow are defined (see below).
|
|
88
|
+
// Subflow/node runners read it via getter at call time, not at factory-construction
|
|
89
|
+
// time, so the undefined-at-construction gap is safe.
|
|
90
|
+
let nodeCtx;
|
|
91
|
+
|
|
92
|
+
// Every pipeline's subflow node now runs under childFlows:true — this is a
|
|
93
|
+
// direct call to the only dispatch path that remains. `runFlow` is a
|
|
94
|
+
// hoisted function declaration defined below — safe to reference here.
|
|
95
|
+
// `kind:"subflow"` covers both the many-children-via-selector shape and
|
|
96
|
+
// single-child use.
|
|
97
|
+
async function executeSubflowNode({ node, nodeId, flowId, state }) {
|
|
98
|
+
// A subflow node's own selector(state) can legitimately need a PRIOR
|
|
99
|
+
// subflow node's aggregated per-subflow results (e.g. PROPOSE's selector
|
|
100
|
+
// reading state._subflowGroups.SCORE to resolve each shortlisted
|
|
101
|
+
// listing's score data) — same reason dispatchCodeEntry
|
|
102
|
+
// (executor-nodes.js) enriches state before calling a "code" node's
|
|
103
|
+
// run(). Subflow dispatch never does this enrichment, so any such
|
|
104
|
+
// selector silently sees undefined subflow groups and resolves to zero
|
|
105
|
+
// items regardless of how many upstream branches actually succeeded.
|
|
106
|
+
//
|
|
107
|
+
// assembleState(baseState: state) computes _subflowGroups here (the
|
|
108
|
+
// same getAllSubflowGroups call, folded in) and additionally overlays
|
|
109
|
+
// each node's resultJson on top of the real stateJson passed as
|
|
110
|
+
// baseState. The overlay is a no-op against real flows (resultJson-derived
|
|
111
|
+
// values are byte-identical to state_json's), so the enrichment stays
|
|
112
|
+
// scoped to the single narrowest read site (node.selector) rather than
|
|
113
|
+
// all 5 root call sites at once.
|
|
114
|
+
const enrichedState = await assembleState(flowId, { runStore, nodeExecutionsStore: nodeExecutions, baseState: state });
|
|
115
|
+
return dispatchSubflowChildren({
|
|
116
|
+
node, nodeId, flowId, state: enrichedState, store, runStore, nodeExecutionsStore: nodeExecutions,
|
|
117
|
+
maybeBudgetGate, runFlow, logger, lineage: _lineage, pipelinesRef, admissionQueueStore,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const { executeNode } = createNodeRunner({
|
|
122
|
+
maybeBudgetGate, dispatchAgentWithRetry, executeSubflowNode,
|
|
123
|
+
api, config: _config, lineage: _lineage, pipelinesDir: _pipelinesDir, logger,
|
|
124
|
+
getNodeCtx: () => nodeCtx,
|
|
125
|
+
runStore, nodeExecutionsStore: nodeExecutions,
|
|
126
|
+
nodeHealthStore, pipelinesRef,
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Create the flow_run row for a new pipeline run and return the record.
|
|
131
|
+
* Does NOT start the execution loop — call `runFlow(record.flowId)` to drive it.
|
|
132
|
+
*/
|
|
133
|
+
async function start(pipelineId, args = {}, triggeredBy = null) {
|
|
134
|
+
const pipeline = pipelinesRef.value.get(pipelineId);
|
|
135
|
+
if (!pipeline) throw new Error(`unknown pipeline: ${pipelineId}`);
|
|
136
|
+
|
|
137
|
+
const runId = randomUUID();
|
|
138
|
+
const initialState = {
|
|
139
|
+
runId,
|
|
140
|
+
startedAt: Date.now(),
|
|
141
|
+
args,
|
|
142
|
+
triggeredBy: triggeredBy ?? null,
|
|
143
|
+
...deepClone(pipeline.state),
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
const record = await store.create({ pipelineId, initialState });
|
|
147
|
+
logger.info(`manifold [${record.flowId.slice(0, 8)}] start: pipeline=${pipelineId} runId=${runId.slice(0, 8)}`);
|
|
148
|
+
|
|
149
|
+
// Per-pipeline singleton gate. Default (unset) is unlimited — the
|
|
150
|
+
// shared admission pool below is the only gate almost every pipeline
|
|
151
|
+
// needs. `allowConcurrent: false` opts a pipeline out of overlap
|
|
152
|
+
// entirely (e.g. a browser-automation poller that would race itself
|
|
153
|
+
// over shared session/gate state): reject this run before it ever
|
|
154
|
+
// competes for an admission slot if another non-terminal run of the
|
|
155
|
+
// same pipelineId already exists.
|
|
156
|
+
if (pipeline.allowConcurrent === false) {
|
|
157
|
+
const others = await runStore.listRunsPage({ pipelineId });
|
|
158
|
+
const active = others.find((r) => r.flowId !== record.flowId && !isTerminal(r.status));
|
|
159
|
+
if (active) {
|
|
160
|
+
logger.info(`manifold [${record.flowId.slice(0, 8)}] start: skipped — ${pipelineId} already has an active run (flowId=${active.flowId.slice(0, 8)}, status=${active.status})`);
|
|
161
|
+
await store.skip({
|
|
162
|
+
flowId: record.flowId,
|
|
163
|
+
expectedRevision: record.revision,
|
|
164
|
+
stateJson: record.stateJson,
|
|
165
|
+
reason: `another run of ${pipelineId} is already active (flowId=${active.flowId}, status=${active.status})`,
|
|
166
|
+
});
|
|
167
|
+
return await store.get(record.flowId);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Whole-flow-run admission — the shared pool (state/admission-queue-
|
|
172
|
+
// store.js), same one subflow child-dispatch draws from. Every new flow
|
|
173
|
+
// competes for a real slot the instant it's created, and queues (rather
|
|
174
|
+
// than being rejected) if the pool is full — the caller gets back a
|
|
175
|
+
// record with status:"queued", exactly like an unadmitted subflow
|
|
176
|
+
// child. state-store.js's
|
|
177
|
+
// release/promotion wiring wakes it the moment a slot frees, no restart
|
|
178
|
+
// or caller retry needed.
|
|
179
|
+
const admission = await admissionQueueStore.admit({ kind: "flow-start", flowId: record.flowId });
|
|
180
|
+
if (admission.status === "queued") {
|
|
181
|
+
logger.info(`manifold [${record.flowId.slice(0, 8)}] start: queued — no admission slot available yet`);
|
|
182
|
+
await store.setQueued({
|
|
183
|
+
flowId: record.flowId,
|
|
184
|
+
expectedRevision: record.revision,
|
|
185
|
+
lastStep: record.lastStep,
|
|
186
|
+
stateJson: record.stateJson,
|
|
187
|
+
waitJson: { kind: "flow-start" },
|
|
188
|
+
statusReason: "queued: waiting for a shared admission slot",
|
|
189
|
+
});
|
|
190
|
+
return await store.get(record.flowId);
|
|
191
|
+
}
|
|
192
|
+
return record;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Drive the run loop until the flow reaches a terminal status or blocks.
|
|
197
|
+
* Safe to call multiple times — re-enters the loop wherever the flow is.
|
|
198
|
+
*
|
|
199
|
+
* `kind` describes why THIS runFlow() call was made — "fresh" | "retry" |
|
|
200
|
+
* "revival" | "recheck" — and is only actually consulted for the specific
|
|
201
|
+
* node the flow was already sitting at (a non-fresh re-entry); every
|
|
202
|
+
* subsequent genuinely-new node this loop advances to resolves to "fresh"
|
|
203
|
+
* regardless of what was passed here (see the execKind computation below).
|
|
204
|
+
* Required, not defaulted — see state/node-executions.js's write() for why
|
|
205
|
+
* a caller must always be explicit about this.
|
|
206
|
+
*/
|
|
207
|
+
async function runFlow(flowId, kind) {
|
|
208
|
+
while (true) {
|
|
209
|
+
const flow = await store.get(flowId);
|
|
210
|
+
if (!flow) {
|
|
211
|
+
logger.warn(`manifold [${flowId.slice(0, 8)}] vanished mid-execution`);
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (isTerminal(flow.status)) {
|
|
215
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] terminal (${flow.status})`);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (isBlocked(flow.status)) {
|
|
219
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] ${flow.status} at ${flow.lastStep} on ${flow.waitJson?.kind ?? "?"}`);
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
// A flow queued for its OWN flow-start or child-dispatch admission
|
|
223
|
+
// (kind:"flow-start" / "child-dispatch") is not yet cleared to run at
|
|
224
|
+
// all — its admission check lives OUTSIDE this loop (in start()'s
|
|
225
|
+
// caller for flow-start, in the parent's dispatch loop for
|
|
226
|
+
// child-dispatch), so a fresh runFlow() call here has no code path
|
|
227
|
+
// that would naturally re-request admission; it would just dispatch
|
|
228
|
+
// lastStep's outbound edge unconditionally, bypassing the pool
|
|
229
|
+
// entirely. Return immediately — admission-queue-store's own
|
|
230
|
+
// release()/promotion wakes it later.
|
|
231
|
+
if (isQueued(flow.status) && (flow.waitJson?.kind === "flow-start" || flow.waitJson?.kind === "child-dispatch")) {
|
|
232
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] queued at ${flow.lastStep} on ${flow.waitJson?.kind}`);
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const state = flow.stateJson ?? {};
|
|
237
|
+
if (state._paused) {
|
|
238
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] paused at ${flow.lastStep}`);
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
const pipelineId = state.pipelineId;
|
|
242
|
+
if (pipelineId && !pipelinesRef.value.get(pipelineId) && typeof ensurePipelinesLoaded === "function") {
|
|
243
|
+
try { await ensurePipelinesLoaded(); } catch (err) {
|
|
244
|
+
logger.warn(`manifold [${flowId.slice(0, 8)}] ensurePipelinesLoaded failed: ${err?.message ?? err}`);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
const pipeline = pipelineId ? pipelinesRef.value.get(pipelineId) : null;
|
|
248
|
+
if (!pipeline) {
|
|
249
|
+
await persistFailRetrying({
|
|
250
|
+
flowId, revision: flow.revision, state,
|
|
251
|
+
message: `pipeline not loaded: ${pipelineId}`,
|
|
252
|
+
failureNode: "<engine:pipeline-load>",
|
|
253
|
+
});
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const lastStep = flow.lastStep ?? "__start__";
|
|
258
|
+
// A cascade leg's own edge function (2-arg mode, see resolveEdge) can
|
|
259
|
+
// read the error the leg just threw — carried here via a transient
|
|
260
|
+
// `_cascadeError` state key set right before the retry-continue below,
|
|
261
|
+
// guarded on nodeId===lastStep so a stale marker from an earlier
|
|
262
|
+
// loop-back is never read once lastStep has moved past it. Survives a
|
|
263
|
+
// gateway restart for free — it's just persisted state_json, same as
|
|
264
|
+
// every other field the executor writes.
|
|
265
|
+
const cascadeError = state?._cascadeError?.nodeId === lastStep ? state._cascadeError : null;
|
|
266
|
+
// Edge functions read via assembleState(baseState: state), not raw
|
|
267
|
+
// state — the single call site every edge decision in the engine goes
|
|
268
|
+
// through (root and child flows alike, since they share this one
|
|
269
|
+
// runFlow loop). A separate local var, not a reassignment of `state`
|
|
270
|
+
// itself — `state` is still what gets persisted below; only the value
|
|
271
|
+
// handed to the edge function changes.
|
|
272
|
+
const edgeState = await assembleState(flowId, { runStore, nodeExecutionsStore: nodeExecutions, baseState: state });
|
|
273
|
+
const nextNodeId = resolveEdge(pipeline.edges, lastStep, edgeState, {
|
|
274
|
+
logger, flowId,
|
|
275
|
+
error: cascadeError ? { message: cascadeError.message, errorType: cascadeError.errorType } : null,
|
|
276
|
+
});
|
|
277
|
+
if (!nextNodeId) {
|
|
278
|
+
// If this exhaustion traces back to a throw already recorded
|
|
279
|
+
// immediately at throw-time (the cascade catch branch below),
|
|
280
|
+
// thread that same invocationId/executionId through so
|
|
281
|
+
// upsertFailureRow's execution_id-based upsert collapses this into
|
|
282
|
+
// the SAME failure_events row instead of creating a second,
|
|
283
|
+
// less-detailed one for the identical logical moment. A genuine
|
|
284
|
+
// non-throw dead-end (unknown node, business-logic exhaustion with
|
|
285
|
+
// no antecedent throw) has no cascadeError and gets the plain
|
|
286
|
+
// generic message, same as always.
|
|
287
|
+
await persistFailRetrying({
|
|
288
|
+
flowId, revision: flow.revision, state,
|
|
289
|
+
message: cascadeError?.message ?? `edge ${lastStep} returned no next node`,
|
|
290
|
+
failureNode: lastStep,
|
|
291
|
+
invocationId: cascadeError?.invocationId ?? null,
|
|
292
|
+
executionId: cascadeError?.executionId ?? null,
|
|
293
|
+
errorType: cascadeError?.errorType ?? null,
|
|
294
|
+
});
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
if (nextNodeId === "__end__") {
|
|
298
|
+
// This root read site carries the same resultJson-replay need as
|
|
299
|
+
// every other: pipeline.display() and successCriteria() both read
|
|
300
|
+
// node output through here. assembleState() adds _subflowGroups AND
|
|
301
|
+
// replays resultJson onto raw `state` — both are needed, because
|
|
302
|
+
// state_json is entry-only and carries no node output of its own.
|
|
303
|
+
const enrichedState = await assembleState(flowId, { runStore, nodeExecutionsStore: nodeExecutions, baseState: state });
|
|
304
|
+
await runAuditIfConfigured({ flowId, pipeline, state: enrichedState, logger });
|
|
305
|
+
// failureStore.recordFromTerminalState's soft-marker scan was
|
|
306
|
+
// deleted — zero soft-error producers remain anywhere in
|
|
307
|
+
// `.pipelines`/`.shared`; every failure now persists directly from
|
|
308
|
+
// this file's own throw-catch paths via recordEngineFailure.
|
|
309
|
+
// finalState is enrichedState-based, not state-based — a node's own
|
|
310
|
+
// output (e.g. a runAudit-only sentinel field like
|
|
311
|
+
// `_successCriteriaReason`) only exists via resultJson replay now,
|
|
312
|
+
// so finalState._successCriteriaReason below would silently read
|
|
313
|
+
// undefined off raw `state` otherwise. Safe to carry _subflowGroups
|
|
314
|
+
// through here too: the persist-layer filter
|
|
315
|
+
// (executor-persist.js's toPersistableState) already narrows
|
|
316
|
+
// whatever's passed to persistFailRetrying/persistSkipRetrying/
|
|
317
|
+
// persistFinishRetrying down to entry-only fields regardless of
|
|
318
|
+
// what finalState itself contains.
|
|
319
|
+
let finalState = enrichedState;
|
|
320
|
+
if (typeof pipeline.display === "function") {
|
|
321
|
+
try {
|
|
322
|
+
const out = await Promise.resolve(pipeline.display(enrichedState));
|
|
323
|
+
if (out && typeof out === "object") {
|
|
324
|
+
finalState = {
|
|
325
|
+
...enrichedState,
|
|
326
|
+
_display: {
|
|
327
|
+
title: typeof out.title === "string" ? out.title : null,
|
|
328
|
+
summary: typeof out.summary === "string" ? out.summary : null,
|
|
329
|
+
funnels: Array.isArray(out.funnels) ? out.funnels : null,
|
|
330
|
+
metrics: Array.isArray(out.metrics) ? out.metrics : null,
|
|
331
|
+
},
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
} catch (err) {
|
|
335
|
+
logger?.warn?.(`manifold [${flowId.slice(0, 8)}] pipeline.display() threw: ${err?.message ?? err}`);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
const successCriteria = pipeline.runAudit?.successCriteria;
|
|
339
|
+
if (typeof successCriteria === "function") {
|
|
340
|
+
let verdict;
|
|
341
|
+
try {
|
|
342
|
+
verdict = await Promise.resolve(successCriteria({ ...enrichedState, ...finalState }));
|
|
343
|
+
} catch (err) {
|
|
344
|
+
logger?.warn?.(`manifold [${flowId.slice(0, 8)}] successCriteria threw: ${err?.message ?? err}`);
|
|
345
|
+
}
|
|
346
|
+
if (verdict === "failed") {
|
|
347
|
+
await persistFailRetrying({
|
|
348
|
+
flowId, revision: flow.revision,
|
|
349
|
+
state: finalState,
|
|
350
|
+
message: finalState._successCriteriaReason ?? "successCriteria: failed",
|
|
351
|
+
failureNode: "__end__",
|
|
352
|
+
});
|
|
353
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] __end__ → failed (successCriteria)`);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
if (verdict === "skipped") {
|
|
357
|
+
await persistSkipRetrying({
|
|
358
|
+
flowId, revision: flow.revision,
|
|
359
|
+
state: finalState,
|
|
360
|
+
reason: finalState._successCriteriaReason ?? "successCriteria: skipped",
|
|
361
|
+
});
|
|
362
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] __end__ → skipped (successCriteria)`);
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
await persistFinishRetrying({ flowId, revision: flow.revision, state: finalState });
|
|
367
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] __end__`);
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
const node = pipeline.nodes[nextNodeId];
|
|
372
|
+
if (!node) {
|
|
373
|
+
await persistFailRetrying({
|
|
374
|
+
flowId, revision: flow.revision, state,
|
|
375
|
+
message: `unknown node: ${nextNodeId}`,
|
|
376
|
+
failureNode: nextNodeId,
|
|
377
|
+
});
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] → ${nextNodeId} (${node.kind})`);
|
|
382
|
+
|
|
383
|
+
const enteredAt = Date.now();
|
|
384
|
+
// Computed once per dispatch, from this flow's
|
|
385
|
+
// own last-dispatched node — see executor-helpers.js's isFreshNodeEntry
|
|
386
|
+
// for the full reasoning. execKind is the actual value threaded to every write()
|
|
387
|
+
// call below in this iteration (only one ever fires per iteration) —
|
|
388
|
+
// a genuinely fresh node entry is always "fresh" regardless of why
|
|
389
|
+
// this runFlow() call happened; only a non-fresh re-entry (the node
|
|
390
|
+
// the flow was already at) "spends" the caller-supplied kind.
|
|
391
|
+
const lastNodeId = await nodeExecutions.findCurrentStep(flowId);
|
|
392
|
+
const forceNew = isFreshNodeEntry(lastNodeId, nextNodeId);
|
|
393
|
+
const execKind = forceNew ? "fresh" : kind;
|
|
394
|
+
// Create the invocation row before execution: "last invocation for
|
|
395
|
+
// this flow" tracks the in-flight node even during execution, not
|
|
396
|
+
// just after completion — this is how the currently-running node is
|
|
397
|
+
// identified.
|
|
398
|
+
const enterResult = await nodeExecutions.enter({
|
|
399
|
+
flowId, pipelineId: state.pipelineId ?? null,
|
|
400
|
+
nodeId: nextNodeId, subflowKey: null, kind: execKind,
|
|
401
|
+
});
|
|
402
|
+
const invocationId = enterResult?.invocationId ?? null;
|
|
403
|
+
const isNew = enterResult?.isNew ?? (execKind === "fresh");
|
|
404
|
+
process.nextTick(() => nodeEmitter.emit("node.executing", {
|
|
405
|
+
flowId, nodeId: nextNodeId,
|
|
406
|
+
pipelineId: state.pipelineId ?? null,
|
|
407
|
+
parentInvocationId: flow.parentInvocationId ?? null,
|
|
408
|
+
}));
|
|
409
|
+
let stateUpdate = {};
|
|
410
|
+
let waitJson = null;
|
|
411
|
+
let blockedInfo = null;
|
|
412
|
+
let skippedInfo = null;
|
|
413
|
+
let reentrant = false;
|
|
414
|
+
let exitedAt = null;
|
|
415
|
+
try {
|
|
416
|
+
// "needs:" resolution is lazy, via ctx.needs(varName) called
|
|
417
|
+
// from inside the node's own run()/promptBuilder()/selector(), NOT
|
|
418
|
+
// checked here ahead of the dispatch. executor-nodes.js/executor-subflow-children.js
|
|
419
|
+
// build that accessor (needs-resolver.js's makeNeedsCtx) and pass
|
|
420
|
+
// it through as part of ctx — nothing to do at this call site
|
|
421
|
+
// beyond letting a NodeBlocked(_kind:"needs-not-ready") thrown from
|
|
422
|
+
// inside executeNode() propagate to the catch block below exactly
|
|
423
|
+
// like any other NodeBlocked (the budget-gate pause already relies
|
|
424
|
+
// on this same path). Known gap, deliberately deferred (needs: is
|
|
425
|
+
// dormant, zero pipelines declare it yet): reviveAgentKindOrphan's
|
|
426
|
+
// own dispatch path does not pass a needs ctx at all, so an agent-
|
|
427
|
+
// kind node that crashes mid-flight can't call ctx.needs() on
|
|
428
|
+
// revival — worth fixing before any pipeline adopts needs: on an
|
|
429
|
+
// agent-kind node, not before.
|
|
430
|
+
const result = await executeNode({ node, nodeId: nextNodeId, state, flowId, invocationId });
|
|
431
|
+
exitedAt = Date.now();
|
|
432
|
+
stateUpdate = result.stateUpdate ?? {};
|
|
433
|
+
waitJson = result.waitJson ?? null;
|
|
434
|
+
blockedInfo = result.blockedInfo ?? null;
|
|
435
|
+
skippedInfo = result.skippedInfo ?? null;
|
|
436
|
+
reentrant = result.reentrant === true;
|
|
437
|
+
} catch (err) {
|
|
438
|
+
exitedAt ??= Date.now();
|
|
439
|
+
if (err instanceof NodeBlocked) {
|
|
440
|
+
blockedInfo = err._blocked;
|
|
441
|
+
} else if (err instanceof GatewayHandoffNeeded) {
|
|
442
|
+
blockedInfo = {
|
|
443
|
+
reason: `node ${nextNodeId} needs gateway context: ${err.message}`,
|
|
444
|
+
needed: [],
|
|
445
|
+
hint: "No automatic resume — requires a scope-gap fix at the call site or manual task-revival.",
|
|
446
|
+
_kind: "needs-gateway-handoff",
|
|
447
|
+
};
|
|
448
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} → needs-gateway-handoff: ${err.message}`);
|
|
449
|
+
} else {
|
|
450
|
+
const msg = err?.message ?? String(err);
|
|
451
|
+
// Duck-typed, not `instanceof NodeFailure` — pipeline/shared code
|
|
452
|
+
// and the engine itself can load executor-errors.js through
|
|
453
|
+
// different module registries (the gateway daemon's data:-URL
|
|
454
|
+
// pipeline loader vs. jiti-transpiled CJS for CLI tooling), which
|
|
455
|
+
// gives structurally-identical but reference-distinct NodeFailure
|
|
456
|
+
// classes; instanceof silently returns false across that
|
|
457
|
+
// boundary even for a genuine NodeFailure. A plain Error never
|
|
458
|
+
// has .errorType set, so this is equivalent and robust either way.
|
|
459
|
+
const errorType = err?.errorType ?? null;
|
|
460
|
+
logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
|
|
461
|
+
// No error/errorType here — flow_node_executions is purely an
|
|
462
|
+
// execution-window/identity ledger; every throw's actual content
|
|
463
|
+
// now goes to failure_events instead, via the recordEngineFailure
|
|
464
|
+
// call(s) below.
|
|
465
|
+
const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
466
|
+
if (pipeline.kind === "cascade") {
|
|
467
|
+
// A cascade-kind flow treats ANY throw like a completed dispatch,
|
|
468
|
+
// not a failure
|
|
469
|
+
// — lastStep advances to the node that just threw (same persist
|
|
470
|
+
// shape as the ordinary success path further below), and it's
|
|
471
|
+
// THAT node's own edge function that decides what happens next,
|
|
472
|
+
// the same way it would after a success. A cascade leg throwing
|
|
473
|
+
// NodeFailure (see executor-errors.js) carries WHY it failed via
|
|
474
|
+
// `errorType` — threaded here into `_cascadeError`, a transient
|
|
475
|
+
// state key the next loop iteration reads (guarded on
|
|
476
|
+
// nodeId===lastStep, see the resolveEdge call above) and passes
|
|
477
|
+
// to a 2-arg edge function as its error argument. This is what
|
|
478
|
+
// lets a cascade's own edge decide retry-vs-fallback-vs-give-up
|
|
479
|
+
// from the REAL thrown error, not a soft-returned field.
|
|
480
|
+
//
|
|
481
|
+
// Every throw persists a permanent failure_events row
|
|
482
|
+
// IMMEDIATELY, right here, regardless of what the edge
|
|
483
|
+
// decides next — not just on eventual give-up. This is the one
|
|
484
|
+
// gap the persistFailRetrying->recordEngineFailure path never
|
|
485
|
+
// covers (a cascade-continuing throw never reaches
|
|
486
|
+
// persistFailRetrying at all). If this attempt turns out to be
|
|
487
|
+
// the cascade's final one, the `!nextNodeId` branch at the top
|
|
488
|
+
// of this loop reuses this SAME invocationId/executionId when
|
|
489
|
+
// it fails the flow, so upsertFailureRow's execution_id upsert
|
|
490
|
+
// collapses onto this row instead of creating a second one.
|
|
491
|
+
try {
|
|
492
|
+
await failureStore?.recordEngineFailure?.({
|
|
493
|
+
flowId, pipelineId: state.pipelineId ?? null, failureNode: nextNodeId,
|
|
494
|
+
message: msg, state, invocationId: written.invocationId ?? null, executionId: written.executionId ?? null, errorType,
|
|
495
|
+
});
|
|
496
|
+
} catch (recErr) {
|
|
497
|
+
logger?.warn?.(`manifold [${flowId.slice(0, 8)}] recordEngineFailure (cascade) threw: ${recErr?.message ?? recErr}`);
|
|
498
|
+
}
|
|
499
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} → cascading (its own edge decides what happens next)`);
|
|
500
|
+
await persistRunningRetrying({
|
|
501
|
+
flowId, revision: flow.revision, lastStep: nextNodeId,
|
|
502
|
+
state: {
|
|
503
|
+
...state,
|
|
504
|
+
_cascadeError: {
|
|
505
|
+
nodeId: nextNodeId, message: msg, errorType,
|
|
506
|
+
invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
|
|
507
|
+
},
|
|
508
|
+
},
|
|
509
|
+
});
|
|
510
|
+
continue;
|
|
511
|
+
}
|
|
512
|
+
await persistFailRetrying({
|
|
513
|
+
flowId, revision: flow.revision, state,
|
|
514
|
+
message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
|
|
515
|
+
invocationId: written.invocationId ?? null, executionId: written.executionId ?? null, errorType,
|
|
516
|
+
});
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
if (blockedInfo) {
|
|
522
|
+
if (!blockedInfo._kind) {
|
|
523
|
+
const msg = blockedInfo.reason ?? `${nextNodeId} blocked without declaring a wait kind`;
|
|
524
|
+
logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
|
|
525
|
+
const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
526
|
+
await persistFailRetrying({
|
|
527
|
+
flowId, revision: flow.revision, state,
|
|
528
|
+
message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
|
|
529
|
+
invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
|
|
530
|
+
});
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
const waitJsonForBlock = {
|
|
534
|
+
kind: blockedInfo._kind,
|
|
535
|
+
nodeId: nextNodeId,
|
|
536
|
+
reason: blockedInfo.reason ?? "node blocked",
|
|
537
|
+
needed: blockedInfo.needed ?? [],
|
|
538
|
+
hint: blockedInfo.hint ?? "",
|
|
539
|
+
...(blockedInfo.responder ? { responder: blockedInfo.responder } : {}),
|
|
540
|
+
...(Array.isArray(blockedInfo.choices) && blockedInfo.choices.length > 0
|
|
541
|
+
? { choices: blockedInfo.choices.map((c) => String(c)) }
|
|
542
|
+
: {}),
|
|
543
|
+
...(Array.isArray(blockedInfo.items) ? { items: blockedInfo.items } : {}),
|
|
544
|
+
renderedAt: Date.now(),
|
|
545
|
+
retry: true,
|
|
546
|
+
};
|
|
547
|
+
try {
|
|
548
|
+
validateGateLayout(waitJsonForBlock.items);
|
|
549
|
+
} catch (err) {
|
|
550
|
+
const msg = err?.message ?? String(err);
|
|
551
|
+
logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
|
|
552
|
+
const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
553
|
+
await persistFailRetrying({
|
|
554
|
+
flowId, revision: flow.revision, state,
|
|
555
|
+
message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
|
|
556
|
+
invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
|
|
557
|
+
});
|
|
558
|
+
return;
|
|
559
|
+
}
|
|
560
|
+
const summary = `${nextNodeId}: ${waitJsonForBlock.reason}`;
|
|
561
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] ${waitJsonForBlock.kind} → waiting (will re-run ${nextNodeId} on resume)\n reason: ${waitJsonForBlock.reason}\n hint: ${waitJsonForBlock.hint}`);
|
|
562
|
+
await persistWaitingRetrying({
|
|
563
|
+
flowId, revision: flow.revision, lastStep,
|
|
564
|
+
blockingNodeId: nextNodeId,
|
|
565
|
+
state,
|
|
566
|
+
waitJson: waitJsonForBlock,
|
|
567
|
+
statusReason: shortSummary(summary),
|
|
568
|
+
});
|
|
569
|
+
nodeExecutions.write({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, error: null, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
if (skippedInfo) {
|
|
574
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] ${nextNodeId}: _skipped — ${skippedInfo.reason}`);
|
|
575
|
+
await persistSkipRetrying({
|
|
576
|
+
flowId, revision: flow.revision,
|
|
577
|
+
state,
|
|
578
|
+
reason: skippedInfo.reason ?? "skipped",
|
|
579
|
+
});
|
|
580
|
+
nodeExecutions.write({ flowId, pipelineId: state.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, error: null, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
581
|
+
return;
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
const freshAfterNode = (await store.get(flowId)) ?? { stateJson: state, revision: flow.revision };
|
|
585
|
+
// entryState is taken directly from freshAfterNode.stateJson, not
|
|
586
|
+
// recomputed as a merge of this node's own output. Every persist call
|
|
587
|
+
// below filters its `state:` argument down to entry/bookkeeping fields
|
|
588
|
+
// only (executor-persist.js's toPersistableState), so a wide merge of
|
|
589
|
+
// the node's output here would never reach the database. A node's own
|
|
590
|
+
// output reaches result_json separately, via writeResult(invocationId,
|
|
591
|
+
// stateUpdate) further down — entirely independent of this variable.
|
|
592
|
+
// freshAfterNode.stateJson is already entry-only (it was itself
|
|
593
|
+
// persisted through the same filter), so it's used directly.
|
|
594
|
+
let entryState = freshAfterNode.stateJson ?? state;
|
|
595
|
+
// This node just completed successfully (no throw, no blockedInfo, no
|
|
596
|
+
// skippedInfo — we're past all of those branches above) — any
|
|
597
|
+
// _cascadeError left over from an earlier failed attempt AT THIS SAME
|
|
598
|
+
// node is now stale. Only ever lives on this entry-only state (a
|
|
599
|
+
// success's own stateUpdate never mentions _cascadeError), so a node
|
|
600
|
+
// retried after a cascade throw and then succeeding would otherwise
|
|
601
|
+
// carry a stale error into its own next edge-function call if that
|
|
602
|
+
// edge checks `error` before `state`. Clear it explicitly.
|
|
603
|
+
if (entryState._cascadeError?.nodeId === nextNodeId) {
|
|
604
|
+
const { _cascadeError, ...rest } = entryState;
|
|
605
|
+
entryState = rest;
|
|
606
|
+
}
|
|
607
|
+
const persistRevision = freshAfterNode.revision ?? flow.revision;
|
|
608
|
+
|
|
609
|
+
if (waitJson) {
|
|
610
|
+
try {
|
|
611
|
+
validateGateLayout(waitJson.items);
|
|
612
|
+
} catch (err) {
|
|
613
|
+
const msg = err?.message ?? String(err);
|
|
614
|
+
logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
|
|
615
|
+
const written = await safeWriteExecution({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
616
|
+
await persistFailRetrying({
|
|
617
|
+
flowId, revision: persistRevision, state: entryState,
|
|
618
|
+
message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
|
|
619
|
+
invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
|
|
620
|
+
});
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
// The wait summary is synthesized by describeWaitFallback(waitJson,
|
|
624
|
+
// nextNodeId) — no pipeline-visible render hook feeds this log line.
|
|
625
|
+
const summary = describeWaitFallback(waitJson, nextNodeId);
|
|
626
|
+
logger.info(`manifold [${flowId.slice(0, 8)}] ${reentrant ? "reentrant gate" : "gate"} → blocked\n${summary}`);
|
|
627
|
+
await persistWaitingRetrying({
|
|
628
|
+
flowId, revision: persistRevision,
|
|
629
|
+
lastStep: reentrant ? lastStep : nextNodeId,
|
|
630
|
+
blockingNodeId: nextNodeId,
|
|
631
|
+
state: entryState,
|
|
632
|
+
waitJson,
|
|
633
|
+
statusReason: shortSummary(summary),
|
|
634
|
+
});
|
|
635
|
+
nodeExecutions.write({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, error: null, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
636
|
+
return;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
// Write this node's own result FIRST and durably, before advancing
|
|
640
|
+
// flow_runs.last_step to nextNodeId via persistRunningRetrying below.
|
|
641
|
+
// last_step is what boot-time orphan revival and the pipeline-
|
|
642
|
+
// concurrency dequeue path both trust as "this node completed" — they
|
|
643
|
+
// jump straight to resolving its outbound edge without re-running
|
|
644
|
+
// apply(). Writing the result first makes that invariant crash-safe:
|
|
645
|
+
// a restart landing before last_step advances leaves last_step
|
|
646
|
+
// pointing at the previous node, so revival re-runs this node's
|
|
647
|
+
// apply() again — a duplicate call, not a silent failure.
|
|
648
|
+
//
|
|
649
|
+
// Also still AWAITED, not fire-and-forget: state_json is entry-only,
|
|
650
|
+
// so the very next loop iteration's assembleState() call (line ~288,
|
|
651
|
+
// feeding the edge function) is the ONLY way this node's own output
|
|
652
|
+
// becomes visible again — if
|
|
653
|
+
// that read raced ahead of this write, the edge function would see
|
|
654
|
+
// this node's output as still-missing, some edges then re-route back
|
|
655
|
+
// to the same node (or worse), producing a tight busy-loop.
|
|
656
|
+
// Awaiting costs one DB round-trip per dispatch, already priced into
|
|
657
|
+
// every other persist call this loop makes.
|
|
658
|
+
const resultWrite = await nodeExecutions.writeResult(invocationId, stateUpdate).catch(() => null);
|
|
659
|
+
if (!resultWrite || !resultWrite.ok) {
|
|
660
|
+
const reason = resultWrite?.reason ?? "threw";
|
|
661
|
+
// "race" means a CONCURRENT writer already committed real content
|
|
662
|
+
// for this exact invocation — advancing is correct, the data
|
|
663
|
+
// genuinely is there, just not from this call. Every other reason
|
|
664
|
+
// ("no-invocation-id"/"no-such-invocation"/"unserializable"/
|
|
665
|
+
// "db-error"/a genuine throw, normalized via the catch above) means
|
|
666
|
+
// NO ONE wrote anything. Advancing last_step in that case would
|
|
667
|
+
// move the flow past this node with its output never durably
|
|
668
|
+
// written: the next runFlow() iteration's assembleState() reads
|
|
669
|
+
// nothing for this node, and resolveEdge() dead-ends with "no next
|
|
670
|
+
// node" — the SCORE_LLM "edge returned no next node" shape, and the
|
|
671
|
+
// boot-revival gap it left uncovered (see reviveAgentKindOrphan
|
|
672
|
+
// below). Fail the flow outright for any non-race reason instead of
|
|
673
|
+
// silently continuing, so the operator sees the true cause.
|
|
674
|
+
if (reason !== "race") {
|
|
675
|
+
const msg = `resultJson write failed (${reason})`;
|
|
676
|
+
logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} ${msg}`);
|
|
677
|
+
const written = await safeWriteExecution({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
678
|
+
await persistFailRetrying({
|
|
679
|
+
flowId, revision: persistRevision, state: entryState,
|
|
680
|
+
message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
|
|
681
|
+
invocationId: written.invocationId ?? invocationId, executionId: written.executionId ?? null,
|
|
682
|
+
});
|
|
683
|
+
return;
|
|
684
|
+
}
|
|
685
|
+
logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} resultJson write failed (race) — another writer already completed this invocation`);
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
const runResult = await persistRunningRetrying({
|
|
689
|
+
flowId, revision: persistRevision, lastStep: nextNodeId,
|
|
690
|
+
state: entryState,
|
|
691
|
+
});
|
|
692
|
+
if (!runResult.applied) {
|
|
693
|
+
const msg = `state persist failed: ${runResult.code}`;
|
|
694
|
+
logger.warn(`manifold [${flowId.slice(0, 8)}] ${nextNodeId} failed: ${msg}`);
|
|
695
|
+
const written = await safeWriteExecution({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
696
|
+
await persistFailRetrying({
|
|
697
|
+
flowId, revision: persistRevision, state: entryState,
|
|
698
|
+
message: `${nextNodeId}: ${msg}`, failureNode: nextNodeId,
|
|
699
|
+
invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
|
|
700
|
+
});
|
|
701
|
+
return;
|
|
702
|
+
}
|
|
703
|
+
nodeExecutions.write({ flowId, pipelineId: entryState.pipelineId ?? null, nodeId: nextNodeId, enteredAt, exitedAt, error: null, kind: execKind, invocationId, isNewInvocation: isNew });
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
// Object passed to `code`-kind node bodies as ctx. `invokePipeline` fires a
|
|
708
|
+
// background flow start and returns immediately. Assigned after start/runFlow
|
|
709
|
+
// are defined; subflow/node runners read it via getter so the gap is safe.
|
|
710
|
+
nodeCtx = Object.freeze({
|
|
711
|
+
async invokePipeline(targetPipelineId, args = {}) {
|
|
712
|
+
const record = await start(targetPipelineId, args);
|
|
713
|
+
// start() does not throw when the shared admission pool is full —
|
|
714
|
+
// it queues instead (see start()'s own comment). Preserves the
|
|
715
|
+
// "singleton-busy" contract at this call site specifically: a caller
|
|
716
|
+
// handling invoked:false is expected to retry invokePipeline itself
|
|
717
|
+
// later, not rely on this record being silently promoted and run out
|
|
718
|
+
// from under it with nobody tracking the outcome — so the just-created
|
|
719
|
+
// queued record is cancelled rather than left to run unobserved.
|
|
720
|
+
if (record.status === "queued") {
|
|
721
|
+
await cancel(record.flowId, "invokePipeline: shared admission pool was full").catch(() => {});
|
|
722
|
+
return {
|
|
723
|
+
invoked: false, reason: "singleton-busy", target: targetPipelineId,
|
|
724
|
+
message: `pipeline "${targetPipelineId}" is at its concurrency limit — queued instead of started`,
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
setImmediate(() => {
|
|
728
|
+
runFlow(record.flowId, "fresh").catch((err) => {
|
|
729
|
+
logger?.warn?.(`manifold invokePipeline(${targetPipelineId}) [${record.flowId.slice(0, 8)}]: runFlow failed: ${err?.message ?? err}`);
|
|
730
|
+
});
|
|
731
|
+
});
|
|
732
|
+
return { invoked: true, target: targetPipelineId, flowId: record.flowId };
|
|
733
|
+
},
|
|
734
|
+
});
|
|
735
|
+
|
|
736
|
+
// resume() is the generic "continue a waiting flow" entry point, called
|
|
737
|
+
// both after a gate decision resolves (gate-resolver.js, routes/
|
|
738
|
+
// operator.js) and by boot-time revival's children-pending check
|
|
739
|
+
// (boot/flow-revival.js) — both are the same underlying mechanism (a
|
|
740
|
+
// node the flow was already sitting at gets re-evaluated), just
|
|
741
|
+
// triggered by different external events, so both resolve to "recheck".
|
|
742
|
+
async function resume(flowId) {
|
|
743
|
+
return runFlow(flowId, "recheck");
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
async function rerun(sourceFlowId, argsOverride) {
|
|
747
|
+
const source = await store.get(sourceFlowId);
|
|
748
|
+
if (!source) throw new Error(`source flow not found: ${sourceFlowId}`);
|
|
749
|
+
return rerunFromSource(source, argsOverride);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
async function rerunFromSource(sourceFlow, argsOverride) {
|
|
753
|
+
if (!sourceFlow) throw new Error(`source flow not found`);
|
|
754
|
+
const sourceFlowId = sourceFlow.flowId;
|
|
755
|
+
const pipelineId = sourceFlow.stateJson?.pipelineId;
|
|
756
|
+
if (!pipelineId) throw new Error(`source flow has no pipelineId in state; cannot determine which pipeline to rerun`);
|
|
757
|
+
const args = argsOverride !== undefined ? argsOverride : (sourceFlow.stateJson?.args ?? {});
|
|
758
|
+
const triggeredBy = {
|
|
759
|
+
kind: "rerun",
|
|
760
|
+
label: `Rerun of ${sourceFlowId.slice(0, 8)}`,
|
|
761
|
+
parentFlowId: sourceFlowId,
|
|
762
|
+
ts: Date.now(),
|
|
763
|
+
};
|
|
764
|
+
const record = await start(pipelineId, args, triggeredBy);
|
|
765
|
+
try {
|
|
766
|
+
const fresh = await store.get(record.flowId);
|
|
767
|
+
if (fresh) {
|
|
768
|
+
await store.setRunning({
|
|
769
|
+
flowId: record.flowId,
|
|
770
|
+
expectedRevision: fresh.revision,
|
|
771
|
+
lastStep: fresh.lastStep,
|
|
772
|
+
stateJson: { ...fresh.stateJson, rerunOf: sourceFlowId },
|
|
773
|
+
});
|
|
774
|
+
}
|
|
775
|
+
} catch (err) {
|
|
776
|
+
logger?.warn?.(`manifold rerun: failed to stamp rerunOf on [${record.flowId.slice(0, 8)}]: ${err?.message ?? err}`);
|
|
777
|
+
}
|
|
778
|
+
return record;
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
async function cancel(flowId, reason = null) {
|
|
782
|
+
const flow = await store.get(flowId);
|
|
783
|
+
if (!flow) return { cancelled: false, reason: "Flow not found." };
|
|
784
|
+
// completeCancellation stamps in-flight branches with cancelledAt, kills any
|
|
785
|
+
// active LLM sessions, then writes status='cancelled' + stamped stateJson to DB.
|
|
786
|
+
return completeCancellation({
|
|
787
|
+
store,
|
|
788
|
+
flowId,
|
|
789
|
+
stateJson: flow.stateJson ?? {},
|
|
790
|
+
lastStep: flow.lastStep,
|
|
791
|
+
revision: flow.revision,
|
|
792
|
+
cfg: _config,
|
|
793
|
+
logger,
|
|
794
|
+
runStore,
|
|
795
|
+
nodeExecutionsStore: nodeExecutions,
|
|
796
|
+
reason,
|
|
797
|
+
});
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
/**
|
|
801
|
+
* Revive an "agent"-kind orphan left mid-flight by a crashed process.
|
|
802
|
+
*
|
|
803
|
+
* Unlike a code-kind orphan (safe to just re-enter via runFlow — the
|
|
804
|
+
* node hadn't produced any side effect yet), an agent-kind node may have
|
|
805
|
+
* already dispatched an agent turn that completed (or partially acted)
|
|
806
|
+
* before the crash. Re-running node.promptBuilder(state) fresh would give
|
|
807
|
+
* the agent no way to know it already tried this once, risking a
|
|
808
|
+
* duplicate side effect. Instead this sends REVIVAL_MSG to the SAME
|
|
809
|
+
* deterministic session key the original dispatch used, so the agent can
|
|
810
|
+
* check its own prior turns and decide whether to finish or resume.
|
|
811
|
+
*/
|
|
812
|
+
// nodeId is the crashed/in-flight node — the caller (boot/flow-revival.js)
|
|
813
|
+
// resolves it from f.currentStep/waitJson.nodeId, since flow.lastStep is
|
|
814
|
+
// the last COMPLETED node, not this one. Falls back to flow.lastStep only
|
|
815
|
+
// for callers that don't pass it explicitly (defensive, not the expected path).
|
|
816
|
+
async function reviveAgentKindOrphan(flow, nodeId = flow.lastStep) {
|
|
817
|
+
const flowId = flow.flowId;
|
|
818
|
+
const flowId8 = flowId.slice(0, 8);
|
|
819
|
+
const state = flow.stateJson ?? {};
|
|
820
|
+
const pipeline = pipelinesRef.value.get(state.pipelineId);
|
|
821
|
+
const node = pipeline?.nodes?.[nodeId];
|
|
822
|
+
if (!node || node.kind !== "agent") {
|
|
823
|
+
logger.warn(`manifold revival [${flowId8}]: reviveAgentKindOrphan called for non-agent node "${nodeId}"`);
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
logger.info(`manifold revival [${flowId8}]: reviving interrupted agent-node "${nodeId}"`);
|
|
828
|
+
const enteredAt = Date.now();
|
|
829
|
+
// Resolved up front (not after executeNode returns) so a tool call made
|
|
830
|
+
// during this revived dispatch can register itself under the SAME
|
|
831
|
+
// invocationId the original, interrupted attempt already created — see
|
|
832
|
+
// tool-handoff.js. No fresh nodeExecutions.enter() call here; the
|
|
833
|
+
// invocation row already exists from the original attempt.
|
|
834
|
+
const invocation = await nodeExecutions.getLatestInvocationForNode(flowId, nodeId, null);
|
|
835
|
+
const invocationId = invocation?.id ?? null;
|
|
836
|
+
let result;
|
|
837
|
+
try {
|
|
838
|
+
result = await executeNode({ node, nodeId, state, flowId, reviveMessage: REVIVAL_MSG, invocationId });
|
|
839
|
+
} catch (err) {
|
|
840
|
+
if (err instanceof GatewayHandoffNeeded) {
|
|
841
|
+
// A request-scope gap in the revived dispatch: block rather than
|
|
842
|
+
// failing the flow outright. No automatic resume exists for this
|
|
843
|
+
// wait kind (see executor-errors.js's GatewayHandoffNeeded
|
|
844
|
+
// docstring) — recovery
|
|
845
|
+
// is either a scope-gap fix at the actual call site or manual
|
|
846
|
+
// task-revival.
|
|
847
|
+
await persistWaitingRetrying({
|
|
848
|
+
flowId, revision: flow.revision, lastStep: nodeId, blockingNodeId: nodeId,
|
|
849
|
+
state,
|
|
850
|
+
waitJson: {
|
|
851
|
+
kind: "needs-gateway-handoff", nodeId, reason: err.message, needed: [],
|
|
852
|
+
hint: "No automatic resume — requires a scope-gap fix at the call site or manual task-revival.",
|
|
853
|
+
renderedAt: Date.now(), retry: true,
|
|
854
|
+
},
|
|
855
|
+
statusReason: shortSummary(`${nodeId}: needs gateway context: ${err.message}`),
|
|
856
|
+
});
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
const msg = err?.message ?? String(err);
|
|
860
|
+
// Duck-typed — see the main loop's identical check above for why
|
|
861
|
+
// `instanceof NodeFailure` isn't reliable across this codebase's
|
|
862
|
+
// module-loading boundaries.
|
|
863
|
+
const errorType = err?.errorType ?? null;
|
|
864
|
+
logger.warn(`manifold revival [${flowId8}]: agent-node "${nodeId}" revival failed: ${msg}`);
|
|
865
|
+
// Revival always continues the exact node that crashed — there's no
|
|
866
|
+
// fresh dispatch decision here (unlike runFlow's loop,
|
|
867
|
+
// which resolves an edge to decide nextNodeId), so kind is hardcoded
|
|
868
|
+
// "revival" rather than derived from isFreshNodeEntry.
|
|
869
|
+
const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId, enteredAt, exitedAt: Date.now(), kind: "revival" });
|
|
870
|
+
await persistFailRetrying({
|
|
871
|
+
flowId, revision: flow.revision, state,
|
|
872
|
+
message: `${nodeId}: revival failed: ${msg}`, failureNode: nodeId,
|
|
873
|
+
invocationId: written.invocationId ?? null, executionId: written.executionId ?? null, errorType,
|
|
874
|
+
});
|
|
875
|
+
return;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
const { stateUpdate = {}, blockedInfo, skippedInfo } = result;
|
|
879
|
+
|
|
880
|
+
if (blockedInfo) {
|
|
881
|
+
if (!blockedInfo._kind) {
|
|
882
|
+
const msg = blockedInfo.reason ?? `${nodeId} blocked without declaring a wait kind`;
|
|
883
|
+
logger.warn(`manifold revival [${flowId8}]: agent-node "${nodeId}" ${msg}`);
|
|
884
|
+
// Same reasoning as above — revival is always a continuation.
|
|
885
|
+
const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId, enteredAt, exitedAt: Date.now(), kind: "revival" });
|
|
886
|
+
await persistFailRetrying({
|
|
887
|
+
flowId, revision: flow.revision, state,
|
|
888
|
+
message: `${nodeId}: ${msg}`, failureNode: nodeId,
|
|
889
|
+
invocationId: written.invocationId ?? null, executionId: written.executionId ?? null,
|
|
890
|
+
});
|
|
891
|
+
return;
|
|
892
|
+
}
|
|
893
|
+
const waitJsonForBlock = {
|
|
894
|
+
kind: blockedInfo._kind, nodeId,
|
|
895
|
+
reason: blockedInfo.reason ?? "node blocked", needed: blockedInfo.needed ?? [],
|
|
896
|
+
hint: blockedInfo.hint ?? "", renderedAt: Date.now(), retry: true,
|
|
897
|
+
};
|
|
898
|
+
await persistWaitingRetrying({
|
|
899
|
+
flowId, revision: flow.revision, lastStep: nodeId, blockingNodeId: nodeId,
|
|
900
|
+
state,
|
|
901
|
+
waitJson: waitJsonForBlock,
|
|
902
|
+
statusReason: shortSummary(`${nodeId}: ${waitJsonForBlock.reason}`),
|
|
903
|
+
});
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
if (skippedInfo) {
|
|
908
|
+
await persistSkipRetrying({
|
|
909
|
+
flowId, revision: flow.revision,
|
|
910
|
+
state,
|
|
911
|
+
reason: skippedInfo.reason ?? "skipped",
|
|
912
|
+
});
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
// No wide stateUpdate merge before persisting — `state` here is
|
|
917
|
+
// already entry-only (freshly read via store.get() upstream), and
|
|
918
|
+
// stateUpdate's own node-output content reaches
|
|
919
|
+
// result_json separately via writeResult(), not through this. See the
|
|
920
|
+
// main dispatch loop's identical cleanup above for the full reasoning.
|
|
921
|
+
let entryState = state;
|
|
922
|
+
// Same staleness clear as the main loop's success path above — an
|
|
923
|
+
// agent-kind cascade leg (e.g. SUBMIT_AGENT) revived and now succeeding
|
|
924
|
+
// shouldn't carry a leftover _cascadeError from its earlier crashed
|
|
925
|
+
// attempt into its own next edge-function call.
|
|
926
|
+
if (entryState._cascadeError?.nodeId === nodeId) {
|
|
927
|
+
const { _cascadeError, ...rest } = entryState;
|
|
928
|
+
entryState = rest;
|
|
929
|
+
}
|
|
930
|
+
// The main loop writes a node's result BEFORE advancing last_step, so a
|
|
931
|
+
// crash-then-revival never reads a stale/empty result for the node it
|
|
932
|
+
// just resumed. This revival path resumes the SAME crashed dispatch (no
|
|
933
|
+
// fresh nodeExecutions.enter() call here — the invocation row already
|
|
934
|
+
// exists from the original, interrupted attempt) and carries no
|
|
935
|
+
// writeResult() call of its own, so it writes the result here: without
|
|
936
|
+
// it, last_step has already advanced to `nodeId` while its result_json
|
|
937
|
+
// stayed empty from the interrupted attempt. The very next runFlow() iteration's
|
|
938
|
+
// assembleState() then read nothing for this node, and resolveEdge()
|
|
939
|
+
// dead-ended with "no next node" — deterministic on every revived
|
|
940
|
+
// agent-node whose outbound edge depends on its own output, not a race.
|
|
941
|
+
const resultWrite = await nodeExecutions.writeResult(invocationId, stateUpdate).catch(() => null);
|
|
942
|
+
if (!resultWrite || !resultWrite.ok) {
|
|
943
|
+
const reason = resultWrite?.reason ?? "threw";
|
|
944
|
+
// "race" means a CONCURRENT writer already committed real content for
|
|
945
|
+
// this exact invocation — advancing is correct, the data genuinely is
|
|
946
|
+
// there, just not from this call. Every other reason (db-error,
|
|
947
|
+
// no-such-invocation, no-invocation-id, unserializable, threw) means
|
|
948
|
+
// NO ONE wrote anything: advancing here would move last_step past
|
|
949
|
+
// this node with its output never durably written. The next runFlow()
|
|
950
|
+
// iteration's assembleState() then reads nothing for this node, and
|
|
951
|
+
// resolveEdge() dead-ends with "no next node" — deterministic given
|
|
952
|
+
// the failure, not a race, and completely swallowing the REAL reason
|
|
953
|
+
// behind a generic edge-resolution message. Fail the flow outright
|
|
954
|
+
// instead — same shape as persistRunningRetrying's own CAS-failure
|
|
955
|
+
// branch just below — so the operator sees the true cause and can
|
|
956
|
+
// retry this one node, rather than a downstream "no next node" that
|
|
957
|
+
// hides it.
|
|
958
|
+
if (reason !== "race") {
|
|
959
|
+
const msg = `resultJson write failed (${reason})`;
|
|
960
|
+
logger.warn(`manifold revival [${flowId8}]: agent-node "${nodeId}" ${msg}`);
|
|
961
|
+
const written = await safeWriteExecution({ flowId, pipelineId: state.pipelineId ?? null, nodeId, enteredAt, exitedAt: Date.now(), kind: "revival" });
|
|
962
|
+
await persistFailRetrying({
|
|
963
|
+
flowId, revision: flow.revision, state,
|
|
964
|
+
message: `${nodeId}: ${msg}`, failureNode: nodeId,
|
|
965
|
+
invocationId: written.invocationId ?? invocationId, executionId: written.executionId ?? null,
|
|
966
|
+
});
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
logger.warn(`manifold revival [${flowId8}]: agent-node "${nodeId}" resultJson write failed (race) — another writer already completed this invocation`);
|
|
970
|
+
}
|
|
971
|
+
await persistRunningRetrying({ flowId, revision: flow.revision, lastStep: nodeId, state: entryState });
|
|
972
|
+
logger.info(`manifold revival [${flowId8}]: agent-node "${nodeId}" revived — continuing`);
|
|
973
|
+
return runFlow(flowId, "revival");
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
return { start, runFlow, resume, cancel, rerun, rerunFromSource, reviveAgentKindOrphan };
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
// ── audit ─────────────────────────────────────────────────────────────────────
|
|
980
|
+
|
|
981
|
+
async function runAuditIfConfigured({ flowId, pipeline, state, logger }) {
|
|
982
|
+
const audit = pipeline?.runAudit;
|
|
983
|
+
if (!audit) return;
|
|
984
|
+
if (typeof audit.runsDir !== "string" || typeof audit.summarize !== "function") {
|
|
985
|
+
logger?.warn?.(`manifold [${flowId.slice(0, 8)}] runAudit malformed (need runsDir:string + summarize:function); skipping`);
|
|
986
|
+
return;
|
|
987
|
+
}
|
|
988
|
+
const runId = state?.runId;
|
|
989
|
+
if (!runId) {
|
|
990
|
+
logger?.warn?.(`manifold [${flowId.slice(0, 8)}] runAudit: state.runId missing; skipping`);
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
993
|
+
try {
|
|
994
|
+
const runDir = path.join(audit.runsDir, runId);
|
|
995
|
+
await mkdir(runDir, { recursive: true });
|
|
996
|
+
await writeFile(path.join(runDir, "final-state.json"), JSON.stringify(state, null, 2), "utf-8");
|
|
997
|
+
let summary;
|
|
998
|
+
try { summary = await Promise.resolve(audit.summarize(state)); }
|
|
999
|
+
catch (err) {
|
|
1000
|
+
logger?.warn?.(`manifold [${flowId.slice(0, 8)}] runAudit.summarize threw: ${err?.message ?? err}`);
|
|
1001
|
+
return;
|
|
1002
|
+
}
|
|
1003
|
+
if (summary && typeof summary === "object") {
|
|
1004
|
+
await writeFile(
|
|
1005
|
+
path.join(audit.runsDir, "log.jsonl"),
|
|
1006
|
+
JSON.stringify(summary) + "\n",
|
|
1007
|
+
{ flag: "a", encoding: "utf-8" },
|
|
1008
|
+
);
|
|
1009
|
+
}
|
|
1010
|
+
} catch (err) {
|
|
1011
|
+
logger?.warn?.(`manifold [${flowId.slice(0, 8)}] runAudit failed: ${err?.message ?? err}`);
|
|
1012
|
+
}
|
|
1013
|
+
}
|