@igorjpimenta/manifold 0.0.0-stage → 2026.11.22-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.migrations/ops/0000_futuristic_karma.sql +111 -0
- package/.migrations/ops/0001_low_maximus.sql +7 -0
- package/.migrations/ops/meta/0000_snapshot.json +763 -0
- package/.migrations/ops/meta/0001_snapshot.json +173 -0
- package/.migrations/ops/meta/_journal.json +20 -0
- package/.migrations/runtime/0000_real_boomer.sql +180 -0
- package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
- package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
- package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
- package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
- package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
- package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
- package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
- package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
- package/.migrations/runtime/0009_lying_shocker.sql +96 -0
- package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
- package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
- package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
- package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
- package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
- package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
- package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
- package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
- package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
- package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
- package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
- package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
- package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
- package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
- package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
- package/.migrations/runtime/meta/_journal.json +97 -0
- package/README.md +1583 -2
- package/RELEASING.md +89 -0
- package/agent/fleet.js +100 -0
- package/bin/cli.js +854 -0
- package/bin/manifold.js +4 -0
- package/boot/flow-revival.js +536 -0
- package/budget/index.js +495 -0
- package/budget/lib.js +59 -0
- package/dashboard/cost.js +528 -0
- package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
- package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
- package/dashboard/dist/index.html +13 -0
- package/dashboard/event-log.js +267 -0
- package/dashboard/index.html +12 -0
- package/dashboard/package-lock.json +2877 -0
- package/dashboard/package.json +37 -0
- package/dashboard/pipeline-graph.js +131 -0
- package/dashboard/postcss.config.js +6 -0
- package/dashboard/search.js +183 -0
- package/dashboard/server.js +1015 -0
- package/dashboard/src/App.tsx +42 -0
- package/dashboard/src/api/client.ts +118 -0
- package/dashboard/src/api/types.ts +510 -0
- package/dashboard/src/api/ws.tsx +112 -0
- package/dashboard/src/components/CancelButton.tsx +63 -0
- package/dashboard/src/components/ClickToCopy.tsx +68 -0
- package/dashboard/src/components/CopyButton.tsx +61 -0
- package/dashboard/src/components/DailyCostTrend.tsx +85 -0
- package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
- package/dashboard/src/components/Funnel.tsx +149 -0
- package/dashboard/src/components/GateForm.tsx +158 -0
- package/dashboard/src/components/GraphView.tsx +302 -0
- package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
- package/dashboard/src/components/Histogram.tsx +292 -0
- package/dashboard/src/components/Layout.tsx +28 -0
- package/dashboard/src/components/NeedsAttention.tsx +98 -0
- package/dashboard/src/components/NodeContextCard.tsx +132 -0
- package/dashboard/src/components/PathTrail.tsx +35 -0
- package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
- package/dashboard/src/components/RerunButton.tsx +46 -0
- package/dashboard/src/components/RunComparison.tsx +53 -0
- package/dashboard/src/components/Sparkline.tsx +94 -0
- package/dashboard/src/components/StatStrip.tsx +147 -0
- package/dashboard/src/components/StateTree.tsx +84 -0
- package/dashboard/src/components/StatusBadge.tsx +18 -0
- package/dashboard/src/components/SubflowPathPills.tsx +38 -0
- package/dashboard/src/components/SubflowSummary.tsx +24 -0
- package/dashboard/src/components/SubflowsCard.tsx +327 -0
- package/dashboard/src/components/TodayStrip.tsx +86 -0
- package/dashboard/src/components/TranscriptPane.tsx +99 -0
- package/dashboard/src/components/TranscriptRow.tsx +69 -0
- package/dashboard/src/components/ui/badge.tsx +38 -0
- package/dashboard/src/components/ui/button.tsx +28 -0
- package/dashboard/src/components/ui/card.tsx +42 -0
- package/dashboard/src/components/ui/skeleton.tsx +9 -0
- package/dashboard/src/components/ui/table.tsx +62 -0
- package/dashboard/src/index.css +26 -0
- package/dashboard/src/lib/anomaly.ts +42 -0
- package/dashboard/src/lib/format.ts +30 -0
- package/dashboard/src/lib/step-history.ts +13 -0
- package/dashboard/src/lib/subflow.ts +154 -0
- package/dashboard/src/lib/use-click-outside.ts +28 -0
- package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
- package/dashboard/src/lib/utils.ts +6 -0
- package/dashboard/src/main.tsx +10 -0
- package/dashboard/src/types/shims.d.ts +4 -0
- package/dashboard/src/views/Home.tsx +478 -0
- package/dashboard/src/views/PipelineDetail.tsx +1185 -0
- package/dashboard/src/views/PipelinesList.tsx +111 -0
- package/dashboard/src/views/RunDetail.tsx +248 -0
- package/dashboard/src/views/Search.tsx +196 -0
- package/dashboard/stats.js +668 -0
- package/dashboard/tailwind.config.ts +43 -0
- package/dashboard/tree-stats.js +12 -0
- package/dashboard/tsconfig.json +25 -0
- package/dashboard/views.js +231 -0
- package/dashboard/vite.config.ts +34 -0
- package/drizzle.config.js +12 -0
- package/drizzle.config.ops.js +12 -0
- package/elevation.js +116 -0
- package/engine/admission-promotion-resume.js +127 -0
- package/engine/agent-dispatch.js +64 -0
- package/engine/child-resume-trigger.js +315 -0
- package/engine/executor-budget.js +87 -0
- package/engine/executor-cancel.js +83 -0
- package/engine/executor-errors.js +122 -0
- package/engine/executor-helpers.js +254 -0
- package/engine/executor-nodes.js +292 -0
- package/engine/executor-persist.js +212 -0
- package/engine/executor-subflow-children.js +275 -0
- package/engine/executor.js +1013 -0
- package/engine/global-concurrency.js +33 -0
- package/engine/loader.js +528 -0
- package/engine/needs-resolver.js +229 -0
- package/engine/pipelines-watcher.js +133 -0
- package/engine/result-replay.js +34 -0
- package/engine/revival-messages.js +11 -0
- package/engine/spawn-child.js +55 -0
- package/engine/wait-for-flow-settlement.js +109 -0
- package/events/catalog.js +26 -0
- package/events/emit.js +44 -0
- package/events/kinds.js +118 -0
- package/events/serve-ws.js +74 -0
- package/events/stream.js +35 -0
- package/events/ws-dialog.js +257 -0
- package/events/ws-frame.js +194 -0
- package/events/ws-run-events.js +70 -0
- package/events.js +32 -0
- package/gates/gate-resolver.js +576 -0
- package/gates/paused-flows.js +55 -0
- package/graphql/columns.js +67 -0
- package/graphql/schema.js +333 -0
- package/index.js +49 -0
- package/mcp/http.js +159 -0
- package/mcp/identity.js +94 -0
- package/mcp/stdio.js +80 -0
- package/mcp/tools.js +245 -0
- package/mcp/wire-ops.js +178 -0
- package/package.json +49 -4
- package/routes/bearer-auth.js +57 -0
- package/routes/discuss-audio.js +87 -0
- package/routes/discuss-component.js +39 -0
- package/routes/discuss-draft.js +159 -0
- package/routes/discuss-helpers.js +20 -0
- package/routes/discuss.js +848 -0
- package/routes/failures.js +104 -0
- package/routes/graphql.js +61 -0
- package/routes/health.js +54 -0
- package/routes/operator-discuss.js +193 -0
- package/routes/operator-resume.js +32 -0
- package/routes/operator-retry.js +301 -0
- package/routes/operator.js +539 -0
- package/routes/pipelines.js +118 -0
- package/routes/runs.js +1129 -0
- package/scripts/ci/import-check.mjs +50 -0
- package/scripts/link-core-self.mjs +31 -0
- package/scripts/with-guard.sh +24 -0
- package/server/app.js +503 -0
- package/server/registry.js +53 -0
- package/server/serve.js +163 -0
- package/state/admission-queue-store.js +325 -0
- package/state/attribution-registries.js +102 -0
- package/state/batch-writer.js +162 -0
- package/state/db-guard.js +63 -0
- package/state/db-health.js +27 -0
- package/state/db.js +122 -0
- package/state/discuss-store.js +299 -0
- package/state/failure-store.js +427 -0
- package/state/migrate.js +104 -0
- package/state/node-cost.js +67 -0
- package/state/node-executions.js +775 -0
- package/state/node-health-store.js +124 -0
- package/state/ops-merge.js +93 -0
- package/state/paths.js +146 -0
- package/state/pipeline-dirs.js +99 -0
- package/state/pipeline-registry.js +449 -0
- package/state/run-store.js +697 -0
- package/state/schema.js +438 -0
- package/state/schema.ops.js +44 -0
- package/state/session-index.js +177 -0
- package/state/session-keys.js +39 -0
- package/state/state-store.js +311 -0
- package/state/subflow-state.js +126 -0
- package/state/tool-handoff.js +169 -0
- package/telemetry/budget-alert.js +353 -0
- package/telemetry/cli.js +331 -0
- package/telemetry/grammar.js +119 -0
- package/telemetry/http-budget.js +132 -0
- package/telemetry/http-cost.js +670 -0
- package/telemetry/http-helpers.js +134 -0
- package/telemetry/http-tree.js +349 -0
- package/telemetry/http.js +44 -0
- package/telemetry/index.js +307 -0
- package/telemetry/ingest.js +134 -0
- package/telemetry/labels.js +115 -0
- package/telemetry/pricer.js +77 -0
- package/telemetry/resolver.js +314 -0
- package/telemetry/store.js +147 -0
- package/utils/run-patch.js +43 -0
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
import { getNodeExecutionsStore } from "../state/node-executions.js";
|
|
2
|
+
import { admitAndSetRunning } from "../engine/child-resume-trigger.js";
|
|
3
|
+
|
|
4
|
+
// The four statuses casSetRunningStmt's CAS excludes (run-store.js) — a flow
|
|
5
|
+
// sitting in one of these needs store.forceBlock before setRunning will
|
|
6
|
+
// accept it. "blocked" and "running" are already fine as-is.
|
|
7
|
+
const CAS_EXCLUDED_STATUSES = new Set(["succeeded", "failed", "cancelled", "skipped"]);
|
|
8
|
+
|
|
9
|
+
// Force a terminal flow (succeeded/failed/cancelled/skipped) into `blocked`
|
|
10
|
+
// so setRunning's CAS will accept it (its WHERE clause excludes those four
|
|
11
|
+
// statuses — see run-store.js's casSetRunningStmt). Goes through the
|
|
12
|
+
// injected `store` (state-store.js → run-store.js's forceBlock CAS) rather
|
|
13
|
+
// than a raw DatabaseSync write — a prior version of this function opened
|
|
14
|
+
// its own `new DatabaseSync(runtime.sqlite path)` here, bypassing `store`/
|
|
15
|
+
// `readStore` entirely, so tests injecting in-memory store doubles could
|
|
16
|
+
// neither observe nor control it (the write landed against a real DB file
|
|
17
|
+
// the doubles knew nothing about, always returning "revision changed
|
|
18
|
+
// during force-block" in tests).
|
|
19
|
+
// Exported so other retry paths can reuse this instead of hitting the same
|
|
20
|
+
// `setRunning failed: terminal` result and having no recovery — a subflow
|
|
21
|
+
// item can finish failed on a flow that has since gone fully terminal (the
|
|
22
|
+
// OTHER items in the same subflow resolved and closed the flow out from under
|
|
23
|
+
// it).
|
|
24
|
+
// Returns { applied: true, nextRevision } or { applied: false, reason }.
|
|
25
|
+
export async function forceBlockOutOfMapFlow({ store, flowId, revision, stateJson, ownerSessionKey }) {
|
|
26
|
+
const result = await store.forceBlock({
|
|
27
|
+
flowId,
|
|
28
|
+
expectedRevision: revision,
|
|
29
|
+
stateJson,
|
|
30
|
+
statusReason: "retry-pending",
|
|
31
|
+
ownerKey: ownerSessionKey,
|
|
32
|
+
});
|
|
33
|
+
if (!result.applied) return { applied: false, reason: `force-block failed: ${result.code ?? "unknown"}` };
|
|
34
|
+
return { applied: true, nextRevision: revision + 1 };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Walks down through a subflow node's spawned children to find the real,
|
|
38
|
+
// non-propagated failure(s) beneath a "subflow-mode-not-satisfied" row
|
|
39
|
+
// (recursing through any further nested propagations), rather than
|
|
40
|
+
// retrying the propagation's own flow — which already dispatched cleanly
|
|
41
|
+
// and would just immediately re-fail with the identical reason. Collects
|
|
42
|
+
// EVERY real failure found, not just the first: under mode:"all" with more
|
|
43
|
+
// than one failed child, fixing only one still leaves the fan-out
|
|
44
|
+
// unsatisfied. `visited` guards a pathological cycle; real topologies are
|
|
45
|
+
// trees, this is defense only.
|
|
46
|
+
async function findRealFailuresBeneath({ flowId, nodeId, runStore, nodeExecutionsStore, failureStore, visited }) {
|
|
47
|
+
const key = `${flowId}:${nodeId}`;
|
|
48
|
+
if (visited.has(key)) return [];
|
|
49
|
+
visited.add(key);
|
|
50
|
+
const itemInvocationIds = await nodeExecutionsStore.listInvocationIdsForNode(flowId, nodeId);
|
|
51
|
+
if (!itemInvocationIds.length) return [];
|
|
52
|
+
const children = await runStore.listForParentInvocations(itemInvocationIds);
|
|
53
|
+
const found = [];
|
|
54
|
+
for (const child of children) {
|
|
55
|
+
if (child.status === "succeeded") continue;
|
|
56
|
+
// Legitimately still pending (e.g. blocked on its own human-gate) has
|
|
57
|
+
// no failure_events row at all — correctly nothing to retry here yet.
|
|
58
|
+
const [childFailure] = await failureStore.list({ flowId: child.flowId, limit: 1 });
|
|
59
|
+
if (!childFailure) continue;
|
|
60
|
+
if (childFailure.error_type === "subflow-mode-not-satisfied") {
|
|
61
|
+
found.push(...await findRealFailuresBeneath({
|
|
62
|
+
flowId: child.flowId, nodeId: childFailure.failure_node, runStore, nodeExecutionsStore, failureStore, visited,
|
|
63
|
+
}));
|
|
64
|
+
} else {
|
|
65
|
+
found.push(childFailure);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return found;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function createRetryFailure({
|
|
72
|
+
store,
|
|
73
|
+
readStore,
|
|
74
|
+
failureStore,
|
|
75
|
+
pipelinesRef,
|
|
76
|
+
executor,
|
|
77
|
+
runStore,
|
|
78
|
+
ownerSessionKey,
|
|
79
|
+
admissionQueueStore,
|
|
80
|
+
logger,
|
|
81
|
+
api,
|
|
82
|
+
}) {
|
|
83
|
+
async function retryFailureBranch({ id, via, awaitCompletion = false }) {
|
|
84
|
+
const failure = await failureStore.get(id);
|
|
85
|
+
if (!failure) return { code: "not_found", reason: "failure event not found" };
|
|
86
|
+
const flowId = failure.flow_id;
|
|
87
|
+
const failureNode = failure.failure_node;
|
|
88
|
+
|
|
89
|
+
// Redirect before falling into the normal same-flow retry below. This
|
|
90
|
+
// flow's own dispatch (SUBMIT/PROPOSE/etc.) already succeeded; retrying
|
|
91
|
+
// IT re-runs a clean dispatch, finds the same still-failed/still-blocked
|
|
92
|
+
// children, and (for the still-failed ones) immediately re-fails with
|
|
93
|
+
// the identical reason within the same tick.
|
|
94
|
+
if (failure.error_type === "subflow-mode-not-satisfied") {
|
|
95
|
+
const realFailures = await findRealFailuresBeneath({
|
|
96
|
+
flowId, nodeId: failureNode, runStore, nodeExecutionsStore: getNodeExecutionsStore(), failureStore, visited: new Set(),
|
|
97
|
+
});
|
|
98
|
+
if (realFailures.length === 0) {
|
|
99
|
+
return { code: "conflict", reason: `no real failure found beneath "${failureNode}" — may already be resolving` };
|
|
100
|
+
}
|
|
101
|
+
const results = [];
|
|
102
|
+
for (const real of realFailures) {
|
|
103
|
+
results.push(await retryFailureBranch({ id: real.id, via, awaitCompletion }));
|
|
104
|
+
}
|
|
105
|
+
const anyOk = results.some((r) => r.code === "ok");
|
|
106
|
+
return {
|
|
107
|
+
code: anyOk ? "ok" : "conflict",
|
|
108
|
+
reason: anyOk ? undefined : "none of the underlying failures could be retried",
|
|
109
|
+
mode: "cascade-redirect",
|
|
110
|
+
redirectedTo: realFailures.map((f) => f.id),
|
|
111
|
+
results,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Every failure — whether at flow's own top level or inside a
|
|
116
|
+
// childFlows:true subflow item — retries the same way: failure.flow_id
|
|
117
|
+
// already names the exact flow to walk and re-run. A subflow item runs
|
|
118
|
+
// as a real child flow with its own flow_id (executor-subflow-children.js),
|
|
119
|
+
// so branch_item_id (sourced from flow_node_invocations.subflow_key, the
|
|
120
|
+
// per-item dispatch key) is purely descriptive here — it doesn't
|
|
121
|
+
// change which flow gets retried.
|
|
122
|
+
{
|
|
123
|
+
// ── Flow-level retry ─────────────────────────────────────────────────────
|
|
124
|
+
const inMapFlow = await store.get(flowId);
|
|
125
|
+
const flow = inMapFlow ?? await readStore.get(flowId);
|
|
126
|
+
if (!flow) return { code: "not_found", reason: "flow gone from registry" };
|
|
127
|
+
const state = flow.stateJson ?? {};
|
|
128
|
+
const pipeline = pipelinesRef.value.get(state.pipelineId);
|
|
129
|
+
if (!pipeline) return { code: "not_found", reason: `pipeline not loaded: ${state.pipelineId}` };
|
|
130
|
+
const FLOW_TERMINAL = new Set(["succeeded", "failed", "cancelled", "blocked", "skipped"]);
|
|
131
|
+
if (!FLOW_TERMINAL.has(flow.status)) {
|
|
132
|
+
return { code: "conflict", reason: `flow is ${flow.status}; refusing to race executor` };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// A failure with no failure_node happened before any node on this flow
|
|
136
|
+
// ever dispatched (failure_node is derived from invocation_id, which is
|
|
137
|
+
// only ever set once a flow_node_invocations row exists — e.g. the
|
|
138
|
+
// flow's own pipeline failed to resolve). There's nothing to partially
|
|
139
|
+
// clear in that case, so the only sane retry target is the whole flow
|
|
140
|
+
// from __start__ — same lastStep a brand-new flow gets from
|
|
141
|
+
// state-store.js's create().
|
|
142
|
+
if (!failureNode) {
|
|
143
|
+
const nextState = JSON.parse(JSON.stringify(state));
|
|
144
|
+
nextState._attemptNumber = (state._attemptNumber ?? 0) + 1;
|
|
145
|
+
let effectiveRevision = flow.revision;
|
|
146
|
+
if (CAS_EXCLUDED_STATUSES.has(flow.status)) {
|
|
147
|
+
const forced = await forceBlockOutOfMapFlow({
|
|
148
|
+
store, flowId, revision: flow.revision, stateJson: nextState, ownerSessionKey,
|
|
149
|
+
});
|
|
150
|
+
if (!forced.applied) return { code: "conflict", reason: forced.reason };
|
|
151
|
+
effectiveRevision = forced.nextRevision;
|
|
152
|
+
}
|
|
153
|
+
const result = await admitAndSetRunning({
|
|
154
|
+
flowId, expectedRevision: effectiveRevision, lastStep: "__start__", stateJson: nextState,
|
|
155
|
+
parentInvocationId: flow.parentInvocationId, store, admissionQueueStore,
|
|
156
|
+
runStore, resume: executor.resume, logger, api,
|
|
157
|
+
});
|
|
158
|
+
if (!result.applied) {
|
|
159
|
+
return { code: "conflict", reason: `setRunning failed: ${result.reason ?? result.code ?? "unknown"}` };
|
|
160
|
+
}
|
|
161
|
+
if (result.queued) {
|
|
162
|
+
logger.info(
|
|
163
|
+
`manifold retry-flow: flowId=${flowId.slice(0, 8)} failureNode=<none> rootNode=__start__ via=${via} queued for admission`
|
|
164
|
+
);
|
|
165
|
+
return { code: "queued", flowId, failureNode: null, rootNode: "__start__", priorStep: null, cleared: [], mode: "flow-restart", via };
|
|
166
|
+
}
|
|
167
|
+
const runRetryFlow = () => executor.runFlow(flowId, "retry").catch((err) => {
|
|
168
|
+
logger.warn(`manifold retry-flow [${flowId.slice(0, 8)}/__start__] runFlow err: ${err?.message ?? err}`);
|
|
169
|
+
});
|
|
170
|
+
if (awaitCompletion) await runRetryFlow();
|
|
171
|
+
else setImmediate(runRetryFlow);
|
|
172
|
+
logger.info(
|
|
173
|
+
`manifold retry-flow: flowId=${flowId.slice(0, 8)} failureNode=<none> rootNode=__start__ via=${via}`
|
|
174
|
+
);
|
|
175
|
+
return { code: "ok", flowId, failureNode: null, rootNode: "__start__", priorStep: null, cleared: [], mode: "flow-restart", via };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Walk topology to resolve rootNode
|
|
179
|
+
const topo = [];
|
|
180
|
+
const visited = new Set();
|
|
181
|
+
let cur = "__start__";
|
|
182
|
+
let safety = 256;
|
|
183
|
+
while (cur && cur !== "__end__" && !visited.has(cur) && safety-- > 0) {
|
|
184
|
+
visited.add(cur);
|
|
185
|
+
topo.push(cur);
|
|
186
|
+
const e = pipeline.edges?.[cur];
|
|
187
|
+
try { cur = typeof e === "function" ? e(state) : (typeof e === "string" ? e : null); }
|
|
188
|
+
catch { cur = null; }
|
|
189
|
+
}
|
|
190
|
+
// subflow_key IS NULL is the structural filter for top-level errors —
|
|
191
|
+
// a real column, not an inferred string shape.
|
|
192
|
+
//
|
|
193
|
+
// queryErrorsByFlow returns EVERY historical errored execution, not
|
|
194
|
+
// just the latest attempt per node — a node that failed once, was
|
|
195
|
+
// retried, and now succeeds still has its old failed row in this
|
|
196
|
+
// list. A raw nodeId Set can't tell "still failing" apart from
|
|
197
|
+
// "failed once, historically." Narrow to nodes whose CURRENT (most
|
|
198
|
+
// recent) execution is the one that errored — getLatestExecutionForNode
|
|
199
|
+
// (added for failure_events invocation-linking) makes this exact
|
|
200
|
+
// instead of inferred from string/nodeId equality alone.
|
|
201
|
+
//
|
|
202
|
+
// queryErrorsByFlow moved from nodeExecutionsStore to failureStore
|
|
203
|
+
// ("Eradicate soft errors") — flow_node_executions no longer stores
|
|
204
|
+
// error content at all, failure_events is the only place "which
|
|
205
|
+
// nodes errored" can be answered from now. getLatestExecutionForNode
|
|
206
|
+
// stays on nodeExecutionsStore below — it's identity/timing only,
|
|
207
|
+
// unaffected.
|
|
208
|
+
const nodeExecutionsStore = getNodeExecutionsStore();
|
|
209
|
+
const allRootErrors = (await failureStore.queryErrorsByFlow(flowId)).filter((e) => e.subflowKey == null);
|
|
210
|
+
const errorNodes = new Set();
|
|
211
|
+
for (const e of allRootErrors) {
|
|
212
|
+
const latest = await nodeExecutionsStore.getLatestExecutionForNode({ flowId, nodeId: e.nodeId, subflowKey: null });
|
|
213
|
+
if (latest?.executionId === e.executionId) errorNodes.add(e.nodeId);
|
|
214
|
+
}
|
|
215
|
+
// failure_node is always derived from a live flow_node_executions
|
|
216
|
+
// entry at the moment the failure was first recorded
|
|
217
|
+
// (state/failure-store.js's flow-level sweep). If it's gone from the
|
|
218
|
+
// CURRENT set of root-level errors, this specific failure has already
|
|
219
|
+
// been resolved — most commonly by an earlier retry of the same
|
|
220
|
+
// failure event — so there is nothing left to retry. Fail clean
|
|
221
|
+
// instead of silently re-running the node.
|
|
222
|
+
//
|
|
223
|
+
// Prefer the exact execution_id anchor when the failure row has one
|
|
224
|
+
// (every failure recorded since this session's failure-linking work
|
|
225
|
+
// does) — ties eligibility to the SPECIFIC execution that produced
|
|
226
|
+
// this failure, not just "some node named failureNode currently has
|
|
227
|
+
// an error" (which errorNodes alone can't distinguish across
|
|
228
|
+
// visit_seq-repeated loop-back re-entries of the same node). Legacy
|
|
229
|
+
// rows recorded before invocation_id/execution_id existed fall back
|
|
230
|
+
// to the errorNodes Set membership check.
|
|
231
|
+
const stillLive = failure.execution_id
|
|
232
|
+
? (await nodeExecutionsStore.getLatestExecutionForNode({ flowId, nodeId: failureNode, subflowKey: null }))?.executionId === failure.execution_id
|
|
233
|
+
: errorNodes.has(failureNode);
|
|
234
|
+
if (!stillLive) {
|
|
235
|
+
return {
|
|
236
|
+
code: "conflict",
|
|
237
|
+
reason: `failure_node "${failureNode}" has no matching error in the flow's current state — this failure was likely already resolved by an earlier retry`,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
let rootNode = failureNode;
|
|
241
|
+
for (const node of topo) { if (errorNodes.has(node)) { rootNode = node; break; } }
|
|
242
|
+
let priorStep = null;
|
|
243
|
+
for (const [step, edge] of Object.entries(pipeline.edges ?? {})) {
|
|
244
|
+
try {
|
|
245
|
+
const next = typeof edge === "function" ? edge(state) : edge;
|
|
246
|
+
if (next === rootNode) { priorStep = step; break; }
|
|
247
|
+
} catch { /* skip */ }
|
|
248
|
+
}
|
|
249
|
+
if (!priorStep) {
|
|
250
|
+
return {
|
|
251
|
+
code: "conflict",
|
|
252
|
+
reason: `no edge in pipeline ${state.pipelineId} resolves to "${rootNode}" — graph may have changed since the flow ran`,
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
const nextState = JSON.parse(JSON.stringify(state));
|
|
256
|
+
const rootIdx = topo.indexOf(rootNode);
|
|
257
|
+
const nodesToClear = rootIdx >= 0 ? topo.slice(rootIdx) : [rootNode];
|
|
258
|
+
for (const node of nodesToClear) { delete nextState[node]; }
|
|
259
|
+
nextState._attemptNumber = (state._attemptNumber ?? 0) + 1;
|
|
260
|
+
let effectiveRevision = flow.revision;
|
|
261
|
+
if (CAS_EXCLUDED_STATUSES.has(flow.status)) {
|
|
262
|
+
const forced = await forceBlockOutOfMapFlow({
|
|
263
|
+
store, flowId, revision: flow.revision, stateJson: nextState, ownerSessionKey,
|
|
264
|
+
});
|
|
265
|
+
if (!forced.applied) return { code: "conflict", reason: forced.reason };
|
|
266
|
+
effectiveRevision = forced.nextRevision;
|
|
267
|
+
}
|
|
268
|
+
const result = await admitAndSetRunning({
|
|
269
|
+
flowId, expectedRevision: effectiveRevision, lastStep: priorStep, stateJson: nextState,
|
|
270
|
+
parentInvocationId: flow.parentInvocationId, store, admissionQueueStore,
|
|
271
|
+
runStore, resume: executor.resume, logger, api,
|
|
272
|
+
});
|
|
273
|
+
if (!result.applied) {
|
|
274
|
+
return { code: "conflict", reason: `setRunning failed: ${result.reason ?? result.code ?? "unknown"}` };
|
|
275
|
+
}
|
|
276
|
+
if (result.queued) {
|
|
277
|
+
logger.info(
|
|
278
|
+
`manifold retry-flow: flowId=${flowId.slice(0, 8)} failureNode=${failureNode} rootNode=${rootNode} priorStep=${priorStep} cleared=[${nodesToClear.join(",")}] via=${via} queued for admission`
|
|
279
|
+
);
|
|
280
|
+
return { code: "queued", flowId, failureNode, rootNode, priorStep, cleared: nodesToClear, mode: "flow-step", via };
|
|
281
|
+
}
|
|
282
|
+
const runRetryFlow = () => executor.runFlow(flowId, "retry").catch((err) => {
|
|
283
|
+
logger.warn(`manifold retry-flow [${flowId.slice(0, 8)}/${rootNode}] runFlow err: ${err?.message ?? err}`);
|
|
284
|
+
});
|
|
285
|
+
// HTTP/Telegram callers pass neither, and need this call to return
|
|
286
|
+
// fast — the flow keeps running in the background via setImmediate.
|
|
287
|
+
// The CLI passes awaitCompletion:true instead: its one-shot process
|
|
288
|
+
// exits right after this function returns, so a merely-SCHEDULED
|
|
289
|
+
// setImmediate callback would just get dropped mid-flight — it needs
|
|
290
|
+
// the retry to have genuinely finished before returning.
|
|
291
|
+
if (awaitCompletion) await runRetryFlow();
|
|
292
|
+
else setImmediate(runRetryFlow);
|
|
293
|
+
logger.info(
|
|
294
|
+
`manifold retry-flow: flowId=${flowId.slice(0, 8)} failureNode=${failureNode} rootNode=${rootNode} priorStep=${priorStep} cleared=[${nodesToClear.join(",")}] via=${via}`
|
|
295
|
+
);
|
|
296
|
+
return { code: "ok", flowId, failureNode, rootNode, priorStep, cleared: nodesToClear, mode: "flow-step", via };
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return { retryFailureBranch };
|
|
301
|
+
}
|