@igorjpimenta/manifold 0.0.0-bootstrap.1 → 0.0.0-stage
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1583
- package/package.json +4 -49
- package/.migrations/ops/0000_futuristic_karma.sql +0 -111
- package/.migrations/ops/0001_low_maximus.sql +0 -7
- package/.migrations/ops/meta/0000_snapshot.json +0 -763
- package/.migrations/ops/meta/0001_snapshot.json +0 -173
- package/.migrations/ops/meta/_journal.json +0 -20
- package/.migrations/runtime/0000_real_boomer.sql +0 -180
- package/.migrations/runtime/0001_hot_dakota_north.sql +0 -2
- package/.migrations/runtime/0002_breezy_vertigo.sql +0 -8
- package/.migrations/runtime/0003_dashing_jack_flag.sql +0 -5
- package/.migrations/runtime/0004_panoramic_ego.sql +0 -1
- package/.migrations/runtime/0005_tidy_ben_grimm.sql +0 -10
- package/.migrations/runtime/0006_wet_nightmare.sql +0 -57
- package/.migrations/runtime/0007_ambiguous_chronomancer.sql +0 -8
- package/.migrations/runtime/0008_wealthy_thundra.sql +0 -301
- package/.migrations/runtime/0009_lying_shocker.sql +0 -96
- package/.migrations/runtime/0010_solid_phantom_reporter.sql +0 -160
- package/.migrations/runtime/0011_windy_demogoblin.sql +0 -175
- package/.migrations/runtime/0012_restore_harness_fks.sql +0 -103
- package/.migrations/runtime/meta/0000_snapshot.json +0 -1181
- package/.migrations/runtime/meta/0001_snapshot.json +0 -1209
- package/.migrations/runtime/meta/0002_snapshot.json +0 -1261
- package/.migrations/runtime/meta/0003_snapshot.json +0 -1292
- package/.migrations/runtime/meta/0004_snapshot.json +0 -1240
- package/.migrations/runtime/meta/0006_snapshot.json +0 -1310
- package/.migrations/runtime/meta/0007_snapshot.json +0 -1362
- package/.migrations/runtime/meta/0008_snapshot.json +0 -1742
- package/.migrations/runtime/meta/0009_snapshot.json +0 -2413
- package/.migrations/runtime/meta/0010_snapshot.json +0 -2562
- package/.migrations/runtime/meta/0011_snapshot.json +0 -2812
- package/.migrations/runtime/meta/0012_snapshot.json +0 -2812
- package/.migrations/runtime/meta/_journal.json +0 -97
- package/RELEASING.md +0 -89
- package/agent/fleet.js +0 -100
- package/bin/cli.js +0 -854
- package/bin/manifold.js +0 -4
- package/boot/flow-revival.js +0 -536
- package/budget/index.js +0 -495
- package/budget/lib.js +0 -59
- package/dashboard/cost.js +0 -528
- package/dashboard/dist/assets/index-CUfp63sj.css +0 -1
- package/dashboard/dist/assets/index-Qy9JMpma.js +0 -371
- package/dashboard/dist/index.html +0 -13
- package/dashboard/event-log.js +0 -267
- package/dashboard/index.html +0 -12
- package/dashboard/package-lock.json +0 -2877
- package/dashboard/package.json +0 -37
- package/dashboard/pipeline-graph.js +0 -131
- package/dashboard/postcss.config.js +0 -6
- package/dashboard/search.js +0 -183
- package/dashboard/server.js +0 -1015
- package/dashboard/src/App.tsx +0 -42
- package/dashboard/src/api/client.ts +0 -118
- package/dashboard/src/api/types.ts +0 -510
- package/dashboard/src/api/ws.tsx +0 -112
- package/dashboard/src/components/CancelButton.tsx +0 -63
- package/dashboard/src/components/ClickToCopy.tsx +0 -68
- package/dashboard/src/components/CopyButton.tsx +0 -61
- package/dashboard/src/components/DailyCostTrend.tsx +0 -85
- package/dashboard/src/components/DailyTimeTrend.tsx +0 -94
- package/dashboard/src/components/Funnel.tsx +0 -149
- package/dashboard/src/components/GateForm.tsx +0 -158
- package/dashboard/src/components/GraphView.tsx +0 -302
- package/dashboard/src/components/GroupPathsSummary.tsx +0 -47
- package/dashboard/src/components/Histogram.tsx +0 -292
- package/dashboard/src/components/Layout.tsx +0 -28
- package/dashboard/src/components/NeedsAttention.tsx +0 -98
- package/dashboard/src/components/NodeContextCard.tsx +0 -132
- package/dashboard/src/components/PathTrail.tsx +0 -35
- package/dashboard/src/components/PipelineCostBreakdown.tsx +0 -277
- package/dashboard/src/components/RerunButton.tsx +0 -46
- package/dashboard/src/components/RunComparison.tsx +0 -53
- package/dashboard/src/components/Sparkline.tsx +0 -94
- package/dashboard/src/components/StatStrip.tsx +0 -147
- package/dashboard/src/components/StateTree.tsx +0 -84
- package/dashboard/src/components/StatusBadge.tsx +0 -18
- package/dashboard/src/components/SubflowPathPills.tsx +0 -38
- package/dashboard/src/components/SubflowSummary.tsx +0 -24
- package/dashboard/src/components/SubflowsCard.tsx +0 -327
- package/dashboard/src/components/TodayStrip.tsx +0 -86
- package/dashboard/src/components/TranscriptPane.tsx +0 -99
- package/dashboard/src/components/TranscriptRow.tsx +0 -69
- package/dashboard/src/components/ui/badge.tsx +0 -38
- package/dashboard/src/components/ui/button.tsx +0 -28
- package/dashboard/src/components/ui/card.tsx +0 -42
- package/dashboard/src/components/ui/skeleton.tsx +0 -9
- package/dashboard/src/components/ui/table.tsx +0 -62
- package/dashboard/src/index.css +0 -26
- package/dashboard/src/lib/anomaly.ts +0 -42
- package/dashboard/src/lib/format.ts +0 -30
- package/dashboard/src/lib/step-history.ts +0 -13
- package/dashboard/src/lib/subflow.ts +0 -154
- package/dashboard/src/lib/use-click-outside.ts +0 -28
- package/dashboard/src/lib/use-ignored-clusters.ts +0 -67
- package/dashboard/src/lib/utils.ts +0 -6
- package/dashboard/src/main.tsx +0 -10
- package/dashboard/src/types/shims.d.ts +0 -4
- package/dashboard/src/views/Home.tsx +0 -478
- package/dashboard/src/views/PipelineDetail.tsx +0 -1185
- package/dashboard/src/views/PipelinesList.tsx +0 -111
- package/dashboard/src/views/RunDetail.tsx +0 -248
- package/dashboard/src/views/Search.tsx +0 -196
- package/dashboard/stats.js +0 -668
- package/dashboard/tailwind.config.ts +0 -43
- package/dashboard/tree-stats.js +0 -12
- package/dashboard/tsconfig.json +0 -25
- package/dashboard/views.js +0 -231
- package/dashboard/vite.config.ts +0 -34
- package/drizzle.config.js +0 -12
- package/drizzle.config.ops.js +0 -12
- package/elevation.js +0 -116
- package/engine/admission-promotion-resume.js +0 -127
- package/engine/agent-dispatch.js +0 -64
- package/engine/child-resume-trigger.js +0 -315
- package/engine/executor-budget.js +0 -87
- package/engine/executor-cancel.js +0 -83
- package/engine/executor-errors.js +0 -122
- package/engine/executor-helpers.js +0 -254
- package/engine/executor-nodes.js +0 -292
- package/engine/executor-persist.js +0 -212
- package/engine/executor-subflow-children.js +0 -275
- package/engine/executor.js +0 -1013
- package/engine/global-concurrency.js +0 -33
- package/engine/loader.js +0 -528
- package/engine/needs-resolver.js +0 -229
- package/engine/pipelines-watcher.js +0 -133
- package/engine/result-replay.js +0 -34
- package/engine/revival-messages.js +0 -11
- package/engine/spawn-child.js +0 -55
- package/engine/wait-for-flow-settlement.js +0 -109
- package/events/catalog.js +0 -26
- package/events/emit.js +0 -44
- package/events/kinds.js +0 -118
- package/events/serve-ws.js +0 -74
- package/events/stream.js +0 -35
- package/events/ws-dialog.js +0 -257
- package/events/ws-frame.js +0 -194
- package/events/ws-run-events.js +0 -70
- package/events.js +0 -32
- package/gates/gate-resolver.js +0 -576
- package/gates/paused-flows.js +0 -55
- package/graphql/columns.js +0 -67
- package/graphql/schema.js +0 -333
- package/index.js +0 -49
- package/mcp/http.js +0 -159
- package/mcp/identity.js +0 -94
- package/mcp/stdio.js +0 -80
- package/mcp/tools.js +0 -245
- package/mcp/wire-ops.js +0 -178
- package/routes/bearer-auth.js +0 -57
- package/routes/discuss-audio.js +0 -87
- package/routes/discuss-component.js +0 -39
- package/routes/discuss-draft.js +0 -159
- package/routes/discuss-helpers.js +0 -20
- package/routes/discuss.js +0 -848
- package/routes/failures.js +0 -104
- package/routes/graphql.js +0 -61
- package/routes/health.js +0 -54
- package/routes/operator-discuss.js +0 -193
- package/routes/operator-resume.js +0 -32
- package/routes/operator-retry.js +0 -301
- package/routes/operator.js +0 -539
- package/routes/pipelines.js +0 -118
- package/routes/runs.js +0 -1129
- package/scripts/ci/import-check.mjs +0 -50
- package/scripts/link-core-self.mjs +0 -31
- package/scripts/with-guard.sh +0 -24
- package/server/app.js +0 -503
- package/server/registry.js +0 -53
- package/server/serve.js +0 -163
- package/state/admission-queue-store.js +0 -325
- package/state/attribution-registries.js +0 -102
- package/state/batch-writer.js +0 -162
- package/state/db-guard.js +0 -63
- package/state/db-health.js +0 -27
- package/state/db.js +0 -122
- package/state/discuss-store.js +0 -299
- package/state/failure-store.js +0 -427
- package/state/migrate.js +0 -104
- package/state/node-cost.js +0 -67
- package/state/node-executions.js +0 -775
- package/state/node-health-store.js +0 -124
- package/state/ops-merge.js +0 -93
- package/state/paths.js +0 -146
- package/state/pipeline-dirs.js +0 -99
- package/state/pipeline-registry.js +0 -449
- package/state/run-store.js +0 -697
- package/state/schema.js +0 -438
- package/state/schema.ops.js +0 -44
- package/state/session-index.js +0 -177
- package/state/session-keys.js +0 -39
- package/state/state-store.js +0 -311
- package/state/subflow-state.js +0 -126
- package/state/tool-handoff.js +0 -169
- package/telemetry/budget-alert.js +0 -353
- package/telemetry/cli.js +0 -331
- package/telemetry/grammar.js +0 -119
- package/telemetry/http-budget.js +0 -132
- package/telemetry/http-cost.js +0 -670
- package/telemetry/http-helpers.js +0 -134
- package/telemetry/http-tree.js +0 -349
- package/telemetry/http.js +0 -44
- package/telemetry/index.js +0 -307
- package/telemetry/ingest.js +0 -134
- package/telemetry/labels.js +0 -115
- package/telemetry/pricer.js +0 -77
- package/telemetry/resolver.js +0 -314
- package/telemetry/store.js +0 -147
- package/utils/run-patch.js +0 -43
package/bin/manifold.js
DELETED
package/boot/flow-revival.js
DELETED
|
@@ -1,536 +0,0 @@
|
|
|
1
|
-
import { completeCancellation } from "../engine/executor-cancel.js";
|
|
2
|
-
import { areSiblingsAllTerminal, flipToRunningAndResume, subflowOutcomeSatisfied } from "../engine/child-resume-trigger.js";
|
|
3
|
-
import { resolveEdge } from "../engine/executor-helpers.js";
|
|
4
|
-
import { wakePromotedRelease, isEngineResumableKind, scheduleDetachedResume } from "../engine/admission-promotion-resume.js";
|
|
5
|
-
import { admissionKindForFlow } from "../engine/child-resume-trigger.js";
|
|
6
|
-
import { TERMINAL_STATUS_SET } from "../state/run-store.js";
|
|
7
|
-
|
|
8
|
-
// A promotion (release()'s side effect of flipping a queued row straight to
|
|
9
|
-
// 'admitted') is supposed to be immediately followed by wakePromotedRelease
|
|
10
|
-
// driving that flow to 'running' — see admission-promotion-resume.js and
|
|
11
|
-
// every release()/releaseByFlow() call site in this codebase. But that
|
|
12
|
-
// follow-through is still two separate steps (a durable DB write, then an
|
|
13
|
-
// async continuation that isn't part of the same transaction): if the
|
|
14
|
-
// continuation never runs (a crash in that exact window) or throws in a way
|
|
15
|
-
// nothing catches, the row stays 'admitted' forever while its flow sits
|
|
16
|
-
// non-terminal and non-running — holding a live slot in the shared pool
|
|
17
|
-
// with nothing left to drive it. The terminal-leak sweep above can't catch
|
|
18
|
-
// this: it only reconciles rows whose owning flow is already terminal, and
|
|
19
|
-
// this flow never gets there — it's stuck exactly where it was queued.
|
|
20
|
-
//
|
|
21
|
-
// minAgeMs guards against a false positive on a promotion that's
|
|
22
|
-
// genuinely still in flight (the continuation hasn't had a chance to run
|
|
23
|
-
// yet) — skip anything admitted more recently than that window. Exported
|
|
24
|
-
// (not folded into reviveOrphanFlows) so index.js can also run it on a
|
|
25
|
-
// periodic timer: reviveOrphanFlows only runs once, at boot, so a leak
|
|
26
|
-
// occurring hours into uptime would otherwise go undetected until the next
|
|
27
|
-
// restart.
|
|
28
|
-
//
|
|
29
|
-
// Also reaps admitted rows of kinds this engine does not admit — see the
|
|
30
|
-
// unresumable-kind pass below for why skipping those is what wedged the whole
|
|
31
|
-
// pool shut, not a safe no-op.
|
|
32
|
-
export async function reconcileStuckAdmissions({ admissionQueueStore, runStore, executor, api, logger, minAgeMs = 30_000 }) {
|
|
33
|
-
if (!admissionQueueStore) return { stuckAdmissionsResumed: 0, stuckAdmissionsFailed: 0, retiredKindReaped: 0 };
|
|
34
|
-
const now = Date.now();
|
|
35
|
-
const admittedRows = await admissionQueueStore.listByStatus("admitted");
|
|
36
|
-
|
|
37
|
-
// Reap admitted rows whose kind this engine does not admit. Such a kind
|
|
38
|
-
// can never be driven: resumePromotedAdmission refuses non-resumable
|
|
39
|
-
// kinds, and every flow transition releases by the flow's CURRENT
|
|
40
|
-
// admission kind, so releaseByFlow never matches such a row either.
|
|
41
|
-
// "Unresumable" therefore makes the row a permanent pool leak, not a safe
|
|
42
|
-
// skip — admit()'s and fillFreeSlots()'s admitted-count queries have no
|
|
43
|
-
// kind filter, so zombie rows count against the ceiling forever and wedge
|
|
44
|
-
// both shut. release() is the reclaim: it frees the slot and, as a side
|
|
45
|
-
// effect, promotes the globally-oldest queued row, which
|
|
46
|
-
// wakePromotedRelease then drives — the freed capacity goes back to work
|
|
47
|
-
// in the same pass instead of idling until some unrelated release.
|
|
48
|
-
// A QUEUED row of such a kind is left here (it holds no slot), but
|
|
49
|
-
// fillFreeSlots() can promote one into 'admitted'; this pass is what
|
|
50
|
-
// reclaims it after that, past the same grace window.
|
|
51
|
-
let retiredKindReaped = 0;
|
|
52
|
-
const resumableRows = [];
|
|
53
|
-
for (const row of admittedRows) {
|
|
54
|
-
if (isEngineResumableKind(row.kind)) { resumableRows.push(row); continue; }
|
|
55
|
-
if (row.admittedAt && now - row.admittedAt < minAgeMs) continue;
|
|
56
|
-
const flow = await runStore.get(row.flowId);
|
|
57
|
-
// A flow still 'running' owns the slot legitimately and releases it
|
|
58
|
-
// through its own transition path; reclaiming the row underneath it
|
|
59
|
-
// would transiently over-admit the pool. Missing or terminal flows are
|
|
60
|
-
// reaped: no live process can release them (the terminal-leak sweep
|
|
61
|
-
// above only runs at boot, and release() is a safe no-op if it already
|
|
62
|
-
// did).
|
|
63
|
-
if (flow?.status === "running") continue;
|
|
64
|
-
try {
|
|
65
|
-
const result = await admissionQueueStore.release(row.id);
|
|
66
|
-
await wakePromotedRelease(result, {
|
|
67
|
-
runStore, resume: executor.resume, logger, api,
|
|
68
|
-
context: `manifold revival: retired-kind reap [${row.id.slice(0, 8)}]`,
|
|
69
|
-
});
|
|
70
|
-
retiredKindReaped++;
|
|
71
|
-
logger.warn(
|
|
72
|
-
`manifold revival: reaped retired-kind '${row.kind}' admission [${row.id.slice(0, 8)}] for flow ` +
|
|
73
|
-
`[${row.flowId.slice(0, 8)}] (${flow ? `status=${flow.status}` : "flow_runs row gone"}) — ` +
|
|
74
|
-
`no code path can drive this kind, slot reclaimed`,
|
|
75
|
-
);
|
|
76
|
-
} catch (err) {
|
|
77
|
-
logger.warn(`manifold revival: retired-kind reap [${row.id.slice(0, 8)}] failed: ${err?.message ?? err}`);
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
// Filter down to genuinely stuck candidates first — cheap, fast DB
|
|
82
|
-
// reads, fine to do one at a time. flipToRunningAndResume itself is the
|
|
83
|
-
// slow, unbounded part: it can resolve a flow all the way to a fresh
|
|
84
|
-
// admission request of its own, which — under a congested pool — waits
|
|
85
|
-
// on yet another slot. Driving that sequentially, one candidate at a
|
|
86
|
-
// time, meant a single candidate stuck behind real contention blocked
|
|
87
|
-
// every other candidate behind it in the same pass indefinitely: this
|
|
88
|
-
// sweep exists specifically to recover flows stuck exactly like that, so
|
|
89
|
-
// serializing on one of them defeated its own purpose under load.
|
|
90
|
-
const candidates = [];
|
|
91
|
-
for (const row of resumableRows) {
|
|
92
|
-
if (row.admittedAt && now - row.admittedAt < minAgeMs) continue;
|
|
93
|
-
const flow = await runStore.get(row.flowId);
|
|
94
|
-
// No flow_runs row at all, or already terminal: not this sweep's job —
|
|
95
|
-
// the terminal-leak reconciliation above (boot-only) or a future run of
|
|
96
|
-
// it handles those. This sweep only exists for "admitted but genuinely
|
|
97
|
-
// stuck non-terminal", so a flow already 'running' is also skipped —
|
|
98
|
-
// that's the healthy, expected state.
|
|
99
|
-
if (!flow || flow.status === "running" || TERMINAL_STATUS_SET.has(flow.status)) continue;
|
|
100
|
-
// The row being 'admitted' only means THIS flow's row for THIS kind
|
|
101
|
-
// holds a live slot — it says nothing about what the flow is currently
|
|
102
|
-
// doing: a flow can be 'queued' on a different resource (a cross-process
|
|
103
|
-
// lock) while its slot is legitimately still counted. Force-driving that
|
|
104
|
-
// would resume it on top of its own in-flight wait. The real, narrow
|
|
105
|
-
// signature of the leak this sweep exists for is a flow queued waiting
|
|
106
|
-
// specifically to be admitted for the SAME kind it already holds — a
|
|
107
|
-
// state that can only exist if a promotion's flip-to-running
|
|
108
|
-
// follow-through was dropped, since nothing else produces it.
|
|
109
|
-
if (flow.waitJson?.kind !== row.kind) continue;
|
|
110
|
-
candidates.push(row);
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
let stuckAdmissionsResumed = 0;
|
|
114
|
-
let stuckAdmissionsFailed = 0;
|
|
115
|
-
await Promise.all(candidates.map(async (row) => {
|
|
116
|
-
logger.warn(
|
|
117
|
-
`manifold revival: admitted ${row.kind} slot [${row.id.slice(0, 8)}] for flow [${row.flowId.slice(0, 8)}] ` +
|
|
118
|
-
`never resumed (status=queued, admitted ${Math.round((now - (row.admittedAt ?? now)) / 60000)}min ago) — force-driving flipToRunningAndResume`,
|
|
119
|
-
);
|
|
120
|
-
try {
|
|
121
|
-
// No admissionQueueStore passed through: this flow already holds the
|
|
122
|
-
// live admission (that's how it got into `admittedRows` above) —
|
|
123
|
-
// passing it would have flipToRunningAndResume call admit() again,
|
|
124
|
-
// which the resume-admitted CLI command's own header comment already
|
|
125
|
-
// flags as unsafe to do for a flow that already holds the slot.
|
|
126
|
-
const result = await flipToRunningAndResume({ parentFlowId: row.flowId, runStore, resume: executor.resume, logger, api });
|
|
127
|
-
if (result.resumed) {
|
|
128
|
-
stuckAdmissionsResumed++;
|
|
129
|
-
} else {
|
|
130
|
-
stuckAdmissionsFailed++;
|
|
131
|
-
logger.warn(`manifold revival: stuck-admission force-resume for [${row.flowId.slice(0, 8)}] did not apply: ${result.reason ?? "unknown"}`);
|
|
132
|
-
}
|
|
133
|
-
} catch (err) {
|
|
134
|
-
stuckAdmissionsFailed++;
|
|
135
|
-
logger.warn(`manifold revival: stuck-admission force-resume for [${row.flowId.slice(0, 8)}] threw: ${err?.message ?? err}`);
|
|
136
|
-
}
|
|
137
|
-
}));
|
|
138
|
-
return { stuckAdmissionsResumed, stuckAdmissionsFailed, retiredKindReaped };
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/**
|
|
142
|
-
* Boot-time orphan recovery for flow_runs left in status='running' (crashed
|
|
143
|
-
* mid-flight) or status='queued' (waiting on a shared
|
|
144
|
-
* admission-pool slot — flow-start or child-dispatch — when the
|
|
145
|
-
* process died) after a gateway crash. Called by the manifold-flow-revival
|
|
146
|
-
* service in index.js.
|
|
147
|
-
*
|
|
148
|
-
* Returns a summary object for observability and testing:
|
|
149
|
-
* { revived: number, revivedAgent: number, queuedForAdmission: number, readmitted: number, recancelled: number, childrenReconciled: number, gatesReconciled: number, admissionLeaksReleased: number, freeSlotsFilled: number }
|
|
150
|
-
*/
|
|
151
|
-
|
|
152
|
-
export async function reviveOrphanFlows({
|
|
153
|
-
all, pipelines, store, logger, executor, createCancelStore,
|
|
154
|
-
runStore, nodeExecutionsStore, maybeResolveFlow, sessionIndex, admissionQueueStore, api,
|
|
155
|
-
// Threaded through to the stuck-promotion reconciliation call below —
|
|
156
|
-
// exists mainly so tests can exercise that pass without waiting out its
|
|
157
|
-
// real-world grace window (a genuine boot's admission rows are all older
|
|
158
|
-
// than this process's own start, so the default is trivially satisfied
|
|
159
|
-
// there;
|
|
160
|
-
// a unit test that admits a row moments before calling this function is
|
|
161
|
-
// not so lucky).
|
|
162
|
-
stuckAdmissionMinAgeMs,
|
|
163
|
-
}) {
|
|
164
|
-
// ── Complete interrupted cancellations ──────────────────────────────────────
|
|
165
|
-
// store.cancel() writes cancel_requested_at first, then status='cancelled',
|
|
166
|
-
// in two separate SQL statements. A crash between them leaves a flow with
|
|
167
|
-
// cancelRequestedAt set but status still 'running'/'blocked'/'waiting'.
|
|
168
|
-
// Re-cancel those before reviving anything else so they don't land in toRevive.
|
|
169
|
-
// Uses the same completeCancellation path as executor.cancel() — step 1 is
|
|
170
|
-
// already done, so f.revision is already the post-requestCancel value and is
|
|
171
|
-
// passed directly to completeCancellation as the expectedRevision for setRunning.
|
|
172
|
-
const cancelledByRevival = new Set();
|
|
173
|
-
const terminalStatuses = new Set(["cancelled", "failed", "succeeded"]);
|
|
174
|
-
const cancelStores = new Map();
|
|
175
|
-
function cancelStoreFor(ownerKey) {
|
|
176
|
-
if (!ownerKey || !createCancelStore) return store;
|
|
177
|
-
if (!cancelStores.has(ownerKey)) cancelStores.set(ownerKey, createCancelStore(ownerKey));
|
|
178
|
-
return cancelStores.get(ownerKey);
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
for (const f of all) {
|
|
182
|
-
if (f.cancelRequestedAt == null) continue;
|
|
183
|
-
if (terminalStatuses.has(f.status)) continue;
|
|
184
|
-
logger.info(
|
|
185
|
-
`manifold revival: completing interrupted cancel [${f.flowId.slice(0, 8)}] status=${f.status}`,
|
|
186
|
-
);
|
|
187
|
-
await completeCancellation({
|
|
188
|
-
store: cancelStoreFor(f.ownerKey),
|
|
189
|
-
flowId: f.flowId,
|
|
190
|
-
stateJson: f.stateJson,
|
|
191
|
-
lastStep: f.lastStep,
|
|
192
|
-
revision: f.revision,
|
|
193
|
-
logger,
|
|
194
|
-
runStore,
|
|
195
|
-
nodeExecutionsStore,
|
|
196
|
-
reason: "cancel requested before restart, completed on revival",
|
|
197
|
-
});
|
|
198
|
-
cancelledByRevival.add(f.flowId);
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
// ── Classify orphans (running AND queued, any wait kind) ───────────────────
|
|
202
|
-
// 'running' orphans: process died mid-flight. 'queued' orphans: process
|
|
203
|
-
// died (or restarted) while the flow was waiting on an internal resource —
|
|
204
|
-
// the shared flow-start/child-dispatch admission pool.
|
|
205
|
-
// All land here uniformly; the skip-direct-dispatch step
|
|
206
|
-
// below and the revive-survivors step's dispatch don't need to know *why*
|
|
207
|
-
// a row was queued, only whether it wants to run.
|
|
208
|
-
//
|
|
209
|
-
// isAgent decides HOW to revive an agent-kind node: reviveAgentKindOrphan()
|
|
210
|
-
// resumes the SAME deterministic session with REVIVAL_MSG, a continuation
|
|
211
|
-
// prompt, instead of re-running promptBuilder fresh (see engine/
|
|
212
|
-
// revival-messages.js) — and it sends that continuation prompt
|
|
213
|
-
// UNCONDITIONALLY, even to a session with zero prior turns (executor-
|
|
214
|
-
// nodes.js's dispatch: `reviveMessage ?? promptBuilder(...)`, with no
|
|
215
|
-
// branch on whether a prior invocation/session actually exists). So
|
|
216
|
-
// isAgent is only correct when a dispatch attempt genuinely started —
|
|
217
|
-
// sending REVIVAL_MSG as literally the first message to a node that was
|
|
218
|
-
// only ever queued, never dispatched, would be a "continue where you left
|
|
219
|
-
// off" prompt with nothing to continue.
|
|
220
|
-
//
|
|
221
|
-
// For a 'running' orphan, targetNodeId comes from
|
|
222
|
-
// nodeExecutionsStore.findCurrentStep() below, which IS race-free proof a
|
|
223
|
-
// dispatch attempt genuinely entered this node: the nodeExecutions.enter()
|
|
224
|
-
// call creates the invocation row BEFORE executeNode() runs, well before
|
|
225
|
-
// dispatchAgentWithRetry gets to recording a flow_sessions entry
|
|
226
|
-
// (session-index.js) or calling subagent.run() — so currentStep alone is
|
|
227
|
-
// sufficient there, without consulting sessionIndex.hasSessionForNode().
|
|
228
|
-
//
|
|
229
|
-
// The 'running' case must not REQUIRE a session record. The flow_sessions
|
|
230
|
-
// record is written well after enter() (past several awaits — budget gate,
|
|
231
|
-
// assembleState, promptBuilder, dispatch-semaphore acquisition), so a crash
|
|
232
|
-
// in that narrow window leaves a genuinely in-flight agent dispatch with a
|
|
233
|
-
// correct targetNodeId but hasSessionForNode()===false, and requiring it
|
|
234
|
-
// misroutes that dispatch into the generic resolveEdge()/runFlow() path
|
|
235
|
-
// below, which has no node output to route on and fails with "edge <NODE>
|
|
236
|
-
// returned no next node". A running flow with no invocation row yet
|
|
237
|
-
// (findCurrentStep() -> null) is the other case, and there falling through
|
|
238
|
-
// to resolveEdge() IS correct; hasSessionForNode() is what still decides
|
|
239
|
-
// isAgent for it, via the !dispatchGenuinelyEntered branch below.
|
|
240
|
-
//
|
|
241
|
-
// For a 'queued' orphan, targetNodeId instead comes from waitJson.nodeId
|
|
242
|
-
// — set by the admission wait before it acquires a slot, which only
|
|
243
|
-
// proves the node was ABOUT to be dispatched, not that dispatch began.
|
|
244
|
-
// hasSessionForNode() still gates isAgent there.
|
|
245
|
-
//
|
|
246
|
-
// f.lastStep is the last COMPLETED node, not the one in flight — using it
|
|
247
|
-
// directly as the crashed node's id is wrong. The real target node is:
|
|
248
|
-
// - queued: f.waitJson.nodeId — the admission wait sets it directly
|
|
249
|
-
// (executor.js) since a queued flow never got far enough to advance
|
|
250
|
-
// lastStep at all.
|
|
251
|
-
// - running: nodeExecutionsStore.findCurrentStep(flowId) — the invocation
|
|
252
|
-
// row is created (enter()) before executeNode() runs, so
|
|
253
|
-
// the last invocation's node_id correctly identifies the in-flight node
|
|
254
|
-
// even across a crash. Falls back to resolveEdge(lastStep) if the query
|
|
255
|
-
// returns null (first iteration of a brand-new flow with no invocations).
|
|
256
|
-
const orphans = [];
|
|
257
|
-
for (const f of all) {
|
|
258
|
-
if (f.status !== "running" && f.status !== "queued") continue;
|
|
259
|
-
if (cancelledByRevival.has(f.flowId)) continue;
|
|
260
|
-
const pipeline = pipelines.get(f.stateJson?.pipelineId);
|
|
261
|
-
const currentStep = f.status === "running"
|
|
262
|
-
? await nodeExecutionsStore?.findCurrentStep(f.flowId)
|
|
263
|
-
: null;
|
|
264
|
-
const targetNodeId = f.waitJson?.nodeId
|
|
265
|
-
?? currentStep
|
|
266
|
-
?? resolveEdge(pipeline?.edges ?? {}, f.lastStep, f.stateJson, { logger, flowId: f.flowId });
|
|
267
|
-
const nodeKind = pipeline?.nodes?.[targetNodeId]?.kind;
|
|
268
|
-
const dispatchGenuinelyEntered = currentStep != null && targetNodeId === currentStep;
|
|
269
|
-
const hasSession = nodeKind === "agent" && !dispatchGenuinelyEntered && sessionIndex != null
|
|
270
|
-
? !!(await sessionIndex.hasSessionForNode(f.flowId, targetNodeId))
|
|
271
|
-
: false;
|
|
272
|
-
const isAgent = nodeKind === "agent" && (dispatchGenuinelyEntered || hasSession);
|
|
273
|
-
orphans.push({ flow: f, isAgent, targetNodeId, nodeKind });
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
// ── Skip direct dispatch for orphans queued on the shared admission
|
|
277
|
-
// pool ──────────────────────────────────────────────────────────────────
|
|
278
|
-
// The orphan-classification step above classified every non-terminal
|
|
279
|
-
// orphan as 'running' or 'queued'.
|
|
280
|
-
// A 'queued' orphan waiting on kind:"flow-start" or "child-dispatch" is
|
|
281
|
-
// tracked in the DURABLE combined admission pool (state/admission-queue-
|
|
282
|
-
// store.js — the same runtime.sqlite every other table here survives a
|
|
283
|
-
// crash in), which already reflects who was genuinely admitted vs still
|
|
284
|
-
// queued the moment the gateway died. Nothing needs to be re-derived: a
|
|
285
|
-
// live queued flow-start/child-dispatch flow is never dispatched
|
|
286
|
-
// directly either (see engine/executor.js's runFlow() isQueued guard) —
|
|
287
|
-
// the same admission-queue-store release()/promotion cycle that wakes a
|
|
288
|
-
// live queued flow wakes a boot-revived one too, once some other admitted
|
|
289
|
-
// flow elsewhere goes blocked or terminal, no special boot-time action
|
|
290
|
-
// needed. Throttling happens atomically at the moment each flow admits,
|
|
291
|
-
// not as a bulk post-hoc displacement correction here.
|
|
292
|
-
//
|
|
293
|
-
// A 'queued' orphan on waitJson.kind:"agent-dispatch" — a kind this engine
|
|
294
|
-
// does not admit (state/admission-queue-store.js's migration deletes its
|
|
295
|
-
// rows) — gave its own run slot back when it queued, so it holds nothing.
|
|
296
|
-
// Re-entering it directly would run it outside the pool; instead it
|
|
297
|
-
// re-requests its own run slot (flipToRunningAndResume with the store:
|
|
298
|
-
// admitted → flipped to running and resumed; pool full → queued on its
|
|
299
|
-
// own kind for the normal promotion cycle).
|
|
300
|
-
//
|
|
301
|
-
const toRevive = [];
|
|
302
|
-
let queuedForAdmission = 0;
|
|
303
|
-
let readmitted = 0;
|
|
304
|
-
for (const o of orphans) {
|
|
305
|
-
const kind = o.flow.waitJson?.kind;
|
|
306
|
-
if (o.flow.status === "queued" && (kind === "flow-start" || kind === "child-dispatch")) {
|
|
307
|
-
logger.info(`manifold revival: [${o.flow.flowId.slice(0, 8)}] left queued for admission (${kind}) — will resume via the normal release/promotion cycle`);
|
|
308
|
-
queuedForAdmission++;
|
|
309
|
-
continue;
|
|
310
|
-
}
|
|
311
|
-
if (o.flow.status === "queued" && kind === "agent-dispatch" && admissionQueueStore && runStore) {
|
|
312
|
-
const result = await flipToRunningAndResume({
|
|
313
|
-
parentFlowId: o.flow.flowId, runStore, admissionQueueStore, logger, api,
|
|
314
|
-
resume: (flowId) => { scheduleDetachedResume(executor.resume, flowId, { logger, api, context: "manifold revival: re-admit" }); },
|
|
315
|
-
});
|
|
316
|
-
logger.info(
|
|
317
|
-
`manifold revival: [${o.flow.flowId.slice(0, 8)}] re-requested its ${admissionKindForFlow(o.flow)} slot — ` +
|
|
318
|
-
`${result.resumed ? "admitted, resuming" : result.reason}`,
|
|
319
|
-
);
|
|
320
|
-
readmitted++;
|
|
321
|
-
continue;
|
|
322
|
-
}
|
|
323
|
-
toRevive.push(o);
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
// ── Revive survivors ────────────────────────────────────────────────────────
|
|
327
|
-
let revivedAgent = 0;
|
|
328
|
-
if (toRevive.length === 0) {
|
|
329
|
-
logger.info("manifold revival: no revivable orphan flows at boot");
|
|
330
|
-
} else {
|
|
331
|
-
logger.info(`manifold revival: reviving ${toRevive.length} orphan flow(s) from lastStep`);
|
|
332
|
-
for (const o of toRevive) {
|
|
333
|
-
const flow = o.flow;
|
|
334
|
-
const pipelineId = flow.stateJson?.pipelineId ?? "?";
|
|
335
|
-
logger.info(`manifold revival: [${flow.flowId.slice(0, 8)}] pipeline=${pipelineId} step=${o.targetNodeId}`);
|
|
336
|
-
if (o.isAgent) {
|
|
337
|
-
revivedAgent++;
|
|
338
|
-
// Resumes the SAME session with a continuation prompt. No admission
|
|
339
|
-
// work: the flow's own run slot (flow-start/child-dispatch) is
|
|
340
|
-
// durable across the crash and still held, and the agent call runs
|
|
341
|
-
// inside it like any other node — it's released through the flow's
|
|
342
|
-
// normal lifecycle once the run blocks or finishes.
|
|
343
|
-
executor.reviveAgentKindOrphan(flow, o.targetNodeId)
|
|
344
|
-
.catch((err) => {
|
|
345
|
-
logger.warn(`manifold revival: reviveAgentKindOrphan [${flow.flowId.slice(0, 8)}] error: ${err?.message ?? err}`);
|
|
346
|
-
});
|
|
347
|
-
} else {
|
|
348
|
-
executor.runFlow(flow.flowId, "revival").catch((err) => {
|
|
349
|
-
logger.warn(`manifold revival: runFlow [${flow.flowId.slice(0, 8)}] error: ${err?.message ?? err}`);
|
|
350
|
-
});
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
// ── Children-pending reconciliation ─────────────────────────────────────────
|
|
356
|
-
// The resume-trigger (state-store.js's finish/skip/fail/cancel) fires
|
|
357
|
-
// on the LAST sibling's terminal write — but if the gateway crashes at
|
|
358
|
-
// exactly that moment, the fire can be lost. Re-check every blocked parent
|
|
359
|
-
// directly against its children's current terminal state and resume if
|
|
360
|
-
// already satisfied, the same idempotent-recheck shape gate-resolver.js's
|
|
361
|
-
// maybeResolveFlow already uses for the analogous gate-side problem. No-op
|
|
362
|
-
// when nodeExecutionsStore/runStore aren't wired in (older callers, tests).
|
|
363
|
-
let childrenReconciled = 0;
|
|
364
|
-
if (runStore && nodeExecutionsStore) {
|
|
365
|
-
for (const f of all) {
|
|
366
|
-
if (f.status !== "blocked") continue;
|
|
367
|
-
if (f.waitJson?.kind !== "children-pending") continue;
|
|
368
|
-
const nodeId = f.waitJson?.nodeId;
|
|
369
|
-
if (!nodeId) continue;
|
|
370
|
-
const { allTerminal, children } = await areSiblingsAllTerminal({
|
|
371
|
-
flowId: f.flowId, nodeId, runStore, nodeExecutionsStore,
|
|
372
|
-
});
|
|
373
|
-
if (!allTerminal) continue;
|
|
374
|
-
childrenReconciled++;
|
|
375
|
-
// Same mode check as the live-process path
|
|
376
|
-
// (child-resume-trigger.js's maybeResumeParent) — a parent orphaned
|
|
377
|
-
// mid-resume by a gateway crash must fail (not resume) if its
|
|
378
|
-
// subflow node's mode isn't satisfied, same as it would have if the
|
|
379
|
-
// crash hadn't happened.
|
|
380
|
-
const mode = pipelines?.get?.(f.pipelineId)?.nodes?.[nodeId]?.mode;
|
|
381
|
-
if (mode && !subflowOutcomeSatisfied(mode, children)) {
|
|
382
|
-
logger.info(`manifold revival: failing [${f.flowId.slice(0, 8)}] — subflow "${nodeId}" (mode:"${mode}") not satisfied`);
|
|
383
|
-
await store.fail({
|
|
384
|
-
flowId: f.flowId, expectedRevision: f.revision, stateJson: f.stateJson,
|
|
385
|
-
statusReason: `subflow "${nodeId}" (mode:"${mode}") not satisfied — ${mode === "any" ? "every child failed" : "at least one child did not succeed"}`,
|
|
386
|
-
});
|
|
387
|
-
continue;
|
|
388
|
-
}
|
|
389
|
-
logger.info(`manifold revival: resuming [${f.flowId.slice(0, 8)}] — all children of ${nodeId} terminal`);
|
|
390
|
-
// Must flip status back to 'running' before calling resume() —
|
|
391
|
-
// executor.resume() is just runFlow(), whose own first check returns
|
|
392
|
-
// immediately for a flow still in status='blocked'. See
|
|
393
|
-
// child-resume-trigger.js's flipToRunningAndResume for why.
|
|
394
|
-
await flipToRunningAndResume({ parentFlowId: f.flowId, runStore, resume: executor.resume, logger, admissionQueueStore, api }).catch((err) => {
|
|
395
|
-
logger.warn(`manifold revival: children-pending resume [${f.flowId.slice(0, 8)}] error: ${err?.message ?? err}`);
|
|
396
|
-
});
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
// ── Stuck-gate-decision reconciliation ──────────────────────────────────────
|
|
401
|
-
// A gate decision is recorded in two steps: (1) write approvalPending[itemId]
|
|
402
|
-
// durably, (2) call maybeResolveFlow → applyResolutionsAndResume, which
|
|
403
|
-
// moves the decision into approvedIds/rejectedIds and resumes the flow.
|
|
404
|
-
// If the process ever dies between those two steps, approvalPending
|
|
405
|
-
// permanently shows a real decision that was never actually applied —
|
|
406
|
-
// the flow stays blocked forever, since nothing else ever re-drives step
|
|
407
|
-
// (2). Same idempotent-recheck shape as the children-pending
|
|
408
|
-
// reconciliation above, applied to the analogous gate-side gap. No-op
|
|
409
|
-
// when maybeResolveFlow isn't wired in (older callers, tests).
|
|
410
|
-
let gatesReconciled = 0;
|
|
411
|
-
if (maybeResolveFlow) {
|
|
412
|
-
for (const f of all) {
|
|
413
|
-
if (f.status !== "blocked") continue;
|
|
414
|
-
const waitKind = f.waitJson?.kind;
|
|
415
|
-
if (waitKind !== "human-gate" && waitKind !== "gate" && waitKind !== "verify-required") continue;
|
|
416
|
-
const items = Array.isArray(f.waitJson?.items) ? f.waitJson.items : [];
|
|
417
|
-
if (items.length === 0) continue;
|
|
418
|
-
const approvalPending = f.stateJson?.approvalPending ?? {};
|
|
419
|
-
const hasUnappliedDecision = items.some(({ itemId }) => {
|
|
420
|
-
const entry = approvalPending[itemId];
|
|
421
|
-
return entry && (entry.decision || entry.sendFailed);
|
|
422
|
-
});
|
|
423
|
-
if (!hasUnappliedDecision) continue;
|
|
424
|
-
logger.info(`manifold revival: [${f.flowId.slice(0, 8)}] has a recorded gate decision never applied — reconciling`);
|
|
425
|
-
gatesReconciled++;
|
|
426
|
-
maybeResolveFlow(f.flowId).catch((err) => {
|
|
427
|
-
logger.warn(`manifold revival: gate reconciliation [${f.flowId.slice(0, 8)}] error: ${err?.message ?? err}`);
|
|
428
|
-
});
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
// ── Admission-pool leak reconciliation ──────────────────────────────────────
|
|
433
|
-
// A flow_admission_queue row (state/admission-queue-store.js) is only ever
|
|
434
|
-
// supposed to exist while its owning flow is genuinely non-terminal —
|
|
435
|
-
// every terminal transition (finish/fail/cancel) releases it via
|
|
436
|
-
// state-store.js's triggerAdmissionRelease. That release silently no-ops
|
|
437
|
-
// if the caller's state-store was constructed without admissionQueueStore
|
|
438
|
-
// wired in, leaving the flow's row "admitted" forever and shrinking the
|
|
439
|
-
// one shared pool (a single combined ceiling across every
|
|
440
|
-
// flow-start/child-dispatch, gateway-wide) for the rest of the process's
|
|
441
|
-
// uptime. Rather than trust every future release call site
|
|
442
|
-
// to stay correctly wired, sweep for the
|
|
443
|
-
// invariant violation itself on every boot: a terminal flow can never
|
|
444
|
-
// legitimately hold a live admission entry, so any row whose owning flow
|
|
445
|
-
// is already terminal is definitionally a leak, regardless of which bug
|
|
446
|
-
// produced it. statusByFlowId is built from `all` (already loaded above)
|
|
447
|
-
// plus cancelledByRevival (flows this very call just finished cancelling,
|
|
448
|
-
// whose stale status in `all` is older than that) — no extra flow_runs reads.
|
|
449
|
-
// Own terminal-status set, not the narrower `terminalStatuses` above
|
|
450
|
-
// (cancelled/failed/succeeded only, scoped to that phase's specific
|
|
451
|
-
// re-cancel check) — this sweep must also catch "skipped" (FILTER's
|
|
452
|
-
// dedup path, run-store.js's own TERMINAL_STATUSES), any of which can
|
|
453
|
-
// equally hold a leaked admission row.
|
|
454
|
-
const admissionTerminalStatuses = new Set(["succeeded", "failed", "cancelled", "skipped"]);
|
|
455
|
-
let admissionLeaksReleased = 0;
|
|
456
|
-
if (admissionQueueStore) {
|
|
457
|
-
const statusByFlowId = new Map(all.map((f) => [f.flowId, f.status]));
|
|
458
|
-
const admissionRows = [
|
|
459
|
-
...(await admissionQueueStore.listByStatus("admitted")),
|
|
460
|
-
...(await admissionQueueStore.listByStatus("queued")),
|
|
461
|
-
];
|
|
462
|
-
for (const row of admissionRows) {
|
|
463
|
-
const owningStatus = cancelledByRevival.has(row.flowId) ? "cancelled" : statusByFlowId.get(row.flowId);
|
|
464
|
-
if (!owningStatus || !admissionTerminalStatuses.has(owningStatus)) continue;
|
|
465
|
-
logger.info(
|
|
466
|
-
`manifold revival: releasing leaked ${row.kind} admission [${row.id.slice(0, 8)}] ` +
|
|
467
|
-
`for already-terminal flow [${row.flowId.slice(0, 8)}] (status=${owningStatus})`,
|
|
468
|
-
);
|
|
469
|
-
try {
|
|
470
|
-
const result = await admissionQueueStore.release(row.id);
|
|
471
|
-
// release() frees this leaked slot and, as a side effect, may
|
|
472
|
-
// promote the globally-oldest queued row into 'admitted' — a
|
|
473
|
-
// DIFFERENT flow than the one this iteration is cleaning up.
|
|
474
|
-
// Every other caller of release()/releaseByFlow() in this codebase
|
|
475
|
-
// drives wakePromotedRelease() on that release, and so must this
|
|
476
|
-
// sweep: calling release() directly and discarding its result would
|
|
477
|
-
// flip the row to 'admitted' with nothing to ever flip the promoted
|
|
478
|
-
// flow's own flow_runs status to 'running' or call runFlow() on it
|
|
479
|
-
// — its admission would look live forever while the flow itself sat
|
|
480
|
-
// frozen exactly where it was queued, permanently shrinking the
|
|
481
|
-
// shared pool by one slot per promotion this sweep caused.
|
|
482
|
-
await wakePromotedRelease(result, {
|
|
483
|
-
runStore, resume: executor.resume, logger, api,
|
|
484
|
-
context: `manifold revival: leak-release promotion [${row.id.slice(0, 8)}]`,
|
|
485
|
-
});
|
|
486
|
-
} catch (err) {
|
|
487
|
-
logger.warn(`manifold revival: admission release [${row.id.slice(0, 8)}] failed: ${err?.message ?? err}`);
|
|
488
|
-
}
|
|
489
|
-
admissionLeaksReleased++;
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
// ── Fill free capacity ──────────────────────────────────────────────────────
|
|
494
|
-
// Every release promotes exactly one row into the slot it frees, so the
|
|
495
|
-
// pool never drops below its limit with work queued — except when
|
|
496
|
-
// admitted rows vanish without a release (a crash between a delete and
|
|
497
|
-
// its promotion, or a migration dropping a whole kind). Promote queued
|
|
498
|
-
// rows into whatever capacity is free now, so the pool doesn't idle below
|
|
499
|
-
// its limit until enough unrelated releases happen.
|
|
500
|
-
let freeSlotsFilled = 0;
|
|
501
|
-
if (admissionQueueStore) {
|
|
502
|
-
try {
|
|
503
|
-
const promoted = await admissionQueueStore.fillFreeSlots();
|
|
504
|
-
for (const p of promoted) {
|
|
505
|
-
await wakePromotedRelease({ released: false, promoted: p }, {
|
|
506
|
-
runStore, resume: executor.resume, logger, api,
|
|
507
|
-
context: `manifold revival: free-slot fill [${p.id.slice(0, 8)}]`,
|
|
508
|
-
});
|
|
509
|
-
}
|
|
510
|
-
freeSlotsFilled = promoted.length;
|
|
511
|
-
if (freeSlotsFilled > 0) logger.info(`manifold revival: filled ${freeSlotsFilled} free admission slot(s) from the queue`);
|
|
512
|
-
} catch (err) {
|
|
513
|
-
logger.warn(`manifold revival: free-slot fill failed: ${err?.message ?? err}`);
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
// ── Stuck-promotion reconciliation ──────────────────────────────────────────
|
|
518
|
-
// The sweep above only catches leaks on already-terminal flows. Run the
|
|
519
|
-
// non-terminal counterpart too (see reconcileStuckAdmissions's own header)
|
|
520
|
-
// so a boot doesn't just inherit the same "left queued for admission —
|
|
521
|
-
// will resume via the normal release/promotion cycle" hope the process
|
|
522
|
-
// that crashed was already relying on.
|
|
523
|
-
const { stuckAdmissionsResumed, stuckAdmissionsFailed, retiredKindReaped } = await reconcileStuckAdmissions({
|
|
524
|
-
admissionQueueStore, runStore, executor, api, logger,
|
|
525
|
-
...(stuckAdmissionMinAgeMs != null ? { minAgeMs: stuckAdmissionMinAgeMs } : {}),
|
|
526
|
-
});
|
|
527
|
-
if (retiredKindReaped > 0) {
|
|
528
|
-
logger.warn(`manifold revival: reaped ${retiredKindReaped} retired-kind admission slot(s) no code path could drive`);
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
return {
|
|
532
|
-
revived: toRevive.length, revivedAgent, queuedForAdmission, readmitted, recancelled: cancelledByRevival.size,
|
|
533
|
-
childrenReconciled, gatesReconciled, admissionLeaksReleased, freeSlotsFilled,
|
|
534
|
-
stuckAdmissionsResumed, stuckAdmissionsFailed, retiredKindReaped,
|
|
535
|
-
};
|
|
536
|
-
}
|