@nanobpm/nano-workforce 0.200.0 → 0.200.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/app/contracts.ts +16 -0
- package/app/deliveryGraphCompiler.ts +9 -3
- package/app/layoutOffThread.test.ts +208 -0
- package/app/layoutOffThread.ts +182 -0
- package/app/layoutWorker.ts +34 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
## [0.200.1](https://github.com/nanobpm/nano-workforce/compare/v0.200.0...v0.200.1) (2026-10-03)
|
|
2
|
+
|
|
3
|
+
### Performance Improvements
|
|
4
|
+
|
|
5
|
+
* **delivery-graph:** run BPMN autolayout off the main thread (worker_threads) so a large graph cannot freeze the app ([#855](https://github.com/nanobpm/nano-workforce/issues/855)) ([d726bca](https://github.com/nanobpm/nano-workforce/commit/d726bcafd1b8e89484c7231a705f287943633a8a)), closes [#852](https://github.com/nanobpm/nano-workforce/issues/852) [#854](https://github.com/nanobpm/nano-workforce/issues/854)
|
|
6
|
+
|
|
1
7
|
## [0.200.0](https://github.com/nanobpm/nano-workforce/compare/v0.199.0...v0.200.0) (2026-10-02)
|
|
2
8
|
|
|
3
9
|
### Features
|
package/app/contracts.ts
CHANGED
|
@@ -104,6 +104,22 @@ export const ENV_CONTRACTS = {
|
|
|
104
104
|
"Clone timeout in milliseconds emitted as `repository.cloneTimeoutMs` in the agent-task envelope so the c8ctl harness raises its 120s default for large-repo provisioning (branch-scoped blobless clones of big monorepos still approach/exceed 120s; issue #694). Inherited by both the review-round and merge paths via the one `repoEnvelopeVars` builder.",
|
|
105
105
|
default: "600000",
|
|
106
106
|
},
|
|
107
|
+
NANO_DELIVERY_LAYOUT_TIMEOUT_MS: {
|
|
108
|
+
category: "env",
|
|
109
|
+
name: "NANO_DELIVERY_LAYOUT_TIMEOUT_MS",
|
|
110
|
+
owner: "app/layoutOffThread.ts",
|
|
111
|
+
semantics:
|
|
112
|
+
"Bound (ms) on the delivery-graph BPMN autolayout (`layoutBpmn` → `bpmn-auto-layout`), which runs OFF the main event loop in a `node:worker_threads` worker (`layoutBpmnOffThread`, issue #854). On expiry the worker is terminated and the compile/launch fails CLEANLY rather than hanging — the inline layout previously froze the whole app (no HTTP, no poll passes) for the entire superlinear run (#852). Shared by both the dispatch (`dispatchDeliveryGraphRun` → `compileDeliveryGraph`) and preview (`previewProposalBpmn`) paths, which both funnel through the one `layoutDeliveryDiagram` entry point.",
|
|
113
|
+
default: "300000",
|
|
114
|
+
},
|
|
115
|
+
NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY: {
|
|
116
|
+
category: "env",
|
|
117
|
+
name: "NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY",
|
|
118
|
+
owner: "app/layoutOffThread.ts",
|
|
119
|
+
semantics:
|
|
120
|
+
"Maximum number of delivery-graph BPMN autolayout workers (`layoutBpmnOffThread`, issue #854) allowed to run CONCURRENTLY. Each layout is a CPU-bound `node:worker_threads` worker running the superlinear `bpmn-auto-layout` pass; overlapping preview/dispatch requests would otherwise spawn an UNBOUNDED number of them and saturate CPU/memory, starving the very event loop this off-thread move exists to protect. Excess layouts queue (FIFO) for a slot via the shared `Semaphore`. Capacity is fixed at first use, so a retune needs a restart (unlike the per-call timeout).",
|
|
121
|
+
default: "2",
|
|
122
|
+
},
|
|
107
123
|
NANO_PR_MAX_ROUNDS: {
|
|
108
124
|
category: "env",
|
|
109
125
|
name: "NANO_PR_MAX_ROUNDS",
|
|
@@ -23,7 +23,6 @@
|
|
|
23
23
|
// parallel gateways for genuine fan-out (>1 downstream) and fan-in (>1 upstream). This slice targets
|
|
24
24
|
// the WIRING/SHAPE — the concrete node bodies land in S4.
|
|
25
25
|
|
|
26
|
-
import { layoutBpmn } from "@nanobpm/urban";
|
|
27
26
|
import type {
|
|
28
27
|
CompileDeliveryGraphErrors,
|
|
29
28
|
CompileDeliveryGraphResult,
|
|
@@ -50,6 +49,7 @@ import {
|
|
|
50
49
|
validateDeliveryGraph,
|
|
51
50
|
} from "./deliveryGraph.ts";
|
|
52
51
|
import { DELIVERY_HUMAN_ELEMENT, GENERIC_HUMAN_FORM } from "./deliveryHuman.ts";
|
|
52
|
+
import { layoutBpmnOffThread } from "./layoutOffThread.ts";
|
|
53
53
|
import { DEFAULT_BACKOFF, DEFAULT_EVERY_MS, DEFAULT_ON_TIMEOUT, DEFAULT_TIMEOUT_MS, isProbeKind, normalizePoll, redactEmbeddedCredential, redactString } from "./readiness.ts";
|
|
54
54
|
import { AGENT_TASK_NS } from "./repoEnvelope.ts";
|
|
55
55
|
import { isoDuration } from "./reviewWait.ts";
|
|
@@ -719,9 +719,15 @@ export async function compileDeliveryGraph(
|
|
|
719
719
|
* `bpmn-auto-layout` is a real runtime dependency of `@nanobpm/urban` (which re-exports `layoutBpmn`),
|
|
720
720
|
* but the toolkit no-ops layout (semantic model unchanged, no DI) when it is somehow absent. That
|
|
721
721
|
* silent no-op is exactly the DI-less bug this fixes, so we FAIL LOUD if the pass produced no diagram.
|
|
722
|
-
* Deterministic given identical input, preserving the compiler's "same JSON → byte-identical XML".
|
|
722
|
+
* Deterministic given identical input, preserving the compiler's "same JSON → byte-identical XML".
|
|
723
|
+
*
|
|
724
|
+
* The autolayout itself runs OFF the main event loop in a `node:worker_threads` worker, bounded by a
|
|
725
|
+
* timeout (`layoutBpmnOffThread` → `app/layoutOffThread.ts`, issue #854): `layoutBpmn` is superlinear
|
|
726
|
+
* and, run inline here, blocked the whole app (no HTTP, no poll passes) for the entire layout — a hard
|
|
727
|
+
* lock-up on a large graph (#852). This is the ONE place the layout runs, so hoisting it off-thread
|
|
728
|
+
* covers BOTH the dispatch and preview paths that funnel through `compileDeliveryGraph`. */
|
|
723
729
|
async function layoutDeliveryDiagram(semanticBpmn: string): Promise<string> {
|
|
724
|
-
const laidOut = await
|
|
730
|
+
const laidOut = await layoutBpmnOffThread(semanticBpmn);
|
|
725
731
|
const start = laidOut.indexOf("<bpmndi:BPMNDiagram");
|
|
726
732
|
const endTag = "</bpmndi:BPMNDiagram>";
|
|
727
733
|
const end = laidOut.lastIndexOf(endTag);
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
// Guard coverage for the off-main-thread BPMN autolayout (issue #854). `layoutBpmn`
|
|
2
|
+
// (`bpmn-auto-layout`) is superlinear and used to run SYNCHRONOUSLY on the app's event loop inside
|
|
3
|
+
// `compileDeliveryGraph` → `layoutDeliveryDiagram`, freezing the whole app (no HTTP, no poll passes)
|
|
4
|
+
// for the entire layout — a 3287ms single gap locally on the 54-node merlin graph, minutes under load
|
|
5
|
+
// (#852). It now runs in a `node:worker_threads` worker bounded by a timeout. These tests assert:
|
|
6
|
+
// • the event loop KEEPS TICKING (bounded max gap) while a large graph lays out off-thread — the
|
|
7
|
+
// core liveness property this fixes,
|
|
8
|
+
// • the timeout TERMINATES a runaway layout and fails CLEANLY (rejects) rather than hanging,
|
|
9
|
+
// • the canonical `compileDeliveryGraph` path still produces a diagram through the off-thread bridge.
|
|
10
|
+
import { test } from "node:test";
|
|
11
|
+
import { assert, assertRejects, assertStringIncludes } from "#test-assert";
|
|
12
|
+
import type { DeliveryGraph } from "../nano-generated/api-io.d.ts";
|
|
13
|
+
import { compileDeliveryGraph } from "./deliveryGraphCompiler.ts";
|
|
14
|
+
import { layoutBpmnOffThread, layoutMaxConcurrency, layoutTimeoutMs, Semaphore } from "./layoutOffThread.ts";
|
|
15
|
+
|
|
16
|
+
/** Build a wide-and-deep delivery graph with `count` agent nodes wired into a chain with periodic
|
|
17
|
+
* fan-out, so the autolayout has enough nodes/edges to spend real CPU time laying it out. */
|
|
18
|
+
function bigGraph(count: number): DeliveryGraph {
|
|
19
|
+
const nodes: DeliveryGraph["nodes"] = [];
|
|
20
|
+
const edges: DeliveryGraph["edges"] = [];
|
|
21
|
+
for (let i = 0; i < count; i++) {
|
|
22
|
+
nodes.push({ id: `n${i}`, kind: "agent", agent: { jobType: "senior:feature", prompt: `slice ${i}` } });
|
|
23
|
+
if (i > 0) edges.push({ from: `n${i - 1}`, to: `n${i}` });
|
|
24
|
+
// Periodic fan-out to widen the graph (more edges → more layout work).
|
|
25
|
+
if (i >= 3 && i % 3 === 0) edges.push({ from: `n${i - 3}`, to: `n${i}` });
|
|
26
|
+
}
|
|
27
|
+
return { name: "big-layout-graph", nodes, edges } as DeliveryGraph;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
test("#854 the main event loop keeps servicing queued work while a large graph lays out off-thread", async () => {
|
|
31
|
+
const graph = bigGraph(60);
|
|
32
|
+
|
|
33
|
+
// ORDER-based liveness proof (not a wall-clock threshold): race a trivial main-thread `setImmediate`
|
|
34
|
+
// checkpoint against the layout. Off-thread, the loop reaches the next tick and fires the immediate
|
|
35
|
+
// in microseconds while the worker chews on the superlinear layout for far longer, so the checkpoint
|
|
36
|
+
// ALWAYS wins. Inline (the regression), `compileDeliveryGraph`'s synchronous `layoutBpmn` call would
|
|
37
|
+
// block the loop for the whole layout and its promise would then resolve via a microtask BEFORE any
|
|
38
|
+
// queued `setImmediate` macrotask could run, so the layout would win — failing this test. Because the
|
|
39
|
+
// worker's real CPU work dwarfs a single queued-tick dispatch by orders of magnitude, the winner is
|
|
40
|
+
// deterministic under any scheduler load (AGENTS.md deterministic-test rule) — no elapsed-time bound.
|
|
41
|
+
let compiled: Awaited<ReturnType<typeof compileDeliveryGraph>> | undefined;
|
|
42
|
+
const checkpoint = new Promise<"checkpoint">((resolve) => {
|
|
43
|
+
setImmediate(() => resolve("checkpoint"));
|
|
44
|
+
});
|
|
45
|
+
const layoutDone = compileDeliveryGraph(graph).then((c): "layout" => {
|
|
46
|
+
compiled = c;
|
|
47
|
+
return "layout";
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
const winner = await Promise.race([checkpoint, layoutDone]);
|
|
51
|
+
assert(
|
|
52
|
+
winner === "checkpoint",
|
|
53
|
+
"a queued main-thread setImmediate must win the race against the off-thread layout; if layout ran " +
|
|
54
|
+
"inline it would block the loop and its promise would resolve before the immediate could fire",
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
await layoutDone; // let the worker finish and tear down so the test leaks no worker/handle
|
|
58
|
+
assert(compiled !== undefined && compiled.ok, `expected ok:true, got ${JSON.stringify(compiled)}`);
|
|
59
|
+
assert(compiled.bpmn.includes("<bpmndi:BPMNDiagram"), "off-thread layout should still attach diagram interchange");
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("#854 a layout that exceeds its timeout fails cleanly (rejects) instead of hanging", async () => {
|
|
63
|
+
// A 1ms bound is unsatisfiable for any real graph, so the worker is terminated and the call rejects.
|
|
64
|
+
const semantic = "<?xml version=\"1.0\"?><bpmn:definitions xmlns:bpmn=\"http://www.omg.org/spec/BPMN/20100524/MODEL\" id=\"d\"><bpmn:process id=\"p\" isExecutable=\"true\"><bpmn:startEvent id=\"s\"/></bpmn:process></bpmn:definitions>";
|
|
65
|
+
const err = await assertRejects(() => layoutBpmnOffThread(semantic, 1));
|
|
66
|
+
assertStringIncludes(err.message, "854");
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("#854 layoutTimeoutMs honours the env knob and ignores garbage", () => {
|
|
70
|
+
assert(layoutTimeoutMs({}) === 300000, "unset → registered default");
|
|
71
|
+
assert(layoutTimeoutMs({ NANO_DELIVERY_LAYOUT_TIMEOUT_MS: "90000" }) === 90000, "valid override honoured");
|
|
72
|
+
assert(layoutTimeoutMs({ NANO_DELIVERY_LAYOUT_TIMEOUT_MS: "nope" }) === 300000, "garbage → default");
|
|
73
|
+
assert(layoutTimeoutMs({ NANO_DELIVERY_LAYOUT_TIMEOUT_MS: "-5" }) === 300000, "non-positive → default");
|
|
74
|
+
// Node's setTimeout ceiling is a signed 32-bit ms value (2_147_483_647); a larger delay wraps to
|
|
75
|
+
// 1ms and would abort every layout immediately, so an overflowing override must degrade to the
|
|
76
|
+
// registered default rather than be honoured (issue #854 review).
|
|
77
|
+
assert(layoutTimeoutMs({ NANO_DELIVERY_LAYOUT_TIMEOUT_MS: "2147483647" }) === 2147483647, "ceiling value honoured");
|
|
78
|
+
assert(layoutTimeoutMs({ NANO_DELIVERY_LAYOUT_TIMEOUT_MS: "2147483648" }) === 300000, "above Node timer ceiling → default");
|
|
79
|
+
assert(layoutTimeoutMs({ NANO_DELIVERY_LAYOUT_TIMEOUT_MS: "999999999999" }) === 300000, "far above ceiling → default");
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("#854 layoutMaxConcurrency honours the env knob and degrades a bad bound to the default", () => {
|
|
83
|
+
assert(layoutMaxConcurrency({}) === 2, "unset → registered default");
|
|
84
|
+
assert(layoutMaxConcurrency({ NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY: "4" }) === 4, "valid override honoured");
|
|
85
|
+
assert(layoutMaxConcurrency({ NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY: "nope" }) === 2, "garbage → default");
|
|
86
|
+
assert(layoutMaxConcurrency({ NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY: "0" }) === 2, "zero (unbounded-by-stall) → default");
|
|
87
|
+
assert(layoutMaxConcurrency({ NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY: "-3" }) === 2, "negative → default");
|
|
88
|
+
// A fractional bound is not a valid holder count; it must degrade rather than round silently.
|
|
89
|
+
assert(layoutMaxConcurrency({ NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY: "1.5" }) === 2, "non-integer → default");
|
|
90
|
+
// An absurdly large bound is effectively "unbounded", which defeats the guard — degrade to default.
|
|
91
|
+
assert(layoutMaxConcurrency({ NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY: "99999" }) === 2, "above ceiling → default");
|
|
92
|
+
assert(layoutMaxConcurrency({ NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY: "1024" }) === 1024, "ceiling value honoured");
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("#854 Semaphore rejects a non-positive / non-integer bound (the guard must never be 0/∞)", () => {
|
|
96
|
+
for (const bad of [0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY]) {
|
|
97
|
+
let threw = false;
|
|
98
|
+
try {
|
|
99
|
+
new Semaphore(bad);
|
|
100
|
+
} catch {
|
|
101
|
+
threw = true;
|
|
102
|
+
}
|
|
103
|
+
assert(threw, `Semaphore(${bad}) should throw — a bad bound would silently defeat the concurrency guard`);
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("#854 Semaphore bounds concurrency to `max` and admits waiters in FIFO order", async () => {
|
|
108
|
+
const sem = new Semaphore(2);
|
|
109
|
+
let active = 0;
|
|
110
|
+
let peak = 0;
|
|
111
|
+
const order: number[] = [];
|
|
112
|
+
|
|
113
|
+
// A holder records its admission order and the live peak, then yields a few microtasks before
|
|
114
|
+
// releasing so later acquirers genuinely have to queue behind the first `max`.
|
|
115
|
+
const run = async (id: number): Promise<void> => {
|
|
116
|
+
const release = await sem.acquire();
|
|
117
|
+
order.push(id);
|
|
118
|
+
active++;
|
|
119
|
+
peak = Math.max(peak, active);
|
|
120
|
+
try {
|
|
121
|
+
for (let i = 0; i < 5; i++) await Promise.resolve();
|
|
122
|
+
} finally {
|
|
123
|
+
active--;
|
|
124
|
+
release();
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
await Promise.all([run(1), run(2), run(3), run(4), run(5)]);
|
|
129
|
+
|
|
130
|
+
assert(peak <= 2, `at most 2 holders may run at once, saw ${peak}`);
|
|
131
|
+
// First two get the free slots immediately; 3,4,5 queue and are admitted in arrival order.
|
|
132
|
+
assert(JSON.stringify(order) === JSON.stringify([1, 2, 3, 4, 5]), `expected FIFO admission, got ${order.join(",")}`);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
test("#854 Semaphore release is idempotent — a double release cannot over-admit", async () => {
|
|
136
|
+
const sem = new Semaphore(1);
|
|
137
|
+
const release = await sem.acquire();
|
|
138
|
+
release();
|
|
139
|
+
release(); // must be a no-op, not a second freed slot
|
|
140
|
+
|
|
141
|
+
// With max=1, exactly one holder may be active. Acquire twice without releasing the first; the
|
|
142
|
+
// second must still be waiting (never admitted by the stray double release above).
|
|
143
|
+
const first = await sem.acquire();
|
|
144
|
+
let secondAdmitted = false;
|
|
145
|
+
const second = sem.acquire().then((rel) => {
|
|
146
|
+
secondAdmitted = true;
|
|
147
|
+
return rel;
|
|
148
|
+
});
|
|
149
|
+
for (let i = 0; i < 10; i++) await Promise.resolve();
|
|
150
|
+
assert(!secondAdmitted, "the second acquire must block — a double release must not have over-admitted");
|
|
151
|
+
first();
|
|
152
|
+
(await second)();
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test("#854 a holder that releases in `finally` frees its slot even when its work throws", async () => {
|
|
156
|
+
// This is the invariant `layoutBpmnOffThread` relies on: it `release()`s in a `finally`, so a
|
|
157
|
+
// rejecting/terminated layout returns its slot and the next queued layout proceeds instead of the
|
|
158
|
+
// gate wedging forever behind the failure. Prove it at max=1: a throwing holder that releases in
|
|
159
|
+
// `finally` must still admit the waiter behind it.
|
|
160
|
+
const sem = new Semaphore(1);
|
|
161
|
+
const failing = (async () => {
|
|
162
|
+
const release = await sem.acquire();
|
|
163
|
+
try {
|
|
164
|
+
throw new Error("layout blew up");
|
|
165
|
+
} finally {
|
|
166
|
+
release();
|
|
167
|
+
}
|
|
168
|
+
})();
|
|
169
|
+
await assertRejects(() => failing);
|
|
170
|
+
|
|
171
|
+
let nextAdmitted = false;
|
|
172
|
+
const next = sem.acquire().then((rel) => {
|
|
173
|
+
nextAdmitted = true;
|
|
174
|
+
rel();
|
|
175
|
+
});
|
|
176
|
+
for (let i = 0; i < 10; i++) await Promise.resolve();
|
|
177
|
+
assert(nextAdmitted, "the slot held by a throwing holder must be freed, admitting the next layout");
|
|
178
|
+
await next;
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test("#854 a synchronous worker-construction failure releases the slot instead of wedging the gate", async () => {
|
|
182
|
+
// Regression for the review finding: `new Worker(...)` can throw synchronously (e.g.
|
|
183
|
+
// `ERR_WORKER_INIT_FAILED` under resource pressure). The slot is acquired BEFORE construction, so if
|
|
184
|
+
// construction isn't inside the release `finally`, each failure permanently consumes a slot and —
|
|
185
|
+
// after `layoutMaxConcurrency()` failures — every later layout queues on the shared singleton gate
|
|
186
|
+
// FOREVER. Force construction to throw more times than the bound: with the leak, the (max+1)-th call
|
|
187
|
+
// hangs on `acquire()` and never reaches the throwing factory; with the fix, every call rejects
|
|
188
|
+
// promptly. A per-call watchdog turns a wedge into a clean assertion failure rather than a suite hang.
|
|
189
|
+
const spawnThrows = (): never => {
|
|
190
|
+
throw new Error("ERR_WORKER_INIT_FAILED (simulated)");
|
|
191
|
+
};
|
|
192
|
+
const calls = layoutMaxConcurrency() + 2; // more than the bound, so a leak would exhaust the gate
|
|
193
|
+
for (let i = 0; i < calls; i++) {
|
|
194
|
+
const watchdog = new Promise<"hung">((resolve) => {
|
|
195
|
+
const t = setTimeout(() => resolve("hung"), 2000);
|
|
196
|
+
t.unref?.();
|
|
197
|
+
});
|
|
198
|
+
const attempt = layoutBpmnOffThread("<x/>", 60000, spawnThrows).then(
|
|
199
|
+
() => "resolved" as const,
|
|
200
|
+
() => "rejected" as const,
|
|
201
|
+
);
|
|
202
|
+
const outcome = await Promise.race([attempt, watchdog]);
|
|
203
|
+
assert(
|
|
204
|
+
outcome === "rejected",
|
|
205
|
+
`construction-failure call #${i + 1} must reject and free its slot, not wedge the gate (got "${outcome}")`,
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
});
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// app/layoutOffThread.ts — the ONE canonical off-main-thread bridge to the CPU-bound BPMN autolayout
|
|
2
|
+
// (`layoutBpmn` → `bpmn-auto-layout`), issue #854.
|
|
3
|
+
//
|
|
4
|
+
// WHY: `layoutBpmn` is superlinear in node/edge count and, run inline on the event loop, blocked the
|
|
5
|
+
// whole app (no HTTP, no poll passes) for the entire layout — ~52s on merlin, >4min under load on a
|
|
6
|
+
// 54-node delivery graph, a hard lock-up the operator had to kill mid-launch (#852). This module runs
|
|
7
|
+
// the layout in a `node:worker_threads` worker so the main loop keeps ticking, and BOUNDS it with a
|
|
8
|
+
// timeout that TERMINATES the worker on expiry so a pathological graph fails the launch CLEANLY
|
|
9
|
+
// instead of hanging forever.
|
|
10
|
+
//
|
|
11
|
+
// `layoutDeliveryDiagram` (app/deliveryGraphCompiler.ts) — the single layout entry point BOTH the
|
|
12
|
+
// dispatch path (`dispatchDeliveryGraphRun` → `compileDeliveryGraph`) and the preview path
|
|
13
|
+
// (`previewProposalBpmn` → `compileDeliveryGraph`) funnel through — calls `layoutBpmnOffThread` here,
|
|
14
|
+
// so there is exactly ONE place the autolayout runs and exactly ONE timeout/termination policy.
|
|
15
|
+
|
|
16
|
+
import { Worker } from "node:worker_threads";
|
|
17
|
+
import { readEnvOr } from "./contracts.ts";
|
|
18
|
+
|
|
19
|
+
/** Node's `setTimeout` delay ceiling (a signed 32-bit ms value). A delay above this silently wraps to
|
|
20
|
+
* 1ms (emitting `TimeoutOverflowWarning`), so an over-large override would abort layouts IMMEDIATELY
|
|
21
|
+
* instead of extending the bound — a value that large is treated as garbage and degrades to the
|
|
22
|
+
* registered default. */
|
|
23
|
+
const MAX_TIMER_MS = 2_147_483_647;
|
|
24
|
+
|
|
25
|
+
/** The registered default, resolved once from the schema (empty env ⇒ registry `default`) so the
|
|
26
|
+
* invalid-override fallback and the timeout message stay single-sourced — not a literal `300000`
|
|
27
|
+
* duplicated here that would drift from `ENV_CONTRACTS`. Mirrors `cloneTimeoutMs` in
|
|
28
|
+
* `app/repoEnvelope.ts`. */
|
|
29
|
+
const envDefaultLayoutTimeoutMs = Number(readEnvOr("NANO_DELIVERY_LAYOUT_TIMEOUT_MS", "300000", {}));
|
|
30
|
+
|
|
31
|
+
/** Resolve the layout timeout (ms) from `NANO_DELIVERY_LAYOUT_TIMEOUT_MS` (registered default 300000
|
|
32
|
+
* = 5 min), degrading to the registered default for any value that is non-positive, garbage, or above
|
|
33
|
+
* Node's timer ceiling ({@link MAX_TIMER_MS} — an overflow would wrap to 1ms and abort every layout).
|
|
34
|
+
* Read per call so an operator can retune it without a restart; the layout is rare (dispatch/preview),
|
|
35
|
+
* so the lookup cost is irrelevant. */
|
|
36
|
+
export function layoutTimeoutMs(env: Record<string, string | undefined> = process.env): number {
|
|
37
|
+
const raw = Number(readEnvOr("NANO_DELIVERY_LAYOUT_TIMEOUT_MS", "300000", env));
|
|
38
|
+
return Number.isFinite(raw) && raw > 0 && raw <= MAX_TIMER_MS ? raw : envDefaultLayoutTimeoutMs;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** A sanity ceiling on the concurrency bound: a value this large is effectively unbounded and is
|
|
42
|
+
* treated as garbage (degrading to the registered default) rather than honoured — the whole point of
|
|
43
|
+
* the bound is to keep concurrent CPU-bound layouts SMALL. */
|
|
44
|
+
const MAX_LAYOUT_CONCURRENCY = 1024;
|
|
45
|
+
|
|
46
|
+
/** The registered default concurrency, resolved once from the schema (empty env ⇒ registry `default`)
|
|
47
|
+
* so the invalid-override fallback stays single-sourced — not a literal duplicated here that would
|
|
48
|
+
* drift from `ENV_CONTRACTS`. Mirrors {@link envDefaultLayoutTimeoutMs}. */
|
|
49
|
+
const envDefaultLayoutMaxConcurrency = Number(readEnvOr("NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY", "2", {}));
|
|
50
|
+
|
|
51
|
+
/** Resolve the max number of concurrent layout workers from `NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY`
|
|
52
|
+
* (registered default 2), degrading to the registered default for any value that is non-integer,
|
|
53
|
+
* non-positive, garbage, or above {@link MAX_LAYOUT_CONCURRENCY}. Mirrors {@link layoutTimeoutMs}'s
|
|
54
|
+
* fail-safe parsing so a bad override can never make the bound 0/∞ (either of which defeats it). */
|
|
55
|
+
export function layoutMaxConcurrency(env: Record<string, string | undefined> = process.env): number {
|
|
56
|
+
const raw = Number(readEnvOr("NANO_DELIVERY_LAYOUT_MAX_CONCURRENCY", "2", env));
|
|
57
|
+
return Number.isInteger(raw) && raw > 0 && raw <= MAX_LAYOUT_CONCURRENCY ? raw : envDefaultLayoutMaxConcurrency;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** A minimal FIFO counting semaphore: at most `max` holders run concurrently, the rest queue in
|
|
61
|
+
* arrival order. Bounds how many CPU-bound layout workers exist at once so a burst of overlapping
|
|
62
|
+
* preview/dispatch layouts cannot spawn an UNBOUNDED number of superlinear workers and saturate
|
|
63
|
+
* CPU/memory — the very starvation this off-thread move exists to prevent (issue #854). Each
|
|
64
|
+
* `acquire()` resolves with an idempotent `release` that hands the freed slot to the next waiter. */
|
|
65
|
+
export class Semaphore {
|
|
66
|
+
private active = 0;
|
|
67
|
+
private readonly waiters: Array<() => void> = [];
|
|
68
|
+
private readonly max: number;
|
|
69
|
+
|
|
70
|
+
constructor(max: number) {
|
|
71
|
+
if (!Number.isInteger(max) || max < 1) {
|
|
72
|
+
throw new Error(`Semaphore: max must be a positive integer, got ${String(max)}`);
|
|
73
|
+
}
|
|
74
|
+
this.max = max;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Wait (FIFO) for a slot, then return a one-shot `release`. Always `release()` in a `finally` so a
|
|
78
|
+
* rejecting holder cannot leak its slot and wedge every queued layout behind it. */
|
|
79
|
+
async acquire(): Promise<() => void> {
|
|
80
|
+
await new Promise<void>((resolve) => {
|
|
81
|
+
if (this.active < this.max) {
|
|
82
|
+
this.active++;
|
|
83
|
+
resolve();
|
|
84
|
+
} else {
|
|
85
|
+
this.waiters.push(() => {
|
|
86
|
+
this.active++;
|
|
87
|
+
resolve();
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
let released = false;
|
|
92
|
+
return () => {
|
|
93
|
+
if (released) return; // idempotent: a double release must not free a slot it never held
|
|
94
|
+
released = true;
|
|
95
|
+
this.active--;
|
|
96
|
+
this.waiters.shift()?.();
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** The ONE shared gate bounding concurrent layout workers, sized lazily on first use from
|
|
102
|
+
* {@link layoutMaxConcurrency}. A singleton so EVERY `layoutBpmnOffThread` call — across both the
|
|
103
|
+
* dispatch and preview paths — contends for the same small pool of slots. */
|
|
104
|
+
let layoutGate: Semaphore | undefined;
|
|
105
|
+
function layoutConcurrencyGate(): Semaphore {
|
|
106
|
+
if (!layoutGate) layoutGate = new Semaphore(layoutMaxConcurrency());
|
|
107
|
+
return layoutGate;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The worker message shape: a laid-out XML payload on success, or a server-side error string. */
|
|
111
|
+
type LayoutWorkerMessage = { ok: true; laidOut: string } | { ok: false; error: string };
|
|
112
|
+
|
|
113
|
+
/** Run `layoutBpmn(semanticBpmn)` in a dedicated `worker_threads` worker, OFF the main event loop,
|
|
114
|
+
* bounded by `timeoutMs`. Resolves with the laid-out XML; rejects (and TERMINATES the worker — the
|
|
115
|
+
* bound that stops a runaway layout) on timeout, a worker error, an early exit, or a layout failure
|
|
116
|
+
* reported by the worker. The worker is always torn down before this resolves/rejects, so no isolate
|
|
117
|
+
* leaks across calls. At most {@link layoutMaxConcurrency} workers run at once (the shared
|
|
118
|
+
* {@link Semaphore} gate) — a burst of overlapping preview/dispatch layouts queues for a slot rather
|
|
119
|
+
* than spawning an unbounded number of superlinear CPU-bound workers that would saturate the host.
|
|
120
|
+
* The timeout covers only the ACTUAL layout (it starts after a slot is acquired), so queueing behind
|
|
121
|
+
* other layouts is never charged against a graph's bound.
|
|
122
|
+
*
|
|
123
|
+
* `spawnWorker` is an injectable seam (default: the real `layoutWorker.ts` worker) so a test can force
|
|
124
|
+
* a SYNCHRONOUS construction failure — `new Worker(...)` can throw (e.g. `ERR_WORKER_INIT_FAILED`
|
|
125
|
+
* under resource pressure) — and prove the slot is still released (see below). */
|
|
126
|
+
export async function layoutBpmnOffThread(
|
|
127
|
+
semanticBpmn: string,
|
|
128
|
+
timeoutMs: number = layoutTimeoutMs(),
|
|
129
|
+
spawnWorker: () => Worker = () =>
|
|
130
|
+
new Worker(new URL("./layoutWorker.ts", import.meta.url), { workerData: { semanticBpmn } }),
|
|
131
|
+
): Promise<string> {
|
|
132
|
+
// Acquire the slot, then wrap EVERYTHING — including worker construction — in the release `finally`.
|
|
133
|
+
// `new Worker(...)` can throw synchronously (e.g. `ERR_WORKER_INIT_FAILED` under resource pressure);
|
|
134
|
+
// if construction sat outside this guard, each such failure would permanently consume a slot and,
|
|
135
|
+
// after `max` failures, wedge every later layout waiting on the gate forever (issue #854 review).
|
|
136
|
+
const release = await layoutConcurrencyGate().acquire();
|
|
137
|
+
try {
|
|
138
|
+
const worker = spawnWorker();
|
|
139
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
140
|
+
try {
|
|
141
|
+
return await new Promise<string>((resolve, reject) => {
|
|
142
|
+
let settled = false;
|
|
143
|
+
const settle = (fn: () => void) => {
|
|
144
|
+
if (settled) return;
|
|
145
|
+
settled = true;
|
|
146
|
+
if (timer) clearTimeout(timer);
|
|
147
|
+
fn();
|
|
148
|
+
};
|
|
149
|
+
timer = setTimeout(() => {
|
|
150
|
+
settle(() =>
|
|
151
|
+
reject(
|
|
152
|
+
new Error(
|
|
153
|
+
`layoutDeliveryDiagram: BPMN autolayout exceeded its ${timeoutMs}ms bound and was aborted — the ` +
|
|
154
|
+
"delivery graph is too large/dense to lay out within the timeout (bpmn-auto-layout is superlinear in " +
|
|
155
|
+
"node/edge count, issue #854). The launch was failed cleanly rather than hung; retune " +
|
|
156
|
+
`\`NANO_DELIVERY_LAYOUT_TIMEOUT_MS\` (default ${envDefaultLayoutTimeoutMs}) or shrink the graph.`,
|
|
157
|
+
),
|
|
158
|
+
),
|
|
159
|
+
);
|
|
160
|
+
}, timeoutMs);
|
|
161
|
+
// A setTimeout in a hot dispatch path should not itself keep a draining process alive.
|
|
162
|
+
timer.unref?.();
|
|
163
|
+
worker.once("message", (msg: LayoutWorkerMessage) => {
|
|
164
|
+
settle(() => {
|
|
165
|
+
if (msg.ok) resolve(msg.laidOut);
|
|
166
|
+
else reject(new Error(msg.error));
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
worker.once("error", (err) => settle(() => reject(err)));
|
|
170
|
+
worker.once("exit", (code) => {
|
|
171
|
+
if (code === 0) return; // a clean exit after a delivered message is expected on teardown
|
|
172
|
+
settle(() => reject(new Error(`layoutDeliveryDiagram: layout worker exited early with code ${code}`)));
|
|
173
|
+
});
|
|
174
|
+
});
|
|
175
|
+
} finally {
|
|
176
|
+
if (timer) clearTimeout(timer);
|
|
177
|
+
await worker.terminate();
|
|
178
|
+
}
|
|
179
|
+
} finally {
|
|
180
|
+
release(); // hand our slot to the next queued layout — ALWAYS, even on a construction throw/reject/terminate
|
|
181
|
+
}
|
|
182
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// app/layoutWorker.ts — the `node:worker_threads` entry that runs the CPU-bound BPMN autolayout
|
|
2
|
+
// (`layoutBpmn` → `bpmn-auto-layout`) OFF the app's main event loop (issue #854).
|
|
3
|
+
//
|
|
4
|
+
// `bpmn-auto-layout` is superlinear in node/edge count: on the 54-node merlin delivery graph it ran
|
|
5
|
+
// ~52s on merlin and >4min under real load, and because it ran SYNCHRONOUSLY on the event loop in
|
|
6
|
+
// `layoutDeliveryDiagram`, the whole app served no HTTP and ran no poll passes for that entire window
|
|
7
|
+
// — the operator saw a hard lock-up (#852). Hoisting the layout into this worker keeps the main loop
|
|
8
|
+
// free to serve pages/API/MCP and tick its poll loop while a large graph lays out; the parent bounds
|
|
9
|
+
// the work with a timeout and TERMINATES this worker on expiry (see `app/layoutOffThread.ts`), so a
|
|
10
|
+
// pathological graph fails the launch cleanly instead of hanging forever.
|
|
11
|
+
//
|
|
12
|
+
// Protocol: `workerData.semanticBpmn` carries the pre-layout semantic BPMN; we post back either the
|
|
13
|
+
// laid-out XML (a string) on success or `{ error }` on failure. We never `process.exit` — resolving
|
|
14
|
+
// the message lets the parent `terminate()` us.
|
|
15
|
+
|
|
16
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
17
|
+
import { layoutBpmn } from "@nanobpm/urban";
|
|
18
|
+
|
|
19
|
+
async function run(): Promise<void> {
|
|
20
|
+
const port = parentPort;
|
|
21
|
+
if (!port) throw new Error("layoutWorker: must be spawned as a worker_threads worker (no parentPort)");
|
|
22
|
+
const { semanticBpmn } = workerData ?? {};
|
|
23
|
+
if (typeof semanticBpmn !== "string") {
|
|
24
|
+
throw new Error("layoutWorker: workerData.semanticBpmn must be a string");
|
|
25
|
+
}
|
|
26
|
+
try {
|
|
27
|
+
const laidOut = await layoutBpmn(semanticBpmn);
|
|
28
|
+
port.postMessage({ ok: true, laidOut });
|
|
29
|
+
} catch (err) {
|
|
30
|
+
port.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
void run();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nanobpm/nano-workforce",
|
|
3
|
-
"version": "0.200.
|
|
3
|
+
"version": "0.200.1",
|
|
4
4
|
"description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "main.ts",
|