@nanobpm/nano-workforce 0.200.5 → 0.200.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/app/deliveryGraphDispatch.test.ts +177 -11
- package/app/deliveryGraphDispatch.ts +72 -3
- package/app/deliveryGraphDispatchLayoutOnce.test.ts +13 -0
- package/app/deliveryGraphRun.test.ts +324 -1
- package/app/deliveryGraphRun.ts +194 -3
- package/app/deliveryRunner.test.ts +22 -0
- package/app/deliveryRunner.ts +15 -3
- package/app/service.ts +52 -3
- package/docs/agent-guide.md +9 -0
- package/operations/dispatchDeliveryGraph.test.ts +23 -1
- package/package.json +2 -2
- package/pages/delivery-graphs/staged.mount.js +17 -6
- package/test/delivery-graphs-staged-slow-dispatch.test.ts +85 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
## [0.200.7](https://github.com/nanobpm/nano-workforce/compare/v0.200.6...v0.200.7) (2026-10-05)
|
|
2
|
+
|
|
3
|
+
### Documentation
|
|
4
|
+
|
|
5
|
+
* **agent-guide:** document the feature-run start endpoint ([#874](https://github.com/nanobpm/nano-workforce/issues/874)) ([faa01f2](https://github.com/nanobpm/nano-workforce/commit/faa01f28da943dcd5fd8695cb18bd8054d08c3f1))
|
|
6
|
+
|
|
7
|
+
## [0.200.6](https://github.com/nanobpm/nano-workforce/compare/v0.200.5...v0.200.6) (2026-10-04)
|
|
8
|
+
|
|
9
|
+
### Bug Fixes
|
|
10
|
+
|
|
11
|
+
* **delivery-graph:** dispatch large graphs without UI timeout or a stranded phantom run ([#853](https://github.com/nanobpm/nano-workforce/issues/853)) ([d1adc47](https://github.com/nanobpm/nano-workforce/commit/d1adc4752690a91cb764ec3a49015f5a46ac7378)), closes [#852](https://github.com/nanobpm/nano-workforce/issues/852) [#852](https://github.com/nanobpm/nano-workforce/issues/852)
|
|
12
|
+
|
|
1
13
|
## [0.200.5](https://github.com/nanobpm/nano-workforce/compare/v0.200.4...v0.200.5) (2026-10-04)
|
|
2
14
|
|
|
3
15
|
### Bug Fixes
|
|
@@ -9,11 +9,13 @@ import { test } from "node:test";
|
|
|
9
9
|
import { assert, assertEquals, assertRejects } from "#test-assert";
|
|
10
10
|
import type { AppApi } from "@nanobpm/urban";
|
|
11
11
|
import { dispatchDeliveryGraphRun } from "./deliveryGraphDispatch.ts";
|
|
12
|
+
import { isStaleLaunchClaim } from "./deliveryGraphRun.ts";
|
|
12
13
|
import { noopLog } from "../test/log.ts";
|
|
13
14
|
|
|
14
|
-
function makeApp(opts: { failIdentityWrite?: boolean } = {}) {
|
|
15
|
+
function makeApp(opts: { failIdentityWrite?: boolean; reconcileReadThrows?: { on: boolean }; liveOriginal?: { processInstanceKey: string; runKey: string | null; processDefinitionId?: string } } = {}) {
|
|
15
16
|
const tables = new Map<string, Record<string, unknown>[]>();
|
|
16
|
-
const started: { processDefinitionId: string }[] = [];
|
|
17
|
+
const started: { processDefinitionId: string; variables?: Record<string, unknown> }[] = [];
|
|
18
|
+
const cancelled: string[] = [];
|
|
17
19
|
const table = (name: string, key: string) => {
|
|
18
20
|
if (opts.failIdentityWrite && name === "delivery_graph_run_identity") {
|
|
19
21
|
// Simulate a transient side-table write error (e.g. a SQLite `disk I/O error`) — NOT a UNIQUE
|
|
@@ -61,31 +63,67 @@ function makeApp(opts: { failIdentityWrite?: boolean } = {}) {
|
|
|
61
63
|
const app = {
|
|
62
64
|
data: {
|
|
63
65
|
table,
|
|
64
|
-
open: () =>
|
|
65
|
-
exec
|
|
66
|
+
open: () => {
|
|
67
|
+
const exec = (sql: string, params: unknown[]) =>
|
|
66
68
|
Promise.resolve().then(() => {
|
|
67
69
|
const cols = [...sql.matchAll(/"(\w+)"\s*=\s*\?/g)].map((m) => m[1]);
|
|
68
|
-
|
|
70
|
+
// The claim CAS (claimRunForLaunch) ends `… WHERE "run_key" = ? AND ("status" <> 'running'
|
|
71
|
+
// OR ("process_key" IS NULL AND "updated_at" < ?))` — the run_key is the 2nd-to-last param,
|
|
72
|
+
// the stale-TTL threshold the last. Model BOTH branches so a stale re-claim wins.
|
|
73
|
+
const isClaimCas = sql.includes('OR ("process_key" IS NULL AND "updated_at" < ?)');
|
|
74
|
+
const runKey = isClaimCas ? params[params.length - 2] : params[params.length - 1];
|
|
75
|
+
const staleBefore = isClaimCas ? (params[params.length - 1] as string) : null;
|
|
69
76
|
const rows = tables.get("delivery_graph_runs") ?? [];
|
|
70
77
|
const row = rows.find((r) => r["run_key"] === runKey);
|
|
71
|
-
|
|
78
|
+
const staleReclaimable =
|
|
79
|
+
isClaimCas && row && row["status"] === "running" && (row["process_key"] == null) && typeof staleBefore === "string" && (row["updated_at"] as string) < staleBefore;
|
|
80
|
+
if (row && (row["status"] !== "running" || staleReclaimable)) {
|
|
72
81
|
for (let i = 0; i < cols.length - 1; i++) row[cols[i]] = params[i];
|
|
73
82
|
return { changed: 1 };
|
|
74
83
|
}
|
|
75
84
|
return { changed: 0 };
|
|
76
|
-
})
|
|
77
|
-
|
|
85
|
+
});
|
|
86
|
+
// The claim runs inside a transaction (`data.open().tx(...)`); model `tx` as a pass-through
|
|
87
|
+
// that hands the same `exec` to the callback (the mock has no real isolation to model).
|
|
88
|
+
return { exec, tx: async (fn: (t: { exec: typeof exec }) => Promise<unknown>) => fn({ exec }) };
|
|
89
|
+
},
|
|
78
90
|
},
|
|
79
91
|
engine: {
|
|
80
92
|
deployResources: () => Promise.resolve([]),
|
|
81
|
-
createInstance: (req: { processDefinitionId: string }) => {
|
|
93
|
+
createInstance: (req: { processDefinitionId: string; variables?: Record<string, unknown> }) => {
|
|
82
94
|
started.push(req);
|
|
83
|
-
return Promise.resolve({ processInstanceKey:
|
|
95
|
+
return Promise.resolve({ processInstanceKey: `PI-${started.length}`, processDefinitionId: req.processDefinitionId });
|
|
96
|
+
},
|
|
97
|
+
// The reconcile-before-relaunch seam (issue #852 review): surface a configured LIVE original
|
|
98
|
+
// instance (a dispatch that died after createInstance but before stamping the key) so the test can
|
|
99
|
+
// assert the relaunch cancels it. `liveOriginal.runKey === null` models an instance whose runKey
|
|
100
|
+
// variable is absent/unreadable — it must NOT be cancelled (unproven ownership).
|
|
101
|
+
searchProcessInstances: (filter?: { processDefinitionId?: string; state?: string }) =>
|
|
102
|
+
// A reconcile READ failure (issue #853 review — thread deliveryGraphDispatch.ts:280): the
|
|
103
|
+
// reconcile pass queries by `processDefinitionId`, so gate the transient throw on that filter.
|
|
104
|
+
opts.reconcileReadThrows?.on && filter?.processDefinitionId !== undefined
|
|
105
|
+
? Promise.reject(new Error("engine unavailable"))
|
|
106
|
+
: Promise.resolve(
|
|
107
|
+
opts.liveOriginal &&
|
|
108
|
+
(filter?.processDefinitionId === undefined || filter.processDefinitionId === (opts.liveOriginal.processDefinitionId ?? filter.processDefinitionId)) &&
|
|
109
|
+
(filter?.state === undefined || filter.state === "ACTIVE")
|
|
110
|
+
? [{ processInstanceKey: opts.liveOriginal.processInstanceKey, state: "ACTIVE" }]
|
|
111
|
+
: [],
|
|
112
|
+
),
|
|
113
|
+
searchVariables: (filter?: { processInstanceKey?: string; name?: string }) =>
|
|
114
|
+
Promise.resolve(
|
|
115
|
+
opts.liveOriginal && filter?.name === "runKey" && filter.processInstanceKey === opts.liveOriginal.processInstanceKey && opts.liveOriginal.runKey !== null
|
|
116
|
+
? [{ variableKey: "v1", name: "runKey", value: JSON.stringify(opts.liveOriginal.runKey), scopeKey: opts.liveOriginal.processInstanceKey, processInstanceKey: opts.liveOriginal.processInstanceKey, isTruncated: false }]
|
|
117
|
+
: [],
|
|
118
|
+
),
|
|
119
|
+
cancelInstance: (req: { processInstanceKey: string }) => {
|
|
120
|
+
cancelled.push(req.processInstanceKey);
|
|
121
|
+
return Promise.resolve();
|
|
84
122
|
},
|
|
85
123
|
},
|
|
86
124
|
log: noopLog(),
|
|
87
125
|
} as unknown as AppApi;
|
|
88
|
-
return { app, started, runs: () => tables.get("delivery_graph_runs") ?? [] };
|
|
126
|
+
return { app, started, cancelled, runs: () => tables.get("delivery_graph_runs") ?? [] };
|
|
89
127
|
}
|
|
90
128
|
|
|
91
129
|
const HUMAN_ONLY = {
|
|
@@ -257,3 +295,131 @@ test("dispatchDeliveryGraphRun: two credential-differing secret graphs with DIST
|
|
|
257
295
|
assertEquals(started.length, 2);
|
|
258
296
|
assertEquals(new Set(runs().map((r) => r.run_key)).size, 2);
|
|
259
297
|
});
|
|
298
|
+
|
|
299
|
+
// ── #852 review: reconcile-before-relaunch (thread deliveryGraphRun.ts:180) ──────────────────────
|
|
300
|
+
// A STALE launch claim can hide a LIVE original instance (the dispatch died after createInstance but
|
|
301
|
+
// before stamping the process key). The relaunch must cancel that original — matched by its seeded
|
|
302
|
+
// `runKey` variable — before starting a second instance of the same side-effecting graph.
|
|
303
|
+
|
|
304
|
+
/** Seed a STALE launch-claim row (running, NULL process_key, updated_at older than the TTL) straight
|
|
305
|
+
* into the mock store, as a crashed dispatch left it. */
|
|
306
|
+
async function seedStaleClaim(app: AppApi, runKey: string, digest: string) {
|
|
307
|
+
const staleAt = new Date(Date.now() - 20 * 60_000).toISOString(); // 20 min ago > 15 min TTL
|
|
308
|
+
await app.data.table("delivery_graph_runs", "run_key").insert({
|
|
309
|
+
run_key: runKey,
|
|
310
|
+
process_key: null,
|
|
311
|
+
process_definition_id: `delivery-graph-${digest}`,
|
|
312
|
+
digest,
|
|
313
|
+
status: "running",
|
|
314
|
+
side_effecting: 1,
|
|
315
|
+
node_count: 2,
|
|
316
|
+
human_node_count: 1,
|
|
317
|
+
side_effect_count: 1,
|
|
318
|
+
title: "t",
|
|
319
|
+
phase: "Running",
|
|
320
|
+
phase_node_id: null,
|
|
321
|
+
human_labels: null,
|
|
322
|
+
created_at: staleAt,
|
|
323
|
+
updated_at: staleAt,
|
|
324
|
+
acknowledged_at: null,
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
test("#852 dispatchDeliveryGraphRun: re-claiming a STALE claim with a LIVE original instance cancels it before relaunch (at-most-once)", async () => {
|
|
329
|
+
// The original dispatch died after createInstance but before stamping the key: the row is a stale
|
|
330
|
+
// claim AND a live instance of this run (runKey "run-x") is still ACTIVE on the engine.
|
|
331
|
+
const { app, started, cancelled } = makeApp({ liveOriginal: { processInstanceKey: "PI-orig", runKey: "run-x" } });
|
|
332
|
+
// First dispatch to learn the digest, on a SEPARATE app so it doesn't disturb the test store.
|
|
333
|
+
const probe = await dispatchDeliveryGraphRun(makeApp().app, SIDE_EFFECTING, { runKey: "run-x", repoless: true });
|
|
334
|
+
assert(probe.ok);
|
|
335
|
+
if (!probe.ok) return;
|
|
336
|
+
await seedStaleClaim(app, "run-x", probe.digest);
|
|
337
|
+
|
|
338
|
+
const res = await dispatchDeliveryGraphRun(app, SIDE_EFFECTING, { runKey: "run-x", repoless: true });
|
|
339
|
+
assertEquals(res.ok, true);
|
|
340
|
+
if (!res.ok) return;
|
|
341
|
+
assertEquals(res.alreadyRunning, false, "the stale claim is re-claimed, not short-circuited");
|
|
342
|
+
assertEquals(cancelled, ["PI-orig"], "the live original instance is cancelled before the relaunch");
|
|
343
|
+
assertEquals(started.length, 1, "exactly one replacement instance launches");
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
test("#852 dispatchDeliveryGraphRun: re-claiming a STALE claim whose only ACTIVE instance is a DIFFERENT run relaunches WITHOUT cancelling it", async () => {
|
|
347
|
+
// A different run of the SAME graph (same content-addressed processDefinitionId) is still ACTIVE.
|
|
348
|
+
// The relaunch must NOT cancel it — the runKey variable proves it is not ours.
|
|
349
|
+
const { app, started, cancelled } = makeApp({ liveOriginal: { processInstanceKey: "PI-other", runKey: "other-run" } });
|
|
350
|
+
const probe = await dispatchDeliveryGraphRun(makeApp().app, SIDE_EFFECTING, { runKey: "run-y", repoless: true });
|
|
351
|
+
assert(probe.ok);
|
|
352
|
+
if (!probe.ok) return;
|
|
353
|
+
await seedStaleClaim(app, "run-y", probe.digest);
|
|
354
|
+
|
|
355
|
+
const res = await dispatchDeliveryGraphRun(app, SIDE_EFFECTING, { runKey: "run-y", repoless: true });
|
|
356
|
+
assertEquals(res.ok, true);
|
|
357
|
+
if (!res.ok) return;
|
|
358
|
+
assertEquals(cancelled, [], "a different run's live instance is never cancelled");
|
|
359
|
+
assertEquals(started.length, 1, "the relaunch still proceeds");
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
test("#852 dispatchDeliveryGraphRun: an explicit-key stale re-claim whose graph DRIFTED still cancels the original (matched by the STALE ROW's digest, not the new one)", async () => {
|
|
363
|
+
// Under an explicit idempotencyKey the re-dispatched graph can differ from the original, so the
|
|
364
|
+
// freshly-compiled digest (d2) diverges from the stale row's digest (d1). The original live instance
|
|
365
|
+
// was deployed under `delivery-graph-d1`; matching it under the NEW digest d2 would miss it and strand
|
|
366
|
+
// it live — a double-launch of side-effecting nodes. Reconcile MUST search under the stale row's own
|
|
367
|
+
// digest. Learn both digests on throwaway apps.
|
|
368
|
+
const probeOrig = await dispatchDeliveryGraphRun(makeApp().app, SIDE_EFFECTING, { runKey: "run-z", repoless: true });
|
|
369
|
+
const probeDrift = await dispatchDeliveryGraphRun(makeApp().app, HUMAN_ONLY, { runKey: "run-z", repoless: true });
|
|
370
|
+
assert(probeOrig.ok && probeDrift.ok);
|
|
371
|
+
if (!probeOrig.ok || !probeDrift.ok) return;
|
|
372
|
+
assert(probeOrig.digest !== probeDrift.digest, "the two graphs must have different digests for this test to bite");
|
|
373
|
+
|
|
374
|
+
// The live original is deployed under the ORIGINAL digest's definition id — NOT the re-dispatched one.
|
|
375
|
+
const { app, started, cancelled } = makeApp({
|
|
376
|
+
liveOriginal: { processInstanceKey: "PI-orig", runKey: "run-z", processDefinitionId: `delivery-graph-${probeOrig.digest}` },
|
|
377
|
+
});
|
|
378
|
+
await seedStaleClaim(app, "run-z", probeOrig.digest);
|
|
379
|
+
|
|
380
|
+
// Re-dispatch the DRIFTED graph under the same explicit key.
|
|
381
|
+
const res = await dispatchDeliveryGraphRun(app, HUMAN_ONLY, { runKey: "run-z", repoless: true });
|
|
382
|
+
assertEquals(res.ok, true);
|
|
383
|
+
if (!res.ok) return;
|
|
384
|
+
assertEquals(res.alreadyRunning, false, "the stale claim is re-claimed, not short-circuited");
|
|
385
|
+
assertEquals(cancelled, ["PI-orig"], "the original (under the stale row's digest) is cancelled despite the graph drift");
|
|
386
|
+
assertEquals(started.length, 1, "exactly one replacement instance launches");
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
// ── #853 review: a reconcile-read failure must leave the row RECLAIMABLE-with-reconcile, never `failed`
|
|
390
|
+
// (thread deliveryGraphDispatch.ts:280) ──────────────────────────────────────────────────────────────
|
|
391
|
+
// If the reconcile read/cancel throws TRANSIENTLY while the original instance is still live, retiring
|
|
392
|
+
// the stale claim to `failed` would let the NEXT dispatch relaunch WITHOUT reconciling (a `failed` row
|
|
393
|
+
// is a plain terminal re-run, not a stale claim), double-launching the side-effecting nodes. The claim
|
|
394
|
+
// must instead revert to a stale launch-claim so the reconcile is retried before any relaunch.
|
|
395
|
+
|
|
396
|
+
test("#853 dispatchDeliveryGraphRun: a reconcile-read failure reverts the row to a STALE claim (not `failed`), launching nothing — so the next dispatch retries reconcile instead of relaunching blind", async () => {
|
|
397
|
+
const reconcileReadThrows = { on: true };
|
|
398
|
+
const { app, started, cancelled, runs } = makeApp({ liveOriginal: { processInstanceKey: "PI-orig", runKey: "run-r" }, reconcileReadThrows });
|
|
399
|
+
const probe = await dispatchDeliveryGraphRun(makeApp().app, SIDE_EFFECTING, { runKey: "run-r", repoless: true });
|
|
400
|
+
assert(probe.ok);
|
|
401
|
+
if (!probe.ok) return;
|
|
402
|
+
await seedStaleClaim(app, "run-r", probe.digest);
|
|
403
|
+
|
|
404
|
+
// Phase 1: the reconcile read throws. The dispatch must propagate the error, launch NOTHING, and
|
|
405
|
+
// leave the row a reclaimable stale claim — NOT retire it to `failed`.
|
|
406
|
+
await assertRejects(() => dispatchDeliveryGraphRun(app, SIDE_EFFECTING, { runKey: "run-r", repoless: true }), Error, "engine unavailable");
|
|
407
|
+
const row = runs().find((r) => r["run_key"] === "run-r");
|
|
408
|
+
assert(row, "the run row still exists");
|
|
409
|
+
assertEquals(row?.["status"], "running", "the row stays a `running` claim, not `failed`");
|
|
410
|
+
assertEquals(row?.["process_key"], null, "the row stays a NULL-key launch claim (still reclaimable as stale)");
|
|
411
|
+
assert(isStaleLaunchClaim(row as Parameters<typeof isStaleLaunchClaim>[0]), "the row is still a STALE launch claim, so the reconcile is retried before any relaunch");
|
|
412
|
+
assertEquals(started.length, 0, "nothing launched while the original may still be live");
|
|
413
|
+
assertEquals(cancelled, [], "nothing cancelled — the read failed before any cancel");
|
|
414
|
+
|
|
415
|
+
// Phase 2: the engine recovers. The NEXT dispatch re-enters reconcile (because the row is still a
|
|
416
|
+
// stale claim), cancels the live original, and relaunches exactly once — proving the row was never
|
|
417
|
+
// stranded in a `failed` state that would have relaunched blind.
|
|
418
|
+
reconcileReadThrows.on = false;
|
|
419
|
+
const res = await dispatchDeliveryGraphRun(app, SIDE_EFFECTING, { runKey: "run-r", repoless: true });
|
|
420
|
+
assertEquals(res.ok, true);
|
|
421
|
+
if (!res.ok) return;
|
|
422
|
+
assertEquals(res.alreadyRunning, false, "the stale claim is re-claimed, not short-circuited");
|
|
423
|
+
assertEquals(cancelled, ["PI-orig"], "the recovered reconcile cancels the live original before relaunch");
|
|
424
|
+
assertEquals(started.length, 1, "exactly one replacement instance launches across the two dispatches");
|
|
425
|
+
});
|
|
@@ -15,7 +15,7 @@ import { createHash } from "node:crypto";
|
|
|
15
15
|
import type { AppApi } from "@nanobpm/urban";
|
|
16
16
|
import type { DeliveryGraph } from "../nano-generated/api-io.d.ts";
|
|
17
17
|
import { canonicalJson, validateDeliveryGraph } from "./deliveryGraph.ts";
|
|
18
|
-
import {
|
|
18
|
+
import { compileDeliveryGraphSemantic, DELIVERY_GRAPH_PROCESS_ID, digestInvisibleRawValues, graphCarriesRedactedSecrets } from "./deliveryGraphCompiler.ts";
|
|
19
19
|
import {
|
|
20
20
|
buildDeliveryGraphRunRow,
|
|
21
21
|
buildHumanLabels,
|
|
@@ -25,6 +25,8 @@ import {
|
|
|
25
25
|
type DeliveryGraphRunIdentity,
|
|
26
26
|
deliveryGraphRunIdentities,
|
|
27
27
|
deliveryGraphRuns,
|
|
28
|
+
isStaleLaunchClaim,
|
|
29
|
+
reconcileOriginalInstanceBeforeRelaunch,
|
|
28
30
|
} from "./deliveryGraphRun.ts";
|
|
29
31
|
import type { DeliveryRunTimeouts } from "./deliveryRunner.ts";
|
|
30
32
|
import { deliveryGraphDigest, runDeliveryGraph } from "./deliveryRunner.ts";
|
|
@@ -97,7 +99,9 @@ export async function dispatchDeliveryGraphRun(
|
|
|
97
99
|
// The graph passed the semantic validator above, so it is safe to narrow to the typed contract.
|
|
98
100
|
// biome-ignore lint/plugin: validated external body narrowed to its contract after validateDeliveryGraph
|
|
99
101
|
const typedGraph = graph as DeliveryGraph;
|
|
100
|
-
|
|
102
|
+
// Semantic compile only (#852): this needs the digest / resolved model / side effects / human stops,
|
|
103
|
+
// not the laid-out BPMN — `runDeliveryGraph` below runs the CPU-bound layout exactly once.
|
|
104
|
+
const compiled = await compileDeliveryGraphSemantic(typedGraph);
|
|
101
105
|
if (!compiled.ok) {
|
|
102
106
|
return { ok: false, errors: compiled.errors };
|
|
103
107
|
}
|
|
@@ -182,7 +186,23 @@ export async function dispatchDeliveryGraphRun(
|
|
|
182
186
|
|
|
183
187
|
// Idempotency short-circuit — a re-dispatch onto a still-running run does NOT double-launch.
|
|
184
188
|
const existing = await runs.get(runKey);
|
|
185
|
-
|
|
189
|
+
// Capture staleness NOW, off the pre-claim snapshot, and reuse that ONE decision below: the claim
|
|
190
|
+
// CAS below rewrites `updated_at`, so a staleness check after it would read the fresh claim and
|
|
191
|
+
// never fire. Only a STALE re-claim needs reconcile-before-relaunch (a fresh claim or a terminal-row
|
|
192
|
+
// re-run has no possibly-live original). Re-deriving staleness a second time here would race the TTL
|
|
193
|
+
// boundary: `isStaleLaunchClaim` reads `Date.now()` afresh, so the row can cross the TTL between this
|
|
194
|
+
// capture and a second check — the short-circuit would then stop firing while `reclaimingStaleClaim`
|
|
195
|
+
// stays false, the CAS below would still reclaim the (now-stale) row, and the reconcile block would
|
|
196
|
+
// be skipped, relaunching WITHOUT cancelling a possibly-live original. One captured decision keeps
|
|
197
|
+
// the short-circuit and the reconcile in lock-step (issue #853 review — thread
|
|
198
|
+
// deliveryGraphDispatch.ts:193).
|
|
199
|
+
const reclaimingStaleClaim = existing != null && isStaleLaunchClaim(existing);
|
|
200
|
+
// Capture the stale claim's original `updated_at` NOW, before the CAS below rewrites it: a
|
|
201
|
+
// reconcile-read failure must restore the row to exactly this past-the-TTL timestamp so it stays a
|
|
202
|
+
// RECLAIMABLE stale claim (see `revertToStaleClaim`). Captured as a value (not read back off
|
|
203
|
+
// `existing` after the CAS) because the CAS refreshes `updated_at` in place.
|
|
204
|
+
const staleClaimUpdatedAt = existing?.updated_at;
|
|
205
|
+
if (existing && existing.status === "running" && !reclaimingStaleClaim) {
|
|
186
206
|
app.log.info("dispatch-delivery-graph short-circuit: already running", { runKey });
|
|
187
207
|
return {
|
|
188
208
|
ok: true,
|
|
@@ -236,6 +256,55 @@ export async function dispatchDeliveryGraphRun(
|
|
|
236
256
|
const { run_key, created_at, ...patch } = failed;
|
|
237
257
|
await runs.update(runKey, patch);
|
|
238
258
|
};
|
|
259
|
+
// Reconcile-FAILURE recovery (issue #853 review — thread deliveryGraphDispatch.ts:280): a reconcile
|
|
260
|
+
// READ/CANCEL failure must NOT retire the row to `failed`. A `failed` row is terminal, so
|
|
261
|
+
// `isStaleLaunchClaim` rejects it and the NEXT dispatch re-runs it as a plain terminal-row relaunch —
|
|
262
|
+
// WITHOUT re-entering reconcile-before-relaunch — double-launching the side-effecting nodes if the
|
|
263
|
+
// original instance is in fact still live (the read failed only transiently). Instead revert the row
|
|
264
|
+
// to the pre-claim STALE launch-claim snapshot so the reconcile is RETRIED before any relaunch: the
|
|
265
|
+
// next dispatch sees a stale claim again (`reclaimingStaleClaim` → reconcile) and the poller sees one
|
|
266
|
+
// too (`reconcileStaleLaunchClaim` cancels the original via `finalize`). This mirrors the poller's own
|
|
267
|
+
// reconcile-failure behaviour, whose `finalize` throw rolls the CAS back to the same stale-claim
|
|
268
|
+
// `running` state rather than failing it — one canonical "a failed reconcile leaves the row
|
|
269
|
+
// reconcile-pending, never terminal" rule across both the dispatch and poller paths.
|
|
270
|
+
const revertToStaleClaim = async () => {
|
|
271
|
+
await runs.update(runKey, { status: "running", phase: DELIVERY_PHASE.RUNNING, process_key: null, process_definition_id: null, updated_at: staleClaimUpdatedAt });
|
|
272
|
+
};
|
|
273
|
+
// Reconcile-before-relaunch (issue #852 review — thread deliveryGraphRun.ts:180): if we just RE-CLAIMED
|
|
274
|
+
// a STALE launch claim (a `running`/NULL-key row past its TTL), the original dispatch may have died
|
|
275
|
+
// AFTER `createInstance` succeeded but BEFORE stamping the process key — leaving a LIVE instance
|
|
276
|
+
// executing this graph's side-effecting nodes. A timeout alone cannot tell that from "died before
|
|
277
|
+
// create". Now that the engine seam exposes it, close the window: cancel any still-ACTIVE instance of
|
|
278
|
+
// this run (matched by its seeded `runKey` variable) BEFORE we launch a replacement, so the relaunch
|
|
279
|
+
// stays at-most-once. Only a STALE re-claim needs this — a fresh claim or a terminal-row re-run has no
|
|
280
|
+
// possibly-live original. The content-addressed definition id is derived from the STALE ROW's own
|
|
281
|
+
// `digest` (`existing.digest`), NOT the freshly-compiled `digest` of THIS dispatch: under an explicit
|
|
282
|
+
// idempotency key the re-dispatched graph can differ (digest drift), and the original was deployed
|
|
283
|
+
// under `delivery-graph-<existing.digest>`. Searching under the new digest would miss the original and
|
|
284
|
+
// strand it live — the exact double-launch this reconcile exists to prevent. Mirrors the poller, which
|
|
285
|
+
// uses the stored `run.digest` (service.ts). `existing` is non-null whenever `reclaimingStaleClaim`.
|
|
286
|
+
if (reclaimingStaleClaim) {
|
|
287
|
+
const originalDigest = existing?.digest ?? digest;
|
|
288
|
+
try {
|
|
289
|
+
const cancelled = await reconcileOriginalInstanceBeforeRelaunch(app.engine, {
|
|
290
|
+
runKey,
|
|
291
|
+
processDefinitionId: `${DELIVERY_GRAPH_PROCESS_ID}-${originalDigest}`,
|
|
292
|
+
});
|
|
293
|
+
if (cancelled.length > 0) {
|
|
294
|
+
app.log.warn("dispatch-delivery-graph cancelled a still-running original instance before relaunch", { runKey, cancelled });
|
|
295
|
+
}
|
|
296
|
+
} catch (err) {
|
|
297
|
+
// A reconciliation read/cancel failure — OR a fail-closed `ReconcileConflictError` (an ACTIVE
|
|
298
|
+
// same-definition instance with no readable run-root `runKey`, e.g. a pre-upgrade legacy original)
|
|
299
|
+
// — must not strand the claim OR waive a blind relaunch through: revert to the pre-claim STALE
|
|
300
|
+
// launch-claim so the reconcile is retried (by the next dispatch or the poller) BEFORE any
|
|
301
|
+
// replacement launches, rather than retiring to `failed` (which a later dispatch would
|
|
302
|
+
// short-circuit past into a possible double-launch).
|
|
303
|
+
await revertToStaleClaim();
|
|
304
|
+
app.log.error("dispatch-delivery-graph reconcile-before-relaunch threw", { runKey });
|
|
305
|
+
throw err;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
239
308
|
// The claim is ours — persist THIS run's lossless identity so a later same-key short-circuit can prove
|
|
240
309
|
// the running run is this exact graph. Upsert: a relaunch off a persisted (e.g. failed) row under an
|
|
241
310
|
// explicit key may carry a different graph, so overwrite the stored fingerprint. This runs AFTER the
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// #852 — a dispatch must run the CPU-bound, superlinear `layoutBpmn` pass ONCE. `dispatchDeliveryGraphRun`
|
|
2
|
+
// needs only the digest / resolved model / side effects / human stops (all on the semantic compile); the
|
|
3
|
+
// laid-out BPMN is produced by `runDeliveryGraph` → `prepareDeliveryGraph`. Calling the layout-bearing
|
|
4
|
+
// `compileDeliveryGraph` here too doubled a 54-node dispatch on merlin to ~105 s (52 s per layout).
|
|
5
|
+
import assert from "node:assert/strict";
|
|
6
|
+
import { readFileSync } from "node:fs";
|
|
7
|
+
import { test } from "node:test";
|
|
8
|
+
|
|
9
|
+
test("#852: dispatchDeliveryGraphRun uses the semantic compile, never the layout-bearing compileDeliveryGraph", () => {
|
|
10
|
+
const src = readFileSync(new URL("./deliveryGraphDispatch.ts", import.meta.url), "utf8").replace(/\/\/.*$|\/\*[\s\S]*?\*\//gm, "");
|
|
11
|
+
assert.doesNotMatch(src, /\bcompileDeliveryGraph\s*\(/, "dispatch must not lay the diagram out (runDeliveryGraph already does)");
|
|
12
|
+
assert.match(src, /\bcompileDeliveryGraphSemantic\s*\(/);
|
|
13
|
+
});
|