@nanobpm/nano-workforce 0.171.7 → 0.171.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
## [0.171.8](https://github.com/nanobpm/nano-workforce/compare/v0.171.7...v0.171.8) (2026-09-01)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
* **agentic:** release job⇄worker correlation on terminal lifecycle event ([#688](https://github.com/nanobpm/nano-workforce/issues/688)) ([58dde6c](https://github.com/nanobpm/nano-workforce/commit/58dde6cf1c91f75828c8a38de52fef9409e3484c)), closes [#661](https://github.com/nanobpm/nano-workforce/issues/661) [#544](https://github.com/nanobpm/nano-workforce/issues/544) [#661](https://github.com/nanobpm/nano-workforce/issues/661)
|
|
6
|
+
|
|
1
7
|
## [0.171.7](https://github.com/nanobpm/nano-workforce/compare/v0.171.6...v0.171.7) (2026-09-01)
|
|
2
8
|
|
|
3
9
|
### Bug Fixes
|
|
@@ -16,16 +16,19 @@ import { fileURLToPath } from "node:url";
|
|
|
16
16
|
import { ConnectionRegistry } from "@nanobpm/agentic/channel";
|
|
17
17
|
import type { Frame } from "@nanobpm/agentic/protocol";
|
|
18
18
|
import { RELAY_FAMILY } from "@nanobpm/agentic/relay";
|
|
19
|
-
import { type SqliteDb, TRANSCRIPT_SCHEMA_SQL } from "@nanobpm/agentic/transcript";
|
|
20
|
-
import { assert, assertEquals } from "#test-assert";
|
|
19
|
+
import { encodeTranscriptEvent, type SqliteDb, TRANSCRIPT_SCHEMA_SQL } from "@nanobpm/agentic/transcript";
|
|
20
|
+
import { assert, assertEquals, assertThrows } from "#test-assert";
|
|
21
21
|
import { noopLog } from "../../../test/log.ts";
|
|
22
22
|
import { CorrelationRegistry, jobStream } from "../correlation.ts";
|
|
23
23
|
import { AgenticCorrelationStore } from "../correlation-store.ts";
|
|
24
|
+
import { createPresenceStore, PresenceRegistry } from "./presence.family.ts";
|
|
24
25
|
import {
|
|
25
26
|
type CorrelationLink,
|
|
26
27
|
createRelayFamily,
|
|
27
28
|
currentRelayTranscriptService,
|
|
29
|
+
engineReconcileMs,
|
|
28
30
|
family as relayFamily,
|
|
31
|
+
guardOverlappingPasses,
|
|
29
32
|
RELAY_FAMILY_NAME,
|
|
30
33
|
RelayTranscriptService,
|
|
31
34
|
sweepIntervalMs,
|
|
@@ -542,6 +545,227 @@ test("#544 element-instance enrichment: an unresolved job (never parked) leaves
|
|
|
542
545
|
service.teardown();
|
|
543
546
|
});
|
|
544
547
|
|
|
548
|
+
/** A `produce` frame whose chunk is a typed transcript LIFECYCLE event at `phase` (#661). */
|
|
549
|
+
const lifecycle = (stream: string, incarnation: number, phase: "open" | "completed" | "exited"): Frame =>
|
|
550
|
+
produce(stream, incarnation, encodeTranscriptEvent({ kind: "lifecycle", phase, offset: 0 }));
|
|
551
|
+
|
|
552
|
+
test("#661 primary release: a terminal lifecycle event clears an idle-but-connected worker's finished job", () => {
|
|
553
|
+
const registry = new ConnectionRegistry();
|
|
554
|
+
const correlation = new CorrelationRegistry();
|
|
555
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
556
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection);
|
|
557
|
+
const p = connect("prod", registry);
|
|
558
|
+
|
|
559
|
+
// The worker relays its job's terminal → linked as active.
|
|
560
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
561
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "the job is active while it runs");
|
|
562
|
+
assertEquals(correlation.count(), 1);
|
|
563
|
+
|
|
564
|
+
// The job ends: the worker emits the terminal `lifecycle` event on the SAME live connection and then
|
|
565
|
+
// goes idle — it does NOT disconnect and does NOT take a new job (no supersede). Before this fix that
|
|
566
|
+
// finished job lingered forever as a phantom active job; now the terminal event releases it.
|
|
567
|
+
hub.handler?.(lifecycle(jobStream("k1"), 1, "completed"), p.conn);
|
|
568
|
+
assertEquals(correlation.jobKeysFor("worker-A"), [], "the finished job is released on the terminal event");
|
|
569
|
+
assertEquals(correlation.count(), 0, "count() drops — no phantom active job");
|
|
570
|
+
|
|
571
|
+
// Assert the RENDERED supply row clears too (not just the registry): the presence snapshot seeds
|
|
572
|
+
// per-worker jobKeys from the correlation registry, so an idle-but-connected worker shows no job.
|
|
573
|
+
const store = createPresenceStore(memoryDb());
|
|
574
|
+
store.ensureSchema();
|
|
575
|
+
store.register({ instance: "worker-A", connectionId: "prod", identity: "leaf", capability: {} });
|
|
576
|
+
const presence = new PresenceRegistry(store, () => new Set(["prod"]));
|
|
577
|
+
const row = presence.snapshot({ jobKeysFor: (i) => correlation.jobKeysFor(i) }).workers[0];
|
|
578
|
+
assertEquals(row.jobKeys, [], "the supply row shows no active job for the idle worker");
|
|
579
|
+
assert(row.live, "the worker is still connected — the connection persists across jobs");
|
|
580
|
+
|
|
581
|
+
// The terminal event itself is captured in the flushed transcript (release runs AFTER the ring append).
|
|
582
|
+
const meta = service.transcriptOf(jobStream("k1"));
|
|
583
|
+
assertEquals(meta?.status, "completed", "the finished job becomes a completed past session");
|
|
584
|
+
assertEquals(service.reattach(jobStream("k1"), 0)?.entries.length, 2, "the terminal event is part of the transcript");
|
|
585
|
+
service.teardown();
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
test("#661 primary release: an `exited` lifecycle also releases; a non-terminal `open` does not", () => {
|
|
589
|
+
const registry = new ConnectionRegistry();
|
|
590
|
+
const correlation = new CorrelationRegistry();
|
|
591
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
592
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection);
|
|
593
|
+
const p = connect("prod", registry);
|
|
594
|
+
|
|
595
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
596
|
+
// A `phase: "open"` lifecycle is NOT terminal — a genuinely active job must not be cleared.
|
|
597
|
+
hub.handler?.(lifecycle(jobStream("k1"), 1, "open"), p.conn);
|
|
598
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "an open lifecycle keeps the active job");
|
|
599
|
+
assertEquals(correlation.count(), 1);
|
|
600
|
+
|
|
601
|
+
// An `exited` lifecycle (a crash/kill the worker still managed to report) IS terminal → released.
|
|
602
|
+
hub.handler?.(lifecycle(jobStream("k1"), 1, "exited"), p.conn);
|
|
603
|
+
assertEquals(correlation.jobKeysFor("worker-A"), [], "an exited lifecycle releases the job");
|
|
604
|
+
assertEquals(correlation.count(), 0);
|
|
605
|
+
service.teardown();
|
|
606
|
+
});
|
|
607
|
+
|
|
608
|
+
test("#661 no-regression: an ordinary (non-lifecycle) chunk never clears a genuinely active job", () => {
|
|
609
|
+
const registry = new ConnectionRegistry();
|
|
610
|
+
const correlation = new CorrelationRegistry();
|
|
611
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
612
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection);
|
|
613
|
+
const p = connect("prod", registry);
|
|
614
|
+
|
|
615
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
616
|
+
// Ordinary terminal output (raw bytes, not a typed envelope) must NOT be read as a job-end signal.
|
|
617
|
+
for (let i = 0; i < 5; i++) hub.handler?.(produce(jobStream("k1"), 1, `output ${i}`), p.conn);
|
|
618
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "a running job stays active across ordinary output");
|
|
619
|
+
assertEquals(correlation.count(), 1);
|
|
620
|
+
assertEquals(service.transcriptOf(jobStream("k1")), undefined, "the live job is not completed");
|
|
621
|
+
service.teardown();
|
|
622
|
+
});
|
|
623
|
+
|
|
624
|
+
test("#661 defensive reconcile: an unclean exit (no lifecycle) whose engine job is gone is dropped by the reconcile pass", async () => {
|
|
625
|
+
const registry = new ConnectionRegistry();
|
|
626
|
+
const correlation = new CorrelationRegistry();
|
|
627
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
628
|
+
// The engine reports job k1 as a live JOB park (element instance ei-1) — until the worker exits
|
|
629
|
+
// UNCLEANLY (crash/kill): it emits no terminal lifecycle event, keeps no connection to reconcile,
|
|
630
|
+
// and the engine park is gone → the resolver returns undefined.
|
|
631
|
+
let parked = true;
|
|
632
|
+
const resolveElementInstance = (jobKey: string) =>
|
|
633
|
+
Promise.resolve(parked && jobKey === "k1" ? "ei-1" : undefined);
|
|
634
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
|
|
635
|
+
resolveElementInstance,
|
|
636
|
+
});
|
|
637
|
+
const p = connect("prod", registry);
|
|
638
|
+
|
|
639
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
640
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "the job links while it runs");
|
|
641
|
+
|
|
642
|
+
// While the engine still parks the job, the reconcile pass leaves a genuinely active job alone.
|
|
643
|
+
await service.reconcileEngineCorrelations();
|
|
644
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "a genuinely active job is NOT cleared by the safety net");
|
|
645
|
+
assertEquals(correlation.count(), 1);
|
|
646
|
+
|
|
647
|
+
// The worker exits uncleanly (no lifecycle event) — its engine park vanishes. The next reconcile
|
|
648
|
+
// pass drops the stale correlation even though nothing on the wire signalled job end.
|
|
649
|
+
parked = false;
|
|
650
|
+
await service.reconcileEngineCorrelations();
|
|
651
|
+
assertEquals(correlation.jobKeysFor("worker-A"), [], "the reconcile pass drops the stale correlation");
|
|
652
|
+
assertEquals(correlation.count(), 0, "no phantom active job survives an unclean exit");
|
|
653
|
+
service.teardown();
|
|
654
|
+
});
|
|
655
|
+
|
|
656
|
+
test("#661 defensive reconcile: a transient engine read failure never falsely releases a job", async () => {
|
|
657
|
+
const registry = new ConnectionRegistry();
|
|
658
|
+
const correlation = new CorrelationRegistry();
|
|
659
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
660
|
+
// The engine read throws (unavailable) — a transient fault must be treated as "unknown, keep it",
|
|
661
|
+
// NEVER as "job gone", or a live job would be wrongly cleared on every engine blip.
|
|
662
|
+
const resolveElementInstance = () => Promise.reject(new Error("engine unavailable"));
|
|
663
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
|
|
664
|
+
resolveElementInstance,
|
|
665
|
+
});
|
|
666
|
+
const p = connect("prod", registry);
|
|
667
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
668
|
+
|
|
669
|
+
await service.reconcileEngineCorrelations();
|
|
670
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "a transient engine failure leaves the job linked");
|
|
671
|
+
assertEquals(correlation.count(), 1);
|
|
672
|
+
service.teardown();
|
|
673
|
+
});
|
|
674
|
+
|
|
675
|
+
test("#661 defensive reconcile: a no-op when no engine resolver is wired", async () => {
|
|
676
|
+
const registry = new ConnectionRegistry();
|
|
677
|
+
const correlation = new CorrelationRegistry();
|
|
678
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
679
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection);
|
|
680
|
+
const p = connect("prod", registry);
|
|
681
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
682
|
+
|
|
683
|
+
// With no element-instance resolver (engine-less host), the safety net cannot query the engine —
|
|
684
|
+
// it must be an inert no-op, leaving the correlation exactly as the primary path manages it.
|
|
685
|
+
await service.reconcileEngineCorrelations();
|
|
686
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "no resolver → the reconcile pass is inert");
|
|
687
|
+
service.teardown();
|
|
688
|
+
});
|
|
689
|
+
|
|
690
|
+
test("#661 engineReconcileMs: defaults, disables on a non-positive/non-finite value, caps at the timer max", () => {
|
|
691
|
+
assertEquals(engineReconcileMs(), 30_000, "an omitted config uses the default cadence");
|
|
692
|
+
assertEquals(engineReconcileMs(5_000), 5_000, "a finite positive value is honoured");
|
|
693
|
+
assertEquals(engineReconcileMs(0), undefined, "zero disables the pass");
|
|
694
|
+
assertEquals(engineReconcileMs(-1), undefined, "a negative value disables the pass");
|
|
695
|
+
assertEquals(engineReconcileMs(Number.NaN), undefined, "a non-finite value disables the pass");
|
|
696
|
+
assertEquals(engineReconcileMs(Number.POSITIVE_INFINITY), undefined, "infinity disables the pass");
|
|
697
|
+
assertEquals(engineReconcileMs(2 ** 32), 2_147_483_647, "a huge value is capped at the Node timer ceiling");
|
|
698
|
+
assertEquals(engineReconcileMs(0.5), 1, "a sub-millisecond positive value floors to 1ms, never 0 (no busy setInterval(0))");
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
test("#661 guardOverlappingPasses: a tick while a pass is in flight is skipped; only one pass runs at a time", async () => {
|
|
702
|
+
let starts = 0;
|
|
703
|
+
let active = 0;
|
|
704
|
+
let maxActive = 0;
|
|
705
|
+
let release!: () => void;
|
|
706
|
+
// A pass that blocks until we release it, so we can hold one "in flight" while further ticks fire.
|
|
707
|
+
const pass = () => {
|
|
708
|
+
starts++;
|
|
709
|
+
active++;
|
|
710
|
+
maxActive = Math.max(maxActive, active);
|
|
711
|
+
return new Promise<void>((resolve) => {
|
|
712
|
+
release = () => {
|
|
713
|
+
active--;
|
|
714
|
+
resolve();
|
|
715
|
+
};
|
|
716
|
+
});
|
|
717
|
+
};
|
|
718
|
+
const tick = guardOverlappingPasses(pass);
|
|
719
|
+
|
|
720
|
+
tick(); // first tick starts a pass — now in flight
|
|
721
|
+
tick(); // skipped while the first pass is still pending
|
|
722
|
+
tick(); // skipped
|
|
723
|
+
assertEquals(starts, 1, "overlapping ticks are dropped while a pass is in flight");
|
|
724
|
+
assertEquals(maxActive, 1, "never more than one pass runs concurrently");
|
|
725
|
+
|
|
726
|
+
release(); // let the first pass settle
|
|
727
|
+
await new Promise((r) => setImmediate(r)); // flush the `finally` that clears the in-flight flag
|
|
728
|
+
|
|
729
|
+
tick(); // a tick after the previous pass settled starts a fresh pass
|
|
730
|
+
assertEquals(starts, 2, "a tick after the in-flight pass settles starts a new pass");
|
|
731
|
+
assertEquals(maxActive, 1, "still only ever one pass at a time");
|
|
732
|
+
release();
|
|
733
|
+
});
|
|
734
|
+
|
|
735
|
+
test("#661 guardOverlappingPasses: a pass whose work rejects but is self-caught still re-arms the guard", async () => {
|
|
736
|
+
// The guard clears its in-flight flag via `.finally`, so it re-arms whether the pass resolves OR
|
|
737
|
+
// rejects — a rejection does NOT wedge the guard. The real hazard of a rejecting pass is an
|
|
738
|
+
// *unhandled rejection* (the tick voids the returned promise), which is why the mount wraps
|
|
739
|
+
// `reconcileEngineCorrelations()` in `.catch`. Here the pass's work rejects but is self-caught
|
|
740
|
+
// (mirroring that wrapper), so there is no unhandled rejection, and we assert the guard re-arms.
|
|
741
|
+
let starts = 0;
|
|
742
|
+
const tick = guardOverlappingPasses(() => {
|
|
743
|
+
starts++;
|
|
744
|
+
// Work that rejects but settles its own error — exactly like the mount's `.catch` wrapper.
|
|
745
|
+
return Promise.reject(new Error("pass work failed")).catch(() => {});
|
|
746
|
+
});
|
|
747
|
+
tick();
|
|
748
|
+
await new Promise((r) => setImmediate(r));
|
|
749
|
+
tick();
|
|
750
|
+
await new Promise((r) => setImmediate(r));
|
|
751
|
+
assertEquals(starts, 2, "a self-caught rejecting pass settles and re-arms the guard for the next tick");
|
|
752
|
+
});
|
|
753
|
+
|
|
754
|
+
test("#661 guardOverlappingPasses: a pass that throws SYNCHRONOUSLY re-arms the guard (no wedge) and stays loud", () => {
|
|
755
|
+
// A synchronous throw from `pass` escapes before `.finally` is attached, so the guard must catch it,
|
|
756
|
+
// re-arm, and re-throw — otherwise a future non-`async` caller/refactor would wedge the guard
|
|
757
|
+
// permanently in-flight. We assert both: the throw propagates (stays loud, not swallowed) AND the
|
|
758
|
+
// next tick starts a fresh pass (the guard re-armed rather than sticking at in-flight).
|
|
759
|
+
let starts = 0;
|
|
760
|
+
const tick = guardOverlappingPasses(() => {
|
|
761
|
+
starts++;
|
|
762
|
+
throw new Error("synchronous boom"); // violates the "must not throw synchronously" contract
|
|
763
|
+
});
|
|
764
|
+
assertThrows(() => tick(), Error, "synchronous boom");
|
|
765
|
+
assertThrows(() => tick(), Error, "synchronous boom");
|
|
766
|
+
assertEquals(starts, 2, "the guard re-armed after a synchronous throw, so the next tick ran the pass");
|
|
767
|
+
});
|
|
768
|
+
|
|
545
769
|
/**
|
|
546
770
|
* A {@link CorrelationLink} wrapper that delegates to a real registry but can be flipped to throw on
|
|
547
771
|
* `link()`/`releaseJob()`, exercising the advisory-resilience contract: `#link`/`#unlink` are
|
|
@@ -24,6 +24,7 @@ import type { ConnectionRegistry } from "@nanobpm/agentic/channel";
|
|
|
24
24
|
import type { Frame } from "@nanobpm/agentic/protocol";
|
|
25
25
|
import { RELAY_FAMILY, RelayHub, type RelayHubOptions } from "@nanobpm/agentic/relay";
|
|
26
26
|
import {
|
|
27
|
+
parseTranscriptEvent,
|
|
27
28
|
type SqliteDb,
|
|
28
29
|
type TranscriptLifecycle,
|
|
29
30
|
type TranscriptRing,
|
|
@@ -50,6 +51,14 @@ const SWEEP_DIVISOR = 4;
|
|
|
50
51
|
* 32-bit timer and Node silently clamps it to 1ms — turning a slow periodic tick into a busy loop. */
|
|
51
52
|
const MAX_TIMER_MS = 2_147_483_647;
|
|
52
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Default cadence (ms) of the defensive engine-reconcile pass (#661) — the safety net that releases a
|
|
56
|
+
* correlation whose engine JOB park is gone but whose terminal `lifecycle` event never arrived (an
|
|
57
|
+
* unclean worker exit). 30s trades a small staleness bound for a light engine-read load; the precise,
|
|
58
|
+
* immediate release stays the terminal-lifecycle path, so this only ever mops up unclean exits.
|
|
59
|
+
*/
|
|
60
|
+
const DEFAULT_ENGINE_RECONCILE_MS = 30_000;
|
|
61
|
+
|
|
53
62
|
/**
|
|
54
63
|
* The retention-sweep cadence (ms) for a given ephemeral-retention window: a fraction of the window,
|
|
55
64
|
* floored at 1ms and — crucially — capped at {@link MAX_TIMER_MS} so a large retention config (e.g.
|
|
@@ -64,12 +73,75 @@ export function sweepIntervalMs(ephemeralRetentionMs: number): number {
|
|
|
64
73
|
return Math.min(MAX_TIMER_MS, Math.max(1, interval));
|
|
65
74
|
}
|
|
66
75
|
|
|
76
|
+
/**
|
|
77
|
+
* The defensive engine-reconcile cadence (ms) for a given config (#661), or `undefined` to DISABLE the
|
|
78
|
+
* pass. An omitted config uses {@link DEFAULT_ENGINE_RECONCILE_MS}; a non-finite or non-positive value
|
|
79
|
+
* (a broken config, or a deliberate opt-out) disables the pass rather than degrading into a 1ms busy
|
|
80
|
+
* loop; a finite positive value is floored at 1ms (so a sub-millisecond config like 0.5 cannot floor
|
|
81
|
+
* to 0 and degrade into a busy `setInterval(0)`) and capped at {@link MAX_TIMER_MS} so a large window
|
|
82
|
+
* cannot overflow Node's 32-bit timer.
|
|
83
|
+
*/
|
|
84
|
+
export function engineReconcileMs(configuredMs?: number): number | undefined {
|
|
85
|
+
const value = configuredMs ?? DEFAULT_ENGINE_RECONCILE_MS;
|
|
86
|
+
if (!Number.isFinite(value) || value <= 0) return undefined;
|
|
87
|
+
return Math.min(MAX_TIMER_MS, Math.max(1, Math.floor(value)));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Wrap an async pass in an in-flight guard so a periodic `setInterval` never fires OVERLAPPING runs
|
|
92
|
+
* (#661). A single pass of {@link RelayTranscriptService.reconcileEngineCorrelations} awaits an engine
|
|
93
|
+
* read per linked job, so a pass can outlast its interval (a small configured cadence, or a slow/large
|
|
94
|
+
* engine read-model); an unguarded `setInterval` would then stack concurrent passes, piling up engine
|
|
95
|
+
* reads and log volume. While a pass is still pending, every subsequent tick is skipped; the next tick
|
|
96
|
+
* after it settles — whether it resolves OR rejects, since the guard clears via `.finally` — starts a
|
|
97
|
+
* fresh pass. The guard clears on BOTH failure modes so it can never wedge in-flight: (1) a
|
|
98
|
+
* *synchronous* throw from `pass` escapes before `.finally` is attached, so it is caught here — the
|
|
99
|
+
* guard is re-armed and the fault is re-thrown so it stays loud rather than being silently swallowed;
|
|
100
|
+
* (2) an async rejection is cleared by `.finally`, but the tick `void`s (does not await) the returned
|
|
101
|
+
* promise, so `pass` must still settle its own rejections or an unhandled rejection results — which is
|
|
102
|
+
* why the mount wraps `reconcileEngineCorrelations()` in `.catch`. Making the guard resilient to (1)
|
|
103
|
+
* removes a subtle footgun for a future caller (or refactor) that returns a non-`async` `pass`. Mirrors
|
|
104
|
+
* the "one pass at a time" discipline the main poll loop enforces by self-scheduling. Returns the tick
|
|
105
|
+
* callback to hand to `setInterval`.
|
|
106
|
+
*/
|
|
107
|
+
export function guardOverlappingPasses(pass: () => Promise<void>): () => void {
|
|
108
|
+
let inFlight = false;
|
|
109
|
+
return () => {
|
|
110
|
+
if (inFlight) return;
|
|
111
|
+
inFlight = true;
|
|
112
|
+
try {
|
|
113
|
+
void pass().finally(() => {
|
|
114
|
+
inFlight = false;
|
|
115
|
+
});
|
|
116
|
+
} catch (err) {
|
|
117
|
+
// A synchronous throw never attaches the `.finally`; re-arm the guard so it does not wedge,
|
|
118
|
+
// then re-throw so a contract violation still surfaces instead of being silently swallowed.
|
|
119
|
+
inFlight = false;
|
|
120
|
+
throw err;
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
67
125
|
/** Read a property off an unknown value without an unsafe `as` cast (mirrors the loader's helper). */
|
|
68
126
|
function readProp(value: unknown, key: string): unknown {
|
|
69
127
|
if (!value || typeof value !== "object") return undefined;
|
|
70
128
|
return Object.hasOwn(value, key) ? Object.getOwnPropertyDescriptor(value, key)?.value : undefined;
|
|
71
129
|
}
|
|
72
130
|
|
|
131
|
+
/**
|
|
132
|
+
* Decode a single relay chunk through the ONE canonical transcript parser and report whether it is a
|
|
133
|
+
* TERMINAL `lifecycle` event (`phase` `completed`/`exited`) — the authoritative "job end" signal a
|
|
134
|
+
* clean agent run emits on its transcript stream (#661). Anything else — raw terminal bytes, a
|
|
135
|
+
* non-envelope JSON value, a `phase: "open"` lifecycle, any other event kind — is not terminal.
|
|
136
|
+
* Reusing {@link parseTranscriptEvent} keeps transcript-vocab knowledge out of the content-agnostic
|
|
137
|
+
* relay ring and off any forked decoder (Derivation Over Duplication): the ring still sees opaque
|
|
138
|
+
* bytes; only this narrow seam classifies them.
|
|
139
|
+
*/
|
|
140
|
+
function isTerminalLifecycleChunk(chunk: string): boolean {
|
|
141
|
+
const event = parseTranscriptEvent({ offset: 0, chunk });
|
|
142
|
+
return event.kind === "lifecycle" && (event.phase === "completed" || event.phase === "exited");
|
|
143
|
+
}
|
|
144
|
+
|
|
73
145
|
/**
|
|
74
146
|
* A resume-from-offset source with no retained chunks — used to flush/complete a stream that a
|
|
75
147
|
* producer opened logically but never wrote to, so its transcript is still stamped `completed`
|
|
@@ -424,11 +496,41 @@ export class RelayTranscriptService {
|
|
|
424
496
|
this.#streams.clear();
|
|
425
497
|
}
|
|
426
498
|
|
|
427
|
-
/** Handle one inbound `relay` frame: reconcile dead producers, observe ownership,
|
|
499
|
+
/** Handle one inbound `relay` frame: reconcile dead producers, observe ownership, delegate, then
|
|
500
|
+
* release on a terminal `lifecycle` event (#661 — AFTER the hub appended the chunk, so the terminal
|
|
501
|
+
* event itself is captured in the flushed transcript). */
|
|
428
502
|
#onFrame(frame: Frame, conn: RelayConnectionCtx): void {
|
|
429
503
|
this.#reconcile();
|
|
430
504
|
this.#observe(frame, conn);
|
|
431
505
|
this.relay.handle(frame, conn);
|
|
506
|
+
this.#observeTerminalLifecycle(frame);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Primary job-end release (#661): when a `produce` frame carries the terminal `lifecycle` event
|
|
511
|
+
* (`phase` `completed`/`exited`), complete its stream so the worker's job⇄instance correlation is
|
|
512
|
+
* released the moment the job ends — even though the worker's relay connection stays open across jobs
|
|
513
|
+
* (the disconnect/supersede release paths miss that idle-after-last-job tail, so an idle worker's
|
|
514
|
+
* finished job would otherwise linger as a phantom active job on its supply row). Runs AFTER the hub
|
|
515
|
+
* appends the chunk to the ring, so the terminal event is part of the flushed transcript. A narrow,
|
|
516
|
+
* self-contained decode at the correlation seam that reuses the ONE canonical
|
|
517
|
+
* {@link parseTranscriptEvent} — the content-agnostic relay ring keeps treating chunks as opaque
|
|
518
|
+
* bytes, and no transcript-vocab knowledge is forked into it. Non-terminal chunks (raw bytes, a
|
|
519
|
+
* `phase: "open"` lifecycle, any other event kind) never complete a live stream, so a genuinely
|
|
520
|
+
* active job is never cleared. Advisory — never throws into the frame handler.
|
|
521
|
+
*/
|
|
522
|
+
#observeTerminalLifecycle(frame: Frame): void {
|
|
523
|
+
if (readProp(frame.payload, "op") !== "produce") return;
|
|
524
|
+
const stream = readProp(frame.payload, "stream");
|
|
525
|
+
if (typeof stream !== "string" || stream === "") return;
|
|
526
|
+
const chunk = readProp(frame.payload, "chunk");
|
|
527
|
+
if (typeof chunk !== "string" || chunk === "") return;
|
|
528
|
+
const state = this.#streams.get(stream);
|
|
529
|
+
// Idempotent: an unknown or already-completed stream needs no (further) release — a second terminal
|
|
530
|
+
// event, or a late one after the disconnect/supersede path already completed the stream, is a no-op.
|
|
531
|
+
if (state === undefined || state.completed) return;
|
|
532
|
+
if (!isTerminalLifecycleChunk(chunk)) return;
|
|
533
|
+
this.completeStream(stream);
|
|
432
534
|
}
|
|
433
535
|
|
|
434
536
|
/** Record `produce` ownership so a producer disconnect can drive ephemeral completion. */
|
|
@@ -624,6 +726,61 @@ export class RelayTranscriptService {
|
|
|
624
726
|
}
|
|
625
727
|
}
|
|
626
728
|
|
|
729
|
+
/**
|
|
730
|
+
* Defensive engine-reconcile safety net (#661): release any linked jobKey whose engine JOB park is
|
|
731
|
+
* no longer live. The precise, fast release is the terminal `lifecycle` event
|
|
732
|
+
* ({@link #observeTerminalLifecycle}), but an UNCLEAN worker exit (crash/kill) can skip that event —
|
|
733
|
+
* and because the worker's relay connection is persistent across jobs, the disconnect release never
|
|
734
|
+
* fires either, so the finished job would linger as a phantom active job on the worker's supply row.
|
|
735
|
+
* This periodic pass asks the engine read model (the same {@link ElementInstanceResolver} the link
|
|
736
|
+
* path uses at #544) whether each linked job is still parked; a job the engine no longer parks
|
|
737
|
+
* (resolver returns `undefined`) is released and its transcript completed. Bounds staleness regardless
|
|
738
|
+
* of whether the worker emitted a clean terminal event — and also covers a worker that emitted nothing
|
|
739
|
+
* and merely went quiet.
|
|
740
|
+
*
|
|
741
|
+
* Advisory and best-effort: a no-op with no resolver wired, and a resolver THROW / REJECTION for a
|
|
742
|
+
* given job is treated as "unknown — keep it linked" (never a false release of a genuinely active
|
|
743
|
+
* job). The linked set is snapshotted before any await so a concurrent completion (a terminal
|
|
744
|
+
* lifecycle event landing mid-pass) cannot corrupt iteration, and each release re-checks the current
|
|
745
|
+
* stream state so a job already released between snapshot and resolution is not double-completed.
|
|
746
|
+
*/
|
|
747
|
+
async reconcileEngineCorrelations(): Promise<void> {
|
|
748
|
+
const resolve = this.#resolveElementInstance;
|
|
749
|
+
if (resolve === undefined) return;
|
|
750
|
+
const linked: { stream: string; jobKey: string; processInstanceKey?: string }[] = [];
|
|
751
|
+
for (const [stream, state] of this.#streams) {
|
|
752
|
+
if (!state.linked || state.completed) continue;
|
|
753
|
+
const jobKey = jobKeyOfStream(stream);
|
|
754
|
+
if (jobKey === undefined) continue;
|
|
755
|
+
const processInstanceKey = this.#correlation()?.resolve?.(jobKey)?.processInstanceKey;
|
|
756
|
+
linked.push({ stream, jobKey, processInstanceKey });
|
|
757
|
+
}
|
|
758
|
+
for (const { stream, jobKey, processInstanceKey } of linked) {
|
|
759
|
+
let activeKey: string | undefined;
|
|
760
|
+
try {
|
|
761
|
+
activeKey = await resolve(jobKey, processInstanceKey);
|
|
762
|
+
} catch (err) {
|
|
763
|
+
// A transient engine read failure must NOT be read as "job gone" — leave the job linked; a
|
|
764
|
+
// later pass (or the terminal lifecycle event) releases it. Advisory, never a false release.
|
|
765
|
+
this.#log.warn("agentic relay engine-reconcile read failed — leaving correlation linked", {
|
|
766
|
+
stream,
|
|
767
|
+
jobKey,
|
|
768
|
+
err: String(err),
|
|
769
|
+
});
|
|
770
|
+
continue;
|
|
771
|
+
}
|
|
772
|
+
// A live JOB park (a resolved element-instance key) means the job is genuinely active — keep it.
|
|
773
|
+
if (activeKey !== undefined) continue;
|
|
774
|
+
// The engine no longer parks this job → it ended (possibly via an unclean exit that skipped the
|
|
775
|
+
// terminal lifecycle event). Re-check the current state — a concurrent completion may already
|
|
776
|
+
// have released it — then release its correlation and flush its transcript.
|
|
777
|
+
const current = this.#streams.get(stream);
|
|
778
|
+
if (current === undefined || current.completed || !current.linked) continue;
|
|
779
|
+
this.#log.info("agentic relay engine-reconcile released a stale correlation", { stream, jobKey });
|
|
780
|
+
this.completeStream(stream);
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
|
|
627
784
|
/**
|
|
628
785
|
* The still-live relay ring for a stream, for the read path to serve BEFORE a durable flush (#486).
|
|
629
786
|
*
|
|
@@ -670,11 +827,18 @@ export function createRelayFamily(options: {
|
|
|
670
827
|
readonly relay?: RelayHubOptions;
|
|
671
828
|
readonly transcript?: TranscriptStoreOptions;
|
|
672
829
|
readonly ensureSchema?: boolean;
|
|
830
|
+
/**
|
|
831
|
+
* Cadence (ms) of the defensive engine-reconcile pass (#661). Defaults to
|
|
832
|
+
* {@link DEFAULT_ENGINE_RECONCILE_MS}. Clamped to Node's 32-bit timer ceiling; a non-positive /
|
|
833
|
+
* non-finite value disables the pass (the terminal-lifecycle release path still runs).
|
|
834
|
+
*/
|
|
835
|
+
readonly engineReconcileIntervalMs?: number;
|
|
673
836
|
/** Called with the live service once mounted, so a driver can drive completion/reattach. */
|
|
674
837
|
readonly onMounted?: (service: RelayTranscriptService) => void;
|
|
675
838
|
} = {}): AgenticFamily {
|
|
676
839
|
let service: RelayTranscriptService | undefined;
|
|
677
840
|
let sweepTimer: ReturnType<typeof setInterval> | undefined;
|
|
841
|
+
let engineReconcileTimer: ReturnType<typeof setInterval> | undefined;
|
|
678
842
|
return {
|
|
679
843
|
name: RELAY_FAMILY_NAME,
|
|
680
844
|
mount(ctx: AgenticContext): void {
|
|
@@ -729,6 +893,31 @@ export function createRelayFamily(options: {
|
|
|
729
893
|
tick();
|
|
730
894
|
}
|
|
731
895
|
|
|
896
|
+
// Defensive engine-reconcile safety net (#661): periodically release any linked correlation
|
|
897
|
+
// whose engine JOB park is gone but whose terminal `lifecycle` event never arrived (an unclean
|
|
898
|
+
// worker exit), so a crashed worker's finished job stops showing as a phantom active job. Only
|
|
899
|
+
// useful when an element-instance resolver is wired (engine read-model access); harmless no-op
|
|
900
|
+
// otherwise. Advisory — a reconcile fault is logged, never thrown, and never keeps the process
|
|
901
|
+
// alive on its own.
|
|
902
|
+
if (ctx.resolveElementInstance !== undefined) {
|
|
903
|
+
const reconcileInterval = engineReconcileMs(options.engineReconcileIntervalMs);
|
|
904
|
+
if (reconcileInterval !== undefined) {
|
|
905
|
+
// In-flight guard: `reconcileEngineCorrelations()` is async and awaits an engine read per
|
|
906
|
+
// linked job, so a pass can outlast `reconcileInterval` (a small interval, or a slow/large
|
|
907
|
+
// engine read-model). Without a guard, `setInterval` would fire overlapping passes that pile
|
|
908
|
+
// up concurrent engine reads and log volume. {@link guardOverlappingPasses} skips a tick while
|
|
909
|
+
// the previous pass is still running so only one reconcile runs at a time — the same "one pass
|
|
910
|
+
// at a time" discipline the main poll loop enforces by self-scheduling.
|
|
911
|
+
const reconcileTick = guardOverlappingPasses(() =>
|
|
912
|
+
(service?.reconcileEngineCorrelations() ?? Promise.resolve()).catch((err: unknown) => {
|
|
913
|
+
ctx.log.warn("agentic relay engine-reconcile failed", { err: String(err) });
|
|
914
|
+
}),
|
|
915
|
+
);
|
|
916
|
+
engineReconcileTimer = setInterval(reconcileTick, reconcileInterval);
|
|
917
|
+
engineReconcileTimer.unref?.();
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
|
|
732
921
|
options.onMounted?.(service);
|
|
733
922
|
},
|
|
734
923
|
teardown(): void {
|
|
@@ -736,6 +925,10 @@ export function createRelayFamily(options: {
|
|
|
736
925
|
clearInterval(sweepTimer);
|
|
737
926
|
sweepTimer = undefined;
|
|
738
927
|
}
|
|
928
|
+
if (engineReconcileTimer !== undefined) {
|
|
929
|
+
clearInterval(engineReconcileTimer);
|
|
930
|
+
engineReconcileTimer = undefined;
|
|
931
|
+
}
|
|
739
932
|
service?.teardown();
|
|
740
933
|
if (currentService === service) setCurrentRelayTranscriptService(undefined);
|
|
741
934
|
service = undefined;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nanobpm/nano-workforce",
|
|
3
|
-
"version": "0.171.
|
|
3
|
+
"version": "0.171.8",
|
|
4
4
|
"description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "main.ts",
|