@nanobpm/nano-workforce 0.171.7 → 0.171.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
## [0.171.9](https://github.com/nanobpm/nano-workforce/compare/v0.171.8...v0.171.9) (2026-09-01)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
* **agentic:** spare a still-active job's relay stream on a mid-job reconnect ([#690](https://github.com/nanobpm/nano-workforce/issues/690)) ([e0bb51e](https://github.com/nanobpm/nano-workforce/commit/e0bb51e5cb4a2e1c92f3760a3a4e9c70b8ef3803)), closes [661/#688](https://github.com/661/nano-workforce/issues/688) [#reconcile](https://github.com/nanobpm/nano-workforce/issues/reconcile) [#689](https://github.com/nanobpm/nano-workforce/issues/689) [#689](https://github.com/nanobpm/nano-workforce/issues/689) [#689](https://github.com/nanobpm/nano-workforce/issues/689)
|
|
6
|
+
|
|
7
|
+
## [0.171.8](https://github.com/nanobpm/nano-workforce/compare/v0.171.7...v0.171.8) (2026-09-01)
|
|
8
|
+
|
|
9
|
+
### Bug Fixes
|
|
10
|
+
|
|
11
|
+
* **agentic:** release job⇄worker correlation on terminal lifecycle event ([#688](https://github.com/nanobpm/nano-workforce/issues/688)) ([58dde6c](https://github.com/nanobpm/nano-workforce/commit/58dde6cf1c91f75828c8a38de52fef9409e3484c)), closes [#661](https://github.com/nanobpm/nano-workforce/issues/661) [#544](https://github.com/nanobpm/nano-workforce/issues/544) [#661](https://github.com/nanobpm/nano-workforce/issues/661)
|
|
12
|
+
|
|
1
13
|
## [0.171.7](https://github.com/nanobpm/nano-workforce/compare/v0.171.6...v0.171.7) (2026-09-01)
|
|
2
14
|
|
|
3
15
|
### Bug Fixes
|
|
@@ -138,6 +138,22 @@ test("instanceForConnection: resolves the worker instance owning a connection (H
|
|
|
138
138
|
assertEquals(registry.instanceForConnection(""), undefined, "empty connection → undefined");
|
|
139
139
|
});
|
|
140
140
|
|
|
141
|
+
test("isInstanceLive: an instance is live while ANY of its connections is open — survives a reconnect (#689)", () => {
|
|
142
|
+
const store = createPresenceStore(memSqlite());
|
|
143
|
+
store.ensureSchema();
|
|
144
|
+
// worker-L reconnected: its OLD connection (cOld) is closed, a NEW one (cNew) is open. Presence is
|
|
145
|
+
// keyed by instance, so both rows exist; only cNew is live.
|
|
146
|
+
store.register({ instance: "worker-L", connectionId: "cOld", identity: "leaf", capability: {} });
|
|
147
|
+
store.register({ instance: "worker-L", connectionId: "cNew", identity: "leaf", capability: {} });
|
|
148
|
+
store.register({ instance: "worker-Gone", connectionId: "cGone", identity: "leaf", capability: {} });
|
|
149
|
+
const registry = new PresenceRegistry(store, () => new Set(["cNew"]));
|
|
150
|
+
|
|
151
|
+
assertEquals(registry.isInstanceLive("worker-L"), true, "live via its new connection despite the old one dropping");
|
|
152
|
+
assertEquals(registry.isInstanceLive("worker-Gone"), false, "no live connection → not live");
|
|
153
|
+
assertEquals(registry.isInstanceLive("unknown"), false, "unknown instance → not live");
|
|
154
|
+
assertEquals(registry.isInstanceLive(""), false, "empty instance → not live");
|
|
155
|
+
});
|
|
156
|
+
|
|
141
157
|
test("attributionOf: resolves a worker instance's durable identity + host for job attribution (#485)", () => {
|
|
142
158
|
const store = createPresenceStore(memSqlite());
|
|
143
159
|
store.ensureSchema();
|
|
@@ -148,6 +148,25 @@ export class PresenceRegistry {
|
|
|
148
148
|
return undefined;
|
|
149
149
|
}
|
|
150
150
|
|
|
151
|
+
/**
|
|
152
|
+
* Whether a worker instance currently has a LIVE hub connection — i.e. the worker is present now,
|
|
153
|
+
* on any connection. Used by the relay slice to distinguish a worker that merely RECONNECTED
|
|
154
|
+
* mid-job (its old producer connection dropped, a new one re-registered under the SAME instance,
|
|
155
|
+
* so the instance is still live) from a worker that truly EXITED (all its connections gone, so the
|
|
156
|
+
* instance is no longer live). The former must NOT complete/archive its still-active job stream on
|
|
157
|
+
* the stale-producer reconcile; the latter must. Presence is keyed by instance across reconnects,
|
|
158
|
+
* so this survives the connection churn that a single-connection liveness check cannot. An empty or
|
|
159
|
+
* unknown instance is not live.
|
|
160
|
+
*/
|
|
161
|
+
isInstanceLive(instance: string): boolean {
|
|
162
|
+
if (instance === "") return false;
|
|
163
|
+
const live = this.#liveConnectionIds();
|
|
164
|
+
for (const row of this.#store.list()) {
|
|
165
|
+
if (row.instance === instance && live.has(row.connectionId)) return true;
|
|
166
|
+
}
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
|
|
151
170
|
/**
|
|
152
171
|
* Resolve a worker instance's durable identity attributes (presence identity + host) for job
|
|
153
172
|
* attribution (#485). Returns the most recently registered matching row's attributes, or undefined
|
|
@@ -16,16 +16,19 @@ import { fileURLToPath } from "node:url";
|
|
|
16
16
|
import { ConnectionRegistry } from "@nanobpm/agentic/channel";
|
|
17
17
|
import type { Frame } from "@nanobpm/agentic/protocol";
|
|
18
18
|
import { RELAY_FAMILY } from "@nanobpm/agentic/relay";
|
|
19
|
-
import { type SqliteDb, TRANSCRIPT_SCHEMA_SQL } from "@nanobpm/agentic/transcript";
|
|
20
|
-
import { assert, assertEquals } from "#test-assert";
|
|
19
|
+
import { encodeTranscriptEvent, type SqliteDb, TRANSCRIPT_SCHEMA_SQL } from "@nanobpm/agentic/transcript";
|
|
20
|
+
import { assert, assertEquals, assertThrows } from "#test-assert";
|
|
21
21
|
import { noopLog } from "../../../test/log.ts";
|
|
22
22
|
import { CorrelationRegistry, jobStream } from "../correlation.ts";
|
|
23
23
|
import { AgenticCorrelationStore } from "../correlation-store.ts";
|
|
24
|
+
import { createPresenceStore, PresenceRegistry } from "./presence.family.ts";
|
|
24
25
|
import {
|
|
25
26
|
type CorrelationLink,
|
|
26
27
|
createRelayFamily,
|
|
27
28
|
currentRelayTranscriptService,
|
|
29
|
+
engineReconcileMs,
|
|
28
30
|
family as relayFamily,
|
|
31
|
+
guardOverlappingPasses,
|
|
29
32
|
RELAY_FAMILY_NAME,
|
|
30
33
|
RelayTranscriptService,
|
|
31
34
|
sweepIntervalMs,
|
|
@@ -150,6 +153,7 @@ function mkCorrelatedService(
|
|
|
150
153
|
attributionForInstance?: (instance: string) => { identity?: string; host?: string } | undefined;
|
|
151
154
|
correlationStore?: AgenticCorrelationStore;
|
|
152
155
|
resolveElementInstance?: (jobKey: string, processInstanceKey?: string) => Promise<string | undefined>;
|
|
156
|
+
isInstanceLive?: (instance: string) => boolean;
|
|
153
157
|
now?: () => string;
|
|
154
158
|
} = {},
|
|
155
159
|
): { service: RelayTranscriptService; hub: CapturingHub } {
|
|
@@ -164,6 +168,7 @@ function mkCorrelatedService(
|
|
|
164
168
|
attributionForInstance: extra.attributionForInstance,
|
|
165
169
|
correlationStore: extra.correlationStore,
|
|
166
170
|
resolveElementInstance: extra.resolveElementInstance,
|
|
171
|
+
isInstanceLive: extra.isInstanceLive,
|
|
167
172
|
now: extra.now,
|
|
168
173
|
});
|
|
169
174
|
return { service, hub };
|
|
@@ -542,6 +547,227 @@ test("#544 element-instance enrichment: an unresolved job (never parked) leaves
|
|
|
542
547
|
service.teardown();
|
|
543
548
|
});
|
|
544
549
|
|
|
550
|
+
/** A `produce` frame whose chunk is a typed transcript LIFECYCLE event at `phase` (#661). */
|
|
551
|
+
const lifecycle = (stream: string, incarnation: number, phase: "open" | "completed" | "exited"): Frame =>
|
|
552
|
+
produce(stream, incarnation, encodeTranscriptEvent({ kind: "lifecycle", phase, offset: 0 }));
|
|
553
|
+
|
|
554
|
+
test("#661 primary release: a terminal lifecycle event clears an idle-but-connected worker's finished job", () => {
|
|
555
|
+
const registry = new ConnectionRegistry();
|
|
556
|
+
const correlation = new CorrelationRegistry();
|
|
557
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
558
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection);
|
|
559
|
+
const p = connect("prod", registry);
|
|
560
|
+
|
|
561
|
+
// The worker relays its job's terminal → linked as active.
|
|
562
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
563
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "the job is active while it runs");
|
|
564
|
+
assertEquals(correlation.count(), 1);
|
|
565
|
+
|
|
566
|
+
// The job ends: the worker emits the terminal `lifecycle` event on the SAME live connection and then
|
|
567
|
+
// goes idle — it does NOT disconnect and does NOT take a new job (no supersede). Before this fix that
|
|
568
|
+
// finished job lingered forever as a phantom active job; now the terminal event releases it.
|
|
569
|
+
hub.handler?.(lifecycle(jobStream("k1"), 1, "completed"), p.conn);
|
|
570
|
+
assertEquals(correlation.jobKeysFor("worker-A"), [], "the finished job is released on the terminal event");
|
|
571
|
+
assertEquals(correlation.count(), 0, "count() drops — no phantom active job");
|
|
572
|
+
|
|
573
|
+
// Assert the RENDERED supply row clears too (not just the registry): the presence snapshot seeds
|
|
574
|
+
// per-worker jobKeys from the correlation registry, so an idle-but-connected worker shows no job.
|
|
575
|
+
const store = createPresenceStore(memoryDb());
|
|
576
|
+
store.ensureSchema();
|
|
577
|
+
store.register({ instance: "worker-A", connectionId: "prod", identity: "leaf", capability: {} });
|
|
578
|
+
const presence = new PresenceRegistry(store, () => new Set(["prod"]));
|
|
579
|
+
const row = presence.snapshot({ jobKeysFor: (i) => correlation.jobKeysFor(i) }).workers[0];
|
|
580
|
+
assertEquals(row.jobKeys, [], "the supply row shows no active job for the idle worker");
|
|
581
|
+
assert(row.live, "the worker is still connected — the connection persists across jobs");
|
|
582
|
+
|
|
583
|
+
// The terminal event itself is captured in the flushed transcript (release runs AFTER the ring append).
|
|
584
|
+
const meta = service.transcriptOf(jobStream("k1"));
|
|
585
|
+
assertEquals(meta?.status, "completed", "the finished job becomes a completed past session");
|
|
586
|
+
assertEquals(service.reattach(jobStream("k1"), 0)?.entries.length, 2, "the terminal event is part of the transcript");
|
|
587
|
+
service.teardown();
|
|
588
|
+
});
|
|
589
|
+
|
|
590
|
+
test("#661 primary release: an `exited` lifecycle also releases; a non-terminal `open` does not", () => {
|
|
591
|
+
const registry = new ConnectionRegistry();
|
|
592
|
+
const correlation = new CorrelationRegistry();
|
|
593
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
594
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection);
|
|
595
|
+
const p = connect("prod", registry);
|
|
596
|
+
|
|
597
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
598
|
+
// A `phase: "open"` lifecycle is NOT terminal — a genuinely active job must not be cleared.
|
|
599
|
+
hub.handler?.(lifecycle(jobStream("k1"), 1, "open"), p.conn);
|
|
600
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "an open lifecycle keeps the active job");
|
|
601
|
+
assertEquals(correlation.count(), 1);
|
|
602
|
+
|
|
603
|
+
// An `exited` lifecycle (a crash/kill the worker still managed to report) IS terminal → released.
|
|
604
|
+
hub.handler?.(lifecycle(jobStream("k1"), 1, "exited"), p.conn);
|
|
605
|
+
assertEquals(correlation.jobKeysFor("worker-A"), [], "an exited lifecycle releases the job");
|
|
606
|
+
assertEquals(correlation.count(), 0);
|
|
607
|
+
service.teardown();
|
|
608
|
+
});
|
|
609
|
+
|
|
610
|
+
test("#661 no-regression: an ordinary (non-lifecycle) chunk never clears a genuinely active job", () => {
|
|
611
|
+
const registry = new ConnectionRegistry();
|
|
612
|
+
const correlation = new CorrelationRegistry();
|
|
613
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
614
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection);
|
|
615
|
+
const p = connect("prod", registry);
|
|
616
|
+
|
|
617
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
618
|
+
// Ordinary terminal output (raw bytes, not a typed envelope) must NOT be read as a job-end signal.
|
|
619
|
+
for (let i = 0; i < 5; i++) hub.handler?.(produce(jobStream("k1"), 1, `output ${i}`), p.conn);
|
|
620
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "a running job stays active across ordinary output");
|
|
621
|
+
assertEquals(correlation.count(), 1);
|
|
622
|
+
assertEquals(service.transcriptOf(jobStream("k1")), undefined, "the live job is not completed");
|
|
623
|
+
service.teardown();
|
|
624
|
+
});
|
|
625
|
+
|
|
626
|
+
test("#661 defensive reconcile: an unclean exit (no lifecycle) whose engine job is gone is dropped by the reconcile pass", async () => {
|
|
627
|
+
const registry = new ConnectionRegistry();
|
|
628
|
+
const correlation = new CorrelationRegistry();
|
|
629
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
630
|
+
// The engine reports job k1 as a live JOB park (element instance ei-1) — until the worker exits
|
|
631
|
+
// UNCLEANLY (crash/kill): it emits no terminal lifecycle event, keeps no connection to reconcile,
|
|
632
|
+
// and the engine park is gone → the resolver returns undefined.
|
|
633
|
+
let parked = true;
|
|
634
|
+
const resolveElementInstance = (jobKey: string) =>
|
|
635
|
+
Promise.resolve(parked && jobKey === "k1" ? "ei-1" : undefined);
|
|
636
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
|
|
637
|
+
resolveElementInstance,
|
|
638
|
+
});
|
|
639
|
+
const p = connect("prod", registry);
|
|
640
|
+
|
|
641
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
642
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "the job links while it runs");
|
|
643
|
+
|
|
644
|
+
// While the engine still parks the job, the reconcile pass leaves a genuinely active job alone.
|
|
645
|
+
await service.reconcileEngineCorrelations();
|
|
646
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "a genuinely active job is NOT cleared by the safety net");
|
|
647
|
+
assertEquals(correlation.count(), 1);
|
|
648
|
+
|
|
649
|
+
// The worker exits uncleanly (no lifecycle event) — its engine park vanishes. The next reconcile
|
|
650
|
+
// pass drops the stale correlation even though nothing on the wire signalled job end.
|
|
651
|
+
parked = false;
|
|
652
|
+
await service.reconcileEngineCorrelations();
|
|
653
|
+
assertEquals(correlation.jobKeysFor("worker-A"), [], "the reconcile pass drops the stale correlation");
|
|
654
|
+
assertEquals(correlation.count(), 0, "no phantom active job survives an unclean exit");
|
|
655
|
+
service.teardown();
|
|
656
|
+
});
|
|
657
|
+
|
|
658
|
+
test("#661 defensive reconcile: a transient engine read failure never falsely releases a job", async () => {
|
|
659
|
+
const registry = new ConnectionRegistry();
|
|
660
|
+
const correlation = new CorrelationRegistry();
|
|
661
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
662
|
+
// The engine read throws (unavailable) — a transient fault must be treated as "unknown, keep it",
|
|
663
|
+
// NEVER as "job gone", or a live job would be wrongly cleared on every engine blip.
|
|
664
|
+
const resolveElementInstance = () => Promise.reject(new Error("engine unavailable"));
|
|
665
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
|
|
666
|
+
resolveElementInstance,
|
|
667
|
+
});
|
|
668
|
+
const p = connect("prod", registry);
|
|
669
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
670
|
+
|
|
671
|
+
await service.reconcileEngineCorrelations();
|
|
672
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "a transient engine failure leaves the job linked");
|
|
673
|
+
assertEquals(correlation.count(), 1);
|
|
674
|
+
service.teardown();
|
|
675
|
+
});
|
|
676
|
+
|
|
677
|
+
test("#661 defensive reconcile: a no-op when no engine resolver is wired", async () => {
|
|
678
|
+
const registry = new ConnectionRegistry();
|
|
679
|
+
const correlation = new CorrelationRegistry();
|
|
680
|
+
const byConnection = new Map([["prod", "worker-A"]]);
|
|
681
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection);
|
|
682
|
+
const p = connect("prod", registry);
|
|
683
|
+
hub.handler?.(produce(jobStream("k1"), 1, "job-1 line"), p.conn);
|
|
684
|
+
|
|
685
|
+
// With no element-instance resolver (engine-less host), the safety net cannot query the engine —
|
|
686
|
+
// it must be an inert no-op, leaving the correlation exactly as the primary path manages it.
|
|
687
|
+
await service.reconcileEngineCorrelations();
|
|
688
|
+
assertEquals(correlation.jobKeysFor("worker-A"), ["k1"], "no resolver → the reconcile pass is inert");
|
|
689
|
+
service.teardown();
|
|
690
|
+
});
|
|
691
|
+
|
|
692
|
+
test("#661 engineReconcileMs: defaults, disables on a non-positive/non-finite value, caps at the timer max", () => {
|
|
693
|
+
assertEquals(engineReconcileMs(), 30_000, "an omitted config uses the default cadence");
|
|
694
|
+
assertEquals(engineReconcileMs(5_000), 5_000, "a finite positive value is honoured");
|
|
695
|
+
assertEquals(engineReconcileMs(0), undefined, "zero disables the pass");
|
|
696
|
+
assertEquals(engineReconcileMs(-1), undefined, "a negative value disables the pass");
|
|
697
|
+
assertEquals(engineReconcileMs(Number.NaN), undefined, "a non-finite value disables the pass");
|
|
698
|
+
assertEquals(engineReconcileMs(Number.POSITIVE_INFINITY), undefined, "infinity disables the pass");
|
|
699
|
+
assertEquals(engineReconcileMs(2 ** 32), 2_147_483_647, "a huge value is capped at the Node timer ceiling");
|
|
700
|
+
assertEquals(engineReconcileMs(0.5), 1, "a sub-millisecond positive value floors to 1ms, never 0 (no busy setInterval(0))");
|
|
701
|
+
});
|
|
702
|
+
|
|
703
|
+
test("#661 guardOverlappingPasses: a tick while a pass is in flight is skipped; only one pass runs at a time", async () => {
|
|
704
|
+
let starts = 0;
|
|
705
|
+
let active = 0;
|
|
706
|
+
let maxActive = 0;
|
|
707
|
+
let release!: () => void;
|
|
708
|
+
// A pass that blocks until we release it, so we can hold one "in flight" while further ticks fire.
|
|
709
|
+
const pass = () => {
|
|
710
|
+
starts++;
|
|
711
|
+
active++;
|
|
712
|
+
maxActive = Math.max(maxActive, active);
|
|
713
|
+
return new Promise<void>((resolve) => {
|
|
714
|
+
release = () => {
|
|
715
|
+
active--;
|
|
716
|
+
resolve();
|
|
717
|
+
};
|
|
718
|
+
});
|
|
719
|
+
};
|
|
720
|
+
const tick = guardOverlappingPasses(pass);
|
|
721
|
+
|
|
722
|
+
tick(); // first tick starts a pass — now in flight
|
|
723
|
+
tick(); // skipped while the first pass is still pending
|
|
724
|
+
tick(); // skipped
|
|
725
|
+
assertEquals(starts, 1, "overlapping ticks are dropped while a pass is in flight");
|
|
726
|
+
assertEquals(maxActive, 1, "never more than one pass runs concurrently");
|
|
727
|
+
|
|
728
|
+
release(); // let the first pass settle
|
|
729
|
+
await new Promise((r) => setImmediate(r)); // flush the `finally` that clears the in-flight flag
|
|
730
|
+
|
|
731
|
+
tick(); // a tick after the previous pass settled starts a fresh pass
|
|
732
|
+
assertEquals(starts, 2, "a tick after the in-flight pass settles starts a new pass");
|
|
733
|
+
assertEquals(maxActive, 1, "still only ever one pass at a time");
|
|
734
|
+
release();
|
|
735
|
+
});
|
|
736
|
+
|
|
737
|
+
test("#661 guardOverlappingPasses: a pass whose work rejects but is self-caught still re-arms the guard", async () => {
|
|
738
|
+
// The guard clears its in-flight flag via `.finally`, so it re-arms whether the pass resolves OR
|
|
739
|
+
// rejects — a rejection does NOT wedge the guard. The real hazard of a rejecting pass is an
|
|
740
|
+
// *unhandled rejection* (the tick voids the returned promise), which is why the mount wraps
|
|
741
|
+
// `reconcileEngineCorrelations()` in `.catch`. Here the pass's work rejects but is self-caught
|
|
742
|
+
// (mirroring that wrapper), so there is no unhandled rejection, and we assert the guard re-arms.
|
|
743
|
+
let starts = 0;
|
|
744
|
+
const tick = guardOverlappingPasses(() => {
|
|
745
|
+
starts++;
|
|
746
|
+
// Work that rejects but settles its own error — exactly like the mount's `.catch` wrapper.
|
|
747
|
+
return Promise.reject(new Error("pass work failed")).catch(() => {});
|
|
748
|
+
});
|
|
749
|
+
tick();
|
|
750
|
+
await new Promise((r) => setImmediate(r));
|
|
751
|
+
tick();
|
|
752
|
+
await new Promise((r) => setImmediate(r));
|
|
753
|
+
assertEquals(starts, 2, "a self-caught rejecting pass settles and re-arms the guard for the next tick");
|
|
754
|
+
});
|
|
755
|
+
|
|
756
|
+
test("#661 guardOverlappingPasses: a pass that throws SYNCHRONOUSLY re-arms the guard (no wedge) and stays loud", () => {
|
|
757
|
+
// A synchronous throw from `pass` escapes before `.finally` is attached, so the guard must catch it,
|
|
758
|
+
// re-arm, and re-throw — otherwise a future non-`async` caller/refactor would wedge the guard
|
|
759
|
+
// permanently in-flight. We assert both: the throw propagates (stays loud, not swallowed) AND the
|
|
760
|
+
// next tick starts a fresh pass (the guard re-armed rather than sticking at in-flight).
|
|
761
|
+
let starts = 0;
|
|
762
|
+
const tick = guardOverlappingPasses(() => {
|
|
763
|
+
starts++;
|
|
764
|
+
throw new Error("synchronous boom"); // violates the "must not throw synchronously" contract
|
|
765
|
+
});
|
|
766
|
+
assertThrows(() => tick(), Error, "synchronous boom");
|
|
767
|
+
assertThrows(() => tick(), Error, "synchronous boom");
|
|
768
|
+
assertEquals(starts, 2, "the guard re-armed after a synchronous throw, so the next tick ran the pass");
|
|
769
|
+
});
|
|
770
|
+
|
|
545
771
|
/**
|
|
546
772
|
* A {@link CorrelationLink} wrapper that delegates to a real registry but can be flipped to throw on
|
|
547
773
|
* `link()`/`releaseJob()`, exercising the advisory-resilience contract: `#link`/`#unlink` are
|
|
@@ -661,6 +887,75 @@ test("H6 correlation write-side: a producer disconnect releases its job correlat
|
|
|
661
887
|
service.teardown();
|
|
662
888
|
});
|
|
663
889
|
|
|
890
|
+
test("#689 mid-job reconnect: a stale producer whose instance is still live keeps its job correlation", () => {
|
|
891
|
+
const registry = new ConnectionRegistry();
|
|
892
|
+
const correlation = new CorrelationRegistry();
|
|
893
|
+
const byConnection = new Map([["conn-old", "worker-L"]]);
|
|
894
|
+
// Derive liveness from REAL connections exactly as production's PresenceRegistry.isInstanceLive
|
|
895
|
+
// does — an instance is live iff some connection attributed to it is still open in the registry —
|
|
896
|
+
// rather than a static stub decoupled from the connection churn. This makes the test exercise the
|
|
897
|
+
// very reconnect race the seam guards: the instance is live only while a real live connection
|
|
898
|
+
// (conn-old, then conn-new) backs it.
|
|
899
|
+
const isInstanceLive = (instance: string): boolean => {
|
|
900
|
+
for (const [connId, inst] of byConnection) {
|
|
901
|
+
if (inst === instance && registry.has(connId)) return true;
|
|
902
|
+
}
|
|
903
|
+
return false;
|
|
904
|
+
};
|
|
905
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
|
|
906
|
+
isInstanceLive,
|
|
907
|
+
});
|
|
908
|
+
const p = connect("conn-old", registry);
|
|
909
|
+
hub.handler?.(produce(jobStream("5749"), 1, "booting agent"), p.conn);
|
|
910
|
+
assertEquals(correlation.jobKeysFor("worker-L"), ["5749"], "the job is linked on first produce");
|
|
911
|
+
|
|
912
|
+
// The producer's WS connection blips (client reconnects). Model the reconnect faithfully: the new
|
|
913
|
+
// connection re-registers under the SAME instance BEFORE the reconcile frame, so there is never a
|
|
914
|
+
// gap where the instance is not live — presence keeps worker-L live across the churn. The old
|
|
915
|
+
// producer connection then drops. A subsequent frame drives #reconcile — which, seeing the instance
|
|
916
|
+
// still live via conn-new, must NOT archive the still-active job.
|
|
917
|
+
byConnection.set("conn-new", "worker-L");
|
|
918
|
+
const p2 = connect("conn-new", registry);
|
|
919
|
+
registry.remove("conn-old");
|
|
920
|
+
const other = connect("cons", registry);
|
|
921
|
+
hub.handler?.(grant(0), other.conn);
|
|
922
|
+
assertEquals(correlation.jobKeysFor("worker-L"), ["5749"], "a mid-job reconnect keeps the correlation");
|
|
923
|
+
assertEquals(service.transcriptOf(jobStream("5749")), undefined, "the still-active stream is NOT archived");
|
|
924
|
+
|
|
925
|
+
// The worker resumes producing on the SAME job over its already-open NEW connection → re-attributed,
|
|
926
|
+
// still one job, transcript still live (not terminal).
|
|
927
|
+
hub.handler?.(produce(jobStream("5749"), 1, "resumed output"), p2.conn);
|
|
928
|
+
assertEquals(correlation.jobKeysFor("worker-L"), ["5749"], "the resumed producer stays linked to the same job");
|
|
929
|
+
assertEquals(service.liveFallback(jobStream("5749"))?.ring !== undefined, true, "the transcript is still live, not completed");
|
|
930
|
+
|
|
931
|
+
// Only once the worker truly EXITS (all its connections gone → isInstanceLive false) does a later
|
|
932
|
+
// reconcile complete the stream and release the correlation.
|
|
933
|
+
registry.remove("conn-new");
|
|
934
|
+
hub.handler?.(grant(0), other.conn);
|
|
935
|
+
assertEquals(correlation.jobKeysFor("worker-L"), [], "a true worker-exit completes + releases the job");
|
|
936
|
+
assertEquals(service.transcriptOf(jobStream("5749"))?.status, "completed", "the exited worker's stream is archived");
|
|
937
|
+
service.teardown();
|
|
938
|
+
});
|
|
939
|
+
|
|
940
|
+
test("#689 mid-job reconnect: with no isInstanceLive wired, a producer disconnect completes as before", () => {
|
|
941
|
+
const registry = new ConnectionRegistry();
|
|
942
|
+
const correlation = new CorrelationRegistry();
|
|
943
|
+
const byConnection = new Map([["prod", "worker-N"]]);
|
|
944
|
+
// Default seam (isInstanceLive omitted → () => false): a producer disconnect completes the stream,
|
|
945
|
+
// preserving the prior always-complete-on-disconnect behaviour for callers that don't wire presence.
|
|
946
|
+
const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection);
|
|
947
|
+
const p = connect("prod", registry);
|
|
948
|
+
hub.handler?.(produce(jobStream("kN"), 1, "x"), p.conn);
|
|
949
|
+
assertEquals(correlation.jobKeysFor("worker-N"), ["kN"]);
|
|
950
|
+
|
|
951
|
+
registry.remove("prod");
|
|
952
|
+
const other = connect("cons", registry);
|
|
953
|
+
hub.handler?.(grant(0), other.conn);
|
|
954
|
+
assertEquals(correlation.jobKeysFor("worker-N"), [], "disconnect completes + releases when liveness is unknown");
|
|
955
|
+
assertEquals(service.transcriptOf(jobStream("kN"))?.status, "completed");
|
|
956
|
+
service.teardown();
|
|
957
|
+
});
|
|
958
|
+
|
|
664
959
|
test("H6 correlation write-side: non-job streams are never linked; a link retries until the instance resolves", () => {
|
|
665
960
|
const registry = new ConnectionRegistry();
|
|
666
961
|
const correlation = new CorrelationRegistry();
|
|
@@ -24,6 +24,7 @@ import type { ConnectionRegistry } from "@nanobpm/agentic/channel";
|
|
|
24
24
|
import type { Frame } from "@nanobpm/agentic/protocol";
|
|
25
25
|
import { RELAY_FAMILY, RelayHub, type RelayHubOptions } from "@nanobpm/agentic/relay";
|
|
26
26
|
import {
|
|
27
|
+
parseTranscriptEvent,
|
|
27
28
|
type SqliteDb,
|
|
28
29
|
type TranscriptLifecycle,
|
|
29
30
|
type TranscriptRing,
|
|
@@ -50,6 +51,14 @@ const SWEEP_DIVISOR = 4;
|
|
|
50
51
|
* 32-bit timer and Node silently clamps it to 1ms — turning a slow periodic tick into a busy loop. */
|
|
51
52
|
const MAX_TIMER_MS = 2_147_483_647;
|
|
52
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Default cadence (ms) of the defensive engine-reconcile pass (#661) — the safety net that releases a
|
|
56
|
+
* correlation whose engine JOB park is gone but whose terminal `lifecycle` event never arrived (an
|
|
57
|
+
* unclean worker exit). 30s trades a small staleness bound for a light engine-read load; the precise,
|
|
58
|
+
* immediate release stays the terminal-lifecycle path, so this only ever mops up unclean exits.
|
|
59
|
+
*/
|
|
60
|
+
const DEFAULT_ENGINE_RECONCILE_MS = 30_000;
|
|
61
|
+
|
|
53
62
|
/**
|
|
54
63
|
* The retention-sweep cadence (ms) for a given ephemeral-retention window: a fraction of the window,
|
|
55
64
|
* floored at 1ms and — crucially — capped at {@link MAX_TIMER_MS} so a large retention config (e.g.
|
|
@@ -64,12 +73,75 @@ export function sweepIntervalMs(ephemeralRetentionMs: number): number {
|
|
|
64
73
|
return Math.min(MAX_TIMER_MS, Math.max(1, interval));
|
|
65
74
|
}
|
|
66
75
|
|
|
76
|
+
/**
|
|
77
|
+
* The defensive engine-reconcile cadence (ms) for a given config (#661), or `undefined` to DISABLE the
|
|
78
|
+
* pass. An omitted config uses {@link DEFAULT_ENGINE_RECONCILE_MS}; a non-finite or non-positive value
|
|
79
|
+
* (a broken config, or a deliberate opt-out) disables the pass rather than degrading into a 1ms busy
|
|
80
|
+
* loop; a finite positive value is floored at 1ms (so a sub-millisecond config like 0.5 cannot floor
|
|
81
|
+
* to 0 and degrade into a busy `setInterval(0)`) and capped at {@link MAX_TIMER_MS} so a large window
|
|
82
|
+
* cannot overflow Node's 32-bit timer.
|
|
83
|
+
*/
|
|
84
|
+
export function engineReconcileMs(configuredMs?: number): number | undefined {
|
|
85
|
+
const value = configuredMs ?? DEFAULT_ENGINE_RECONCILE_MS;
|
|
86
|
+
if (!Number.isFinite(value) || value <= 0) return undefined;
|
|
87
|
+
return Math.min(MAX_TIMER_MS, Math.max(1, Math.floor(value)));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Wrap an async pass in an in-flight guard so a periodic `setInterval` never fires OVERLAPPING runs
|
|
92
|
+
* (#661). A single pass of {@link RelayTranscriptService.reconcileEngineCorrelations} awaits an engine
|
|
93
|
+
* read per linked job, so a pass can outlast its interval (a small configured cadence, or a slow/large
|
|
94
|
+
* engine read-model); an unguarded `setInterval` would then stack concurrent passes, piling up engine
|
|
95
|
+
* reads and log volume. While a pass is still pending, every subsequent tick is skipped; the next tick
|
|
96
|
+
* after it settles — whether it resolves OR rejects, since the guard clears via `.finally` — starts a
|
|
97
|
+
* fresh pass. The guard clears on BOTH failure modes so it can never wedge in-flight: (1) a
|
|
98
|
+
* *synchronous* throw from `pass` escapes before `.finally` is attached, so it is caught here — the
|
|
99
|
+
* guard is re-armed and the fault is re-thrown so it stays loud rather than being silently swallowed;
|
|
100
|
+
* (2) an async rejection is cleared by `.finally`, but the tick `void`s (does not await) the returned
|
|
101
|
+
* promise, so `pass` must still settle its own rejections or an unhandled rejection results — which is
|
|
102
|
+
* why the mount wraps `reconcileEngineCorrelations()` in `.catch`. Making the guard resilient to (1)
|
|
103
|
+
* removes a subtle footgun for a future caller (or refactor) that returns a non-`async` `pass`. Mirrors
|
|
104
|
+
* the "one pass at a time" discipline the main poll loop enforces by self-scheduling. Returns the tick
|
|
105
|
+
* callback to hand to `setInterval`.
|
|
106
|
+
*/
|
|
107
|
+
export function guardOverlappingPasses(pass: () => Promise<void>): () => void {
|
|
108
|
+
let inFlight = false;
|
|
109
|
+
return () => {
|
|
110
|
+
if (inFlight) return;
|
|
111
|
+
inFlight = true;
|
|
112
|
+
try {
|
|
113
|
+
void pass().finally(() => {
|
|
114
|
+
inFlight = false;
|
|
115
|
+
});
|
|
116
|
+
} catch (err) {
|
|
117
|
+
// A synchronous throw never attaches the `.finally`; re-arm the guard so it does not wedge,
|
|
118
|
+
// then re-throw so a contract violation still surfaces instead of being silently swallowed.
|
|
119
|
+
inFlight = false;
|
|
120
|
+
throw err;
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
67
125
|
/** Read a property off an unknown value without an unsafe `as` cast (mirrors the loader's helper). */
|
|
68
126
|
function readProp(value: unknown, key: string): unknown {
|
|
69
127
|
if (!value || typeof value !== "object") return undefined;
|
|
70
128
|
return Object.hasOwn(value, key) ? Object.getOwnPropertyDescriptor(value, key)?.value : undefined;
|
|
71
129
|
}
|
|
72
130
|
|
|
131
|
+
/**
|
|
132
|
+
* Decode a single relay chunk through the ONE canonical transcript parser and report whether it is a
|
|
133
|
+
* TERMINAL `lifecycle` event (`phase` `completed`/`exited`) — the authoritative "job end" signal a
|
|
134
|
+
* clean agent run emits on its transcript stream (#661). Anything else — raw terminal bytes, a
|
|
135
|
+
* non-envelope JSON value, a `phase: "open"` lifecycle, any other event kind — is not terminal.
|
|
136
|
+
* Reusing {@link parseTranscriptEvent} keeps transcript-vocab knowledge out of the content-agnostic
|
|
137
|
+
* relay ring and off any forked decoder (Derivation Over Duplication): the ring still sees opaque
|
|
138
|
+
* bytes; only this narrow seam classifies them.
|
|
139
|
+
*/
|
|
140
|
+
function isTerminalLifecycleChunk(chunk: string): boolean {
|
|
141
|
+
const event = parseTranscriptEvent({ offset: 0, chunk });
|
|
142
|
+
return event.kind === "lifecycle" && (event.phase === "completed" || event.phase === "exited");
|
|
143
|
+
}
|
|
144
|
+
|
|
73
145
|
/**
|
|
74
146
|
* A resume-from-offset source with no retained chunks — used to flush/complete a stream that a
|
|
75
147
|
* producer opened logically but never wrote to, so its transcript is still stamped `completed`
|
|
@@ -180,6 +252,19 @@ export interface RelayTranscriptServiceOptions {
|
|
|
180
252
|
* ({@link currentPresenceRegistry}). A resolver returning undefined → no linking (advisory).
|
|
181
253
|
*/
|
|
182
254
|
readonly instanceForConnection?: (connectionId: string) => string | undefined;
|
|
255
|
+
/**
|
|
256
|
+
* Whether a worker instance is still live on any hub connection (#689). Wired to the presence
|
|
257
|
+
* registry's {@link PresenceRegistry.isInstanceLive}. The disconnect-driven reconcile uses it to
|
|
258
|
+
* spare a still-active job's stream when its worker merely RECONNECTED mid-job (old producer
|
|
259
|
+
* connection dropped, a new one re-registered under the same instance): completing then would
|
|
260
|
+
* archive a live job's transcript and release its correlation, wedging the cockpit (the reconnected
|
|
261
|
+
* worker's produce frames hit a terminal `completed` stream and are ignored) while the harness
|
|
262
|
+
* keeps the engine lease. When wired to presence, a true worker-exit still completes: presence
|
|
263
|
+
* has dropped the instance, so this returns false. Omitted (`() => false`, the static default) →
|
|
264
|
+
* the prior always-complete-on-disconnect behaviour — every disconnect completes, reconnect
|
|
265
|
+
* included (the seam never consults presence, so its value does not vary).
|
|
266
|
+
*/
|
|
267
|
+
readonly isInstanceLive?: (instance: string) => boolean;
|
|
183
268
|
/**
|
|
184
269
|
* Resolve a producing worker instance's durable identity attributes (identity / host) — read at
|
|
185
270
|
* job-completion time and persisted with the attribution so a PAST session stays attributable to a
|
|
@@ -245,6 +330,8 @@ export class RelayTranscriptService {
|
|
|
245
330
|
readonly #correlation: () => CorrelationLink | undefined;
|
|
246
331
|
/** The connection → producing-instance resolver (H6, #149). */
|
|
247
332
|
readonly #instanceForConnection: (connectionId: string) => string | undefined;
|
|
333
|
+
/** Whether a worker instance is still live on any connection (#689) — gates disconnect completion. */
|
|
334
|
+
readonly #isInstanceLive: (instance: string) => boolean;
|
|
248
335
|
/** Resolve a worker instance's durable identity attributes for attribution (#485). */
|
|
249
336
|
readonly #attributionForInstance: (instance: string) => WorkerAttribution | undefined;
|
|
250
337
|
/** The durable worker-attribution store, or undefined when unpersisted (#485). */
|
|
@@ -259,6 +346,7 @@ export class RelayTranscriptService {
|
|
|
259
346
|
this.#log = options.log;
|
|
260
347
|
this.#correlation = options.correlation ?? currentCorrelation;
|
|
261
348
|
this.#instanceForConnection = options.instanceForConnection ?? (() => undefined);
|
|
349
|
+
this.#isInstanceLive = options.isInstanceLive ?? (() => false);
|
|
262
350
|
this.#attributionForInstance = options.attributionForInstance ?? (() => undefined);
|
|
263
351
|
this.#resolveElementInstance = options.resolveElementInstance;
|
|
264
352
|
this.#now = options.now ?? (() => new Date().toISOString());
|
|
@@ -424,11 +512,41 @@ export class RelayTranscriptService {
|
|
|
424
512
|
this.#streams.clear();
|
|
425
513
|
}
|
|
426
514
|
|
|
427
|
-
/** Handle one inbound `relay` frame: reconcile dead producers, observe ownership,
|
|
515
|
+
/** Handle one inbound `relay` frame: reconcile dead producers, observe ownership, delegate, then
|
|
516
|
+
* release on a terminal `lifecycle` event (#661 — AFTER the hub appended the chunk, so the terminal
|
|
517
|
+
* event itself is captured in the flushed transcript). */
|
|
428
518
|
#onFrame(frame: Frame, conn: RelayConnectionCtx): void {
|
|
429
519
|
this.#reconcile();
|
|
430
520
|
this.#observe(frame, conn);
|
|
431
521
|
this.relay.handle(frame, conn);
|
|
522
|
+
this.#observeTerminalLifecycle(frame);
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* Primary job-end release (#661): when a `produce` frame carries the terminal `lifecycle` event
|
|
527
|
+
* (`phase` `completed`/`exited`), complete its stream so the worker's job⇄instance correlation is
|
|
528
|
+
* released the moment the job ends — even though the worker's relay connection stays open across jobs
|
|
529
|
+
* (the disconnect/supersede release paths miss that idle-after-last-job tail, so an idle worker's
|
|
530
|
+
* finished job would otherwise linger as a phantom active job on its supply row). Runs AFTER the hub
|
|
531
|
+
* appends the chunk to the ring, so the terminal event is part of the flushed transcript. A narrow,
|
|
532
|
+
* self-contained decode at the correlation seam that reuses the ONE canonical
|
|
533
|
+
* {@link parseTranscriptEvent} — the content-agnostic relay ring keeps treating chunks as opaque
|
|
534
|
+
* bytes, and no transcript-vocab knowledge is forked into it. Non-terminal chunks (raw bytes, a
|
|
535
|
+
* `phase: "open"` lifecycle, any other event kind) never complete a live stream, so a genuinely
|
|
536
|
+
* active job is never cleared. Advisory — never throws into the frame handler.
|
|
537
|
+
*/
|
|
538
|
+
#observeTerminalLifecycle(frame: Frame): void {
|
|
539
|
+
if (readProp(frame.payload, "op") !== "produce") return;
|
|
540
|
+
const stream = readProp(frame.payload, "stream");
|
|
541
|
+
if (typeof stream !== "string" || stream === "") return;
|
|
542
|
+
const chunk = readProp(frame.payload, "chunk");
|
|
543
|
+
if (typeof chunk !== "string" || chunk === "") return;
|
|
544
|
+
const state = this.#streams.get(stream);
|
|
545
|
+
// Idempotent: an unknown or already-completed stream needs no (further) release — a second terminal
|
|
546
|
+
// event, or a late one after the disconnect/supersede path already completed the stream, is a no-op.
|
|
547
|
+
if (state === undefined || state.completed) return;
|
|
548
|
+
if (!isTerminalLifecycleChunk(chunk)) return;
|
|
549
|
+
this.completeStream(stream);
|
|
432
550
|
}
|
|
433
551
|
|
|
434
552
|
/** Record `produce` ownership so a producer disconnect can drive ephemeral completion. */
|
|
@@ -616,6 +734,19 @@ export class RelayTranscriptService {
|
|
|
616
734
|
#reconcile(): void {
|
|
617
735
|
for (const [stream, state] of this.#streams) {
|
|
618
736
|
if (state.producer !== undefined && !this.#registry.has(state.producer)) {
|
|
737
|
+
// Producer connection gone. Normally that means the job it was relaying ended — release the
|
|
738
|
+
// correlation and flush+complete its ephemeral transcript. BUT a worker that merely
|
|
739
|
+
// RECONNECTED mid-job (#689) also loses its old producer connection while its job keeps
|
|
740
|
+
// running (the harness holds the engine lease and extends it). Presence re-registers the SAME
|
|
741
|
+
// instance under the new connection, so the instance stays live even though this specific
|
|
742
|
+
// producer connection is gone. Completing then would archive a still-active job's transcript
|
|
743
|
+
// to `historical` and drop its correlation — and because a completed stream is terminal
|
|
744
|
+
// (`#observe` ignores later frames), the reconnected worker could NEVER re-correlate: the
|
|
745
|
+
// cockpit shows the worker idle with a frozen transcript while the job is genuinely running.
|
|
746
|
+
// So spare the stream while its instance is still live; the next `produce` re-attributes the
|
|
747
|
+
// live connection (via `#observe`), a NEW job supersedes it (via `#link`), or — once the
|
|
748
|
+
// worker truly exits — presence drops the instance and a later reconcile completes it.
|
|
749
|
+
if (state.instance !== undefined && this.#isInstanceLive(state.instance)) continue;
|
|
619
750
|
// Producer connection gone → the job it was relaying ended: release its correlation.
|
|
620
751
|
this.#unlink(stream, state);
|
|
621
752
|
// ...and flush+complete an ephemeral, not-yet-completed transcript exactly as before.
|
|
@@ -624,6 +755,61 @@ export class RelayTranscriptService {
|
|
|
624
755
|
}
|
|
625
756
|
}
|
|
626
757
|
|
|
758
|
+
/**
|
|
759
|
+
* Defensive engine-reconcile safety net (#661): release any linked jobKey whose engine JOB park is
|
|
760
|
+
* no longer live. The precise, fast release is the terminal `lifecycle` event
|
|
761
|
+
* ({@link #observeTerminalLifecycle}), but an UNCLEAN worker exit (crash/kill) can skip that event —
|
|
762
|
+
* and because the worker's relay connection is persistent across jobs, the disconnect release never
|
|
763
|
+
* fires either, so the finished job would linger as a phantom active job on the worker's supply row.
|
|
764
|
+
* This periodic pass asks the engine read model (the same {@link ElementInstanceResolver} the link
|
|
765
|
+
* path uses at #544) whether each linked job is still parked; a job the engine no longer parks
|
|
766
|
+
* (resolver returns `undefined`) is released and its transcript completed. Bounds staleness regardless
|
|
767
|
+
* of whether the worker emitted a clean terminal event — and also covers a worker that emitted nothing
|
|
768
|
+
* and merely went quiet.
|
|
769
|
+
*
|
|
770
|
+
* Advisory and best-effort: a no-op with no resolver wired, and a resolver THROW / REJECTION for a
|
|
771
|
+
* given job is treated as "unknown — keep it linked" (never a false release of a genuinely active
|
|
772
|
+
* job). The linked set is snapshotted before any await so a concurrent completion (a terminal
|
|
773
|
+
* lifecycle event landing mid-pass) cannot corrupt iteration, and each release re-checks the current
|
|
774
|
+
* stream state so a job already released between snapshot and resolution is not double-completed.
|
|
775
|
+
*/
|
|
776
|
+
async reconcileEngineCorrelations(): Promise<void> {
|
|
777
|
+
const resolve = this.#resolveElementInstance;
|
|
778
|
+
if (resolve === undefined) return;
|
|
779
|
+
const linked: { stream: string; jobKey: string; processInstanceKey?: string }[] = [];
|
|
780
|
+
for (const [stream, state] of this.#streams) {
|
|
781
|
+
if (!state.linked || state.completed) continue;
|
|
782
|
+
const jobKey = jobKeyOfStream(stream);
|
|
783
|
+
if (jobKey === undefined) continue;
|
|
784
|
+
const processInstanceKey = this.#correlation()?.resolve?.(jobKey)?.processInstanceKey;
|
|
785
|
+
linked.push({ stream, jobKey, processInstanceKey });
|
|
786
|
+
}
|
|
787
|
+
for (const { stream, jobKey, processInstanceKey } of linked) {
|
|
788
|
+
let activeKey: string | undefined;
|
|
789
|
+
try {
|
|
790
|
+
activeKey = await resolve(jobKey, processInstanceKey);
|
|
791
|
+
} catch (err) {
|
|
792
|
+
// A transient engine read failure must NOT be read as "job gone" — leave the job linked; a
|
|
793
|
+
// later pass (or the terminal lifecycle event) releases it. Advisory, never a false release.
|
|
794
|
+
this.#log.warn("agentic relay engine-reconcile read failed — leaving correlation linked", {
|
|
795
|
+
stream,
|
|
796
|
+
jobKey,
|
|
797
|
+
err: String(err),
|
|
798
|
+
});
|
|
799
|
+
continue;
|
|
800
|
+
}
|
|
801
|
+
// A live JOB park (a resolved element-instance key) means the job is genuinely active — keep it.
|
|
802
|
+
if (activeKey !== undefined) continue;
|
|
803
|
+
// The engine no longer parks this job → it ended (possibly via an unclean exit that skipped the
|
|
804
|
+
// terminal lifecycle event). Re-check the current state — a concurrent completion may already
|
|
805
|
+
// have released it — then release its correlation and flush its transcript.
|
|
806
|
+
const current = this.#streams.get(stream);
|
|
807
|
+
if (current === undefined || current.completed || !current.linked) continue;
|
|
808
|
+
this.#log.info("agentic relay engine-reconcile released a stale correlation", { stream, jobKey });
|
|
809
|
+
this.completeStream(stream);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
|
|
627
813
|
/**
|
|
628
814
|
* The still-live relay ring for a stream, for the read path to serve BEFORE a durable flush (#486).
|
|
629
815
|
*
|
|
@@ -670,11 +856,18 @@ export function createRelayFamily(options: {
|
|
|
670
856
|
readonly relay?: RelayHubOptions;
|
|
671
857
|
readonly transcript?: TranscriptStoreOptions;
|
|
672
858
|
readonly ensureSchema?: boolean;
|
|
859
|
+
/**
|
|
860
|
+
* Cadence (ms) of the defensive engine-reconcile pass (#661). Defaults to
|
|
861
|
+
* {@link DEFAULT_ENGINE_RECONCILE_MS}. Clamped to Node's 32-bit timer ceiling; a non-positive /
|
|
862
|
+
* non-finite value disables the pass (the terminal-lifecycle release path still runs).
|
|
863
|
+
*/
|
|
864
|
+
readonly engineReconcileIntervalMs?: number;
|
|
673
865
|
/** Called with the live service once mounted, so a driver can drive completion/reattach. */
|
|
674
866
|
readonly onMounted?: (service: RelayTranscriptService) => void;
|
|
675
867
|
} = {}): AgenticFamily {
|
|
676
868
|
let service: RelayTranscriptService | undefined;
|
|
677
869
|
let sweepTimer: ReturnType<typeof setInterval> | undefined;
|
|
870
|
+
let engineReconcileTimer: ReturnType<typeof setInterval> | undefined;
|
|
678
871
|
return {
|
|
679
872
|
name: RELAY_FAMILY_NAME,
|
|
680
873
|
mount(ctx: AgenticContext): void {
|
|
@@ -693,6 +886,11 @@ export function createRelayFamily(options: {
|
|
|
693
886
|
// are read per call, so this works regardless of family mount order (relay may mount before
|
|
694
887
|
// presence/correlation). Absent registries → no linking, still advisory-correct.
|
|
695
888
|
instanceForConnection: (connectionId) => currentPresenceRegistry()?.instanceForConnection(connectionId),
|
|
889
|
+
// #689: is the producing worker instance still live on ANY connection? Gates the
|
|
890
|
+
// disconnect-driven reconcile so a mid-job RECONNECT (old producer connection dropped, the
|
|
891
|
+
// same instance re-registered) does not archive a still-active job's stream and wedge its
|
|
892
|
+
// correlation. Read per call for the same mount-order independence as the resolvers above.
|
|
893
|
+
isInstanceLive: (instance) => currentPresenceRegistry()?.isInstanceLive(instance) ?? false,
|
|
696
894
|
// #485: resolve a completed job's worker attribution (presence identity/host) from the live
|
|
697
895
|
// presence registry, read per call for the same mount-order independence. Absent → attribution
|
|
698
896
|
// records instance only.
|
|
@@ -729,6 +927,31 @@ export function createRelayFamily(options: {
|
|
|
729
927
|
tick();
|
|
730
928
|
}
|
|
731
929
|
|
|
930
|
+
// Defensive engine-reconcile safety net (#661): periodically release any linked correlation
|
|
931
|
+
// whose engine JOB park is gone but whose terminal `lifecycle` event never arrived (an unclean
|
|
932
|
+
// worker exit), so a crashed worker's finished job stops showing as a phantom active job. Only
|
|
933
|
+
// useful when an element-instance resolver is wired (engine read-model access); harmless no-op
|
|
934
|
+
// otherwise. Advisory — a reconcile fault is logged, never thrown, and never keeps the process
|
|
935
|
+
// alive on its own.
|
|
936
|
+
if (ctx.resolveElementInstance !== undefined) {
|
|
937
|
+
const reconcileInterval = engineReconcileMs(options.engineReconcileIntervalMs);
|
|
938
|
+
if (reconcileInterval !== undefined) {
|
|
939
|
+
// In-flight guard: `reconcileEngineCorrelations()` is async and awaits an engine read per
|
|
940
|
+
// linked job, so a pass can outlast `reconcileInterval` (a small interval, or a slow/large
|
|
941
|
+
// engine read-model). Without a guard, `setInterval` would fire overlapping passes that pile
|
|
942
|
+
// up concurrent engine reads and log volume. {@link guardOverlappingPasses} skips a tick while
|
|
943
|
+
// the previous pass is still running so only one reconcile runs at a time — the same "one pass
|
|
944
|
+
// at a time" discipline the main poll loop enforces by self-scheduling.
|
|
945
|
+
const reconcileTick = guardOverlappingPasses(() =>
|
|
946
|
+
(service?.reconcileEngineCorrelations() ?? Promise.resolve()).catch((err: unknown) => {
|
|
947
|
+
ctx.log.warn("agentic relay engine-reconcile failed", { err: String(err) });
|
|
948
|
+
}),
|
|
949
|
+
);
|
|
950
|
+
engineReconcileTimer = setInterval(reconcileTick, reconcileInterval);
|
|
951
|
+
engineReconcileTimer.unref?.();
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
|
|
732
955
|
options.onMounted?.(service);
|
|
733
956
|
},
|
|
734
957
|
teardown(): void {
|
|
@@ -736,6 +959,10 @@ export function createRelayFamily(options: {
|
|
|
736
959
|
clearInterval(sweepTimer);
|
|
737
960
|
sweepTimer = undefined;
|
|
738
961
|
}
|
|
962
|
+
if (engineReconcileTimer !== undefined) {
|
|
963
|
+
clearInterval(engineReconcileTimer);
|
|
964
|
+
engineReconcileTimer = undefined;
|
|
965
|
+
}
|
|
739
966
|
service?.teardown();
|
|
740
967
|
if (currentService === service) setCurrentRelayTranscriptService(undefined);
|
|
741
968
|
service = undefined;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nanobpm/nano-workforce",
|
|
3
|
-
"version": "0.171.
|
|
3
|
+
"version": "0.171.9",
|
|
4
4
|
"description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "main.ts",
|