instar 1.3.334 → 1.3.336
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/FailureEpisodeLatch.d.ts +50 -0
- package/dist/core/FailureEpisodeLatch.d.ts.map +1 -0
- package/dist/core/FailureEpisodeLatch.js +59 -0
- package/dist/core/FailureEpisodeLatch.js.map +1 -0
- package/dist/core/HttpLiveTailTransport.d.ts +19 -0
- package/dist/core/HttpLiveTailTransport.d.ts.map +1 -1
- package/dist/core/HttpLiveTailTransport.js +25 -4
- package/dist/core/HttpLiveTailTransport.js.map +1 -1
- package/dist/core/MultiMachineCoordinator.d.ts +26 -0
- package/dist/core/MultiMachineCoordinator.d.ts.map +1 -1
- package/dist/core/MultiMachineCoordinator.js +70 -4
- package/dist/core/MultiMachineCoordinator.js.map +1 -1
- package/dist/core/ReplyMarkerTransport.d.ts +19 -0
- package/dist/core/ReplyMarkerTransport.d.ts.map +1 -1
- package/dist/core/ReplyMarkerTransport.js +25 -4
- package/dist/core/ReplyMarkerTransport.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +3 -3
- package/upgrades/1.3.335.md +53 -0
- package/upgrades/1.3.336.md +45 -0
- package/upgrades/side-effects/heartbeat-writer-signal.md +55 -0
- package/upgrades/side-effects/sibling-transport-brakes.md +43 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FailureEpisodeLatch — the generic failure-episode accountant behind the
|
|
3
|
+
* "No Unbounded Loops" (P19) Eternal Sentinel condition 4: a loop that retries
|
|
4
|
+
* forever must (a) log state CHANGES, not attempts, and (b) raise ONE
|
|
5
|
+
* sustained-failure signal per episode, then stay quiet while retrying.
|
|
6
|
+
*
|
|
7
|
+
* Third night-of-2026-06-05 incarnation of the episode-latch shape
|
|
8
|
+
* (SlowRetrySentinelEscalation in lifeline; the inline live-tail stale latch)
|
|
9
|
+
* — extracted to core as the canonical reusable form so future loops stop
|
|
10
|
+
* re-implementing it <!-- tracked: CMT-1109 -->. Pure: injectable clock, three
|
|
11
|
+
* fields of state, no I/O.
|
|
12
|
+
*
|
|
13
|
+
* Usage per attempt:
|
|
14
|
+
* const f = latch.recordFailure(); // on a failed attempt
|
|
15
|
+
* if (f.firstOfEpisode) log("X became failing: ...");
|
|
16
|
+
* if (f.shouldSignal) reportDegradationOnce(...); // fires ONCE per episode
|
|
17
|
+
* ...
|
|
18
|
+
* const s = latch.recordSuccess(); // on a successful attempt
|
|
19
|
+
* if (s.recovered) log(`X recovered after ${s.failures} failures`);
|
|
20
|
+
*/
|
|
21
|
+
export interface FailureEpisodeLatchOpts {
|
|
22
|
+
/** Sustained-failure threshold before the one-per-episode signal. */
|
|
23
|
+
signalAfterMs: number;
|
|
24
|
+
now?: () => number;
|
|
25
|
+
}
|
|
26
|
+
export interface FailureRecord {
|
|
27
|
+
/** True exactly once — the attempt that started this episode. */
|
|
28
|
+
firstOfEpisode: boolean;
|
|
29
|
+
/** True exactly once per episode — the first failed attempt at/after the threshold. */
|
|
30
|
+
shouldSignal: boolean;
|
|
31
|
+
/** Episode duration so far (ms). */
|
|
32
|
+
failingForMs: number;
|
|
33
|
+
/** Consecutive failures including this one. */
|
|
34
|
+
failures: number;
|
|
35
|
+
}
|
|
36
|
+
export declare class FailureEpisodeLatch {
|
|
37
|
+
private readonly signalAfterMs;
|
|
38
|
+
private readonly now;
|
|
39
|
+
private failingSince;
|
|
40
|
+
private failures;
|
|
41
|
+
/** Episode key (failingSince value) the signal already fired for — null = armed. */
|
|
42
|
+
private signaledFor;
|
|
43
|
+
constructor(opts: FailureEpisodeLatchOpts);
|
|
44
|
+
recordFailure(): FailureRecord;
|
|
45
|
+
recordSuccess(): {
|
|
46
|
+
recovered: boolean;
|
|
47
|
+
failures: number;
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=FailureEpisodeLatch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FailureEpisodeLatch.d.ts","sourceRoot":"","sources":["../../src/core/FailureEpisodeLatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,uBAAuB;IACtC,qEAAqE;IACrE,aAAa,EAAE,MAAM,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,iEAAiE;IACjE,cAAc,EAAE,OAAO,CAAC;IACxB,uFAAuF;IACvF,YAAY,EAAE,OAAO,CAAC;IACtB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IAKnC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,QAAQ,CAAK;IACrB,oFAAoF;IACpF,OAAO,CAAC,WAAW,CAAuB;gBAE9B,IAAI,EAAE,uBAAuB;IAKzC,aAAa,IAAI,aAAa;IAc9B,aAAa,IAAI;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;CAQ1D"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FailureEpisodeLatch — the generic failure-episode accountant behind the
|
|
3
|
+
* "No Unbounded Loops" (P19) Eternal Sentinel condition 4: a loop that retries
|
|
4
|
+
* forever must (a) log state CHANGES, not attempts, and (b) raise ONE
|
|
5
|
+
* sustained-failure signal per episode, then stay quiet while retrying.
|
|
6
|
+
*
|
|
7
|
+
* Third night-of-2026-06-05 incarnation of the episode-latch shape
|
|
8
|
+
* (SlowRetrySentinelEscalation in lifeline; the inline live-tail stale latch)
|
|
9
|
+
* — extracted to core as the canonical reusable form so future loops stop
|
|
10
|
+
* re-implementing it <!-- tracked: CMT-1109 -->. Pure: injectable clock, three
|
|
11
|
+
* fields of state, no I/O.
|
|
12
|
+
*
|
|
13
|
+
* Usage per attempt:
|
|
14
|
+
* const f = latch.recordFailure(); // on a failed attempt
|
|
15
|
+
* if (f.firstOfEpisode) log("X became failing: ...");
|
|
16
|
+
* if (f.shouldSignal) reportDegradationOnce(...); // fires ONCE per episode
|
|
17
|
+
* ...
|
|
18
|
+
* const s = latch.recordSuccess(); // on a successful attempt
|
|
19
|
+
* if (s.recovered) log(`X recovered after ${s.failures} failures`);
|
|
20
|
+
*/
|
|
21
|
+
export class FailureEpisodeLatch {
|
|
22
|
+
signalAfterMs;
|
|
23
|
+
now;
|
|
24
|
+
// null sentinels, NOT 0 — an episode legitimately starting at clock value 0
|
|
25
|
+
// (tests with injected clocks; epoch-0 edge) must not collide with the
|
|
26
|
+
// "no episode" state. (The Dawn "zero is falsy" lesson, caught by the P19
|
|
27
|
+
// sustained-failure test before this ever shipped.)
|
|
28
|
+
failingSince = null;
|
|
29
|
+
failures = 0;
|
|
30
|
+
/** Episode key (failingSince value) the signal already fired for — null = armed. */
|
|
31
|
+
signaledFor = null;
|
|
32
|
+
constructor(opts) {
|
|
33
|
+
this.signalAfterMs = opts.signalAfterMs;
|
|
34
|
+
this.now = opts.now ?? Date.now;
|
|
35
|
+
}
|
|
36
|
+
recordFailure() {
|
|
37
|
+
const now = this.now();
|
|
38
|
+
const firstOfEpisode = this.failingSince === null;
|
|
39
|
+
if (this.failingSince === null)
|
|
40
|
+
this.failingSince = now;
|
|
41
|
+
this.failures++;
|
|
42
|
+
const failingForMs = now - this.failingSince;
|
|
43
|
+
let shouldSignal = false;
|
|
44
|
+
if (failingForMs >= this.signalAfterMs && this.signaledFor !== this.failingSince) {
|
|
45
|
+
this.signaledFor = this.failingSince;
|
|
46
|
+
shouldSignal = true;
|
|
47
|
+
}
|
|
48
|
+
return { firstOfEpisode, shouldSignal, failingForMs, failures: this.failures };
|
|
49
|
+
}
|
|
50
|
+
recordSuccess() {
|
|
51
|
+
const failures = this.failures;
|
|
52
|
+
const recovered = failures > 0;
|
|
53
|
+
this.failingSince = null;
|
|
54
|
+
this.failures = 0;
|
|
55
|
+
this.signaledFor = null;
|
|
56
|
+
return { recovered, failures };
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=FailureEpisodeLatch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FailureEpisodeLatch.js","sourceRoot":"","sources":["../../src/core/FailureEpisodeLatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAmBH,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAS;IACtB,GAAG,CAAe;IACnC,4EAA4E;IAC5E,uEAAuE;IACvE,0EAA0E;IAC1E,oDAAoD;IAC5C,YAAY,GAAkB,IAAI,CAAC;IACnC,QAAQ,GAAG,CAAC,CAAC;IACrB,oFAAoF;IAC5E,WAAW,GAAkB,IAAI,CAAC;IAE1C,YAAY,IAA6B;QACvC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QACxC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IAClC,CAAC;IAED,aAAa;QACX,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC;QAClD,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI;YAAE,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,MAAM,YAAY,GAAG,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC;QAC7C,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,YAAY,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;YACjF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC;YACrC,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;IACjF,CAAC;IAED,aAAa;QACX,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,MAAM,SAAS,GAAG,QAAQ,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;IACjC,CAAC;CACF"}
|
|
@@ -59,6 +59,20 @@ export interface HttpLiveTailTransportDeps {
|
|
|
59
59
|
fetchImpl?: typeof fetch;
|
|
60
60
|
/** How recent a successful broadcast counts as "reachable". Default = 60s. */
|
|
61
61
|
reachabilityWindowMs?: number;
|
|
62
|
+
/**
|
|
63
|
+
* Per-request abort timeout (P19 brake — a hung socket must not hold a
|
|
64
|
+
* flush open indefinitely). Default 30s: sized ABOVE the fleet's documented
|
|
65
|
+
* 5–40s receiver-stall envelope (the #874 reviewer's sizing rationale) so a
|
|
66
|
+
* slow-but-alive standby is not converted into "unreachable".
|
|
67
|
+
*/
|
|
68
|
+
requestTimeoutMs?: number;
|
|
69
|
+
/**
|
|
70
|
+
* Coarse-reminder interval for the per-peer failure log gate (P19 brake).
|
|
71
|
+
* Default 360: with #867's per-topic backoff the attempt rate is bounded,
|
|
72
|
+
* but N topics against one down peer still meant N lines per backoff window
|
|
73
|
+
* — the gate collapses that to first/Nth/recovery per peer.
|
|
74
|
+
*/
|
|
75
|
+
failureLogEveryN?: number;
|
|
62
76
|
now?: () => number;
|
|
63
77
|
logger?: (msg: string) => void;
|
|
64
78
|
}
|
|
@@ -73,7 +87,12 @@ export declare class HttpLiveTailTransport {
|
|
|
73
87
|
private readonly d;
|
|
74
88
|
private lastBroadcastOkAt;
|
|
75
89
|
private readonly windowMs;
|
|
90
|
+
private readonly requestTimeoutMs;
|
|
91
|
+
/** State-change failure logging (first/Nth/recovery) — never per-attempt. */
|
|
92
|
+
private readonly logGate;
|
|
76
93
|
constructor(deps: HttpLiveTailTransportDeps);
|
|
94
|
+
private logFailure;
|
|
95
|
+
private logSuccess;
|
|
77
96
|
private now;
|
|
78
97
|
private log;
|
|
79
98
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HttpLiveTailTransport.d.ts","sourceRoot":"","sources":["../../src/core/HttpLiveTailTransport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;
|
|
1
|
+
{"version":3,"file":"HttpLiveTailTransport.d.ts","sourceRoot":"","sources":["../../src/core/HttpLiveTailTransport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAKH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAE/D,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,GAAG,EAAE,MAAM,CAAC;IACZ,6EAA6E;IAC7E,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,qFAAqF;AACrF,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,sBAAsB,CAAC;IAC5B,2EAA2E;IAC3E,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,yFAAyF;IACzF,KAAK,EAAE,MAAM,YAAY,EAAE,CAAC;IAC5B,yEAAyE;IACzE,YAAY,EAAE,MAAM,MAAM,CAAC;IAC3B;;;OAGG;IACH,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,KAAK,sBAAsB,CAAC;IACpF,iDAAiD;IACjD,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,8EAA8E;IAC9E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ,qDAAqD;IACrD,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAA4B;IAC9C,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAEjC,IAAI,EAAE,yBAAyB;IAO3C,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,UAAU;IAKlB,OAAO,CAAC,GAAG;IAGX,OAAO,CAAC,GAAG;IAIX;;;;OAIG;IACG,SAAS,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,OAAO,CAAC;IA4C5D,uFAAuF;IACvF,WAAW,IAAI,OAAO;CAGvB"}
|
|
@@ -28,14 +28,30 @@
|
|
|
28
28
|
* redact→encrypt→post logic is unit-testable without a network or real keys.
|
|
29
29
|
*/
|
|
30
30
|
import { signRequest } from '../server/machineAuth.js';
|
|
31
|
+
import { PeerFailureLogGate } from './PeerFailureLogGate.js';
|
|
31
32
|
import { redactForLiveTail } from './liveTailRedaction.js';
|
|
32
33
|
export class HttpLiveTailTransport {
|
|
33
34
|
d;
|
|
34
35
|
lastBroadcastOkAt = 0;
|
|
35
36
|
windowMs;
|
|
37
|
+
requestTimeoutMs;
|
|
38
|
+
/** State-change failure logging (first/Nth/recovery) — never per-attempt. */
|
|
39
|
+
logGate;
|
|
36
40
|
constructor(deps) {
|
|
37
41
|
this.d = deps;
|
|
38
42
|
this.windowMs = deps.reachabilityWindowMs ?? 60_000;
|
|
43
|
+
this.requestTimeoutMs = deps.requestTimeoutMs ?? 30_000;
|
|
44
|
+
this.logGate = new PeerFailureLogGate(deps.failureLogEveryN ?? 360);
|
|
45
|
+
}
|
|
46
|
+
logFailure(key, detail) {
|
|
47
|
+
const line = this.logGate.failed(key, detail);
|
|
48
|
+
if (line)
|
|
49
|
+
this.log(line);
|
|
50
|
+
}
|
|
51
|
+
logSuccess(key) {
|
|
52
|
+
const line = this.logGate.succeeded(key);
|
|
53
|
+
if (line)
|
|
54
|
+
this.log(line);
|
|
39
55
|
}
|
|
40
56
|
now() {
|
|
41
57
|
return (this.d.now ?? Date.now)();
|
|
@@ -73,14 +89,19 @@ export class HttpLiveTailTransport {
|
|
|
73
89
|
method: 'POST',
|
|
74
90
|
headers: { 'Content-Type': 'application/json', ...headers },
|
|
75
91
|
body: JSON.stringify(body),
|
|
92
|
+
// P19 brake: a hung socket aborts instead of holding the flush open.
|
|
93
|
+
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
|
76
94
|
});
|
|
77
|
-
if (res && res.ok)
|
|
95
|
+
if (res && res.ok) {
|
|
78
96
|
anyOk = true;
|
|
79
|
-
|
|
80
|
-
|
|
97
|
+
this.logSuccess(`live-tail to ${peer.machineId}`);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
this.logFailure(`live-tail to ${peer.machineId}`, `status ${res?.status} (seq=${flush.seq})`);
|
|
101
|
+
}
|
|
81
102
|
}
|
|
82
103
|
catch (err) {
|
|
83
|
-
this.
|
|
104
|
+
this.logFailure(`live-tail to ${peer.machineId}`, err instanceof Error ? err.message : String(err));
|
|
84
105
|
}
|
|
85
106
|
}));
|
|
86
107
|
if (anyOk)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HttpLiveTailTransport.js","sourceRoot":"","sources":["../../src/core/HttpLiveTailTransport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"HttpLiveTailTransport.js","sourceRoot":"","sources":["../../src/core/HttpLiveTailTransport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AA8D3D,MAAM,OAAO,qBAAqB;IACf,CAAC,CAA4B;IACtC,iBAAiB,GAAG,CAAC,CAAC;IACb,QAAQ,CAAS;IACjB,gBAAgB,CAAS;IAC1C,6EAA6E;IAC5D,OAAO,CAAqB;IAE7C,YAAY,IAA+B;QACzC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,oBAAoB,IAAI,MAAM,CAAC;QACpD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,MAAM,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAC,CAAC;IACtE,CAAC;IAEO,UAAU,CAAC,GAAW,EAAE,MAAc;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC9C,IAAI,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IACO,UAAU,CAAC,GAAW;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAEO,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,CAAC;IACO,GAAG,CAAC,CAAS;QACnB,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,SAAS,CAAC,KAAyB;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,+EAA+E;QAC/E,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,QAAQ,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,GAAG,CAAC,YAAY,QAAQ,CAAC,aAAa,eAAe,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC1G,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC;QAC5C,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,IAAI,CAAC;gBACH,sEAAsE;gBACtE,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAiC;oBACzC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,gBAAgB,EAAE,QAAQ,CAAC,OAAO,EAAE;iBACvF,CAAC;gBACF,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC;gBACrG,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,EAAE;oBAC1E,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,OAAO,EAAE;oBAC3D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAC1B,qEAAqE;oBACrE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC;iBACnD,CAAC,CAAC;gBACH,IAAI,GAAG,IAAK,GAAgB,CAAC,EAAE,EAAE,CAAC;oBAChC,KAAK,GAAG,IAAI,CAAC;oBACb,IAAI,CAAC,UAAU,CAAC,gBAAgB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,UAAU,CAAC,gBAAgB,IAAI,CAAC,SAAS,EAAE,EAAE,UAAW,GAAgB,EAAE,MAAM,SAAS,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;gBAC9G,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,UAAU,CAAC,gBAAgB,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACtG,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,IAAI,KAAK;YAAE,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,uFAAuF;IACvF,WAAW;QACT,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,QAAQ,CAAC;IAC9D,CAAC;CACF"}
|
|
@@ -60,6 +60,20 @@ export declare class MultiMachineCoordinator extends EventEmitter {
|
|
|
60
60
|
private _identity;
|
|
61
61
|
private _enabled;
|
|
62
62
|
private heartbeatWriteTimer;
|
|
63
|
+
/**
|
|
64
|
+
* Heartbeat-write failure episode accounting ("No Unbounded Loops" / P19,
|
|
65
|
+
* Eternal Sentinel condition 4). writeHeartbeat() throws raw fs errors
|
|
66
|
+
* (ENOSPC, EACCES) — pre-fix, a throw inside the 2-min timer tick escaped as
|
|
67
|
+
* an uncaughtException and CRASHED the awake holder (the worst possible cost
|
|
68
|
+
* for one failed attempt), while a hypothetical swallowed failure would have
|
|
69
|
+
* gone silent until a peer force-failed-over. Now: each tick's write is
|
|
70
|
+
* guarded, the first failure of an episode logs once, a sustained episode
|
|
71
|
+
* raises ONE degradation signal (before the peer failover horizon), recovery
|
|
72
|
+
* logs once and re-arms. The write keeps being attempted every tick forever
|
|
73
|
+
* — this writer is the awake machine's liveness voice; persistence is the
|
|
74
|
+
* point.
|
|
75
|
+
*/
|
|
76
|
+
private readonly hbWriteEpisode;
|
|
63
77
|
private heartbeatCheckTimer;
|
|
64
78
|
/** Integrated-Being v1 — tracks whether we've already emitted the
|
|
65
79
|
* per-machine-ledger warning this boot. Spec §Multi-machine. */
|
|
@@ -265,6 +279,18 @@ export declare class MultiMachineCoordinator extends EventEmitter {
|
|
|
265
279
|
* Start periodic heartbeat writes (awake machine only).
|
|
266
280
|
*/
|
|
267
281
|
private startHeartbeatWriter;
|
|
282
|
+
/**
|
|
283
|
+
* ETERNAL SENTINEL (declared per "No Unbounded Loops" / P19): the heartbeat
|
|
284
|
+
* writer is the awake machine's liveness voice — it must keep attempting
|
|
285
|
+
* every tick forever (rate floor = HEARTBEAT_WRITE_INTERVAL_MS, constant
|
|
286
|
+
* cost). Its brakes live here: a write failure can no longer escape the
|
|
287
|
+
* timer tick as an uncaughtException (pre-fix: ENOSPC at the wrong moment
|
|
288
|
+
* CRASHED the awake holder), failure logging is state-change-bounded (first
|
|
289
|
+
* + recovery, one line each), and a sustained episode raises ONE degradation
|
|
290
|
+
* signal — sized at 6min (3 failed cycles) so the operator hears about it
|
|
291
|
+
* BEFORE the peer's ~15min heartbeat-expiry failover horizon.
|
|
292
|
+
*/
|
|
293
|
+
private writeHeartbeatGuarded;
|
|
268
294
|
/**
|
|
269
295
|
* Start periodic heartbeat monitoring (all machines).
|
|
270
296
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MultiMachineCoordinator.d.ts","sourceRoot":"","sources":["../../src/core/MultiMachineCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"MultiMachineCoordinator.d.ts","sourceRoot":"","sources":["../../src/core/MultiMachineCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAI9D,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAErG,0EAA0E;AAC1E,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,8FAA8F;IAC9F,eAAe,EAAE,OAAO,GAAG,WAAW,GAAG,gBAAgB,CAAC;IAC1D,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAYD,MAAM,WAAW,iBAAiB;IAChC,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,YAAY,CAAC,EAAE,kBAAkB,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,wDAAwD;IACxD,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,yDAAyD;IACzD,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,0CAA0C;IAC1C,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,6BAA6B;IAC7B,UAAU,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;CAC1D;AAID,qBAAa,uBAAwB,SAAQ,YAAY;IACvD,OAAO,CAAC,eAAe,CAAyB;IAChD,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,KAAK,CAA0B;IACvC,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA+C;IAC1E;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0D;IACzF,OAAO,CAAC,mBAAmB,CAA+C;IAC1E;qEACiE;IACjE,OAAO,CAAC,6BAA6B,CAAkB;IACvD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAiC;IACzD,OAAO,CAAC,YAAY,CAAkB;IACtC;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,gBAAgB,CAAkB;IAC1C;;;;;;;;;;OAUG;IACH,OAAO,CAAC,gBAAgB,CAAuF;gBAEnG,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,iBAAiB;IAc1D,uDAAuD;IACvD,IAAI,OAAO,IAAI,OAAO,CAA0B;IAEhD,mCAAmC;IACnC,IAAI,IAAI,IAAI,WAAW,CAAuB;IAE9C,yDAAyD;IACzD,IAAI,QAAQ,IAAI,eAAe,GAAG,IAAI,CAA2B;IAEjE,2DAA2D;IAC3D,IAAI,OAAO,IAAI,OAAO,CAAmC;IAEzD;;;;;;;;;;;;OAYG;IACH,IAAI,kBAAkB,IAAI,OAAO,CAEhC;IAED,0DAA0D;IAC1D,IAAI,gBAAgB,IAAI,gBAAgB,CAEvC;IAED,kDAAkD;IAClD,IAAI,QAAQ;;;;;MAOX;IAID;;;OAGG;IACH,KAAK,IAAI,WAAW;IAuGpB;;OAEG;IACH;;;;OAIG;IACH,OAAO,CAAC,sCAAsC;IAiB9C,IAAI,IAAI,IAAI;IAmBZ;;;OAGG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IA0CpC;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAgCrC;;;OAGG;IACH,oBAAoB,IAAI,OAAO;IAoB/B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,EAAE,gBAAgB,GAAG,IAAI;IAIlD,kFAAkF;IAClF,UAAU,IAAI,OAAO;IAIrB;;;;;;;;;OASG;IACH,QAAQ,IAAI,OAAO;IAInB;;;;OAIG;IACH,aAAa,IAAI,MAAM;IAIvB;;;;;OAKG;IACH,aAAa,IAAI,sBAAsB;IA6BvC;;;OAGG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBtC;;;;;;OAMG;IACG,qBAAqB,CAAC,kBAAkB,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAOzE;;;OAGG;YACW,SAAS;IAsBvB,wEAAwE;IACxE,OAAO,CAAC,oBAAoB;IA6B5B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAqB1B;;;;;;OAMG;YACW,aAAa;IA0C3B;;;;;;;;OAQG;IACH,OAAO,CAAC,+BAA+B;IAoBvC;;;;;;;;;;;;;;;;;;;;;OAqBG;YACW,0BAA0B;IAgExC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IA4B5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,qBAAqB;IA0B7B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAc7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CA2B7B"}
|
|
@@ -20,6 +20,8 @@ import { HeartbeatManager } from './HeartbeatManager.js';
|
|
|
20
20
|
import { SecurityLog } from './SecurityLog.js';
|
|
21
21
|
import { NonceStore } from './NonceStore.js';
|
|
22
22
|
import { SEAMLESSNESS_PROTOCOL_VERSION } from './seamlessnessConfig.js';
|
|
23
|
+
import { FailureEpisodeLatch } from './FailureEpisodeLatch.js';
|
|
24
|
+
import { DegradationReporter } from '../monitoring/DegradationReporter.js';
|
|
23
25
|
// ── Constants ────────────────────────────────────────────────────────
|
|
24
26
|
const HEARTBEAT_WRITE_INTERVAL_MS = 2 * 60_000; // Write heartbeat every 2 min
|
|
25
27
|
const HEARTBEAT_CHECK_INTERVAL_MS = 2 * 60_000; // Check heartbeat every 2 min
|
|
@@ -38,6 +40,20 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
38
40
|
_identity = null;
|
|
39
41
|
_enabled = false;
|
|
40
42
|
heartbeatWriteTimer = null;
|
|
43
|
+
/**
|
|
44
|
+
* Heartbeat-write failure episode accounting ("No Unbounded Loops" / P19,
|
|
45
|
+
* Eternal Sentinel condition 4). writeHeartbeat() throws raw fs errors
|
|
46
|
+
* (ENOSPC, EACCES) — pre-fix, a throw inside the 2-min timer tick escaped as
|
|
47
|
+
* an uncaughtException and CRASHED the awake holder (the worst possible cost
|
|
48
|
+
* for one failed attempt), while a hypothetical swallowed failure would have
|
|
49
|
+
* gone silent until a peer force-failed-over. Now: each tick's write is
|
|
50
|
+
* guarded, the first failure of an episode logs once, a sustained episode
|
|
51
|
+
* raises ONE degradation signal (before the peer failover horizon), recovery
|
|
52
|
+
* logs once and re-arms. The write keeps being attempted every tick forever
|
|
53
|
+
* — this writer is the awake machine's liveness voice; persistence is the
|
|
54
|
+
* point.
|
|
55
|
+
*/
|
|
56
|
+
hbWriteEpisode = new FailureEpisodeLatch({ signalAfterMs: 6 * 60_000 });
|
|
41
57
|
heartbeatCheckTimer = null;
|
|
42
58
|
/** Integrated-Being v1 — tracks whether we've already emitted the
|
|
43
59
|
* per-machine-ledger warning this boot. Spec §Multi-machine. */
|
|
@@ -267,8 +283,21 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
267
283
|
this._role = 'awake';
|
|
268
284
|
// Update registry
|
|
269
285
|
this.identityManager.updateRole(this._identity.machineId, 'awake');
|
|
270
|
-
// Write initial heartbeat
|
|
271
|
-
|
|
286
|
+
// Write initial heartbeat. This is the ONE call site where a write failure
|
|
287
|
+
// is NOT retryable-in-place (second-pass reviewer): a promotion that cannot
|
|
288
|
+
// voice its liveness must ABORT CLEANLY — completing it silently would
|
|
289
|
+
// leave this machine serving as awake with no heartbeat, and (pre-fix) a
|
|
290
|
+
// raw throw left the role flipped + registry updated with no writer
|
|
291
|
+
// running. Roll both back, then rethrow for the caller.
|
|
292
|
+
try {
|
|
293
|
+
this.heartbeatManager.writeHeartbeat();
|
|
294
|
+
}
|
|
295
|
+
catch (err) {
|
|
296
|
+
this._role = oldRole;
|
|
297
|
+
this.identityManager.updateRole(this._identity.machineId, oldRole);
|
|
298
|
+
console.error(`[MultiMachine] promotion aborted — initial heartbeat write failed (${err instanceof Error ? err.message : String(err)}); role rolled back to '${oldRole}'`);
|
|
299
|
+
throw err;
|
|
300
|
+
}
|
|
272
301
|
// Start heartbeat writer
|
|
273
302
|
this.startHeartbeatWriter();
|
|
274
303
|
// Enable writes on StateManager
|
|
@@ -704,11 +733,11 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
704
733
|
clearInterval(this.heartbeatWriteTimer);
|
|
705
734
|
}
|
|
706
735
|
// Write immediately
|
|
707
|
-
this.
|
|
736
|
+
this.writeHeartbeatGuarded();
|
|
708
737
|
// Then every 2 minutes
|
|
709
738
|
this.heartbeatWriteTimer = setInterval(() => {
|
|
710
739
|
if (this._role === 'awake') {
|
|
711
|
-
this.
|
|
740
|
+
this.writeHeartbeatGuarded();
|
|
712
741
|
// Touch lastSeen in registry
|
|
713
742
|
if (this._identity) {
|
|
714
743
|
try {
|
|
@@ -724,6 +753,43 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
724
753
|
this.heartbeatWriteTimer.unref();
|
|
725
754
|
}
|
|
726
755
|
}
|
|
756
|
+
/**
|
|
757
|
+
* ETERNAL SENTINEL (declared per "No Unbounded Loops" / P19): the heartbeat
|
|
758
|
+
* writer is the awake machine's liveness voice — it must keep attempting
|
|
759
|
+
* every tick forever (rate floor = HEARTBEAT_WRITE_INTERVAL_MS, constant
|
|
760
|
+
* cost). Its brakes live here: a write failure can no longer escape the
|
|
761
|
+
* timer tick as an uncaughtException (pre-fix: ENOSPC at the wrong moment
|
|
762
|
+
* CRASHED the awake holder), failure logging is state-change-bounded (first
|
|
763
|
+
* + recovery, one line each), and a sustained episode raises ONE degradation
|
|
764
|
+
* signal — sized at 6min (3 failed cycles) so the operator hears about it
|
|
765
|
+
* BEFORE the peer's ~15min heartbeat-expiry failover horizon.
|
|
766
|
+
*/
|
|
767
|
+
writeHeartbeatGuarded() {
|
|
768
|
+
try {
|
|
769
|
+
this.heartbeatManager.writeHeartbeat();
|
|
770
|
+
const s = this.hbWriteEpisode.recordSuccess();
|
|
771
|
+
if (s.recovered) {
|
|
772
|
+
console.log(`[MultiMachine] heartbeat write recovered after ${s.failures} consecutive failures`);
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
catch (err) {
|
|
776
|
+
const f = this.hbWriteEpisode.recordFailure();
|
|
777
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
778
|
+
if (f.firstOfEpisode) {
|
|
779
|
+
console.error(`[MultiMachine] heartbeat write FAILED (${msg}) — retrying every ${HEARTBEAT_WRITE_INTERVAL_MS / 60_000}min; peers may failover if this persists`);
|
|
780
|
+
}
|
|
781
|
+
if (f.shouldSignal) {
|
|
782
|
+
console.error(`[MultiMachine] heartbeat write failing for ${Math.round(f.failingForMs / 60_000)}min (${f.failures} consecutive) — signaling once; retries continue`);
|
|
783
|
+
DegradationReporter.getInstance().report({
|
|
784
|
+
feature: 'MultiMachine.heartbeatWrite',
|
|
785
|
+
primary: "Awake machine persists its liveness heartbeat so peers don't failover",
|
|
786
|
+
fallback: `Heartbeat writes failing for ~${Math.round(f.failingForMs / 60_000)}min (${f.failures} consecutive: ${msg}); retries continue every ${HEARTBEAT_WRITE_INTERVAL_MS / 60_000}min`,
|
|
787
|
+
reason: 'Heartbeat file write throwing (disk full / permissions / path issue)',
|
|
788
|
+
impact: 'If this persists past the heartbeat expiry window, a standby peer will treat this machine as dead and fail over while it is still serving.',
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
}
|
|
727
793
|
/**
|
|
728
794
|
* Start periodic heartbeat monitoring (all machines).
|
|
729
795
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MultiMachineCoordinator.js","sourceRoot":"","sources":["../../src/core/MultiMachineCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,OAAO,EAAE,6BAA6B,EAAE,MAAM,yBAAyB,CAAC;AAgBxE,wEAAwE;AAExE,MAAM,2BAA2B,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,8BAA8B;AAC9E,MAAM,2BAA2B,GAAG,CAAC,GAAG,MAAM,CAAC,CAAE,8BAA8B;AAC/E,MAAM,2BAA2B,GAAG,EAAE,GAAG,MAAM,CAAC,CAAE,yBAAyB;AAC3E,mFAAmF;AACnF,MAAM,8BAA8B,GAAG,KAAK,CAAC;AAsB7C,wEAAwE;AAExE,MAAM,OAAO,uBAAwB,SAAQ,YAAY;IAC/C,eAAe,CAAyB;IACxC,gBAAgB,CAAmB;IACnC,WAAW,CAAc;IACzB,UAAU,CAAa;IACvB,KAAK,CAAe;IACpB,MAAM,CAAoB;IAC1B,KAAK,GAAgB,SAAS,CAAC;IAC/B,SAAS,GAA2B,IAAI,CAAC;IACzC,QAAQ,GAAY,KAAK,CAAC;IAC1B,mBAAmB,GAA0C,IAAI,CAAC;IAClE,mBAAmB,GAA0C,IAAI,CAAC;IAC1E;qEACiE;IACzD,6BAA6B,GAAY,KAAK,CAAC;IACvD;;;;;OAKG;IACK,gBAAgB,GAA4B,IAAI,CAAC;IACjD,YAAY,GAAY,KAAK,CAAC;IACtC;;;;;;;OAOG;IACK,cAAc,GAAyC,IAAI,CAAC;IAC5D,YAAY,GAAY,KAAK,CAAC;IAC9B,kBAAkB,GAAY,KAAK,CAAC;IACpC,gBAAgB,GAAY,KAAK,CAAC;IAC1C;;;;;;;;;;OAUG;IACK,gBAAgB,GAAkF,IAAI,CAAC;IAE/G,YAAY,KAAmB,EAAE,MAAyB;QACxD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,IAAI,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAEtE,4DAA4D;QAC5D,IAAI,CAAC,gBAAgB,GAAG,IAAW,CAAC,CAAC,yBAAyB;IAChE,CAAC;IAED,oEAAoE;IAEpE,uDAAuD;IACvD,IAAI,OAAO,KAAc,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEhD,mCAAmC;IACnC,IAAI,IAAI,KAAkB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAE9C,yDAAyD;IACzD,IAAI,QAAQ,KAA6B,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAEjE,2DAA2D;IAC3D,IAAI,OAAO,KAAc,OAAO,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC;IAEzD;;;;;;;;;;;;OAYG;IACH,IAAI,kBAAkB;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,eAAe,KAAK,KAAK,CAAC;IAC7D,CAAC;IAED,0DAA0D;IAC1D,IAAI,gBAAgB;QAClB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,gBAAgB,IAAI,iBAAiB,CAAC;IACzE,CAAC;IAED,kDAAkD;IAClD,IAAI,QAAQ;QACV,OAAO;YACL,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC;IACJ,CAAC;IAED,oEAAoE;IAEpE;;;OAGG;IACH,KAAK;QACH,mCAAmC;QACnC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,EAAE,CAAC;YACxC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;YACtB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,gCAAgC;YACtD,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;QAErD,wEAAwE;QACxE,uEAAuE;QACvE,qEAAqE;QACrE,2DAA2D;QAC3D,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAErE,oDAAoD;QACpD,oEAAoE;QACpE,wEAAwE;QACxE,6CAA6C;QAC7C,IAAI,CAAC,sCAAsC,EAAE,CAAC;QAE9C,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QAE9B,8CAA8C;QAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,sBAAsB,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;QACpF,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,YAAY,IAAI,IAAI,CAAC;QACpE,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE;YAC3F,OAAO,EAAE,YAAY;YACrB,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAEnC,gEAAgE;QAChE,4DAA4D;QAC5D,iEAAiE;QACjE,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;YACrB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACnE,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC,gCAAgC;YAC7D,yEAAyE;YAEzE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;gBACtB,KAAK,EAAE,qBAAqB;gBAC5B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,IAAI,EAAE,IAAI,CAAC,KAAK;gBAChB,gBAAgB,EAAE,aAAa;aAChC,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,6CAA6C,IAAI,CAAC,SAAS,CAAC,SAAS,eAAe,CAAC,CAAC;YAClG,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QAED,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK,GAAG,OAAO,EAAE,IAAI,IAAI,SAAS,CAAC;QAExC,2CAA2C;QAC3C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;QAE9D,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC3B,sDAAsD;YACtD,IAAI,cAAc,CAAC,MAAM,KAAK,SAAS,IAAI,cAAc,CAAC,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC9F,iDAAiD;gBACjD,OAAO,CAAC,GAAG,CAAC,mCAAmC,cAAc,CAAC,MAAM,6CAA6C,CAAC,CAAC;gBACnH,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;gBACvB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACvE,CAAC;iBAAM,CAAC;gBACN,4DAA4D;gBAC5D,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,CAAC;QACH,CAAC;aAAM,CAAC;YACN,8CAA8C;YAC9C,IAAI,cAAc,CAAC,MAAM,KAAK,SAAS,IAAI,cAAc,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC/E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;gBAC9D,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;oBAC1B,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC3E,IAAI,CAAC,cAAc,CAAC,qBAAqB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;QACH,CAAC;QAED,yCAAyC;QACzC,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QAED,uEAAuE;QACvE,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAE7B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACtB,KAAK,EAAE,qBAAqB;YAC5B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACnC,IAAI,EAAE,IAAI,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH;;;;OAIG;IACK,sCAAsC;QAC5C,IAAI,IAAI,CAAC,6BAA6B;YAAE,OAAO;QAC/C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YAC1D,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CACV,yCAAyC,KAAK,aAAa;oBAC3D,mFAAmF,CACpF,CAAC;gBACF,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kDAAkD;QACpD,CAAC;IACH,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAClC,CAAC;QACD,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IAC5B,CAAC;IAED,oEAAoE;IAEpE;;;OAGG;IACH,cAAc,CAAC,MAAc;QAC3B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QAErB,kBAAkB;QAClB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAEnE,0BAA0B;QAC1B,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;QAEvC,yBAAyB;QACzB,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,gCAAgC;QAChC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAE9B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACtB,KAAK,EAAE,iBAAiB;YACxB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACnC,IAAI,EAAE,OAAO;YACb,EAAE,EAAE,OAAO;YACX,MAAM;SACP,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACrB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,MAAc;QAC5B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QAEvB,kBAAkB;QAClB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAErE,wBAAwB;QACxB,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAClC,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAE7B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACtB,KAAK,EAAE,iBAAiB;YACxB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACnC,IAAI,EAAE,OAAO;YACb,EAAE,EAAE,SAAS;YACb,MAAM;SACP,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,sCAAsC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,oEAAoE;IAEpE;;;OAGG;IACH,oBAAoB;QAClB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC,CAAC,kCAAkC;QACpE,iDAAiD;QACjD,IAAI,IAAI,CAAC,gBAAgB,KAAK,aAAa;YAAE,OAAO,KAAK,CAAC;QAE1D,sEAAsE;QACtE,wEAAwE;QACxE,yEAAyE;QACzE,+DAA+D;QAC/D,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC,CAAC,iBAAiB;QAC1D,yDAAyD;QACzD,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC;IAC9C,CAAC;IAED,oEAAoE;IAEpE;;;;;OAKG;IACH,sBAAsB,CAAC,EAAoB;QACzC,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED,kFAAkF;IAClF,UAAU;QACR,OAAO,IAAI,CAAC,gBAAgB,EAAE,UAAU,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC;IACvE,CAAC;IAED;;;;;;;;;OASG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,gBAAgB,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;;;OAKG;IACH,aAAa;QACX,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;YAChD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;oBAAE,iBAAiB,EAAE,CAAC;YAC9C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,yDAAyD,CAAC,CAAC;QAErE,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,WAAW,IAAI,KAAK,CAAC;QAClE,MAAM,eAAe,GAA8C,aAAa;YAC9E,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC;gBAClD,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,OAAO,CAAC;QAEd,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,WAAW,EAAE,IAAI,CAAC,gBAAgB,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACzG,UAAU,EAAE,IAAI,CAAC,gBAAgB,EAAE,YAAY,EAAE,IAAI,CAAC;YACtD,UAAU,EAAE,KAAK;YACjB,eAAe;YACf,eAAe,EAAE,6BAA6B;YAC9C,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO;QACnC,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,qEAAqE;YACrE,IAAI,CAAC,oBAAoB,CAAC,yBAAyB,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;YAChD,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QAC1C,CAAC;QACD,6EAA6E;QAC7E,wEAAwE;QACxE,yEAAyE;QACzE,8EAA8E;QAC9E,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,qBAAqB,CAAC,kBAA0B;QACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,KAAK,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;QAClF,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;QAC3C,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC5B,mEAAmE;gBACnE,uEAAuE;gBACvE,IAAI,CAAC,oBAAoB,CAAC,yBAAyB,CAAC,CAAC;YACvD,CAAC;iBAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,EAAE,CAAC;gBAC9C,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;gBACpC,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;gBAChD,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,gEAAgE;QAClE,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,wEAAwE;IAChE,oBAAoB,CAAC,MAAc;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,OAAO,GAAgB,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,IAAI,OAAO,KAAK,IAAI,CAAC,KAAK;YAAE,OAAO;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QACrB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACnE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,KAAK;YAAE,IAAI,CAAC,oBAAoB,EAAE,CAAC;aAClC,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAClC,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACtB,KAAK,EAAE,iBAAiB;YACxB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACnC,IAAI,EAAE,OAAO;YACb,EAAE,EAAE,OAAO;YACX,MAAM,EAAE,SAAS,MAAM,EAAE;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC1C,IAAI,KAAK;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;;YAC3B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,oCAAoC,OAAO,KAAK,MAAM,GAAG,CAAC,CAAC;IACzE,CAAC;IAED,oEAAoE;IAEpE;;;;;;;OAOG;IACK,kBAAkB;QACxB,uEAAuE;QACvE,4EAA4E;QAC5E,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE;YAAE,OAAO;QACxI,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,CAAC,kBAAkB;QACnD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC7F,MAAM,GAAG,GAAG,GAAG,EAAE;YACf,uEAAuE;YACvE,iEAAiE;YACjE,IAAI,IAAI,CAAC,gBAAgB;gBAAE,OAAO;YAClC,sEAAsE;YACtE,oBAAoB;YACpB,MAAM,MAAM,GAAG,IAAI,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC;YACzD,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACjF,IAAI,IAAI,CAAC,cAAc,CAAC,KAAK;gBAAE,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC7D,CAAC,CAAC;QACF,GAAG,EAAE,CAAC;IACR,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,aAAa,CAAC,GAAe;QACzC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAAC,GAAG,EAAE,CAAC;YAAC,OAAO;QAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,gBAAiB,CAAC,aAAa,EAAE,CAAC;YAC7C,2EAA2E;YAC3E,4EAA4E;YAC5E,2EAA2E;YAC3E,uEAAuE;YACvE,2EAA2E;YAC3E,0EAA0E;YAC1E,0EAA0E;YAC1E,wEAAwE;YACxE,4EAA4E;YAC5E,6EAA6E;YAC7E,yEAAyE;YACzE,sDAAsD;YACtD,IAAI,IAAI,CAAC,gBAAiB,CAAC,iBAAiB,EAAE,EAAE,CAAC;gBAC/C,wEAAwE;gBACxE,6EAA6E;gBAC7E,6EAA6E;gBAC7E,4EAA4E;gBAC5E,8EAA8E;gBAC9E,2EAA2E;gBAC3E,4EAA4E;gBAC5E,wEAAwE;gBACxE,IAAI,IAAI,CAAC,gBAAiB,CAAC,mBAAmB,EAAE,EAAE,CAAC;oBACjD,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;gBAC1C,CAAC;gBACD,IAAI,CAAC,+BAA+B,EAAE,CAAC;gBACvC,wEAAwE;gBACxE,uEAAuE;gBACvE,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;YAC1C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4DAA4D;QAC9D,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,GAAG,EAAE,CAAC;QACR,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,+BAA+B;QACrC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;QACtG,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAC/B,OAAO,CAAC,IAAI,CACV,gEAAgE,IAAK,CAAC,MAAM,GAAG;gBAC/E,gBAAgB,IAAK,CAAC,KAAK,wBAAwB,QAAQ,mCAAmC,CAC/F,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,IAAK,CAAC,MAAM,EAAE,SAAS,EAAE,IAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5F,CAAC;aAAM,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACK,KAAK,CAAC,0BAA0B;QACtC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QACtD,MAAM,qBAAqB,GAAG,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC;QACtD,wEAAwE;QACxE,8EAA8E;QAC9E,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;QACtG,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,0EAA0E;YAC1E,mEAAmE;YACnE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,OAAO;QACT,CAAC;QACD,MAAM,UAAU,GAAG,IAAK,CAAC,MAAM,CAAC;QAChC,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB;QAC5E,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;YACvE,IAAI,CAAC,gBAAgB,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QAC5F,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACjC,EAAE,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,uBAAuB;QAC7E,MAAM,KAAK,GAAG,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,KAAK,IAAI,CAAC;QAClC,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;YACjB,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,gBAAgB,CAAC,2BAA2B,EAAE,CAAC;gBAC1D,OAAO,CAAC,GAAG,CACT,8CAA8C,UAAU,aAAa,QAAQ,GAAG;oBAChF,sDAAsD,CACvD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;gBACnC,IAAI,CAAC,oBAAoB,CAAC,sBAAsB,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,CACT,kDAAkD,MAAM,aAAa,QAAQ,GAAG;oBAChF,yDAAyD,CAC1D,CAAC;YACJ,CAAC;YACD,EAAE,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,sDAAsD;QAC5E,CAAC;QACD,yEAAyE;QACzE,8EAA8E;QAC9E,gFAAgF;QAChF,IAAI,EAAE,CAAC,MAAM,IAAI,qBAAqB,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;YACxD,EAAE,CAAC,SAAS,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE;gBAChC,UAAU;gBACV,MAAM;gBACN,KAAK;gBACL,cAAc,EAAE,UAAU,KAAK,EAAE;gBACjC,MAAM,EAAE,8CAA8C,EAAE,CAAC,MAAM,iCAAiC;aACjG,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CACV,yDAAyD,EAAE,CAAC,MAAM,YAAY;gBAC9E,sBAAsB,KAAK,aAAa,UAAU,GAAG,CACtD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,oEAAoE;IAEpE;;OAEG;IACK,oBAAoB;QAC1B,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1C,CAAC;QAED,oBAAoB;QACpB,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;QAEvC,uBAAuB;QACvB,IAAI,CAAC,mBAAmB,GAAG,WAAW,CAAC,GAAG,EAAE;YAC1C,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;gBAC3B,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;gBACvC,6BAA6B;gBAC7B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACnB,IAAI,CAAC;wBACH,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;oBAC9D,CAAC;oBAAC,MAAM,CAAC;wBACP,qDAAqD;oBACvD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,qBAAqB;QAC3B,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,WAAW,CAAC,GAAG,EAAE;YAC1C,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC9B,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,+CAA+C;QAC/C,IAAI,IAAI,CAAC,gBAAgB,KAAK,aAAa;YAAE,OAAO;QAEpD,yEAAyE;QACzE,sEAAsE;QACtE,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC3B,iDAAiD;YACjD,IAAI,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,EAAE,CAAC;gBACzC,IAAI,CAAC,eAAe,CAAC,uCAAuC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;YAC9D,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;gBAC1B,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;gBACvC,IAAI,CAAC,cAAc,CAAC,kBAAkB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"MultiMachineCoordinator.js","sourceRoot":"","sources":["../../src/core/MultiMachineCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,OAAO,EAAE,6BAA6B,EAAE,MAAM,yBAAyB,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAC;AAgB3E,wEAAwE;AAExE,MAAM,2BAA2B,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,8BAA8B;AAC9E,MAAM,2BAA2B,GAAG,CAAC,GAAG,MAAM,CAAC,CAAE,8BAA8B;AAC/E,MAAM,2BAA2B,GAAG,EAAE,GAAG,MAAM,CAAC,CAAE,yBAAyB;AAC3E,mFAAmF;AACnF,MAAM,8BAA8B,GAAG,KAAK,CAAC;AAsB7C,wEAAwE;AAExE,MAAM,OAAO,uBAAwB,SAAQ,YAAY;IAC/C,eAAe,CAAyB;IACxC,gBAAgB,CAAmB;IACnC,WAAW,CAAc;IACzB,UAAU,CAAa;IACvB,KAAK,CAAe;IACpB,MAAM,CAAoB;IAC1B,KAAK,GAAgB,SAAS,CAAC;IAC/B,SAAS,GAA2B,IAAI,CAAC;IACzC,QAAQ,GAAY,KAAK,CAAC;IAC1B,mBAAmB,GAA0C,IAAI,CAAC;IAC1E;;;;;;;;;;;;OAYG;IACc,cAAc,GAAG,IAAI,mBAAmB,CAAC,EAAE,aAAa,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;IACjF,mBAAmB,GAA0C,IAAI,CAAC;IAC1E;qEACiE;IACzD,6BAA6B,GAAY,KAAK,CAAC;IACvD;;;;;OAKG;IACK,gBAAgB,GAA4B,IAAI,CAAC;IACjD,YAAY,GAAY,KAAK,CAAC;IACtC;;;;;;;OAOG;IACK,cAAc,GAAyC,IAAI,CAAC;IAC5D,YAAY,GAAY,KAAK,CAAC;IAC9B,kBAAkB,GAAY,KAAK,CAAC;IACpC,gBAAgB,GAAY,KAAK,CAAC;IAC1C;;;;;;;;;;OAUG;IACK,gBAAgB,GAAkF,IAAI,CAAC;IAE/G,YAAY,KAAmB,EAAE,MAAyB;QACxD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,IAAI,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAEtE,4DAA4D;QAC5D,IAAI,CAAC,gBAAgB,GAAG,IAAW,CAAC,CAAC,yBAAyB;IAChE,CAAC;IAED,oEAAoE;IAEpE,uDAAuD;IACvD,IAAI,OAAO,KAAc,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEhD,mCAAmC;IACnC,IAAI,IAAI,KAAkB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAE9C,yDAAyD;IACzD,IAAI,QAAQ,KAA6B,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAEjE,2DAA2D;IAC3D,IAAI,OAAO,KAAc,OAAO,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC;IAEzD;;;;;;;;;;;;OAYG;IACH,IAAI,kBAAkB;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,eAAe,KAAK,KAAK,CAAC;IAC7D,CAAC;IAED,0DAA0D;IAC1D,IAAI,gBAAgB;QAClB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,gBAAgB,IAAI,iBAAiB,CAAC;IACzE,CAAC;IAED,kDAAkD;IAClD,IAAI,QAAQ;QACV,OAAO;YACL,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC;IACJ,CAAC;IAED,oEAAoE;IAEpE;;;OAGG;IACH,KAAK;QACH,mCAAmC;QACnC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,EAAE,CAAC;YACxC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;YACtB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,gCAAgC;YACtD,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;QAErD,wEAAwE;QACxE,uEAAuE;QACvE,qEAAqE;QACrE,2DAA2D;QAC3D,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAErE,oDAAoD;QACpD,oEAAoE;QACpE,wEAAwE;QACxE,6CAA6C;QAC7C,IAAI,CAAC,sCAAsC,EAAE,CAAC;QAE9C,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QAE9B,8CAA8C;QAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,sBAAsB,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;QACpF,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,YAAY,IAAI,IAAI,CAAC;QACpE,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE;YAC3F,OAAO,EAAE,YAAY;YACrB,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAEnC,gEAAgE;QAChE,4DAA4D;QAC5D,iEAAiE;QACjE,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;YACrB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACnE,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC,gCAAgC;YAC7D,yEAAyE;YAEzE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;gBACtB,KAAK,EAAE,qBAAqB;gBAC5B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,IAAI,EAAE,IAAI,CAAC,KAAK;gBAChB,gBAAgB,EAAE,aAAa;aAChC,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,6CAA6C,IAAI,CAAC,SAAS,CAAC,SAAS,eAAe,CAAC,CAAC;YAClG,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QAED,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK,GAAG,OAAO,EAAE,IAAI,IAAI,SAAS,CAAC;QAExC,2CAA2C;QAC3C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;QAE9D,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC3B,sDAAsD;YACtD,IAAI,cAAc,CAAC,MAAM,KAAK,SAAS,IAAI,cAAc,CAAC,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC9F,iDAAiD;gBACjD,OAAO,CAAC,GAAG,CAAC,mCAAmC,cAAc,CAAC,MAAM,6CAA6C,CAAC,CAAC;gBACnH,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;gBACvB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACvE,CAAC;iBAAM,CAAC;gBACN,4DAA4D;gBAC5D,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,CAAC;QACH,CAAC;aAAM,CAAC;YACN,8CAA8C;YAC9C,IAAI,cAAc,CAAC,MAAM,KAAK,SAAS,IAAI,cAAc,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC/E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;gBAC9D,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;oBAC1B,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC3E,IAAI,CAAC,cAAc,CAAC,qBAAqB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;QACH,CAAC;QAED,yCAAyC;QACzC,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QAED,uEAAuE;QACvE,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAE7B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACtB,KAAK,EAAE,qBAAqB;YAC5B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACnC,IAAI,EAAE,IAAI,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH;;;;OAIG;IACK,sCAAsC;QAC5C,IAAI,IAAI,CAAC,6BAA6B;YAAE,OAAO;QAC/C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YAC1D,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CACV,yCAAyC,KAAK,aAAa;oBAC3D,mFAAmF,CACpF,CAAC;gBACF,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kDAAkD;QACpD,CAAC;IACH,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAClC,CAAC;QACD,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IAC5B,CAAC;IAED,oEAAoE;IAEpE;;;OAGG;IACH,cAAc,CAAC,MAAc;QAC3B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QAErB,kBAAkB;QAClB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAEnE,2EAA2E;QAC3E,4EAA4E;QAC5E,uEAAuE;QACvE,yEAAyE;QACzE,oEAAoE;QACpE,wDAAwD;QACxD,IAAI,CAAC;YACH,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;YACrB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACnE,OAAO,CAAC,KAAK,CAAC,sEAAsE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,2BAA2B,OAAO,GAAG,CAAC,CAAC;YAC3K,MAAM,GAAG,CAAC;QACZ,CAAC;QAED,yBAAyB;QACzB,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,gCAAgC;QAChC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAE9B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACtB,KAAK,EAAE,iBAAiB;YACxB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACnC,IAAI,EAAE,OAAO;YACb,EAAE,EAAE,OAAO;YACX,MAAM;SACP,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACrB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,MAAc;QAC5B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QAEvB,kBAAkB;QAClB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAErE,wBAAwB;QACxB,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAClC,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAE7B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACtB,KAAK,EAAE,iBAAiB;YACxB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACnC,IAAI,EAAE,OAAO;YACb,EAAE,EAAE,SAAS;YACb,MAAM;SACP,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,sCAAsC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,oEAAoE;IAEpE;;;OAGG;IACH,oBAAoB;QAClB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC,CAAC,kCAAkC;QACpE,iDAAiD;QACjD,IAAI,IAAI,CAAC,gBAAgB,KAAK,aAAa;YAAE,OAAO,KAAK,CAAC;QAE1D,sEAAsE;QACtE,wEAAwE;QACxE,yEAAyE;QACzE,+DAA+D;QAC/D,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC,CAAC,iBAAiB;QAC1D,yDAAyD;QACzD,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC;IAC9C,CAAC;IAED,oEAAoE;IAEpE;;;;;OAKG;IACH,sBAAsB,CAAC,EAAoB;QACzC,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED,kFAAkF;IAClF,UAAU;QACR,OAAO,IAAI,CAAC,gBAAgB,EAAE,UAAU,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC;IACvE,CAAC;IAED;;;;;;;;;OASG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,gBAAgB,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;;;OAKG;IACH,aAAa;QACX,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;YAChD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;oBAAE,iBAAiB,EAAE,CAAC;YAC9C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,yDAAyD,CAAC,CAAC;QAErE,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,WAAW,IAAI,KAAK,CAAC;QAClE,MAAM,eAAe,GAA8C,aAAa;YAC9E,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC;gBAClD,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,OAAO,CAAC;QAEd,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,WAAW,EAAE,IAAI,CAAC,gBAAgB,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACzG,UAAU,EAAE,IAAI,CAAC,gBAAgB,EAAE,YAAY,EAAE,IAAI,CAAC;YACtD,UAAU,EAAE,KAAK;YACjB,eAAe;YACf,eAAe,EAAE,6BAA6B;YAC9C,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO;QACnC,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,qEAAqE;YACrE,IAAI,CAAC,oBAAoB,CAAC,yBAAyB,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;YAChD,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QAC1C,CAAC;QACD,6EAA6E;QAC7E,wEAAwE;QACxE,yEAAyE;QACzE,8EAA8E;QAC9E,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,qBAAqB,CAAC,kBAA0B;QACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,KAAK,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;QAClF,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;QAC3C,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC5B,mEAAmE;gBACnE,uEAAuE;gBACvE,IAAI,CAAC,oBAAoB,CAAC,yBAAyB,CAAC,CAAC;YACvD,CAAC;iBAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,EAAE,CAAC;gBAC9C,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;gBACpC,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;gBAChD,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,gEAAgE;QAClE,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,wEAAwE;IAChE,oBAAoB,CAAC,MAAc;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,OAAO,GAAgB,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,IAAI,OAAO,KAAK,IAAI,CAAC,KAAK;YAAE,OAAO;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QACrB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACnE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,KAAK;YAAE,IAAI,CAAC,oBAAoB,EAAE,CAAC;aAClC,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAClC,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACtB,KAAK,EAAE,iBAAiB;YACxB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACnC,IAAI,EAAE,OAAO;YACb,EAAE,EAAE,OAAO;YACX,MAAM,EAAE,SAAS,MAAM,EAAE;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC1C,IAAI,KAAK;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;;YAC3B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,oCAAoC,OAAO,KAAK,MAAM,GAAG,CAAC,CAAC;IACzE,CAAC;IAED,oEAAoE;IAEpE;;;;;;;OAOG;IACK,kBAAkB;QACxB,uEAAuE;QACvE,4EAA4E;QAC5E,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE;YAAE,OAAO;QACxI,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,CAAC,kBAAkB;QACnD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC7F,MAAM,GAAG,GAAG,GAAG,EAAE;YACf,uEAAuE;YACvE,iEAAiE;YACjE,IAAI,IAAI,CAAC,gBAAgB;gBAAE,OAAO;YAClC,sEAAsE;YACtE,oBAAoB;YACpB,MAAM,MAAM,GAAG,IAAI,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC;YACzD,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACjF,IAAI,IAAI,CAAC,cAAc,CAAC,KAAK;gBAAE,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC7D,CAAC,CAAC;QACF,GAAG,EAAE,CAAC;IACR,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,aAAa,CAAC,GAAe;QACzC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAAC,GAAG,EAAE,CAAC;YAAC,OAAO;QAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,gBAAiB,CAAC,aAAa,EAAE,CAAC;YAC7C,2EAA2E;YAC3E,4EAA4E;YAC5E,2EAA2E;YAC3E,uEAAuE;YACvE,2EAA2E;YAC3E,0EAA0E;YAC1E,0EAA0E;YAC1E,wEAAwE;YACxE,4EAA4E;YAC5E,6EAA6E;YAC7E,yEAAyE;YACzE,sDAAsD;YACtD,IAAI,IAAI,CAAC,gBAAiB,CAAC,iBAAiB,EAAE,EAAE,CAAC;gBAC/C,wEAAwE;gBACxE,6EAA6E;gBAC7E,6EAA6E;gBAC7E,4EAA4E;gBAC5E,8EAA8E;gBAC9E,2EAA2E;gBAC3E,4EAA4E;gBAC5E,wEAAwE;gBACxE,IAAI,IAAI,CAAC,gBAAiB,CAAC,mBAAmB,EAAE,EAAE,CAAC;oBACjD,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;gBAC1C,CAAC;gBACD,IAAI,CAAC,+BAA+B,EAAE,CAAC;gBACvC,wEAAwE;gBACxE,uEAAuE;gBACvE,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;YAC1C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4DAA4D;QAC9D,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,GAAG,EAAE,CAAC;QACR,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,+BAA+B;QACrC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;QACtG,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAC/B,OAAO,CAAC,IAAI,CACV,gEAAgE,IAAK,CAAC,MAAM,GAAG;gBAC/E,gBAAgB,IAAK,CAAC,KAAK,wBAAwB,QAAQ,mCAAmC,CAC/F,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,IAAK,CAAC,MAAM,EAAE,SAAS,EAAE,IAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5F,CAAC;aAAM,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACK,KAAK,CAAC,0BAA0B;QACtC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QACtD,MAAM,qBAAqB,GAAG,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC;QACtD,wEAAwE;QACxE,8EAA8E;QAC9E,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;QACtG,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,0EAA0E;YAC1E,mEAAmE;YACnE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,OAAO;QACT,CAAC;QACD,MAAM,UAAU,GAAG,IAAK,CAAC,MAAM,CAAC;QAChC,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB;QAC5E,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;YACvE,IAAI,CAAC,gBAAgB,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QAC5F,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACjC,EAAE,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,uBAAuB;QAC7E,MAAM,KAAK,GAAG,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,KAAK,IAAI,CAAC;QAClC,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;YACjB,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,gBAAgB,CAAC,2BAA2B,EAAE,CAAC;gBAC1D,OAAO,CAAC,GAAG,CACT,8CAA8C,UAAU,aAAa,QAAQ,GAAG;oBAChF,sDAAsD,CACvD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;gBACnC,IAAI,CAAC,oBAAoB,CAAC,sBAAsB,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,CACT,kDAAkD,MAAM,aAAa,QAAQ,GAAG;oBAChF,yDAAyD,CAC1D,CAAC;YACJ,CAAC;YACD,EAAE,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,sDAAsD;QAC5E,CAAC;QACD,yEAAyE;QACzE,8EAA8E;QAC9E,gFAAgF;QAChF,IAAI,EAAE,CAAC,MAAM,IAAI,qBAAqB,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;YACxD,EAAE,CAAC,SAAS,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE;gBAChC,UAAU;gBACV,MAAM;gBACN,KAAK;gBACL,cAAc,EAAE,UAAU,KAAK,EAAE;gBACjC,MAAM,EAAE,8CAA8C,EAAE,CAAC,MAAM,iCAAiC;aACjG,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CACV,yDAAyD,EAAE,CAAC,MAAM,YAAY;gBAC9E,sBAAsB,KAAK,aAAa,UAAU,GAAG,CACtD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,oEAAoE;IAEpE;;OAEG;IACK,oBAAoB;QAC1B,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1C,CAAC;QAED,oBAAoB;QACpB,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAE7B,uBAAuB;QACvB,IAAI,CAAC,mBAAmB,GAAG,WAAW,CAAC,GAAG,EAAE;YAC1C,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;gBAC3B,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAC7B,6BAA6B;gBAC7B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACnB,IAAI,CAAC;wBACH,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;oBAC9D,CAAC;oBAAC,MAAM,CAAC;wBACP,qDAAqD;oBACvD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACK,qBAAqB;QAC3B,IAAI,CAAC;YACH,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YAC9C,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC,QAAQ,uBAAuB,CAAC,CAAC;YACnG,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;gBACrB,OAAO,CAAC,KAAK,CAAC,0CAA0C,GAAG,sBAAsB,2BAA2B,GAAG,MAAM,0CAA0C,CAAC,CAAC;YACnK,CAAC;YACD,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;gBACnB,OAAO,CAAC,KAAK,CAAC,8CAA8C,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,kDAAkD,CAAC,CAAC;gBACrK,mBAAmB,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC;oBACvC,OAAO,EAAE,6BAA6B;oBACtC,OAAO,EAAE,uEAAuE;oBAChF,QAAQ,EAAE,iCAAiC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,iBAAiB,GAAG,6BAA6B,2BAA2B,GAAG,MAAM,KAAK;oBAC1L,MAAM,EAAE,sEAAsE;oBAC9E,MAAM,EAAE,4IAA4I;iBACrJ,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,qBAAqB;QAC3B,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,WAAW,CAAC,GAAG,EAAE;YAC1C,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC9B,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,+CAA+C;QAC/C,IAAI,IAAI,CAAC,gBAAgB,KAAK,aAAa;YAAE,OAAO;QAEpD,yEAAyE;QACzE,sEAAsE;QACtE,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC3B,iDAAiD;YACjD,IAAI,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,EAAE,CAAC;gBACzC,IAAI,CAAC,eAAe,CAAC,uCAAuC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;YAC9D,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;gBAC1B,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;gBACvC,IAAI,CAAC,cAAc,CAAC,kBAAkB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -40,12 +40,31 @@ export interface ReplyMarkerTransportDeps {
|
|
|
40
40
|
/** Monotonic per-request sequence for machine-auth replay protection. */
|
|
41
41
|
nextSequence: () => number;
|
|
42
42
|
fetchImpl?: typeof fetch;
|
|
43
|
+
/**
|
|
44
|
+
* Per-request abort timeout (P19 brake — a hung socket must not hold the
|
|
45
|
+
* reply-commit path open). Default 30s: sized ABOVE the fleet's documented
|
|
46
|
+
* 5–40s receiver-stall envelope (the #874 reviewer's sizing rationale — a
|
|
47
|
+
* slow-but-alive peer must not read as unreachable).
|
|
48
|
+
*/
|
|
49
|
+
requestTimeoutMs?: number;
|
|
50
|
+
/**
|
|
51
|
+
* Coarse-reminder interval for the per-peer failure log gate (P19 brake).
|
|
52
|
+
* Default 50 — markers flow at REPLY cadence (not a fixed 5s tick), so the
|
|
53
|
+
* reminder lands roughly every 50 failed replies; first-failure and
|
|
54
|
+
* recovery lines carry the state changes regardless.
|
|
55
|
+
*/
|
|
56
|
+
failureLogEveryN?: number;
|
|
43
57
|
logger?: (msg: string) => void;
|
|
44
58
|
}
|
|
45
59
|
export declare class ReplyMarkerTransport {
|
|
46
60
|
private readonly d;
|
|
61
|
+
private readonly requestTimeoutMs;
|
|
62
|
+
/** State-change failure logging (first/Nth/recovery) — never per-attempt. */
|
|
63
|
+
private readonly logGate;
|
|
47
64
|
constructor(deps: ReplyMarkerTransportDeps);
|
|
48
65
|
private log;
|
|
66
|
+
private logFailure;
|
|
67
|
+
private logSuccess;
|
|
49
68
|
/**
|
|
50
69
|
* POST the marker to every standby peer over the signed channel. Resolves true
|
|
51
70
|
* if at least one peer accepted (a live medium carried it); false if none was
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ReplyMarkerTransport.d.ts","sourceRoot":"","sources":["../../src/core/ReplyMarkerTransport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;
|
|
1
|
+
{"version":3,"file":"ReplyMarkerTransport.d.ts","sourceRoot":"","sources":["../../src/core/ReplyMarkerTransport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAKH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,eAAe,EAAE,CAAC;IAC/B,yEAAyE;IACzE,YAAY,EAAE,MAAM,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAA2B;IAC7C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAEjC,IAAI,EAAE,wBAAwB;IAM1C,OAAO,CAAC,GAAG;IAIX,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,UAAU;IAKlB;;;;;;OAMG;IACG,SAAS,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;CA8BvD"}
|
|
@@ -21,14 +21,30 @@
|
|
|
21
21
|
* injected for testability.
|
|
22
22
|
*/
|
|
23
23
|
import { signRequest } from '../server/machineAuth.js';
|
|
24
|
+
import { PeerFailureLogGate } from './PeerFailureLogGate.js';
|
|
24
25
|
export class ReplyMarkerTransport {
|
|
25
26
|
d;
|
|
27
|
+
requestTimeoutMs;
|
|
28
|
+
/** State-change failure logging (first/Nth/recovery) — never per-attempt. */
|
|
29
|
+
logGate;
|
|
26
30
|
constructor(deps) {
|
|
27
31
|
this.d = deps;
|
|
32
|
+
this.requestTimeoutMs = deps.requestTimeoutMs ?? 30_000;
|
|
33
|
+
this.logGate = new PeerFailureLogGate(deps.failureLogEveryN ?? 50);
|
|
28
34
|
}
|
|
29
35
|
log(m) {
|
|
30
36
|
this.d.logger?.(`[reply-marker] ${m}`);
|
|
31
37
|
}
|
|
38
|
+
logFailure(key, detail) {
|
|
39
|
+
const line = this.logGate.failed(key, detail);
|
|
40
|
+
if (line)
|
|
41
|
+
this.log(line);
|
|
42
|
+
}
|
|
43
|
+
logSuccess(key) {
|
|
44
|
+
const line = this.logGate.succeeded(key);
|
|
45
|
+
if (line)
|
|
46
|
+
this.log(line);
|
|
47
|
+
}
|
|
32
48
|
/**
|
|
33
49
|
* POST the marker to every standby peer over the signed channel. Resolves true
|
|
34
50
|
* if at least one peer accepted (a live medium carried it); false if none was
|
|
@@ -50,14 +66,19 @@ export class ReplyMarkerTransport {
|
|
|
50
66
|
method: 'POST',
|
|
51
67
|
headers: { 'Content-Type': 'application/json', ...headers },
|
|
52
68
|
body: JSON.stringify(body),
|
|
69
|
+
// P19 brake: a hung socket aborts instead of holding the reply-commit path.
|
|
70
|
+
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
|
53
71
|
});
|
|
54
|
-
if (res && res.ok)
|
|
72
|
+
if (res && res.ok) {
|
|
55
73
|
anyOk = true;
|
|
56
|
-
|
|
57
|
-
|
|
74
|
+
this.logSuccess(`marker to ${peer.machineId}`);
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
this.logFailure(`marker to ${peer.machineId}`, `status ${res?.status} (marker ${marker.dedupeKey})`);
|
|
78
|
+
}
|
|
58
79
|
}
|
|
59
80
|
catch (err) {
|
|
60
|
-
this.
|
|
81
|
+
this.logFailure(`marker to ${peer.machineId}`, err instanceof Error ? err.message : String(err));
|
|
61
82
|
}
|
|
62
83
|
}));
|
|
63
84
|
return anyOk;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ReplyMarkerTransport.js","sourceRoot":"","sources":["../../src/core/ReplyMarkerTransport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"ReplyMarkerTransport.js","sourceRoot":"","sources":["../../src/core/ReplyMarkerTransport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAyC7D,MAAM,OAAO,oBAAoB;IACd,CAAC,CAA2B;IAC5B,gBAAgB,CAAS;IAC1C,6EAA6E;IAC5D,OAAO,CAAqB;IAE7C,YAAY,IAA8B;QACxC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;QACd,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,MAAM,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC;IAEO,GAAG,CAAC,CAAS;QACnB,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC;IAEO,UAAU,CAAC,GAAW,EAAE,MAAc;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC9C,IAAI,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IACO,UAAU,CAAC,GAAW;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,SAAS,CAAC,MAAmB;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC;QAC5C,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC;gBACrG,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,qBAAqB,EAAE;oBAC/E,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,OAAO,EAAE;oBAC3D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAC1B,4EAA4E;oBAC5E,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC;iBACnD,CAAC,CAAC;gBACH,IAAI,GAAG,IAAK,GAAgB,CAAC,EAAE,EAAE,CAAC;oBAChC,KAAK,GAAG,IAAI,CAAC;oBACb,IAAI,CAAC,UAAU,CAAC,aAAa,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,UAAU,CAAC,aAAa,IAAI,CAAC,SAAS,EAAE,EAAE,UAAW,GAAgB,EAAE,MAAM,YAAY,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;gBACrH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,UAAU,CAAC,aAAa,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACnG,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-06-06T03:
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-06-06T03:58:05.940Z",
|
|
5
|
+
"instarVersion": "1.3.336",
|
|
6
6
|
"entryCount": 199,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -1594,7 +1594,7 @@
|
|
|
1594
1594
|
"type": "subsystem",
|
|
1595
1595
|
"domain": "coordination",
|
|
1596
1596
|
"sourcePath": "src/core/MultiMachineCoordinator.ts",
|
|
1597
|
-
"contentHash": "
|
|
1597
|
+
"contentHash": "b1e9707dcdaacefc9577b5341a61afd7d2c99d5aac6764c97eef85b0d37b7f06",
|
|
1598
1598
|
"since": "2025-01-01"
|
|
1599
1599
|
},
|
|
1600
1600
|
"subsystem:backup-manager": {
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
Audit fix #4 under "No Unbounded Loops" (P19): the multi-machine heartbeat
|
|
9
|
+
writer. Grounding revealed the audit lead ("silent failure") was backwards and
|
|
10
|
+
worse — `HeartbeatManager.writeHeartbeat()` throws raw fs errors and the 2-min
|
|
11
|
+
timer tick called it UNGUARDED, so a transient disk error (ENOSPC/EACCES)
|
|
12
|
+
escaped as an uncaughtException and CRASHED the awake holder; in
|
|
13
|
+
`promoteToAwake` a throw aborted promotion midway (role flipped, writer never
|
|
14
|
+
started). Now: the periodic + boot writes go through a guarded funnel with the
|
|
15
|
+
new `FailureEpisodeLatch` (pure core class — the canonical extraction of
|
|
16
|
+
tonight's episode-latch pattern): first-failure log, one DegradationReporter
|
|
17
|
+
signal per episode at 6min sustained (before the peer's 15min failover
|
|
18
|
+
horizon), recovery log + re-arm, retries forever (declared Eternal Sentinel).
|
|
19
|
+
`promoteToAwake`'s initial write is rollback-and-rethrow per the adversarial
|
|
20
|
+
reviewer's finding: a promotion that cannot voice its liveness aborts cleanly
|
|
21
|
+
(role + registry rolled back) instead of silently completing voiceless.
|
|
22
|
+
|
|
23
|
+
## What to Tell Your User
|
|
24
|
+
|
|
25
|
+
If you run me on more than one machine: a disk hiccup can no longer crash the
|
|
26
|
+
machine that's actively serving you (previously a full disk at the wrong
|
|
27
|
+
moment did exactly that, every 2 minutes), machine takeovers can't
|
|
28
|
+
half-complete anymore, and if my "I'm alive" signal ever starts failing you
|
|
29
|
+
get a health-log note within 6 minutes — before the other machine would
|
|
30
|
+
assume I'm dead and take over.
|
|
31
|
+
|
|
32
|
+
## Summary of New Capabilities
|
|
33
|
+
|
|
34
|
+
- Heartbeat-write resilience: transient write failures are absorbed, bounded-
|
|
35
|
+
logged, and surfaced once per outage (`MultiMachine.heartbeatWrite` in the
|
|
36
|
+
degradation log); clean promotion abort on a failed initial write. No
|
|
37
|
+
configuration needed.
|
|
38
|
+
- `FailureEpisodeLatch` (core): reusable episode-latch for P19 condition-4
|
|
39
|
+
signals — first/threshold-once/recovery accounting, null-sentinel-safe.
|
|
40
|
+
|
|
41
|
+
## Evidence
|
|
42
|
+
|
|
43
|
+
Loop-safety audit lead (CMT-1109) re-grounded at source: unguarded
|
|
44
|
+
`writeHeartbeat()` in the `setInterval` tick + `server.ts`'s FATAL
|
|
45
|
+
uncaughtException path for fs errors = crash vector (worse than the claimed
|
|
46
|
+
silence). Adversarial second-pass: narrow OBJECT on the promote call site —
|
|
47
|
+
APPLIED pre-commit (rollback-and-rethrow); probes 2–4 CONCUR (CLI one-shot raw
|
|
48
|
+
calls safe; 6min < 15min failover horizon verified against
|
|
49
|
+
DEFAULT_FAILOVER_TIMEOUT_MS; DegradationReporter cannot throw into the catch).
|
|
50
|
+
Tests: 10 green in FailureEpisodeLatch.test.ts incl. the P19 bound (a week of
|
|
51
|
+
2-min failures → exactly 1 signal) — which caught a real zero-sentinel bug
|
|
52
|
+
pre-ship — + wiring pins (exactly two sanctioned raw call sites); neighbor
|
|
53
|
+
suites green (coordinator 22, leasePull 3); tsc clean.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
Audit fix #5 under "No Unbounded Loops" (P19), completing the mesh-transport
|
|
9
|
+
brake set #874 started: `HttpLiveTailTransport` (conversation-tail streaming)
|
|
10
|
+
and `ReplyMarkerTransport` (cross-machine "already answered" markers) get
|
|
11
|
+
`AbortSignal.timeout` on their fetches (30s default — the #874 reviewer's
|
|
12
|
+
stall-envelope sizing, applied from the start) and `PeerFailureLogGate`
|
|
13
|
+
state-change failure logging (first / every-Nth / recovery; previously one
|
|
14
|
+
line per failed attempt, and silent for non-ok responses). Verified: neither
|
|
15
|
+
transport feeds lease renewal/suspend (`isReachable()`'s only consumer is the
|
|
16
|
+
lease wire), so the timeout carries zero authority coupling. The reviewer
|
|
17
|
+
confirmed the marker-drop trade-off under a 30s abort is pre-accepted by the
|
|
18
|
+
marker's documented design (fire-and-forget; provider redelivery + dedup gate
|
|
19
|
+
+ synced ledger are the backstops).
|
|
20
|
+
|
|
21
|
+
## What to Tell Your User
|
|
22
|
+
|
|
23
|
+
If you run me on more than one machine: the last two machine-to-machine sync
|
|
24
|
+
channels now have the same protections as the rest — a stuck network
|
|
25
|
+
connection can't silently jam them, and a down machine produces a handful of
|
|
26
|
+
meaningful log lines instead of one per failed attempt. All four cross-machine
|
|
27
|
+
wires now follow the loop-safety standard.
|
|
28
|
+
|
|
29
|
+
## Summary of New Capabilities
|
|
30
|
+
|
|
31
|
+
- Hung-socket protection + bounded failure logging on the live-tail and
|
|
32
|
+
reply-marker transports (no configuration needed; recovery and
|
|
33
|
+
rejecting-peer visibility included).
|
|
34
|
+
|
|
35
|
+
## Evidence
|
|
36
|
+
|
|
37
|
+
CMT-1109 audit, mechanical completion of the #874 pattern (line-level
|
|
38
|
+
CONCUR-reviewed tonight) on two lower-blast-radius transports. Focused
|
|
39
|
+
adversarial second-pass: CONCUR on all probes (Two-Generals widening
|
|
40
|
+
pre-accepted by design — broadcast is void-discarded at its only callsite;
|
|
41
|
+
gate memory peer-bounded; alternating ok/fail chatter is diagnostic at reply
|
|
42
|
+
cadence; local encryptFor throws disambiguate via error detail). Tests: 6 new
|
|
43
|
+
green (AbortSignal pins both transports; P19 bounds — 25 failed flushes → 3
|
|
44
|
+
lines, 12 rejected markers → 2 lines; recovery-once; steady-success silence)
|
|
45
|
+
+ pre-existing transport suites untouched and green; tsc clean.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# Side-Effects Review — Heartbeat-Writer Guard
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `heartbeat-writer-signal`
|
|
4
|
+
**Date:** `2026-06-06`
|
|
5
|
+
**Author:** `Echo (instar-dev agent, autonomous session per Justin's direction)`
|
|
6
|
+
**Second-pass reviewer:** `adversarial reviewer subagent — narrow OBJECT on the promoteToAwake call site, APPLIED pre-commit (rollback-and-rethrow); all other probes CONCUR`
|
|
7
|
+
|
|
8
|
+
## Summary of the change
|
|
9
|
+
|
|
10
|
+
The multi-machine heartbeat writer's three call paths get three deliberate behaviors: (1) the 2-min timer tick and (2) the boot-immediate write go through `writeHeartbeatGuarded()` — try/catch + the new pure `FailureEpisodeLatch` (first-failure log once, ONE DegradationReporter signal per episode at 6min sustained, recovery log + re-arm, retries forever; declared Eternal Sentinel per P19); (3) `promoteToAwake`'s initial write is rollback-and-rethrow — a failed write aborts the promotion cleanly (`_role` + registry rolled back) instead of (pre-fix) dying mid-transition or (first draft) silently completing voiceless. Files: `FailureEpisodeLatch.ts` (new), `MultiMachineCoordinator.ts`, tests.
|
|
11
|
+
|
|
12
|
+
## Decision-point inventory
|
|
13
|
+
|
|
14
|
+
- 2-min tick + boot-immediate write — **modify (guarded)** — a write failure can no longer escape as uncaughtException (pre-fix: server FATAL crash on transient fs error); write cadence unchanged.
|
|
15
|
+
- `promoteToAwake` initial write — **modify (abort-clean)** — failure now rolls back role + registry and rethrows; pre-fix it threw AFTER flipping both (half-promotion).
|
|
16
|
+
- `FailureEpisodeLatch` — **add** — pure signal accountant, no authority.
|
|
17
|
+
|
|
18
|
+
## 1. Over-block
|
|
19
|
+
|
|
20
|
+
`promoteToAwake` now refuses to complete when the initial heartbeat write fails — the only "block," and it is the reviewer-mandated correct behavior: the alternative (silent voiceless promotion) invites a peer failover into dual-awake; the pre-fix behavior (throw after flipping role+registry) was a half-promotion. Failover paths that call promote with a genuinely broken disk now fail loudly at the transition where the operator can see it, with state consistent.
|
|
21
|
+
|
|
22
|
+
## 2. Under-block
|
|
23
|
+
|
|
24
|
+
(a) In heartbeat-only coordination mode (no lease attached — not the production default), a machine whose writes start failing AFTER a successful promotion serves voiceless until the 15min expiry triggers a peer failover; the dual-awake window is bounded by `shouldDemote` on the next check cycle, and the 6min signal precedes the horizon (reviewer probe 1 analysis — lease-attached mode resolves this via epochs regardless). (b) Three raw `writeHeartbeat()` calls remain in `machine.ts` CLI one-shots — no timer vector, errors surface to the CLI naturally (reviewer probe 2: no fix needed). (c) Consolidating the two earlier inline episode latches (lifeline supervisor; live-tail) onto `FailureEpisodeLatch` is deliberate follow-up, not this PR — refactoring just-merged code mid-series adds risk for zero behavior <!-- tracked: CMT-1109 -->.
|
|
25
|
+
|
|
26
|
+
## 3. Level-of-abstraction fit
|
|
27
|
+
|
|
28
|
+
`FailureEpisodeLatch` lands in core (importable by both core and lifeline consumers — the layering that blocked reusing the lifeline-resident sibling), as the canonical extraction of a pattern now shipped four times in one night per Friction-Is-a-Spec. The guard lives in the coordinator that owns the timer; `HeartbeatManager.writeHeartbeat()` itself stays throwing (CLI callers WANT the throw; the policy decision belongs to each caller, which is exactly what the three-site design encodes).
|
|
29
|
+
|
|
30
|
+
## 4. Signal vs authority compliance
|
|
31
|
+
|
|
32
|
+
**Required reference:** `docs/signal-vs-authority.md`
|
|
33
|
+
|
|
34
|
+
- [x] No — the latch is a pure signal accountant. The behavioral changes are failure-HANDLING policy at existing call sites: the tick guard removes an accidental authority (a disk error crashing the server), and the promote rollback makes an existing implicit abort CONSISTENT (state no longer half-flipped). No new decision-maker.
|
|
35
|
+
|
|
36
|
+
## 5. Interactions
|
|
37
|
+
|
|
38
|
+
- **Crash vector closed:** the tick's throw → `uncaughtException` → `server.ts` FATAL path (fs errors are not in the non-fatal allowlist) — traced pre-fix; the guard removes it.
|
|
39
|
+
- **Signal-before-failover:** 6min threshold (3 failed 2-min cycles) < 15min `DEFAULT_FAILOVER_TIMEOUT_MS` expiry — operator hears before a peer acts (reviewer probe 3, verified at source).
|
|
40
|
+
- **DegradationReporter in the catch:** cannot throw back (console + in-memory push + best-effort-wrapped disk persist + fire-and-forget dispatch; reviewer probe 4); 10+ core files already use the singleton.
|
|
41
|
+
- **promoteToAwake consumers:** no test pinned the pre-fix throw-after-flip behavior (reviewer grep); coordinator suite (22) green.
|
|
42
|
+
|
|
43
|
+
## 6. External surfaces / 7. Rollback
|
|
44
|
+
|
|
45
|
+
Logs + one degradation record per episode; no API/schema/config/persistent state; no migration. Rollback = revert; the crash vector and the half-promotion return.
|
|
46
|
+
|
|
47
|
+
## Conclusion
|
|
48
|
+
|
|
49
|
+
The audit lead said "silent failure"; grounding found a crash vector — the verify-every-lead discipline paying again. The fix gives each of three call sites its correct failure policy, extracts the night's episode-latch pattern to its canonical reusable home, and was materially improved by the adversarial pass (the promote rollback) and by its own P19 test (the zero-sentinel bug, caught pre-ship).
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Phase 5 — Second-pass review (multi-machine role authority → required)
|
|
54
|
+
|
|
55
|
+
An adversarial reviewer probed: (1) the promotion semantics change — OBJECT: the first draft's guarded swallow in `promoteToAwake` silently completed a voiceless promotion; prescribed rollback-and-rethrow, APPLIED (with the registry rollback included, which even the pre-fix crash lacked); analyzed heartbeat-only vs lease-attached failover authority end-to-end; (2) remaining raw callsites — CLI one-shots only, safe; (3) 6min signal vs 15min failover horizon — verified at source; (4) DegradationReporter throw-safety in the catch — safe, precedented. Ran the latch + coordinator + leasePull suites and tsc — green/clean. **Verdict: CONCUR with the applied fix.**
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Side-Effects Review — Sibling-Transport Brakes
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `sibling-transport-brakes`
|
|
4
|
+
**Date:** `2026-06-06`
|
|
5
|
+
**Author:** `Echo (instar-dev agent, autonomous session per Justin's direction)`
|
|
6
|
+
**Second-pass reviewer:** `focused adversarial reviewer subagent — CONCUR on all four probes (the #874 pattern itself carries tonight's line-level CONCUR)`
|
|
7
|
+
|
|
8
|
+
## Summary of the change
|
|
9
|
+
|
|
10
|
+
The #874 brake pattern applied verbatim to the two remaining mesh transports: `HttpLiveTailTransport` and `ReplyMarkerTransport` get `AbortSignal.timeout` (30s default, the #874-corrected sizing above the fleet's 5–40s receiver-stall envelope) on their fetches, and `PeerFailureLogGate` state-change failure logging (live-tail N=360; reply-marker N=50 for reply cadence) replacing per-attempt lines, with non-ok responses now gated-logged. Files: both transports, one test file.
|
|
11
|
+
|
|
12
|
+
## Decision-point inventory
|
|
13
|
+
|
|
14
|
+
- Both transports' `broadcast()` — **modify (bounded)** — same requests, same return semantics; failures abort at the timeout instead of hanging; logging gated.
|
|
15
|
+
|
|
16
|
+
## 1. Over-block / 2. Under-block
|
|
17
|
+
|
|
18
|
+
(a) A 30s abort can drop a reply-marker a slower wait might have delivered — widening the marker's documented Two-Generals residual. Reviewer-traced: `broadcast()`'s return is void-discarded at its ONLY callsite (fire-and-forget by design); provider redelivery + the dedup gate + git-committed ledger state are the explicit backstops. Pre-accepted, not new risk. (b) Perfectly alternating ok/fail logs one transition pair per cycle — at reply cadence this is bounded and is diagnostic signal for a genuinely unstable peer (reviewer: not a blocker; an across-resets suppressor would mask real instability). (c) A local `encryptFor` throw logs under the peer's key — imprecise label, but the error detail (the real exception message) disambiguates and the operational meaning ("this peer's flush did not deliver") is identical. (d) Remaining audit item is the SessionRouter design <!-- tracked: CMT-1109 -->.
|
|
19
|
+
|
|
20
|
+
## 3. Level-of-abstraction fit / 4. Signal vs authority
|
|
21
|
+
|
|
22
|
+
Brakes in the transports that generate the cost; cadence stays with callers; reuses the canonical gate class rather than re-implementing. **Signal-only** per `docs/signal-vs-authority.md`: log shaping + failure-timing bounds. Critically verified: neither transport feeds lease renewal/suspend (`.isReachable()`'s only src consumer is `LeaseCoordinator` via the LEASE transport) — the #874 reviewer's false-self-suspend concern structurally cannot apply here.
|
|
23
|
+
|
|
24
|
+
## 5. Interactions
|
|
25
|
+
|
|
26
|
+
- **Exactly-once (the marker's job):** unchanged — the timeout affects only the best-effort fast path; the dedup gate and ledger sync carry correctness (reviewer probe 1).
|
|
27
|
+
- **#867's per-topic backoff:** composes — backoff bounds live-tail attempt RATE per topic; the gate bounds LOG volume per peer across topics.
|
|
28
|
+
- **Old format pins:** none — pre-existing transport suites pass untouched.
|
|
29
|
+
- **Node engines / AbortSignal-vs-test-clocks:** identical to #874 (verified there; same package.json).
|
|
30
|
+
|
|
31
|
+
## 6. External surfaces / 7. Rollback
|
|
32
|
+
|
|
33
|
+
Logs only; no API/schema/config/persistent state; no migration (in-code defaults). Rollback = revert; the hang exposure and per-attempt logging return.
|
|
34
|
+
|
|
35
|
+
## Conclusion
|
|
36
|
+
|
|
37
|
+
Completes the mesh-transport brake set: all four cross-machine wires (lease #874, heartbeat #881, live-tail and reply-marker here) now carry P19's brakes. Mechanical extension of a twice-validated pattern, with the one novel risk (marker-drop under timeout) traced to a documented, backstopped design acceptance.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## Phase 5 — Second-pass review (cross-machine exactly-once adjacency → performed)
|
|
42
|
+
|
|
43
|
+
A focused adversarial pass probed only what is novel beyond #874's reviewed pattern: (1) the Two-Generals widening from a 30s marker abort — traced `broadcast()`'s only callsite (void-discarded, fire-and-forget, comment-documented backstops): pre-accepted by design; (2) gate key granularity + alternating ok/fail behavior — peer-bounded memory, transition chatter acceptable-and-diagnostic at reply cadence; (3) local `encryptFor` throws misattributed to the peer — disambiguated by the error detail, operationally equivalent; (4) ran the new + pre-existing transport suites (15/15) and tsc (clean). **Verdict: CONCUR.**
|