instar 1.3.690 → 1.3.692
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +9 -0
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +7 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/MultiMachineCoordinator.d.ts +40 -0
- package/dist/core/MultiMachineCoordinator.d.ts.map +1 -1
- package/dist/core/MultiMachineCoordinator.js +96 -1
- package/dist/core/MultiMachineCoordinator.js.map +1 -1
- package/dist/core/nobodyPollingActuator.d.ts +61 -0
- package/dist/core/nobodyPollingActuator.d.ts.map +1 -0
- package/dist/core/nobodyPollingActuator.js +61 -0
- package/dist/core/nobodyPollingActuator.js.map +1 -0
- package/dist/core/zombieRelinquish.d.ts +90 -0
- package/dist/core/zombieRelinquish.d.ts.map +1 -0
- package/dist/core/zombieRelinquish.js +95 -0
- package/dist/core/zombieRelinquish.js.map +1 -0
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +3 -3
- package/upgrades/1.3.691.md +28 -0
- package/upgrades/1.3.692.md +25 -0
- package/upgrades/side-effects/mesh-self-heal-g1-core.md +32 -0
- package/upgrades/side-effects/mesh-self-heal-g2-enforce.md +38 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MESH-SELF-HEAL G2 — enforce-mode ACTUATOR (dependency-injected, testable).
|
|
3
|
+
*
|
|
4
|
+
* Spec: MESH-SELF-HEAL-SPEC §3.2; build plan: MESH-SELF-HEAL-G2-BUILD.md.
|
|
5
|
+
*
|
|
6
|
+
* This is the ENFORCE half of G2: given a `claim` decision from
|
|
7
|
+
* `decideNobodyPollingClaim`, actually take poll-ownership — win the fenced
|
|
8
|
+
* epoch-CAS, RE-VERIFY live local poll-success (CAS-win is necessary-not-sufficient,
|
|
9
|
+
* Adv2-F1), then either start polling (drive the existing poll-follows-lease lever)
|
|
10
|
+
* or relinquish + self-exclude. The delicate cross-machine authority (the exact
|
|
11
|
+
* code whose hasty version caused the 2026-06-27 tug-of-war) is isolated behind
|
|
12
|
+
* INJECTED PORTS so it is unit-testable WITHOUT a live coordinator/lifeline, and so
|
|
13
|
+
* the dryRun gate is a single, auditable chokepoint.
|
|
14
|
+
*
|
|
15
|
+
* Ships DARK + dryRun-first. In dryRun it records "would actuate" and performs ZERO
|
|
16
|
+
* side effects (no CAS acquire, no poll-lever write) — pure observation. Only an
|
|
17
|
+
* explicit `dryRun:false` (the enforce promotion) ever touches the lease or polling.
|
|
18
|
+
*
|
|
19
|
+
* SECOND-PASS REVIEW REQUIRED before merge (this holds poll-ownership authority).
|
|
20
|
+
*/
|
|
21
|
+
import { decidePostCasSelfReverify } from './nobodyPollingRecovery.js';
|
|
22
|
+
/**
|
|
23
|
+
* Actuate a G2 claim decision. Pure control-flow over injected ports; the only
|
|
24
|
+
* authority is exercised through the ports, and ONLY when `dryRun` is false and the
|
|
25
|
+
* decision is a genuine self-claim. Idempotent per call.
|
|
26
|
+
*/
|
|
27
|
+
export async function applyNobodyPollingRecovery(args) {
|
|
28
|
+
const { decision, dryRun, ports, ledger, nowIso, log } = args;
|
|
29
|
+
// Only a genuine self-claim actuates. stand-down / veto / fail-closed / hold /
|
|
30
|
+
// escalate / await / no-op are non-actuating (the decision recorder already
|
|
31
|
+
// counted them); the actuator must NEVER touch the lease for those.
|
|
32
|
+
if (!decision.selfClaims || decision.action !== 'claim') {
|
|
33
|
+
return { result: 'no-action', reason: `decision=${decision.action}` };
|
|
34
|
+
}
|
|
35
|
+
// DARK / dryRun: observe only. Record the counterfactual; perform NO side effect.
|
|
36
|
+
if (dryRun) {
|
|
37
|
+
log?.(`[g2-actuator] DRY-RUN would claim poll-ownership (epoch n+1) — no CAS, no poll-lever write`);
|
|
38
|
+
return { result: 'dry-run-would-claim', reason: 'dry-run-observe-only' };
|
|
39
|
+
}
|
|
40
|
+
// ENFORCE: win the fenced epoch-CAS. Losing means a peer won the same episode —
|
|
41
|
+
// stand down (the single-claimant invariant holds via the fence, not our guess).
|
|
42
|
+
const won = await ports.acquireFencedCas();
|
|
43
|
+
if (!won) {
|
|
44
|
+
return { result: 'cas-lost', reason: 'another-machine-won-the-fenced-epoch' };
|
|
45
|
+
}
|
|
46
|
+
// CAS-win is necessary but NOT sufficient (Adv2-F1): re-verify OWN live poll
|
|
47
|
+
// freshness (current+local) before committing to serve.
|
|
48
|
+
const reverify = decidePostCasSelfReverify({ localPollSucceededFresh: ports.localPollSucceededFresh() });
|
|
49
|
+
if (!reverify.commit) {
|
|
50
|
+
ports.relinquishAndSelfExclude();
|
|
51
|
+
ledger.recordSelfExclusion(nowIso);
|
|
52
|
+
log?.(`[g2-actuator] won CAS but self-unfit on re-verify → relinquished + self-excluded`);
|
|
53
|
+
return { result: 'self-excluded', reason: reverify.reason };
|
|
54
|
+
}
|
|
55
|
+
// Committed: drive the poll-follows-lease lever to START polling at the won epoch.
|
|
56
|
+
// (The caller's post-claim live-verify confirms lifeline-poll-active.json advances.)
|
|
57
|
+
ports.startPolling(ports.currentEpoch());
|
|
58
|
+
log?.(`[g2-actuator] claimed poll-ownership + started polling at epoch ${ports.currentEpoch()}`);
|
|
59
|
+
return { result: 'claimed-serving', reason: 'cas-won-self-reverified-fit' };
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=nobodyPollingActuator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nobodyPollingActuator.js","sourceRoot":"","sources":["../../src/core/nobodyPollingActuator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAoCvE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAOhD;IACC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAE9D,+EAA+E;IAC/E,4EAA4E;IAC5E,oEAAoE;IACpE,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACxD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;IACxE,CAAC;IAED,kFAAkF;IAClF,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,EAAE,CAAC,4FAA4F,CAAC,CAAC;QACpG,OAAO,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;IAC3E,CAAC;IAED,gFAAgF;IAChF,iFAAiF;IACjF,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,gBAAgB,EAAE,CAAC;IAC3C,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,sCAAsC,EAAE,CAAC;IAChF,CAAC;IAED,6EAA6E;IAC7E,wDAAwD;IACxD,MAAM,QAAQ,GAAG,yBAAyB,CAAC,EAAE,uBAAuB,EAAE,KAAK,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IACzG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrB,KAAK,CAAC,wBAAwB,EAAE,CAAC;QACjC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACnC,GAAG,EAAE,CAAC,kFAAkF,CAAC,CAAC;QAC1F,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC9D,CAAC;IAED,mFAAmF;IACnF,qFAAqF;IACrF,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;IACzC,GAAG,EAAE,CAAC,mEAAmE,KAAK,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACjG,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,6BAA6B,EAAE,CAAC;AAC9E,CAAC"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MESH-SELF-HEAL G1 — lease↔job binding via a THREE-SIGNAL liveness model (pure core).
|
|
3
|
+
*
|
|
4
|
+
* Spec: MESH-SELF-HEAL-SPEC §3.1. The DEEPEST fix: the "in charge" lease means two
|
|
5
|
+
* things at once — "I'm the coordinator" AND "I'm actually serving you" — and
|
|
6
|
+
* conflating them is what let the 2026-06-19/27 zombie hold the badge while serving
|
|
7
|
+
* nothing. G1 binds the two: a holder that has stopped doing the JOB must relinquish
|
|
8
|
+
* the badge, decided from THREE machine-local, monotonic liveness signals sourced
|
|
9
|
+
* from the lifeline's ACTUAL-poll truth (FD10: server-intent ≠ lifeline-actual is
|
|
10
|
+
* exactly what hid the incident).
|
|
11
|
+
*
|
|
12
|
+
* This module is the PURE decision (no I/O) so every branch is unit-testable on both
|
|
13
|
+
* sides. The caller supplies freshness booleans (computed by comparing each
|
|
14
|
+
* machine-local monotonic watermark against `jobStaleThresholdMs` on its OWN clock —
|
|
15
|
+
* never a cross-process/cross-machine monotonic subtraction), the debounce result,
|
|
16
|
+
* and POSITIVE peer-evidence of a global outage. It is SIGNAL/decision only; the
|
|
17
|
+
* caller actuates via F3's `relinquishAndBroadcast()` (signed tombstone). Ships DARK.
|
|
18
|
+
*
|
|
19
|
+
* The three signals (all machine-local, evaluated by the machine about ITSELF):
|
|
20
|
+
* - pollAttemptedMonoMs — a getUpdates cycle was attempted (alive + trying)
|
|
21
|
+
* - pollSucceededMonoMs — a getUpdates response succeeded (channel reachable)
|
|
22
|
+
* - serveProgressedMonoMs — a fetched update was dispatched/served (end-to-end)
|
|
23
|
+
*/
|
|
24
|
+
export interface ZombieRelinquishInput {
|
|
25
|
+
/** This machine currently holds the fenced lease. */
|
|
26
|
+
holdsLease: boolean;
|
|
27
|
+
/** The `active` lease role (NOT observe-only/deferential — those are F3's path,
|
|
28
|
+
* not G1; finding Int-LOW). G1 only ever acts on an active holder. */
|
|
29
|
+
isActiveLeaseRole: boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Updates are PENDING = `lastFetchedUpdateId > lastServedUpdateId` (the two
|
|
32
|
+
* poll-side counters) — NOT serve-queue depth, NOT the Telegram-acked offset
|
|
33
|
+
* (Adv2-F5/Adv3-F-D: the poller advances the offset on FETCH, so "offset not
|
|
34
|
+
* advanced" would falsely read non-pending even when serve silently dropped).
|
|
35
|
+
*/
|
|
36
|
+
pending: boolean;
|
|
37
|
+
/** pollAttemptedMonoMs fresh (within jobStaleThresholdMs, own clock). */
|
|
38
|
+
pollAttemptedFresh: boolean;
|
|
39
|
+
/** pollSucceededMonoMs fresh. */
|
|
40
|
+
pollSucceededFresh: boolean;
|
|
41
|
+
/** serveProgressedMonoMs fresh (boot-epoch-fenced — a prior incarnation's stamp
|
|
42
|
+
* reads as NOT fresh; round-4 Adv4-B). */
|
|
43
|
+
serveProgressedFresh: boolean;
|
|
44
|
+
/** The relevant-staleness has PERSISTED across N fresh evaluator ticks (Adv-F8 —
|
|
45
|
+
* defeats an evaluator-resume false trip). The caller debounces. */
|
|
46
|
+
staleConfirmed: boolean;
|
|
47
|
+
/**
|
|
48
|
+
* POSITIVE evidence the outage is GLOBAL: a fresh, signed heartbeat from ≥1 live
|
|
49
|
+
* peer in-window that ALSO reports pollSucceeded-stale. "I can't hear any peer"
|
|
50
|
+
* must NEVER set this true (Adv2-F2 — a heartbeat-transport partition is not a
|
|
51
|
+
* global outage; misreading it would HOLD and protect a real zombie).
|
|
52
|
+
*/
|
|
53
|
+
peerConfirmsGlobalOutage: boolean;
|
|
54
|
+
}
|
|
55
|
+
export type ZombieRelinquishAction = 'not-applicable' | 'healthy' | 'await-confirm' | 'hold-global' | 'relinquish' | 'relinquish-wedged';
|
|
56
|
+
export interface ZombieRelinquishDecision {
|
|
57
|
+
action: ZombieRelinquishAction;
|
|
58
|
+
/** Should the caller run relinquishAndBroadcast() now? */
|
|
59
|
+
relinquish: boolean;
|
|
60
|
+
/** Short machine-readable reason for the audit log. */
|
|
61
|
+
reason: string;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Decide whether an active lease-HOLDER is a zombie that must relinquish. Pure;
|
|
65
|
+
* the caller actuates only on `relinquish` (quiesce the renew path + draw a
|
|
66
|
+
* tombstone nonce > the highest it will emit this epoch, then relinquishAndBroadcast).
|
|
67
|
+
*/
|
|
68
|
+
export declare function decideZombieRelinquish(input: ZombieRelinquishInput): ZombieRelinquishDecision;
|
|
69
|
+
/**
|
|
70
|
+
* G1 soak/observability ledger — evaluable evidence (mirrors G2/G3 close-the-loop).
|
|
71
|
+
* Counts each decision-class transition. SIGNAL only.
|
|
72
|
+
*/
|
|
73
|
+
export interface ZombieRelinquishLedgerSummary {
|
|
74
|
+
evaluations: number;
|
|
75
|
+
relinquishedLocal: number;
|
|
76
|
+
relinquishedWedged: number;
|
|
77
|
+
heldGlobal: number;
|
|
78
|
+
healthy: number;
|
|
79
|
+
awaitConfirm: number;
|
|
80
|
+
firstAt: string | null;
|
|
81
|
+
lastAt: string | null;
|
|
82
|
+
}
|
|
83
|
+
export declare class ZombieRelinquishLedger {
|
|
84
|
+
private s;
|
|
85
|
+
record(decision: ZombieRelinquishDecision, nowIso: string): void;
|
|
86
|
+
summary(): ZombieRelinquishLedgerSummary;
|
|
87
|
+
}
|
|
88
|
+
/** Process-wide shared G1 ledger (mirrors sharedG2NobodyPollingLedger). */
|
|
89
|
+
export declare const sharedG1ZombieRelinquishLedger: ZombieRelinquishLedger;
|
|
90
|
+
//# sourceMappingURL=zombieRelinquish.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"zombieRelinquish.d.ts","sourceRoot":"","sources":["../../src/core/zombieRelinquish.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,MAAM,WAAW,qBAAqB;IACpC,qDAAqD;IACrD,UAAU,EAAE,OAAO,CAAC;IACpB;2EACuE;IACvE,iBAAiB,EAAE,OAAO,CAAC;IAC3B;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,yEAAyE;IACzE,kBAAkB,EAAE,OAAO,CAAC;IAC5B,iCAAiC;IACjC,kBAAkB,EAAE,OAAO,CAAC;IAC5B;+CAC2C;IAC3C,oBAAoB,EAAE,OAAO,CAAC;IAC9B;yEACqE;IACrE,cAAc,EAAE,OAAO,CAAC;IACxB;;;;;OAKG;IACH,wBAAwB,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,MAAM,sBAAsB,GAC9B,gBAAgB,GAChB,SAAS,GACT,eAAe,GACf,aAAa,GACb,YAAY,GACZ,mBAAmB,CAAC;AAExB,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,sBAAsB,CAAC;IAC/B,0DAA0D;IAC1D,UAAU,EAAE,OAAO,CAAC;IACpB,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,qBAAqB,GAAG,wBAAwB,CAsC7F;AAED;;;GAGG;AACH,MAAM,WAAW,6BAA6B;IAC5C,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,CAAC,CAGP;IAEF,MAAM,CAAC,QAAQ,EAAE,wBAAwB,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAchE,OAAO,IAAI,6BAA6B;CAGzC;AAED,2EAA2E;AAC3E,eAAO,MAAM,8BAA8B,wBAA+B,CAAC"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MESH-SELF-HEAL G1 — lease↔job binding via a THREE-SIGNAL liveness model (pure core).
|
|
3
|
+
*
|
|
4
|
+
* Spec: MESH-SELF-HEAL-SPEC §3.1. The DEEPEST fix: the "in charge" lease means two
|
|
5
|
+
* things at once — "I'm the coordinator" AND "I'm actually serving you" — and
|
|
6
|
+
* conflating them is what let the 2026-06-19/27 zombie hold the badge while serving
|
|
7
|
+
* nothing. G1 binds the two: a holder that has stopped doing the JOB must relinquish
|
|
8
|
+
* the badge, decided from THREE machine-local, monotonic liveness signals sourced
|
|
9
|
+
* from the lifeline's ACTUAL-poll truth (FD10: server-intent ≠ lifeline-actual is
|
|
10
|
+
* exactly what hid the incident).
|
|
11
|
+
*
|
|
12
|
+
* This module is the PURE decision (no I/O) so every branch is unit-testable on both
|
|
13
|
+
* sides. The caller supplies freshness booleans (computed by comparing each
|
|
14
|
+
* machine-local monotonic watermark against `jobStaleThresholdMs` on its OWN clock —
|
|
15
|
+
* never a cross-process/cross-machine monotonic subtraction), the debounce result,
|
|
16
|
+
* and POSITIVE peer-evidence of a global outage. It is SIGNAL/decision only; the
|
|
17
|
+
* caller actuates via F3's `relinquishAndBroadcast()` (signed tombstone). Ships DARK.
|
|
18
|
+
*
|
|
19
|
+
* The three signals (all machine-local, evaluated by the machine about ITSELF):
|
|
20
|
+
* - pollAttemptedMonoMs — a getUpdates cycle was attempted (alive + trying)
|
|
21
|
+
* - pollSucceededMonoMs — a getUpdates response succeeded (channel reachable)
|
|
22
|
+
* - serveProgressedMonoMs — a fetched update was dispatched/served (end-to-end)
|
|
23
|
+
*/
|
|
24
|
+
/**
|
|
25
|
+
* Decide whether an active lease-HOLDER is a zombie that must relinquish. Pure;
|
|
26
|
+
* the caller actuates only on `relinquish` (quiesce the renew path + draw a
|
|
27
|
+
* tombstone nonce > the highest it will emit this epoch, then relinquishAndBroadcast).
|
|
28
|
+
*/
|
|
29
|
+
export function decideZombieRelinquish(input) {
|
|
30
|
+
const { holdsLease, isActiveLeaseRole, pending, pollAttemptedFresh, pollSucceededFresh, serveProgressedFresh, staleConfirmed, peerConfirmsGlobalOutage, } = input;
|
|
31
|
+
// G1 only acts on an ACTIVE holder. A non-holder / observe-only / deferential
|
|
32
|
+
// machine is handled elsewhere (F3 / G2), never here.
|
|
33
|
+
if (!holdsLease || !isActiveLeaseRole) {
|
|
34
|
+
return { action: 'not-applicable', relinquish: false, reason: 'not-an-active-holder' };
|
|
35
|
+
}
|
|
36
|
+
// The relevant liveness signal: end-to-end SERVE progress when updates are
|
|
37
|
+
// pending, else the poll-SUCCESS heartbeat when idle (no pending work to serve).
|
|
38
|
+
const relevantStale = pending ? !serveProgressedFresh : !pollSucceededFresh;
|
|
39
|
+
if (!relevantStale) {
|
|
40
|
+
return { action: 'healthy', relinquish: false, reason: pending ? 'serve-progressing' : 'poll-succeeding' };
|
|
41
|
+
}
|
|
42
|
+
// Debounce: a single stale read is not a zombie (an evaluator-resume blip).
|
|
43
|
+
if (!staleConfirmed) {
|
|
44
|
+
return { action: 'await-confirm', relinquish: false, reason: 'stale-not-yet-confirmed' };
|
|
45
|
+
}
|
|
46
|
+
// The poll LOOP itself is wedged (not even attempting) → relinquish unconditionally;
|
|
47
|
+
// a wedged holder serves nothing and HOLD cannot be justified.
|
|
48
|
+
if (!pollAttemptedFresh) {
|
|
49
|
+
return { action: 'relinquish-wedged', relinquish: true, reason: 'poll-loop-wedged-pollattempted-stale' };
|
|
50
|
+
}
|
|
51
|
+
// pollAttempted is fresh (alive + trying) but the channel/serve is stale. Only a
|
|
52
|
+
// POSITIVE peer confirmation that the outage is shared justifies HOLD; absent it,
|
|
53
|
+
// treat as LOCAL failure and relinquish (the safe direction — G2 picks a server).
|
|
54
|
+
if (peerConfirmsGlobalOutage) {
|
|
55
|
+
return { action: 'hold-global', relinquish: false, reason: 'global-outage-peer-confirmed-hold' };
|
|
56
|
+
}
|
|
57
|
+
return { action: 'relinquish', relinquish: true, reason: 'local-failure-relinquish-safe-direction' };
|
|
58
|
+
}
|
|
59
|
+
export class ZombieRelinquishLedger {
|
|
60
|
+
s = {
|
|
61
|
+
evaluations: 0, relinquishedLocal: 0, relinquishedWedged: 0, heldGlobal: 0,
|
|
62
|
+
healthy: 0, awaitConfirm: 0, firstAt: null, lastAt: null,
|
|
63
|
+
};
|
|
64
|
+
record(decision, nowIso) {
|
|
65
|
+
if (decision.action === 'not-applicable')
|
|
66
|
+
return; // a non-holder is a non-event
|
|
67
|
+
this.s.evaluations += 1;
|
|
68
|
+
switch (decision.action) {
|
|
69
|
+
case 'relinquish':
|
|
70
|
+
this.s.relinquishedLocal += 1;
|
|
71
|
+
break;
|
|
72
|
+
case 'relinquish-wedged':
|
|
73
|
+
this.s.relinquishedWedged += 1;
|
|
74
|
+
break;
|
|
75
|
+
case 'hold-global':
|
|
76
|
+
this.s.heldGlobal += 1;
|
|
77
|
+
break;
|
|
78
|
+
case 'healthy':
|
|
79
|
+
this.s.healthy += 1;
|
|
80
|
+
break;
|
|
81
|
+
case 'await-confirm':
|
|
82
|
+
this.s.awaitConfirm += 1;
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
if (this.s.firstAt === null)
|
|
86
|
+
this.s.firstAt = nowIso;
|
|
87
|
+
this.s.lastAt = nowIso;
|
|
88
|
+
}
|
|
89
|
+
summary() {
|
|
90
|
+
return { ...this.s };
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** Process-wide shared G1 ledger (mirrors sharedG2NobodyPollingLedger). */
|
|
94
|
+
export const sharedG1ZombieRelinquishLedger = new ZombieRelinquishLedger();
|
|
95
|
+
//# sourceMappingURL=zombieRelinquish.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"zombieRelinquish.js","sourceRoot":"","sources":["../../src/core/zombieRelinquish.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAkDH;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,KAA4B;IACjE,MAAM,EACJ,UAAU,EAAE,iBAAiB,EAAE,OAAO,EACtC,kBAAkB,EAAE,kBAAkB,EAAE,oBAAoB,EAC5D,cAAc,EAAE,wBAAwB,GACzC,GAAG,KAAK,CAAC;IAEV,8EAA8E;IAC9E,sDAAsD;IACtD,IAAI,CAAC,UAAU,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtC,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;IACzF,CAAC;IAED,2EAA2E;IAC3E,iFAAiF;IACjF,MAAM,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAC5E,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,iBAAiB,EAAE,CAAC;IAC7G,CAAC;IAED,4EAA4E;IAC5E,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,yBAAyB,EAAE,CAAC;IAC3F,CAAC;IAED,qFAAqF;IACrF,+DAA+D;IAC/D,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,sCAAsC,EAAE,CAAC;IAC3G,CAAC;IAED,iFAAiF;IACjF,kFAAkF;IAClF,kFAAkF;IAClF,IAAI,wBAAwB,EAAE,CAAC;QAC7B,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAC;IACnG,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,yCAAyC,EAAE,CAAC;AACvG,CAAC;AAiBD,MAAM,OAAO,sBAAsB;IACzB,CAAC,GAAkC;QACzC,WAAW,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;QAC1E,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI;KACzD,CAAC;IAEF,MAAM,CAAC,QAAkC,EAAE,MAAc;QACvD,IAAI,QAAQ,CAAC,MAAM,KAAK,gBAAgB;YAAE,OAAO,CAAC,8BAA8B;QAChF,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC;QACxB,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxB,KAAK,YAAY;gBAAE,IAAI,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,CAAC;gBAAC,MAAM;YACxD,KAAK,mBAAmB;gBAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,CAAC;gBAAC,MAAM;YAChE,KAAK,aAAa;gBAAE,IAAI,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;gBAAC,MAAM;YAClD,KAAK,SAAS;gBAAE,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;gBAAC,MAAM;YAC3C,KAAK,eAAe;gBAAE,IAAI,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC;gBAAC,MAAM;QACxD,CAAC;QACD,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI;YAAE,IAAI,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,CAAC;QACrD,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,OAAO;QACL,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC;IACvB,CAAC;CACF;AAED,2EAA2E;AAC3E,MAAM,CAAC,MAAM,8BAA8B,GAAG,IAAI,sBAAsB,EAAE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-06-
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-06-28T10:05:54.303Z",
|
|
5
|
+
"instarVersion": "1.3.692",
|
|
6
6
|
"entryCount": 202,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -1618,7 +1618,7 @@
|
|
|
1618
1618
|
"type": "subsystem",
|
|
1619
1619
|
"domain": "coordination",
|
|
1620
1620
|
"sourcePath": "src/core/MultiMachineCoordinator.ts",
|
|
1621
|
-
"contentHash": "
|
|
1621
|
+
"contentHash": "db0a3a53e3b3fc6d8c9593918944cba27c1faccd5a369d71bf1e555cf20bb028",
|
|
1622
1622
|
"since": "2025-01-01"
|
|
1623
1623
|
},
|
|
1624
1624
|
"subsystem:backup-manager": {
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
Mesh Self-Heal **G2 enforce** (MESH-SELF-HEAL-SPEC §3.2) — the take-over actuation for the nobody-serving alarm. When NO machine is polling Telegram (the zombie-lease-holder state behind the message-drop incidents), exactly ONE fit machine now takes over poll-ownership — never zero (drops), never two (the Telegram 409 poll-war).
|
|
9
|
+
|
|
10
|
+
- `src/core/nobodyPollingActuator.ts` (`applyNobodyPollingRecovery`) — dependency-injected actuator: win the fenced epoch-CAS → re-verify own poll-freshness (CAS-win is necessary-not-sufficient) → start polling, else relinquish + self-exclude.
|
|
11
|
+
- `MultiMachineCoordinator.evaluateNobodyPolling()` — wires the detector + single-claimant election to the actuator with the real lease ports (`acquireIfEligible` / `currentEpoch` / `writeLeasePollIntent` / `relinquishAndBroadcast`), debounced (confirm=3) and reentrancy-guarded.
|
|
12
|
+
- `server.ts` `refreshPool` — runs it on the existing 30s pool cadence, fire-and-forget.
|
|
13
|
+
|
|
14
|
+
Ships **dark + dryRun-first** (`multiMachine.nobodyPollingRecovery`, OMITS `enabled` → dev-gated). In dryRun it detects + elects + records the soak counterfactual but performs ZERO side effects (no lease acquire, no poll-lever write) — verified by an independent second-pass review (the high-risk lease-authority gate). Flipping `dryRun:false` is the deliberate enforce promotion, gated on two tracked prerequisites: the real pollSucceeded watermark + the peer-confirmed global-outage plumbing (see MESH-SELF-HEAL-G2-BUILD.md).
|
|
15
|
+
|
|
16
|
+
## What to Tell Your User
|
|
17
|
+
|
|
18
|
+
When this is turned on (it ships off, in observe-only mode for now), it fixes the "nobody is polling, so my messages silently drop" problem on multi-machine setups: if the in-charge machine goes quiet, exactly one healthy machine automatically takes over polling — no tug-of-war, no duplicates, no action needed from you. Single-machine setups are unaffected.
|
|
19
|
+
|
|
20
|
+
## Summary of New Capabilities
|
|
21
|
+
|
|
22
|
+
- `multiMachine.nobodyPollingRecovery` `{ dryRun }` — opt-in (dev-gated) nobody-polling take-over: detect → elect one machine → acquire the fenced lease → start polling. Dark + dryRun by default (observe-only; zero side effects until a deliberate enforce promotion).
|
|
23
|
+
|
|
24
|
+
## Evidence
|
|
25
|
+
|
|
26
|
+
- `tests/unit/nobodyPollingActuator.test.ts` — 5 tests, both sides of every boundary incl. the critical dryRun=zero-side-effects safety invariant (no CAS, no poll-write) and CAS-lost / self-excluded / claimed-serving.
|
|
27
|
+
- Builds on the merged G2 core (18 unit) + observe route (4 integration). Dark-gate lint golden map updated for the +7 line shift. Typecheck clean; full G2 suite green.
|
|
28
|
+
- Independent Phase-5 second-pass review: CONCUR (dryRun invariant holds; single-claimant is the fenced CAS; no tug-of-war). Its one minor concern (reentrancy guard) was fixed in this commit.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
Mesh Self-Heal **G1 — core decision logic** (increment 1 of the lease↔job binding, MESH-SELF-HEAL-SPEC §3.1). Adds `src/core/zombieRelinquish.ts`: the PURE, deterministic decision for the deepest fix — the "in charge" lease currently means two things at once ("I'm the coordinator" AND "I'm actually serving you"), and conflating them is what let a machine hold the badge while serving nothing. G1 binds them: an active holder that has stopped doing the JOB must relinquish.
|
|
9
|
+
|
|
10
|
+
- `decideZombieRelinquish` — three-signal liveness (pollAttempted / pollSucceeded / serveProgressed, all machine-local, about the machine ITSELF): `not-applicable` for a non-active-holder; `healthy`; `await-confirm` (debounced); `relinquish-wedged` (poll loop dead → unconditional); `hold-global` (only on POSITIVE peer evidence the outage is shared — "can't hear a peer" ≠ global); `relinquish` (confirmed LOCAL failure → safe direction, G2 backstops). Pending work keys on serve-progress; idle keys on poll-success.
|
|
11
|
+
- `ZombieRelinquishLedger` — evaluable soak evidence (mirrors G2/G3).
|
|
12
|
+
|
|
13
|
+
Pure decision logic, NOT yet wired — ZERO runtime effect until the next increment (the three-signal watermark plumbing incl. the new `state/serve-progress.json` record + the tickLease holder-branch wiring + F3 relinquish actuation) consumes it.
|
|
14
|
+
|
|
15
|
+
## What to Tell Your User
|
|
16
|
+
|
|
17
|
+
Nothing changes yet — inert internal decision logic for the deepest part of the multi-machine self-heal, not active anywhere. When the full feature lands, a machine that holds the "in charge" badge but has quietly stopped fetching/serving your messages will automatically hand the badge off (instead of sitting on it while messages drop) — and exactly one healthy machine takes over. No action needed.
|
|
18
|
+
|
|
19
|
+
## Summary of New Capabilities
|
|
20
|
+
|
|
21
|
+
- None user-facing in this increment. New internal module `src/core/zombieRelinquish.ts` (pure decision core) consumed by a later wiring increment; no route, config flag, or runtime surface added yet.
|
|
22
|
+
|
|
23
|
+
## Evidence
|
|
24
|
+
|
|
25
|
+
- `tests/unit/zombieRelinquish.test.ts` — 11 unit tests covering both sides of every branch: not-a-holder / observe-only → not-applicable; healthy (pending→serve, idle→poll); await-confirm; relinquish-wedged; hold-global (peer-confirmed); relinquish (no peer evidence = safe direction); fetched-and-dropped zombie; ledger accounting. Typecheck clean.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Side-Effects Review — Mesh Self-Heal G1 (core decision logic)
|
|
2
|
+
|
|
3
|
+
**Change:** The PURE decision core of G1 (lease↔job binding via a three-signal liveness model) from MESH-SELF-HEAL-SPEC §3.1 — `src/core/zombieRelinquish.ts` (`decideZombieRelinquish` + `ZombieRelinquishLedger`) + 11 unit tests. This is increment 1 of G1: the decision only. It is NOT YET WIRED to any tick, watermark plumbing, or actuation — ZERO runtime effect until a later increment consumes it.
|
|
4
|
+
|
|
5
|
+
**Decision point?** The module encodes a decision (is an active holder a zombie that must relinquish?), but nothing CALLS it yet, so there is no live decision-point in this increment. Signal-vs-authority assessed (Q4).
|
|
6
|
+
|
|
7
|
+
## 1. Over-block
|
|
8
|
+
N/A — unwired. By design it biases AGAINST relinquishing: `not-applicable` for a non-active-holder; `healthy` when the relevant signal is fresh; `await-confirm` until the staleness persists across N ticks. It only relinquishes on a CONFIRMED zombie.
|
|
9
|
+
|
|
10
|
+
## 2. Under-block
|
|
11
|
+
As pure logic, none in-scope. The KNOWN not-yet-built pieces (tracked, not orphan-deferred — see MESH-SELF-HEAL-G2-BUILD.md / spec §3.1): the three-signal watermark plumbing (`pollAttempted/SucceededMonoMs` from `lifeline-poll-active.json`; `serveProgressedMonoMs` in a NEW single-writer `state/serve-progress.json` with a boot-epoch fence), the `lastFetched>lastServed` pending counters, the per-tick debounce, the positive-peer-evidence global-outage signal, and the tickLease holder-branch wiring + F3 `relinquishAndBroadcast` actuation (with the tombstone-nonce quiesce). Until those land, this module is inert.
|
|
12
|
+
|
|
13
|
+
## 3. Level-of-abstraction fit
|
|
14
|
+
Correct. A pure decision module mirroring `nobodyPollingRecovery.ts` (G2) / `leaseGatedSpawn.ts` (G3) — no I/O, fully unit-testable, consumed by a thin wiring layer later. It encodes the spec's exact rule: pending → serveProgressed signal, idle → pollSucceeded signal; wedged poll loop → unconditional relinquish; "can't hear a peer" ≠ global (only positive peer evidence HOLDs).
|
|
15
|
+
|
|
16
|
+
## 4. Signal vs authority compliance
|
|
17
|
+
COMPLIANT. Pure function, no I/O, no authority. The actual authority (`relinquishAndBroadcast` — a signed tombstone) lives in the lease coordinator, which the future wiring calls; this module only DECIDES. The decision keys ONLY on machine-local signals about the machine ITSELF (never a peer-observed value driving a relinquish; finding Sec-F1) — the structural defense against a skew/partition-induced false relinquish.
|
|
18
|
+
|
|
19
|
+
## 5. Interactions
|
|
20
|
+
None in this increment (unwired). Intended (wiring increment): reads the lifeline-actual truth (FD10) + the new serve-progress record; the relinquish runs F3's path; it composes with G2 (a relinquished holder lets G2's single-claimant pick a server) and G3 (the lease it binds is the same fence G3's spawn gate reads).
|
|
21
|
+
|
|
22
|
+
## 6. External surfaces
|
|
23
|
+
None yet. No route, config flag, or watermark write in this increment — pure logic. The exported `sharedG1ZombieRelinquishLedger` singleton mirrors the G2/G3 ledgers; written/read only once the wiring lands.
|
|
24
|
+
|
|
25
|
+
## 7. Multi-machine posture (Cross-Machine Coherence)
|
|
26
|
+
This IS the deepest cross-machine coherence fix — binding "holds the lease" to "actually serving" so a zombie holder self-relinquishes. The three watermarks are machine-local, never replicated (Sca-F1), evaluated by each machine about ITSELF (skew-immune — no foreign-monotonic subtraction). The decision is local; coherence emerges because a zombie relinquishes and G2's fenced single-claimant picks the successor.
|
|
27
|
+
|
|
28
|
+
## 8. Rollback cost
|
|
29
|
+
Trivial — revert the commit. Unreferenced by any runtime path, so removing it cannot affect a running agent. No config, no migration, no state.
|
|
30
|
+
|
|
31
|
+
## Second-pass review
|
|
32
|
+
Not triggered for this increment (unwired pure logic, no live decision-point / block-allow authority / session-lifecycle touch). The Phase-5 second-pass IS required for the WIRING increment (it consumes lease-relinquish authority + adds cross-process watermark plumbing) and is flagged for that increment.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Side-Effects Review — Mesh Self-Heal G2 (enforce actuation)
|
|
2
|
+
|
|
3
|
+
**Change:** The ENFORCE half of G2 (MESH-SELF-HEAL-SPEC §3.2) — actually taking over poll-ownership when nobody is polling. HIGH-RISK: it holds cross-machine poll-ownership authority (the area whose hasty version caused the 2026-06-27 tug-of-war). Pieces:
|
|
4
|
+
- `src/core/nobodyPollingActuator.ts` (`applyNobodyPollingRecovery`) — dependency-injected actuator: dryRun gate → `acquireFencedCas` → `decidePostCasSelfReverify` → `startPolling` / `relinquishAndSelfExclude`. 5 unit tests.
|
|
5
|
+
- `MultiMachineCoordinator.evaluateNobodyPolling()` — the wiring: gate (`nobodyPollingRecoveryCfg`) → B5 verdict → debounce (`_g2SilenceStreak`, confirm=3) → `decideNobodyPollingClaim` → record → actuate (real ports: `leaseCoordinator.acquireIfEligible` / `currentEpoch` / `writeLeasePollIntent(true)` / `relinquishAndBroadcast`). Reentrancy-guarded (`_g2Evaluating`).
|
|
6
|
+
- `server.ts` `refreshPool` — the 30s cadence (fire-and-forget, gated inside the coordinator).
|
|
7
|
+
- `ConfigDefaults`: `multiMachine.nobodyPollingRecovery: { dryRun: true }` (OMITS `enabled` → dev-gated). Dark-gate lint golden map updated (+7 line shift).
|
|
8
|
+
|
|
9
|
+
**Decision point?** YES — it acquires the fenced lease + starts polling. Signal-vs-authority + the dryRun safety invariant are the load-bearing concerns.
|
|
10
|
+
|
|
11
|
+
## 1. Over-block
|
|
12
|
+
N/A (it grants poll-ownership, never blocks). It biases hard against acting: only a confirmed real silence where this machine is the deterministic single claimant AND wins the fenced CAS AND re-verifies its own poll-freshness ever serves.
|
|
13
|
+
|
|
14
|
+
## 2. Under-block
|
|
15
|
+
Ships dark+dryRun (default), so nothing actuates until a deliberate `dryRun:false`. **Enforce-ENABLE prerequisites (TRACKED, not orphan-deferred — see MESH-SELF-HEAL-G2-BUILD.md + the enforce note below):** (1) the real `pollSucceededMonoMs`/serve-progress watermark for `localPollSucceededFresh` (currently a lifeline-liveness approximation — consulted ONLY when dryRun:false); (2) peer-confirmed `globalOutageEvidence` plumbing (currently hard-coded false → a confirmed silence proceeds to elect, the local-failure-safe direction). Both are inert under dryRun.
|
|
16
|
+
|
|
17
|
+
## 3. Level-of-abstraction fit
|
|
18
|
+
The authority lives in the coordinator (it owns the leaseCoordinator); the server only supplies the cadence + capacities. The actuator isolates the delicate flow behind injected ports (testable without a live coordinator; the dryRun gate is a single chokepoint). Reuses `acquireIfEligible` (the SAME fence `tickLease` uses) — not a parallel acquire path.
|
|
19
|
+
|
|
20
|
+
## 4. Signal vs authority compliance
|
|
21
|
+
COMPLIANT with the safety design: the only authority (acquire CAS + start polling) is exercised through the ports ONLY when `dryRun:false` AND the decision is a genuine self-claim. The election is deterministic + machine-agnostic, but it is NOT the authority — the **fenced CAS is the single-claimant gate** (even if two machines diverge on the election under partition, `acquireIfEligible` admits exactly one; the loser gets `cas-lost` and stands down). `decidePostCasSelfReverify` enforces "CAS-win necessary-not-sufficient."
|
|
22
|
+
|
|
23
|
+
## 5. Interactions
|
|
24
|
+
- vs `tickLease`: a won CAS makes this machine the holder → `tickLease`'s non-holder acquire converges (doesn't fight); a successful claim flips the verdict to `ok` (self-terminating). Debounce + post-CAS reverify damp flap.
|
|
25
|
+
- Reentrancy: `_g2Evaluating` prevents a >30s eval overlapping the 30s cadence (which would inflate the silence streak pre-await). Added per the 2nd-pass review.
|
|
26
|
+
- Cadence: fire-and-forget with `.catch()` — a slow/failed eval never blocks or fails `refreshPool`.
|
|
27
|
+
|
|
28
|
+
## 6. External surfaces
|
|
29
|
+
No new route (the observe `/mesh-selfheal/g2` from the prior PR reads the same ledger). New config path `multiMachine.nobodyPollingRecovery` (dev-gated). When `dryRun:false` it WRITES the lease (poll-ownership) — the reason this is dark-first + 2nd-pass-gated.
|
|
30
|
+
|
|
31
|
+
## 7. Multi-machine posture (Cross-Machine Coherence)
|
|
32
|
+
This IS the cross-machine recovery actuator. Coherence is the deterministic election + the fenced CAS (one winner, pool-wide). The watermarks it reads are machine-local (skew-immune). Single-machine = strict no-op (no leaseCoordinator/peers → `skipped`).
|
|
33
|
+
|
|
34
|
+
## 8. Rollback cost
|
|
35
|
+
Trivial — ships dark+dryRun (strict no-op until a deliberate enable). Revert the commit, or leave the flag off. No migration, no state. Even enabled-in-dryRun performs zero side effects.
|
|
36
|
+
|
|
37
|
+
## Second-Pass Review (REQUIRED — high-risk: lease/poll-ownership authority)
|
|
38
|
+
An independent reviewer audited the actuator + coordinator method + cadence. **Verdict: CONCUR.** Verified: (a) the dryRun invariant HOLDS — the gate sits strictly before every port call; default `dryRun:true` (`?? true`) + dev-gated dark-on-fleet; test pins zero side-effects; (b) single-claimant is genuinely the fenced CAS, not the election guess — a partition divergence still admits exactly one (loser → `cas-lost`); (c) no tug-of-war — the won CAS converges with `tickLease`, debounce + self-reverify damp flap; (d) fail directions correct, hardcoded `globalOutageEvidence:false` is the documented local-safe direction + inert under dryRun. **One concern raised (minor, non-blocking given dark+dryRun): no reentrancy guard** on `evaluateNobodyPolling` — a >30s eval could overlap the 30s tick and inflate `_g2SilenceStreak`. **RESOLVED in this commit** (added `_g2Evaluating` guard, mirroring `leaseTicking`). The reviewer's two enforce-ENABLE prerequisites (the pollSucceeded watermark; the globalOutageEvidence plumbing) are tracked in MESH-SELF-HEAL-G2-BUILD.md as the gate before `dryRun:false`.
|