instar 1.3.627 → 1.3.629
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/subscriptions.js +50 -0
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +9 -0
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +28 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/EnrollmentWizard.d.ts +8 -0
- package/dist/core/EnrollmentWizard.d.ts.map +1 -1
- package/dist/core/EnrollmentWizard.js +12 -0
- package/dist/core/EnrollmentWizard.js.map +1 -1
- package/dist/core/FencedLease.d.ts +23 -4
- package/dist/core/FencedLease.d.ts.map +1 -1
- package/dist/core/FencedLease.js +44 -4
- package/dist/core/FencedLease.js.map +1 -1
- package/dist/core/LeaseCoordinator.d.ts +47 -0
- package/dist/core/LeaseCoordinator.d.ts.map +1 -1
- package/dist/core/LeaseCoordinator.js +89 -7
- package/dist/core/LeaseCoordinator.js.map +1 -1
- package/dist/core/MultiMachineCoordinator.d.ts +73 -0
- package/dist/core/MultiMachineCoordinator.d.ts.map +1 -1
- package/dist/core/MultiMachineCoordinator.js +221 -6
- package/dist/core/MultiMachineCoordinator.js.map +1 -1
- package/dist/core/PendingLoginStore.d.ts.map +1 -1
- package/dist/core/PendingLoginStore.js +21 -2
- package/dist/core/PendingLoginStore.js.map +1 -1
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +18 -1
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +10 -0
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/types.d.ts +66 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/scaffold/templates.d.ts.map +1 -1
- package/dist/scaffold/templates.js +2 -0
- package/dist/scaffold/templates.js.map +1 -1
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +112 -0
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +64 -64
- package/src/scaffold/templates.ts +2 -0
- package/upgrades/1.3.628.md +23 -0
- package/upgrades/1.3.629.md +31 -0
- package/upgrades/side-effects/matrix-cell-operator-cancel.md +151 -0
- package/upgrades/side-effects/multi-machine-lease-self-heal.md +45 -0
|
@@ -41,6 +41,17 @@ export class LeaseCoordinator {
|
|
|
41
41
|
* while it is not superseded by a higher epoch.
|
|
42
42
|
*/
|
|
43
43
|
selfIssued = null;
|
|
44
|
+
/**
|
|
45
|
+
* F2 (staleHolderTakeover) — per-holder freshness on the OBSERVER's own
|
|
46
|
+
* monotonic clock: the time we last saw that holder's signed nonce watermark
|
|
47
|
+
* ADVANCE (a renewing holder bumps its nonce each renew). Stamped only when a
|
|
48
|
+
* VERIFIED observed lease's nonce strictly exceeds the last we recorded for
|
|
49
|
+
* that holder. `lastObservedNonce` is the per-holder high-water nonce that
|
|
50
|
+
* gates the stamp. Single-clock ⇒ clock-skew immune; the holder cannot forge
|
|
51
|
+
* freshness (the nonce is inside its Ed25519 signature).
|
|
52
|
+
*/
|
|
53
|
+
freshObservedMonoMs = new Map();
|
|
54
|
+
lastObservedNonce = new Map();
|
|
44
55
|
constructor(deps) {
|
|
45
56
|
this.d = deps;
|
|
46
57
|
this.fl = deps.lease;
|
|
@@ -103,9 +114,20 @@ export class LeaseCoordinator {
|
|
|
103
114
|
const { [obs.lease.holder]: _self, ...nonceFloor } = obs.lastNonceByHolder;
|
|
104
115
|
void _self;
|
|
105
116
|
const decision = this.fl.acceptTunnelLease(obs.lease, git.epoch, nonceFloor);
|
|
106
|
-
if (decision.accept
|
|
107
|
-
|
|
108
|
-
epoch
|
|
117
|
+
if (decision.accept) {
|
|
118
|
+
// F2 freshness stamp (VERIFIED fold-in): a renewing holder bumps its
|
|
119
|
+
// signed nonce each renew (same OR higher epoch), so its watermark
|
|
120
|
+
// advances; a non-renewing holder's stops. Stamp the OBSERVER's own
|
|
121
|
+
// monotonic time when a peer holder's nonce watermark strictly advances.
|
|
122
|
+
const prevNonce = this.lastObservedNonce.get(obs.lease.holder) ?? -1;
|
|
123
|
+
if (obs.lease.holder !== this.selfMachineId && obs.lease.nonce > prevNonce) {
|
|
124
|
+
this.lastObservedNonce.set(obs.lease.holder, obs.lease.nonce);
|
|
125
|
+
this.freshObservedMonoMs.set(obs.lease.holder, this.monotonicNow());
|
|
126
|
+
}
|
|
127
|
+
if (obs.lease.epoch > epoch) {
|
|
128
|
+
bestLease = obs.lease;
|
|
129
|
+
epoch = obs.lease.epoch;
|
|
130
|
+
}
|
|
109
131
|
}
|
|
110
132
|
}
|
|
111
133
|
}
|
|
@@ -140,7 +162,25 @@ export class LeaseCoordinator {
|
|
|
140
162
|
return this.effectiveView().epoch;
|
|
141
163
|
}
|
|
142
164
|
currentHolder() {
|
|
143
|
-
|
|
165
|
+
const lease = this.effectiveView().lease;
|
|
166
|
+
// F3 — a RELEASED tombstone declares "epoch N released, not held": it names
|
|
167
|
+
// no live holder, so peers stop deferring to a muted ex-holder's zombie. A
|
|
168
|
+
// higher-epoch normal acquisition still strictly dominates (epoch fold).
|
|
169
|
+
if (!lease || lease.released)
|
|
170
|
+
return null;
|
|
171
|
+
return lease.holder;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* F4 — is `machineId` the current holder of a LIVE (non-expired, non-released)
|
|
175
|
+
* lease we observe? The deferential-standby health gate: a non-preferred machine
|
|
176
|
+
* defers to its preferred peer ONLY while this is true, so a frozen/down/released
|
|
177
|
+
* preferred never strands coverage (the non-preferred then acquires normally).
|
|
178
|
+
*/
|
|
179
|
+
isHolderHealthy(machineId) {
|
|
180
|
+
const view = this.effectiveView();
|
|
181
|
+
if (!view.lease || view.lease.released || view.lease.holder !== machineId)
|
|
182
|
+
return false;
|
|
183
|
+
return !this.fl.isExpired(view.lease, this.now());
|
|
144
184
|
}
|
|
145
185
|
/**
|
|
146
186
|
* The current effective-view signed lease (max of tunnel-observed, git-committed,
|
|
@@ -175,6 +215,48 @@ export class LeaseCoordinator {
|
|
|
175
215
|
this.d.store.forceLocalExpiry?.();
|
|
176
216
|
this.log('relinquished self-lease (contested tie-break loser) — winner may now advance to N+1');
|
|
177
217
|
}
|
|
218
|
+
/**
|
|
219
|
+
* F2 — build the StaleHolderTakeoverOpts for `canAcquire`, or `undefined` when
|
|
220
|
+
* the flag is off / the candidate is not a takeable peer (so canAcquire stays
|
|
221
|
+
* byte-for-byte the legacy behavior). Uses this holder's observed freshness on
|
|
222
|
+
* the OBSERVER's own monotonic clock.
|
|
223
|
+
*/
|
|
224
|
+
staleHolderOpts(currentLease) {
|
|
225
|
+
const cfg = this.d.staleHolderTakeover?.();
|
|
226
|
+
if (!cfg?.enabled)
|
|
227
|
+
return undefined;
|
|
228
|
+
if (!currentLease || currentLease.holder === this.selfMachineId)
|
|
229
|
+
return undefined;
|
|
230
|
+
return {
|
|
231
|
+
monotonicNowMs: this.monotonicNow(),
|
|
232
|
+
freshObservedMonoMs: this.freshObservedMonoMs.get(currentLease.holder),
|
|
233
|
+
nonRenewalThresholdMs: this.fl.ttlMs * Math.max(1, cfg.nonRenewalMissedObservations),
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* F3 — relinquish a lease THIS machine holds AND broadcast a SIGNED tombstone
|
|
238
|
+
* so peers stop deferring to a muted ex-holder's zombie. Called level-triggered
|
|
239
|
+
* by the coordinator when this machine is observe-only yet still the named
|
|
240
|
+
* holder (the 2026-06-19 silent-standby zombie). The tombstone is
|
|
241
|
+
* `released:true` at our epoch with a fresh nonce strictly greater than our last
|
|
242
|
+
* (so a concurrent in-flight renewal cannot out-nonce it) and a past expiry; a
|
|
243
|
+
* higher-epoch normal takeover always strictly dominates it. Idempotent: a no-op
|
|
244
|
+
* (just local relinquish) when we are not the named holder.
|
|
245
|
+
*/
|
|
246
|
+
async relinquishAndBroadcast() {
|
|
247
|
+
const view = this.effectiveView();
|
|
248
|
+
if (!view.lease || view.lease.holder !== this.selfMachineId || view.lease.released) {
|
|
249
|
+
// Not the live holder (or already a tombstone) → just clear local state.
|
|
250
|
+
this.relinquish();
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
const tombstone = this.fl.signLease(view.lease.epoch, view.lease.acquiredAt, new Date(this.now()).toISOString(), // past/now expiry — released regardless
|
|
254
|
+
this.nextNonce(), // strictly > our last renewal nonce
|
|
255
|
+
true);
|
|
256
|
+
this.relinquish(); // clear our own self-hold first (forceLocalExpiry preserves the epoch floor)
|
|
257
|
+
await this.broadcast(tombstone);
|
|
258
|
+
this.log(`relinquished + broadcast tombstone for epoch ${view.lease.epoch} (silent-standby release)`);
|
|
259
|
+
}
|
|
178
260
|
/**
|
|
179
261
|
* Force a ONE-TIME epoch advance to resolve a same-epoch contested tie (spec
|
|
180
262
|
* §Problem A — the WINNER side, lower `machineId`). Unlike acquireIfEligible(),
|
|
@@ -284,7 +366,7 @@ export class LeaseCoordinator {
|
|
|
284
366
|
if (view.lease && view.lease.holder === this.selfMachineId && !this.fl.isExpired(view.lease, this.now())) {
|
|
285
367
|
return this.renew();
|
|
286
368
|
}
|
|
287
|
-
const decision = this.fl.canAcquire(view.lease, dead, this.now());
|
|
369
|
+
const decision = this.fl.canAcquire(view.lease, dead, this.now(), this.staleHolderOpts(view.lease));
|
|
288
370
|
if (!decision.can) {
|
|
289
371
|
this.log(`acquire skipped: ${decision.reason}`);
|
|
290
372
|
return false;
|
|
@@ -304,8 +386,8 @@ export class LeaseCoordinator {
|
|
|
304
386
|
if (observedEpoch >= candidate.epoch) {
|
|
305
387
|
this.log(`CAS lost to epoch ${observedEpoch} (our candidate ${candidate.epoch}) — yielding`);
|
|
306
388
|
this.emitEpoch(observedEpoch);
|
|
307
|
-
// If the winner is a presumed-dead/expired holder we'll retry; else stop.
|
|
308
|
-
if (!this.fl.canAcquire(res.observed.lease, dead, this.now()).can)
|
|
389
|
+
// If the winner is a presumed-dead/expired/non-renewing holder we'll retry; else stop.
|
|
390
|
+
if (!this.fl.canAcquire(res.observed.lease, dead, this.now(), this.staleHolderOpts(res.observed.lease)).can)
|
|
309
391
|
return false;
|
|
310
392
|
}
|
|
311
393
|
retries++;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LeaseCoordinator.js","sourceRoot":"","sources":["../../src/core/LeaseCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAyFH,MAAM,OAAO,gBAAgB;IACV,CAAC,CAAuB;IACxB,EAAE,CAAc;IACzB,YAAY,GAAG,CAAC,CAAC;IACzB;;;;;;OAMG;IACK,iBAAiB,GAAG,CAAC,CAAC;IACtB,iBAAiB,GAAG,CAAC,CAAC;IACtB,SAAS,GAAG,KAAK,CAAC;IAC1B;;;;;;OAMG;IACK,UAAU,GAAuB,IAAI,CAAC;IAE9C,YAAY,IAA0B;QACpC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;QACd,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;QACrB,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAEO,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,CAAC;IACD;;;;;OAKG;IACK,YAAY;QAClB,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;QACtD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,QAAU,CAAC,CAAC;IACtD,CAAC;IACD;;;OAGG;IACK,WAAW;QACjB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAC/C,CAAC;IACO,GAAG,CAAC,CAAS;QACnB,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IACO,SAAS;QACf,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC;IAC/B,CAAC;IACD,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,aAAa;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;QAC1B,IAAI,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,wEAAwE;gBACxE,0EAA0E;gBAC1E,0EAA0E;gBAC1E,2EAA2E;gBAC3E,2EAA2E;gBAC3E,yEAAyE;gBACzE,yEAAyE;gBACzE,uEAAuE;gBACvE,2EAA2E;gBAC3E,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,GAAG,CAAC,iBAAiB,CAAC;gBAC3E,KAAK,KAAK,CAAC;gBACX,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;gBAC7E,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC;oBAC/C,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;oBACtB,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QACD,2EAA2E;QAC3E,yEAAyE;QACzE,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;YACvG,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;YAC5B,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChC,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;IAC1D,CAAC;IAED,6EAA6E;IAC7E,UAAU;QACR,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/E,2EAA2E;QAC3E,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,oEAAoE;QACpE,0EAA0E;QAC1E,wDAAwD;QACxD,IAAI,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAC/E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,+DAA+D;IAC/D,YAAY;QACV,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;IACpC,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;IACpD,CAAC;IAED;;;;;;OAMG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU;QACR,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,qFAAqF,CAAC,CAAC;IAClG,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,2BAA2B;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,iFAAiF;QACjF,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,qBAAqB,SAAS,CAAC,KAAK,+CAA+C,CAAC,CAAC;YAC9F,OAAO,IAAI,CAAC;QACd,CAAC;QACD,6EAA6E;QAC7E,qDAAqD;QACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,2CAA2C,GAAG,CAAC,QAAQ,CAAC,KAAK,8BAA8B,CAAC,CAAC;QACtG,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC,aAAa,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,YAAY,KAAK,UAAU,CAAC;IAC3D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,YAAY;YAAE,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;IACtE,CAAC;IAED;;;;;OAKG;IACH,iBAAiB;QACf,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,mBAAmB;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACxB,oEAAoE;QACpE,IAAI,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACtD,+EAA+E;QAC/E,8EAA8E;QAC9E,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,oEAAoE;YACpE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;QAC1C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAClC,qDAAqD;YACrD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;gBACzG,OAAO,IAAI,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAClE,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAClB,IAAI,CAAC,GAAG,CAAC,oBAAoB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBAChD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;gBAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAChC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,2BAA2B,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,uEAAuE;YACvE,MAAM,aAAa,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;YACzC,IAAI,aAAa,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,qBAAqB,aAAa,mBAAmB,SAAS,CAAC,KAAK,cAAc,CAAC,CAAC;gBAC7F,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBAC9B,0EAA0E;gBAC1E,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG;oBAAE,OAAO,KAAK,CAAC;YAClF,CAAC;YACD,OAAO,EAAE,CAAC;YACV,IAAI,IAAI,CAAC,EAAE,CAAC,4BAA4B,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;gBACpF,IAAI,CAAC,GAAG,CAAC,0BAA0B,OAAO,2BAA2B,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,CAAC;gBAC5F,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,gBAAgB,CAAC,kBAA0B;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;QAC1C,IAAI,MAAM,IAAI,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,CAAC,+BAA+B;QAC9C,CAAC;QACD,IAAI,MAAM,IAAI,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,uCAAuC,kBAAkB,0BAA0B,MAAM,EAAE,CAAC,CAAC;YACtG,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,sCAAsC,SAAS,CAAC,KAAK,gBAAgB,kBAAkB,GAAG,CAAC,CAAC;YACrG,OAAO,IAAI,CAAC;QACd,CAAC;QACD,gFAAgF;QAChF,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,qCAAqC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QACpE,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa;YAAE,OAAO,KAAK,CAAC;QAE1E,wEAAwE;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,CAC/B,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAClD,IAAI,CAAC,SAAS,EAAE,CACjB,CAAC;QAEF,4EAA4E;QAC5E,2EAA2E;QAC3E,wEAAwE;QACxE,uEAAuE;QACvE,oEAAoE;QACpE,IAAI,SAAkB,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClB,SAAS,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QACxE,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC;YAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YACjE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,CACpB,gCAAgC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,sBAAsB,IAAI,CAAC,EAAE,CAAC,KAAK,+BAA+B,CACnI,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,yEAAyE;QACzE,sEAAsE;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAkB;QACxC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO;QAC3B,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChD,IAAI,EAAE;gBAAE,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,qBAAqB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,KAAa;QAC7B,IAAI,KAAK,KAAK,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACrC,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;YAC/B,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,yBAAyB,CAAC,kBAA0B;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3E,IAAI,UAAU,IAAI,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5E,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;gBAClB,kBAAkB;gBAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACzB,MAAM,EAAE,gFAAgF;aACzF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"LeaseCoordinator.js","sourceRoot":"","sources":["../../src/core/LeaseCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAiGH,MAAM,OAAO,gBAAgB;IACV,CAAC,CAAuB;IACxB,EAAE,CAAc;IACzB,YAAY,GAAG,CAAC,CAAC;IACzB;;;;;;OAMG;IACK,iBAAiB,GAAG,CAAC,CAAC;IACtB,iBAAiB,GAAG,CAAC,CAAC;IACtB,SAAS,GAAG,KAAK,CAAC;IAC1B;;;;;;OAMG;IACK,UAAU,GAAuB,IAAI,CAAC;IAC9C;;;;;;;;OAQG;IACK,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEtD,YAAY,IAA0B;QACpC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;QACd,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;QACrB,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAEO,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,CAAC;IACD;;;;;OAKG;IACK,YAAY;QAClB,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;QACtD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,QAAU,CAAC,CAAC;IACtD,CAAC;IACD;;;OAGG;IACK,WAAW;QACjB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAC/C,CAAC;IACO,GAAG,CAAC,CAAS;QACnB,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IACO,SAAS;QACf,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC;IAC/B,CAAC;IACD,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,aAAa;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;QAC1B,IAAI,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,wEAAwE;gBACxE,0EAA0E;gBAC1E,0EAA0E;gBAC1E,2EAA2E;gBAC3E,2EAA2E;gBAC3E,yEAAyE;gBACzE,yEAAyE;gBACzE,uEAAuE;gBACvE,2EAA2E;gBAC3E,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,GAAG,CAAC,iBAAiB,CAAC;gBAC3E,KAAK,KAAK,CAAC;gBACX,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;gBAC7E,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACpB,qEAAqE;oBACrE,mEAAmE;oBACnE,oEAAoE;oBACpE,yEAAyE;oBACzE,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;oBACrE,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;wBAC3E,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAC9D,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;oBACtE,CAAC;oBACD,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC;wBAC5B,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;wBACtB,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC;oBAC1B,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,2EAA2E;QAC3E,yEAAyE;QACzE,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;YACvG,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;YAC5B,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChC,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;IAC1D,CAAC;IAED,6EAA6E;IAC7E,UAAU;QACR,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/E,2EAA2E;QAC3E,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,oEAAoE;QACpE,0EAA0E;QAC1E,wDAAwD;QACxD,IAAI,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAC/E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,+DAA+D;IAC/D,YAAY;QACV,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;IACpC,CAAC;IAED,aAAa;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;QACzC,4EAA4E;QAC5E,2EAA2E;QAC3E,yEAAyE;QACzE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC1C,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,SAAiB;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACxF,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpD,CAAC;IAED;;;;;;OAMG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU;QACR,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,qFAAqF,CAAC,CAAC;IAClG,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,YAA4C;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,GAAG,EAAE,OAAO;YAAE,OAAO,SAAS,CAAC;QACpC,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa;YAAE,OAAO,SAAS,CAAC;QAClF,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,YAAY,EAAE;YACnC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC;YACtE,qBAAqB,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,4BAA4B,CAAC;SACrF,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,sBAAsB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnF,yEAAyE;YACzE,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,CACjC,IAAI,CAAC,KAAK,CAAC,KAAK,EAChB,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,wCAAwC;QAC5E,IAAI,CAAC,SAAS,EAAE,EAAoB,oCAAoC;QACxE,IAAI,CACL,CAAC;QACF,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,6EAA6E;QAChG,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,gDAAgD,IAAI,CAAC,KAAK,CAAC,KAAK,2BAA2B,CAAC,CAAC;IACxG,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,2BAA2B;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,iFAAiF;QACjF,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,qBAAqB,SAAS,CAAC,KAAK,+CAA+C,CAAC,CAAC;YAC9F,OAAO,IAAI,CAAC;QACd,CAAC;QACD,6EAA6E;QAC7E,qDAAqD;QACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,2CAA2C,GAAG,CAAC,QAAQ,CAAC,KAAK,8BAA8B,CAAC,CAAC;QACtG,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC,aAAa,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,YAAY,KAAK,UAAU,CAAC;IAC3D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,YAAY;YAAE,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;IACtE,CAAC;IAED;;;;;OAKG;IACH,iBAAiB;QACf,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,mBAAmB;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACxB,oEAAoE;QACpE,IAAI,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACtD,+EAA+E;QAC/E,8EAA8E;QAC9E,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,oEAAoE;YACpE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;QAC1C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAClC,qDAAqD;YACrD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;gBACzG,OAAO,IAAI,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACpG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAClB,IAAI,CAAC,GAAG,CAAC,oBAAoB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBAChD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;gBAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAChC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,2BAA2B,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,uEAAuE;YACvE,MAAM,aAAa,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;YACzC,IAAI,aAAa,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,qBAAqB,aAAa,mBAAmB,SAAS,CAAC,KAAK,cAAc,CAAC,CAAC;gBAC7F,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBAC9B,uFAAuF;gBACvF,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG;oBAAE,OAAO,KAAK,CAAC;YAC5H,CAAC;YACD,OAAO,EAAE,CAAC;YACV,IAAI,IAAI,CAAC,EAAE,CAAC,4BAA4B,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;gBACpF,IAAI,CAAC,GAAG,CAAC,0BAA0B,OAAO,2BAA2B,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,CAAC;gBAC5F,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,gBAAgB,CAAC,kBAA0B;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;QAC1C,IAAI,MAAM,IAAI,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,CAAC,+BAA+B;QAC9C,CAAC;QACD,IAAI,MAAM,IAAI,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,uCAAuC,kBAAkB,0BAA0B,MAAM,EAAE,CAAC,CAAC;YACtG,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,sCAAsC,SAAS,CAAC,KAAK,gBAAgB,kBAAkB,GAAG,CAAC,CAAC;YACrG,OAAO,IAAI,CAAC;QACd,CAAC;QACD,gFAAgF;QAChF,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,qCAAqC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QACpE,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa;YAAE,OAAO,KAAK,CAAC;QAE1E,wEAAwE;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,CAC/B,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAClD,IAAI,CAAC,SAAS,EAAE,CACjB,CAAC;QAEF,4EAA4E;QAC5E,2EAA2E;QAC3E,wEAAwE;QACxE,uEAAuE;QACvE,oEAAoE;QACpE,IAAI,SAAkB,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClB,SAAS,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QACxE,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC;YAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YACjE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,CACpB,gCAAgC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,sBAAsB,IAAI,CAAC,EAAE,CAAC,KAAK,+BAA+B,CACnI,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,yEAAyE;QACzE,sEAAsE;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAkB;QACxC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO;QAC3B,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChD,IAAI,EAAE;gBAAE,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,qBAAqB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,KAAa;QAC7B,IAAI,KAAK,KAAK,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACrC,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;YAC/B,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,yBAAyB,CAAC,kBAA0B;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3E,IAAI,UAAU,IAAI,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5E,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;gBAClB,kBAAkB;gBAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACzB,MAAM,EAAE,gFAAgF;aACzF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
|
|
@@ -32,6 +32,18 @@ export interface MultiMachineSyncStatus {
|
|
|
32
32
|
splitBrainState: 'clear' | 'contested' | 'self-suspended';
|
|
33
33
|
protocolVersion: number;
|
|
34
34
|
awakeMachineCount: number;
|
|
35
|
+
/**
|
|
36
|
+
* multi-machine-lease-self-heal observability (Agent Awareness). F1 tick-watchdog
|
|
37
|
+
* health — answers "did the watchdog fire?" / "is it disarmed?". `lastTickAgeMs`
|
|
38
|
+
* is the monotonic age of the last main-tick run (a large value = the tick has
|
|
39
|
+
* stalled). `preferredAwakeMachineId` echoes the F4 config (null = off).
|
|
40
|
+
*/
|
|
41
|
+
leaseTickWatchdog?: {
|
|
42
|
+
lastTickAgeMs: number;
|
|
43
|
+
reArmCount: number;
|
|
44
|
+
disarmed: boolean;
|
|
45
|
+
};
|
|
46
|
+
preferredAwakeMachineId?: string | null;
|
|
35
47
|
}
|
|
36
48
|
export interface CoordinatorConfig {
|
|
37
49
|
/** State directory (.instar) */
|
|
@@ -110,6 +122,23 @@ export declare class MultiMachineCoordinator extends EventEmitter {
|
|
|
110
122
|
* K-cycle escalation. Cleared on the falling edge (contested resolved).
|
|
111
123
|
*/
|
|
112
124
|
private contestedEpisode;
|
|
125
|
+
/**
|
|
126
|
+
* multi-machine-lease-self-heal F1 — the tick self-heal watchdog state.
|
|
127
|
+
* `lastTickRunMonoMs` is stamped on the OBSERVER's own monotonic clock at the
|
|
128
|
+
* TOP of checkHeartbeatAndAct (before any early-return, so a solo agent stamps
|
|
129
|
+
* a healthy advancing value and the watchdog stays a no-op). `*StartMonoMs`
|
|
130
|
+
* stamp when a reentrancy guard is taken, so the watchdog can distinguish a
|
|
131
|
+
* legitimately-slow in-flight tick (leave alone) from a stuck guard (reset).
|
|
132
|
+
* `watchdogReArmTimes` is a rolling window of re-arm timestamps for self-disarm.
|
|
133
|
+
*/
|
|
134
|
+
private tickWatchdogTimer;
|
|
135
|
+
private lastTickRunMonoMs;
|
|
136
|
+
private leaseTickStartMonoMs;
|
|
137
|
+
private leasePullStartMonoMs;
|
|
138
|
+
private watchdogReArmTimes;
|
|
139
|
+
private watchdogDisarmed;
|
|
140
|
+
/** F3 — per-incarnation latch so a silent standby relinquishes its held lease once. */
|
|
141
|
+
private silentStandbyRelinquished;
|
|
113
142
|
constructor(state: StateManager, config: CoordinatorConfig);
|
|
114
143
|
/** Whether multi-machine is enabled (has identity). */
|
|
115
144
|
get enabled(): boolean;
|
|
@@ -133,6 +162,28 @@ export declare class MultiMachineCoordinator extends EventEmitter {
|
|
|
133
162
|
* such a machine is a deliberate un-mute (telegramPolling → true), not auto.
|
|
134
163
|
*/
|
|
135
164
|
get isLeaseObserveOnly(): boolean;
|
|
165
|
+
/**
|
|
166
|
+
* multi-machine-lease-self-heal M3 — the first-class lease-participation mode,
|
|
167
|
+
* decoupled from the overloaded `telegramPolling` flag. Explicit
|
|
168
|
+
* `leaseSelfHeal.leaseRole` wins; otherwise derive from `telegramPolling`
|
|
169
|
+
* (===false ⇒ 'observe-only', else 'active') for back-compat. ('deferential'
|
|
170
|
+
* is an F4 concept handled separately; only 'observe-only' gates acquisition
|
|
171
|
+
* here so F4's down-preferred failover is never accidentally suppressed.)
|
|
172
|
+
*/
|
|
173
|
+
private resolvedLeaseRole;
|
|
174
|
+
/**
|
|
175
|
+
* F4 (preferred-awake, opt-in; null = off) — should THIS machine defer (not
|
|
176
|
+
* contend for the lease) right now? True iff a `preferredAwakeMachineId` is
|
|
177
|
+
* configured, it names a DIFFERENT machine (we are NOT the preferred), and that
|
|
178
|
+
* preferred machine is currently a HEALTHY holder. A machine that IS the
|
|
179
|
+
* preferred never defers; a non-preferred machine defers only while the preferred
|
|
180
|
+
* is healthy, so the preferred going down never strands coverage. Safe under any
|
|
181
|
+
* config (no tie-break override, so a divergent config degrades to the existing
|
|
182
|
+
* lower-machineId baseline rather than flapping).
|
|
183
|
+
*/
|
|
184
|
+
private shouldDeferToPreferred;
|
|
185
|
+
/** F4 — the single shared health predicate: is the preferred machine a live holder? */
|
|
186
|
+
private preferredIsHealthy;
|
|
136
187
|
/** The coordination mode (default: 'primary-standby'). */
|
|
137
188
|
get coordinationMode(): CoordinationMode;
|
|
138
189
|
/** The underlying managers (for route wiring). */
|
|
@@ -291,6 +342,28 @@ export declare class MultiMachineCoordinator extends EventEmitter {
|
|
|
291
342
|
* BEFORE the peer's ~15min heartbeat-expiry failover horizon.
|
|
292
343
|
*/
|
|
293
344
|
private writeHeartbeatGuarded;
|
|
345
|
+
/** Monotonic milliseconds (NTP-step / sleep-resume immune), like LeaseCoordinator §L−1. */
|
|
346
|
+
private monoNowMs;
|
|
347
|
+
/** Resolved F1 watchdog config (defaults baked in; read live so disable needs no restart). */
|
|
348
|
+
private get tickWatchdogCfg();
|
|
349
|
+
/**
|
|
350
|
+
* F1a — bound a tick-path network await so a never-settling call (the proven
|
|
351
|
+
* 2026-06-19 freeze: a hung fetch left `leaseTicking` stuck true forever) can
|
|
352
|
+
* NEVER hang the tick. ALL tick-path awaits route through this one helper, so
|
|
353
|
+
* the "no unbounded tick await" invariant is structural + grep-auditable.
|
|
354
|
+
*/
|
|
355
|
+
private withTickTimeout;
|
|
356
|
+
/**
|
|
357
|
+
* F1b — the monotonic-clocked tick watchdog. Detects a stalled main tick loop
|
|
358
|
+
* (lost timer / stuck reentrancy guard) and re-arms it. SAFE-BY-CONSTRUCTION:
|
|
359
|
+
* never crashes (try/catch), never touches authority/epoch/suspend state, only
|
|
360
|
+
* resets a reentrancy guard whose in-flight tick is ALSO older than the ceiling
|
|
361
|
+
* (so a legitimately-slow live tick is never preempted), reads `enabled` live,
|
|
362
|
+
* and self-disarms (one DegradationReporter signal) if it re-arms too often.
|
|
363
|
+
* A true event-loop stall freezes this timer too — that case is layer-2 (the
|
|
364
|
+
* out-of-process fleet/launchd watchdog), as documented in the spec.
|
|
365
|
+
*/
|
|
366
|
+
private runTickWatchdog;
|
|
294
367
|
/**
|
|
295
368
|
* Start periodic heartbeat monitoring (all machines).
|
|
296
369
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MultiMachineCoordinator.d.ts","sourceRoot":"","sources":["../../src/core/MultiMachineCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAI9D,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAErG,0EAA0E;AAC1E,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,8FAA8F;IAC9F,eAAe,EAAE,OAAO,GAAG,WAAW,GAAG,gBAAgB,CAAC;IAC1D,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"MultiMachineCoordinator.d.ts","sourceRoot":"","sources":["../../src/core/MultiMachineCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAI9D,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAErG,0EAA0E;AAC1E,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,8FAA8F;IAC9F,eAAe,EAAE,OAAO,GAAG,WAAW,GAAG,gBAAgB,CAAC;IAC1D,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC;IACrF,uBAAuB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AAaD,MAAM,WAAW,iBAAiB;IAChC,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,YAAY,CAAC,EAAE,kBAAkB,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,wDAAwD;IACxD,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,yDAAyD;IACzD,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,0CAA0C;IAC1C,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,6BAA6B;IAC7B,UAAU,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;CAC1D;AAID,qBAAa,uBAAwB,SAAQ,YAAY;IACvD,OAAO,CAAC,eAAe,CAAyB;IAChD,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,KAAK,CAA0B;IACvC,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA+C;IAC1E;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0D;IACzF,OAAO,CAAC,mBAAmB,CAA+C;IAC1E;qEACiE;IACjE,OAAO,CAAC,6BAA6B,CAAkB;IACvD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAiC;IACzD,OAAO,CAAC,YAAY,CAAkB;IACtC;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,gBAAgB,CAAkB;IAC1C;;;;;;;;;;OAUG;IACH,OAAO,CAAC,gBAAgB,CAAuF;IAE/G;;;;;;;;OAQG;IACH,OAAO,CAAC,iBAAiB,CAA+C;IACxE,OAAO,CAAC,iBAAiB,CAAa;IACtC,OAAO,CAAC,oBAAoB,CAAa;IACzC,OAAO,CAAC,oBAAoB,CAAa;IACzC,OAAO,CAAC,kBAAkB,CAAgB;IAC1C,OAAO,CAAC,gBAAgB,CAAkB;IAC1C,uFAAuF;IACvF,OAAO,CAAC,yBAAyB,CAAkB;gBAEvC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,iBAAiB;IAc1D,uDAAuD;IACvD,IAAI,OAAO,IAAI,OAAO,CAA0B;IAEhD,mCAAmC;IACnC,IAAI,IAAI,IAAI,WAAW,CAAuB;IAE9C,yDAAyD;IACzD,IAAI,QAAQ,IAAI,eAAe,GAAG,IAAI,CAA2B;IAEjE,2DAA2D;IAC3D,IAAI,OAAO,IAAI,OAAO,CAAmC;IAEzD;;;;;;;;;;;;OAYG;IACH,IAAI,kBAAkB,IAAI,OAAO,CAEhC;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;IAMzB;;;;;;;;;OASG;IACH,OAAO,CAAC,sBAAsB;IAO9B,uFAAuF;IACvF,OAAO,CAAC,kBAAkB;IAI1B,0DAA0D;IAC1D,IAAI,gBAAgB,IAAI,gBAAgB,CAEvC;IAED,kDAAkD;IAClD,IAAI,QAAQ;;;;;MAOX;IAID;;;OAGG;IACH,KAAK,IAAI,WAAW;IAuGpB;;OAEG;IACH;;;;OAIG;IACH,OAAO,CAAC,sCAAsC;IAiB9C,IAAI,IAAI,IAAI;IAuBZ;;;OAGG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IA0CpC;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAgCrC;;;OAGG;IACH,oBAAoB,IAAI,OAAO;IAoB/B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,EAAE,gBAAgB,GAAG,IAAI;IAIlD,kFAAkF;IAClF,UAAU,IAAI,OAAO;IAIrB;;;;;;;;;OASG;IACH,QAAQ,IAAI,OAAO;IAInB;;;;OAIG;IACH,aAAa,IAAI,MAAM;IAIvB;;;;;OAKG;IACH,aAAa,IAAI,sBAAsB;IAqCvC;;;OAGG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAmBtC;;;;;;OAMG;IACG,qBAAqB,CAAC,kBAAkB,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAOzE;;;OAGG;YACW,SAAS;IA+CvB,wEAAwE;IACxE,OAAO,CAAC,oBAAoB;IA6B5B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAqB1B;;;;;;OAMG;YACW,aAAa;IA6C3B;;;;;;;;OAQG;IACH,OAAO,CAAC,+BAA+B;IAoBvC;;;;;;;;;;;;;;;;;;;;;OAqBG;YACW,0BAA0B;IAgExC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IA4B5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,qBAAqB;IA4B7B,2FAA2F;IAC3F,OAAO,CAAC,SAAS;IAIjB,8FAA8F;IAC9F,OAAO,KAAK,eAAe,GAS1B;IAED;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAiBvB;;;;;;;;;OASG;IACH,OAAO,CAAC,eAAe;IAgDvB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAyB7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CA+B7B"}
|
|
@@ -25,6 +25,7 @@ import { DegradationReporter } from '../monitoring/DegradationReporter.js';
|
|
|
25
25
|
// ── Constants ────────────────────────────────────────────────────────
|
|
26
26
|
const HEARTBEAT_WRITE_INTERVAL_MS = 2 * 60_000; // Write heartbeat every 2 min
|
|
27
27
|
const HEARTBEAT_CHECK_INTERVAL_MS = 2 * 60_000; // Check heartbeat every 2 min
|
|
28
|
+
const TICK_WATCHDOG_INTERVAL_MS = 60_000; // F1b — independent tick-stall watchdog cadence
|
|
28
29
|
const DEFAULT_FAILOVER_TIMEOUT_MS = 15 * 60_000; // 15 min before failover
|
|
29
30
|
/** Cross-Machine Coherence — default active lease-PULL cadence over the tunnel. */
|
|
30
31
|
const DEFAULT_LEASE_PULL_INTERVAL_MS = 5_000;
|
|
@@ -90,6 +91,23 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
90
91
|
* K-cycle escalation. Cleared on the falling edge (contested resolved).
|
|
91
92
|
*/
|
|
92
93
|
contestedEpisode = null;
|
|
94
|
+
/**
|
|
95
|
+
* multi-machine-lease-self-heal F1 — the tick self-heal watchdog state.
|
|
96
|
+
* `lastTickRunMonoMs` is stamped on the OBSERVER's own monotonic clock at the
|
|
97
|
+
* TOP of checkHeartbeatAndAct (before any early-return, so a solo agent stamps
|
|
98
|
+
* a healthy advancing value and the watchdog stays a no-op). `*StartMonoMs`
|
|
99
|
+
* stamp when a reentrancy guard is taken, so the watchdog can distinguish a
|
|
100
|
+
* legitimately-slow in-flight tick (leave alone) from a stuck guard (reset).
|
|
101
|
+
* `watchdogReArmTimes` is a rolling window of re-arm timestamps for self-disarm.
|
|
102
|
+
*/
|
|
103
|
+
tickWatchdogTimer = null;
|
|
104
|
+
lastTickRunMonoMs = 0;
|
|
105
|
+
leaseTickStartMonoMs = 0;
|
|
106
|
+
leasePullStartMonoMs = 0;
|
|
107
|
+
watchdogReArmTimes = [];
|
|
108
|
+
watchdogDisarmed = false;
|
|
109
|
+
/** F3 — per-incarnation latch so a silent standby relinquishes its held lease once. */
|
|
110
|
+
silentStandbyRelinquished = false;
|
|
93
111
|
constructor(state, config) {
|
|
94
112
|
super();
|
|
95
113
|
this.state = state;
|
|
@@ -123,7 +141,43 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
123
141
|
* such a machine is a deliberate un-mute (telegramPolling → true), not auto.
|
|
124
142
|
*/
|
|
125
143
|
get isLeaseObserveOnly() {
|
|
126
|
-
return this.
|
|
144
|
+
return this.resolvedLeaseRole() === 'observe-only';
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* multi-machine-lease-self-heal M3 — the first-class lease-participation mode,
|
|
148
|
+
* decoupled from the overloaded `telegramPolling` flag. Explicit
|
|
149
|
+
* `leaseSelfHeal.leaseRole` wins; otherwise derive from `telegramPolling`
|
|
150
|
+
* (===false ⇒ 'observe-only', else 'active') for back-compat. ('deferential'
|
|
151
|
+
* is an F4 concept handled separately; only 'observe-only' gates acquisition
|
|
152
|
+
* here so F4's down-preferred failover is never accidentally suppressed.)
|
|
153
|
+
*/
|
|
154
|
+
resolvedLeaseRole() {
|
|
155
|
+
const explicit = this.config.multiMachine?.leaseSelfHeal?.leaseRole;
|
|
156
|
+
if (explicit === 'active' || explicit === 'observe-only' || explicit === 'deferential')
|
|
157
|
+
return explicit;
|
|
158
|
+
return this.config.multiMachine?.telegramPolling === false ? 'observe-only' : 'active';
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* F4 (preferred-awake, opt-in; null = off) — should THIS machine defer (not
|
|
162
|
+
* contend for the lease) right now? True iff a `preferredAwakeMachineId` is
|
|
163
|
+
* configured, it names a DIFFERENT machine (we are NOT the preferred), and that
|
|
164
|
+
* preferred machine is currently a HEALTHY holder. A machine that IS the
|
|
165
|
+
* preferred never defers; a non-preferred machine defers only while the preferred
|
|
166
|
+
* is healthy, so the preferred going down never strands coverage. Safe under any
|
|
167
|
+
* config (no tie-break override, so a divergent config degrades to the existing
|
|
168
|
+
* lower-machineId baseline rather than flapping).
|
|
169
|
+
*/
|
|
170
|
+
shouldDeferToPreferred() {
|
|
171
|
+
const pref = this.config.multiMachine?.leaseSelfHeal?.preferredAwakeMachineId;
|
|
172
|
+
if (!pref || !this._identity || !this.leaseCoordinator)
|
|
173
|
+
return false; // F4 off / no identity
|
|
174
|
+
if (pref === this._identity.machineId)
|
|
175
|
+
return false; // WE are preferred → never defer
|
|
176
|
+
return this.preferredIsHealthy(pref);
|
|
177
|
+
}
|
|
178
|
+
/** F4 — the single shared health predicate: is the preferred machine a live holder? */
|
|
179
|
+
preferredIsHealthy(machineId) {
|
|
180
|
+
return this.leaseCoordinator?.isHolderHealthy(machineId) ?? false;
|
|
127
181
|
}
|
|
128
182
|
/** The coordination mode (default: 'primary-standby'). */
|
|
129
183
|
get coordinationMode() {
|
|
@@ -264,6 +318,10 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
264
318
|
clearInterval(this.heartbeatCheckTimer);
|
|
265
319
|
this.heartbeatCheckTimer = null;
|
|
266
320
|
}
|
|
321
|
+
if (this.tickWatchdogTimer) {
|
|
322
|
+
clearInterval(this.tickWatchdogTimer);
|
|
323
|
+
this.tickWatchdogTimer = null;
|
|
324
|
+
}
|
|
267
325
|
this.leasePullStopped = true;
|
|
268
326
|
if (this.leasePullTimer) {
|
|
269
327
|
clearTimeout(this.leasePullTimer);
|
|
@@ -432,6 +490,14 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
432
490
|
splitBrainState,
|
|
433
491
|
protocolVersion: SEAMLESSNESS_PROTOCOL_VERSION,
|
|
434
492
|
awakeMachineCount,
|
|
493
|
+
leaseTickWatchdog: this.leaseCoordinator
|
|
494
|
+
? {
|
|
495
|
+
lastTickAgeMs: this.lastTickRunMonoMs > 0 ? this.monoNowMs() - this.lastTickRunMonoMs : -1,
|
|
496
|
+
reArmCount: this.watchdogReArmTimes.length,
|
|
497
|
+
disarmed: this.watchdogDisarmed,
|
|
498
|
+
}
|
|
499
|
+
: undefined,
|
|
500
|
+
preferredAwakeMachineId: this.config.multiMachine?.leaseSelfHeal?.preferredAwakeMachineId ?? null,
|
|
435
501
|
};
|
|
436
502
|
}
|
|
437
503
|
/**
|
|
@@ -445,6 +511,10 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
445
511
|
// Silent standby: do NOT acquire — only observe the primary's lease.
|
|
446
512
|
this.reconcileRoleToLease('lease-init-observe-only');
|
|
447
513
|
}
|
|
514
|
+
else if (this.shouldDeferToPreferred()) {
|
|
515
|
+
// F4 — boot as a deferential standby to a healthy preferred peer (no contend).
|
|
516
|
+
this.reconcileRoleToLease('lease-init-defer-preferred');
|
|
517
|
+
}
|
|
448
518
|
else {
|
|
449
519
|
await this.leaseCoordinator.acquireIfEligible();
|
|
450
520
|
this.reconcileRoleToLease('lease-init');
|
|
@@ -477,26 +547,50 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
477
547
|
if (!this.leaseCoordinator || this.leaseTicking)
|
|
478
548
|
return;
|
|
479
549
|
this.leaseTicking = true;
|
|
550
|
+
this.leaseTickStartMonoMs = this.monoNowMs(); // F1b — for the watchdog's ceiling-gated guard reset
|
|
480
551
|
try {
|
|
481
552
|
if (this.isLeaseObserveOnly) {
|
|
553
|
+
// F3 (silentStandbyRelinquish, DARK) — LEVEL-TRIGGERED: a silent standby
|
|
554
|
+
// that is STILL the named holder (the 2026-06-19 zombie: muted while
|
|
555
|
+
// holding epoch N) relinquishes + broadcasts a signed tombstone ONCE per
|
|
556
|
+
// incarnation, so peers stop deferring to it. Config flips need a restart,
|
|
557
|
+
// and the zombie is a persisted prior-process record, so this fires on the
|
|
558
|
+
// observe-only tick — not on an (absent) transition event.
|
|
559
|
+
if (this.config.multiMachine?.leaseSelfHeal?.silentStandbyRelinquish?.enabled &&
|
|
560
|
+
!this.silentStandbyRelinquished &&
|
|
561
|
+
this._identity &&
|
|
562
|
+
this.leaseCoordinator.currentHolder() === this._identity.machineId) {
|
|
563
|
+
this.silentStandbyRelinquished = true;
|
|
564
|
+
await this.withTickTimeout('relinquishAndBroadcast', () => this.leaseCoordinator.relinquishAndBroadcast());
|
|
565
|
+
}
|
|
482
566
|
// Silent standby: never acquire/renew — just reconcile role to the
|
|
483
567
|
// observed holder (effectiveView folds the primary's broadcast lease).
|
|
484
568
|
this.reconcileRoleToLease('lease-tick-observe-only');
|
|
485
569
|
}
|
|
486
570
|
else if (this.leaseCoordinator.holdsLease()) {
|
|
487
|
-
await
|
|
571
|
+
// F1a — bounded await: a hung renew can never wedge the tick.
|
|
572
|
+
await this.withTickTimeout('renew', () => this.leaseCoordinator.renew());
|
|
488
573
|
this.reconcileRoleToLease('lease-tick');
|
|
489
574
|
}
|
|
575
|
+
else if (this.shouldDeferToPreferred()) {
|
|
576
|
+
// F4 (preferred-awake, opt-in) — we are NON-preferred and observe the
|
|
577
|
+
// preferred machine holding a HEALTHY lease: defer, do not contend. If the
|
|
578
|
+
// preferred goes down/unhealthy, shouldDeferToPreferred() flips false and we
|
|
579
|
+
// acquire normally next tick (no coverage stranding, no flap — a deferential
|
|
580
|
+
// machine creates no contention to leapfrog, so no agreement-gossip needed).
|
|
581
|
+
this.reconcileRoleToLease('lease-tick-defer-preferred');
|
|
582
|
+
}
|
|
490
583
|
else {
|
|
491
|
-
await this.leaseCoordinator.acquireIfEligible();
|
|
584
|
+
await this.withTickTimeout('acquireIfEligible', () => this.leaseCoordinator.acquireIfEligible());
|
|
492
585
|
this.reconcileRoleToLease('lease-tick');
|
|
493
586
|
}
|
|
494
587
|
}
|
|
495
588
|
catch {
|
|
496
|
-
// @silent-fallback-ok — a tick failure is retried next interval
|
|
589
|
+
// @silent-fallback-ok — a tick failure (incl. a bounded-await timeout) is retried next interval
|
|
497
590
|
}
|
|
498
591
|
finally {
|
|
499
592
|
this.leaseTicking = false;
|
|
593
|
+
this.leaseTickStartMonoMs = 0;
|
|
500
594
|
}
|
|
501
595
|
}
|
|
502
596
|
/** Bring _role + read-only into line with whether we hold the lease. */
|
|
@@ -577,8 +671,10 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
577
671
|
return;
|
|
578
672
|
}
|
|
579
673
|
this.leasePulling = true;
|
|
674
|
+
this.leasePullStartMonoMs = this.monoNowMs(); // F1b — watchdog ceiling-gated guard reset
|
|
580
675
|
try {
|
|
581
|
-
await
|
|
676
|
+
// F1a — bounded await: a hung peer pull can never wedge the pull loop.
|
|
677
|
+
await this.withTickTimeout('pullFromPeers', () => this.leaseCoordinator.pullFromPeers());
|
|
582
678
|
// Only reconcile role / surface split-brain when a peer lease was actually
|
|
583
679
|
// OBSERVED. A solo machine (no peers, or no peer lease seen this boot) must
|
|
584
680
|
// NEVER be demoted by the pull loop on a transient self-lease lapse — that
|
|
@@ -610,10 +706,11 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
610
706
|
}
|
|
611
707
|
}
|
|
612
708
|
catch {
|
|
613
|
-
// @silent-fallback-ok — a pull failure is retried next tick
|
|
709
|
+
// @silent-fallback-ok — a pull failure (incl. a bounded-await timeout) is retried next tick
|
|
614
710
|
}
|
|
615
711
|
finally {
|
|
616
712
|
this.leasePulling = false;
|
|
713
|
+
this.leasePullStartMonoMs = 0;
|
|
617
714
|
arm();
|
|
618
715
|
}
|
|
619
716
|
}
|
|
@@ -790,6 +887,109 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
790
887
|
}
|
|
791
888
|
}
|
|
792
889
|
}
|
|
890
|
+
// ── multi-machine-lease-self-heal F1 — tick self-heal ─────────────
|
|
891
|
+
/** Monotonic milliseconds (NTP-step / sleep-resume immune), like LeaseCoordinator §L−1. */
|
|
892
|
+
monoNowMs() {
|
|
893
|
+
return Number(process.hrtime.bigint() / 1000000n);
|
|
894
|
+
}
|
|
895
|
+
/** Resolved F1 watchdog config (defaults baked in; read live so disable needs no restart). */
|
|
896
|
+
get tickWatchdogCfg() {
|
|
897
|
+
const w = this.config.multiMachine?.leaseSelfHeal?.tickWatchdog;
|
|
898
|
+
const staleFactor = Math.max(2, w?.staleFactorMissedTicks ?? 5);
|
|
899
|
+
return {
|
|
900
|
+
enabled: w?.enabled ?? true,
|
|
901
|
+
staleMs: HEARTBEAT_CHECK_INTERVAL_MS * staleFactor,
|
|
902
|
+
awaitTimeoutMs: Math.max(1000, w?.awaitTimeoutMs ?? 20_000),
|
|
903
|
+
maxReArmsPerHour: Math.max(2, w?.maxReArmsPerHour ?? 6),
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
/**
|
|
907
|
+
* F1a — bound a tick-path network await so a never-settling call (the proven
|
|
908
|
+
* 2026-06-19 freeze: a hung fetch left `leaseTicking` stuck true forever) can
|
|
909
|
+
* NEVER hang the tick. ALL tick-path awaits route through this one helper, so
|
|
910
|
+
* the "no unbounded tick await" invariant is structural + grep-auditable.
|
|
911
|
+
*/
|
|
912
|
+
withTickTimeout(label, fn) {
|
|
913
|
+
const ms = this.tickWatchdogCfg.awaitTimeoutMs;
|
|
914
|
+
return new Promise((resolve, reject) => {
|
|
915
|
+
let settled = false;
|
|
916
|
+
const t = setTimeout(() => {
|
|
917
|
+
if (settled)
|
|
918
|
+
return;
|
|
919
|
+
settled = true;
|
|
920
|
+
reject(new Error(`tick-await timeout after ${ms}ms: ${label}`));
|
|
921
|
+
}, ms);
|
|
922
|
+
if (typeof t.unref === 'function')
|
|
923
|
+
t.unref();
|
|
924
|
+
fn().then((v) => { if (!settled) {
|
|
925
|
+
settled = true;
|
|
926
|
+
clearTimeout(t);
|
|
927
|
+
resolve(v);
|
|
928
|
+
} }, (e) => { if (!settled) {
|
|
929
|
+
settled = true;
|
|
930
|
+
clearTimeout(t);
|
|
931
|
+
reject(e);
|
|
932
|
+
} });
|
|
933
|
+
});
|
|
934
|
+
}
|
|
935
|
+
/**
|
|
936
|
+
* F1b — the monotonic-clocked tick watchdog. Detects a stalled main tick loop
|
|
937
|
+
* (lost timer / stuck reentrancy guard) and re-arms it. SAFE-BY-CONSTRUCTION:
|
|
938
|
+
* never crashes (try/catch), never touches authority/epoch/suspend state, only
|
|
939
|
+
* resets a reentrancy guard whose in-flight tick is ALSO older than the ceiling
|
|
940
|
+
* (so a legitimately-slow live tick is never preempted), reads `enabled` live,
|
|
941
|
+
* and self-disarms (one DegradationReporter signal) if it re-arms too often.
|
|
942
|
+
* A true event-loop stall freezes this timer too — that case is layer-2 (the
|
|
943
|
+
* out-of-process fleet/launchd watchdog), as documented in the spec.
|
|
944
|
+
*/
|
|
945
|
+
runTickWatchdog() {
|
|
946
|
+
try {
|
|
947
|
+
const cfg = this.tickWatchdogCfg;
|
|
948
|
+
if (!cfg.enabled || this.watchdogDisarmed)
|
|
949
|
+
return;
|
|
950
|
+
// No lease coordinator (solo / non-git mesh) ⇒ nothing to self-heal.
|
|
951
|
+
if (!this.leaseCoordinator)
|
|
952
|
+
return;
|
|
953
|
+
const now = this.monoNowMs();
|
|
954
|
+
// lastTickRunMonoMs is stamped at the TOP of checkHeartbeatAndAct; if it
|
|
955
|
+
// has not advanced within the stale window, the main loop is stalled.
|
|
956
|
+
if (this.lastTickRunMonoMs > 0 && now - this.lastTickRunMonoMs <= cfg.staleMs)
|
|
957
|
+
return;
|
|
958
|
+
// Ceiling-gated guard reset: only clear a guard whose in-flight tick is
|
|
959
|
+
// ALSO older than the ceiling (a stuck guard, not a slow-but-live tick).
|
|
960
|
+
if (this.leaseTicking && this.leaseTickStartMonoMs > 0 && now - this.leaseTickStartMonoMs > cfg.staleMs) {
|
|
961
|
+
this.leaseTicking = false;
|
|
962
|
+
}
|
|
963
|
+
if (this.leasePulling && this.leasePullStartMonoMs > 0 && now - this.leasePullStartMonoMs > cfg.staleMs) {
|
|
964
|
+
this.leasePulling = false;
|
|
965
|
+
}
|
|
966
|
+
// Re-arm the main monitor (clears + recreates the interval).
|
|
967
|
+
this.startHeartbeatMonitor();
|
|
968
|
+
this.lastTickRunMonoMs = now; // reset so we don't re-fire next tick on the same stall
|
|
969
|
+
// Self-disarm bookkeeping (rolling 1h window).
|
|
970
|
+
this.watchdogReArmTimes.push(now);
|
|
971
|
+
const hourAgo = now - 3_600_000;
|
|
972
|
+
this.watchdogReArmTimes = this.watchdogReArmTimes.filter((t) => t >= hourAgo);
|
|
973
|
+
console.log(`[MultiMachine] lease-tick watchdog: stalled >${cfg.staleMs}ms — re-armed (${this.watchdogReArmTimes.length}/${cfg.maxReArmsPerHour} this hour)`);
|
|
974
|
+
this.emit('tickStallRecovered', { reArmCount: this.watchdogReArmTimes.length });
|
|
975
|
+
if (this.watchdogReArmTimes.length > cfg.maxReArmsPerHour) {
|
|
976
|
+
this.watchdogDisarmed = true;
|
|
977
|
+
console.error('[MultiMachine] lease-tick watchdog SELF-DISARMED — re-arming too often; the tick itself is the incident');
|
|
978
|
+
DegradationReporter.getInstance().report({
|
|
979
|
+
feature: 'MultiMachine.leaseTickWatchdog',
|
|
980
|
+
primary: 'The lease-tick self-heal watchdog re-arms a stalled coordinator tick',
|
|
981
|
+
fallback: `Watchdog re-armed >${cfg.maxReArmsPerHour}×/hour and has self-disarmed; the lease tick is repeatedly stalling`,
|
|
982
|
+
reason: 'A persistently-stalling lease tick (transport hang / event-loop pressure) the in-process watchdog cannot durably fix',
|
|
983
|
+
impact: 'The awake-machine election may be unstable; investigate the coordinator/transport. The out-of-process fleet watchdog remains the backstop.',
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
catch (err) {
|
|
988
|
+
// @silent-fallback-ok — the watchdog must NEVER throw out of its interval
|
|
989
|
+
// callback (an uncaughtException would turn a partial wedge into a crash).
|
|
990
|
+
console.error(`[MultiMachine] tick watchdog error (ignored): ${err instanceof Error ? err.message : String(err)}`);
|
|
991
|
+
}
|
|
992
|
+
}
|
|
793
993
|
/**
|
|
794
994
|
* Start periodic heartbeat monitoring (all machines).
|
|
795
995
|
*/
|
|
@@ -803,11 +1003,26 @@ export class MultiMachineCoordinator extends EventEmitter {
|
|
|
803
1003
|
if (this.heartbeatCheckTimer.unref) {
|
|
804
1004
|
this.heartbeatCheckTimer.unref();
|
|
805
1005
|
}
|
|
1006
|
+
// F1b — arm the independent tick watchdog ONCE (shorter 60s cadence so
|
|
1007
|
+
// detection latency isn't gated by the 2-min main cadence). startHeartbeat-
|
|
1008
|
+
// Monitor is also called BY the watchdog to re-arm the main timer, so guard
|
|
1009
|
+
// against stacking a second watchdog on re-arm.
|
|
1010
|
+
if (!this.tickWatchdogTimer) {
|
|
1011
|
+
this.tickWatchdogTimer = setInterval(() => {
|
|
1012
|
+
this.runTickWatchdog();
|
|
1013
|
+
}, TICK_WATCHDOG_INTERVAL_MS);
|
|
1014
|
+
if (this.tickWatchdogTimer.unref)
|
|
1015
|
+
this.tickWatchdogTimer.unref();
|
|
1016
|
+
}
|
|
806
1017
|
}
|
|
807
1018
|
/**
|
|
808
1019
|
* Check the heartbeat and take action if needed.
|
|
809
1020
|
*/
|
|
810
1021
|
checkHeartbeatAndAct() {
|
|
1022
|
+
// F1b — stamp the monotonic liveness mark FIRST, before any early-return, so
|
|
1023
|
+
// a solo / no-leaseCoordinator agent still advances it and the watchdog never
|
|
1024
|
+
// re-arms there (genuine no-op on single-machine agents).
|
|
1025
|
+
this.lastTickRunMonoMs = this.monoNowMs();
|
|
811
1026
|
if (!this._identity)
|
|
812
1027
|
return;
|
|
813
1028
|
// Independent mode: no failover/demotion logic
|