instar 1.3.627 → 1.3.629

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dashboard/subscriptions.js +50 -0
  2. package/dist/commands/server.d.ts.map +1 -1
  3. package/dist/commands/server.js +9 -0
  4. package/dist/commands/server.js.map +1 -1
  5. package/dist/config/ConfigDefaults.d.ts.map +1 -1
  6. package/dist/config/ConfigDefaults.js +28 -0
  7. package/dist/config/ConfigDefaults.js.map +1 -1
  8. package/dist/core/EnrollmentWizard.d.ts +8 -0
  9. package/dist/core/EnrollmentWizard.d.ts.map +1 -1
  10. package/dist/core/EnrollmentWizard.js +12 -0
  11. package/dist/core/EnrollmentWizard.js.map +1 -1
  12. package/dist/core/FencedLease.d.ts +23 -4
  13. package/dist/core/FencedLease.d.ts.map +1 -1
  14. package/dist/core/FencedLease.js +44 -4
  15. package/dist/core/FencedLease.js.map +1 -1
  16. package/dist/core/LeaseCoordinator.d.ts +47 -0
  17. package/dist/core/LeaseCoordinator.d.ts.map +1 -1
  18. package/dist/core/LeaseCoordinator.js +89 -7
  19. package/dist/core/LeaseCoordinator.js.map +1 -1
  20. package/dist/core/MultiMachineCoordinator.d.ts +73 -0
  21. package/dist/core/MultiMachineCoordinator.d.ts.map +1 -1
  22. package/dist/core/MultiMachineCoordinator.js +221 -6
  23. package/dist/core/MultiMachineCoordinator.js.map +1 -1
  24. package/dist/core/PendingLoginStore.d.ts.map +1 -1
  25. package/dist/core/PendingLoginStore.js +21 -2
  26. package/dist/core/PendingLoginStore.js.map +1 -1
  27. package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
  28. package/dist/core/PostUpdateMigrator.js +18 -1
  29. package/dist/core/PostUpdateMigrator.js.map +1 -1
  30. package/dist/core/devGatedFeatures.d.ts.map +1 -1
  31. package/dist/core/devGatedFeatures.js +10 -0
  32. package/dist/core/devGatedFeatures.js.map +1 -1
  33. package/dist/core/types.d.ts +66 -0
  34. package/dist/core/types.d.ts.map +1 -1
  35. package/dist/core/types.js.map +1 -1
  36. package/dist/scaffold/templates.d.ts.map +1 -1
  37. package/dist/scaffold/templates.js +2 -0
  38. package/dist/scaffold/templates.js.map +1 -1
  39. package/dist/server/routes.d.ts.map +1 -1
  40. package/dist/server/routes.js +112 -0
  41. package/dist/server/routes.js.map +1 -1
  42. package/package.json +1 -1
  43. package/src/data/builtin-manifest.json +64 -64
  44. package/src/scaffold/templates.ts +2 -0
  45. package/upgrades/1.3.628.md +23 -0
  46. package/upgrades/1.3.629.md +31 -0
  47. package/upgrades/side-effects/matrix-cell-operator-cancel.md +151 -0
  48. package/upgrades/side-effects/multi-machine-lease-self-heal.md +45 -0
@@ -41,6 +41,17 @@ export class LeaseCoordinator {
41
41
  * while it is not superseded by a higher epoch.
42
42
  */
43
43
  selfIssued = null;
44
+ /**
45
+ * F2 (staleHolderTakeover) — per-holder freshness on the OBSERVER's own
46
+ * monotonic clock: the time we last saw that holder's signed nonce watermark
47
+ * ADVANCE (a renewing holder bumps its nonce each renew). Stamped only when a
48
+ * VERIFIED observed lease's nonce strictly exceeds the last we recorded for
49
+ * that holder. `lastObservedNonce` is the per-holder high-water nonce that
50
+ * gates the stamp. Single-clock ⇒ clock-skew immune; the holder cannot forge
51
+ * freshness (the nonce is inside its Ed25519 signature).
52
+ */
53
+ freshObservedMonoMs = new Map();
54
+ lastObservedNonce = new Map();
44
55
  constructor(deps) {
45
56
  this.d = deps;
46
57
  this.fl = deps.lease;
@@ -103,9 +114,20 @@ export class LeaseCoordinator {
103
114
  const { [obs.lease.holder]: _self, ...nonceFloor } = obs.lastNonceByHolder;
104
115
  void _self;
105
116
  const decision = this.fl.acceptTunnelLease(obs.lease, git.epoch, nonceFloor);
106
- if (decision.accept && obs.lease.epoch > epoch) {
107
- bestLease = obs.lease;
108
- epoch = obs.lease.epoch;
117
+ if (decision.accept) {
118
+ // F2 freshness stamp (VERIFIED fold-in): a renewing holder bumps its
119
+ // signed nonce each renew (same OR higher epoch), so its watermark
120
+ // advances; a non-renewing holder's stops. Stamp the OBSERVER's own
121
+ // monotonic time when a peer holder's nonce watermark strictly advances.
122
+ const prevNonce = this.lastObservedNonce.get(obs.lease.holder) ?? -1;
123
+ if (obs.lease.holder !== this.selfMachineId && obs.lease.nonce > prevNonce) {
124
+ this.lastObservedNonce.set(obs.lease.holder, obs.lease.nonce);
125
+ this.freshObservedMonoMs.set(obs.lease.holder, this.monotonicNow());
126
+ }
127
+ if (obs.lease.epoch > epoch) {
128
+ bestLease = obs.lease;
129
+ epoch = obs.lease.epoch;
130
+ }
109
131
  }
110
132
  }
111
133
  }
@@ -140,7 +162,25 @@ export class LeaseCoordinator {
140
162
  return this.effectiveView().epoch;
141
163
  }
142
164
  currentHolder() {
143
- return this.effectiveView().lease?.holder ?? null;
165
+ const lease = this.effectiveView().lease;
166
+ // F3 — a RELEASED tombstone declares "epoch N released, not held": it names
167
+ // no live holder, so peers stop deferring to a muted ex-holder's zombie. A
168
+ // higher-epoch normal acquisition still strictly dominates (epoch fold).
169
+ if (!lease || lease.released)
170
+ return null;
171
+ return lease.holder;
172
+ }
173
+ /**
174
+ * F4 — is `machineId` the current holder of a LIVE (non-expired, non-released)
175
+ * lease we observe? The deferential-standby health gate: a non-preferred machine
176
+ * defers to its preferred peer ONLY while this is true, so a frozen/down/released
177
+ * preferred never strands coverage (the non-preferred then acquires normally).
178
+ */
179
+ isHolderHealthy(machineId) {
180
+ const view = this.effectiveView();
181
+ if (!view.lease || view.lease.released || view.lease.holder !== machineId)
182
+ return false;
183
+ return !this.fl.isExpired(view.lease, this.now());
144
184
  }
145
185
  /**
146
186
  * The current effective-view signed lease (max of tunnel-observed, git-committed,
@@ -175,6 +215,48 @@ export class LeaseCoordinator {
175
215
  this.d.store.forceLocalExpiry?.();
176
216
  this.log('relinquished self-lease (contested tie-break loser) — winner may now advance to N+1');
177
217
  }
218
+ /**
219
+ * F2 — build the StaleHolderTakeoverOpts for `canAcquire`, or `undefined` when
220
+ * the flag is off / the candidate is not a takeable peer (so canAcquire stays
221
+ * byte-for-byte the legacy behavior). Uses this holder's observed freshness on
222
+ * the OBSERVER's own monotonic clock.
223
+ */
224
+ staleHolderOpts(currentLease) {
225
+ const cfg = this.d.staleHolderTakeover?.();
226
+ if (!cfg?.enabled)
227
+ return undefined;
228
+ if (!currentLease || currentLease.holder === this.selfMachineId)
229
+ return undefined;
230
+ return {
231
+ monotonicNowMs: this.monotonicNow(),
232
+ freshObservedMonoMs: this.freshObservedMonoMs.get(currentLease.holder),
233
+ nonRenewalThresholdMs: this.fl.ttlMs * Math.max(1, cfg.nonRenewalMissedObservations),
234
+ };
235
+ }
236
+ /**
237
+ * F3 — relinquish a lease THIS machine holds AND broadcast a SIGNED tombstone
238
+ * so peers stop deferring to a muted ex-holder's zombie. Called level-triggered
239
+ * by the coordinator when this machine is observe-only yet still the named
240
+ * holder (the 2026-06-19 silent-standby zombie). The tombstone is
241
+ * `released:true` at our epoch with a fresh nonce strictly greater than our last
242
+ * (so a concurrent in-flight renewal cannot out-nonce it) and a past expiry; a
243
+ * higher-epoch normal takeover always strictly dominates it. Idempotent: a no-op
244
+ * (just local relinquish) when we are not the named holder.
245
+ */
246
+ async relinquishAndBroadcast() {
247
+ const view = this.effectiveView();
248
+ if (!view.lease || view.lease.holder !== this.selfMachineId || view.lease.released) {
249
+ // Not the live holder (or already a tombstone) → just clear local state.
250
+ this.relinquish();
251
+ return;
252
+ }
253
+ const tombstone = this.fl.signLease(view.lease.epoch, view.lease.acquiredAt, new Date(this.now()).toISOString(), // past/now expiry — released regardless
254
+ this.nextNonce(), // strictly > our last renewal nonce
255
+ true);
256
+ this.relinquish(); // clear our own self-hold first (forceLocalExpiry preserves the epoch floor)
257
+ await this.broadcast(tombstone);
258
+ this.log(`relinquished + broadcast tombstone for epoch ${view.lease.epoch} (silent-standby release)`);
259
+ }
178
260
  /**
179
261
  * Force a ONE-TIME epoch advance to resolve a same-epoch contested tie (spec
180
262
  * §Problem A — the WINNER side, lower `machineId`). Unlike acquireIfEligible(),
@@ -284,7 +366,7 @@ export class LeaseCoordinator {
284
366
  if (view.lease && view.lease.holder === this.selfMachineId && !this.fl.isExpired(view.lease, this.now())) {
285
367
  return this.renew();
286
368
  }
287
- const decision = this.fl.canAcquire(view.lease, dead, this.now());
369
+ const decision = this.fl.canAcquire(view.lease, dead, this.now(), this.staleHolderOpts(view.lease));
288
370
  if (!decision.can) {
289
371
  this.log(`acquire skipped: ${decision.reason}`);
290
372
  return false;
@@ -304,8 +386,8 @@ export class LeaseCoordinator {
304
386
  if (observedEpoch >= candidate.epoch) {
305
387
  this.log(`CAS lost to epoch ${observedEpoch} (our candidate ${candidate.epoch}) — yielding`);
306
388
  this.emitEpoch(observedEpoch);
307
- // If the winner is a presumed-dead/expired holder we'll retry; else stop.
308
- if (!this.fl.canAcquire(res.observed.lease, dead, this.now()).can)
389
+ // If the winner is a presumed-dead/expired/non-renewing holder we'll retry; else stop.
390
+ if (!this.fl.canAcquire(res.observed.lease, dead, this.now(), this.staleHolderOpts(res.observed.lease)).can)
309
391
  return false;
310
392
  }
311
393
  retries++;
@@ -1 +1 @@
1
- {"version":3,"file":"LeaseCoordinator.js","sourceRoot":"","sources":["../../src/core/LeaseCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAyFH,MAAM,OAAO,gBAAgB;IACV,CAAC,CAAuB;IACxB,EAAE,CAAc;IACzB,YAAY,GAAG,CAAC,CAAC;IACzB;;;;;;OAMG;IACK,iBAAiB,GAAG,CAAC,CAAC;IACtB,iBAAiB,GAAG,CAAC,CAAC;IACtB,SAAS,GAAG,KAAK,CAAC;IAC1B;;;;;;OAMG;IACK,UAAU,GAAuB,IAAI,CAAC;IAE9C,YAAY,IAA0B;QACpC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;QACd,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;QACrB,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAEO,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,CAAC;IACD;;;;;OAKG;IACK,YAAY;QAClB,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;QACtD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,QAAU,CAAC,CAAC;IACtD,CAAC;IACD;;;OAGG;IACK,WAAW;QACjB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAC/C,CAAC;IACO,GAAG,CAAC,CAAS;QACnB,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IACO,SAAS;QACf,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC;IAC/B,CAAC;IACD,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,aAAa;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;QAC1B,IAAI,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,wEAAwE;gBACxE,0EAA0E;gBAC1E,0EAA0E;gBAC1E,2EAA2E;gBAC3E,2EAA2E;gBAC3E,yEAAyE;gBACzE,yEAAyE;gBACzE,uEAAuE;gBACvE,2EAA2E;gBAC3E,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,GAAG,CAAC,iBAAiB,CAAC;gBAC3E,KAAK,KAAK,CAAC;gBACX,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;gBAC7E,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC;oBAC/C,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;oBACtB,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QACD,2EAA2E;QAC3E,yEAAyE;QACzE,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;YACvG,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;YAC5B,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChC,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;IAC1D,CAAC;IAED,6EAA6E;IAC7E,UAAU;QACR,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/E,2EAA2E;QAC3E,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,oEAAoE;QACpE,0EAA0E;QAC1E,wDAAwD;QACxD,IAAI,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAC/E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,+DAA+D;IAC/D,YAAY;QACV,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;IACpC,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;IACpD,CAAC;IAED;;;;;;OAMG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU;QACR,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,qFAAqF,CAAC,CAAC;IAClG,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,2BAA2B;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,iFAAiF;QACjF,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,qBAAqB,SAAS,CAAC,KAAK,+CAA+C,CAAC,CAAC;YAC9F,OAAO,IAAI,CAAC;QACd,CAAC;QACD,6EAA6E;QAC7E,qDAAqD;QACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,2CAA2C,GAAG,CAAC,QAAQ,CAAC,KAAK,8BAA8B,CAAC,CAAC;QACtG,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC,aAAa,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,YAAY,KAAK,UAAU,CAAC;IAC3D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,YAAY;YAAE,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;IACtE,CAAC;IAED;;;;;OAKG;IACH,iBAAiB;QACf,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,mBAAmB;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACxB,oEAAoE;QACpE,IAAI,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACtD,+EAA+E;QAC/E,8EAA8E;QAC9E,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,oEAAoE;YACpE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;QAC1C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAClC,qDAAqD;YACrD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;gBACzG,OAAO,IAAI,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAClE,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAClB,IAAI,CAAC,GAAG,CAAC,oBAAoB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBAChD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;gBAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAChC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,2BAA2B,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,uEAAuE;YACvE,MAAM,aAAa,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;YACzC,IAAI,aAAa,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,qBAAqB,aAAa,mBAAmB,SAAS,CAAC,KAAK,cAAc,CAAC,CAAC;gBAC7F,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBAC9B,0EAA0E;gBAC1E,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG;oBAAE,OAAO,KAAK,CAAC;YAClF,CAAC;YACD,OAAO,EAAE,CAAC;YACV,IAAI,IAAI,CAAC,EAAE,CAAC,4BAA4B,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;gBACpF,IAAI,CAAC,GAAG,CAAC,0BAA0B,OAAO,2BAA2B,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,CAAC;gBAC5F,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,gBAAgB,CAAC,kBAA0B;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;QAC1C,IAAI,MAAM,IAAI,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,CAAC,+BAA+B;QAC9C,CAAC;QACD,IAAI,MAAM,IAAI,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,uCAAuC,kBAAkB,0BAA0B,MAAM,EAAE,CAAC,CAAC;YACtG,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,sCAAsC,SAAS,CAAC,KAAK,gBAAgB,kBAAkB,GAAG,CAAC,CAAC;YACrG,OAAO,IAAI,CAAC;QACd,CAAC;QACD,gFAAgF;QAChF,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,qCAAqC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QACpE,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa;YAAE,OAAO,KAAK,CAAC;QAE1E,wEAAwE;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,CAC/B,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAClD,IAAI,CAAC,SAAS,EAAE,CACjB,CAAC;QAEF,4EAA4E;QAC5E,2EAA2E;QAC3E,wEAAwE;QACxE,uEAAuE;QACvE,oEAAoE;QACpE,IAAI,SAAkB,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClB,SAAS,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QACxE,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC;YAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YACjE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,CACpB,gCAAgC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,sBAAsB,IAAI,CAAC,EAAE,CAAC,KAAK,+BAA+B,CACnI,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,yEAAyE;QACzE,sEAAsE;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAkB;QACxC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO;QAC3B,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChD,IAAI,EAAE;gBAAE,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,qBAAqB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,KAAa;QAC7B,IAAI,KAAK,KAAK,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACrC,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;YAC/B,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,yBAAyB,CAAC,kBAA0B;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3E,IAAI,UAAU,IAAI,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5E,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;gBAClB,kBAAkB;gBAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACzB,MAAM,EAAE,gFAAgF;aACzF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
1
+ {"version":3,"file":"LeaseCoordinator.js","sourceRoot":"","sources":["../../src/core/LeaseCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAiGH,MAAM,OAAO,gBAAgB;IACV,CAAC,CAAuB;IACxB,EAAE,CAAc;IACzB,YAAY,GAAG,CAAC,CAAC;IACzB;;;;;;OAMG;IACK,iBAAiB,GAAG,CAAC,CAAC;IACtB,iBAAiB,GAAG,CAAC,CAAC;IACtB,SAAS,GAAG,KAAK,CAAC;IAC1B;;;;;;OAMG;IACK,UAAU,GAAuB,IAAI,CAAC;IAC9C;;;;;;;;OAQG;IACK,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEtD,YAAY,IAA0B;QACpC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;QACd,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;QACrB,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAEO,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,CAAC;IACD;;;;;OAKG;IACK,YAAY;QAClB,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;QACtD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,QAAU,CAAC,CAAC;IACtD,CAAC;IACD;;;OAGG;IACK,WAAW;QACjB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAC/C,CAAC;IACO,GAAG,CAAC,CAAS;QACnB,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IACO,SAAS;QACf,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC;IAC/B,CAAC;IACD,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,aAAa;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;QAC1B,IAAI,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,wEAAwE;gBACxE,0EAA0E;gBAC1E,0EAA0E;gBAC1E,2EAA2E;gBAC3E,2EAA2E;gBAC3E,yEAAyE;gBACzE,yEAAyE;gBACzE,uEAAuE;gBACvE,2EAA2E;gBAC3E,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,GAAG,CAAC,iBAAiB,CAAC;gBAC3E,KAAK,KAAK,CAAC;gBACX,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;gBAC7E,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACpB,qEAAqE;oBACrE,mEAAmE;oBACnE,oEAAoE;oBACpE,yEAAyE;oBACzE,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;oBACrE,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;wBAC3E,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAC9D,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;oBACtE,CAAC;oBACD,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC;wBAC5B,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;wBACtB,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC;oBAC1B,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,2EAA2E;QAC3E,yEAAyE;QACzE,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;YACvG,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;YAC5B,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChC,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;IAC1D,CAAC;IAED,6EAA6E;IAC7E,UAAU;QACR,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/E,2EAA2E;QAC3E,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,oEAAoE;QACpE,0EAA0E;QAC1E,wDAAwD;QACxD,IAAI,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAC/E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,+DAA+D;IAC/D,YAAY;QACV,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;IACpC,CAAC;IAED,aAAa;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;QACzC,4EAA4E;QAC5E,2EAA2E;QAC3E,yEAAyE;QACzE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC1C,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,SAAiB;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACxF,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpD,CAAC;IAED;;;;;;OAMG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU;QACR,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,qFAAqF,CAAC,CAAC;IAClG,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,YAA4C;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,GAAG,EAAE,OAAO;YAAE,OAAO,SAAS,CAAC;QACpC,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa;YAAE,OAAO,SAAS,CAAC;QAClF,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,YAAY,EAAE;YACnC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC;YACtE,qBAAqB,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,4BAA4B,CAAC;SACrF,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,sBAAsB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnF,yEAAyE;YACzE,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,CACjC,IAAI,CAAC,KAAK,CAAC,KAAK,EAChB,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,wCAAwC;QAC5E,IAAI,CAAC,SAAS,EAAE,EAAoB,oCAAoC;QACxE,IAAI,CACL,CAAC;QACF,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,6EAA6E;QAChG,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,gDAAgD,IAAI,CAAC,KAAK,CAAC,KAAK,2BAA2B,CAAC,CAAC;IACxG,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,2BAA2B;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,iFAAiF;QACjF,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,qBAAqB,SAAS,CAAC,KAAK,+CAA+C,CAAC,CAAC;YAC9F,OAAO,IAAI,CAAC;QACd,CAAC;QACD,6EAA6E;QAC7E,qDAAqD;QACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,2CAA2C,GAAG,CAAC,QAAQ,CAAC,KAAK,8BAA8B,CAAC,CAAC;QACtG,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC,aAAa,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,YAAY,KAAK,UAAU,CAAC;IAC3D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,YAAY;YAAE,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;IACtE,CAAC;IAED;;;;;OAKG;IACH,iBAAiB;QACf,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,mBAAmB;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACxB,oEAAoE;QACpE,IAAI,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACtD,+EAA+E;QAC/E,8EAA8E;QAC9E,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,oEAAoE;YACpE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;QAC1C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAClC,qDAAqD;YACrD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;gBACzG,OAAO,IAAI,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACpG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAClB,IAAI,CAAC,GAAG,CAAC,oBAAoB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBAChD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;gBAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAChC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,2BAA2B,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,uEAAuE;YACvE,MAAM,aAAa,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;YACzC,IAAI,aAAa,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,qBAAqB,aAAa,mBAAmB,SAAS,CAAC,KAAK,cAAc,CAAC,CAAC;gBAC7F,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBAC9B,uFAAuF;gBACvF,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG;oBAAE,OAAO,KAAK,CAAC;YAC5H,CAAC;YACD,OAAO,EAAE,CAAC;YACV,IAAI,IAAI,CAAC,EAAE,CAAC,4BAA4B,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;gBACpF,IAAI,CAAC,GAAG,CAAC,0BAA0B,OAAO,2BAA2B,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,CAAC;gBAC5F,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,gBAAgB,CAAC,kBAA0B;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC;QAC1C,IAAI,MAAM,IAAI,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,CAAC,+BAA+B;QAC9C,CAAC;QACD,IAAI,MAAM,IAAI,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,uCAAuC,kBAAkB,0BAA0B,MAAM,EAAE,CAAC,CAAC;YACtG,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,sCAAsC,SAAS,CAAC,KAAK,gBAAgB,kBAAkB,GAAG,CAAC,CAAC;YACrG,OAAO,IAAI,CAAC;QACd,CAAC;QACD,gFAAgF;QAChF,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,qCAAqC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QACpE,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa;YAAE,OAAO,KAAK,CAAC;QAE1E,wEAAwE;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,CAC/B,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAClD,IAAI,CAAC,SAAS,EAAE,CACjB,CAAC;QAEF,4EAA4E;QAC5E,2EAA2E;QAC3E,wEAAwE;QACxE,uEAAuE;QACvE,oEAAoE;QACpE,IAAI,SAAkB,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAClB,SAAS,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QACxE,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC;YAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YACjE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,CACpB,gCAAgC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,sBAAsB,IAAI,CAAC,EAAE,CAAC,KAAK,+BAA+B,CACnI,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,yEAAyE;QACzE,sEAAsE;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAkB;QACxC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO;QAC3B,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChD,IAAI,EAAE;gBAAE,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,qBAAqB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,KAAa;QAC7B,IAAI,KAAK,KAAK,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACrC,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;YAC/B,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,yBAAyB,CAAC,kBAA0B;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3E,IAAI,UAAU,IAAI,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5E,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;gBAClB,kBAAkB;gBAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACzB,MAAM,EAAE,gFAAgF;aACzF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
@@ -32,6 +32,18 @@ export interface MultiMachineSyncStatus {
32
32
  splitBrainState: 'clear' | 'contested' | 'self-suspended';
33
33
  protocolVersion: number;
34
34
  awakeMachineCount: number;
35
+ /**
36
+ * multi-machine-lease-self-heal observability (Agent Awareness). F1 tick-watchdog
37
+ * health — answers "did the watchdog fire?" / "is it disarmed?". `lastTickAgeMs`
38
+ * is the monotonic age of the last main-tick run (a large value = the tick has
39
+ * stalled). `preferredAwakeMachineId` echoes the F4 config (null = off).
40
+ */
41
+ leaseTickWatchdog?: {
42
+ lastTickAgeMs: number;
43
+ reArmCount: number;
44
+ disarmed: boolean;
45
+ };
46
+ preferredAwakeMachineId?: string | null;
35
47
  }
36
48
  export interface CoordinatorConfig {
37
49
  /** State directory (.instar) */
@@ -110,6 +122,23 @@ export declare class MultiMachineCoordinator extends EventEmitter {
110
122
  * K-cycle escalation. Cleared on the falling edge (contested resolved).
111
123
  */
112
124
  private contestedEpisode;
125
+ /**
126
+ * multi-machine-lease-self-heal F1 — the tick self-heal watchdog state.
127
+ * `lastTickRunMonoMs` is stamped on the OBSERVER's own monotonic clock at the
128
+ * TOP of checkHeartbeatAndAct (before any early-return, so a solo agent stamps
129
+ * a healthy advancing value and the watchdog stays a no-op). `*StartMonoMs`
130
+ * stamp when a reentrancy guard is taken, so the watchdog can distinguish a
131
+ * legitimately-slow in-flight tick (leave alone) from a stuck guard (reset).
132
+ * `watchdogReArmTimes` is a rolling window of re-arm timestamps for self-disarm.
133
+ */
134
+ private tickWatchdogTimer;
135
+ private lastTickRunMonoMs;
136
+ private leaseTickStartMonoMs;
137
+ private leasePullStartMonoMs;
138
+ private watchdogReArmTimes;
139
+ private watchdogDisarmed;
140
+ /** F3 — per-incarnation latch so a silent standby relinquishes its held lease once. */
141
+ private silentStandbyRelinquished;
113
142
  constructor(state: StateManager, config: CoordinatorConfig);
114
143
  /** Whether multi-machine is enabled (has identity). */
115
144
  get enabled(): boolean;
@@ -133,6 +162,28 @@ export declare class MultiMachineCoordinator extends EventEmitter {
133
162
  * such a machine is a deliberate un-mute (telegramPolling → true), not auto.
134
163
  */
135
164
  get isLeaseObserveOnly(): boolean;
165
+ /**
166
+ * multi-machine-lease-self-heal M3 — the first-class lease-participation mode,
167
+ * decoupled from the overloaded `telegramPolling` flag. Explicit
168
+ * `leaseSelfHeal.leaseRole` wins; otherwise derive from `telegramPolling`
169
+ * (===false ⇒ 'observe-only', else 'active') for back-compat. ('deferential'
170
+ * is an F4 concept handled separately; only 'observe-only' gates acquisition
171
+ * here so F4's down-preferred failover is never accidentally suppressed.)
172
+ */
173
+ private resolvedLeaseRole;
174
+ /**
175
+ * F4 (preferred-awake, opt-in; null = off) — should THIS machine defer (not
176
+ * contend for the lease) right now? True iff a `preferredAwakeMachineId` is
177
+ * configured, it names a DIFFERENT machine (we are NOT the preferred), and that
178
+ * preferred machine is currently a HEALTHY holder. A machine that IS the
179
+ * preferred never defers; a non-preferred machine defers only while the preferred
180
+ * is healthy, so the preferred going down never strands coverage. Safe under any
181
+ * config (no tie-break override, so a divergent config degrades to the existing
182
+ * lower-machineId baseline rather than flapping).
183
+ */
184
+ private shouldDeferToPreferred;
185
+ /** F4 — the single shared health predicate: is the preferred machine a live holder? */
186
+ private preferredIsHealthy;
136
187
  /** The coordination mode (default: 'primary-standby'). */
137
188
  get coordinationMode(): CoordinationMode;
138
189
  /** The underlying managers (for route wiring). */
@@ -291,6 +342,28 @@ export declare class MultiMachineCoordinator extends EventEmitter {
291
342
  * BEFORE the peer's ~15min heartbeat-expiry failover horizon.
292
343
  */
293
344
  private writeHeartbeatGuarded;
345
+ /** Monotonic milliseconds (NTP-step / sleep-resume immune), like LeaseCoordinator §L−1. */
346
+ private monoNowMs;
347
+ /** Resolved F1 watchdog config (defaults baked in; read live so disable needs no restart). */
348
+ private get tickWatchdogCfg();
349
+ /**
350
+ * F1a — bound a tick-path network await so a never-settling call (the proven
351
+ * 2026-06-19 freeze: a hung fetch left `leaseTicking` stuck true forever) can
352
+ * NEVER hang the tick. ALL tick-path awaits route through this one helper, so
353
+ * the "no unbounded tick await" invariant is structural + grep-auditable.
354
+ */
355
+ private withTickTimeout;
356
+ /**
357
+ * F1b — the monotonic-clocked tick watchdog. Detects a stalled main tick loop
358
+ * (lost timer / stuck reentrancy guard) and re-arms it. SAFE-BY-CONSTRUCTION:
359
+ * never crashes (try/catch), never touches authority/epoch/suspend state, only
360
+ * resets a reentrancy guard whose in-flight tick is ALSO older than the ceiling
361
+ * (so a legitimately-slow live tick is never preempted), reads `enabled` live,
362
+ * and self-disarms (one DegradationReporter signal) if it re-arms too often.
363
+ * A true event-loop stall freezes this timer too — that case is layer-2 (the
364
+ * out-of-process fleet/launchd watchdog), as documented in the spec.
365
+ */
366
+ private runTickWatchdog;
294
367
  /**
295
368
  * Start periodic heartbeat monitoring (all machines).
296
369
  */
@@ -1 +1 @@
1
- {"version":3,"file":"MultiMachineCoordinator.d.ts","sourceRoot":"","sources":["../../src/core/MultiMachineCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAI9D,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAErG,0EAA0E;AAC1E,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,8FAA8F;IAC9F,eAAe,EAAE,OAAO,GAAG,WAAW,GAAG,gBAAgB,CAAC;IAC1D,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAYD,MAAM,WAAW,iBAAiB;IAChC,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,YAAY,CAAC,EAAE,kBAAkB,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,wDAAwD;IACxD,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,yDAAyD;IACzD,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,0CAA0C;IAC1C,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,6BAA6B;IAC7B,UAAU,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;CAC1D;AAID,qBAAa,uBAAwB,SAAQ,YAAY;IACvD,OAAO,CAAC,eAAe,CAAyB;IAChD,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,KAAK,CAA0B;IACvC,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA+C;IAC1E;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0D;IACzF,OAAO,CAAC,mBAAmB,CAA+C;IAC1E;qEACiE;IACjE,OAAO,CAAC,6BAA6B,CAAkB;IACvD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAiC;IACzD,OAAO,CAAC,YAAY,CAAkB;IACtC;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,gBAAgB,CAAkB;IAC1C;;;;;;;;;;OAUG;IACH,OAAO,CAAC,gBAAgB,CAAuF;gBAEnG,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,iBAAiB;IAc1D,uDAAuD;IACvD,IAAI,OAAO,IAAI,OAAO,CAA0B;IAEhD,mCAAmC;IACnC,IAAI,IAAI,IAAI,WAAW,CAAuB;IAE9C,yDAAyD;IACzD,IAAI,QAAQ,IAAI,eAAe,GAAG,IAAI,CAA2B;IAEjE,2DAA2D;IAC3D,IAAI,OAAO,IAAI,OAAO,CAAmC;IAEzD;;;;;;;;;;;;OAYG;IACH,IAAI,kBAAkB,IAAI,OAAO,CAEhC;IAED,0DAA0D;IAC1D,IAAI,gBAAgB,IAAI,gBAAgB,CAEvC;IAED,kDAAkD;IAClD,IAAI,QAAQ;;;;;MAOX;IAID;;;OAGG;IACH,KAAK,IAAI,WAAW;IAuGpB;;OAEG;IACH;;;;OAIG;IACH,OAAO,CAAC,sCAAsC;IAiB9C,IAAI,IAAI,IAAI;IAmBZ;;;OAGG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IA0CpC;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAgCrC;;;OAGG;IACH,oBAAoB,IAAI,OAAO;IAoB/B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,EAAE,gBAAgB,GAAG,IAAI;IAIlD,kFAAkF;IAClF,UAAU,IAAI,OAAO;IAIrB;;;;;;;;;OASG;IACH,QAAQ,IAAI,OAAO;IAInB;;;;OAIG;IACH,aAAa,IAAI,MAAM;IAIvB;;;;;OAKG;IACH,aAAa,IAAI,sBAAsB;IA6BvC;;;OAGG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBtC;;;;;;OAMG;IACG,qBAAqB,CAAC,kBAAkB,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAOzE;;;OAGG;YACW,SAAS;IAsBvB,wEAAwE;IACxE,OAAO,CAAC,oBAAoB;IA6B5B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAqB1B;;;;;;OAMG;YACW,aAAa;IA0C3B;;;;;;;;OAQG;IACH,OAAO,CAAC,+BAA+B;IAoBvC;;;;;;;;;;;;;;;;;;;;;OAqBG;YACW,0BAA0B;IAgExC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IA4B5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,qBAAqB;IA0B7B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAc7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CA2B7B"}
1
+ {"version":3,"file":"MultiMachineCoordinator.d.ts","sourceRoot":"","sources":["../../src/core/MultiMachineCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAI9D,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAErG,0EAA0E;AAC1E,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,8FAA8F;IAC9F,eAAe,EAAE,OAAO,GAAG,WAAW,GAAG,gBAAgB,CAAC;IAC1D,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC;IACrF,uBAAuB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AAaD,MAAM,WAAW,iBAAiB;IAChC,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,YAAY,CAAC,EAAE,kBAAkB,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,wDAAwD;IACxD,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,yDAAyD;IACzD,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,0CAA0C;IAC1C,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,6BAA6B;IAC7B,UAAU,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;CAC1D;AAID,qBAAa,uBAAwB,SAAQ,YAAY;IACvD,OAAO,CAAC,eAAe,CAAyB;IAChD,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,KAAK,CAA0B;IACvC,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA+C;IAC1E;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0D;IACzF,OAAO,CAAC,mBAAmB,CAA+C;IAC1E;qEACiE;IACjE,OAAO,CAAC,6BAA6B,CAAkB;IACvD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAiC;IACzD,OAAO,CAAC,YAAY,CAAkB;IACtC;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,gBAAgB,CAAkB;IAC1C;;;;;;;;;;OAUG;IACH,OAAO,CAAC,gBAAgB,CAAuF;IAE/G;;;;;;;;OAQG;IACH,OAAO,CAAC,iBAAiB,CAA+C;IACxE,OAAO,CAAC,iBAAiB,CAAa;IACtC,OAAO,CAAC,oBAAoB,CAAa;IACzC,OAAO,CAAC,oBAAoB,CAAa;IACzC,OAAO,CAAC,kBAAkB,CAAgB;IAC1C,OAAO,CAAC,gBAAgB,CAAkB;IAC1C,uFAAuF;IACvF,OAAO,CAAC,yBAAyB,CAAkB;gBAEvC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,iBAAiB;IAc1D,uDAAuD;IACvD,IAAI,OAAO,IAAI,OAAO,CAA0B;IAEhD,mCAAmC;IACnC,IAAI,IAAI,IAAI,WAAW,CAAuB;IAE9C,yDAAyD;IACzD,IAAI,QAAQ,IAAI,eAAe,GAAG,IAAI,CAA2B;IAEjE,2DAA2D;IAC3D,IAAI,OAAO,IAAI,OAAO,CAAmC;IAEzD;;;;;;;;;;;;OAYG;IACH,IAAI,kBAAkB,IAAI,OAAO,CAEhC;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;IAMzB;;;;;;;;;OASG;IACH,OAAO,CAAC,sBAAsB;IAO9B,uFAAuF;IACvF,OAAO,CAAC,kBAAkB;IAI1B,0DAA0D;IAC1D,IAAI,gBAAgB,IAAI,gBAAgB,CAEvC;IAED,kDAAkD;IAClD,IAAI,QAAQ;;;;;MAOX;IAID;;;OAGG;IACH,KAAK,IAAI,WAAW;IAuGpB;;OAEG;IACH;;;;OAIG;IACH,OAAO,CAAC,sCAAsC;IAiB9C,IAAI,IAAI,IAAI;IAuBZ;;;OAGG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IA0CpC;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAgCrC;;;OAGG;IACH,oBAAoB,IAAI,OAAO;IAoB/B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,EAAE,gBAAgB,GAAG,IAAI;IAIlD,kFAAkF;IAClF,UAAU,IAAI,OAAO;IAIrB;;;;;;;;;OASG;IACH,QAAQ,IAAI,OAAO;IAInB;;;;OAIG;IACH,aAAa,IAAI,MAAM;IAIvB;;;;;OAKG;IACH,aAAa,IAAI,sBAAsB;IAqCvC;;;OAGG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAmBtC;;;;;;OAMG;IACG,qBAAqB,CAAC,kBAAkB,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAOzE;;;OAGG;YACW,SAAS;IA+CvB,wEAAwE;IACxE,OAAO,CAAC,oBAAoB;IA6B5B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAqB1B;;;;;;OAMG;YACW,aAAa;IA6C3B;;;;;;;;OAQG;IACH,OAAO,CAAC,+BAA+B;IAoBvC;;;;;;;;;;;;;;;;;;;;;OAqBG;YACW,0BAA0B;IAgExC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IA4B5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,qBAAqB;IA4B7B,2FAA2F;IAC3F,OAAO,CAAC,SAAS;IAIjB,8FAA8F;IAC9F,OAAO,KAAK,eAAe,GAS1B;IAED;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAiBvB;;;;;;;;;OASG;IACH,OAAO,CAAC,eAAe;IAgDvB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAyB7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CA+B7B"}
@@ -25,6 +25,7 @@ import { DegradationReporter } from '../monitoring/DegradationReporter.js';
25
25
  // ── Constants ────────────────────────────────────────────────────────
26
26
  const HEARTBEAT_WRITE_INTERVAL_MS = 2 * 60_000; // Write heartbeat every 2 min
27
27
  const HEARTBEAT_CHECK_INTERVAL_MS = 2 * 60_000; // Check heartbeat every 2 min
28
+ const TICK_WATCHDOG_INTERVAL_MS = 60_000; // F1b — independent tick-stall watchdog cadence
28
29
  const DEFAULT_FAILOVER_TIMEOUT_MS = 15 * 60_000; // 15 min before failover
29
30
  /** Cross-Machine Coherence — default active lease-PULL cadence over the tunnel. */
30
31
  const DEFAULT_LEASE_PULL_INTERVAL_MS = 5_000;
@@ -90,6 +91,23 @@ export class MultiMachineCoordinator extends EventEmitter {
90
91
  * K-cycle escalation. Cleared on the falling edge (contested resolved).
91
92
  */
92
93
  contestedEpisode = null;
94
+ /**
95
+ * multi-machine-lease-self-heal F1 — the tick self-heal watchdog state.
96
+ * `lastTickRunMonoMs` is stamped on the OBSERVER's own monotonic clock at the
97
+ * TOP of checkHeartbeatAndAct (before any early-return, so a solo agent stamps
98
+ * a healthy advancing value and the watchdog stays a no-op). `*StartMonoMs`
99
+ * stamp when a reentrancy guard is taken, so the watchdog can distinguish a
100
+ * legitimately-slow in-flight tick (leave alone) from a stuck guard (reset).
101
+ * `watchdogReArmTimes` is a rolling window of re-arm timestamps for self-disarm.
102
+ */
103
+ tickWatchdogTimer = null;
104
+ lastTickRunMonoMs = 0;
105
+ leaseTickStartMonoMs = 0;
106
+ leasePullStartMonoMs = 0;
107
+ watchdogReArmTimes = [];
108
+ watchdogDisarmed = false;
109
+ /** F3 — per-incarnation latch so a silent standby relinquishes its held lease once. */
110
+ silentStandbyRelinquished = false;
93
111
  constructor(state, config) {
94
112
  super();
95
113
  this.state = state;
@@ -123,7 +141,43 @@ export class MultiMachineCoordinator extends EventEmitter {
123
141
  * such a machine is a deliberate un-mute (telegramPolling → true), not auto.
124
142
  */
125
143
  get isLeaseObserveOnly() {
126
- return this.config.multiMachine?.telegramPolling === false;
144
+ return this.resolvedLeaseRole() === 'observe-only';
145
+ }
146
+ /**
147
+ * multi-machine-lease-self-heal M3 — the first-class lease-participation mode,
148
+ * decoupled from the overloaded `telegramPolling` flag. Explicit
149
+ * `leaseSelfHeal.leaseRole` wins; otherwise derive from `telegramPolling`
150
+ * (===false ⇒ 'observe-only', else 'active') for back-compat. ('deferential'
151
+ * is an F4 concept handled separately; only 'observe-only' gates acquisition
152
+ * here so F4's down-preferred failover is never accidentally suppressed.)
153
+ */
154
+ resolvedLeaseRole() {
155
+ const explicit = this.config.multiMachine?.leaseSelfHeal?.leaseRole;
156
+ if (explicit === 'active' || explicit === 'observe-only' || explicit === 'deferential')
157
+ return explicit;
158
+ return this.config.multiMachine?.telegramPolling === false ? 'observe-only' : 'active';
159
+ }
160
+ /**
161
+ * F4 (preferred-awake, opt-in; null = off) — should THIS machine defer (not
162
+ * contend for the lease) right now? True iff a `preferredAwakeMachineId` is
163
+ * configured, it names a DIFFERENT machine (we are NOT the preferred), and that
164
+ * preferred machine is currently a HEALTHY holder. A machine that IS the
165
+ * preferred never defers; a non-preferred machine defers only while the preferred
166
+ * is healthy, so the preferred going down never strands coverage. Safe under any
167
+ * config (no tie-break override, so a divergent config degrades to the existing
168
+ * lower-machineId baseline rather than flapping).
169
+ */
170
+ shouldDeferToPreferred() {
171
+ const pref = this.config.multiMachine?.leaseSelfHeal?.preferredAwakeMachineId;
172
+ if (!pref || !this._identity || !this.leaseCoordinator)
173
+ return false; // F4 off / no identity
174
+ if (pref === this._identity.machineId)
175
+ return false; // WE are preferred → never defer
176
+ return this.preferredIsHealthy(pref);
177
+ }
178
+ /** F4 — the single shared health predicate: is the preferred machine a live holder? */
179
+ preferredIsHealthy(machineId) {
180
+ return this.leaseCoordinator?.isHolderHealthy(machineId) ?? false;
127
181
  }
128
182
  /** The coordination mode (default: 'primary-standby'). */
129
183
  get coordinationMode() {
@@ -264,6 +318,10 @@ export class MultiMachineCoordinator extends EventEmitter {
264
318
  clearInterval(this.heartbeatCheckTimer);
265
319
  this.heartbeatCheckTimer = null;
266
320
  }
321
+ if (this.tickWatchdogTimer) {
322
+ clearInterval(this.tickWatchdogTimer);
323
+ this.tickWatchdogTimer = null;
324
+ }
267
325
  this.leasePullStopped = true;
268
326
  if (this.leasePullTimer) {
269
327
  clearTimeout(this.leasePullTimer);
@@ -432,6 +490,14 @@ export class MultiMachineCoordinator extends EventEmitter {
432
490
  splitBrainState,
433
491
  protocolVersion: SEAMLESSNESS_PROTOCOL_VERSION,
434
492
  awakeMachineCount,
493
+ leaseTickWatchdog: this.leaseCoordinator
494
+ ? {
495
+ lastTickAgeMs: this.lastTickRunMonoMs > 0 ? this.monoNowMs() - this.lastTickRunMonoMs : -1,
496
+ reArmCount: this.watchdogReArmTimes.length,
497
+ disarmed: this.watchdogDisarmed,
498
+ }
499
+ : undefined,
500
+ preferredAwakeMachineId: this.config.multiMachine?.leaseSelfHeal?.preferredAwakeMachineId ?? null,
435
501
  };
436
502
  }
437
503
  /**
@@ -445,6 +511,10 @@ export class MultiMachineCoordinator extends EventEmitter {
445
511
  // Silent standby: do NOT acquire — only observe the primary's lease.
446
512
  this.reconcileRoleToLease('lease-init-observe-only');
447
513
  }
514
+ else if (this.shouldDeferToPreferred()) {
515
+ // F4 — boot as a deferential standby to a healthy preferred peer (no contend).
516
+ this.reconcileRoleToLease('lease-init-defer-preferred');
517
+ }
448
518
  else {
449
519
  await this.leaseCoordinator.acquireIfEligible();
450
520
  this.reconcileRoleToLease('lease-init');
@@ -477,26 +547,50 @@ export class MultiMachineCoordinator extends EventEmitter {
477
547
  if (!this.leaseCoordinator || this.leaseTicking)
478
548
  return;
479
549
  this.leaseTicking = true;
550
+ this.leaseTickStartMonoMs = this.monoNowMs(); // F1b — for the watchdog's ceiling-gated guard reset
480
551
  try {
481
552
  if (this.isLeaseObserveOnly) {
553
+ // F3 (silentStandbyRelinquish, DARK) — LEVEL-TRIGGERED: a silent standby
554
+ // that is STILL the named holder (the 2026-06-19 zombie: muted while
555
+ // holding epoch N) relinquishes + broadcasts a signed tombstone ONCE per
556
+ // incarnation, so peers stop deferring to it. Config flips need a restart,
557
+ // and the zombie is a persisted prior-process record, so this fires on the
558
+ // observe-only tick — not on an (absent) transition event.
559
+ if (this.config.multiMachine?.leaseSelfHeal?.silentStandbyRelinquish?.enabled &&
560
+ !this.silentStandbyRelinquished &&
561
+ this._identity &&
562
+ this.leaseCoordinator.currentHolder() === this._identity.machineId) {
563
+ this.silentStandbyRelinquished = true;
564
+ await this.withTickTimeout('relinquishAndBroadcast', () => this.leaseCoordinator.relinquishAndBroadcast());
565
+ }
482
566
  // Silent standby: never acquire/renew — just reconcile role to the
483
567
  // observed holder (effectiveView folds the primary's broadcast lease).
484
568
  this.reconcileRoleToLease('lease-tick-observe-only');
485
569
  }
486
570
  else if (this.leaseCoordinator.holdsLease()) {
487
- await this.leaseCoordinator.renew();
571
+ // F1a — bounded await: a hung renew can never wedge the tick.
572
+ await this.withTickTimeout('renew', () => this.leaseCoordinator.renew());
488
573
  this.reconcileRoleToLease('lease-tick');
489
574
  }
575
+ else if (this.shouldDeferToPreferred()) {
576
+ // F4 (preferred-awake, opt-in) — we are NON-preferred and observe the
577
+ // preferred machine holding a HEALTHY lease: defer, do not contend. If the
578
+ // preferred goes down/unhealthy, shouldDeferToPreferred() flips false and we
579
+ // acquire normally next tick (no coverage stranding, no flap — a deferential
580
+ // machine creates no contention to leapfrog, so no agreement-gossip needed).
581
+ this.reconcileRoleToLease('lease-tick-defer-preferred');
582
+ }
490
583
  else {
491
- await this.leaseCoordinator.acquireIfEligible();
584
+ await this.withTickTimeout('acquireIfEligible', () => this.leaseCoordinator.acquireIfEligible());
492
585
  this.reconcileRoleToLease('lease-tick');
493
586
  }
494
587
  }
495
588
  catch {
496
- // @silent-fallback-ok — a tick failure is retried next interval
589
+ // @silent-fallback-ok — a tick failure (incl. a bounded-await timeout) is retried next interval
497
590
  }
498
591
  finally {
499
592
  this.leaseTicking = false;
593
+ this.leaseTickStartMonoMs = 0;
500
594
  }
501
595
  }
502
596
  /** Bring _role + read-only into line with whether we hold the lease. */
@@ -577,8 +671,10 @@ export class MultiMachineCoordinator extends EventEmitter {
577
671
  return;
578
672
  }
579
673
  this.leasePulling = true;
674
+ this.leasePullStartMonoMs = this.monoNowMs(); // F1b — watchdog ceiling-gated guard reset
580
675
  try {
581
- await this.leaseCoordinator.pullFromPeers();
676
+ // F1a — bounded await: a hung peer pull can never wedge the pull loop.
677
+ await this.withTickTimeout('pullFromPeers', () => this.leaseCoordinator.pullFromPeers());
582
678
  // Only reconcile role / surface split-brain when a peer lease was actually
583
679
  // OBSERVED. A solo machine (no peers, or no peer lease seen this boot) must
584
680
  // NEVER be demoted by the pull loop on a transient self-lease lapse — that
@@ -610,10 +706,11 @@ export class MultiMachineCoordinator extends EventEmitter {
610
706
  }
611
707
  }
612
708
  catch {
613
- // @silent-fallback-ok — a pull failure is retried next tick
709
+ // @silent-fallback-ok — a pull failure (incl. a bounded-await timeout) is retried next tick
614
710
  }
615
711
  finally {
616
712
  this.leasePulling = false;
713
+ this.leasePullStartMonoMs = 0;
617
714
  arm();
618
715
  }
619
716
  }
@@ -790,6 +887,109 @@ export class MultiMachineCoordinator extends EventEmitter {
790
887
  }
791
888
  }
792
889
  }
890
+ // ── multi-machine-lease-self-heal F1 — tick self-heal ─────────────
891
+ /** Monotonic milliseconds (NTP-step / sleep-resume immune), like LeaseCoordinator §L−1. */
892
+ monoNowMs() {
893
+ return Number(process.hrtime.bigint() / 1000000n);
894
+ }
895
+ /** Resolved F1 watchdog config (defaults baked in; read live so disable needs no restart). */
896
+ get tickWatchdogCfg() {
897
+ const w = this.config.multiMachine?.leaseSelfHeal?.tickWatchdog;
898
+ const staleFactor = Math.max(2, w?.staleFactorMissedTicks ?? 5);
899
+ return {
900
+ enabled: w?.enabled ?? true,
901
+ staleMs: HEARTBEAT_CHECK_INTERVAL_MS * staleFactor,
902
+ awaitTimeoutMs: Math.max(1000, w?.awaitTimeoutMs ?? 20_000),
903
+ maxReArmsPerHour: Math.max(2, w?.maxReArmsPerHour ?? 6),
904
+ };
905
+ }
906
+ /**
907
+ * F1a — bound a tick-path network await so a never-settling call (the proven
908
+ * 2026-06-19 freeze: a hung fetch left `leaseTicking` stuck true forever) can
909
+ * NEVER hang the tick. ALL tick-path awaits route through this one helper, so
910
+ * the "no unbounded tick await" invariant is structural + grep-auditable.
911
+ */
912
+ withTickTimeout(label, fn) {
913
+ const ms = this.tickWatchdogCfg.awaitTimeoutMs;
914
+ return new Promise((resolve, reject) => {
915
+ let settled = false;
916
+ const t = setTimeout(() => {
917
+ if (settled)
918
+ return;
919
+ settled = true;
920
+ reject(new Error(`tick-await timeout after ${ms}ms: ${label}`));
921
+ }, ms);
922
+ if (typeof t.unref === 'function')
923
+ t.unref();
924
+ fn().then((v) => { if (!settled) {
925
+ settled = true;
926
+ clearTimeout(t);
927
+ resolve(v);
928
+ } }, (e) => { if (!settled) {
929
+ settled = true;
930
+ clearTimeout(t);
931
+ reject(e);
932
+ } });
933
+ });
934
+ }
935
+ /**
936
+ * F1b — the monotonic-clocked tick watchdog. Detects a stalled main tick loop
937
+ * (lost timer / stuck reentrancy guard) and re-arms it. SAFE-BY-CONSTRUCTION:
938
+ * never crashes (try/catch), never touches authority/epoch/suspend state, only
939
+ * resets a reentrancy guard whose in-flight tick is ALSO older than the ceiling
940
+ * (so a legitimately-slow live tick is never preempted), reads `enabled` live,
941
+ * and self-disarms (one DegradationReporter signal) if it re-arms too often.
942
+ * A true event-loop stall freezes this timer too — that case is layer-2 (the
943
+ * out-of-process fleet/launchd watchdog), as documented in the spec.
944
+ */
945
+ runTickWatchdog() {
946
+ try {
947
+ const cfg = this.tickWatchdogCfg;
948
+ if (!cfg.enabled || this.watchdogDisarmed)
949
+ return;
950
+ // No lease coordinator (solo / non-git mesh) ⇒ nothing to self-heal.
951
+ if (!this.leaseCoordinator)
952
+ return;
953
+ const now = this.monoNowMs();
954
+ // lastTickRunMonoMs is stamped at the TOP of checkHeartbeatAndAct; if it
955
+ // has not advanced within the stale window, the main loop is stalled.
956
+ if (this.lastTickRunMonoMs > 0 && now - this.lastTickRunMonoMs <= cfg.staleMs)
957
+ return;
958
+ // Ceiling-gated guard reset: only clear a guard whose in-flight tick is
959
+ // ALSO older than the ceiling (a stuck guard, not a slow-but-live tick).
960
+ if (this.leaseTicking && this.leaseTickStartMonoMs > 0 && now - this.leaseTickStartMonoMs > cfg.staleMs) {
961
+ this.leaseTicking = false;
962
+ }
963
+ if (this.leasePulling && this.leasePullStartMonoMs > 0 && now - this.leasePullStartMonoMs > cfg.staleMs) {
964
+ this.leasePulling = false;
965
+ }
966
+ // Re-arm the main monitor (clears + recreates the interval).
967
+ this.startHeartbeatMonitor();
968
+ this.lastTickRunMonoMs = now; // reset so we don't re-fire next tick on the same stall
969
+ // Self-disarm bookkeeping (rolling 1h window).
970
+ this.watchdogReArmTimes.push(now);
971
+ const hourAgo = now - 3_600_000;
972
+ this.watchdogReArmTimes = this.watchdogReArmTimes.filter((t) => t >= hourAgo);
973
+ console.log(`[MultiMachine] lease-tick watchdog: stalled >${cfg.staleMs}ms — re-armed (${this.watchdogReArmTimes.length}/${cfg.maxReArmsPerHour} this hour)`);
974
+ this.emit('tickStallRecovered', { reArmCount: this.watchdogReArmTimes.length });
975
+ if (this.watchdogReArmTimes.length > cfg.maxReArmsPerHour) {
976
+ this.watchdogDisarmed = true;
977
+ console.error('[MultiMachine] lease-tick watchdog SELF-DISARMED — re-arming too often; the tick itself is the incident');
978
+ DegradationReporter.getInstance().report({
979
+ feature: 'MultiMachine.leaseTickWatchdog',
980
+ primary: 'The lease-tick self-heal watchdog re-arms a stalled coordinator tick',
981
+ fallback: `Watchdog re-armed >${cfg.maxReArmsPerHour}×/hour and has self-disarmed; the lease tick is repeatedly stalling`,
982
+ reason: 'A persistently-stalling lease tick (transport hang / event-loop pressure) the in-process watchdog cannot durably fix',
983
+ impact: 'The awake-machine election may be unstable; investigate the coordinator/transport. The out-of-process fleet watchdog remains the backstop.',
984
+ });
985
+ }
986
+ }
987
+ catch (err) {
988
+ // @silent-fallback-ok — the watchdog must NEVER throw out of its interval
989
+ // callback (an uncaughtException would turn a partial wedge into a crash).
990
+ console.error(`[MultiMachine] tick watchdog error (ignored): ${err instanceof Error ? err.message : String(err)}`);
991
+ }
992
+ }
793
993
  /**
794
994
  * Start periodic heartbeat monitoring (all machines).
795
995
  */
@@ -803,11 +1003,26 @@ export class MultiMachineCoordinator extends EventEmitter {
803
1003
  if (this.heartbeatCheckTimer.unref) {
804
1004
  this.heartbeatCheckTimer.unref();
805
1005
  }
1006
+ // F1b — arm the independent tick watchdog ONCE (shorter 60s cadence so
1007
+ // detection latency isn't gated by the 2-min main cadence). startHeartbeat-
1008
+ // Monitor is also called BY the watchdog to re-arm the main timer, so guard
1009
+ // against stacking a second watchdog on re-arm.
1010
+ if (!this.tickWatchdogTimer) {
1011
+ this.tickWatchdogTimer = setInterval(() => {
1012
+ this.runTickWatchdog();
1013
+ }, TICK_WATCHDOG_INTERVAL_MS);
1014
+ if (this.tickWatchdogTimer.unref)
1015
+ this.tickWatchdogTimer.unref();
1016
+ }
806
1017
  }
807
1018
  /**
808
1019
  * Check the heartbeat and take action if needed.
809
1020
  */
810
1021
  checkHeartbeatAndAct() {
1022
+ // F1b — stamp the monotonic liveness mark FIRST, before any early-return, so
1023
+ // a solo / no-leaseCoordinator agent still advances it and the watchdog never
1024
+ // re-arms there (genuine no-op on single-machine agents).
1025
+ this.lastTickRunMonoMs = this.monoNowMs();
811
1026
  if (!this._identity)
812
1027
  return;
813
1028
  // Independent mode: no failover/demotion logic