instar 1.3.633 → 1.3.635
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +59 -4
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +18 -1
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/LeaseCoordinator.d.ts +7 -0
- package/dist/core/LeaseCoordinator.d.ts.map +1 -1
- package/dist/core/LeaseCoordinator.js +9 -0
- package/dist/core/LeaseCoordinator.js.map +1 -1
- package/dist/core/MachinePoolRegistry.d.ts +8 -0
- package/dist/core/MachinePoolRegistry.d.ts.map +1 -1
- package/dist/core/MachinePoolRegistry.js +1 -0
- package/dist/core/MachinePoolRegistry.js.map +1 -1
- package/dist/core/MultiMachineCoordinator.d.ts +69 -0
- package/dist/core/MultiMachineCoordinator.d.ts.map +1 -1
- package/dist/core/MultiMachineCoordinator.js +183 -0
- package/dist/core/MultiMachineCoordinator.js.map +1 -1
- package/dist/core/churnBreaker.d.ts +68 -0
- package/dist/core/churnBreaker.d.ts.map +1 -0
- package/dist/core/churnBreaker.js +98 -0
- package/dist/core/churnBreaker.js.map +1 -0
- package/dist/core/clockSkewAlarm.d.ts +61 -0
- package/dist/core/clockSkewAlarm.d.ts.map +1 -0
- package/dist/core/clockSkewAlarm.js +68 -0
- package/dist/core/clockSkewAlarm.js.map +1 -0
- package/dist/core/configCoherence.d.ts +39 -0
- package/dist/core/configCoherence.d.ts.map +1 -0
- package/dist/core/configCoherence.js +52 -0
- package/dist/core/configCoherence.js.map +1 -0
- package/dist/core/leaseLiveness.d.ts +42 -0
- package/dist/core/leaseLiveness.d.ts.map +1 -0
- package/dist/core/leaseLiveness.js +34 -0
- package/dist/core/leaseLiveness.js.map +1 -0
- package/dist/core/pollIntent.d.ts +65 -0
- package/dist/core/pollIntent.d.ts.map +1 -0
- package/dist/core/pollIntent.js +112 -0
- package/dist/core/pollIntent.js.map +1 -0
- package/dist/core/pollerCount.d.ts +61 -0
- package/dist/core/pollerCount.d.ts.map +1 -0
- package/dist/core/pollerCount.js +62 -0
- package/dist/core/pollerCount.js.map +1 -0
- package/dist/core/types.d.ts +57 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/lifeline/TelegramLifeline.d.ts +18 -0
- package/dist/lifeline/TelegramLifeline.d.ts.map +1 -1
- package/dist/lifeline/TelegramLifeline.js +99 -0
- package/dist/lifeline/TelegramLifeline.js.map +1 -1
- package/dist/lifeline/pollDecision.d.ts +47 -0
- package/dist/lifeline/pollDecision.d.ts.map +1 -0
- package/dist/lifeline/pollDecision.js +50 -0
- package/dist/lifeline/pollDecision.js.map +1 -0
- package/dist/monitoring/CommitmentTracker.d.ts +15 -0
- package/dist/monitoring/CommitmentTracker.d.ts.map +1 -1
- package/dist/monitoring/CommitmentTracker.js +59 -22
- package/dist/monitoring/CommitmentTracker.js.map +1 -1
- package/dist/monitoring/DegradationReporter.d.ts +10 -0
- package/dist/monitoring/DegradationReporter.d.ts.map +1 -1
- package/dist/monitoring/DegradationReporter.js +45 -17
- package/dist/monitoring/DegradationReporter.js.map +1 -1
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +17 -0
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +49 -49
- package/upgrades/1.3.634.md +59 -0
- package/upgrades/1.3.635.md +53 -0
- package/upgrades/side-effects/commitment-tracker-verify-batches-saves.md +109 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b1-lifeline-consumer.md +43 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b1-poll-decision.md +38 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b1-poll-intent-producer.md +37 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b2-churn-breaker.md +42 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b3-renew-timer.md +43 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b4-clockskew-alarm-core.md +25 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b4-d9-freeze-vs-skew.md +71 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b4-skew-immune-liveness.md +41 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b5-poller-count-route.md +24 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b5-poller-count.md +38 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-b5-pollingactive-heartbeat.md +38 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-ci-housekeeping.md +17 -0
- package/upgrades/side-effects/mm-lease-poll-robustness-config-coherence.md +23 -0
- package/upgrades/1.3.633.md +0 -37
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* B4 (multimachine-lease-poll-robustness, Decision 9) — the clock-skew alarm
|
|
3
|
+
* decision.
|
|
4
|
+
*
|
|
5
|
+
* The mesh RPC rejects a signed envelope whose timestamp is >30s off the
|
|
6
|
+
* receiver's clock (`MeshRpc.verifyEnvelope`). When two machines' clocks drift
|
|
7
|
+
* past that, the cross-machine handshake silently breaks (the 2026-06-20
|
|
8
|
+
* post-reboot incident: a transient skew 403'd every lease/heartbeat RPC, the lease
|
|
9
|
+
* couldn't settle, and nobody was told). This decides whether to raise an
|
|
10
|
+
* EARLY-WARNING — at a margin BELOW the 30s reject cliff — so an operator hears
|
|
11
|
+
* about drift before the handshake fails, not after.
|
|
12
|
+
*
|
|
13
|
+
* Two N=2 subtleties this encodes:
|
|
14
|
+
* - Attribution: with only two machines and no third reference, a measured
|
|
15
|
+
* offset is RELATIVE — each sees the other as skewed. So each machine checks
|
|
16
|
+
* its OWN NTP sync and, when ITS clock is unsynced, blames ITSELF rather than
|
|
17
|
+
* finger-pointing the peer (Decision 9). Only when our own clock is verified
|
|
18
|
+
* synced do we point at the peer.
|
|
19
|
+
* - Hysteresis: the measured offset is a noisy signal near the threshold;
|
|
20
|
+
* alarm at `alarmThresholdMs`, clear only below `clearThresholdMs`, so it
|
|
21
|
+
* doesn't flap the attention surface.
|
|
22
|
+
*
|
|
23
|
+
* Pure + deterministic → fully unit-testable. SIGNAL only — never widens the
|
|
24
|
+
* MeshRpc reject (replay-safety) and never gates; it raises an advisory alarm.
|
|
25
|
+
*/
|
|
26
|
+
export function evaluateClockSkew(i) {
|
|
27
|
+
const mag = Math.abs(i.observedOffsetMs);
|
|
28
|
+
// Hysteresis: once alarming, stay until below the clear threshold; otherwise
|
|
29
|
+
// only start alarming at/above the alarm threshold.
|
|
30
|
+
const alarming = i.currentlyAlarming ? mag >= i.clearThresholdMs : mag >= i.alarmThresholdMs;
|
|
31
|
+
if (!alarming) {
|
|
32
|
+
return { alarming: false, blame: 'unknown', reason: `clock offset ${Math.round(mag)}ms within tolerance` };
|
|
33
|
+
}
|
|
34
|
+
// Freeze-vs-skew disambiguation (the 2026-06-20 live lesson). If THIS machine's
|
|
35
|
+
// event loop just starved, a large apparent offset is most likely the freeze
|
|
36
|
+
// making pre-freeze timestamps look stale — NOT genuine peer clock drift. Defer
|
|
37
|
+
// to the starvation signal (which has its own surface) and SUPPRESS the peer
|
|
38
|
+
// alarm. Safe direction: a persistent genuine skew re-alarms once the freeze
|
|
39
|
+
// window ages out; we never raise a misleading "peer skew" during our own freeze.
|
|
40
|
+
const freshFreeze = i.recentLocalStarvationAgeMs !== undefined &&
|
|
41
|
+
i.recentLocalStarvationAgeMs <= (i.starvationFreshnessMs ?? 30_000);
|
|
42
|
+
if (freshFreeze) {
|
|
43
|
+
return {
|
|
44
|
+
alarming: false,
|
|
45
|
+
blame: 'local-freeze',
|
|
46
|
+
reason: `clock offset ${Math.round(mag)}ms coincides with a LOCAL event-loop starvation ${Math.round(i.recentLocalStarvationAgeMs)}ms ago — apparent staleness is freeze-induced (pre-freeze timestamps verified post-freeze), not peer skew; deferring to the starvation signal`,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
// Attribution (N=2). If our own clock is unsynced, the fault is plausibly OURS
|
|
50
|
+
// — blame self. If ours is verified synced, point at the peer. If we couldn't
|
|
51
|
+
// probe our own sync, stay 'unknown' (never a confident finger-point).
|
|
52
|
+
let blame;
|
|
53
|
+
let reason;
|
|
54
|
+
if (i.ownNtpSynced === false) {
|
|
55
|
+
blame = 'self';
|
|
56
|
+
reason = `clock offset ${Math.round(mag)}ms AND my own clock is not NTP-synced — fix my clock`;
|
|
57
|
+
}
|
|
58
|
+
else if (i.ownNtpSynced === true) {
|
|
59
|
+
blame = 'peer';
|
|
60
|
+
reason = `clock offset ${Math.round(mag)}ms; my clock is NTP-synced, so the peer's clock is likely drifting — mesh RPC will start failing past ${30}s`;
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
blame = 'unknown';
|
|
64
|
+
reason = `clock offset ${Math.round(mag)}ms with one of us drifting (own NTP status unknown) — mesh RPC at risk`;
|
|
65
|
+
}
|
|
66
|
+
return { alarming: true, blame, reason };
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=clockSkewAlarm.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"clockSkewAlarm.js","sourceRoot":"","sources":["../../src/core/clockSkewAlarm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAuCH,MAAM,UAAU,iBAAiB,CAAC,CAAkB;IAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;IACzC,6EAA6E;IAC7E,oDAAoD;IACpD,MAAM,QAAQ,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,gBAAgB,CAAC;IAC7F,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC;IAC7G,CAAC;IACD,gFAAgF;IAChF,6EAA6E;IAC7E,gFAAgF;IAChF,6EAA6E;IAC7E,6EAA6E;IAC7E,kFAAkF;IAClF,MAAM,WAAW,GACf,CAAC,CAAC,0BAA0B,KAAK,SAAS;QAC1C,CAAC,CAAC,0BAA0B,IAAI,CAAC,CAAC,CAAC,qBAAqB,IAAI,MAAM,CAAC,CAAC;IACtE,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO;YACL,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,cAAc;YACrB,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,mDAAmD,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,0BAA2B,CAAC,+IAA+I;SACnR,CAAC;IACJ,CAAC;IACD,+EAA+E;IAC/E,8EAA8E;IAC9E,uEAAuE;IACvE,IAAI,KAAgB,CAAC;IACrB,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC,CAAC,YAAY,KAAK,KAAK,EAAE,CAAC;QAC7B,KAAK,GAAG,MAAM,CAAC;QACf,MAAM,GAAG,gBAAgB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,sDAAsD,CAAC;IACjG,CAAC;SAAM,IAAI,CAAC,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QACnC,KAAK,GAAG,MAAM,CAAC;QACf,MAAM,GAAG,gBAAgB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,yGAAyG,EAAE,GAAG,CAAC;IACzJ,CAAC;SAAM,CAAC;QACN,KAAK,GAAG,SAAS,CAAC;QAClB,MAAM,GAAG,gBAAgB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,wEAAwE,CAAC;IACnH,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3C,CAAC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 2 #7 (multimachine-lease-poll-robustness audit) — startup config-coherence
|
|
3
|
+
* checks. Surfaces incoherent multi-machine config combinations that silently
|
|
4
|
+
* degrade behavior (the 2026-06-20 audit: meshTransport disabled while session
|
|
5
|
+
* transfer is live left the pair on single-rope fragility under transfer load —
|
|
6
|
+
* the worst-of-both state my morning band-aid created).
|
|
7
|
+
*
|
|
8
|
+
* SIGNAL only — these are WARNINGS, never a startup REJECT. A hard reject would
|
|
9
|
+
* refuse fleet boot on a config the audit showed is the SHIPPED default in some
|
|
10
|
+
* combinations (F-CLAMP1: default TTL 60s < tick 120s would itself trip a naive
|
|
11
|
+
* invariant). So the checker returns warnings the caller logs (+ optionally an
|
|
12
|
+
* Attention item); it never throws.
|
|
13
|
+
*
|
|
14
|
+
* Pure + deterministic → fully unit-testable.
|
|
15
|
+
*/
|
|
16
|
+
export interface ConfigCoherenceWarning {
|
|
17
|
+
code: string;
|
|
18
|
+
message: string;
|
|
19
|
+
}
|
|
20
|
+
interface MultiMachineLike {
|
|
21
|
+
meshTransport?: {
|
|
22
|
+
enabled?: boolean;
|
|
23
|
+
priorities?: Record<string, number>;
|
|
24
|
+
};
|
|
25
|
+
sessionPool?: {
|
|
26
|
+
stage?: string;
|
|
27
|
+
enabled?: boolean;
|
|
28
|
+
};
|
|
29
|
+
leaseTtlMs?: number;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* @param mm the resolved `multiMachine` config block.
|
|
33
|
+
* @param isMultiMachine whether this agent actually runs multi-machine (has a
|
|
34
|
+
* machine identity / a peer). On a single-machine agent these combinations are
|
|
35
|
+
* harmless no-ops, so we don't warn.
|
|
36
|
+
*/
|
|
37
|
+
export declare function checkMultiMachineConfigCoherence(mm: MultiMachineLike | undefined, isMultiMachine: boolean): ConfigCoherenceWarning[];
|
|
38
|
+
export {};
|
|
39
|
+
//# sourceMappingURL=configCoherence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"configCoherence.d.ts","sourceRoot":"","sources":["../../src/core/configCoherence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,gBAAgB;IACxB,aAAa,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC;IAC3E,WAAW,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACpD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,wBAAgB,gCAAgC,CAC9C,EAAE,EAAE,gBAAgB,GAAG,SAAS,EAChC,cAAc,EAAE,OAAO,GACtB,sBAAsB,EAAE,CAgC1B"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 2 #7 (multimachine-lease-poll-robustness audit) — startup config-coherence
|
|
3
|
+
* checks. Surfaces incoherent multi-machine config combinations that silently
|
|
4
|
+
* degrade behavior (the 2026-06-20 audit: meshTransport disabled while session
|
|
5
|
+
* transfer is live left the pair on single-rope fragility under transfer load —
|
|
6
|
+
* the worst-of-both state my morning band-aid created).
|
|
7
|
+
*
|
|
8
|
+
* SIGNAL only — these are WARNINGS, never a startup REJECT. A hard reject would
|
|
9
|
+
* refuse fleet boot on a config the audit showed is the SHIPPED default in some
|
|
10
|
+
* combinations (F-CLAMP1: default TTL 60s < tick 120s would itself trip a naive
|
|
11
|
+
* invariant). So the checker returns warnings the caller logs (+ optionally an
|
|
12
|
+
* Attention item); it never throws.
|
|
13
|
+
*
|
|
14
|
+
* Pure + deterministic → fully unit-testable.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* @param mm the resolved `multiMachine` config block.
|
|
18
|
+
* @param isMultiMachine whether this agent actually runs multi-machine (has a
|
|
19
|
+
* machine identity / a peer). On a single-machine agent these combinations are
|
|
20
|
+
* harmless no-ops, so we don't warn.
|
|
21
|
+
*/
|
|
22
|
+
export function checkMultiMachineConfigCoherence(mm, isMultiMachine) {
|
|
23
|
+
const warnings = [];
|
|
24
|
+
if (!mm || !isMultiMachine)
|
|
25
|
+
return warnings;
|
|
26
|
+
// 1) Mesh transport disabled while session transfer is LIVE — the worst-of-both
|
|
27
|
+
// state: the pool actively moves sessions but the transport reverts to a
|
|
28
|
+
// single (flaky) rope, reintroducing the lease flap under transfer load.
|
|
29
|
+
const transferLive = mm.sessionPool?.enabled !== false && mm.sessionPool?.stage === 'live-transfer';
|
|
30
|
+
if (mm.meshTransport?.enabled === false && transferLive) {
|
|
31
|
+
warnings.push({
|
|
32
|
+
code: 'mesh-off-while-live-transfer',
|
|
33
|
+
message: 'multiMachine.meshTransport.enabled=false while sessionPool.stage=live-transfer on a multi-machine agent — ' +
|
|
34
|
+
'session transfer is active but the mesh is single-rope, reintroducing the lease flap. Set meshTransport.enabled=true.',
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
// 2) Mesh rope priorities must be DISTINCT positive integers (a tie makes rope
|
|
38
|
+
// selection nondeterministic).
|
|
39
|
+
const pri = mm.meshTransport?.priorities;
|
|
40
|
+
if (pri && typeof pri === 'object') {
|
|
41
|
+
const vals = Object.values(pri).filter((v) => typeof v === 'number');
|
|
42
|
+
const bad = vals.filter((v) => !Number.isInteger(v) || v <= 0);
|
|
43
|
+
if (bad.length > 0) {
|
|
44
|
+
warnings.push({ code: 'mesh-priority-nonpositive', message: `multiMachine.meshTransport.priorities must be positive integers; got ${JSON.stringify(bad)}.` });
|
|
45
|
+
}
|
|
46
|
+
if (new Set(vals).size !== vals.length) {
|
|
47
|
+
warnings.push({ code: 'mesh-priority-collision', message: `multiMachine.meshTransport.priorities has duplicate values (${JSON.stringify(vals)}) — rope selection is nondeterministic.` });
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return warnings;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=configCoherence.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"configCoherence.js","sourceRoot":"","sources":["../../src/core/configCoherence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAaH;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAC9C,EAAgC,EAChC,cAAuB;IAEvB,MAAM,QAAQ,GAA6B,EAAE,CAAC;IAC9C,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc;QAAE,OAAO,QAAQ,CAAC;IAE5C,gFAAgF;IAChF,4EAA4E;IAC5E,4EAA4E;IAC5E,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,KAAK,eAAe,CAAC;IACpG,IAAI,EAAE,CAAC,aAAa,EAAE,OAAO,KAAK,KAAK,IAAI,YAAY,EAAE,CAAC;QACxD,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,8BAA8B;YACpC,OAAO,EACL,4GAA4G;gBAC5G,uHAAuH;SAC1H,CAAC,CAAC;IACL,CAAC;IAED,+EAA+E;IAC/E,kCAAkC;IAClC,MAAM,GAAG,GAAG,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC;IACzC,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,OAAO,EAAE,wEAAwE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAChK,CAAC;QACD,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YACvC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,yBAAyB,EAAE,OAAO,EAAE,+DAA+D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC;QAC5L,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* B4 (multimachine-lease-poll-robustness, Decision 10) — skew-immune peer
|
|
3
|
+
* liveness for the lease layer.
|
|
4
|
+
*
|
|
5
|
+
* The lease's `presumedDeadHolders` / `allPeersPresumedGone` historically derive
|
|
6
|
+
* liveness from the registry `lastSeen` — the PEER's OWN wall clock. Under clock
|
|
7
|
+
* skew (the 2026-06-20 post-reboot incident) a +Ns-fast peer looks MORE alive
|
|
8
|
+
* than it is (delaying a needed failover) and a −Ns-slow peer looks DEADER than
|
|
9
|
+
* it is (triggering a false failover → the flap). The skew-immune source is the
|
|
10
|
+
* router's OWN observation clock (`routerReceivedAt`), held by MachinePoolRegistry.
|
|
11
|
+
*
|
|
12
|
+
* This pure decision keeps the conservative direction throughout: when in doubt,
|
|
13
|
+
* a peer is NOT presumed dead (a wrongful "dead" verdict causes a takeover →
|
|
14
|
+
* split-brain, the worse failure). It is flag-gated; with `skewImmune:false` it is
|
|
15
|
+
* byte-for-byte the legacy `lastSeen`-threshold behavior.
|
|
16
|
+
*/
|
|
17
|
+
export interface PeerLivenessInputs {
|
|
18
|
+
/** Registry `lastSeen` parsed to ms (the peer's own wall clock). null/NaN ⇒ unknown. */
|
|
19
|
+
lastSeenMs: number | null;
|
|
20
|
+
/**
|
|
21
|
+
* Does the in-process MachinePoolRegistry hold a `routerReceivedAt` for this
|
|
22
|
+
* peer THIS incarnation? false ⇒ known-on-disk-but-not-yet-observed (a fresh
|
|
23
|
+
* boot before the first heartbeat) → the skew-immune source has no opinion yet,
|
|
24
|
+
* so we fall back to lastSeen rather than wrongly presume-dead a peer we simply
|
|
25
|
+
* haven't heard from yet (the convergence-review edge).
|
|
26
|
+
*/
|
|
27
|
+
routerObserved: boolean;
|
|
28
|
+
/** The registry's skew-immune online verdict (now − routerReceivedAt < failoverThreshold). */
|
|
29
|
+
routerOnline: boolean;
|
|
30
|
+
/** Caller's now (ms). */
|
|
31
|
+
nowMs: number;
|
|
32
|
+
/** Liveness horizon (ms). */
|
|
33
|
+
failoverThresholdMs: number;
|
|
34
|
+
/** Flag: use the skew-immune router source when it has an opinion. */
|
|
35
|
+
skewImmune: boolean;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* True iff the peer should be PRESUMED DEAD (eligible for the lease to act as if
|
|
39
|
+
* it is gone). Conservative: only on positive evidence of staleness.
|
|
40
|
+
*/
|
|
41
|
+
export declare function isPeerPresumedDead(i: PeerLivenessInputs): boolean;
|
|
42
|
+
//# sourceMappingURL=leaseLiveness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"leaseLiveness.d.ts","sourceRoot":"","sources":["../../src/core/leaseLiveness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,WAAW,kBAAkB;IACjC,wFAAwF;IACxF,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B;;;;;;OAMG;IACH,cAAc,EAAE,OAAO,CAAC;IACxB,8FAA8F;IAC9F,YAAY,EAAE,OAAO,CAAC;IACtB,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,6BAA6B;IAC7B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sEAAsE;IACtE,UAAU,EAAE,OAAO,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAWjE"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* B4 (multimachine-lease-poll-robustness, Decision 10) — skew-immune peer
|
|
3
|
+
* liveness for the lease layer.
|
|
4
|
+
*
|
|
5
|
+
* The lease's `presumedDeadHolders` / `allPeersPresumedGone` historically derive
|
|
6
|
+
* liveness from the registry `lastSeen` — the PEER's OWN wall clock. Under clock
|
|
7
|
+
* skew (the 2026-06-20 post-reboot incident) a +Ns-fast peer looks MORE alive
|
|
8
|
+
* than it is (delaying a needed failover) and a −Ns-slow peer looks DEADER than
|
|
9
|
+
* it is (triggering a false failover → the flap). The skew-immune source is the
|
|
10
|
+
* router's OWN observation clock (`routerReceivedAt`), held by MachinePoolRegistry.
|
|
11
|
+
*
|
|
12
|
+
* This pure decision keeps the conservative direction throughout: when in doubt,
|
|
13
|
+
* a peer is NOT presumed dead (a wrongful "dead" verdict causes a takeover →
|
|
14
|
+
* split-brain, the worse failure). It is flag-gated; with `skewImmune:false` it is
|
|
15
|
+
* byte-for-byte the legacy `lastSeen`-threshold behavior.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* True iff the peer should be PRESUMED DEAD (eligible for the lease to act as if
|
|
19
|
+
* it is gone). Conservative: only on positive evidence of staleness.
|
|
20
|
+
*/
|
|
21
|
+
export function isPeerPresumedDead(i) {
|
|
22
|
+
// PRIMARY — skew-immune router liveness, but ONLY when the router actually has
|
|
23
|
+
// an observation for this peer this incarnation. observed-but-stale ⇒ dead.
|
|
24
|
+
if (i.skewImmune && i.routerObserved) {
|
|
25
|
+
return !i.routerOnline;
|
|
26
|
+
}
|
|
27
|
+
// FALLBACK — legacy lastSeen threshold (flag off, OR peer not yet observed).
|
|
28
|
+
// Unknown/unparseable lastSeen ⇒ NOT dead (conservative; a takeover on a peer
|
|
29
|
+
// we can't measure is exactly the split-brain risk we refuse).
|
|
30
|
+
if (i.lastSeenMs == null || Number.isNaN(i.lastSeenMs))
|
|
31
|
+
return false;
|
|
32
|
+
return i.nowMs - i.lastSeenMs > i.failoverThresholdMs;
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=leaseLiveness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"leaseLiveness.js","sourceRoot":"","sources":["../../src/core/leaseLiveness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAuBH;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,CAAqB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QACrC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC;IACzB,CAAC;IACD,6EAA6E;IAC7E,8EAA8E;IAC9E,+DAA+D;IAC/D,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IACrE,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,mBAAmB,CAAC;AACxD,CAAC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* B1 (multimachine-lease-poll-robustness, Decisions 4/5/6) — the cross-process
|
|
3
|
+
* poll-intent file. The SERVER (which holds the fenced lease) writes its
|
|
4
|
+
* lease-derived poll intent here on every role transition; the LIFELINE (a
|
|
5
|
+
* separate process that owns the Telegram socket) reads it to decide whether to
|
|
6
|
+
* poll (via decidePollAction). This is the bridge that makes poll-ownership
|
|
7
|
+
* FOLLOW the lease at runtime instead of a static boot-time flag.
|
|
8
|
+
*
|
|
9
|
+
* Integrity (Decision 5): the record carries the writing server's `serverPid` +
|
|
10
|
+
* `bootId` + `ts`. The lifeline IGNORES an intent (→ "no current opinion", null)
|
|
11
|
+
* that is stale (older than a bound) or whose writer pid is dead — so a stale
|
|
12
|
+
* `{shouldPoll:true}` left on disk after a crash can NOT resurrect a poller, and a
|
|
13
|
+
* stale `{shouldPoll:false}` can NOT wrongly silence a live awake machine. The
|
|
14
|
+
* server writes `{shouldPoll:false}` at boot (before its role is known), so the
|
|
15
|
+
* default on-disk state is the safe one (mute). A graceful-shutdown mute is NOT
|
|
16
|
+
* relied upon: a crashed/exited writer is covered by the consumer's dead-pid +
|
|
17
|
+
* staleness gate (`effectivePollIntent` → null → the lifeline HOLDs).
|
|
18
|
+
*
|
|
19
|
+
* Threat model: local same-uid IPC (parity with TelegramPollOwnerLease) — never
|
|
20
|
+
* network-reachable. Atomic write (tmp + rename) so the lifeline never reads a
|
|
21
|
+
* torn record.
|
|
22
|
+
*/
|
|
23
|
+
export interface PollIntentRecord {
|
|
24
|
+
shouldPoll: boolean;
|
|
25
|
+
leaseEpoch: number;
|
|
26
|
+
role: 'awake' | 'standby';
|
|
27
|
+
serverPid: number;
|
|
28
|
+
bootId: string;
|
|
29
|
+
/** epoch ms when written. */
|
|
30
|
+
ts: number;
|
|
31
|
+
}
|
|
32
|
+
/** Absolute path to the intent file under a state dir (`.instar`). */
|
|
33
|
+
export declare function pollIntentPath(stateDir: string): string;
|
|
34
|
+
/** Atomic write (tmp + rename). Never throws to the caller's hot path — callers wrap. */
|
|
35
|
+
export declare function writePollIntent(stateDir: string, rec: PollIntentRecord): void;
|
|
36
|
+
/** Read + parse the intent record. Returns null on missing/corrupt. */
|
|
37
|
+
export declare function readPollIntent(stateDir: string): PollIntentRecord | null;
|
|
38
|
+
export interface PollIntentFreshnessInputs {
|
|
39
|
+
nowMs: number;
|
|
40
|
+
/** Max age before the record is "no current opinion". */
|
|
41
|
+
maxStaleMs: number;
|
|
42
|
+
/** Is the writing server pid alive? (caller probes process liveness.) */
|
|
43
|
+
serverPidAlive: boolean;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The EFFECTIVE poll intent the lifeline should act on:
|
|
47
|
+
* - true / false — a fresh, live-writer record's shouldPoll.
|
|
48
|
+
* - null — "no current opinion": missing, stale (> maxStaleMs), or the
|
|
49
|
+
* writing server pid is dead. decidePollAction treats null as
|
|
50
|
+
* HOLD (never a surprise stop, never a blind start).
|
|
51
|
+
*
|
|
52
|
+
* Pure — the I/O (read + pid probe) is the caller's; this decides trust.
|
|
53
|
+
*/
|
|
54
|
+
export declare function effectivePollIntent(rec: PollIntentRecord | null, i: PollIntentFreshnessInputs): boolean | null;
|
|
55
|
+
export interface PollActiveRecord {
|
|
56
|
+
pollingActive: boolean;
|
|
57
|
+
pid: number;
|
|
58
|
+
ts: number;
|
|
59
|
+
}
|
|
60
|
+
export declare function pollActivePath(stateDir: string): string;
|
|
61
|
+
export declare function writePollActive(stateDir: string, pollingActive: boolean): void;
|
|
62
|
+
export declare function readPollActive(stateDir: string): PollActiveRecord | null;
|
|
63
|
+
/** True iff a pid is alive (probe). Used to discount a crashed writer's record. */
|
|
64
|
+
export declare function pidAlive(pid: number): boolean;
|
|
65
|
+
//# sourceMappingURL=pollIntent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pollIntent.d.ts","sourceRoot":"","sources":["../../src/core/pollIntent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAKH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,OAAO,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;CACZ;AAID,sEAAsE;AACtE,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED,yFAAyF;AACzF,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,gBAAgB,GAAG,IAAI,CAK7E;AAED,uEAAuE;AACvE,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAYxE;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,yDAAyD;IACzD,UAAU,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,cAAc,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,gBAAgB,GAAG,IAAI,EAC5B,CAAC,EAAE,yBAAyB,GAC3B,OAAO,GAAG,IAAI,CAKhB;AASD,MAAM,WAAW,gBAAgB;IAAG,aAAa,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;CAAE;AAEtF,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,GAAG,IAAI,CAK9E;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAUxE;AAED,mFAAmF;AACnF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAM7C"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* B1 (multimachine-lease-poll-robustness, Decisions 4/5/6) — the cross-process
|
|
3
|
+
* poll-intent file. The SERVER (which holds the fenced lease) writes its
|
|
4
|
+
* lease-derived poll intent here on every role transition; the LIFELINE (a
|
|
5
|
+
* separate process that owns the Telegram socket) reads it to decide whether to
|
|
6
|
+
* poll (via decidePollAction). This is the bridge that makes poll-ownership
|
|
7
|
+
* FOLLOW the lease at runtime instead of a static boot-time flag.
|
|
8
|
+
*
|
|
9
|
+
* Integrity (Decision 5): the record carries the writing server's `serverPid` +
|
|
10
|
+
* `bootId` + `ts`. The lifeline IGNORES an intent (→ "no current opinion", null)
|
|
11
|
+
* that is stale (older than a bound) or whose writer pid is dead — so a stale
|
|
12
|
+
* `{shouldPoll:true}` left on disk after a crash can NOT resurrect a poller, and a
|
|
13
|
+
* stale `{shouldPoll:false}` can NOT wrongly silence a live awake machine. The
|
|
14
|
+
* server writes `{shouldPoll:false}` at boot (before its role is known), so the
|
|
15
|
+
* default on-disk state is the safe one (mute). A graceful-shutdown mute is NOT
|
|
16
|
+
* relied upon: a crashed/exited writer is covered by the consumer's dead-pid +
|
|
17
|
+
* staleness gate (`effectivePollIntent` → null → the lifeline HOLDs).
|
|
18
|
+
*
|
|
19
|
+
* Threat model: local same-uid IPC (parity with TelegramPollOwnerLease) — never
|
|
20
|
+
* network-reachable. Atomic write (tmp + rename) so the lifeline never reads a
|
|
21
|
+
* torn record.
|
|
22
|
+
*/
|
|
23
|
+
import { writeFileSync, readFileSync, renameSync, existsSync } from 'node:fs';
|
|
24
|
+
import { join } from 'node:path';
|
|
25
|
+
const FILE = 'telegram-poll-intent.json';
|
|
26
|
+
/** Absolute path to the intent file under a state dir (`.instar`). */
|
|
27
|
+
export function pollIntentPath(stateDir) {
|
|
28
|
+
return join(stateDir, FILE);
|
|
29
|
+
}
|
|
30
|
+
/** Atomic write (tmp + rename). Never throws to the caller's hot path — callers wrap. */
|
|
31
|
+
export function writePollIntent(stateDir, rec) {
|
|
32
|
+
const p = pollIntentPath(stateDir);
|
|
33
|
+
const tmp = `${p}.tmp.${rec.serverPid}`;
|
|
34
|
+
writeFileSync(tmp, JSON.stringify(rec), 'utf8');
|
|
35
|
+
renameSync(tmp, p);
|
|
36
|
+
}
|
|
37
|
+
/** Read + parse the intent record. Returns null on missing/corrupt. */
|
|
38
|
+
export function readPollIntent(stateDir) {
|
|
39
|
+
const p = pollIntentPath(stateDir);
|
|
40
|
+
if (!existsSync(p))
|
|
41
|
+
return null;
|
|
42
|
+
try {
|
|
43
|
+
const rec = JSON.parse(readFileSync(p, 'utf8'));
|
|
44
|
+
if (typeof rec?.shouldPoll !== 'boolean' || typeof rec?.serverPid !== 'number' || typeof rec?.ts !== 'number') {
|
|
45
|
+
return null; // shape check — a partial/legacy record is "no opinion"
|
|
46
|
+
}
|
|
47
|
+
return rec;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* The EFFECTIVE poll intent the lifeline should act on:
|
|
55
|
+
* - true / false — a fresh, live-writer record's shouldPoll.
|
|
56
|
+
* - null — "no current opinion": missing, stale (> maxStaleMs), or the
|
|
57
|
+
* writing server pid is dead. decidePollAction treats null as
|
|
58
|
+
* HOLD (never a surprise stop, never a blind start).
|
|
59
|
+
*
|
|
60
|
+
* Pure — the I/O (read + pid probe) is the caller's; this decides trust.
|
|
61
|
+
*/
|
|
62
|
+
export function effectivePollIntent(rec, i) {
|
|
63
|
+
if (!rec)
|
|
64
|
+
return null;
|
|
65
|
+
if (!i.serverPidAlive)
|
|
66
|
+
return null; // writer is dead → its opinion is stale
|
|
67
|
+
if (i.nowMs - rec.ts > i.maxStaleMs)
|
|
68
|
+
return null; // too old → no current opinion
|
|
69
|
+
return rec.shouldPoll;
|
|
70
|
+
}
|
|
71
|
+
// ── lifeline-poll-active (B5 source — Decision 6) ───────────────────────────
|
|
72
|
+
// The LIFELINE (the process that actually owns the getUpdates socket) writes its
|
|
73
|
+
// REAL poll state here; B5's exactly-one-listener guard reads THIS (the truth),
|
|
74
|
+
// not the server's intent (a wish). Same local-same-uid IPC + atomic-write posture.
|
|
75
|
+
const POLL_ACTIVE_FILE = 'lifeline-poll-active.json';
|
|
76
|
+
export function pollActivePath(stateDir) {
|
|
77
|
+
return join(stateDir, POLL_ACTIVE_FILE);
|
|
78
|
+
}
|
|
79
|
+
export function writePollActive(stateDir, pollingActive) {
|
|
80
|
+
const p = pollActivePath(stateDir);
|
|
81
|
+
const tmp = `${p}.tmp.${process.pid}`;
|
|
82
|
+
writeFileSync(tmp, JSON.stringify({ pollingActive, pid: process.pid, ts: Date.now() }), 'utf8');
|
|
83
|
+
renameSync(tmp, p);
|
|
84
|
+
}
|
|
85
|
+
export function readPollActive(stateDir) {
|
|
86
|
+
const p = pollActivePath(stateDir);
|
|
87
|
+
if (!existsSync(p))
|
|
88
|
+
return null;
|
|
89
|
+
try {
|
|
90
|
+
const rec = JSON.parse(readFileSync(p, 'utf8'));
|
|
91
|
+
if (typeof rec?.pollingActive !== 'boolean' || typeof rec?.ts !== 'number')
|
|
92
|
+
return null;
|
|
93
|
+
return rec;
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/** True iff a pid is alive (probe). Used to discount a crashed writer's record. */
|
|
100
|
+
export function pidAlive(pid) {
|
|
101
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
102
|
+
return false;
|
|
103
|
+
try {
|
|
104
|
+
process.kill(pid, 0);
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
// EPERM = exists but not ours (still alive); ESRCH = gone.
|
|
109
|
+
return err.code === 'EPERM';
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=pollIntent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pollIntent.js","sourceRoot":"","sources":["../../src/core/pollIntent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC9E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAYjC,MAAM,IAAI,GAAG,2BAA2B,CAAC;AAEzC,sEAAsE;AACtE,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,OAAO,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,GAAqB;IACrE,MAAM,CAAC,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,GAAG,CAAC,SAAS,EAAE,CAAC;IACxC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;IAChD,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AACrB,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,CAAC,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAqB,CAAC;QACpE,IAAI,OAAO,GAAG,EAAE,UAAU,KAAK,SAAS,IAAI,OAAO,GAAG,EAAE,SAAS,KAAK,QAAQ,IAAI,OAAO,GAAG,EAAE,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC9G,OAAO,IAAI,CAAC,CAAC,wDAAwD;QACvE,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAUD;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CACjC,GAA4B,EAC5B,CAA4B;IAE5B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,IAAI,CAAC,CAAC,CAAC,cAAc;QAAE,OAAO,IAAI,CAAC,CAAC,wCAAwC;IAC5E,IAAI,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC,CAAC,+BAA+B;IACjF,OAAO,GAAG,CAAC,UAAU,CAAC;AACxB,CAAC;AAED,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,oFAAoF;AAEpF,MAAM,gBAAgB,GAAG,2BAA2B,CAAC;AAIrD,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,OAAO,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,aAAsB;IACtE,MAAM,CAAC,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;IACtC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAA6B,CAAC,EAAE,MAAM,CAAC,CAAC;IAC3H,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,CAAC,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAqB,CAAC;QACpE,IAAI,OAAO,GAAG,EAAE,aAAa,KAAK,SAAS,IAAI,OAAO,GAAG,EAAE,EAAE,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACxF,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,IAAI,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAAC,OAAO,IAAI,CAAC;IAAC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACtD,2DAA2D;QAC3D,OAAQ,GAA6B,CAAC,IAAI,KAAK,OAAO,CAAC;IACzD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* B5 (multimachine-lease-poll-robustness, Decision 11) — the exactly-one-listener
|
|
3
|
+
* decision.
|
|
4
|
+
*
|
|
5
|
+
* Telegram allows exactly ONE getUpdates poller per bot token. Zero pollers =
|
|
6
|
+
* silence (the 3.5h incident); ≥2 = a 409-conflict war + split handling. This
|
|
7
|
+
* pure decision answers "how many of my machines are polling?" THREE-valued so a
|
|
8
|
+
* dark peer never produces a false alarm:
|
|
9
|
+
* - ok exactly one FRESH poller observed, no peer dark/unknown.
|
|
10
|
+
* - dual ≥2 pollers positively observed, OR a local Telegram 409
|
|
11
|
+
* (partition-immune evidence of a 2nd poller even when the
|
|
12
|
+
* peer's heartbeat is dark — the case heartbeat-counting alone
|
|
13
|
+
* can't see).
|
|
14
|
+
* - silence zero pollers, and every peer is fresh + known (so it's a real
|
|
15
|
+
* zero, not a visibility gap).
|
|
16
|
+
* - indeterminate a peer is dark or reports an unknown poll state, so the exact
|
|
17
|
+
* count can't be confirmed — surface "can't confirm ingress",
|
|
18
|
+
* NEVER a false silence/ok alarm (the adversarial-review rule).
|
|
19
|
+
*
|
|
20
|
+
* Pure + deterministic → fully unit-testable. SIGNAL only (observe), never gates.
|
|
21
|
+
*/
|
|
22
|
+
export interface PollerObservation {
|
|
23
|
+
machineId: string;
|
|
24
|
+
/**
|
|
25
|
+
* Whether this machine's lifeline is ACTUALLY polling. `undefined` = an older
|
|
26
|
+
* peer that doesn't emit the field yet (mid-rollout) OR genuinely unknown —
|
|
27
|
+
* treated as a visibility gap (→ indeterminate), never as "not polling".
|
|
28
|
+
*/
|
|
29
|
+
pollingActive: boolean | undefined;
|
|
30
|
+
/** Heartbeat-fresh (not dark). A dark peer's poll state is unknowable. */
|
|
31
|
+
fresh: boolean;
|
|
32
|
+
}
|
|
33
|
+
export type PollerCountVerdict = 'ok' | 'dual' | 'silence' | 'indeterminate';
|
|
34
|
+
export interface PollerCountResult {
|
|
35
|
+
verdict: PollerCountVerdict;
|
|
36
|
+
/** Pollers positively observed (fresh AND pollingActive === true). */
|
|
37
|
+
activePollers: number;
|
|
38
|
+
/** True if any peer is dark or reports an unknown poll state. */
|
|
39
|
+
hasVisibilityGap: boolean;
|
|
40
|
+
/** One-line reason for the surface. */
|
|
41
|
+
reason: string;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* @param observations one row per pool machine (INCLUDING self — self is always
|
|
45
|
+
* fresh + a known poll state).
|
|
46
|
+
* @param localSaw409 did THIS machine's lifeline recently get a Telegram 409
|
|
47
|
+
* (someone else is polling the same token)? Partition-immune dual evidence.
|
|
48
|
+
*/
|
|
49
|
+
export declare function evaluatePollerCount(observations: PollerObservation[], localSaw409: boolean): PollerCountResult;
|
|
50
|
+
/**
|
|
51
|
+
* Adapter — evaluate the exactly-one-listener verdict directly over the pool's
|
|
52
|
+
* MachineCapacity rows (from `?scope=pool`). `online` is the freshness signal
|
|
53
|
+
* (a dark peer → not fresh → indeterminate), `pollingActive` the truth field
|
|
54
|
+
* (absent on an older peer → unknown). This is what a `/guards` surface calls.
|
|
55
|
+
*/
|
|
56
|
+
export declare function poolPollerVerdict(capacities: Array<{
|
|
57
|
+
machineId: string;
|
|
58
|
+
online?: boolean;
|
|
59
|
+
pollingActive?: boolean;
|
|
60
|
+
}>, localSaw409: boolean): PollerCountResult;
|
|
61
|
+
//# sourceMappingURL=pollerCount.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pollerCount.d.ts","sourceRoot":"","sources":["../../src/core/pollerCount.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,aAAa,EAAE,OAAO,GAAG,SAAS,CAAC;IACnC,0EAA0E;IAC1E,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,GAAG,MAAM,GAAG,SAAS,GAAG,eAAe,CAAC;AAE7E,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,kBAAkB,CAAC;IAC5B,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,iEAAiE;IACjE,gBAAgB,EAAE,OAAO,CAAC;IAC1B,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,YAAY,EAAE,iBAAiB,EAAE,EACjC,WAAW,EAAE,OAAO,GACnB,iBAAiB,CAyBnB;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,UAAU,EAAE,KAAK,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,aAAa,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACnF,WAAW,EAAE,OAAO,GACnB,iBAAiB,CAKnB"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* B5 (multimachine-lease-poll-robustness, Decision 11) — the exactly-one-listener
|
|
3
|
+
* decision.
|
|
4
|
+
*
|
|
5
|
+
* Telegram allows exactly ONE getUpdates poller per bot token. Zero pollers =
|
|
6
|
+
* silence (the 3.5h incident); ≥2 = a 409-conflict war + split handling. This
|
|
7
|
+
* pure decision answers "how many of my machines are polling?" THREE-valued so a
|
|
8
|
+
* dark peer never produces a false alarm:
|
|
9
|
+
* - ok exactly one FRESH poller observed, no peer dark/unknown.
|
|
10
|
+
* - dual ≥2 pollers positively observed, OR a local Telegram 409
|
|
11
|
+
* (partition-immune evidence of a 2nd poller even when the
|
|
12
|
+
* peer's heartbeat is dark — the case heartbeat-counting alone
|
|
13
|
+
* can't see).
|
|
14
|
+
* - silence zero pollers, and every peer is fresh + known (so it's a real
|
|
15
|
+
* zero, not a visibility gap).
|
|
16
|
+
* - indeterminate a peer is dark or reports an unknown poll state, so the exact
|
|
17
|
+
* count can't be confirmed — surface "can't confirm ingress",
|
|
18
|
+
* NEVER a false silence/ok alarm (the adversarial-review rule).
|
|
19
|
+
*
|
|
20
|
+
* Pure + deterministic → fully unit-testable. SIGNAL only (observe), never gates.
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* @param observations one row per pool machine (INCLUDING self — self is always
|
|
24
|
+
* fresh + a known poll state).
|
|
25
|
+
* @param localSaw409 did THIS machine's lifeline recently get a Telegram 409
|
|
26
|
+
* (someone else is polling the same token)? Partition-immune dual evidence.
|
|
27
|
+
*/
|
|
28
|
+
export function evaluatePollerCount(observations, localSaw409) {
|
|
29
|
+
const activePollers = observations.filter((o) => o.fresh && o.pollingActive === true).length;
|
|
30
|
+
const hasVisibilityGap = observations.some((o) => !o.fresh || o.pollingActive === undefined);
|
|
31
|
+
// 409 is ground truth that a 2nd poller exists — even if its heartbeat is dark.
|
|
32
|
+
if (localSaw409) {
|
|
33
|
+
return { verdict: 'dual', activePollers, hasVisibilityGap,
|
|
34
|
+
reason: 'Telegram 409 conflict — a second machine is polling the same bot token' };
|
|
35
|
+
}
|
|
36
|
+
// We positively SEE ≥2 pollers → dual regardless of any unknown peers.
|
|
37
|
+
if (activePollers >= 2) {
|
|
38
|
+
return { verdict: 'dual', activePollers, hasVisibilityGap,
|
|
39
|
+
reason: `${activePollers} machines are polling Telegram (dual-poll → 409 war)` };
|
|
40
|
+
}
|
|
41
|
+
// A visibility gap means we can't assert exactly-one or zero → indeterminate.
|
|
42
|
+
if (hasVisibilityGap) {
|
|
43
|
+
return { verdict: 'indeterminate', activePollers, hasVisibilityGap,
|
|
44
|
+
reason: 'cannot confirm ingress — a peer is unreachable or on an older version' };
|
|
45
|
+
}
|
|
46
|
+
// Everyone fresh + known, <2 active.
|
|
47
|
+
if (activePollers === 1) {
|
|
48
|
+
return { verdict: 'ok', activePollers, hasVisibilityGap, reason: 'exactly one machine is polling Telegram' };
|
|
49
|
+
}
|
|
50
|
+
return { verdict: 'silence', activePollers, hasVisibilityGap,
|
|
51
|
+
reason: 'NO machine is polling Telegram — inbound is not being received' };
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Adapter — evaluate the exactly-one-listener verdict directly over the pool's
|
|
55
|
+
* MachineCapacity rows (from `?scope=pool`). `online` is the freshness signal
|
|
56
|
+
* (a dark peer → not fresh → indeterminate), `pollingActive` the truth field
|
|
57
|
+
* (absent on an older peer → unknown). This is what a `/guards` surface calls.
|
|
58
|
+
*/
|
|
59
|
+
export function poolPollerVerdict(capacities, localSaw409) {
|
|
60
|
+
return evaluatePollerCount(capacities.map((c) => ({ machineId: c.machineId, pollingActive: c.pollingActive, fresh: !!c.online })), localSaw409);
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=pollerCount.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pollerCount.js","sourceRoot":"","sources":["../../src/core/pollerCount.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AA0BH;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CACjC,YAAiC,EACjC,WAAoB;IAEpB,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IAC7F,MAAM,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC;IAE7F,gFAAgF;IAChF,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,gBAAgB;YACvD,MAAM,EAAE,wEAAwE,EAAE,CAAC;IACvF,CAAC;IACD,uEAAuE;IACvE,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,gBAAgB;YACvD,MAAM,EAAE,GAAG,aAAa,sDAAsD,EAAE,CAAC;IACrF,CAAC;IACD,8EAA8E;IAC9E,IAAI,gBAAgB,EAAE,CAAC;QACrB,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB;YAChE,MAAM,EAAE,uEAAuE,EAAE,CAAC;IACtF,CAAC;IACD,qCAAqC;IACrC,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,EAAE,yCAAyC,EAAE,CAAC;IAC/G,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,gBAAgB;QAC1D,MAAM,EAAE,gEAAgE,EAAE,CAAC;AAC/E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAC/B,UAAmF,EACnF,WAAoB;IAEpB,OAAO,mBAAmB,CACxB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,aAAa,EAAE,CAAC,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EACtG,WAAW,CACZ,CAAC;AACJ,CAAC"}
|