@antseed/cli 0.1.144 → 0.1.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/proxy/buyer-proxy.d.ts +59 -9
- package/dist/proxy/buyer-proxy.d.ts.map +1 -1
- package/dist/proxy/buyer-proxy.js +391 -47
- package/dist/proxy/buyer-proxy.js.map +1 -1
- package/dist/proxy/buyer-proxy.test.js +339 -11
- package/dist/proxy/buyer-proxy.test.js.map +1 -1
- package/dist/proxy/peer-attribution.d.ts +100 -0
- package/dist/proxy/peer-attribution.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.js +175 -0
- package/dist/proxy/peer-attribution.js.map +1 -0
- package/dist/proxy/peer-attribution.test.d.ts +2 -0
- package/dist/proxy/peer-attribution.test.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.test.js +140 -0
- package/dist/proxy/peer-attribution.test.js.map +1 -0
- package/dist/proxy/peer-health.d.ts +73 -0
- package/dist/proxy/peer-health.d.ts.map +1 -0
- package/dist/proxy/peer-health.js +182 -0
- package/dist/proxy/peer-health.js.map +1 -0
- package/dist/proxy/peer-health.test.d.ts +2 -0
- package/dist/proxy/peer-health.test.d.ts.map +1 -0
- package/dist/proxy/peer-health.test.js +197 -0
- package/dist/proxy/peer-health.test.js.map +1 -0
- package/package.json +5 -5
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import test from 'node:test';
|
|
3
|
+
import { PeerAttributionTracker, CORROBORATION_WINDOW_MS, CORRELATION_WINDOW_MS, HEARTBEAT_MS, SUSPEND_JUMP_MS, } from './peer-attribution.js';
|
|
4
|
+
const A = 'a'.repeat(40);
|
|
5
|
+
const B = 'b'.repeat(40);
|
|
6
|
+
const C = 'c'.repeat(40);
|
|
7
|
+
const D = 'd'.repeat(40);
|
|
8
|
+
function fail(tracker, peerId, now) {
|
|
9
|
+
return tracker.classify({ peerId, reasonEscalates: true, fault: 'unknown', now });
|
|
10
|
+
}
|
|
11
|
+
test('a failure never escalates without a prior success anywhere', () => {
|
|
12
|
+
const tracker = new PeerAttributionTracker();
|
|
13
|
+
const { verdict } = fail(tracker, A, 1_000_000);
|
|
14
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'no-corroborating-success' });
|
|
15
|
+
});
|
|
16
|
+
test('a recent success on another peer lets the failure escalate', () => {
|
|
17
|
+
const tracker = new PeerAttributionTracker();
|
|
18
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
19
|
+
const { verdict } = fail(tracker, A, 1_001_000);
|
|
20
|
+
assert.deepEqual(verdict, { escalate: true });
|
|
21
|
+
});
|
|
22
|
+
test('a success on the failing peer itself does not vouch for the buyer', () => {
|
|
23
|
+
const tracker = new PeerAttributionTracker();
|
|
24
|
+
tracker.recordSuccess(A, 1_000_000);
|
|
25
|
+
const { verdict } = fail(tracker, A, 1_001_000);
|
|
26
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'no-corroborating-success' });
|
|
27
|
+
});
|
|
28
|
+
test('a success older than the corroboration window stops vouching', () => {
|
|
29
|
+
const tracker = new PeerAttributionTracker();
|
|
30
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
31
|
+
const { verdict } = fail(tracker, A, 1_000_000 + CORROBORATION_WINDOW_MS + 1);
|
|
32
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'no-corroborating-success' });
|
|
33
|
+
});
|
|
34
|
+
test('three distinct peers failing together blames the buyer and rolls them back', () => {
|
|
35
|
+
const tracker = new PeerAttributionTracker();
|
|
36
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
37
|
+
assert.deepEqual(fail(tracker, A, 1_000_100).verdict, { escalate: true });
|
|
38
|
+
assert.deepEqual(fail(tracker, B, 1_000_200).verdict, { escalate: true });
|
|
39
|
+
const third = fail(tracker, C, 1_000_300);
|
|
40
|
+
assert.deepEqual(third.verdict, { escalate: false, suppressedBy: 'correlated-failure' });
|
|
41
|
+
assert.deepEqual([...third.rollbackPeerIds].sort(), [A, B, C].sort());
|
|
42
|
+
});
|
|
43
|
+
test('repeated failures on a single peer do not trip correlation suppression', () => {
|
|
44
|
+
const tracker = new PeerAttributionTracker();
|
|
45
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
46
|
+
for (let i = 1; i <= 5; i += 1) {
|
|
47
|
+
const { verdict } = fail(tracker, A, 1_000_000 + i * 1_000);
|
|
48
|
+
assert.deepEqual(verdict, { escalate: true }, `failure ${i} should still escalate`);
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
test('failures spread beyond the correlation window are not correlated', () => {
|
|
52
|
+
const tracker = new PeerAttributionTracker();
|
|
53
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
54
|
+
// Three distinct peers, but far enough apart that they are three separate
|
|
55
|
+
// incidents rather than one buyer-side outage. Each is judged on its own.
|
|
56
|
+
const spacing = CORRELATION_WINDOW_MS + 1_000;
|
|
57
|
+
assert.deepEqual(fail(tracker, A, 1_000_100).verdict, { escalate: true });
|
|
58
|
+
assert.deepEqual(fail(tracker, B, 1_000_100 + spacing).verdict, { escalate: true });
|
|
59
|
+
assert.deepEqual(fail(tracker, C, 1_000_100 + spacing * 2).verdict, { escalate: true });
|
|
60
|
+
});
|
|
61
|
+
test('suppression persists for the window after a correlation trip', () => {
|
|
62
|
+
const tracker = new PeerAttributionTracker();
|
|
63
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
64
|
+
fail(tracker, A, 1_000_100);
|
|
65
|
+
fail(tracker, B, 1_000_200);
|
|
66
|
+
fail(tracker, C, 1_000_300);
|
|
67
|
+
const later = fail(tracker, A, 1_000_400);
|
|
68
|
+
assert.deepEqual(later.verdict, { escalate: false, suppressedBy: 'correlated-failure' });
|
|
69
|
+
});
|
|
70
|
+
test('a buyer-attributed fault never escalates, even with corroboration', () => {
|
|
71
|
+
const tracker = new PeerAttributionTracker();
|
|
72
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
73
|
+
const { verdict } = tracker.classify({
|
|
74
|
+
peerId: A, reasonEscalates: true, fault: 'buyer', now: 1_001_000,
|
|
75
|
+
});
|
|
76
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'buyer-fault' });
|
|
77
|
+
});
|
|
78
|
+
test('a non-escalating reason is recorded but never cools a peer down', () => {
|
|
79
|
+
const tracker = new PeerAttributionTracker();
|
|
80
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
81
|
+
const { verdict } = tracker.classify({
|
|
82
|
+
peerId: A, reasonEscalates: false, fault: 'peer', now: 1_001_000,
|
|
83
|
+
});
|
|
84
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'non-escalating-reason' });
|
|
85
|
+
});
|
|
86
|
+
test('a wall-clock jump reads as suspend, quarantines, and rolls back the storm', () => {
|
|
87
|
+
const tracker = new PeerAttributionTracker();
|
|
88
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
89
|
+
assert.equal(tracker.onHeartbeat(1_000_000), null);
|
|
90
|
+
// Two peers fail as the machine goes down; the jump is noticed on the next tick.
|
|
91
|
+
fail(tracker, A, 1_000_100);
|
|
92
|
+
fail(tracker, B, 1_000_200);
|
|
93
|
+
const wake = 1_000_200 + HEARTBEAT_MS + SUSPEND_JUMP_MS + 1_000;
|
|
94
|
+
const result = tracker.onHeartbeat(wake);
|
|
95
|
+
assert.ok(result, 'expected the clock jump to be detected');
|
|
96
|
+
assert.deepEqual([...result.rollbackPeerIds].sort(), [A, B].sort());
|
|
97
|
+
const afterWake = fail(tracker, C, wake + 100);
|
|
98
|
+
assert.deepEqual(afterWake.verdict, { escalate: false, suppressedBy: 'suspend-detected' });
|
|
99
|
+
});
|
|
100
|
+
test('a normal heartbeat cadence is not mistaken for suspend', () => {
|
|
101
|
+
const tracker = new PeerAttributionTracker();
|
|
102
|
+
assert.equal(tracker.onHeartbeat(1_000_000), null);
|
|
103
|
+
assert.equal(tracker.onHeartbeat(1_000_000 + HEARTBEAT_MS), null);
|
|
104
|
+
assert.equal(tracker.onHeartbeat(1_000_000 + HEARTBEAT_MS * 2 + 500), null);
|
|
105
|
+
});
|
|
106
|
+
test('isBuyerHealthy tracks corroboration and suppression', () => {
|
|
107
|
+
const tracker = new PeerAttributionTracker();
|
|
108
|
+
assert.equal(tracker.isBuyerHealthy(1_000_000), false);
|
|
109
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
110
|
+
assert.equal(tracker.isBuyerHealthy(1_000_100), true);
|
|
111
|
+
assert.equal(tracker.isBuyerHealthy(1_000_000 + CORROBORATION_WINDOW_MS + 1), false);
|
|
112
|
+
fail(tracker, A, 1_000_100);
|
|
113
|
+
fail(tracker, B, 1_000_200);
|
|
114
|
+
fail(tracker, C, 1_000_300);
|
|
115
|
+
assert.equal(tracker.isBuyerHealthy(1_000_400), false);
|
|
116
|
+
});
|
|
117
|
+
test('snapshot reports the active suppression window and clears once it lapses', () => {
|
|
118
|
+
const tracker = new PeerAttributionTracker();
|
|
119
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
120
|
+
fail(tracker, A, 1_000_100);
|
|
121
|
+
fail(tracker, B, 1_000_200);
|
|
122
|
+
fail(tracker, C, 1_000_300);
|
|
123
|
+
const during = tracker.snapshot(1_000_400);
|
|
124
|
+
assert.ok(during.suppressedUntil > 1_000_400);
|
|
125
|
+
assert.equal(during.lastSuppressedBy, 'correlated-failure');
|
|
126
|
+
assert.equal(during.lastAnySuccessAt, 1_000_000);
|
|
127
|
+
const after = tracker.snapshot(1_000_300 + 120_000);
|
|
128
|
+
assert.equal(after.suppressedUntil, 0);
|
|
129
|
+
});
|
|
130
|
+
test('a backwards clock jump does not strand failures in the correlation buffer', () => {
|
|
131
|
+
const tracker = new PeerAttributionTracker();
|
|
132
|
+
tracker.recordSuccess(D, 2_000_000);
|
|
133
|
+
fail(tracker, A, 2_000_100);
|
|
134
|
+
fail(tracker, B, 2_000_200);
|
|
135
|
+
// Clock steps backwards past the earlier marks; they must be discarded, so
|
|
136
|
+
// this third distinct peer does not trip correlation on stale data.
|
|
137
|
+
const { verdict } = fail(tracker, C, 1_000_000);
|
|
138
|
+
assert.notEqual(verdict.escalate === false ? verdict.suppressedBy : null, 'correlated-failure');
|
|
139
|
+
});
|
|
140
|
+
//# sourceMappingURL=peer-attribution.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"peer-attribution.test.js","sourceRoot":"","sources":["../../src/proxy/peer-attribution.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAA;AACvC,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,EACrB,YAAY,EACZ,eAAe,GAChB,MAAM,uBAAuB,CAAA;AAE9B,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AACxB,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AACxB,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AACxB,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AAExB,SAAS,IAAI,CAAC,OAA+B,EAAE,MAAc,EAAE,GAAW;IACxE,OAAO,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAA;AACnF,CAAC;AAED,IAAI,CAAC,4DAA4D,EAAE,GAAG,EAAE;IACtE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC/C,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,0BAA0B,EAAE,CAAC,CAAA;AAC1F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,4DAA4D,EAAE,GAAG,EAAE;IACtE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC/C,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;AAC/C,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,mEAAmE,EAAE,GAAG,EAAE;IAC7E,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC/C,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,0BAA0B,EAAE,CAAC,CAAA;AAC1F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,8DAA8D,EAAE,GAAG,EAAE;IACxE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,GAAG,uBAAuB,GAAG,CAAC,CAAC,CAAA;IAC7E,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,0BAA0B,EAAE,CAAC,CAAA;AAC1F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,4EAA4E,EAAE,GAAG,EAAE;IACtF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IAEnC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IACzE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAEzE,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IACzC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE,CAAC,CAAA;IACxF,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;AACvE,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,wEAAwE,EAAE,GAAG,EAAE;IAClF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,CAAA;QAC3D,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,wBAAwB,CAAC,CAAA;IACrF,CAAC;AACH,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,kEAAkE,EAAE,GAAG,EAAE;IAC5E,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IAEnC,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,OAAO,GAAG,qBAAqB,GAAG,KAAK,CAAA;IAC7C,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IACzE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IACnF,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;AACzF,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,8DAA8D,EAAE,GAAG,EAAE;IACxE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAE3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IACzC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE,CAAC,CAAA;AAC1F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,mEAAmE,EAAE,GAAG,EAAE;IAC7E,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,MAAM,EAAE,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS;KACjE,CAAC,CAAA;IACF,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,CAAA;AAC7E,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,iEAAiE,EAAE,GAAG,EAAE;IAC3E,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,MAAM,EAAE,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS;KACjE,CAAC,CAAA;IACF,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,uBAAuB,EAAE,CAAC,CAAA;AACvF,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,2EAA2E,EAAE,GAAG,EAAE;IACrF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAA;IAElD,iFAAiF;IACjF,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAE3B,MAAM,IAAI,GAAG,SAAS,GAAG,YAAY,GAAG,eAAe,GAAG,KAAK,CAAA;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;IACxC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,wCAAwC,CAAC,CAAA;IAC3D,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IAEnE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,GAAG,GAAG,CAAC,CAAA;IAC9C,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,CAAC,CAAA;AAC5F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,wDAAwD,EAAE,GAAG,EAAE;IAClE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAA;IAClD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,GAAG,YAAY,CAAC,EAAE,IAAI,CAAC,CAAA;IACjE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,GAAG,YAAY,GAAG,CAAC,GAAG,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;AAC7E,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,qDAAqD,EAAE,GAAG,EAAE;IAC/D,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAA;IAEtD,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAA;IACrD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,GAAG,uBAAuB,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;IAEpF,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAA;AACxD,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,0EAA0E,EAAE,GAAG,EAAE;IACpF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAE3B,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;IAC1C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,CAAA;IAC7C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,CAAA;IAC3D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAA;IAEhD,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,GAAG,OAAO,CAAC,CAAA;IACnD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAA;AACxC,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,2EAA2E,EAAE,GAAG,EAAE;IACrF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAE3B,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC/C,MAAM,CAAC,QAAQ,CACb,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EACxD,oBAAoB,CACrB,CAAA;AACH,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-peer failure bookkeeping and cooldown state for the buyer proxy.
|
|
3
|
+
*
|
|
4
|
+
* A peer that stops responding should stop being picked, but only temporarily:
|
|
5
|
+
* peers go offline for legitimate reasons, so there is no permanent auto-ban.
|
|
6
|
+
* Whether a failure is even allowed to escalate is decided separately, by
|
|
7
|
+
* `peer-attribution.ts` — this module owns the state machine, not the blame.
|
|
8
|
+
*
|
|
9
|
+
* Cooldown is always advisory. The proxy still dispatches to a cooling-down
|
|
10
|
+
* peer when a request names it, so a pinned conversation keeps working and no
|
|
11
|
+
* amount of cooldown can deadlock routing.
|
|
12
|
+
*/
|
|
13
|
+
/** Failures older than this start a fresh streak rather than compounding. */
|
|
14
|
+
export declare const PEER_HEALTH_WINDOW_MS: number;
|
|
15
|
+
/** Concurrent requests failing together count as one episode, not several. */
|
|
16
|
+
export declare const PEER_HEALTH_COALESCE_MS = 1000;
|
|
17
|
+
/** Ceiling on any single cooldown, and the sanity bound for persisted values. */
|
|
18
|
+
export declare const PEER_HEALTH_MAX_COOLDOWN_MS: number;
|
|
19
|
+
/** How long an idle entry is kept before it is forgotten entirely. */
|
|
20
|
+
export declare const PEER_HEALTH_RETENTION_MS: number;
|
|
21
|
+
/** Upper bound on tracked peers, newest failures retained. */
|
|
22
|
+
export declare const PEER_HEALTH_MAX_ENTRIES = 512;
|
|
23
|
+
export type PeerFailureReason =
|
|
24
|
+
/** Transport threw, timed out, or the connection died mid-request. */
|
|
25
|
+
'request-failed'
|
|
26
|
+
/** Seller answered 500/502/503/504. */
|
|
27
|
+
| 'seller-5xx'
|
|
28
|
+
/** Seller answered 408. */
|
|
29
|
+
| 'seller-timeout'
|
|
30
|
+
/** Seller answered 429 — capacity pressure, not death. Never escalates. */
|
|
31
|
+
| 'seller-busy'
|
|
32
|
+
/** Our own wallet, deposits, config or state machine. Never escalates. */
|
|
33
|
+
| 'buyer-local';
|
|
34
|
+
/** Whether a reason is even a candidate for cooling a peer down. */
|
|
35
|
+
export declare function reasonEscalates(reason: PeerFailureReason): boolean;
|
|
36
|
+
export type PeerHealthEntry = {
|
|
37
|
+
failureStreak: number;
|
|
38
|
+
windowStartedAt: number;
|
|
39
|
+
/** Start of the current coalesced failure episode. */
|
|
40
|
+
episodeStartedAt: number;
|
|
41
|
+
lastFailureAt: number;
|
|
42
|
+
lastReason: PeerFailureReason;
|
|
43
|
+
/** 0 when the peer is not cooling down. */
|
|
44
|
+
cooldownUntil: number;
|
|
45
|
+
/** 0 when the peer has never answered us. */
|
|
46
|
+
lastSuccessAt: number;
|
|
47
|
+
};
|
|
48
|
+
export declare function computeCooldownMs(failureStreak: number): number;
|
|
49
|
+
/**
|
|
50
|
+
* Fold a failure into a peer's health.
|
|
51
|
+
*
|
|
52
|
+
* `escalate` comes from the attribution gates: when false the failure is still
|
|
53
|
+
* recorded for diagnostics and UI, but the streak and cooldown stay put.
|
|
54
|
+
*/
|
|
55
|
+
export declare function recordPeerFailureEntry(previous: PeerHealthEntry | undefined, reason: PeerFailureReason, now: number, escalate: boolean): PeerHealthEntry;
|
|
56
|
+
/**
|
|
57
|
+
* Any response from a peer is proof of life, so a success wipes the streak and
|
|
58
|
+
* releases the cooldown. `lastSuccessAt` is kept for diagnostics.
|
|
59
|
+
*/
|
|
60
|
+
export declare function clearPeerHealthEntry(previous: PeerHealthEntry | undefined, now: number): PeerHealthEntry;
|
|
61
|
+
export declare function isCoolingDown(entry: PeerHealthEntry | undefined, now: number): boolean;
|
|
62
|
+
/** Drop stale entries and cap the map, keeping the most recently active peers. */
|
|
63
|
+
export declare function prunePeerHealth(entries: Map<string, PeerHealthEntry>, now: number): Map<string, PeerHealthEntry>;
|
|
64
|
+
/**
|
|
65
|
+
* Parse the `peerHealth` map out of a buyer.state.json blob.
|
|
66
|
+
*
|
|
67
|
+
* Cooldown survives a restart — a peer that died ten seconds before we exited is
|
|
68
|
+
* still dead — but is never extended or resurrected by one: anything expired,
|
|
69
|
+
* impossibly far in the future, or stamped after `now` is discarded.
|
|
70
|
+
*/
|
|
71
|
+
export declare function parsePersistedPeerHealth(parsed: unknown, now?: number): Map<string, PeerHealthEntry>;
|
|
72
|
+
export declare function serializePeerHealth(entries: Map<string, PeerHealthEntry>): Record<string, PeerHealthEntry>;
|
|
73
|
+
//# sourceMappingURL=peer-health.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"peer-health.d.ts","sourceRoot":"","sources":["../../src/proxy/peer-health.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAUH,6EAA6E;AAC7E,eAAO,MAAM,qBAAqB,QAAa,CAAA;AAC/C,8EAA8E;AAC9E,eAAO,MAAM,uBAAuB,OAAQ,CAAA;AAC5C,iFAAiF;AACjF,eAAO,MAAM,2BAA2B,QAAa,CAAA;AACrD,sEAAsE;AACtE,eAAO,MAAM,wBAAwB,QAAmB,CAAA;AACxD,8DAA8D;AAC9D,eAAO,MAAM,uBAAuB,MAAM,CAAA;AAE1C,MAAM,MAAM,iBAAiB;AAC3B,sEAAsE;AACpE,gBAAgB;AAClB,uCAAuC;GACrC,YAAY;AACd,2BAA2B;GACzB,gBAAgB;AAClB,2EAA2E;GACzE,aAAa;AACf,0EAA0E;GACxE,aAAa,CAAA;AAMjB,oEAAoE;AACpE,wBAAgB,eAAe,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAElE;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,MAAM,CAAA;IACvB,sDAAsD;IACtD,gBAAgB,EAAE,MAAM,CAAA;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,iBAAiB,CAAA;IAC7B,2CAA2C;IAC3C,aAAa,EAAE,MAAM,CAAA;IACrB,6CAA6C;IAC7C,aAAa,EAAE,MAAM,CAAA;CACtB,CAAA;AAED,wBAAgB,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,CAI/D;AAcD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,eAAe,GAAG,SAAS,EACrC,MAAM,EAAE,iBAAiB,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,GAChB,eAAe,CA+BjB;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,eAAe,GAAG,SAAS,EACrC,GAAG,EAAE,MAAM,GACV,eAAe,CASjB;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,eAAe,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAItF;AAUD,kFAAkF;AAClF,wBAAgB,eAAe,CAC7B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,EACrC,GAAG,EAAE,MAAM,GACV,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAS9B;AAMD;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,OAAO,EACf,GAAG,GAAE,MAAmB,GACvB,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CA+C9B;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,GACpC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAEjC"}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-peer failure bookkeeping and cooldown state for the buyer proxy.
|
|
3
|
+
*
|
|
4
|
+
* A peer that stops responding should stop being picked, but only temporarily:
|
|
5
|
+
* peers go offline for legitimate reasons, so there is no permanent auto-ban.
|
|
6
|
+
* Whether a failure is even allowed to escalate is decided separately, by
|
|
7
|
+
* `peer-attribution.ts` — this module owns the state machine, not the blame.
|
|
8
|
+
*
|
|
9
|
+
* Cooldown is always advisory. The proxy still dispatches to a cooling-down
|
|
10
|
+
* peer when a request names it, so a pinned conversation keeps working and no
|
|
11
|
+
* amount of cooldown can deadlock routing.
|
|
12
|
+
*/
|
|
13
|
+
// The escalation curve is defined once in
|
|
14
|
+
// `packages/router-core/src/peer-metrics.ts` (`computeFailureCooldownMs`).
|
|
15
|
+
// These constants mirror it; router-core is not a dependency of the CLI and a
|
|
16
|
+
// six-line curve does not justify making it one. Keep the two in step.
|
|
17
|
+
const MAX_FAILURES = 3;
|
|
18
|
+
const BASE_COOLDOWN_MS = 30_000;
|
|
19
|
+
const MAX_DOUBLINGS = 4;
|
|
20
|
+
/** Failures older than this start a fresh streak rather than compounding. */
|
|
21
|
+
export const PEER_HEALTH_WINDOW_MS = 5 * 60_000;
|
|
22
|
+
/** Concurrent requests failing together count as one episode, not several. */
|
|
23
|
+
export const PEER_HEALTH_COALESCE_MS = 1_000;
|
|
24
|
+
/** Ceiling on any single cooldown, and the sanity bound for persisted values. */
|
|
25
|
+
export const PEER_HEALTH_MAX_COOLDOWN_MS = 8 * 60_000;
|
|
26
|
+
/** How long an idle entry is kept before it is forgotten entirely. */
|
|
27
|
+
export const PEER_HEALTH_RETENTION_MS = 24 * 60 * 60_000;
|
|
28
|
+
/** Upper bound on tracked peers, newest failures retained. */
|
|
29
|
+
export const PEER_HEALTH_MAX_ENTRIES = 512;
|
|
30
|
+
const FAILURE_REASONS = [
|
|
31
|
+
'request-failed', 'seller-5xx', 'seller-timeout', 'seller-busy', 'buyer-local',
|
|
32
|
+
];
|
|
33
|
+
/** Whether a reason is even a candidate for cooling a peer down. */
|
|
34
|
+
export function reasonEscalates(reason) {
|
|
35
|
+
return reason === 'request-failed' || reason === 'seller-5xx' || reason === 'seller-timeout';
|
|
36
|
+
}
|
|
37
|
+
export function computeCooldownMs(failureStreak) {
|
|
38
|
+
if (!Number.isFinite(failureStreak) || failureStreak < MAX_FAILURES)
|
|
39
|
+
return 0;
|
|
40
|
+
const penaltyPower = Math.min(MAX_DOUBLINGS, Math.trunc(failureStreak) - MAX_FAILURES);
|
|
41
|
+
return Math.min(BASE_COOLDOWN_MS * (2 ** penaltyPower), PEER_HEALTH_MAX_COOLDOWN_MS);
|
|
42
|
+
}
|
|
43
|
+
function emptyEntry() {
|
|
44
|
+
return {
|
|
45
|
+
failureStreak: 0,
|
|
46
|
+
windowStartedAt: 0,
|
|
47
|
+
episodeStartedAt: 0,
|
|
48
|
+
lastFailureAt: 0,
|
|
49
|
+
lastReason: 'request-failed',
|
|
50
|
+
cooldownUntil: 0,
|
|
51
|
+
lastSuccessAt: 0,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Fold a failure into a peer's health.
|
|
56
|
+
*
|
|
57
|
+
* `escalate` comes from the attribution gates: when false the failure is still
|
|
58
|
+
* recorded for diagnostics and UI, but the streak and cooldown stay put.
|
|
59
|
+
*/
|
|
60
|
+
export function recordPeerFailureEntry(previous, reason, now, escalate) {
|
|
61
|
+
const prev = previous ?? emptyEntry();
|
|
62
|
+
if (!escalate) {
|
|
63
|
+
return { ...prev, lastFailureAt: now, lastReason: reason };
|
|
64
|
+
}
|
|
65
|
+
// Two requests to the same dead peer failing together are one episode; without
|
|
66
|
+
// this they would skip a doubling step each time concurrency rises.
|
|
67
|
+
const episodeStartedAt = prev.episodeStartedAt;
|
|
68
|
+
if (episodeStartedAt > 0 && now - episodeStartedAt < PEER_HEALTH_COALESCE_MS && now >= prev.lastFailureAt) {
|
|
69
|
+
return { ...prev, lastFailureAt: now, lastReason: reason };
|
|
70
|
+
}
|
|
71
|
+
const windowLapsed = prev.lastFailureAt <= 0
|
|
72
|
+
|| now - prev.lastFailureAt > PEER_HEALTH_WINDOW_MS
|
|
73
|
+
// Clock stepped backwards; the old window is meaningless.
|
|
74
|
+
|| now < prev.lastFailureAt;
|
|
75
|
+
const failureStreak = windowLapsed ? 1 : prev.failureStreak + 1;
|
|
76
|
+
const cooldownMs = computeCooldownMs(failureStreak);
|
|
77
|
+
return {
|
|
78
|
+
failureStreak,
|
|
79
|
+
windowStartedAt: windowLapsed ? now : (prev.windowStartedAt || now),
|
|
80
|
+
episodeStartedAt: now,
|
|
81
|
+
lastFailureAt: now,
|
|
82
|
+
lastReason: reason,
|
|
83
|
+
cooldownUntil: cooldownMs > 0 ? now + cooldownMs : 0,
|
|
84
|
+
lastSuccessAt: prev.lastSuccessAt,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Any response from a peer is proof of life, so a success wipes the streak and
|
|
89
|
+
* releases the cooldown. `lastSuccessAt` is kept for diagnostics.
|
|
90
|
+
*/
|
|
91
|
+
export function clearPeerHealthEntry(previous, now) {
|
|
92
|
+
return {
|
|
93
|
+
...(previous ?? emptyEntry()),
|
|
94
|
+
failureStreak: 0,
|
|
95
|
+
windowStartedAt: 0,
|
|
96
|
+
episodeStartedAt: 0,
|
|
97
|
+
cooldownUntil: 0,
|
|
98
|
+
lastSuccessAt: now,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
export function isCoolingDown(entry, now) {
|
|
102
|
+
if (!entry || entry.cooldownUntil <= now)
|
|
103
|
+
return false;
|
|
104
|
+
// A corrupt or clock-skewed future value must not hold a peer hostage.
|
|
105
|
+
return entry.cooldownUntil - now <= PEER_HEALTH_MAX_COOLDOWN_MS;
|
|
106
|
+
}
|
|
107
|
+
/** Whether an entry still carries information worth persisting. */
|
|
108
|
+
function isEntryMeaningful(entry, now) {
|
|
109
|
+
if (entry.cooldownUntil > now)
|
|
110
|
+
return true;
|
|
111
|
+
const lastActivity = Math.max(entry.lastFailureAt, entry.lastSuccessAt);
|
|
112
|
+
if (lastActivity <= 0)
|
|
113
|
+
return false;
|
|
114
|
+
return now - lastActivity < PEER_HEALTH_RETENTION_MS;
|
|
115
|
+
}
|
|
116
|
+
/** Drop stale entries and cap the map, keeping the most recently active peers. */
|
|
117
|
+
export function prunePeerHealth(entries, now) {
|
|
118
|
+
const kept = [...entries.entries()].filter(([, entry]) => isEntryMeaningful(entry, now));
|
|
119
|
+
if (kept.length > PEER_HEALTH_MAX_ENTRIES) {
|
|
120
|
+
kept.sort((a, b) => Math.max(b[1].lastFailureAt, b[1].lastSuccessAt)
|
|
121
|
+
- Math.max(a[1].lastFailureAt, a[1].lastSuccessAt));
|
|
122
|
+
kept.length = PEER_HEALTH_MAX_ENTRIES;
|
|
123
|
+
}
|
|
124
|
+
return new Map(kept);
|
|
125
|
+
}
|
|
126
|
+
function finiteNonNegative(value) {
|
|
127
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Parse the `peerHealth` map out of a buyer.state.json blob.
|
|
131
|
+
*
|
|
132
|
+
* Cooldown survives a restart — a peer that died ten seconds before we exited is
|
|
133
|
+
* still dead — but is never extended or resurrected by one: anything expired,
|
|
134
|
+
* impossibly far in the future, or stamped after `now` is discarded.
|
|
135
|
+
*/
|
|
136
|
+
export function parsePersistedPeerHealth(parsed, now = Date.now()) {
|
|
137
|
+
const result = new Map();
|
|
138
|
+
if (!parsed || typeof parsed !== 'object')
|
|
139
|
+
return result;
|
|
140
|
+
const raw = parsed.peerHealth;
|
|
141
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw))
|
|
142
|
+
return result;
|
|
143
|
+
for (const [rawPeerId, rawEntry] of Object.entries(raw)) {
|
|
144
|
+
const peerId = rawPeerId.toLowerCase();
|
|
145
|
+
if (!/^[0-9a-f]{40}$/.test(peerId))
|
|
146
|
+
continue;
|
|
147
|
+
if (!rawEntry || typeof rawEntry !== 'object' || Array.isArray(rawEntry))
|
|
148
|
+
continue;
|
|
149
|
+
const entry = rawEntry;
|
|
150
|
+
const lastFailureAt = finiteNonNegative(entry.lastFailureAt);
|
|
151
|
+
const lastSuccessAt = finiteNonNegative(entry.lastSuccessAt);
|
|
152
|
+
// Timestamps ahead of the clock mean the entry cannot be reasoned about.
|
|
153
|
+
if (lastFailureAt > now || lastSuccessAt > now)
|
|
154
|
+
continue;
|
|
155
|
+
const rawStreak = finiteNonNegative(entry.failureStreak);
|
|
156
|
+
const failureStreak = Math.min(64, Math.trunc(rawStreak));
|
|
157
|
+
const lastReason = FAILURE_REASONS.includes(entry.lastReason)
|
|
158
|
+
? entry.lastReason
|
|
159
|
+
: 'request-failed';
|
|
160
|
+
const rawCooldown = finiteNonNegative(entry.cooldownUntil);
|
|
161
|
+
const cooldownUntil = rawCooldown > now && rawCooldown - now <= PEER_HEALTH_MAX_COOLDOWN_MS
|
|
162
|
+
? rawCooldown
|
|
163
|
+
: 0;
|
|
164
|
+
const candidate = {
|
|
165
|
+
failureStreak,
|
|
166
|
+
windowStartedAt: Math.min(finiteNonNegative(entry.windowStartedAt), now),
|
|
167
|
+
episodeStartedAt: Math.min(finiteNonNegative(entry.episodeStartedAt) || lastFailureAt, now),
|
|
168
|
+
lastFailureAt,
|
|
169
|
+
lastReason,
|
|
170
|
+
cooldownUntil,
|
|
171
|
+
lastSuccessAt,
|
|
172
|
+
};
|
|
173
|
+
if (!isEntryMeaningful(candidate, now))
|
|
174
|
+
continue;
|
|
175
|
+
result.set(peerId, candidate);
|
|
176
|
+
}
|
|
177
|
+
return prunePeerHealth(result, now);
|
|
178
|
+
}
|
|
179
|
+
export function serializePeerHealth(entries) {
|
|
180
|
+
return Object.fromEntries(entries);
|
|
181
|
+
}
|
|
182
|
+
//# sourceMappingURL=peer-health.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"peer-health.js","sourceRoot":"","sources":["../../src/proxy/peer-health.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,0CAA0C;AAC1C,2EAA2E;AAC3E,8EAA8E;AAC9E,uEAAuE;AACvE,MAAM,YAAY,GAAG,CAAC,CAAA;AACtB,MAAM,gBAAgB,GAAG,MAAM,CAAA;AAC/B,MAAM,aAAa,GAAG,CAAC,CAAA;AAEvB,6EAA6E;AAC7E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,GAAG,MAAM,CAAA;AAC/C,8EAA8E;AAC9E,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,CAAA;AAC5C,iFAAiF;AACjF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,GAAG,MAAM,CAAA;AACrD,sEAAsE;AACtE,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,GAAG,EAAE,GAAG,MAAM,CAAA;AACxD,8DAA8D;AAC9D,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAA;AAc1C,MAAM,eAAe,GAAiC;IACpD,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa;CAC/E,CAAA;AAED,oEAAoE;AACpE,MAAM,UAAU,eAAe,CAAC,MAAyB;IACvD,OAAO,MAAM,KAAK,gBAAgB,IAAI,MAAM,KAAK,YAAY,IAAI,MAAM,KAAK,gBAAgB,CAAA;AAC9F,CAAC;AAeD,MAAM,UAAU,iBAAiB,CAAC,aAAqB;IACrD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,YAAY;QAAE,OAAO,CAAC,CAAA;IAC7E,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,YAAY,CAAC,CAAA;IACtF,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,EAAE,2BAA2B,CAAC,CAAA;AACtF,CAAC;AAED,SAAS,UAAU;IACjB,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,eAAe,EAAE,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,gBAAgB;QAC5B,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,CAAC;KACjB,CAAA;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACpC,QAAqC,EACrC,MAAyB,EACzB,GAAW,EACX,QAAiB;IAEjB,MAAM,IAAI,GAAG,QAAQ,IAAI,UAAU,EAAE,CAAA;IAErC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,CAAA;IAC5D,CAAC;IAED,+EAA+E;IAC/E,oEAAoE;IACpE,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAA;IAC9C,IAAI,gBAAgB,GAAG,CAAC,IAAI,GAAG,GAAG,gBAAgB,GAAG,uBAAuB,IAAI,GAAG,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QAC1G,OAAO,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,CAAA;IAC5D,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC;WACvC,GAAG,GAAG,IAAI,CAAC,aAAa,GAAG,qBAAqB;QACnD,0DAA0D;WACvD,GAAG,GAAG,IAAI,CAAC,aAAa,CAAA;IAE7B,MAAM,aAAa,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAA;IAC/D,MAAM,UAAU,GAAG,iBAAiB,CAAC,aAAa,CAAC,CAAA;IAEnD,OAAO;QACL,aAAa;QACb,eAAe,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,GAAG,CAAC;QACnE,gBAAgB,EAAE,GAAG;QACrB,aAAa,EAAE,GAAG;QAClB,UAAU,EAAE,MAAM;QAClB,aAAa,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACpD,aAAa,EAAE,IAAI,CAAC,aAAa;KAClC,CAAA;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,QAAqC,EACrC,GAAW;IAEX,OAAO;QACL,GAAG,CAAC,QAAQ,IAAI,UAAU,EAAE,CAAC;QAC7B,aAAa,EAAE,CAAC;QAChB,eAAe,EAAE,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,GAAG;KACnB,CAAA;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAkC,EAAE,GAAW;IAC3E,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,aAAa,IAAI,GAAG;QAAE,OAAO,KAAK,CAAA;IACtD,uEAAuE;IACvE,OAAO,KAAK,CAAC,aAAa,GAAG,GAAG,IAAI,2BAA2B,CAAA;AACjE,CAAC;AAED,mEAAmE;AACnE,SAAS,iBAAiB,CAAC,KAAsB,EAAE,GAAW;IAC5D,IAAI,KAAK,CAAC,aAAa,GAAG,GAAG;QAAE,OAAO,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,aAAa,CAAC,CAAA;IACvE,IAAI,YAAY,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IACnC,OAAO,GAAG,GAAG,YAAY,GAAG,wBAAwB,CAAA;AACtD,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,eAAe,CAC7B,OAAqC,EACrC,GAAW;IAEX,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAA;IACxF,IAAI,IAAI,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACjB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;cAC9C,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAA;QACrD,IAAI,CAAC,MAAM,GAAG,uBAAuB,CAAA;IACvC,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,CAAA;AACtB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;AACtF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CACtC,MAAe,EACf,MAAc,IAAI,CAAC,GAAG,EAAE;IAExB,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAA;IACjD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAExD,MAAM,GAAG,GAAI,MAAmC,CAAC,UAAU,CAAA;IAC3D,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,MAAM,CAAA;IAExE,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAA8B,CAAC,EAAE,CAAC;QACnF,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,CAAA;QACtC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,SAAQ;QAC5C,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAAE,SAAQ;QAElF,MAAM,KAAK,GAAG,QAAmC,CAAA;QACjD,MAAM,aAAa,GAAG,iBAAiB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;QAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;QAE5D,yEAAyE;QACzE,IAAI,aAAa,GAAG,GAAG,IAAI,aAAa,GAAG,GAAG;YAAE,SAAQ;QAExD,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;QACxD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAA;QACzD,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,UAA+B,CAAC;YAChF,CAAC,CAAE,KAAK,CAAC,UAAgC;YACzC,CAAC,CAAC,gBAAgB,CAAA;QAEpB,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;QAC1D,MAAM,aAAa,GAAG,WAAW,GAAG,GAAG,IAAI,WAAW,GAAG,GAAG,IAAI,2BAA2B;YACzF,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,CAAC,CAAA;QAEL,MAAM,SAAS,GAAoB;YACjC,aAAa;YACb,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,GAAG,CAAC;YACxE,gBAAgB,EAAE,IAAI,CAAC,GAAG,CACxB,iBAAiB,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,aAAa,EAC1D,GAAG,CACJ;YACD,aAAa;YACb,UAAU;YACV,aAAa;YACb,aAAa;SACd,CAAA;QACD,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,GAAG,CAAC;YAAE,SAAQ;QAChD,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAAqC;IAErC,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;AACpC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"peer-health.test.d.ts","sourceRoot":"","sources":["../../src/proxy/peer-health.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import test from 'node:test';
|
|
3
|
+
import { recordPeerFailureEntry, clearPeerHealthEntry, isCoolingDown, parsePersistedPeerHealth, prunePeerHealth, computeCooldownMs, reasonEscalates, PEER_HEALTH_WINDOW_MS, PEER_HEALTH_COALESCE_MS, PEER_HEALTH_MAX_COOLDOWN_MS, PEER_HEALTH_RETENTION_MS, PEER_HEALTH_MAX_ENTRIES, } from './peer-health.js';
|
|
4
|
+
const NOW = 1_700_000_000_000;
|
|
5
|
+
const A = 'a'.repeat(40);
|
|
6
|
+
function escalatingFailures(count, startAt = NOW, stepMs = 2_000) {
|
|
7
|
+
let entry;
|
|
8
|
+
for (let i = 0; i < count; i += 1) {
|
|
9
|
+
entry = recordPeerFailureEntry(entry, 'request-failed', startAt + i * stepMs, true);
|
|
10
|
+
}
|
|
11
|
+
return entry;
|
|
12
|
+
}
|
|
13
|
+
test('reasonEscalates only admits reasons that indicate the peer is down', () => {
|
|
14
|
+
assert.equal(reasonEscalates('request-failed'), true);
|
|
15
|
+
assert.equal(reasonEscalates('seller-5xx'), true);
|
|
16
|
+
assert.equal(reasonEscalates('seller-timeout'), true);
|
|
17
|
+
assert.equal(reasonEscalates('seller-busy'), false);
|
|
18
|
+
assert.equal(reasonEscalates('buyer-local'), false);
|
|
19
|
+
});
|
|
20
|
+
test('computeCooldownMs matches the shared escalation curve and its ceiling', () => {
|
|
21
|
+
assert.equal(computeCooldownMs(2), 0);
|
|
22
|
+
assert.equal(computeCooldownMs(3), 30_000);
|
|
23
|
+
assert.equal(computeCooldownMs(4), 60_000);
|
|
24
|
+
assert.equal(computeCooldownMs(5), 120_000);
|
|
25
|
+
assert.equal(computeCooldownMs(7), 480_000);
|
|
26
|
+
assert.equal(computeCooldownMs(50), PEER_HEALTH_MAX_COOLDOWN_MS);
|
|
27
|
+
});
|
|
28
|
+
test('a peer cools down only on the third failure in the window', () => {
|
|
29
|
+
const after1 = recordPeerFailureEntry(undefined, 'request-failed', NOW, true);
|
|
30
|
+
assert.equal(after1.failureStreak, 1);
|
|
31
|
+
assert.equal(isCoolingDown(after1, NOW), false);
|
|
32
|
+
const after2 = recordPeerFailureEntry(after1, 'request-failed', NOW + 2_000, true);
|
|
33
|
+
assert.equal(isCoolingDown(after2, NOW + 2_000), false);
|
|
34
|
+
const after3 = recordPeerFailureEntry(after2, 'request-failed', NOW + 4_000, true);
|
|
35
|
+
assert.equal(after3.failureStreak, 3);
|
|
36
|
+
assert.equal(after3.cooldownUntil, NOW + 4_000 + 30_000);
|
|
37
|
+
assert.equal(isCoolingDown(after3, NOW + 4_000), true);
|
|
38
|
+
});
|
|
39
|
+
test('further failures double the cooldown', () => {
|
|
40
|
+
const after4 = escalatingFailures(4);
|
|
41
|
+
assert.equal(after4.cooldownUntil - after4.lastFailureAt, 60_000);
|
|
42
|
+
const after5 = escalatingFailures(5);
|
|
43
|
+
assert.equal(after5.cooldownUntil - after5.lastFailureAt, 120_000);
|
|
44
|
+
});
|
|
45
|
+
test('a non-escalating failure updates diagnostics but never the streak or cooldown', () => {
|
|
46
|
+
const escalated = escalatingFailures(2);
|
|
47
|
+
const suppressed = recordPeerFailureEntry(escalated, 'seller-busy', NOW + 10_000, false);
|
|
48
|
+
assert.equal(suppressed.failureStreak, escalated.failureStreak);
|
|
49
|
+
assert.equal(suppressed.cooldownUntil, escalated.cooldownUntil);
|
|
50
|
+
assert.equal(suppressed.lastReason, 'seller-busy');
|
|
51
|
+
assert.equal(suppressed.lastFailureAt, NOW + 10_000);
|
|
52
|
+
});
|
|
53
|
+
test('a diagnostic-only failure does not coalesce away the next escalating failure', () => {
|
|
54
|
+
const diagnostic = recordPeerFailureEntry(undefined, 'seller-busy', NOW, false);
|
|
55
|
+
const escalated = recordPeerFailureEntry(diagnostic, 'request-failed', NOW + 100, true);
|
|
56
|
+
assert.equal(escalated.failureStreak, 1);
|
|
57
|
+
assert.equal(escalated.episodeStartedAt, NOW + 100);
|
|
58
|
+
});
|
|
59
|
+
test('failures inside the coalesce window count as a single episode', () => {
|
|
60
|
+
let entry = recordPeerFailureEntry(undefined, 'request-failed', NOW, true);
|
|
61
|
+
entry = recordPeerFailureEntry(entry, 'request-failed', NOW + 100, true);
|
|
62
|
+
entry = recordPeerFailureEntry(entry, 'request-failed', NOW + 200, true);
|
|
63
|
+
assert.equal(entry.failureStreak, 1);
|
|
64
|
+
assert.equal(isCoolingDown(entry, NOW + 200), false);
|
|
65
|
+
});
|
|
66
|
+
test('the coalesce window is anchored to the episode start', () => {
|
|
67
|
+
let entry = recordPeerFailureEntry(undefined, 'request-failed', NOW, true);
|
|
68
|
+
entry = recordPeerFailureEntry(entry, 'request-failed', NOW + PEER_HEALTH_COALESCE_MS - 100, true);
|
|
69
|
+
entry = recordPeerFailureEntry(entry, 'request-failed', NOW + PEER_HEALTH_COALESCE_MS + 100, true);
|
|
70
|
+
assert.equal(entry.failureStreak, 2);
|
|
71
|
+
assert.equal(entry.episodeStartedAt, NOW + PEER_HEALTH_COALESCE_MS + 100);
|
|
72
|
+
});
|
|
73
|
+
test('a lapsed window starts a fresh streak instead of compounding', () => {
|
|
74
|
+
const twoFailures = escalatingFailures(2);
|
|
75
|
+
const muchLater = recordPeerFailureEntry(twoFailures, 'request-failed', twoFailures.lastFailureAt + PEER_HEALTH_WINDOW_MS + 1, true);
|
|
76
|
+
assert.equal(muchLater.failureStreak, 1);
|
|
77
|
+
assert.equal(muchLater.cooldownUntil, 0);
|
|
78
|
+
});
|
|
79
|
+
test('a backwards clock jump resets the window rather than compounding', () => {
|
|
80
|
+
const twoFailures = escalatingFailures(2);
|
|
81
|
+
const backwards = recordPeerFailureEntry(twoFailures, 'request-failed', NOW - 60_000, true);
|
|
82
|
+
assert.equal(backwards.failureStreak, 1);
|
|
83
|
+
assert.equal(backwards.cooldownUntil, 0);
|
|
84
|
+
});
|
|
85
|
+
test('a success clears the streak and releases the cooldown', () => {
|
|
86
|
+
const cooling = escalatingFailures(3);
|
|
87
|
+
assert.equal(isCoolingDown(cooling, cooling.lastFailureAt), true);
|
|
88
|
+
const recovered = clearPeerHealthEntry(cooling, cooling.lastFailureAt + 1_000);
|
|
89
|
+
assert.equal(recovered.failureStreak, 0);
|
|
90
|
+
assert.equal(recovered.cooldownUntil, 0);
|
|
91
|
+
assert.equal(recovered.lastSuccessAt, cooling.lastFailureAt + 1_000);
|
|
92
|
+
assert.equal(isCoolingDown(recovered, recovered.lastSuccessAt), false);
|
|
93
|
+
});
|
|
94
|
+
test('isCoolingDown rejects an impossibly distant cooldown', () => {
|
|
95
|
+
const entry = {
|
|
96
|
+
...escalatingFailures(3),
|
|
97
|
+
cooldownUntil: NOW + PEER_HEALTH_MAX_COOLDOWN_MS * 10,
|
|
98
|
+
};
|
|
99
|
+
assert.equal(isCoolingDown(entry, NOW), false);
|
|
100
|
+
});
|
|
101
|
+
test('parsePersistedPeerHealth ignores junk input', () => {
|
|
102
|
+
assert.equal(parsePersistedPeerHealth(null, NOW).size, 0);
|
|
103
|
+
assert.equal(parsePersistedPeerHealth('nope', NOW).size, 0);
|
|
104
|
+
assert.equal(parsePersistedPeerHealth({}, NOW).size, 0);
|
|
105
|
+
assert.equal(parsePersistedPeerHealth({ peerHealth: [] }, NOW).size, 0);
|
|
106
|
+
assert.equal(parsePersistedPeerHealth({ peerHealth: 'x' }, NOW).size, 0);
|
|
107
|
+
});
|
|
108
|
+
test('parsePersistedPeerHealth rejects malformed peer ids and entries', () => {
|
|
109
|
+
const parsed = parsePersistedPeerHealth({
|
|
110
|
+
peerHealth: {
|
|
111
|
+
'not-a-peer-id': { lastFailureAt: NOW - 1_000, failureStreak: 3 },
|
|
112
|
+
[`0x${'a'.repeat(40)}`]: { lastFailureAt: NOW - 1_000, failureStreak: 3 },
|
|
113
|
+
[A]: 'not-an-object',
|
|
114
|
+
},
|
|
115
|
+
}, NOW);
|
|
116
|
+
assert.equal(parsed.size, 0);
|
|
117
|
+
});
|
|
118
|
+
test('parsePersistedPeerHealth restores a live cooldown unchanged', () => {
|
|
119
|
+
const cooldownUntil = NOW + 20_000;
|
|
120
|
+
const parsed = parsePersistedPeerHealth({
|
|
121
|
+
peerHealth: {
|
|
122
|
+
[A.toUpperCase()]: {
|
|
123
|
+
failureStreak: 3, windowStartedAt: NOW - 5_000, lastFailureAt: NOW - 1_000,
|
|
124
|
+
lastReason: 'seller-5xx', cooldownUntil, lastSuccessAt: 0,
|
|
125
|
+
},
|
|
126
|
+
},
|
|
127
|
+
}, NOW);
|
|
128
|
+
const entry = parsed.get(A);
|
|
129
|
+
assert.ok(entry);
|
|
130
|
+
assert.equal(entry.cooldownUntil, cooldownUntil);
|
|
131
|
+
assert.equal(entry.failureStreak, 3);
|
|
132
|
+
assert.equal(entry.lastReason, 'seller-5xx');
|
|
133
|
+
});
|
|
134
|
+
test('parsePersistedPeerHealth drops an expired or impossible cooldown', () => {
|
|
135
|
+
const expired = parsePersistedPeerHealth({
|
|
136
|
+
peerHealth: { [A]: { failureStreak: 3, lastFailureAt: NOW - 1_000, cooldownUntil: NOW - 1 } },
|
|
137
|
+
}, NOW);
|
|
138
|
+
assert.equal(expired.get(A)?.cooldownUntil, 0);
|
|
139
|
+
const absurd = parsePersistedPeerHealth({
|
|
140
|
+
peerHealth: {
|
|
141
|
+
[A]: {
|
|
142
|
+
failureStreak: 3,
|
|
143
|
+
lastFailureAt: NOW - 1_000,
|
|
144
|
+
cooldownUntil: NOW + PEER_HEALTH_MAX_COOLDOWN_MS * 100,
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
}, NOW);
|
|
148
|
+
assert.equal(absurd.get(A)?.cooldownUntil, 0);
|
|
149
|
+
});
|
|
150
|
+
test('parsePersistedPeerHealth discards entries stamped in the future', () => {
|
|
151
|
+
const parsed = parsePersistedPeerHealth({
|
|
152
|
+
peerHealth: { [A]: { failureStreak: 3, lastFailureAt: NOW + 60_000 } },
|
|
153
|
+
}, NOW);
|
|
154
|
+
assert.equal(parsed.size, 0);
|
|
155
|
+
});
|
|
156
|
+
test('parsePersistedPeerHealth coerces non-finite numbers and unknown reasons', () => {
|
|
157
|
+
const parsed = parsePersistedPeerHealth({
|
|
158
|
+
peerHealth: {
|
|
159
|
+
[A]: {
|
|
160
|
+
failureStreak: 'lots', windowStartedAt: Number.NaN, lastFailureAt: NOW - 1_000,
|
|
161
|
+
lastReason: 'made-up-reason', cooldownUntil: Number.POSITIVE_INFINITY,
|
|
162
|
+
lastSuccessAt: -5,
|
|
163
|
+
},
|
|
164
|
+
},
|
|
165
|
+
}, NOW);
|
|
166
|
+
const entry = parsed.get(A);
|
|
167
|
+
assert.ok(entry);
|
|
168
|
+
assert.equal(entry.failureStreak, 0);
|
|
169
|
+
assert.equal(entry.windowStartedAt, 0);
|
|
170
|
+
assert.equal(entry.lastReason, 'request-failed');
|
|
171
|
+
assert.equal(entry.cooldownUntil, 0);
|
|
172
|
+
assert.equal(entry.lastSuccessAt, 0);
|
|
173
|
+
});
|
|
174
|
+
test('parsePersistedPeerHealth forgets entries past the retention window', () => {
|
|
175
|
+
const parsed = parsePersistedPeerHealth({
|
|
176
|
+
peerHealth: { [A]: { failureStreak: 3, lastFailureAt: NOW - PEER_HEALTH_RETENTION_MS - 1 } },
|
|
177
|
+
}, NOW);
|
|
178
|
+
assert.equal(parsed.size, 0);
|
|
179
|
+
});
|
|
180
|
+
test('prunePeerHealth caps the map keeping the most recently active peers', () => {
|
|
181
|
+
const entries = new Map();
|
|
182
|
+
const total = PEER_HEALTH_MAX_ENTRIES + 10;
|
|
183
|
+
for (let i = 0; i < total; i += 1) {
|
|
184
|
+
const peerId = i.toString(16).padStart(40, '0');
|
|
185
|
+
entries.set(peerId, {
|
|
186
|
+
failureStreak: 1, windowStartedAt: 0, lastFailureAt: NOW - (total - i) * 1_000,
|
|
187
|
+
episodeStartedAt: NOW - (total - i) * 1_000,
|
|
188
|
+
lastReason: 'request-failed', cooldownUntil: 0, lastSuccessAt: 0,
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
const pruned = prunePeerHealth(entries, NOW);
|
|
192
|
+
assert.equal(pruned.size, PEER_HEALTH_MAX_ENTRIES);
|
|
193
|
+
// The freshest peer survives; the stalest is evicted.
|
|
194
|
+
assert.ok(pruned.has((total - 1).toString(16).padStart(40, '0')));
|
|
195
|
+
assert.equal(pruned.has((0).toString(16).padStart(40, '0')), false);
|
|
196
|
+
});
|
|
197
|
+
//# sourceMappingURL=peer-health.test.js.map
|