@antseed/cli 0.1.143 → 0.1.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/cli/commands/config/index.d.ts.map +1 -1
- package/dist/cli/commands/config/index.js +9 -0
- package/dist/cli/commands/config/index.js.map +1 -1
- package/dist/cli/commands/seller/capability-prompts.d.ts +17 -0
- package/dist/cli/commands/seller/capability-prompts.d.ts.map +1 -0
- package/dist/cli/commands/seller/capability-prompts.js +123 -0
- package/dist/cli/commands/seller/capability-prompts.js.map +1 -0
- package/dist/cli/commands/seller/capability-prompts.test.d.ts +2 -0
- package/dist/cli/commands/seller/capability-prompts.test.d.ts.map +1 -0
- package/dist/cli/commands/seller/capability-prompts.test.js +99 -0
- package/dist/cli/commands/seller/capability-prompts.test.js.map +1 -0
- package/dist/cli/commands/seller/setup.d.ts.map +1 -1
- package/dist/cli/commands/seller/setup.js +12 -0
- package/dist/cli/commands/seller/setup.js.map +1 -1
- package/dist/cli/commands/seller/start.d.ts +3 -2
- package/dist/cli/commands/seller/start.d.ts.map +1 -1
- package/dist/cli/commands/seller/start.js +39 -3
- package/dist/cli/commands/seller/start.js.map +1 -1
- package/dist/cli/commands/seller/start.test.js +46 -1
- package/dist/cli/commands/seller/start.test.js.map +1 -1
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +6 -0
- package/dist/config/loader.js.map +1 -1
- package/dist/config/loader.test.js +33 -0
- package/dist/config/loader.test.js.map +1 -1
- package/dist/config/service-metadata.d.ts +5 -0
- package/dist/config/service-metadata.d.ts.map +1 -0
- package/dist/config/service-metadata.js +47 -0
- package/dist/config/service-metadata.js.map +1 -0
- package/dist/config/service-metadata.test.d.ts +2 -0
- package/dist/config/service-metadata.test.d.ts.map +1 -0
- package/dist/config/service-metadata.test.js +18 -0
- package/dist/config/service-metadata.test.js.map +1 -0
- package/dist/config/types.d.ts +5 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/config/validation.d.ts.map +1 -1
- package/dist/config/validation.js +2 -0
- package/dist/config/validation.js.map +1 -1
- package/dist/proxy/buyer-proxy.d.ts +76 -9
- package/dist/proxy/buyer-proxy.d.ts.map +1 -1
- package/dist/proxy/buyer-proxy.js +508 -68
- package/dist/proxy/buyer-proxy.js.map +1 -1
- package/dist/proxy/buyer-proxy.test.js +416 -13
- package/dist/proxy/buyer-proxy.test.js.map +1 -1
- package/dist/proxy/conversation-store.d.ts +6 -1
- package/dist/proxy/conversation-store.d.ts.map +1 -1
- package/dist/proxy/conversation-store.js +9 -2
- package/dist/proxy/conversation-store.js.map +1 -1
- package/dist/proxy/conversation-store.test.js +24 -0
- package/dist/proxy/conversation-store.test.js.map +1 -1
- package/dist/proxy/peer-attribution.d.ts +100 -0
- package/dist/proxy/peer-attribution.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.js +175 -0
- package/dist/proxy/peer-attribution.js.map +1 -0
- package/dist/proxy/peer-attribution.test.d.ts +2 -0
- package/dist/proxy/peer-attribution.test.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.test.js +140 -0
- package/dist/proxy/peer-attribution.test.js.map +1 -0
- package/dist/proxy/peer-health.d.ts +73 -0
- package/dist/proxy/peer-health.d.ts.map +1 -0
- package/dist/proxy/peer-health.js +182 -0
- package/dist/proxy/peer-health.js.map +1 -0
- package/dist/proxy/peer-health.test.d.ts +2 -0
- package/dist/proxy/peer-health.test.d.ts.map +1 -0
- package/dist/proxy/peer-health.test.js +197 -0
- package/dist/proxy/peer-health.test.js.map +1 -0
- package/dist/proxy/routing-parameters.test.d.ts +2 -0
- package/dist/proxy/routing-parameters.test.d.ts.map +1 -0
- package/dist/proxy/routing-parameters.test.js +97 -0
- package/dist/proxy/routing-parameters.test.js.map +1 -0
- package/dist/proxy/routing.d.ts +8 -0
- package/dist/proxy/routing.d.ts.map +1 -1
- package/dist/proxy/routing.js +36 -1
- package/dist/proxy/routing.js.map +1 -1
- package/dist/proxy/service-api-adapter.d.ts +1 -1
- package/dist/proxy/service-api-adapter.d.ts.map +1 -1
- package/dist/proxy/service-api-adapter.js +1 -1
- package/dist/proxy/service-api-adapter.js.map +1 -1
- package/package.json +6 -6
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deciding whether a failed request is the *peer's* fault.
|
|
3
|
+
*
|
|
4
|
+
* The dispatch path funnels nearly every error into a single catch, where a
|
|
5
|
+
* dead seller is indistinguishable from a dropped wifi connection, a suspended
|
|
6
|
+
* laptop, an unreachable chain RPC, or one of our own state-machine bugs. If we
|
|
7
|
+
* cooled a peer down on every failure, closing the laptop lid would exile the
|
|
8
|
+
* entire peer set — and the cooldown is persisted, so it would survive the
|
|
9
|
+
* restart too.
|
|
10
|
+
*
|
|
11
|
+
* So attribution is evidence-based and fails safe: a failure escalates a peer's
|
|
12
|
+
* cooldown only when we have positive evidence that the buyer itself is
|
|
13
|
+
* healthy. Three independent gates provide that evidence, and each one alone is
|
|
14
|
+
* enough to stop a buyer-side outage from mass-penalizing innocent peers.
|
|
15
|
+
*/
|
|
16
|
+
/** A success on another peer older than this no longer vouches for the buyer. */
|
|
17
|
+
export declare const CORROBORATION_WINDOW_MS: number;
|
|
18
|
+
/** Distinct peers that must fail together before we blame ourselves instead. */
|
|
19
|
+
export declare const CORRELATED_DISTINCT_PEERS = 3;
|
|
20
|
+
/** How close together those failures must land to count as one outage. */
|
|
21
|
+
export declare const CORRELATION_WINDOW_MS = 60000;
|
|
22
|
+
/** How long we keep suppressing after concluding the buyer is the problem. */
|
|
23
|
+
export declare const CORRELATION_SUPPRESSION_MS = 60000;
|
|
24
|
+
/** Heartbeat cadence used to notice the wall clock jumping forward. */
|
|
25
|
+
export declare const HEARTBEAT_MS = 10000;
|
|
26
|
+
/** Extra slack past the heartbeat before a gap reads as suspend, not lag. */
|
|
27
|
+
export declare const SUSPEND_JUMP_MS = 30000;
|
|
28
|
+
/** Quarantine after a detected suspend, covering the wake-up failure storm. */
|
|
29
|
+
export declare const SUSPEND_QUARANTINE_MS = 60000;
|
|
30
|
+
/**
|
|
31
|
+
* Slack applied before the last good heartbeat when rolling back a suspend. The
|
|
32
|
+
* failures that matter are the ones recorded during the untrusted gap, plus any
|
|
33
|
+
* that landed just as the machine was going down.
|
|
34
|
+
*/
|
|
35
|
+
export declare const ROLLBACK_LOOKBACK_MS = 15000;
|
|
36
|
+
export type AttributionSuppressionReason = 'no-corroborating-success' | 'correlated-failure' | 'suspend-detected' | 'buyer-fault' | 'non-escalating-reason';
|
|
37
|
+
export type AttributionVerdict = {
|
|
38
|
+
escalate: true;
|
|
39
|
+
} | {
|
|
40
|
+
escalate: false;
|
|
41
|
+
suppressedBy: AttributionSuppressionReason;
|
|
42
|
+
};
|
|
43
|
+
export type AttributionInput = {
|
|
44
|
+
peerId: string;
|
|
45
|
+
/** False for reasons that are informational only, e.g. 429 or a buyer fault. */
|
|
46
|
+
reasonEscalates: boolean;
|
|
47
|
+
/** Fault attribution derived from the thrown error, when it is known. */
|
|
48
|
+
fault?: 'buyer' | 'peer' | 'unknown';
|
|
49
|
+
now: number;
|
|
50
|
+
};
|
|
51
|
+
export type AttributionSnapshot = {
|
|
52
|
+
lastAnySuccessAt: number;
|
|
53
|
+
/** 0 when not currently suppressing. */
|
|
54
|
+
suppressedUntil: number;
|
|
55
|
+
lastSuppressedBy: AttributionSuppressionReason | null;
|
|
56
|
+
};
|
|
57
|
+
/**
|
|
58
|
+
* Tracks the buyer-health evidence needed to attribute failures. Deliberately
|
|
59
|
+
* holds no peer-health state of its own: it answers "may this failure count
|
|
60
|
+
* against the peer?" and hands back the set of peers whose recent failures the
|
|
61
|
+
* caller should roll back when it turns out the buyer was the problem.
|
|
62
|
+
*/
|
|
63
|
+
export declare class PeerAttributionTracker {
|
|
64
|
+
private _lastAnySuccessAt;
|
|
65
|
+
private readonly _lastSuccessByPeer;
|
|
66
|
+
private _recentFailures;
|
|
67
|
+
private _suppressedUntil;
|
|
68
|
+
private _lastSuppressedBy;
|
|
69
|
+
private _lastHeartbeatAt;
|
|
70
|
+
/**
|
|
71
|
+
* Record proof of life. Any HTTP response from a peer counts, including error
|
|
72
|
+
* statuses — the peer answered, so our network, DHT, RPC and wallet are fine.
|
|
73
|
+
*/
|
|
74
|
+
recordSuccess(peerId: string, now: number): void;
|
|
75
|
+
/**
|
|
76
|
+
* Decide whether a failure may escalate, and register it toward correlation
|
|
77
|
+
* detection. Returns the peers to roll back when this failure is the one that
|
|
78
|
+
* reveals a buyer-side outage.
|
|
79
|
+
*/
|
|
80
|
+
classify(input: AttributionInput): {
|
|
81
|
+
verdict: AttributionVerdict;
|
|
82
|
+
rollbackPeerIds: string[];
|
|
83
|
+
};
|
|
84
|
+
/**
|
|
85
|
+
* Heartbeat tick. A wall-clock gap far larger than the cadence means the
|
|
86
|
+
* process was suspended; every pending timeout and keepalive fires at once on
|
|
87
|
+
* wake, so quarantine and roll back the resulting storm.
|
|
88
|
+
*
|
|
89
|
+
* Returns the peers to roll back, or null when nothing unusual happened.
|
|
90
|
+
*/
|
|
91
|
+
onHeartbeat(now: number): {
|
|
92
|
+
rollbackPeerIds: string[];
|
|
93
|
+
} | null;
|
|
94
|
+
snapshot(now: number): AttributionSnapshot;
|
|
95
|
+
/** Whether the buyer currently has evidence it is healthy at all. */
|
|
96
|
+
isBuyerHealthy(now: number): boolean;
|
|
97
|
+
private _hasCorroboratingSuccess;
|
|
98
|
+
private _pruneFailures;
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=peer-attribution.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"peer-attribution.d.ts","sourceRoot":"","sources":["../../src/proxy/peer-attribution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,iFAAiF;AACjF,eAAO,MAAM,uBAAuB,QAAc,CAAA;AAClD,gFAAgF;AAChF,eAAO,MAAM,yBAAyB,IAAI,CAAA;AAC1C,0EAA0E;AAC1E,eAAO,MAAM,qBAAqB,QAAS,CAAA;AAC3C,8EAA8E;AAC9E,eAAO,MAAM,0BAA0B,QAAS,CAAA;AAChD,uEAAuE;AACvE,eAAO,MAAM,YAAY,QAAS,CAAA;AAClC,6EAA6E;AAC7E,eAAO,MAAM,eAAe,QAAS,CAAA;AACrC,+EAA+E;AAC/E,eAAO,MAAM,qBAAqB,QAAS,CAAA;AAC3C;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,QAAS,CAAA;AAE1C,MAAM,MAAM,4BAA4B,GACpC,0BAA0B,GAC1B,oBAAoB,GACpB,kBAAkB,GAClB,aAAa,GACb,uBAAuB,CAAA;AAE3B,MAAM,MAAM,kBAAkB,GAC1B;IAAE,QAAQ,EAAE,IAAI,CAAA;CAAE,GAClB;IAAE,QAAQ,EAAE,KAAK,CAAC;IAAC,YAAY,EAAE,4BAA4B,CAAA;CAAE,CAAA;AAEnE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAA;IACd,gFAAgF;IAChF,eAAe,EAAE,OAAO,CAAA;IACxB,yEAAyE;IACzE,KAAK,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAAA;IACpC,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAID,MAAM,MAAM,mBAAmB,GAAG;IAChC,gBAAgB,EAAE,MAAM,CAAA;IACxB,wCAAwC;IACxC,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,4BAA4B,GAAG,IAAI,CAAA;CACtD,CAAA;AAKD;;;;;GAKG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,iBAAiB,CAAI;IAC7B,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA4B;IAC/D,OAAO,CAAC,eAAe,CAAoB;IAC3C,OAAO,CAAC,gBAAgB,CAAI;IAC5B,OAAO,CAAC,iBAAiB,CAA4C;IACrE,OAAO,CAAC,gBAAgB,CAAsB;IAE9C;;;OAGG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAYhD;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,gBAAgB,GAAG;QAAE,OAAO,EAAE,kBAAkB,CAAC;QAAC,eAAe,EAAE,MAAM,EAAE,CAAA;KAAE;IAsD7F;;;;;;OAMG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG;QAAE,eAAe,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI;IAyB9D,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,mBAAmB;IAQ1C,qEAAqE;IACrE,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAMpC,OAAO,CAAC,wBAAwB;IAQhC,OAAO,CAAC,cAAc;CAMvB"}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deciding whether a failed request is the *peer's* fault.
|
|
3
|
+
*
|
|
4
|
+
* The dispatch path funnels nearly every error into a single catch, where a
|
|
5
|
+
* dead seller is indistinguishable from a dropped wifi connection, a suspended
|
|
6
|
+
* laptop, an unreachable chain RPC, or one of our own state-machine bugs. If we
|
|
7
|
+
* cooled a peer down on every failure, closing the laptop lid would exile the
|
|
8
|
+
* entire peer set — and the cooldown is persisted, so it would survive the
|
|
9
|
+
* restart too.
|
|
10
|
+
*
|
|
11
|
+
* So attribution is evidence-based and fails safe: a failure escalates a peer's
|
|
12
|
+
* cooldown only when we have positive evidence that the buyer itself is
|
|
13
|
+
* healthy. Three independent gates provide that evidence, and each one alone is
|
|
14
|
+
* enough to stop a buyer-side outage from mass-penalizing innocent peers.
|
|
15
|
+
*/
|
|
16
|
+
/** A success on another peer older than this no longer vouches for the buyer. */
|
|
17
|
+
export const CORROBORATION_WINDOW_MS = 10 * 60_000;
|
|
18
|
+
/** Distinct peers that must fail together before we blame ourselves instead. */
|
|
19
|
+
export const CORRELATED_DISTINCT_PEERS = 3;
|
|
20
|
+
/** How close together those failures must land to count as one outage. */
|
|
21
|
+
export const CORRELATION_WINDOW_MS = 60_000;
|
|
22
|
+
/** How long we keep suppressing after concluding the buyer is the problem. */
|
|
23
|
+
export const CORRELATION_SUPPRESSION_MS = 60_000;
|
|
24
|
+
/** Heartbeat cadence used to notice the wall clock jumping forward. */
|
|
25
|
+
export const HEARTBEAT_MS = 10_000;
|
|
26
|
+
/** Extra slack past the heartbeat before a gap reads as suspend, not lag. */
|
|
27
|
+
export const SUSPEND_JUMP_MS = 30_000;
|
|
28
|
+
/** Quarantine after a detected suspend, covering the wake-up failure storm. */
|
|
29
|
+
export const SUSPEND_QUARANTINE_MS = 60_000;
|
|
30
|
+
/**
|
|
31
|
+
* Slack applied before the last good heartbeat when rolling back a suspend. The
|
|
32
|
+
* failures that matter are the ones recorded during the untrusted gap, plus any
|
|
33
|
+
* that landed just as the machine was going down.
|
|
34
|
+
*/
|
|
35
|
+
export const ROLLBACK_LOOKBACK_MS = 15_000;
|
|
36
|
+
/** Bound on remembered per-peer successes; only the freshest ones matter. */
|
|
37
|
+
const MAX_TRACKED_SUCCESS_PEERS = 512;
|
|
38
|
+
/**
|
|
39
|
+
* Tracks the buyer-health evidence needed to attribute failures. Deliberately
|
|
40
|
+
* holds no peer-health state of its own: it answers "may this failure count
|
|
41
|
+
* against the peer?" and hands back the set of peers whose recent failures the
|
|
42
|
+
* caller should roll back when it turns out the buyer was the problem.
|
|
43
|
+
*/
|
|
44
|
+
export class PeerAttributionTracker {
|
|
45
|
+
_lastAnySuccessAt = 0;
|
|
46
|
+
_lastSuccessByPeer = new Map();
|
|
47
|
+
_recentFailures = [];
|
|
48
|
+
_suppressedUntil = 0;
|
|
49
|
+
_lastSuppressedBy = null;
|
|
50
|
+
_lastHeartbeatAt = null;
|
|
51
|
+
/**
|
|
52
|
+
* Record proof of life. Any HTTP response from a peer counts, including error
|
|
53
|
+
* statuses — the peer answered, so our network, DHT, RPC and wallet are fine.
|
|
54
|
+
*/
|
|
55
|
+
recordSuccess(peerId, now) {
|
|
56
|
+
// Re-insert so map order stays least-recently-successful first.
|
|
57
|
+
this._lastSuccessByPeer.delete(peerId);
|
|
58
|
+
this._lastSuccessByPeer.set(peerId, now);
|
|
59
|
+
while (this._lastSuccessByPeer.size > MAX_TRACKED_SUCCESS_PEERS) {
|
|
60
|
+
const oldest = this._lastSuccessByPeer.keys().next();
|
|
61
|
+
if (oldest.done)
|
|
62
|
+
break;
|
|
63
|
+
this._lastSuccessByPeer.delete(oldest.value);
|
|
64
|
+
}
|
|
65
|
+
if (now > this._lastAnySuccessAt)
|
|
66
|
+
this._lastAnySuccessAt = now;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Decide whether a failure may escalate, and register it toward correlation
|
|
70
|
+
* detection. Returns the peers to roll back when this failure is the one that
|
|
71
|
+
* reveals a buyer-side outage.
|
|
72
|
+
*/
|
|
73
|
+
classify(input) {
|
|
74
|
+
const { peerId, reasonEscalates, fault, now } = input;
|
|
75
|
+
if (fault === 'buyer') {
|
|
76
|
+
return { verdict: { escalate: false, suppressedBy: 'buyer-fault' }, rollbackPeerIds: [] };
|
|
77
|
+
}
|
|
78
|
+
if (!reasonEscalates) {
|
|
79
|
+
return { verdict: { escalate: false, suppressedBy: 'non-escalating-reason' }, rollbackPeerIds: [] };
|
|
80
|
+
}
|
|
81
|
+
this._pruneFailures(now);
|
|
82
|
+
this._recentFailures.push({ peerId, at: now });
|
|
83
|
+
// Gate 1c: the machine slept. Nothing recorded around the wake-up is
|
|
84
|
+
// trustworthy, including failures that landed just before we noticed.
|
|
85
|
+
if (now < this._suppressedUntil && this._lastSuppressedBy === 'suspend-detected') {
|
|
86
|
+
return {
|
|
87
|
+
verdict: { escalate: false, suppressedBy: 'suspend-detected' },
|
|
88
|
+
rollbackPeerIds: [],
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
// Gate 1b: several distinct peers failing at once means the common factor
|
|
92
|
+
// is us. Trip suppression and undo what this outage already recorded.
|
|
93
|
+
const distinctPeers = new Set(this._recentFailures.map((f) => f.peerId));
|
|
94
|
+
if (distinctPeers.size >= CORRELATED_DISTINCT_PEERS) {
|
|
95
|
+
this._suppressedUntil = now + CORRELATION_SUPPRESSION_MS;
|
|
96
|
+
this._lastSuppressedBy = 'correlated-failure';
|
|
97
|
+
return {
|
|
98
|
+
verdict: { escalate: false, suppressedBy: 'correlated-failure' },
|
|
99
|
+
rollbackPeerIds: [...distinctPeers],
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
if (now < this._suppressedUntil) {
|
|
103
|
+
return {
|
|
104
|
+
verdict: { escalate: false, suppressedBy: this._lastSuppressedBy ?? 'correlated-failure' },
|
|
105
|
+
rollbackPeerIds: [],
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
// Gate 1a: without a recent success on some *other* peer we have no proof
|
|
109
|
+
// the buyer is healthy, so we cannot fairly blame this one. A buyer that
|
|
110
|
+
// only ever talks to one peer never escalates it — correct, since there is
|
|
111
|
+
// nothing to fail over to anyway.
|
|
112
|
+
if (!this._hasCorroboratingSuccess(peerId, now)) {
|
|
113
|
+
return {
|
|
114
|
+
verdict: { escalate: false, suppressedBy: 'no-corroborating-success' },
|
|
115
|
+
rollbackPeerIds: [],
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
return { verdict: { escalate: true }, rollbackPeerIds: [] };
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Heartbeat tick. A wall-clock gap far larger than the cadence means the
|
|
122
|
+
* process was suspended; every pending timeout and keepalive fires at once on
|
|
123
|
+
* wake, so quarantine and roll back the resulting storm.
|
|
124
|
+
*
|
|
125
|
+
* Returns the peers to roll back, or null when nothing unusual happened.
|
|
126
|
+
*/
|
|
127
|
+
onHeartbeat(now) {
|
|
128
|
+
const previous = this._lastHeartbeatAt;
|
|
129
|
+
this._lastHeartbeatAt = now;
|
|
130
|
+
if (previous === null)
|
|
131
|
+
return null;
|
|
132
|
+
const elapsed = now - previous;
|
|
133
|
+
if (elapsed <= HEARTBEAT_MS + SUSPEND_JUMP_MS)
|
|
134
|
+
return null;
|
|
135
|
+
this._suppressedUntil = now + SUSPEND_QUARANTINE_MS;
|
|
136
|
+
this._lastSuppressedBy = 'suspend-detected';
|
|
137
|
+
// Everything recorded since the last trustworthy tick is suspect: the
|
|
138
|
+
// suspend itself is what killed those requests, not the peers.
|
|
139
|
+
const untrustedFrom = previous - ROLLBACK_LOOKBACK_MS;
|
|
140
|
+
const rollbackPeerIds = [
|
|
141
|
+
...new Set(this._recentFailures
|
|
142
|
+
.filter((f) => f.at >= untrustedFrom)
|
|
143
|
+
.map((f) => f.peerId)),
|
|
144
|
+
];
|
|
145
|
+
this._recentFailures = [];
|
|
146
|
+
return { rollbackPeerIds };
|
|
147
|
+
}
|
|
148
|
+
snapshot(now) {
|
|
149
|
+
return {
|
|
150
|
+
lastAnySuccessAt: this._lastAnySuccessAt,
|
|
151
|
+
suppressedUntil: now < this._suppressedUntil ? this._suppressedUntil : 0,
|
|
152
|
+
lastSuppressedBy: this._lastSuppressedBy,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
/** Whether the buyer currently has evidence it is healthy at all. */
|
|
156
|
+
isBuyerHealthy(now) {
|
|
157
|
+
return now >= this._suppressedUntil
|
|
158
|
+
&& this._lastAnySuccessAt > 0
|
|
159
|
+
&& now - this._lastAnySuccessAt <= CORROBORATION_WINDOW_MS;
|
|
160
|
+
}
|
|
161
|
+
_hasCorroboratingSuccess(failedPeerId, now) {
|
|
162
|
+
for (const [peerId, at] of this._lastSuccessByPeer) {
|
|
163
|
+
if (peerId === failedPeerId)
|
|
164
|
+
continue;
|
|
165
|
+
if (now - at <= CORROBORATION_WINDOW_MS && at <= now)
|
|
166
|
+
return true;
|
|
167
|
+
}
|
|
168
|
+
return false;
|
|
169
|
+
}
|
|
170
|
+
_pruneFailures(now) {
|
|
171
|
+
// A backwards clock jump would otherwise strand marks in the buffer.
|
|
172
|
+
this._recentFailures = this._recentFailures.filter((f) => f.at <= now && now - f.at <= CORRELATION_WINDOW_MS);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=peer-attribution.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"peer-attribution.js","sourceRoot":"","sources":["../../src/proxy/peer-attribution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,iFAAiF;AACjF,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,MAAM,CAAA;AAClD,gFAAgF;AAChF,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAA;AAC1C,0EAA0E;AAC1E,MAAM,CAAC,MAAM,qBAAqB,GAAG,MAAM,CAAA;AAC3C,8EAA8E;AAC9E,MAAM,CAAC,MAAM,0BAA0B,GAAG,MAAM,CAAA;AAChD,uEAAuE;AACvE,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,CAAA;AAClC,6EAA6E;AAC7E,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAA;AACrC,+EAA+E;AAC/E,MAAM,CAAC,MAAM,qBAAqB,GAAG,MAAM,CAAA;AAC3C;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAA;AA+B1C,6EAA6E;AAC7E,MAAM,yBAAyB,GAAG,GAAG,CAAA;AAErC;;;;;GAKG;AACH,MAAM,OAAO,sBAAsB;IACzB,iBAAiB,GAAG,CAAC,CAAA;IACZ,kBAAkB,GAAG,IAAI,GAAG,EAAkB,CAAA;IACvD,eAAe,GAAkB,EAAE,CAAA;IACnC,gBAAgB,GAAG,CAAC,CAAA;IACpB,iBAAiB,GAAwC,IAAI,CAAA;IAC7D,gBAAgB,GAAkB,IAAI,CAAA;IAE9C;;;OAGG;IACH,aAAa,CAAC,MAAc,EAAE,GAAW;QACvC,gEAAgE;QAChE,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QACtC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QACxC,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,GAAG,yBAAyB,EAAE,CAAC;YAChE,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;YACpD,IAAI,MAAM,CAAC,IAAI;gBAAE,MAAK;YACtB,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC9C,CAAC;QACD,IAAI,GAAG,GAAG,IAAI,CAAC,iBAAiB;YAAE,IAAI,CAAC,iBAAiB,GAAG,GAAG,CAAA;IAChE,CAAC;IAED;;;;OAIG;IACH,QAAQ,CAAC,KAAuB;QAC9B,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,KAAK,CAAA;QAErD,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACtB,OAAO,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,CAAA;QAC3F,CAAC;QACD,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,uBAAuB,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,CAAA;QACrG,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAA;QACxB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;QAE9C,qEAAqE;QACrE,sEAAsE;QACtE,IAAI,GAAG,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,iBAAiB,KAAK,kBAAkB,EAAE,CAAC;YACjF,OAAO;gBACL,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE;gBAC9D,eAAe,EAAE,EAAE;aACpB,CAAA;QACH,CAAC;QAED,0EAA0E;QAC1E,sEAAsE;QACtE,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QACxE,IAAI,aAAa,CAAC,IAAI,IAAI,yBAAyB,EAAE,CAAC;YACpD,IAAI,CAAC,gBAAgB,GAAG,GAAG,GAAG,0BAA0B,CAAA;YACxD,IAAI,CAAC,iBAAiB,GAAG,oBAAoB,CAAA;YAC7C,OAAO;gBACL,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE;gBAChE,eAAe,EAAE,CAAC,GAAG,aAAa,CAAC;aACpC,CAAA;QACH,CAAC;QACD,IAAI,GAAG,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAChC,OAAO;gBACL,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,CAAC,iBAAiB,IAAI,oBAAoB,EAAE;gBAC1F,eAAe,EAAE,EAAE;aACpB,CAAA;QACH,CAAC;QAED,0EAA0E;QAC1E,yEAAyE;QACzE,2EAA2E;QAC3E,kCAAkC;QAClC,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YAChD,OAAO;gBACL,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,0BAA0B,EAAE;gBACtE,eAAe,EAAE,EAAE;aACpB,CAAA;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,CAAA;IAC7D,CAAC;IAED;;;;;;OAMG;IACH,WAAW,CAAC,GAAW;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAA;QACtC,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAA;QAC3B,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA;QAElC,MAAM,OAAO,GAAG,GAAG,GAAG,QAAQ,CAAA;QAC9B,IAAI,OAAO,IAAI,YAAY,GAAG,eAAe;YAAE,OAAO,IAAI,CAAA;QAE1D,IAAI,CAAC,gBAAgB,GAAG,GAAG,GAAG,qBAAqB,CAAA;QACnD,IAAI,CAAC,iBAAiB,GAAG,kBAAkB,CAAA;QAE3C,sEAAsE;QACtE,+DAA+D;QAC/D,MAAM,aAAa,GAAG,QAAQ,GAAG,oBAAoB,CAAA;QACrD,MAAM,eAAe,GAAG;YACtB,GAAG,IAAI,GAAG,CACR,IAAI,CAAC,eAAe;iBACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,aAAa,CAAC;iBACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CACxB;SACF,CAAA;QACD,IAAI,CAAC,eAAe,GAAG,EAAE,CAAA;QACzB,OAAO,EAAE,eAAe,EAAE,CAAA;IAC5B,CAAC;IAED,QAAQ,CAAC,GAAW;QAClB,OAAO;YACL,gBAAgB,EAAE,IAAI,CAAC,iBAAiB;YACxC,eAAe,EAAE,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACxE,gBAAgB,EAAE,IAAI,CAAC,iBAAiB;SACzC,CAAA;IACH,CAAC;IAED,qEAAqE;IACrE,cAAc,CAAC,GAAW;QACxB,OAAO,GAAG,IAAI,IAAI,CAAC,gBAAgB;eAC9B,IAAI,CAAC,iBAAiB,GAAG,CAAC;eAC1B,GAAG,GAAG,IAAI,CAAC,iBAAiB,IAAI,uBAAuB,CAAA;IAC9D,CAAC;IAEO,wBAAwB,CAAC,YAAoB,EAAE,GAAW;QAChE,KAAK,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,MAAM,KAAK,YAAY;gBAAE,SAAQ;YACrC,IAAI,GAAG,GAAG,EAAE,IAAI,uBAAuB,IAAI,EAAE,IAAI,GAAG;gBAAE,OAAO,IAAI,CAAA;QACnE,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,cAAc,CAAC,GAAW;QAChC,qEAAqE;QACrE,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,EAAE,IAAI,qBAAqB,CAC1D,CAAA;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"peer-attribution.test.d.ts","sourceRoot":"","sources":["../../src/proxy/peer-attribution.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import test from 'node:test';
|
|
3
|
+
import { PeerAttributionTracker, CORROBORATION_WINDOW_MS, CORRELATION_WINDOW_MS, HEARTBEAT_MS, SUSPEND_JUMP_MS, } from './peer-attribution.js';
|
|
4
|
+
const A = 'a'.repeat(40);
|
|
5
|
+
const B = 'b'.repeat(40);
|
|
6
|
+
const C = 'c'.repeat(40);
|
|
7
|
+
const D = 'd'.repeat(40);
|
|
8
|
+
function fail(tracker, peerId, now) {
|
|
9
|
+
return tracker.classify({ peerId, reasonEscalates: true, fault: 'unknown', now });
|
|
10
|
+
}
|
|
11
|
+
test('a failure never escalates without a prior success anywhere', () => {
|
|
12
|
+
const tracker = new PeerAttributionTracker();
|
|
13
|
+
const { verdict } = fail(tracker, A, 1_000_000);
|
|
14
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'no-corroborating-success' });
|
|
15
|
+
});
|
|
16
|
+
test('a recent success on another peer lets the failure escalate', () => {
|
|
17
|
+
const tracker = new PeerAttributionTracker();
|
|
18
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
19
|
+
const { verdict } = fail(tracker, A, 1_001_000);
|
|
20
|
+
assert.deepEqual(verdict, { escalate: true });
|
|
21
|
+
});
|
|
22
|
+
test('a success on the failing peer itself does not vouch for the buyer', () => {
|
|
23
|
+
const tracker = new PeerAttributionTracker();
|
|
24
|
+
tracker.recordSuccess(A, 1_000_000);
|
|
25
|
+
const { verdict } = fail(tracker, A, 1_001_000);
|
|
26
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'no-corroborating-success' });
|
|
27
|
+
});
|
|
28
|
+
test('a success older than the corroboration window stops vouching', () => {
|
|
29
|
+
const tracker = new PeerAttributionTracker();
|
|
30
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
31
|
+
const { verdict } = fail(tracker, A, 1_000_000 + CORROBORATION_WINDOW_MS + 1);
|
|
32
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'no-corroborating-success' });
|
|
33
|
+
});
|
|
34
|
+
test('three distinct peers failing together blames the buyer and rolls them back', () => {
|
|
35
|
+
const tracker = new PeerAttributionTracker();
|
|
36
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
37
|
+
assert.deepEqual(fail(tracker, A, 1_000_100).verdict, { escalate: true });
|
|
38
|
+
assert.deepEqual(fail(tracker, B, 1_000_200).verdict, { escalate: true });
|
|
39
|
+
const third = fail(tracker, C, 1_000_300);
|
|
40
|
+
assert.deepEqual(third.verdict, { escalate: false, suppressedBy: 'correlated-failure' });
|
|
41
|
+
assert.deepEqual([...third.rollbackPeerIds].sort(), [A, B, C].sort());
|
|
42
|
+
});
|
|
43
|
+
test('repeated failures on a single peer do not trip correlation suppression', () => {
|
|
44
|
+
const tracker = new PeerAttributionTracker();
|
|
45
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
46
|
+
for (let i = 1; i <= 5; i += 1) {
|
|
47
|
+
const { verdict } = fail(tracker, A, 1_000_000 + i * 1_000);
|
|
48
|
+
assert.deepEqual(verdict, { escalate: true }, `failure ${i} should still escalate`);
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
test('failures spread beyond the correlation window are not correlated', () => {
|
|
52
|
+
const tracker = new PeerAttributionTracker();
|
|
53
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
54
|
+
// Three distinct peers, but far enough apart that they are three separate
|
|
55
|
+
// incidents rather than one buyer-side outage. Each is judged on its own.
|
|
56
|
+
const spacing = CORRELATION_WINDOW_MS + 1_000;
|
|
57
|
+
assert.deepEqual(fail(tracker, A, 1_000_100).verdict, { escalate: true });
|
|
58
|
+
assert.deepEqual(fail(tracker, B, 1_000_100 + spacing).verdict, { escalate: true });
|
|
59
|
+
assert.deepEqual(fail(tracker, C, 1_000_100 + spacing * 2).verdict, { escalate: true });
|
|
60
|
+
});
|
|
61
|
+
test('suppression persists for the window after a correlation trip', () => {
|
|
62
|
+
const tracker = new PeerAttributionTracker();
|
|
63
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
64
|
+
fail(tracker, A, 1_000_100);
|
|
65
|
+
fail(tracker, B, 1_000_200);
|
|
66
|
+
fail(tracker, C, 1_000_300);
|
|
67
|
+
const later = fail(tracker, A, 1_000_400);
|
|
68
|
+
assert.deepEqual(later.verdict, { escalate: false, suppressedBy: 'correlated-failure' });
|
|
69
|
+
});
|
|
70
|
+
test('a buyer-attributed fault never escalates, even with corroboration', () => {
|
|
71
|
+
const tracker = new PeerAttributionTracker();
|
|
72
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
73
|
+
const { verdict } = tracker.classify({
|
|
74
|
+
peerId: A, reasonEscalates: true, fault: 'buyer', now: 1_001_000,
|
|
75
|
+
});
|
|
76
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'buyer-fault' });
|
|
77
|
+
});
|
|
78
|
+
test('a non-escalating reason is recorded but never cools a peer down', () => {
|
|
79
|
+
const tracker = new PeerAttributionTracker();
|
|
80
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
81
|
+
const { verdict } = tracker.classify({
|
|
82
|
+
peerId: A, reasonEscalates: false, fault: 'peer', now: 1_001_000,
|
|
83
|
+
});
|
|
84
|
+
assert.deepEqual(verdict, { escalate: false, suppressedBy: 'non-escalating-reason' });
|
|
85
|
+
});
|
|
86
|
+
test('a wall-clock jump reads as suspend, quarantines, and rolls back the storm', () => {
|
|
87
|
+
const tracker = new PeerAttributionTracker();
|
|
88
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
89
|
+
assert.equal(tracker.onHeartbeat(1_000_000), null);
|
|
90
|
+
// Two peers fail as the machine goes down; the jump is noticed on the next tick.
|
|
91
|
+
fail(tracker, A, 1_000_100);
|
|
92
|
+
fail(tracker, B, 1_000_200);
|
|
93
|
+
const wake = 1_000_200 + HEARTBEAT_MS + SUSPEND_JUMP_MS + 1_000;
|
|
94
|
+
const result = tracker.onHeartbeat(wake);
|
|
95
|
+
assert.ok(result, 'expected the clock jump to be detected');
|
|
96
|
+
assert.deepEqual([...result.rollbackPeerIds].sort(), [A, B].sort());
|
|
97
|
+
const afterWake = fail(tracker, C, wake + 100);
|
|
98
|
+
assert.deepEqual(afterWake.verdict, { escalate: false, suppressedBy: 'suspend-detected' });
|
|
99
|
+
});
|
|
100
|
+
test('a normal heartbeat cadence is not mistaken for suspend', () => {
|
|
101
|
+
const tracker = new PeerAttributionTracker();
|
|
102
|
+
assert.equal(tracker.onHeartbeat(1_000_000), null);
|
|
103
|
+
assert.equal(tracker.onHeartbeat(1_000_000 + HEARTBEAT_MS), null);
|
|
104
|
+
assert.equal(tracker.onHeartbeat(1_000_000 + HEARTBEAT_MS * 2 + 500), null);
|
|
105
|
+
});
|
|
106
|
+
test('isBuyerHealthy tracks corroboration and suppression', () => {
|
|
107
|
+
const tracker = new PeerAttributionTracker();
|
|
108
|
+
assert.equal(tracker.isBuyerHealthy(1_000_000), false);
|
|
109
|
+
tracker.recordSuccess(B, 1_000_000);
|
|
110
|
+
assert.equal(tracker.isBuyerHealthy(1_000_100), true);
|
|
111
|
+
assert.equal(tracker.isBuyerHealthy(1_000_000 + CORROBORATION_WINDOW_MS + 1), false);
|
|
112
|
+
fail(tracker, A, 1_000_100);
|
|
113
|
+
fail(tracker, B, 1_000_200);
|
|
114
|
+
fail(tracker, C, 1_000_300);
|
|
115
|
+
assert.equal(tracker.isBuyerHealthy(1_000_400), false);
|
|
116
|
+
});
|
|
117
|
+
test('snapshot reports the active suppression window and clears once it lapses', () => {
|
|
118
|
+
const tracker = new PeerAttributionTracker();
|
|
119
|
+
tracker.recordSuccess(D, 1_000_000);
|
|
120
|
+
fail(tracker, A, 1_000_100);
|
|
121
|
+
fail(tracker, B, 1_000_200);
|
|
122
|
+
fail(tracker, C, 1_000_300);
|
|
123
|
+
const during = tracker.snapshot(1_000_400);
|
|
124
|
+
assert.ok(during.suppressedUntil > 1_000_400);
|
|
125
|
+
assert.equal(during.lastSuppressedBy, 'correlated-failure');
|
|
126
|
+
assert.equal(during.lastAnySuccessAt, 1_000_000);
|
|
127
|
+
const after = tracker.snapshot(1_000_300 + 120_000);
|
|
128
|
+
assert.equal(after.suppressedUntil, 0);
|
|
129
|
+
});
|
|
130
|
+
test('a backwards clock jump does not strand failures in the correlation buffer', () => {
|
|
131
|
+
const tracker = new PeerAttributionTracker();
|
|
132
|
+
tracker.recordSuccess(D, 2_000_000);
|
|
133
|
+
fail(tracker, A, 2_000_100);
|
|
134
|
+
fail(tracker, B, 2_000_200);
|
|
135
|
+
// Clock steps backwards past the earlier marks; they must be discarded, so
|
|
136
|
+
// this third distinct peer does not trip correlation on stale data.
|
|
137
|
+
const { verdict } = fail(tracker, C, 1_000_000);
|
|
138
|
+
assert.notEqual(verdict.escalate === false ? verdict.suppressedBy : null, 'correlated-failure');
|
|
139
|
+
});
|
|
140
|
+
//# sourceMappingURL=peer-attribution.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"peer-attribution.test.js","sourceRoot":"","sources":["../../src/proxy/peer-attribution.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAA;AACvC,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,EACrB,YAAY,EACZ,eAAe,GAChB,MAAM,uBAAuB,CAAA;AAE9B,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AACxB,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AACxB,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AACxB,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AAExB,SAAS,IAAI,CAAC,OAA+B,EAAE,MAAc,EAAE,GAAW;IACxE,OAAO,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAA;AACnF,CAAC;AAED,IAAI,CAAC,4DAA4D,EAAE,GAAG,EAAE;IACtE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC/C,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,0BAA0B,EAAE,CAAC,CAAA;AAC1F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,4DAA4D,EAAE,GAAG,EAAE;IACtE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC/C,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;AAC/C,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,mEAAmE,EAAE,GAAG,EAAE;IAC7E,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC/C,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,0BAA0B,EAAE,CAAC,CAAA;AAC1F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,8DAA8D,EAAE,GAAG,EAAE;IACxE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,GAAG,uBAAuB,GAAG,CAAC,CAAC,CAAA;IAC7E,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,0BAA0B,EAAE,CAAC,CAAA;AAC1F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,4EAA4E,EAAE,GAAG,EAAE;IACtF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IAEnC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IACzE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAEzE,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IACzC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE,CAAC,CAAA;IACxF,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;AACvE,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,wEAAwE,EAAE,GAAG,EAAE;IAClF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,CAAA;QAC3D,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,wBAAwB,CAAC,CAAA;IACrF,CAAC;AACH,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,kEAAkE,EAAE,GAAG,EAAE;IAC5E,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IAEnC,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,OAAO,GAAG,qBAAqB,GAAG,KAAK,CAAA;IAC7C,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IACzE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IACnF,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;AACzF,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,8DAA8D,EAAE,GAAG,EAAE;IACxE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAE3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IACzC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE,CAAC,CAAA;AAC1F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,mEAAmE,EAAE,GAAG,EAAE;IAC7E,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,MAAM,EAAE,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS;KACjE,CAAC,CAAA;IACF,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,CAAA;AAC7E,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,iEAAiE,EAAE,GAAG,EAAE;IAC3E,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,MAAM,EAAE,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS;KACjE,CAAC,CAAA;IACF,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,uBAAuB,EAAE,CAAC,CAAA;AACvF,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,2EAA2E,EAAE,GAAG,EAAE;IACrF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAA;IAElD,iFAAiF;IACjF,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAE3B,MAAM,IAAI,GAAG,SAAS,GAAG,YAAY,GAAG,eAAe,GAAG,KAAK,CAAA;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;IACxC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,wCAAwC,CAAC,CAAA;IAC3D,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IAEnE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,GAAG,GAAG,CAAC,CAAA;IAC9C,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,CAAC,CAAA;AAC5F,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,wDAAwD,EAAE,GAAG,EAAE;IAClE,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAA;IAClD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,GAAG,YAAY,CAAC,EAAE,IAAI,CAAC,CAAA;IACjE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,GAAG,YAAY,GAAG,CAAC,GAAG,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;AAC7E,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,qDAAqD,EAAE,GAAG,EAAE;IAC/D,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAA;IAEtD,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAA;IACrD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,GAAG,uBAAuB,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;IAEpF,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAA;AACxD,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,0EAA0E,EAAE,GAAG,EAAE;IACpF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAE3B,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;IAC1C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,CAAA;IAC7C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,CAAA;IAC3D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAA;IAEhD,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,GAAG,OAAO,CAAC,CAAA;IACnD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAA;AACxC,CAAC,CAAC,CAAA;AAEF,IAAI,CAAC,2EAA2E,EAAE,GAAG,EAAE;IACrF,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAA;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAE3B,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC/C,MAAM,CAAC,QAAQ,CACb,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EACxD,oBAAoB,CACrB,CAAA;AACH,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-peer failure bookkeeping and cooldown state for the buyer proxy.
|
|
3
|
+
*
|
|
4
|
+
* A peer that stops responding should stop being picked, but only temporarily:
|
|
5
|
+
* peers go offline for legitimate reasons, so there is no permanent auto-ban.
|
|
6
|
+
* Whether a failure is even allowed to escalate is decided separately, by
|
|
7
|
+
* `peer-attribution.ts` — this module owns the state machine, not the blame.
|
|
8
|
+
*
|
|
9
|
+
* Cooldown is always advisory. The proxy still dispatches to a cooling-down
|
|
10
|
+
* peer when a request names it, so a pinned conversation keeps working and no
|
|
11
|
+
* amount of cooldown can deadlock routing.
|
|
12
|
+
*/
|
|
13
|
+
/** Failures older than this start a fresh streak rather than compounding. */
|
|
14
|
+
export declare const PEER_HEALTH_WINDOW_MS: number;
|
|
15
|
+
/** Concurrent requests failing together count as one episode, not several. */
|
|
16
|
+
export declare const PEER_HEALTH_COALESCE_MS = 1000;
|
|
17
|
+
/** Ceiling on any single cooldown, and the sanity bound for persisted values. */
|
|
18
|
+
export declare const PEER_HEALTH_MAX_COOLDOWN_MS: number;
|
|
19
|
+
/** How long an idle entry is kept before it is forgotten entirely. */
|
|
20
|
+
export declare const PEER_HEALTH_RETENTION_MS: number;
|
|
21
|
+
/** Upper bound on tracked peers, newest failures retained. */
|
|
22
|
+
export declare const PEER_HEALTH_MAX_ENTRIES = 512;
|
|
23
|
+
export type PeerFailureReason =
|
|
24
|
+
/** Transport threw, timed out, or the connection died mid-request. */
|
|
25
|
+
'request-failed'
|
|
26
|
+
/** Seller answered 500/502/503/504. */
|
|
27
|
+
| 'seller-5xx'
|
|
28
|
+
/** Seller answered 408. */
|
|
29
|
+
| 'seller-timeout'
|
|
30
|
+
/** Seller answered 429 — capacity pressure, not death. Never escalates. */
|
|
31
|
+
| 'seller-busy'
|
|
32
|
+
/** Our own wallet, deposits, config or state machine. Never escalates. */
|
|
33
|
+
| 'buyer-local';
|
|
34
|
+
/** Whether a reason is even a candidate for cooling a peer down. */
|
|
35
|
+
export declare function reasonEscalates(reason: PeerFailureReason): boolean;
|
|
36
|
+
export type PeerHealthEntry = {
|
|
37
|
+
failureStreak: number;
|
|
38
|
+
windowStartedAt: number;
|
|
39
|
+
/** Start of the current coalesced failure episode. */
|
|
40
|
+
episodeStartedAt: number;
|
|
41
|
+
lastFailureAt: number;
|
|
42
|
+
lastReason: PeerFailureReason;
|
|
43
|
+
/** 0 when the peer is not cooling down. */
|
|
44
|
+
cooldownUntil: number;
|
|
45
|
+
/** 0 when the peer has never answered us. */
|
|
46
|
+
lastSuccessAt: number;
|
|
47
|
+
};
|
|
48
|
+
export declare function computeCooldownMs(failureStreak: number): number;
|
|
49
|
+
/**
|
|
50
|
+
* Fold a failure into a peer's health.
|
|
51
|
+
*
|
|
52
|
+
* `escalate` comes from the attribution gates: when false the failure is still
|
|
53
|
+
* recorded for diagnostics and UI, but the streak and cooldown stay put.
|
|
54
|
+
*/
|
|
55
|
+
export declare function recordPeerFailureEntry(previous: PeerHealthEntry | undefined, reason: PeerFailureReason, now: number, escalate: boolean): PeerHealthEntry;
|
|
56
|
+
/**
|
|
57
|
+
* Any response from a peer is proof of life, so a success wipes the streak and
|
|
58
|
+
* releases the cooldown. `lastSuccessAt` is kept for diagnostics.
|
|
59
|
+
*/
|
|
60
|
+
export declare function clearPeerHealthEntry(previous: PeerHealthEntry | undefined, now: number): PeerHealthEntry;
|
|
61
|
+
export declare function isCoolingDown(entry: PeerHealthEntry | undefined, now: number): boolean;
|
|
62
|
+
/** Drop stale entries and cap the map, keeping the most recently active peers. */
|
|
63
|
+
export declare function prunePeerHealth(entries: Map<string, PeerHealthEntry>, now: number): Map<string, PeerHealthEntry>;
|
|
64
|
+
/**
|
|
65
|
+
* Parse the `peerHealth` map out of a buyer.state.json blob.
|
|
66
|
+
*
|
|
67
|
+
* Cooldown survives a restart — a peer that died ten seconds before we exited is
|
|
68
|
+
* still dead — but is never extended or resurrected by one: anything expired,
|
|
69
|
+
* impossibly far in the future, or stamped after `now` is discarded.
|
|
70
|
+
*/
|
|
71
|
+
export declare function parsePersistedPeerHealth(parsed: unknown, now?: number): Map<string, PeerHealthEntry>;
|
|
72
|
+
export declare function serializePeerHealth(entries: Map<string, PeerHealthEntry>): Record<string, PeerHealthEntry>;
|
|
73
|
+
//# sourceMappingURL=peer-health.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"peer-health.d.ts","sourceRoot":"","sources":["../../src/proxy/peer-health.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAUH,6EAA6E;AAC7E,eAAO,MAAM,qBAAqB,QAAa,CAAA;AAC/C,8EAA8E;AAC9E,eAAO,MAAM,uBAAuB,OAAQ,CAAA;AAC5C,iFAAiF;AACjF,eAAO,MAAM,2BAA2B,QAAa,CAAA;AACrD,sEAAsE;AACtE,eAAO,MAAM,wBAAwB,QAAmB,CAAA;AACxD,8DAA8D;AAC9D,eAAO,MAAM,uBAAuB,MAAM,CAAA;AAE1C,MAAM,MAAM,iBAAiB;AAC3B,sEAAsE;AACpE,gBAAgB;AAClB,uCAAuC;GACrC,YAAY;AACd,2BAA2B;GACzB,gBAAgB;AAClB,2EAA2E;GACzE,aAAa;AACf,0EAA0E;GACxE,aAAa,CAAA;AAMjB,oEAAoE;AACpE,wBAAgB,eAAe,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAElE;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,MAAM,CAAA;IACvB,sDAAsD;IACtD,gBAAgB,EAAE,MAAM,CAAA;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,iBAAiB,CAAA;IAC7B,2CAA2C;IAC3C,aAAa,EAAE,MAAM,CAAA;IACrB,6CAA6C;IAC7C,aAAa,EAAE,MAAM,CAAA;CACtB,CAAA;AAED,wBAAgB,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,CAI/D;AAcD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,eAAe,GAAG,SAAS,EACrC,MAAM,EAAE,iBAAiB,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,GAChB,eAAe,CA+BjB;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,eAAe,GAAG,SAAS,EACrC,GAAG,EAAE,MAAM,GACV,eAAe,CASjB;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,eAAe,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAItF;AAUD,kFAAkF;AAClF,wBAAgB,eAAe,CAC7B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,EACrC,GAAG,EAAE,MAAM,GACV,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAS9B;AAMD;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,OAAO,EACf,GAAG,GAAE,MAAmB,GACvB,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CA+C9B;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,GACpC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAEjC"}
|