@antseed/node 0.2.153 → 0.2.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/health/gas-health-monitor.d.ts +19 -0
- package/dist/health/gas-health-monitor.d.ts.map +1 -1
- package/dist/health/gas-health-monitor.js +21 -0
- package/dist/health/gas-health-monitor.js.map +1 -1
- package/dist/health/model-health-checker.d.ts +52 -0
- package/dist/health/model-health-checker.d.ts.map +1 -1
- package/dist/health/model-health-checker.js +139 -7
- package/dist/health/model-health-checker.js.map +1 -1
- package/dist/health/seller-events.d.ts +24 -0
- package/dist/health/seller-events.d.ts.map +1 -0
- package/dist/health/seller-events.js +71 -0
- package/dist/health/seller-events.js.map +1 -0
- package/dist/health/upstream-credit.d.ts +8 -0
- package/dist/health/upstream-credit.d.ts.map +1 -0
- package/dist/health/upstream-credit.js +22 -0
- package/dist/health/upstream-credit.js.map +1 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/interfaces/seller-provider.d.ts +4 -1
- package/dist/interfaces/seller-provider.d.ts.map +1 -1
- package/dist/metering/index.d.ts +1 -1
- package/dist/metering/index.d.ts.map +1 -1
- package/dist/metering/index.js.map +1 -1
- package/dist/metering/seller-session-tracker.d.ts +7 -0
- package/dist/metering/seller-session-tracker.d.ts.map +1 -1
- package/dist/metering/seller-session-tracker.js +29 -0
- package/dist/metering/seller-session-tracker.js.map +1 -1
- package/dist/metering/storage.d.ts +80 -0
- package/dist/metering/storage.d.ts.map +1 -1
- package/dist/metering/storage.js +97 -2
- package/dist/metering/storage.js.map +1 -1
- package/dist/node.d.ts +14 -0
- package/dist/node.d.ts.map +1 -1
- package/dist/node.js +29 -9
- package/dist/node.js.map +1 -1
- package/dist/payments/channel-store.d.ts +10 -0
- package/dist/payments/channel-store.d.ts.map +1 -1
- package/dist/payments/channel-store.js +17 -0
- package/dist/payments/channel-store.js.map +1 -1
- package/dist/seller-request-handler.d.ts.map +1 -1
- package/dist/seller-request-handler.js +8 -0
- package/dist/seller-request-handler.js.map +1 -1
- package/dist/storage/migrate.d.ts +6 -0
- package/dist/storage/migrate.d.ts.map +1 -1
- package/dist/storage/migrate.js +11 -0
- package/dist/storage/migrate.js.map +1 -1
- package/dist/storage/migrations/metering/006_add_event_service.d.ts +4 -0
- package/dist/storage/migrations/metering/006_add_event_service.d.ts.map +1 -0
- package/dist/storage/migrations/metering/006_add_event_service.js +11 -0
- package/dist/storage/migrations/metering/006_add_event_service.js.map +1 -0
- package/dist/storage/migrations/metering/007_add_event_detail.d.ts +4 -0
- package/dist/storage/migrations/metering/007_add_event_detail.d.ts.map +1 -0
- package/dist/storage/migrations/metering/007_add_event_detail.js +12 -0
- package/dist/storage/migrations/metering/007_add_event_detail.js.map +1 -0
- package/dist/storage/migrations/metering/008_add_event_request_id.d.ts +4 -0
- package/dist/storage/migrations/metering/008_add_event_request_id.d.ts.map +1 -0
- package/dist/storage/migrations/metering/008_add_event_request_id.js +10 -0
- package/dist/storage/migrations/metering/008_add_event_request_id.js.map +1 -0
- package/dist/storage/migrations/metering/009_index_receipts_by_event.d.ts +4 -0
- package/dist/storage/migrations/metering/009_index_receipts_by_event.d.ts.map +1 -0
- package/dist/storage/migrations/metering/009_index_receipts_by_event.js +9 -0
- package/dist/storage/migrations/metering/009_index_receipts_by_event.js.map +1 -0
- package/dist/storage/migrations/metering/010_repair_free_tier_usage.d.ts +8 -0
- package/dist/storage/migrations/metering/010_repair_free_tier_usage.d.ts.map +1 -0
- package/dist/storage/migrations/metering/010_repair_free_tier_usage.js +35 -0
- package/dist/storage/migrations/metering/010_repair_free_tier_usage.js.map +1 -0
- package/dist/storage/migrations/metering/index.d.ts.map +1 -1
- package/dist/storage/migrations/metering/index.js +6 -1
- package/dist/storage/migrations/metering/index.js.map +1 -1
- package/dist/types/metering.d.ts +8 -0
- package/dist/types/metering.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -7,6 +7,12 @@ export declare const DEFAULT_GAS_CHECK_INTERVAL_MS = 60000;
|
|
|
7
7
|
* which rejects every buyer while the peer keeps looking discoverable.
|
|
8
8
|
*/
|
|
9
9
|
export declare const DEFAULT_MIN_GAS_BALANCE_WEI = 50000000000000n;
|
|
10
|
+
/** How often a well-funded wallet is checked. */
|
|
11
|
+
export declare const DEFAULT_RELAXED_GAS_CHECK_INTERVAL_MS: number;
|
|
12
|
+
/** The relaxed interval never exceeds this many regular intervals, so a drained wallet is noticed within a few checks. */
|
|
13
|
+
export declare const MAX_RELAXED_INTERVAL_MULTIPLE = 5;
|
|
14
|
+
/** A balance this many times the minimum counts as well funded (the default of `relaxedBalanceFactor`). */
|
|
15
|
+
export declare const RELAXED_BALANCE_FACTOR = 20n;
|
|
10
16
|
export interface GasHealthEvent {
|
|
11
17
|
status: 'depleted' | 'restored';
|
|
12
18
|
/** Wallet that pays gas for seller transactions (the peer identity wallet). */
|
|
@@ -28,6 +34,15 @@ export interface GasHealthMonitorConfig {
|
|
|
28
34
|
getBalance: (address: string) => Promise<bigint>;
|
|
29
35
|
intervalMs?: number;
|
|
30
36
|
minBalanceWei?: bigint;
|
|
37
|
+
/**
|
|
38
|
+
* While the last balance is at least `RELAXED_BALANCE_FACTOR` × the minimum,
|
|
39
|
+
* check only this often (default 5 minutes, at most 5 regular intervals).
|
|
40
|
+
* A wallet drained in between is noticed that much later, so a seller
|
|
41
|
+
* settling heavily lowers it or raises `relaxedBalanceFactor`.
|
|
42
|
+
*/
|
|
43
|
+
relaxedIntervalMs?: number;
|
|
44
|
+
/** How many times the minimum counts as well funded (default `RELAXED_BALANCE_FACTOR`). */
|
|
45
|
+
relaxedBalanceFactor?: bigint;
|
|
31
46
|
/**
|
|
32
47
|
* Invoked when the wallet crosses the threshold in either direction.
|
|
33
48
|
* Callers should pause/resume advertising here (e.g.
|
|
@@ -51,9 +66,12 @@ export declare class GasHealthMonitor {
|
|
|
51
66
|
private readonly _address;
|
|
52
67
|
private readonly _getBalance;
|
|
53
68
|
private readonly _intervalMs;
|
|
69
|
+
private readonly _relaxedIntervalMs;
|
|
70
|
+
private readonly _relaxedBalanceFactor;
|
|
54
71
|
private readonly _minBalanceWei;
|
|
55
72
|
private readonly _onChange;
|
|
56
73
|
private _timer;
|
|
74
|
+
private _stopped;
|
|
57
75
|
private _checkInFlight;
|
|
58
76
|
private _depleted;
|
|
59
77
|
private _lastBalanceWei;
|
|
@@ -62,6 +80,7 @@ export declare class GasHealthMonitor {
|
|
|
62
80
|
/** Start periodic checks. The first check runs immediately (async). */
|
|
63
81
|
start(): void;
|
|
64
82
|
stop(): void;
|
|
83
|
+
private _relaxed;
|
|
65
84
|
/** Whether the last successful reading was below the threshold. */
|
|
66
85
|
get depleted(): boolean;
|
|
67
86
|
getSnapshot(): GasHealthSnapshot;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gas-health-monitor.d.ts","sourceRoot":"","sources":["../../src/health/gas-health-monitor.ts"],"names":[],"mappings":"AAGA,gFAAgF;AAChF,eAAO,MAAM,6BAA6B,QAAS,CAAC;AACpD;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,kBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"gas-health-monitor.d.ts","sourceRoot":"","sources":["../../src/health/gas-health-monitor.ts"],"names":[],"mappings":"AAGA,gFAAgF;AAChF,eAAO,MAAM,6BAA6B,QAAS,CAAC;AACpD;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,kBAAsB,CAAC;AAC/D,iDAAiD;AACjD,eAAO,MAAM,qCAAqC,QAAa,CAAC;AAChE,0HAA0H;AAC1H,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAC/C,2GAA2G;AAC3G,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,UAAU,GAAG,UAAU,CAAC;IAChC,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,OAAO,CAAC;IAClB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,sBAAsB;IACrC,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAC;IAChB,8EAA8E;IAC9E,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACjD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,2FAA2F;IAC3F,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqC;IAC/D,OAAO,CAAC,MAAM,CAA+C;IAC7D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,cAAc,CAAuB;gBAEjC,MAAM,EAAE,sBAAsB;IAU1C,uEAAuE;IACvE,KAAK,IAAI,IAAI;IAYb,IAAI,IAAI,IAAI;IASZ,OAAO,CAAC,QAAQ;IAIhB,mEAAmE;IACnE,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,WAAW,IAAI,iBAAiB;IAUhC,6EAA6E;IACvE,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YAQjB,MAAM;IAwBpB,OAAO,CAAC,WAAW;CAiBpB"}
|
|
@@ -9,6 +9,12 @@ export const DEFAULT_GAS_CHECK_INTERVAL_MS = 60_000;
|
|
|
9
9
|
* which rejects every buyer while the peer keeps looking discoverable.
|
|
10
10
|
*/
|
|
11
11
|
export const DEFAULT_MIN_GAS_BALANCE_WEI = 50000000000000n;
|
|
12
|
+
/** How often a well-funded wallet is checked. */
|
|
13
|
+
export const DEFAULT_RELAXED_GAS_CHECK_INTERVAL_MS = 5 * 60_000;
|
|
14
|
+
/** The relaxed interval never exceeds this many regular intervals, so a drained wallet is noticed within a few checks. */
|
|
15
|
+
export const MAX_RELAXED_INTERVAL_MULTIPLE = 5;
|
|
16
|
+
/** A balance this many times the minimum counts as well funded (the default of `relaxedBalanceFactor`). */
|
|
17
|
+
export const RELAXED_BALANCE_FACTOR = 20n;
|
|
12
18
|
/**
|
|
13
19
|
* Periodically reads the seller wallet's ETH balance and reports when it drops
|
|
14
20
|
* below the minimum needed to fund on-chain settlement. A seller with no gas
|
|
@@ -25,9 +31,12 @@ export class GasHealthMonitor {
|
|
|
25
31
|
_address;
|
|
26
32
|
_getBalance;
|
|
27
33
|
_intervalMs;
|
|
34
|
+
_relaxedIntervalMs;
|
|
35
|
+
_relaxedBalanceFactor;
|
|
28
36
|
_minBalanceWei;
|
|
29
37
|
_onChange;
|
|
30
38
|
_timer = null;
|
|
39
|
+
_stopped = false;
|
|
31
40
|
_checkInFlight = null;
|
|
32
41
|
_depleted = false;
|
|
33
42
|
_lastBalanceWei = null;
|
|
@@ -36,26 +45,36 @@ export class GasHealthMonitor {
|
|
|
36
45
|
this._address = config.address;
|
|
37
46
|
this._getBalance = config.getBalance;
|
|
38
47
|
this._intervalMs = config.intervalMs ?? DEFAULT_GAS_CHECK_INTERVAL_MS;
|
|
48
|
+
this._relaxedIntervalMs = Math.min(this._intervalMs * MAX_RELAXED_INTERVAL_MULTIPLE, Math.max(this._intervalMs, config.relaxedIntervalMs ?? DEFAULT_RELAXED_GAS_CHECK_INTERVAL_MS));
|
|
39
49
|
this._minBalanceWei = config.minBalanceWei ?? DEFAULT_MIN_GAS_BALANCE_WEI;
|
|
50
|
+
this._relaxedBalanceFactor = config.relaxedBalanceFactor ?? RELAXED_BALANCE_FACTOR;
|
|
40
51
|
this._onChange = config.onChange;
|
|
41
52
|
}
|
|
42
53
|
/** Start periodic checks. The first check runs immediately (async). */
|
|
43
54
|
start() {
|
|
44
55
|
if (this._timer)
|
|
45
56
|
return;
|
|
57
|
+
this._stopped = false;
|
|
46
58
|
void this.checkNow();
|
|
47
59
|
this._timer = setInterval(() => {
|
|
60
|
+
if (this._relaxed() && this._lastCheckedAt !== null && Date.now() - this._lastCheckedAt < this._relaxedIntervalMs)
|
|
61
|
+
return;
|
|
48
62
|
void this.checkNow();
|
|
49
63
|
}, this._intervalMs);
|
|
50
64
|
// Never keep the process alive just for gas checks.
|
|
51
65
|
this._timer.unref?.();
|
|
52
66
|
}
|
|
53
67
|
stop() {
|
|
68
|
+
// A reading still in flight is dropped, so nothing reports a change after shutdown.
|
|
69
|
+
this._stopped = true;
|
|
54
70
|
if (this._timer) {
|
|
55
71
|
clearInterval(this._timer);
|
|
56
72
|
this._timer = null;
|
|
57
73
|
}
|
|
58
74
|
}
|
|
75
|
+
_relaxed() {
|
|
76
|
+
return this._lastBalanceWei !== null && this._lastBalanceWei >= this._minBalanceWei * this._relaxedBalanceFactor;
|
|
77
|
+
}
|
|
59
78
|
/** Whether the last successful reading was below the threshold. */
|
|
60
79
|
get depleted() {
|
|
61
80
|
return this._depleted;
|
|
@@ -88,6 +107,8 @@ export class GasHealthMonitor {
|
|
|
88
107
|
debugWarn(`[GasHealth] Balance check failed for ${this._address}: ${err instanceof Error ? err.message : err}`);
|
|
89
108
|
return;
|
|
90
109
|
}
|
|
110
|
+
if (this._stopped)
|
|
111
|
+
return;
|
|
91
112
|
this._lastBalanceWei = balanceWei;
|
|
92
113
|
this._lastCheckedAt = Date.now();
|
|
93
114
|
const depleted = balanceWei < this._minBalanceWei;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gas-health-monitor.js","sourceRoot":"","sources":["../../src/health/gas-health-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAExD,gFAAgF;AAChF,MAAM,CAAC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AACpD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,eAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"gas-health-monitor.js","sourceRoot":"","sources":["../../src/health/gas-health-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAExD,gFAAgF;AAChF,MAAM,CAAC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AACpD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,eAAmB,CAAC;AAC/D,iDAAiD;AACjD,MAAM,CAAC,MAAM,qCAAqC,GAAG,CAAC,GAAG,MAAM,CAAC;AAChE,0HAA0H;AAC1H,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC;AAC/C,2GAA2G;AAC3G,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AA0C1C;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,gBAAgB;IACV,QAAQ,CAAS;IACjB,WAAW,CAAuC;IAClD,WAAW,CAAS;IACpB,kBAAkB,CAAS;IAC3B,qBAAqB,CAAS;IAC9B,cAAc,CAAS;IACvB,SAAS,CAAqC;IACvD,MAAM,GAA0C,IAAI,CAAC;IACrD,QAAQ,GAAG,KAAK,CAAC;IACjB,cAAc,GAAyB,IAAI,CAAC;IAC5C,SAAS,GAAG,KAAK,CAAC;IAClB,eAAe,GAAkB,IAAI,CAAC;IACtC,cAAc,GAAkB,IAAI,CAAC;IAE7C,YAAY,MAA8B;QACxC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,IAAI,6BAA6B,CAAC;QACtE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,6BAA6B,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,iBAAiB,IAAI,qCAAqC,CAAC,CAAC,CAAC;QACpL,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,aAAa,IAAI,2BAA2B,CAAC;QAC1E,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,oBAAoB,IAAI,sBAAsB,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC;IACnC,CAAC;IAED,uEAAuE;IACvE,KAAK;QACH,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE;YAC7B,IAAI,IAAI,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,kBAAkB;gBAAE,OAAO;YAC1H,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACrB,oDAAoD;QACpD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IACxB,CAAC;IAED,IAAI;QACF,oFAAoF;QACpF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACrB,CAAC;IACH,CAAC;IAEO,QAAQ;QACd,OAAO,IAAI,CAAC,eAAe,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;IACnH,CAAC;IAED,mEAAmE;IACnE,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,WAAW;QACT,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,cAAc,EAAE,IAAI,CAAC,eAAe;YACpC,aAAa,EAAE,IAAI,CAAC,cAAc;YAClC,aAAa,EAAE,IAAI,CAAC,cAAc;SACnC,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC,cAAc,CAAC;QACpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAC/C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,MAAM;QAClB,IAAI,UAAkB,CAAC;QACvB,IAAI,CAAC;YACH,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wEAAwE;YACxE,SAAS,CAAC,wCAAwC,IAAI,CAAC,QAAQ,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YAChH,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;QAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEjC,MAAM,QAAQ,GAAG,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC;QAClD,IAAI,QAAQ,KAAK,IAAI,CAAC,SAAS;YAAE,OAAO;QACxC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,WAAW,CAAC;YACf,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU;YAC1C,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,UAAU;YACV,aAAa,EAAE,IAAI,CAAC,cAAc;SACnC,CAAC,CAAC;IACL,CAAC;IAEO,WAAW,CAAC,KAAqB;QACvC,QAAQ,CACN,eAAe,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG;cAC7C,IAAI,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,iBAAiB,WAAW,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,CAC5F,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,MAAM,IAAI,OAAQ,MAAwB,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;gBACnE,MAAwB,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACtC,SAAS,CAAC,gCAAgC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;gBACxF,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,CAAC,gCAAgC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;CACF"}
|
|
@@ -21,17 +21,44 @@ export interface ModelHealthTarget {
|
|
|
21
21
|
* doesn't run a full agent loop. Defaults to `provider`.
|
|
22
22
|
*/
|
|
23
23
|
probeProvider?: Provider;
|
|
24
|
+
/**
|
|
25
|
+
* The seller config's name for this provider instance. Several instances
|
|
26
|
+
* can share a plugin and so a `provider.name` (four upstreams on the openai
|
|
27
|
+
* plugin); this tells them apart in snapshots and events. Defaults to `provider.name`.
|
|
28
|
+
*/
|
|
29
|
+
id?: string;
|
|
24
30
|
}
|
|
25
31
|
export interface ModelHealthEvent {
|
|
26
32
|
provider: string;
|
|
33
|
+
/** The target's `id` (the configured provider name). */
|
|
34
|
+
providerId: string;
|
|
27
35
|
service: string;
|
|
28
36
|
status: 'removed' | 'restored';
|
|
29
37
|
consecutiveFailures: number;
|
|
30
38
|
/** Short description of the most recent probe outcome. */
|
|
31
39
|
detail: string;
|
|
32
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* An upstream account's credit changed: out of credit (an HTTP 402 or a
|
|
43
|
+
* billing error from a probe, an account check or a live request), below the
|
|
44
|
+
* low-balance line (from the account check), or funded again.
|
|
45
|
+
*/
|
|
46
|
+
export interface UpstreamAccountEvent {
|
|
47
|
+
provider: string;
|
|
48
|
+
/** The target's `id` (the configured provider name). */
|
|
49
|
+
providerId: string;
|
|
50
|
+
status: 'out_of_credit' | 'low_balance' | 'funded';
|
|
51
|
+
/** The upstream's error, or the balance read. */
|
|
52
|
+
detail: string;
|
|
53
|
+
/** USD balance, when the provider can read it. */
|
|
54
|
+
balanceUsd: number | null;
|
|
55
|
+
/** The upstream's services (advertised or withdrawn). */
|
|
56
|
+
services: string[];
|
|
57
|
+
}
|
|
33
58
|
export interface ServiceHealthSnapshot {
|
|
34
59
|
provider: string;
|
|
60
|
+
/** The target's `id` (the configured provider name). */
|
|
61
|
+
providerId: string;
|
|
35
62
|
service: string;
|
|
36
63
|
advertised: boolean;
|
|
37
64
|
consecutiveFailures: number;
|
|
@@ -49,6 +76,12 @@ export interface ModelHealthCheckerConfig {
|
|
|
49
76
|
* re-sign seller metadata here (e.g. `node.refreshSellerMetadata()`).
|
|
50
77
|
*/
|
|
51
78
|
onChange?: (event: ModelHealthEvent) => void | Promise<void>;
|
|
79
|
+
/** Invoked when an upstream account runs out of credit, drops below `lowBalanceUsd`, or is funded again. */
|
|
80
|
+
onUpstreamAccount?: (event: UpstreamAccountEvent) => void | Promise<void>;
|
|
81
|
+
/** Invoked after each sweep, so changes from one round can be reported together. */
|
|
82
|
+
onSweepComplete?: () => void;
|
|
83
|
+
/** USD balance under which a provider that reads its balance reports `low_balance`; a function is read on every check. */
|
|
84
|
+
lowBalanceUsd?: number | (() => number | undefined);
|
|
52
85
|
}
|
|
53
86
|
type ProbeOutcome = 'healthy' | 'unhealthy' | 'inconclusive';
|
|
54
87
|
/**
|
|
@@ -70,9 +103,18 @@ export declare class ModelHealthChecker {
|
|
|
70
103
|
private readonly _probeTimeoutMs;
|
|
71
104
|
private readonly _onChange;
|
|
72
105
|
private readonly _states;
|
|
106
|
+
/** Stable per-instance ids: providers are told apart by instance, not by name. */
|
|
107
|
+
private readonly _providerIds;
|
|
73
108
|
private _timer;
|
|
74
109
|
private _sweepInFlight;
|
|
75
110
|
private _stopped;
|
|
111
|
+
private readonly _onUpstreamAccount;
|
|
112
|
+
private readonly _lowBalanceUsd;
|
|
113
|
+
private readonly _onSweepComplete;
|
|
114
|
+
/** Each upstream's credit as last seen; unset until something said. */
|
|
115
|
+
private readonly _accounts;
|
|
116
|
+
/** Services withdrawn before `onChange` could be told; reported on the first start. */
|
|
117
|
+
private _withdrawnAtInit;
|
|
76
118
|
constructor(config: ModelHealthCheckerConfig);
|
|
77
119
|
/** Start periodic sweeps. The first sweep runs immediately (async). */
|
|
78
120
|
start(): void;
|
|
@@ -92,6 +134,16 @@ export declare class ModelHealthChecker {
|
|
|
92
134
|
private _checkServices;
|
|
93
135
|
private _stateFor;
|
|
94
136
|
private _probeService;
|
|
137
|
+
/**
|
|
138
|
+
* A live request's upstream answer (the seller's own traffic): an out-of-credit
|
|
139
|
+
* error is reported at once instead of waiting for the probes, and a billable
|
|
140
|
+
* success after one means the account pays again (polling or downloading a
|
|
141
|
+
* video already paid for is not billable and proves nothing).
|
|
142
|
+
*/
|
|
143
|
+
reportUpstreamResponse(provider: Provider, statusCode: number, body: string, billable?: boolean): void;
|
|
144
|
+
private _lowLine;
|
|
145
|
+
/** Records an upstream's credit and reports a change; the first reading of a funded account is not a change. */
|
|
146
|
+
private _setAccount;
|
|
95
147
|
private _recordFailure;
|
|
96
148
|
private _removeService;
|
|
97
149
|
private _restoreService;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-health-checker.d.ts","sourceRoot":"","sources":["../../src/health/model-health-checker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAA8C,MAAM,kCAAkC,CAAC;AAC7G,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAA4C,KAAK,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"model-health-checker.d.ts","sourceRoot":"","sources":["../../src/health/model-health-checker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAA8C,MAAM,kCAAkC,CAAC;AAC7G,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAA4C,KAAK,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAI5G,yFAAyF;AACzF,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAC3D,mEAAmE;AACnE,eAAO,MAAM,sCAAsC,IAAI,CAAC;AACxD,6EAA6E;AAC7E,eAAO,MAAM,qCAAqC,QAAS,CAAC;AAE5D,MAAM,WAAW,iBAAiB;IAChC;;;;;OAKG;IACH,QAAQ,EAAE,QAAQ,CAAC;IACnB;;;;OAIG;IACH,aAAa,CAAC,EAAE,QAAQ,CAAC;IACzB;;;;OAIG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,GAAG,UAAU,CAAC;IAC/B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,0DAA0D;IAC1D,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,eAAe,GAAG,aAAa,GAAG,QAAQ,CAAC;IACnD,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,OAAO,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,wBAAwB;IACvC,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7D,4GAA4G;IAC5G,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1E,oFAAoF;IACpF,eAAe,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7B,0HAA0H;IAC1H,aAAa,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC;CACrD;AAID,KAAK,YAAY,GAAG,SAAS,GAAG,WAAW,GAAG,cAAc,CAAC;AAgB7D;;;;;;;;;;;GAWG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAuC;IACjE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyC;IACjE,kFAAkF;IAClF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA+B;IAC5D,OAAO,CAAC,MAAM,CAA+C;IAC7D,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAgD;IACnF,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA4C;IAC3E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA8C;IAC/E,uEAAuE;IACvE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqC;IAC/D,uFAAuF;IACvF,OAAO,CAAC,gBAAgB,CAA4B;gBAExC,MAAM,EAAE,wBAAwB;IAyB5C,uEAAuE;IACvE,KAAK,IAAI,IAAI;IAsBb,IAAI,IAAI,IAAI;IAQZ,oEAAoE;IAC9D,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAc/B,WAAW,IAAI,qBAAqB,EAAE;YAaxB,SAAS;IA0CvB;;;;;;;OAOG;YACW,cAAc;IA0D5B,OAAO,CAAC,SAAS;YA0BH,aAAa;IA+D3B;;;;;OAKG;IACH,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,UAAO,GAAG,IAAI;IAUnG,OAAO,CAAC,QAAQ;IAKhB,gHAAgH;IAChH,OAAO,CAAC,WAAW;IA2BnB,OAAO,CAAC,cAAc;IAWtB,OAAO,CAAC,cAAc;IA2BtB,OAAO,CAAC,eAAe;IAiBvB,OAAO,CAAC,WAAW;CAepB;AAOD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAEzE;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,kBAAkB,GAAG,qBAAqB,CAkD5G;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,YAAY,CAOpE"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isNativeVideoProtocol, isRoutingProtocol } from '../types/service-api.js';
|
|
2
2
|
import { debugLog, debugWarn } from '../utils/debug.js';
|
|
3
|
+
import { isLiveResponseOutOfCredit, isUpstreamOutOfCredit } from './upstream-credit.js';
|
|
3
4
|
/** Default sweep interval. Each sweep sends one 1-token probe per advertised service. */
|
|
4
5
|
export const DEFAULT_HEALTH_CHECK_INTERVAL_MS = 5 * 60_000;
|
|
5
6
|
/** Consecutive probe failures before a service is unadvertised. */
|
|
@@ -25,15 +26,27 @@ export class ModelHealthChecker {
|
|
|
25
26
|
_probeTimeoutMs;
|
|
26
27
|
_onChange;
|
|
27
28
|
_states = new Map();
|
|
29
|
+
/** Stable per-instance ids: providers are told apart by instance, not by name. */
|
|
30
|
+
_providerIds = new Map();
|
|
28
31
|
_timer = null;
|
|
29
32
|
_sweepInFlight = null;
|
|
30
33
|
_stopped = false;
|
|
34
|
+
_onUpstreamAccount;
|
|
35
|
+
_lowBalanceUsd;
|
|
36
|
+
_onSweepComplete;
|
|
37
|
+
/** Each upstream's credit as last seen; unset until something said. */
|
|
38
|
+
_accounts = new Map();
|
|
39
|
+
/** Services withdrawn before `onChange` could be told; reported on the first start. */
|
|
40
|
+
_withdrawnAtInit = [];
|
|
31
41
|
constructor(config) {
|
|
32
42
|
this._targets = config.targets;
|
|
33
43
|
this._intervalMs = config.intervalMs ?? DEFAULT_HEALTH_CHECK_INTERVAL_MS;
|
|
34
44
|
this._failureThreshold = Math.max(1, config.failureThreshold ?? DEFAULT_HEALTH_CHECK_FAILURE_THRESHOLD);
|
|
35
45
|
this._probeTimeoutMs = config.probeTimeoutMs ?? DEFAULT_HEALTH_CHECK_PROBE_TIMEOUT_MS;
|
|
36
46
|
this._onChange = config.onChange;
|
|
47
|
+
this._onUpstreamAccount = config.onUpstreamAccount;
|
|
48
|
+
this._lowBalanceUsd = config.lowBalanceUsd;
|
|
49
|
+
this._onSweepComplete = config.onSweepComplete;
|
|
37
50
|
// Providers retained after an OAuth initialization failure must pass a real
|
|
38
51
|
// probe before any service is advertised. Keep their services in recovery
|
|
39
52
|
// state rather than dropping the provider permanently.
|
|
@@ -41,10 +54,11 @@ export class ModelHealthChecker {
|
|
|
41
54
|
if (provider.healthCheckAvailable !== false)
|
|
42
55
|
continue;
|
|
43
56
|
for (const [index, service] of provider.services.entries()) {
|
|
44
|
-
const state = this._stateFor(provider
|
|
57
|
+
const state = this._stateFor(provider, service);
|
|
45
58
|
state.removed = true;
|
|
46
59
|
state.removedAtIndex = index;
|
|
47
60
|
state.lastDetail = 'Awaiting successful probe after initialization failure';
|
|
61
|
+
this._withdrawnAtInit.push(state);
|
|
48
62
|
}
|
|
49
63
|
provider.services.splice(0);
|
|
50
64
|
}
|
|
@@ -54,6 +68,18 @@ export class ModelHealthChecker {
|
|
|
54
68
|
if (this._timer)
|
|
55
69
|
return;
|
|
56
70
|
this._stopped = false;
|
|
71
|
+
for (const state of this._withdrawnAtInit.splice(0)) {
|
|
72
|
+
if (!state.removed)
|
|
73
|
+
continue;
|
|
74
|
+
this._emitChange({
|
|
75
|
+
provider: state.providerName,
|
|
76
|
+
providerId: state.providerId,
|
|
77
|
+
service: state.service,
|
|
78
|
+
status: 'removed',
|
|
79
|
+
consecutiveFailures: state.consecutiveFailures,
|
|
80
|
+
detail: state.lastDetail ?? 'initialization failed',
|
|
81
|
+
});
|
|
82
|
+
}
|
|
57
83
|
void this.runSweep();
|
|
58
84
|
this._timer = setInterval(() => {
|
|
59
85
|
void this.runSweep();
|
|
@@ -74,12 +100,21 @@ export class ModelHealthChecker {
|
|
|
74
100
|
return this._sweepInFlight;
|
|
75
101
|
this._sweepInFlight = this._runSweep().finally(() => {
|
|
76
102
|
this._sweepInFlight = null;
|
|
103
|
+
if (this._stopped || !this._onSweepComplete)
|
|
104
|
+
return;
|
|
105
|
+
try {
|
|
106
|
+
this._onSweepComplete();
|
|
107
|
+
}
|
|
108
|
+
catch (err) {
|
|
109
|
+
debugWarn(`[ModelHealth] onSweepComplete failed: ${err instanceof Error ? err.message : err}`);
|
|
110
|
+
}
|
|
77
111
|
});
|
|
78
112
|
return this._sweepInFlight;
|
|
79
113
|
}
|
|
80
114
|
getSnapshot() {
|
|
81
115
|
return Array.from(this._states.values()).map((state) => ({
|
|
82
116
|
provider: state.providerName,
|
|
117
|
+
providerId: state.providerId,
|
|
83
118
|
service: state.service,
|
|
84
119
|
advertised: !state.removed,
|
|
85
120
|
consecutiveFailures: state.consecutiveFailures,
|
|
@@ -109,7 +144,7 @@ export class ModelHealthChecker {
|
|
|
109
144
|
// the live array on state transitions.
|
|
110
145
|
const services = new Set(provider.services);
|
|
111
146
|
for (const state of this._states.values()) {
|
|
112
|
-
if (state.
|
|
147
|
+
if (state.provider === provider && state.removed) {
|
|
113
148
|
services.add(state.service);
|
|
114
149
|
}
|
|
115
150
|
}
|
|
@@ -151,6 +186,20 @@ export class ModelHealthChecker {
|
|
|
151
186
|
: undefined,
|
|
152
187
|
]);
|
|
153
188
|
const balance = account?.status === 'available' ? account.balanceUsd : undefined;
|
|
189
|
+
if (account?.status === 'unavailable' && (account.outOfCredit === true || isUpstreamOutOfCredit(null, account.reason))) {
|
|
190
|
+
this._setAccount(provider, 'out', account.reason, null);
|
|
191
|
+
}
|
|
192
|
+
else if (account?.status === 'available') {
|
|
193
|
+
const low = this._lowLine();
|
|
194
|
+
// Without a balance, an answering account endpoint is no proof of credit: only a billable success clears "out".
|
|
195
|
+
const unproven = balance === undefined && this._accounts.get(provider) === 'out';
|
|
196
|
+
if (balance !== undefined && balance <= 0)
|
|
197
|
+
this._setAccount(provider, 'out', `Balance $${balance.toFixed(2)}`, balance);
|
|
198
|
+
else if (balance !== undefined && low !== undefined && balance < low)
|
|
199
|
+
this._setAccount(provider, 'low', `Balance $${balance.toFixed(2)}, under $${low.toFixed(2)}`, balance);
|
|
200
|
+
else if (!unproven)
|
|
201
|
+
this._setAccount(provider, 'ok', balance === undefined ? 'Account check passed' : `Balance $${balance.toFixed(2)}`, balance ?? null);
|
|
202
|
+
}
|
|
154
203
|
const combine = (service) => {
|
|
155
204
|
const model = models === undefined
|
|
156
205
|
? undefined
|
|
@@ -164,7 +213,7 @@ export class ModelHealthChecker {
|
|
|
164
213
|
if (this._stopped)
|
|
165
214
|
return;
|
|
166
215
|
const result = combine(service);
|
|
167
|
-
const state = this._stateFor(provider
|
|
216
|
+
const state = this._stateFor(provider, service);
|
|
168
217
|
if (result.status === 'unknown') {
|
|
169
218
|
debugLog(`[ModelHealth] ${provider.name}/${service}: inconclusive (${result.reason ?? 'no result'})`);
|
|
170
219
|
continue;
|
|
@@ -183,12 +232,19 @@ export class ModelHealthChecker {
|
|
|
183
232
|
this._restoreService(provider, state);
|
|
184
233
|
}
|
|
185
234
|
}
|
|
186
|
-
_stateFor(
|
|
187
|
-
|
|
235
|
+
_stateFor(provider, service) {
|
|
236
|
+
let id = this._providerIds.get(provider);
|
|
237
|
+
if (id === undefined) {
|
|
238
|
+
id = this._providerIds.size;
|
|
239
|
+
this._providerIds.set(provider, id);
|
|
240
|
+
}
|
|
241
|
+
const key = `${id}\0${service}`;
|
|
188
242
|
let state = this._states.get(key);
|
|
189
243
|
if (!state) {
|
|
190
244
|
state = {
|
|
191
|
-
|
|
245
|
+
provider,
|
|
246
|
+
providerName: provider.name,
|
|
247
|
+
providerId: this._targets.find((target) => target.provider === provider)?.id ?? provider.name,
|
|
192
248
|
service,
|
|
193
249
|
consecutiveFailures: 0,
|
|
194
250
|
removed: false,
|
|
@@ -204,9 +260,10 @@ export class ModelHealthChecker {
|
|
|
204
260
|
async _probeService(target, service) {
|
|
205
261
|
const provider = target.provider;
|
|
206
262
|
const probeProvider = target.probeProvider ?? provider;
|
|
207
|
-
const state = this._stateFor(provider
|
|
263
|
+
const state = this._stateFor(provider, service);
|
|
208
264
|
state.lastProbeAt = Date.now();
|
|
209
265
|
let outcome;
|
|
266
|
+
let errorText = null;
|
|
210
267
|
try {
|
|
211
268
|
const protocol = resolveProbeProtocol(provider, service);
|
|
212
269
|
if (!supportsHealthProbe(protocol)) {
|
|
@@ -219,6 +276,9 @@ export class ModelHealthChecker {
|
|
|
219
276
|
const response = await withTimeout(probeProvider.handleRequest(request), this._probeTimeoutMs, `health probe for ${service} timed out after ${this._probeTimeoutMs}ms`);
|
|
220
277
|
state.lastStatusCode = response.statusCode;
|
|
221
278
|
outcome = classifyProbeStatus(response.statusCode);
|
|
279
|
+
// Classified on the body itself, not the 200-byte summary: a billing code can follow a long message.
|
|
280
|
+
if (outcome !== 'healthy')
|
|
281
|
+
errorText = decodeBody(response.body, 16_384);
|
|
222
282
|
state.lastDetail = outcome === 'healthy'
|
|
223
283
|
? `HTTP ${response.statusCode}`
|
|
224
284
|
: `HTTP ${response.statusCode}: ${summarizeBody(response.body)}`;
|
|
@@ -226,7 +286,14 @@ export class ModelHealthChecker {
|
|
|
226
286
|
catch (err) {
|
|
227
287
|
state.lastStatusCode = null;
|
|
228
288
|
state.lastDetail = err instanceof Error ? err.message : String(err);
|
|
289
|
+
errorText = state.lastDetail;
|
|
290
|
+
outcome = 'unhealthy';
|
|
291
|
+
}
|
|
292
|
+
// A billing error is a failure whatever its status (Anthropic answers one with a 400);
|
|
293
|
+
// a 404 or 422 is about the probe itself, so its text never marks the account.
|
|
294
|
+
if (outcome !== 'healthy' && state.lastStatusCode !== 404 && state.lastStatusCode !== 422 && isUpstreamOutOfCredit(state.lastStatusCode, errorText)) {
|
|
229
295
|
outcome = 'unhealthy';
|
|
296
|
+
this._setAccount(provider, 'out', state.lastDetail ?? 'out of credit', null);
|
|
230
297
|
}
|
|
231
298
|
if (outcome === 'inconclusive') {
|
|
232
299
|
// The endpoint answered but the probe result proves nothing either way
|
|
@@ -236,6 +303,9 @@ export class ModelHealthChecker {
|
|
|
236
303
|
return;
|
|
237
304
|
}
|
|
238
305
|
if (outcome === 'healthy') {
|
|
306
|
+
// An answered completion means the account pays again (its balance is unknown here).
|
|
307
|
+
if (this._accounts.get(provider) === 'out')
|
|
308
|
+
this._setAccount(provider, 'ok', state.lastDetail ?? 'probe succeeded', null);
|
|
239
309
|
state.consecutiveFailures = 0;
|
|
240
310
|
if (state.removed) {
|
|
241
311
|
this._restoreService(provider, state);
|
|
@@ -244,6 +314,58 @@ export class ModelHealthChecker {
|
|
|
244
314
|
}
|
|
245
315
|
this._recordFailure(provider, state);
|
|
246
316
|
}
|
|
317
|
+
/**
|
|
318
|
+
* A live request's upstream answer (the seller's own traffic): an out-of-credit
|
|
319
|
+
* error is reported at once instead of waiting for the probes, and a billable
|
|
320
|
+
* success after one means the account pays again (polling or downloading a
|
|
321
|
+
* video already paid for is not billable and proves nothing).
|
|
322
|
+
*/
|
|
323
|
+
reportUpstreamResponse(provider, statusCode, body, billable = true) {
|
|
324
|
+
const target = this._targets.find((entry) => entry.provider === provider || entry.probeProvider === provider);
|
|
325
|
+
if (!target || this._stopped)
|
|
326
|
+
return;
|
|
327
|
+
if (statusCode >= 400 && isLiveResponseOutOfCredit(statusCode, body)) {
|
|
328
|
+
this._setAccount(target.provider, 'out', `HTTP ${statusCode}: ${summarizeBody(new TextEncoder().encode(body))}`, null);
|
|
329
|
+
}
|
|
330
|
+
else if (statusCode < 400 && billable && this._accounts.get(target.provider) === 'out') {
|
|
331
|
+
this._setAccount(target.provider, 'ok', `HTTP ${statusCode} on a request`, null);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
_lowLine() {
|
|
335
|
+
const value = typeof this._lowBalanceUsd === 'function' ? this._lowBalanceUsd() : this._lowBalanceUsd;
|
|
336
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : undefined;
|
|
337
|
+
}
|
|
338
|
+
/** Records an upstream's credit and reports a change; the first reading of a funded account is not a change. */
|
|
339
|
+
_setAccount(provider, next, detail, balanceUsd) {
|
|
340
|
+
const previous = this._accounts.get(provider);
|
|
341
|
+
if (previous === next)
|
|
342
|
+
return;
|
|
343
|
+
this._accounts.set(provider, next);
|
|
344
|
+
if (next === 'ok' && previous === undefined)
|
|
345
|
+
return;
|
|
346
|
+
const target = this._targets.find((entry) => entry.provider === provider);
|
|
347
|
+
const services = [...new Set([...provider.services, ...[...this._states.values()].filter((state) => state.provider === provider).map((state) => state.service)])];
|
|
348
|
+
const event = {
|
|
349
|
+
provider: provider.name,
|
|
350
|
+
providerId: target?.id ?? provider.name,
|
|
351
|
+
status: next === 'out' ? 'out_of_credit' : next === 'low' ? 'low_balance' : 'funded',
|
|
352
|
+
detail,
|
|
353
|
+
balanceUsd,
|
|
354
|
+
services,
|
|
355
|
+
};
|
|
356
|
+
debugWarn(`[ModelHealth] ${event.providerId}: upstream ${event.status} (${detail})`);
|
|
357
|
+
if (!this._onUpstreamAccount)
|
|
358
|
+
return;
|
|
359
|
+
try {
|
|
360
|
+
const result = this._onUpstreamAccount(event);
|
|
361
|
+
if (result && typeof result.catch === 'function') {
|
|
362
|
+
result.catch((err) => debugWarn(`[ModelHealth] onUpstreamAccount failed: ${err instanceof Error ? err.message : err}`));
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
catch (err) {
|
|
366
|
+
debugWarn(`[ModelHealth] onUpstreamAccount failed: ${err instanceof Error ? err.message : err}`);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
247
369
|
_recordFailure(provider, state) {
|
|
248
370
|
state.consecutiveFailures += 1;
|
|
249
371
|
debugWarn(`[ModelHealth] ${provider.name}/${state.service}: health check failed `
|
|
@@ -271,6 +393,7 @@ export class ModelHealthChecker {
|
|
|
271
393
|
}
|
|
272
394
|
this._emitChange({
|
|
273
395
|
provider: provider.name,
|
|
396
|
+
providerId: state.providerId,
|
|
274
397
|
service: state.service,
|
|
275
398
|
status: 'removed',
|
|
276
399
|
consecutiveFailures: state.consecutiveFailures,
|
|
@@ -286,6 +409,7 @@ export class ModelHealthChecker {
|
|
|
286
409
|
provider.healthCheckAvailable = true;
|
|
287
410
|
this._emitChange({
|
|
288
411
|
provider: provider.name,
|
|
412
|
+
providerId: state.providerId,
|
|
289
413
|
service: state.service,
|
|
290
414
|
status: 'restored',
|
|
291
415
|
consecutiveFailures: 0,
|
|
@@ -395,6 +519,14 @@ export function classifyProbeStatus(statusCode) {
|
|
|
395
519
|
return 'unhealthy';
|
|
396
520
|
return 'inconclusive';
|
|
397
521
|
}
|
|
522
|
+
function decodeBody(body, maxBytes) {
|
|
523
|
+
try {
|
|
524
|
+
return new TextDecoder().decode(body.slice(0, maxBytes));
|
|
525
|
+
}
|
|
526
|
+
catch {
|
|
527
|
+
return '';
|
|
528
|
+
}
|
|
529
|
+
}
|
|
398
530
|
function summarizeBody(body) {
|
|
399
531
|
try {
|
|
400
532
|
return new TextDecoder().decode(body.slice(0, 200)).replace(/\s+/g, ' ').trim() || '(empty body)';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-health-checker.js","sourceRoot":"","sources":["../../src/health/model-health-checker.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAA2B,MAAM,yBAAyB,CAAC;AAC5G,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAExD,yFAAyF;AACzF,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAAC,GAAG,MAAM,CAAC;AAC3D,mEAAmE;AACnE,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAC,CAAC;AACxD,6EAA6E;AAC7E,MAAM,CAAC,MAAM,qCAAqC,GAAG,MAAM,CAAC;AA8D5D;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,kBAAkB;IACZ,QAAQ,CAAsB;IAC9B,WAAW,CAAS;IACpB,iBAAiB,CAAS;IAC1B,eAAe,CAAS;IACxB,SAAS,CAAuC;IAChD,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACzD,MAAM,GAA0C,IAAI,CAAC;IACrD,cAAc,GAAyB,IAAI,CAAC;IAC5C,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,MAAgC;QAC1C,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,IAAI,gCAAgC,CAAC;QACzE,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,gBAAgB,IAAI,sCAAsC,CAAC,CAAC;QACxG,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,cAAc,IAAI,qCAAqC,CAAC;QACtF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,4EAA4E;QAC5E,0EAA0E;QAC1E,uDAAuD;QACvD,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,IAAI,QAAQ,CAAC,oBAAoB,KAAK,KAAK;gBAAE,SAAS;YACtD,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBACrD,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;gBACrB,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC;gBAC7B,KAAK,CAAC,UAAU,GAAG,wDAAwD,CAAC;YAC9E,CAAC;YACD,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,KAAK;QACH,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE;YAC7B,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACrB,uDAAuD;QACvD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IACxB,CAAC;IAED,IAAI;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACrB,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC,cAAc,CAAC;QACpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAClD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,WAAW;QACT,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACvD,QAAQ,EAAE,KAAK,CAAC,YAAY;YAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,CAAC,KAAK,CAAC,OAAO;YAC1B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,UAAU,EAAE,KAAK,CAAC,UAAU;SAC7B,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,QAAQ;gBAAE,OAAO;YAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YAEjC,0DAA0D;YAC1D,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACxC,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;oBACrC,QAAQ,CAAC,0BAA0B,QAAQ,CAAC,IAAI,kBAAkB,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC;oBACvG,SAAS;gBACX,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,yDAAyD;YAC3D,CAAC;YAED,mEAAmE;YACnE,mEAAmE;YACnE,uCAAuC;YACvC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACpD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,YAAY,KAAK,QAAQ,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;oBAC1D,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,MAAM,YAAY,GAAG,CAAC,SAAoB,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,kBAAkB,IAAI,SAAS,EAAE,oBAAoB,CAAC,CAAC;YACzH,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC,CAAC,QAAQ,CAAC;YAC5F,MAAM,OAAO,GAAG,YAAY,CAAC,aAAa,CAAC;gBACzC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBAClG,CAAC,CAAC,EAAE,CAAC;YACP,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;YAC5D,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,IAAI,IAAI,CAAC,QAAQ;oBAAE,OAAO;gBAC1B,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,SAAS;gBACxC,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,cAAc,CAAC,MAAyB,EAAE,aAAuB,EAAE,QAAkB;QACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,MAAM,KAAK,GAAG,CAAI,KAAa,EAAE,KAAuB,EAAE,EAAE,CAAC,WAAW,CACtE,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAC7B,IAAI,CAAC,eAAe,EACpB,GAAG,KAAK,QAAQ,QAAQ,CAAC,IAAI,oBAAoB,IAAI,CAAC,eAAe,IAAI,CAC1E,CAAC;QACF,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC1C,aAAa,CAAC,oBAAoB;gBAChC,CAAC,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,oBAAqB,EAAE,CAAC;qBAClE,KAAK,CAAC,CAAC,GAAG,EAAyB,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,yBAAyB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC9I,CAAC,CAAC,SAAS;YACb,aAAa,CAAC,kBAAkB;gBAC9B,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,kBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;gBAC3F,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,OAAO,EAAE,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,OAAO,GAAG,CAAC,OAAe,EAAyB,EAAE;YACzD,MAAM,KAAK,GAAoC,MAAM,KAAK,SAAS;gBACjE,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;YACrF,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAmC,EAAE,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;YAC3G,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,aAAa,CAAC;mBAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;mBACrD,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;QACpD,CAAC,CAAC;QACF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,QAAQ;gBAAE,OAAO;YAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACrD,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChC,QAAQ,CAAC,iBAAiB,QAAQ,CAAC,IAAI,IAAI,OAAO,mBAAmB,MAAM,CAAC,MAAM,IAAI,WAAW,GAAG,CAAC,CAAC;gBACtG,SAAS;YACX,CAAC;YACD,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC;YAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;gBACpC,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;gBACjC,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACrC,SAAS;YACX,CAAC;YACD,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,OAAO;gBAAE,SAAS;YAC7B,KAAK,CAAC,UAAU,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,kCAAkC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAC5H,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,YAAoB,EAAE,OAAe;QACrD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,OAAO,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG;gBACN,YAAY;gBACZ,OAAO;gBACP,mBAAmB,EAAE,CAAC;gBACtB,OAAO,EAAE,KAAK;gBACd,cAAc,EAAE,CAAC;gBACjB,WAAW,EAAE,IAAI;gBACjB,cAAc,EAAE,IAAI;gBACpB,UAAU,EAAE,IAAI;aACjB,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAyB,EAAE,OAAe;QACpE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,QAAQ,CAAC;QACvD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,IAAI,OAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACzD,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC5B,KAAK,CAAC,UAAU,GAAG,iDAAiD,QAAQ,EAAE,CAAC;gBAC/E,QAAQ,CAAC,iBAAiB,QAAQ,CAAC,IAAI,IAAI,OAAO,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;gBAC3E,OAAO;YACT,CAAC;YACD,MAAM,OAAO,GAAG,uBAAuB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC3D,MAAM,QAAQ,GAAG,MAAM,WAAW,CAChC,aAAa,CAAC,aAAa,CAAC,OAAO,CAAC,EACpC,IAAI,CAAC,eAAe,EACpB,oBAAoB,OAAO,oBAAoB,IAAI,CAAC,eAAe,IAAI,CACxE,CAAC;YACF,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC;YAC3C,OAAO,GAAG,mBAAmB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACnD,KAAK,CAAC,UAAU,GAAG,OAAO,KAAK,SAAS;gBACtC,CAAC,CAAC,QAAQ,QAAQ,CAAC,UAAU,EAAE;gBAC/B,CAAC,CAAC,QAAQ,QAAQ,CAAC,UAAU,KAAK,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACrE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC;YAC5B,KAAK,CAAC,UAAU,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACpE,OAAO,GAAG,WAAW,CAAC;QACxB,CAAC;QAED,IAAI,OAAO,KAAK,cAAc,EAAE,CAAC;YAC/B,uEAAuE;YACvE,uEAAuE;YACvE,wCAAwC;YACxC,QAAQ,CAAC,iBAAiB,QAAQ,CAAC,IAAI,IAAI,OAAO,mBAAmB,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC;YAC1F,OAAO;QACT,CAAC;QAED,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAC9B,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBAClB,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACxC,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACvC,CAAC;IAEO,cAAc,CAAC,QAAkB,EAAE,KAAyB;QAClE,KAAK,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAC/B,SAAS,CACP,iBAAiB,QAAQ,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,wBAAwB;cACrE,IAAI,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,iBAAiB,OAAO,KAAK,CAAC,UAAU,EAAE,CACnF,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1E,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,QAAkB,EAAE,KAAyB;QAClE,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QACnC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,6EAA6E;YAC7E,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;YACrB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,QAAQ,CAAC,oBAAoB,GAAG,KAAK,CAAC;YACxC,CAAC;YACD,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC1B,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;QACrB,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC;QAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,oBAAoB,GAAG,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,CAAC,WAAW,CAAC;YACf,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,SAAS;YACjB,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,MAAM,EAAE,KAAK,CAAC,UAAU,IAAI,cAAc;SAC3C,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,QAAkB,EAAE,KAAyB;QACnE,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QACnC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACrF,CAAC;QACD,QAAQ,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACrC,IAAI,CAAC,WAAW,CAAC;YACf,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,UAAU;YAClB,mBAAmB,EAAE,CAAC;YACtB,MAAM,EAAE,KAAK,CAAC,UAAU,IAAI,iBAAiB;SAC9C,CAAC,CAAC;IACL,CAAC;IAEO,WAAW,CAAC,KAAuB;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC;QAC7E,QAAQ,CAAC,iBAAiB,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,IAAI,MAAM,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACzF,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,MAAM,IAAI,OAAQ,MAAwB,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;gBACnE,MAAwB,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACtC,SAAS,CAAC,kCAAkC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1F,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,CAAC,kCAAkC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;CACF;AAED,+EAA+E;AAC/E,SAAS,oBAAoB,CAAC,QAAkB,EAAE,OAAe;IAC/D,OAAO,QAAQ,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,yBAAyB,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAA4B;IAC9D,OAAO,QAAQ,KAAK,eAAe,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;AAC1G,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAe,EAAE,QAA4B;IACnF,IAAI,IAAY,CAAC;IACjB,IAAI,IAA6B,CAAC;IAClC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,oBAAoB;YACvB,IAAI,GAAG,cAAc,CAAC;YACtB,IAAI,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YACxF,MAAM;QACR,KAAK,kBAAkB;YACrB,IAAI,GAAG,eAAe,CAAC;YACvB,sEAAsE;YACtE,yEAAyE;YACzE,IAAI,GAAG;gBACL,KAAK,EAAE,OAAO;gBACd,KAAK,EAAE,CAAC;wBACN,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;qBAChD,CAAC;gBACF,iBAAiB,EAAE,EAAE;aACtB,CAAC;YACF,MAAM;QACR,KAAK,oBAAoB;YACvB,IAAI,GAAG,iBAAiB,CAAC;YACzB,IAAI,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;YACzD,MAAM;QACR,KAAK,yBAAyB;YAC5B,IAAI,GAAG,sBAAsB,CAAC;YAC9B,IAAI,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YACxF,MAAM;QACR,KAAK,oBAAoB;YACvB,IAAI,GAAG,eAAe,CAAC;YACvB,iEAAiE;YACjE,kBAAkB;YAClB,IAAI,GAAG;gBACL,KAAK,EAAE,OAAO;gBACd,KAAK,EAAE,MAAM;gBACb,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,6BAA6B,EAAE,EAAE;aACjF,CAAC;YACF,MAAM;QACR;YACE,MAAM,IAAI,KAAK,CAAC,uCAAuC,QAAQ,WAAW,CAAC,CAAC;IAChF,CAAC;IACD,OAAO;QACL,SAAS,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;QACzF,MAAM,EAAE,MAAM;QACd,IAAI;QACJ,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KACrD,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CAAC,UAAkB;IACpD,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,GAAG,GAAG;QAAE,OAAO,SAAS,CAAC;IAC5D,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QAC/G,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,IAAI,UAAU,IAAI,GAAG;QAAE,OAAO,WAAW,CAAC;IAC1C,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,SAAS,aAAa,CAAC,IAAgB;IACrC,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,cAAc,CAAC;IACpG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,mBAAmB,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAI,OAAmB,EAAE,SAAiB,EAAE,OAAe;IACnF,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;QAC/C,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAChE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,yEAAyE;QACzE,6BAA6B;QAC7B,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"model-health-checker.js","sourceRoot":"","sources":["../../src/health/model-health-checker.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAA2B,MAAM,yBAAyB,CAAC;AAC5G,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAExF,yFAAyF;AACzF,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAAC,GAAG,MAAM,CAAC;AAC3D,mEAAmE;AACnE,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAC,CAAC;AACxD,6EAA6E;AAC7E,MAAM,CAAC,MAAM,qCAAqC,GAAG,MAAM,CAAC;AAqG5D;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,kBAAkB;IACZ,QAAQ,CAAsB;IAC9B,WAAW,CAAS;IACpB,iBAAiB,CAAS;IAC1B,eAAe,CAAS;IACxB,SAAS,CAAuC;IAChD,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACjE,kFAAkF;IACjE,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;IACpD,MAAM,GAA0C,IAAI,CAAC;IACrD,cAAc,GAAyB,IAAI,CAAC;IAC5C,QAAQ,GAAG,KAAK,CAAC;IACR,kBAAkB,CAAgD;IAClE,cAAc,CAA4C;IAC1D,gBAAgB,CAA8C;IAC/E,uEAAuE;IACtD,SAAS,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC/D,uFAAuF;IAC/E,gBAAgB,GAAyB,EAAE,CAAC;IAEpD,YAAY,MAAgC;QAC1C,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,IAAI,gCAAgC,CAAC;QACzE,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,gBAAgB,IAAI,sCAAsC,CAAC,CAAC;QACxG,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,cAAc,IAAI,qCAAqC,CAAC;QACtF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,eAAe,CAAC;QAC/C,4EAA4E;QAC5E,0EAA0E;QAC1E,uDAAuD;QACvD,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,IAAI,QAAQ,CAAC,oBAAoB,KAAK,KAAK;gBAAE,SAAS;YACtD,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAChD,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;gBACrB,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC;gBAC7B,KAAK,CAAC,UAAU,GAAG,wDAAwD,CAAC;gBAC5E,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpC,CAAC;YACD,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,KAAK;QACH,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,KAAK,CAAC,OAAO;gBAAE,SAAS;YAC7B,IAAI,CAAC,WAAW,CAAC;gBACf,QAAQ,EAAE,KAAK,CAAC,YAAY;gBAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,MAAM,EAAE,SAAS;gBACjB,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;gBAC9C,MAAM,EAAE,KAAK,CAAC,UAAU,IAAI,uBAAuB;aACpD,CAAC,CAAC;QACL,CAAC;QACD,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE;YAC7B,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACrB,uDAAuD;QACvD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IACxB,CAAC;IAED,IAAI;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACrB,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC,cAAc,CAAC;QACpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAClD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;YAC3B,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO;YACpD,IAAI,CAAC;gBACH,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,CAAC,yCAAyC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YACjG,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,WAAW;QACT,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACvD,QAAQ,EAAE,KAAK,CAAC,YAAY;YAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,CAAC,KAAK,CAAC,OAAO;YAC1B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,UAAU,EAAE,KAAK,CAAC,UAAU;SAC7B,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,QAAQ;gBAAE,OAAO;YAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YAEjC,0DAA0D;YAC1D,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACxC,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;oBACrC,QAAQ,CAAC,0BAA0B,QAAQ,CAAC,IAAI,kBAAkB,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC;oBACvG,SAAS;gBACX,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,yDAAyD;YAC3D,CAAC;YAED,mEAAmE;YACnE,mEAAmE;YACnE,uCAAuC;YACvC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACpD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;oBACjD,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,MAAM,YAAY,GAAG,CAAC,SAAoB,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,kBAAkB,IAAI,SAAS,EAAE,oBAAoB,CAAC,CAAC;YACzH,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC,CAAC,QAAQ,CAAC;YAC5F,MAAM,OAAO,GAAG,YAAY,CAAC,aAAa,CAAC;gBACzC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBAClG,CAAC,CAAC,EAAE,CAAC;YACP,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;YAC5D,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,IAAI,IAAI,CAAC,QAAQ;oBAAE,OAAO;gBAC1B,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,SAAS;gBACxC,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,cAAc,CAAC,MAAyB,EAAE,aAAuB,EAAE,QAAkB;QACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,MAAM,KAAK,GAAG,CAAI,KAAa,EAAE,KAAuB,EAAE,EAAE,CAAC,WAAW,CACtE,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAC7B,IAAI,CAAC,eAAe,EACpB,GAAG,KAAK,QAAQ,QAAQ,CAAC,IAAI,oBAAoB,IAAI,CAAC,eAAe,IAAI,CAC1E,CAAC;QACF,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC1C,aAAa,CAAC,oBAAoB;gBAChC,CAAC,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,oBAAqB,EAAE,CAAC;qBAClE,KAAK,CAAC,CAAC,GAAG,EAAyB,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,yBAAyB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC9I,CAAC,CAAC,SAAS;YACb,aAAa,CAAC,kBAAkB;gBAC9B,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,kBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;gBAC3F,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,OAAO,EAAE,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,IAAI,OAAO,EAAE,MAAM,KAAK,aAAa,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,qBAAqB,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YACvH,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC1D,CAAC;aAAM,IAAI,OAAO,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,gHAAgH;YAChH,MAAM,QAAQ,GAAG,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK,CAAC;YACjF,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC;gBAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,YAAY,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;iBACnH,IAAI,OAAO,KAAK,SAAS,IAAI,GAAG,KAAK,SAAS,IAAI,OAAO,GAAG,GAAG;gBAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,YAAY,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;iBACxK,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,IAAI,IAAI,CAAC,CAAC;QAC3J,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,OAAe,EAAyB,EAAE;YACzD,MAAM,KAAK,GAAoC,MAAM,KAAK,SAAS;gBACjE,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;YACrF,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAmC,EAAE,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;YAC3G,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,aAAa,CAAC;mBAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;mBACrD,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;QACpD,CAAC,CAAC;QACF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,QAAQ;gBAAE,OAAO;YAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChC,QAAQ,CAAC,iBAAiB,QAAQ,CAAC,IAAI,IAAI,OAAO,mBAAmB,MAAM,CAAC,MAAM,IAAI,WAAW,GAAG,CAAC,CAAC;gBACtG,SAAS;YACX,CAAC;YACD,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC;YAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;gBACpC,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;gBACjC,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACrC,SAAS;YACX,CAAC;YACD,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,OAAO;gBAAE,SAAS;YAC7B,KAAK,CAAC,UAAU,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,kCAAkC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAC5H,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,QAAkB,EAAE,OAAe;QACnD,IAAI,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACrB,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,EAAE,CAAC;QAChC,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG;gBACN,QAAQ;gBACR,YAAY,EAAE,QAAQ,CAAC,IAAI;gBAC3B,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,EAAE,IAAI,QAAQ,CAAC,IAAI;gBAC7F,OAAO;gBACP,mBAAmB,EAAE,CAAC;gBACtB,OAAO,EAAE,KAAK;gBACd,cAAc,EAAE,CAAC;gBACjB,WAAW,EAAE,IAAI;gBACjB,cAAc,EAAE,IAAI;gBACpB,UAAU,EAAE,IAAI;aACjB,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAyB,EAAE,OAAe;QACpE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,QAAQ,CAAC;QACvD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,IAAI,OAAqB,CAAC;QAC1B,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACzD,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC5B,KAAK,CAAC,UAAU,GAAG,iDAAiD,QAAQ,EAAE,CAAC;gBAC/E,QAAQ,CAAC,iBAAiB,QAAQ,CAAC,IAAI,IAAI,OAAO,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;gBAC3E,OAAO;YACT,CAAC;YACD,MAAM,OAAO,GAAG,uBAAuB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC3D,MAAM,QAAQ,GAAG,MAAM,WAAW,CAChC,aAAa,CAAC,aAAa,CAAC,OAAO,CAAC,EACpC,IAAI,CAAC,eAAe,EACpB,oBAAoB,OAAO,oBAAoB,IAAI,CAAC,eAAe,IAAI,CACxE,CAAC;YACF,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC;YAC3C,OAAO,GAAG,mBAAmB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACnD,qGAAqG;YACrG,IAAI,OAAO,KAAK,SAAS;gBAAE,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACzE,KAAK,CAAC,UAAU,GAAG,OAAO,KAAK,SAAS;gBACtC,CAAC,CAAC,QAAQ,QAAQ,CAAC,UAAU,EAAE;gBAC/B,CAAC,CAAC,QAAQ,QAAQ,CAAC,UAAU,KAAK,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACrE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC;YAC5B,KAAK,CAAC,UAAU,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACpE,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC;YAC7B,OAAO,GAAG,WAAW,CAAC;QACxB,CAAC;QAED,uFAAuF;QACvF,+EAA+E;QAC/E,IAAI,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,GAAG,IAAI,KAAK,CAAC,cAAc,KAAK,GAAG,IAAI,qBAAqB,CAAC,KAAK,CAAC,cAAc,EAAE,SAAS,CAAC,EAAE,CAAC;YACpJ,OAAO,GAAG,WAAW,CAAC;YACtB,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,UAAU,IAAI,eAAe,EAAE,IAAI,CAAC,CAAC;QAC/E,CAAC;QACD,IAAI,OAAO,KAAK,cAAc,EAAE,CAAC;YAC/B,uEAAuE;YACvE,uEAAuE;YACvE,wCAAwC;YACxC,QAAQ,CAAC,iBAAiB,QAAQ,CAAC,IAAI,IAAI,OAAO,mBAAmB,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC;YAC1F,OAAO;QACT,CAAC;QAED,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,qFAAqF;YACrF,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK;gBAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,IAAI,iBAAiB,EAAE,IAAI,CAAC,CAAC;YAC1H,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAC9B,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBAClB,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACxC,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACH,sBAAsB,CAAC,QAAkB,EAAE,UAAkB,EAAE,IAAY,EAAE,QAAQ,GAAG,IAAI;QAC1F,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC;QAC9G,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QACrC,IAAI,UAAU,IAAI,GAAG,IAAI,yBAAyB,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC;YACrE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,UAAU,KAAK,aAAa,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACzH,CAAC;aAAM,IAAI,UAAU,GAAG,GAAG,IAAI,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAE,CAAC;YACzF,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,UAAU,eAAe,EAAE,IAAI,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAEO,QAAQ;QACd,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,cAAc,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC;QACtG,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9F,CAAC;IAED,gHAAgH;IACxG,WAAW,CAAC,QAAkB,EAAE,IAAkB,EAAE,MAAc,EAAE,UAAyB;QACnG,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9C,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO;QAC9B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;QAC1E,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAClK,MAAM,KAAK,GAAyB;YAClC,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,UAAU,EAAE,MAAM,EAAE,EAAE,IAAI,QAAQ,CAAC,IAAI;YACvC,MAAM,EAAE,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ;YACpF,MAAM;YACN,UAAU;YACV,QAAQ;SACT,CAAC;QACF,SAAS,CAAC,iBAAiB,KAAK,CAAC,UAAU,cAAc,KAAK,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,CAAC;QACrF,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACrC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAC9C,IAAI,MAAM,IAAI,OAAQ,MAAwB,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;gBACnE,MAAwB,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,2CAA2C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC7I,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,CAAC,2CAA2C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACnG,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,QAAkB,EAAE,KAAyB;QAClE,KAAK,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAC/B,SAAS,CACP,iBAAiB,QAAQ,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,wBAAwB;cACrE,IAAI,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,iBAAiB,OAAO,KAAK,CAAC,UAAU,EAAE,CACnF,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1E,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,QAAkB,EAAE,KAAyB;QAClE,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QACnC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,6EAA6E;YAC7E,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;YACrB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,QAAQ,CAAC,oBAAoB,GAAG,KAAK,CAAC;YACxC,CAAC;YACD,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC1B,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;QACrB,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC;QAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,oBAAoB,GAAG,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,CAAC,WAAW,CAAC;YACf,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,SAAS;YACjB,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,MAAM,EAAE,KAAK,CAAC,UAAU,IAAI,cAAc;SAC3C,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,QAAkB,EAAE,KAAyB;QACnE,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QACnC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACrF,CAAC;QACD,QAAQ,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACrC,IAAI,CAAC,WAAW,CAAC;YACf,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,UAAU;YAClB,mBAAmB,EAAE,CAAC;YACtB,MAAM,EAAE,KAAK,CAAC,UAAU,IAAI,iBAAiB;SAC9C,CAAC,CAAC;IACL,CAAC;IAEO,WAAW,CAAC,KAAuB;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC;QAC7E,QAAQ,CAAC,iBAAiB,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,IAAI,MAAM,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACzF,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,MAAM,IAAI,OAAQ,MAAwB,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;gBACnE,MAAwB,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACtC,SAAS,CAAC,kCAAkC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1F,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,CAAC,kCAAkC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;CACF;AAED,+EAA+E;AAC/E,SAAS,oBAAoB,CAAC,QAAkB,EAAE,OAAe;IAC/D,OAAO,QAAQ,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,yBAAyB,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAA4B;IAC9D,OAAO,QAAQ,KAAK,eAAe,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;AAC1G,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAe,EAAE,QAA4B;IACnF,IAAI,IAAY,CAAC;IACjB,IAAI,IAA6B,CAAC;IAClC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,oBAAoB;YACvB,IAAI,GAAG,cAAc,CAAC;YACtB,IAAI,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YACxF,MAAM;QACR,KAAK,kBAAkB;YACrB,IAAI,GAAG,eAAe,CAAC;YACvB,sEAAsE;YACtE,yEAAyE;YACzE,IAAI,GAAG;gBACL,KAAK,EAAE,OAAO;gBACd,KAAK,EAAE,CAAC;wBACN,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;qBAChD,CAAC;gBACF,iBAAiB,EAAE,EAAE;aACtB,CAAC;YACF,MAAM;QACR,KAAK,oBAAoB;YACvB,IAAI,GAAG,iBAAiB,CAAC;YACzB,IAAI,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;YACzD,MAAM;QACR,KAAK,yBAAyB;YAC5B,IAAI,GAAG,sBAAsB,CAAC;YAC9B,IAAI,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YACxF,MAAM;QACR,KAAK,oBAAoB;YACvB,IAAI,GAAG,eAAe,CAAC;YACvB,iEAAiE;YACjE,kBAAkB;YAClB,IAAI,GAAG;gBACL,KAAK,EAAE,OAAO;gBACd,KAAK,EAAE,MAAM;gBACb,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,6BAA6B,EAAE,EAAE;aACjF,CAAC;YACF,MAAM;QACR;YACE,MAAM,IAAI,KAAK,CAAC,uCAAuC,QAAQ,WAAW,CAAC,CAAC;IAChF,CAAC;IACD,OAAO;QACL,SAAS,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;QACzF,MAAM,EAAE,MAAM;QACd,IAAI;QACJ,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KACrD,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CAAC,UAAkB;IACpD,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,GAAG,GAAG;QAAE,OAAO,SAAS,CAAC;IAC5D,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QAC/G,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,IAAI,UAAU,IAAI,GAAG;QAAE,OAAO,WAAW,CAAC;IAC1C,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,SAAS,UAAU,CAAC,IAAgB,EAAE,QAAgB;IACpD,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAgB;IACrC,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,cAAc,CAAC;IACpG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,mBAAmB,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAI,OAAmB,EAAE,SAAiB,EAAE,OAAe;IACnF,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;QAC/C,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAChE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,yEAAyE;QACzE,6BAA6B;QAC7B,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Seller event journal: `<dataDir>/seller-events.jsonl`, newest last. */
|
|
2
|
+
export declare const SELLER_EVENTS_FILE = "seller-events.jsonl";
|
|
3
|
+
/** The journal is rotated to `seller-events.1.jsonl` past this size. */
|
|
4
|
+
export declare const SELLER_EVENTS_MAX_BYTES = 1000000;
|
|
5
|
+
export type SellerEventKind = 'started' | 'stopped' | 'crashed' | 'model_down' | 'model_up' | 'gas_low' | 'gas_ok' | 'idle' | 'active' | 'upstream_credit' | 'upstream_credit_ok' | 'upstream_balance_low' | 'upstream_balance_ok' | 'test';
|
|
6
|
+
export type SellerEventSeverity = 'info' | 'warning' | 'critical';
|
|
7
|
+
export interface SellerEvent {
|
|
8
|
+
/** Unix milliseconds. */
|
|
9
|
+
at: number;
|
|
10
|
+
kind: SellerEventKind;
|
|
11
|
+
severity: SellerEventSeverity;
|
|
12
|
+
/** One line, plain text, safe to post to a chat. */
|
|
13
|
+
title: string;
|
|
14
|
+
detail?: string;
|
|
15
|
+
provider?: string;
|
|
16
|
+
/** The configured provider name, when several instances share a plugin name. */
|
|
17
|
+
providerId?: string;
|
|
18
|
+
service?: string;
|
|
19
|
+
}
|
|
20
|
+
/** Append one event, rotating the journal once it passes SELLER_EVENTS_MAX_BYTES. */
|
|
21
|
+
export declare function appendSellerEvent(dataDir: string, event: SellerEvent): Promise<void>;
|
|
22
|
+
/** The latest `limit` events, newest first (reads the rotated file when the current one is short). */
|
|
23
|
+
export declare function readSellerEvents(dataDir: string, limit?: number): Promise<SellerEvent[]>;
|
|
24
|
+
//# sourceMappingURL=seller-events.d.ts.map
|