@juspay/neurolink 10.1.2 → 10.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/browser/neurolink.min.js +389 -389
- package/dist/cli/commands/proxy.js +85 -14
- package/dist/lib/proxy/proxyPaths.d.ts +1 -0
- package/dist/lib/proxy/proxyPaths.js +5 -0
- package/dist/lib/proxy/usageStats.d.ts +58 -4
- package/dist/lib/proxy/usageStats.js +639 -79
- package/dist/lib/types/proxy.d.ts +39 -0
- package/dist/proxy/proxyPaths.d.ts +1 -0
- package/dist/proxy/proxyPaths.js +5 -0
- package/dist/proxy/usageStats.d.ts +58 -4
- package/dist/proxy/usageStats.js +639 -79
- package/dist/types/proxy.d.ts +39 -0
- package/package.json +3 -2
package/dist/proxy/usageStats.js
CHANGED
|
@@ -1,98 +1,658 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Proxy
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Proxy usage statistics with restart- and handoff-safe persistence.
|
|
3
|
+
*
|
|
4
|
+
* Request handlers update memory synchronously. Small deltas are merged into a
|
|
5
|
+
* shared snapshot asynchronously, under a cross-process lock, so overlapping
|
|
6
|
+
* rolling workers cannot overwrite each other's counters.
|
|
5
7
|
*/
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
export function recordAttempt(accountLabel, accountType) {
|
|
19
|
-
stats.totalAttempts++;
|
|
20
|
-
const acct = ensureAccount(accountLabel, accountType);
|
|
21
|
-
acct.attemptCount++;
|
|
22
|
-
acct.lastAttemptAt = Date.now();
|
|
8
|
+
import { randomUUID } from "node:crypto";
|
|
9
|
+
import { mkdir, open, readFile, readdir, rename, rm, stat, } from "node:fs/promises";
|
|
10
|
+
import { basename, dirname, join } from "node:path";
|
|
11
|
+
import { AsyncMutex } from "../utils/asyncMutex.js";
|
|
12
|
+
import { writeJsonSnapshotAtomically } from "./snapshotPersistence.js";
|
|
13
|
+
const SNAPSHOT_SCHEMA_VERSION = 1;
|
|
14
|
+
const DEFAULT_FLUSH_INTERVAL_MS = 1_000;
|
|
15
|
+
const DEFAULT_LOCK_TIMEOUT_MS = 2_000;
|
|
16
|
+
const DEFAULT_STALE_LOCK_MS = 30_000;
|
|
17
|
+
const LOCK_RETRY_MS = 25;
|
|
18
|
+
const MAX_CORRUPT_SNAPSHOTS = 3;
|
|
19
|
+
class InvalidProxyStatsSnapshotError extends Error {
|
|
23
20
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
21
|
+
function emptyStats(startedAt) {
|
|
22
|
+
return {
|
|
23
|
+
startedAt,
|
|
24
|
+
totalAttempts: 0,
|
|
25
|
+
totalAttemptErrors: 0,
|
|
26
|
+
totalRequests: 0,
|
|
27
|
+
totalSuccess: 0,
|
|
28
|
+
totalErrors: 0,
|
|
29
|
+
totalRateLimits: 0,
|
|
30
|
+
totalTransientRateLimits: 0,
|
|
31
|
+
totalQuotaRateLimits: 0,
|
|
32
|
+
accounts: {},
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function cloneAccount(account) {
|
|
36
|
+
return { ...account };
|
|
37
|
+
}
|
|
38
|
+
function cloneStats(value) {
|
|
39
|
+
return {
|
|
40
|
+
...value,
|
|
41
|
+
accounts: Object.fromEntries(Object.entries(value.accounts).map(([label, account]) => [
|
|
42
|
+
label,
|
|
43
|
+
cloneAccount(account),
|
|
44
|
+
])),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function mergeAccountStats(left, right) {
|
|
48
|
+
if (!left) {
|
|
49
|
+
return cloneAccount(right);
|
|
30
50
|
}
|
|
51
|
+
return {
|
|
52
|
+
label: right.label || left.label,
|
|
53
|
+
type: right.type || left.type,
|
|
54
|
+
attemptCount: left.attemptCount + right.attemptCount,
|
|
55
|
+
attemptErrorCount: left.attemptErrorCount + right.attemptErrorCount,
|
|
56
|
+
successCount: left.successCount + right.successCount,
|
|
57
|
+
errorCount: left.errorCount + right.errorCount,
|
|
58
|
+
rateLimitCount: left.rateLimitCount + right.rateLimitCount,
|
|
59
|
+
transientRateLimitCount: left.transientRateLimitCount + right.transientRateLimitCount,
|
|
60
|
+
quotaRateLimitCount: left.quotaRateLimitCount + right.quotaRateLimitCount,
|
|
61
|
+
lastAttemptAt: Math.max(left.lastAttemptAt, right.lastAttemptAt),
|
|
62
|
+
...(left.lastErrorAt || right.lastErrorAt
|
|
63
|
+
? {
|
|
64
|
+
lastErrorAt: Math.max(left.lastErrorAt ?? 0, right.lastErrorAt ?? 0),
|
|
65
|
+
}
|
|
66
|
+
: {}),
|
|
67
|
+
};
|
|
31
68
|
}
|
|
32
|
-
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
69
|
+
function mergeStats(left, right) {
|
|
70
|
+
const accounts = Object.fromEntries(Object.entries(left.accounts).map(([label, account]) => [
|
|
71
|
+
label,
|
|
72
|
+
cloneAccount(account),
|
|
73
|
+
]));
|
|
74
|
+
for (const [label, account] of Object.entries(right.accounts)) {
|
|
75
|
+
accounts[label] = mergeAccountStats(accounts[label], account);
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
startedAt: Math.min(left.startedAt, right.startedAt),
|
|
79
|
+
totalAttempts: left.totalAttempts + right.totalAttempts,
|
|
80
|
+
totalAttemptErrors: left.totalAttemptErrors + right.totalAttemptErrors,
|
|
81
|
+
totalRequests: left.totalRequests + right.totalRequests,
|
|
82
|
+
totalSuccess: left.totalSuccess + right.totalSuccess,
|
|
83
|
+
totalErrors: left.totalErrors + right.totalErrors,
|
|
84
|
+
totalRateLimits: left.totalRateLimits + right.totalRateLimits,
|
|
85
|
+
totalTransientRateLimits: left.totalTransientRateLimits + right.totalTransientRateLimits,
|
|
86
|
+
totalQuotaRateLimits: left.totalQuotaRateLimits + right.totalQuotaRateLimits,
|
|
87
|
+
accounts,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
function finiteNonNegativeInteger(value) {
|
|
91
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
92
|
+
}
|
|
93
|
+
function validAccountStats(value) {
|
|
94
|
+
if (!value || typeof value !== "object") {
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
const candidate = value;
|
|
98
|
+
return (typeof candidate.label === "string" &&
|
|
99
|
+
typeof candidate.type === "string" &&
|
|
100
|
+
finiteNonNegativeInteger(candidate.attemptCount) &&
|
|
101
|
+
finiteNonNegativeInteger(candidate.attemptErrorCount) &&
|
|
102
|
+
finiteNonNegativeInteger(candidate.successCount) &&
|
|
103
|
+
finiteNonNegativeInteger(candidate.errorCount) &&
|
|
104
|
+
finiteNonNegativeInteger(candidate.rateLimitCount) &&
|
|
105
|
+
finiteNonNegativeInteger(candidate.transientRateLimitCount) &&
|
|
106
|
+
finiteNonNegativeInteger(candidate.quotaRateLimitCount) &&
|
|
107
|
+
finiteNonNegativeInteger(candidate.lastAttemptAt) &&
|
|
108
|
+
(candidate.lastErrorAt === undefined ||
|
|
109
|
+
finiteNonNegativeInteger(candidate.lastErrorAt)));
|
|
110
|
+
}
|
|
111
|
+
function validStats(value) {
|
|
112
|
+
if (!value || typeof value !== "object") {
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
const candidate = value;
|
|
116
|
+
const counters = [
|
|
117
|
+
candidate.startedAt,
|
|
118
|
+
candidate.totalAttempts,
|
|
119
|
+
candidate.totalAttemptErrors,
|
|
120
|
+
candidate.totalRequests,
|
|
121
|
+
candidate.totalSuccess,
|
|
122
|
+
candidate.totalErrors,
|
|
123
|
+
candidate.totalRateLimits,
|
|
124
|
+
candidate.totalTransientRateLimits,
|
|
125
|
+
candidate.totalQuotaRateLimits,
|
|
126
|
+
];
|
|
127
|
+
if (!counters.every(finiteNonNegativeInteger) ||
|
|
128
|
+
!candidate.accounts ||
|
|
129
|
+
typeof candidate.accounts !== "object") {
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
return Object.entries(candidate.accounts).every(([label, account]) => validAccountStats(account) && label === account.label);
|
|
133
|
+
}
|
|
134
|
+
function validSnapshot(value) {
|
|
135
|
+
if (!value || typeof value !== "object") {
|
|
136
|
+
return false;
|
|
137
|
+
}
|
|
138
|
+
const candidate = value;
|
|
139
|
+
return (candidate.schemaVersion === SNAPSHOT_SCHEMA_VERSION &&
|
|
140
|
+
finiteNonNegativeInteger(candidate.revision) &&
|
|
141
|
+
finiteNonNegativeInteger(candidate.updatedAt) &&
|
|
142
|
+
validStats(candidate.stats));
|
|
143
|
+
}
|
|
144
|
+
function isMissingFileError(error) {
|
|
145
|
+
return error?.code === "ENOENT";
|
|
146
|
+
}
|
|
147
|
+
function isCorruptSnapshotError(error) {
|
|
148
|
+
return (error instanceof SyntaxError ||
|
|
149
|
+
error instanceof InvalidProxyStatsSnapshotError);
|
|
150
|
+
}
|
|
151
|
+
function processIsRunning(pid) {
|
|
152
|
+
try {
|
|
153
|
+
process.kill(pid, 0);
|
|
154
|
+
return true;
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
return error.code === "EPERM";
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
async function sleep(ms) {
|
|
161
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
162
|
+
}
|
|
163
|
+
async function readLockOwner(lockPath) {
|
|
164
|
+
try {
|
|
165
|
+
const parsed = JSON.parse(await readFile(lockPath, "utf8"));
|
|
166
|
+
if (!parsed || typeof parsed !== "object") {
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
const candidate = parsed;
|
|
170
|
+
if (typeof candidate.token !== "string" ||
|
|
171
|
+
!finiteNonNegativeInteger(candidate.pid) ||
|
|
172
|
+
candidate.pid === 0 ||
|
|
173
|
+
!finiteNonNegativeInteger(candidate.acquiredAt)) {
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
return candidate;
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
async function removeAbandonedLock(lockPath, staleLockMs, now) {
|
|
183
|
+
const owner = await readLockOwner(lockPath);
|
|
184
|
+
try {
|
|
185
|
+
const lockStat = await stat(lockPath);
|
|
186
|
+
const oldEnough = now - lockStat.mtimeMs >= staleLockMs;
|
|
187
|
+
const ownerIsRunning = owner ? processIsRunning(owner.pid) : false;
|
|
188
|
+
if (owner && ownerIsRunning && !oldEnough) {
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
191
|
+
if ((owner && !ownerIsRunning) || oldEnough) {
|
|
192
|
+
await rm(lockPath, { force: true });
|
|
193
|
+
return true;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
return isMissingFileError(error);
|
|
198
|
+
}
|
|
199
|
+
return false;
|
|
200
|
+
}
|
|
201
|
+
async function acquireFileLock(lockPath, timeoutMs, staleLockMs, now) {
|
|
202
|
+
await mkdir(dirname(lockPath), { recursive: true, mode: 0o700 });
|
|
203
|
+
const deadline = Date.now() + timeoutMs;
|
|
204
|
+
while (true) {
|
|
205
|
+
const owner = {
|
|
206
|
+
token: randomUUID(),
|
|
207
|
+
pid: process.pid,
|
|
208
|
+
acquiredAt: now(),
|
|
209
|
+
};
|
|
210
|
+
let handle;
|
|
211
|
+
try {
|
|
212
|
+
handle = await open(lockPath, "wx", 0o600);
|
|
213
|
+
await handle.writeFile(JSON.stringify(owner));
|
|
214
|
+
const acquiredHandle = handle;
|
|
215
|
+
return async () => {
|
|
216
|
+
await acquiredHandle.close().catch(() => undefined);
|
|
217
|
+
const current = await readLockOwner(lockPath);
|
|
218
|
+
if (current?.token === owner.token) {
|
|
219
|
+
await rm(lockPath, { force: true }).catch(() => undefined);
|
|
220
|
+
}
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
catch (error) {
|
|
224
|
+
if (handle) {
|
|
225
|
+
await handle.close().catch(() => undefined);
|
|
226
|
+
await rm(lockPath, { force: true }).catch(() => undefined);
|
|
227
|
+
}
|
|
228
|
+
if (error.code !== "EEXIST") {
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
if (await removeAbandonedLock(lockPath, staleLockMs, now())) {
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
if (Date.now() >= deadline) {
|
|
236
|
+
throw new Error(`Timed out acquiring proxy stats lock ${lockPath}`);
|
|
237
|
+
}
|
|
238
|
+
await sleep(LOCK_RETRY_MS);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
export class ProxyUsageStatsStore {
|
|
242
|
+
now;
|
|
243
|
+
flushIntervalMs;
|
|
244
|
+
lockTimeoutMs;
|
|
245
|
+
staleLockMs;
|
|
246
|
+
filePath;
|
|
247
|
+
stats;
|
|
248
|
+
pending;
|
|
249
|
+
pendingMutations = 0;
|
|
250
|
+
inFlightMutations = 0;
|
|
251
|
+
revision = 0;
|
|
252
|
+
flushTimer = null;
|
|
253
|
+
flushMutex = new AsyncMutex();
|
|
254
|
+
lastFlushedAt;
|
|
255
|
+
lastReconciledAt;
|
|
256
|
+
lastRecoveryAt;
|
|
257
|
+
lastError;
|
|
258
|
+
constructor(options = {}) {
|
|
259
|
+
this.now = options.now ?? Date.now;
|
|
260
|
+
this.flushIntervalMs = options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS;
|
|
261
|
+
this.lockTimeoutMs = options.lockTimeoutMs ?? DEFAULT_LOCK_TIMEOUT_MS;
|
|
262
|
+
this.staleLockMs = options.staleLockMs ?? DEFAULT_STALE_LOCK_MS;
|
|
263
|
+
this.filePath = options.filePath;
|
|
264
|
+
const startedAt = this.now();
|
|
265
|
+
this.stats = emptyStats(startedAt);
|
|
266
|
+
this.pending = emptyStats(startedAt);
|
|
267
|
+
}
|
|
268
|
+
async initialize(filePath = this.filePath ?? "") {
|
|
269
|
+
this.cancelFlushTimer();
|
|
270
|
+
this.filePath = filePath || undefined;
|
|
271
|
+
const startedAt = this.now();
|
|
272
|
+
this.stats = emptyStats(startedAt);
|
|
273
|
+
this.pending = emptyStats(startedAt);
|
|
274
|
+
this.pendingMutations = 0;
|
|
275
|
+
this.inFlightMutations = 0;
|
|
276
|
+
this.revision = 0;
|
|
277
|
+
this.lastFlushedAt = undefined;
|
|
278
|
+
this.lastReconciledAt = undefined;
|
|
279
|
+
this.lastRecoveryAt = undefined;
|
|
280
|
+
this.lastError = undefined;
|
|
281
|
+
if (!this.filePath) {
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
try {
|
|
285
|
+
const snapshot = await this.readSnapshot();
|
|
286
|
+
if (snapshot) {
|
|
287
|
+
this.stats = cloneStats(snapshot.stats);
|
|
288
|
+
this.pending = emptyStats(this.now());
|
|
289
|
+
this.revision = snapshot.revision;
|
|
290
|
+
this.lastFlushedAt = snapshot.updatedAt;
|
|
291
|
+
}
|
|
292
|
+
this.lastReconciledAt = this.now();
|
|
293
|
+
}
|
|
294
|
+
catch (error) {
|
|
295
|
+
if (isCorruptSnapshotError(error)) {
|
|
296
|
+
try {
|
|
297
|
+
const recoveredSnapshot = await this.recoverCorruptSnapshot();
|
|
298
|
+
if (recoveredSnapshot) {
|
|
299
|
+
this.applySnapshot(recoveredSnapshot);
|
|
300
|
+
}
|
|
301
|
+
this.lastReconciledAt = this.now();
|
|
302
|
+
}
|
|
303
|
+
catch (recoveryError) {
|
|
304
|
+
this.lastError = this.describeError(recoveryError);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
this.lastError = this.describeError(error);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
recordAttempt(accountLabel, accountType) {
|
|
313
|
+
const attemptedAt = this.now();
|
|
314
|
+
this.applyAttempt(this.stats, accountLabel, accountType, attemptedAt);
|
|
315
|
+
this.applyAttempt(this.pending, accountLabel, accountType, attemptedAt);
|
|
316
|
+
this.markMutation();
|
|
317
|
+
}
|
|
318
|
+
recordFinalSuccess(accountLabel, accountType) {
|
|
319
|
+
this.applyFinalSuccess(this.stats, accountLabel, accountType);
|
|
320
|
+
this.applyFinalSuccess(this.pending, accountLabel, accountType);
|
|
321
|
+
this.markMutation();
|
|
322
|
+
}
|
|
323
|
+
recordAttemptError(accountLabel, accountType, status, rateLimitKind) {
|
|
324
|
+
const failedAt = this.now();
|
|
325
|
+
this.applyAttemptError(this.stats, accountLabel, accountType, status, rateLimitKind, failedAt);
|
|
326
|
+
this.applyAttemptError(this.pending, accountLabel, accountType, status, rateLimitKind, failedAt);
|
|
327
|
+
this.markMutation();
|
|
328
|
+
}
|
|
329
|
+
recordFinalError(_status, accountLabel, accountType) {
|
|
330
|
+
const failedAt = this.now();
|
|
331
|
+
this.applyFinalError(this.stats, accountLabel, accountType, failedAt);
|
|
332
|
+
this.applyFinalError(this.pending, accountLabel, accountType, failedAt);
|
|
333
|
+
this.markMutation();
|
|
334
|
+
}
|
|
335
|
+
getStats() {
|
|
336
|
+
return cloneStats(this.stats);
|
|
337
|
+
}
|
|
338
|
+
getAccountStats(label) {
|
|
339
|
+
const account = this.stats.accounts[label];
|
|
340
|
+
return account ? cloneAccount(account) : undefined;
|
|
341
|
+
}
|
|
342
|
+
getPersistenceStatus() {
|
|
343
|
+
return {
|
|
344
|
+
enabled: !!this.filePath,
|
|
345
|
+
filePath: this.filePath ?? null,
|
|
346
|
+
revision: this.revision,
|
|
347
|
+
pendingMutations: this.pendingMutations,
|
|
348
|
+
inFlightMutations: this.inFlightMutations,
|
|
349
|
+
unpersistedMutations: this.pendingMutations + this.inFlightMutations,
|
|
350
|
+
lastFlushedAt: this.lastFlushedAt ?? null,
|
|
351
|
+
lastReconciledAt: this.lastReconciledAt ?? null,
|
|
352
|
+
lastRecoveryAt: this.lastRecoveryAt ?? null,
|
|
353
|
+
lastError: this.lastError ?? null,
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
async flush() {
|
|
357
|
+
if (!this.filePath ||
|
|
358
|
+
(this.pendingMutations === 0 && this.inFlightMutations === 0)) {
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
this.cancelFlushTimer();
|
|
362
|
+
await this.flushMutex.runExclusive(async () => {
|
|
363
|
+
if (!this.filePath || this.pendingMutations === 0) {
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
const delta = this.pending;
|
|
367
|
+
const deltaMutations = this.pendingMutations;
|
|
368
|
+
this.pending = emptyStats(this.now());
|
|
369
|
+
this.pendingMutations = 0;
|
|
370
|
+
this.inFlightMutations = deltaMutations;
|
|
371
|
+
const lockPath = `${this.filePath}.lock`;
|
|
372
|
+
let releaseLock;
|
|
373
|
+
try {
|
|
374
|
+
releaseLock = await acquireFileLock(lockPath, this.lockTimeoutMs, this.staleLockMs, this.now);
|
|
375
|
+
let existing;
|
|
376
|
+
try {
|
|
377
|
+
existing = await this.readSnapshot();
|
|
378
|
+
}
|
|
379
|
+
catch (error) {
|
|
380
|
+
if (!isCorruptSnapshotError(error)) {
|
|
381
|
+
throw error;
|
|
382
|
+
}
|
|
383
|
+
await this.quarantineCorruptSnapshot();
|
|
384
|
+
existing = null;
|
|
385
|
+
}
|
|
386
|
+
const persisted = existing?.stats ?? emptyStats(delta.startedAt);
|
|
387
|
+
const merged = mergeStats(persisted, delta);
|
|
388
|
+
const revision = (existing?.revision ?? 0) + 1;
|
|
389
|
+
const updatedAt = this.now();
|
|
390
|
+
await writeJsonSnapshotAtomically(this.filePath, {
|
|
391
|
+
schemaVersion: SNAPSHOT_SCHEMA_VERSION,
|
|
392
|
+
revision,
|
|
393
|
+
updatedAt,
|
|
394
|
+
stats: merged,
|
|
395
|
+
}, 0o600);
|
|
396
|
+
this.revision = revision;
|
|
397
|
+
this.lastFlushedAt = updatedAt;
|
|
398
|
+
this.lastReconciledAt = updatedAt;
|
|
399
|
+
this.lastError = undefined;
|
|
400
|
+
this.stats = mergeStats(merged, this.pending);
|
|
401
|
+
this.inFlightMutations = 0;
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
this.pending = mergeStats(delta, this.pending);
|
|
405
|
+
this.pendingMutations += deltaMutations;
|
|
406
|
+
this.inFlightMutations = 0;
|
|
407
|
+
this.lastError = this.describeError(error);
|
|
408
|
+
this.scheduleFlush();
|
|
409
|
+
}
|
|
410
|
+
finally {
|
|
411
|
+
await releaseLock?.();
|
|
412
|
+
}
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
async reconcile() {
|
|
416
|
+
if (!this.filePath) {
|
|
417
|
+
return this.getStats();
|
|
418
|
+
}
|
|
419
|
+
return this.flushMutex.runExclusive(async () => {
|
|
420
|
+
try {
|
|
421
|
+
const snapshot = await this.readSnapshot();
|
|
422
|
+
if (snapshot) {
|
|
423
|
+
this.stats = mergeStats(snapshot.stats, this.pending);
|
|
424
|
+
this.revision = snapshot.revision;
|
|
425
|
+
this.lastFlushedAt = snapshot.updatedAt;
|
|
426
|
+
}
|
|
427
|
+
this.lastReconciledAt = this.now();
|
|
428
|
+
this.lastError = undefined;
|
|
429
|
+
}
|
|
430
|
+
catch (error) {
|
|
431
|
+
this.lastError = this.describeError(error);
|
|
432
|
+
}
|
|
433
|
+
return this.getStats();
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
resetMemory() {
|
|
437
|
+
this.cancelFlushTimer();
|
|
438
|
+
const startedAt = this.now();
|
|
439
|
+
this.stats = emptyStats(startedAt);
|
|
440
|
+
this.pending = emptyStats(startedAt);
|
|
441
|
+
this.pendingMutations = 0;
|
|
442
|
+
this.inFlightMutations = 0;
|
|
443
|
+
this.revision = 0;
|
|
444
|
+
this.lastFlushedAt = undefined;
|
|
445
|
+
this.lastReconciledAt = undefined;
|
|
446
|
+
this.lastRecoveryAt = undefined;
|
|
447
|
+
this.lastError = undefined;
|
|
448
|
+
}
|
|
449
|
+
async resetForTests() {
|
|
450
|
+
await this.flushMutex.runExclusive(async () => {
|
|
451
|
+
this.resetMemory();
|
|
452
|
+
this.filePath = undefined;
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
markMutation() {
|
|
456
|
+
this.pendingMutations += 1;
|
|
457
|
+
this.scheduleFlush();
|
|
458
|
+
}
|
|
459
|
+
applyAttempt(target, accountLabel, accountType, attemptedAt) {
|
|
460
|
+
target.totalAttempts += 1;
|
|
461
|
+
const account = this.ensureAccount(target, accountLabel, accountType);
|
|
462
|
+
account.attemptCount += 1;
|
|
463
|
+
account.lastAttemptAt = attemptedAt;
|
|
464
|
+
}
|
|
465
|
+
applyFinalSuccess(target, accountLabel, accountType) {
|
|
466
|
+
target.totalRequests += 1;
|
|
467
|
+
target.totalSuccess += 1;
|
|
468
|
+
if (accountLabel && accountType) {
|
|
469
|
+
this.ensureAccount(target, accountLabel, accountType).successCount += 1;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
applyAttemptError(target, accountLabel, accountType, status, rateLimitKind, failedAt) {
|
|
473
|
+
const account = this.ensureAccount(target, accountLabel, accountType);
|
|
474
|
+
target.totalAttemptErrors += 1;
|
|
475
|
+
account.attemptErrorCount += 1;
|
|
476
|
+
account.lastErrorAt = failedAt;
|
|
477
|
+
if (status !== 429) {
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
target.totalRateLimits += 1;
|
|
481
|
+
account.rateLimitCount += 1;
|
|
40
482
|
if (rateLimitKind === "transient") {
|
|
41
|
-
|
|
42
|
-
|
|
483
|
+
target.totalTransientRateLimits += 1;
|
|
484
|
+
account.transientRateLimitCount += 1;
|
|
43
485
|
}
|
|
44
486
|
else if (rateLimitKind === "quota") {
|
|
45
|
-
|
|
46
|
-
|
|
487
|
+
target.totalQuotaRateLimits += 1;
|
|
488
|
+
account.quotaRateLimitCount += 1;
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
applyFinalError(target, accountLabel, accountType, failedAt) {
|
|
492
|
+
target.totalRequests += 1;
|
|
493
|
+
target.totalErrors += 1;
|
|
494
|
+
if (accountLabel && accountType) {
|
|
495
|
+
const account = this.ensureAccount(target, accountLabel, accountType);
|
|
496
|
+
account.errorCount += 1;
|
|
497
|
+
account.lastErrorAt = failedAt;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
ensureAccount(target, label, type) {
|
|
501
|
+
if (!target.accounts[label]) {
|
|
502
|
+
target.accounts[label] = {
|
|
503
|
+
label,
|
|
504
|
+
type,
|
|
505
|
+
attemptCount: 0,
|
|
506
|
+
attemptErrorCount: 0,
|
|
507
|
+
successCount: 0,
|
|
508
|
+
errorCount: 0,
|
|
509
|
+
rateLimitCount: 0,
|
|
510
|
+
transientRateLimitCount: 0,
|
|
511
|
+
quotaRateLimitCount: 0,
|
|
512
|
+
lastAttemptAt: 0,
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
return target.accounts[label];
|
|
516
|
+
}
|
|
517
|
+
scheduleFlush() {
|
|
518
|
+
if (!this.filePath || this.flushTimer) {
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
this.flushTimer = setTimeout(() => {
|
|
522
|
+
this.flushTimer = null;
|
|
523
|
+
void this.flush().catch((error) => {
|
|
524
|
+
this.lastError = this.describeError(error);
|
|
525
|
+
this.scheduleFlush();
|
|
526
|
+
});
|
|
527
|
+
}, this.flushIntervalMs);
|
|
528
|
+
this.flushTimer.unref?.();
|
|
529
|
+
}
|
|
530
|
+
cancelFlushTimer() {
|
|
531
|
+
if (this.flushTimer) {
|
|
532
|
+
clearTimeout(this.flushTimer);
|
|
533
|
+
this.flushTimer = null;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
async readSnapshot() {
|
|
537
|
+
if (!this.filePath) {
|
|
538
|
+
return null;
|
|
539
|
+
}
|
|
540
|
+
let raw;
|
|
541
|
+
try {
|
|
542
|
+
raw = await readFile(this.filePath, "utf8");
|
|
543
|
+
}
|
|
544
|
+
catch (error) {
|
|
545
|
+
if (isMissingFileError(error)) {
|
|
546
|
+
return null;
|
|
547
|
+
}
|
|
548
|
+
throw error;
|
|
549
|
+
}
|
|
550
|
+
const parsed = JSON.parse(raw);
|
|
551
|
+
if (!validSnapshot(parsed)) {
|
|
552
|
+
throw new InvalidProxyStatsSnapshotError(`Invalid proxy stats snapshot ${this.filePath}`);
|
|
553
|
+
}
|
|
554
|
+
return parsed;
|
|
555
|
+
}
|
|
556
|
+
applySnapshot(snapshot) {
|
|
557
|
+
this.stats = cloneStats(snapshot.stats);
|
|
558
|
+
this.pending = emptyStats(this.now());
|
|
559
|
+
this.revision = snapshot.revision;
|
|
560
|
+
this.lastFlushedAt = snapshot.updatedAt;
|
|
561
|
+
}
|
|
562
|
+
async recoverCorruptSnapshot() {
|
|
563
|
+
if (!this.filePath) {
|
|
564
|
+
return null;
|
|
565
|
+
}
|
|
566
|
+
const releaseLock = await acquireFileLock(`${this.filePath}.lock`, this.lockTimeoutMs, this.staleLockMs, this.now);
|
|
567
|
+
try {
|
|
568
|
+
try {
|
|
569
|
+
return await this.readSnapshot();
|
|
570
|
+
}
|
|
571
|
+
catch (error) {
|
|
572
|
+
if (!isCorruptSnapshotError(error)) {
|
|
573
|
+
throw error;
|
|
574
|
+
}
|
|
575
|
+
await this.quarantineCorruptSnapshot();
|
|
576
|
+
return null;
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
finally {
|
|
580
|
+
await releaseLock();
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
async quarantineCorruptSnapshot() {
|
|
584
|
+
if (!this.filePath) {
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
const recoveredAt = this.now();
|
|
588
|
+
const quarantinePath = `${this.filePath}.corrupt.${recoveredAt}.${randomUUID()}`;
|
|
589
|
+
try {
|
|
590
|
+
await rename(this.filePath, quarantinePath);
|
|
591
|
+
}
|
|
592
|
+
catch (error) {
|
|
593
|
+
if (!isMissingFileError(error)) {
|
|
594
|
+
throw error;
|
|
595
|
+
}
|
|
596
|
+
return;
|
|
47
597
|
}
|
|
598
|
+
this.lastRecoveryAt = recoveredAt;
|
|
599
|
+
this.lastError = undefined;
|
|
600
|
+
await this.pruneCorruptSnapshots();
|
|
48
601
|
}
|
|
602
|
+
async pruneCorruptSnapshots() {
|
|
603
|
+
if (!this.filePath) {
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
const directory = dirname(this.filePath);
|
|
607
|
+
const prefix = `${basename(this.filePath)}.corrupt.`;
|
|
608
|
+
const quarantined = (await readdir(directory))
|
|
609
|
+
.filter((entry) => entry.startsWith(prefix))
|
|
610
|
+
.sort()
|
|
611
|
+
.reverse();
|
|
612
|
+
await Promise.all(quarantined
|
|
613
|
+
.slice(MAX_CORRUPT_SNAPSHOTS)
|
|
614
|
+
.map((entry) => rm(join(directory, entry), { force: true })));
|
|
615
|
+
}
|
|
616
|
+
describeError(error) {
|
|
617
|
+
return (error instanceof Error ? error.message : String(error)).slice(0, 1_000);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
const defaultStore = new ProxyUsageStatsStore();
|
|
621
|
+
export async function initUsageStats(filePath) {
|
|
622
|
+
await defaultStore.initialize(filePath);
|
|
623
|
+
}
|
|
624
|
+
export function recordAttempt(accountLabel, accountType) {
|
|
625
|
+
defaultStore.recordAttempt(accountLabel, accountType);
|
|
626
|
+
}
|
|
627
|
+
export function recordFinalSuccess(accountLabel, accountType) {
|
|
628
|
+
defaultStore.recordFinalSuccess(accountLabel, accountType);
|
|
629
|
+
}
|
|
630
|
+
export function recordAttemptError(accountLabel, accountType, status, rateLimitKind) {
|
|
631
|
+
defaultStore.recordAttemptError(accountLabel, accountType, status, rateLimitKind);
|
|
49
632
|
}
|
|
50
633
|
export function recordFinalError(_status, accountLabel, accountType) {
|
|
51
|
-
|
|
52
|
-
stats.totalErrors++;
|
|
53
|
-
if (accountLabel && accountType) {
|
|
54
|
-
const acct = ensureAccount(accountLabel, accountType);
|
|
55
|
-
acct.errorCount++;
|
|
56
|
-
acct.lastErrorAt = Date.now();
|
|
57
|
-
}
|
|
634
|
+
defaultStore.recordFinalError(_status, accountLabel, accountType);
|
|
58
635
|
}
|
|
59
636
|
export function getStats() {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
return { ...stats, accounts };
|
|
637
|
+
return defaultStore.getStats();
|
|
638
|
+
}
|
|
639
|
+
export async function getReconciledStats() {
|
|
640
|
+
return defaultStore.reconcile();
|
|
65
641
|
}
|
|
66
642
|
export function getAccountStats(label) {
|
|
67
|
-
|
|
68
|
-
return account ? { ...account } : undefined;
|
|
643
|
+
return defaultStore.getAccountStats(label);
|
|
69
644
|
}
|
|
645
|
+
export function getUsageStatsPersistenceStatus() {
|
|
646
|
+
return defaultStore.getPersistenceStatus();
|
|
647
|
+
}
|
|
648
|
+
export async function flushUsageStats() {
|
|
649
|
+
await defaultStore.flush();
|
|
650
|
+
}
|
|
651
|
+
/** Reset process-local counters while intentionally preserving durable state. */
|
|
70
652
|
export function resetStats() {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
stats.totalErrors = 0;
|
|
77
|
-
stats.totalRateLimits = 0;
|
|
78
|
-
stats.totalTransientRateLimits = 0;
|
|
79
|
-
stats.totalQuotaRateLimits = 0;
|
|
80
|
-
stats.accounts = {};
|
|
81
|
-
}
|
|
82
|
-
function ensureAccount(label, type) {
|
|
83
|
-
if (!stats.accounts[label]) {
|
|
84
|
-
stats.accounts[label] = {
|
|
85
|
-
label,
|
|
86
|
-
type,
|
|
87
|
-
attemptCount: 0,
|
|
88
|
-
attemptErrorCount: 0,
|
|
89
|
-
successCount: 0,
|
|
90
|
-
errorCount: 0,
|
|
91
|
-
rateLimitCount: 0,
|
|
92
|
-
transientRateLimitCount: 0,
|
|
93
|
-
quotaRateLimitCount: 0,
|
|
94
|
-
lastAttemptAt: 0,
|
|
95
|
-
};
|
|
96
|
-
}
|
|
97
|
-
return stats.accounts[label];
|
|
653
|
+
defaultStore.resetMemory();
|
|
654
|
+
}
|
|
655
|
+
/** Disconnect persistence and clear singleton state between isolated tests. */
|
|
656
|
+
export async function resetUsageStatsForTests() {
|
|
657
|
+
await defaultStore.resetForTests();
|
|
98
658
|
}
|