@juspay/neurolink 10.4.2 → 10.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/auth/tokenStore.d.ts +7 -0
- package/dist/auth/tokenStore.js +54 -0
- package/dist/browser/neurolink.min.js +377 -380
- package/dist/cli/commands/proxy.d.ts +2 -1
- package/dist/cli/commands/proxy.js +290 -63
- package/dist/cli/commands/proxyAnalyze.js +10 -1
- package/dist/lib/auth/tokenStore.d.ts +7 -0
- package/dist/lib/auth/tokenStore.js +54 -0
- package/dist/lib/proxy/proxyAnalysis.js +136 -12
- package/dist/lib/proxy/proxyTranslationEngine.d.ts +1 -0
- package/dist/lib/proxy/proxyTranslationEngine.js +56 -12
- package/dist/lib/proxy/rawStreamCapture.js +47 -1
- package/dist/lib/proxy/requestLogger.d.ts +2 -0
- package/dist/lib/proxy/requestLogger.js +72 -13
- package/dist/lib/proxy/rollingProxyServer.js +79 -8
- package/dist/lib/proxy/rollingWorkerProcess.js +20 -15
- package/dist/lib/proxy/rollingWorkerProtocol.js +3 -0
- package/dist/lib/proxy/rollingWorkerSupervisor.d.ts +1 -0
- package/dist/lib/proxy/rollingWorkerSupervisor.js +31 -11
- package/dist/lib/proxy/runtimeConfig.d.ts +1 -0
- package/dist/lib/proxy/runtimeConfig.js +20 -5
- package/dist/lib/proxy/socketWorkerRuntime.js +41 -13
- package/dist/lib/proxy/sseInterceptor.js +47 -1
- package/dist/lib/proxy/tokenRefresh.d.ts +6 -0
- package/dist/lib/proxy/tokenRefresh.js +70 -15
- package/dist/lib/proxy/usageStats.d.ts +25 -3
- package/dist/lib/proxy/usageStats.js +546 -55
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +2 -0
- package/dist/lib/server/routes/claudeProxyRoutes.js +261 -297
- package/dist/lib/types/proxy.d.ts +90 -1
- package/dist/proxy/proxyAnalysis.js +136 -12
- package/dist/proxy/proxyTranslationEngine.d.ts +1 -0
- package/dist/proxy/proxyTranslationEngine.js +56 -12
- package/dist/proxy/rawStreamCapture.js +47 -1
- package/dist/proxy/requestLogger.d.ts +2 -0
- package/dist/proxy/requestLogger.js +72 -13
- package/dist/proxy/rollingProxyServer.js +79 -8
- package/dist/proxy/rollingWorkerProcess.js +20 -15
- package/dist/proxy/rollingWorkerProtocol.js +3 -0
- package/dist/proxy/rollingWorkerSupervisor.d.ts +1 -0
- package/dist/proxy/rollingWorkerSupervisor.js +31 -11
- package/dist/proxy/runtimeConfig.d.ts +1 -0
- package/dist/proxy/runtimeConfig.js +20 -5
- package/dist/proxy/socketWorkerRuntime.js +41 -13
- package/dist/proxy/sseInterceptor.js +47 -1
- package/dist/proxy/tokenRefresh.d.ts +6 -0
- package/dist/proxy/tokenRefresh.js +70 -15
- package/dist/proxy/usageStats.d.ts +25 -3
- package/dist/proxy/usageStats.js +546 -55
- package/dist/server/routes/claudeProxyRoutes.d.ts +2 -0
- package/dist/server/routes/claudeProxyRoutes.js +261 -297
- package/dist/types/proxy.d.ts +90 -1
- package/package.json +1 -1
|
@@ -22,7 +22,7 @@ import { ProxyRuntimeConfigStore } from "../../lib/proxy/runtimeConfig.js";
|
|
|
22
22
|
* string) recorded when the supervisor we expect at `pid` was launched.
|
|
23
23
|
* When provided, this is cross-checked against `pid`'s actual OS-reported
|
|
24
24
|
* start time (see {@link getProcessStartTime}) — the args match alone
|
|
25
|
-
* (
|
|
25
|
+
* (the `neurolink proxy start` command is present) is not airtight, since an
|
|
26
26
|
* unrelated process (e.g. a shell running this very test suite, or a
|
|
27
27
|
* coincidentally-named script) could match it too. A pid recycled by such
|
|
28
28
|
* a process would have a start time far from the recorded supervisor
|
|
@@ -57,6 +57,7 @@ export declare function createProxyStartApp(params: {
|
|
|
57
57
|
readiness: ProxyReadinessState;
|
|
58
58
|
}>;
|
|
59
59
|
export declare const proxyStartCommand: CommandModule<object, ProxyStartArgs>;
|
|
60
|
+
export declare function sanitizeProxyStatusTerminalErrorMessage(message: string): string;
|
|
60
61
|
export declare const proxyStatusCommand: CommandModule<object, ProxyStatusArgs>;
|
|
61
62
|
export declare const proxyTelemetryCommand: CommandModule<object, ProxyTelemetryArgs>;
|
|
62
63
|
export declare const proxyGuardCommand: CommandModule<object, ProxyGuardArgs>;
|
|
@@ -12,11 +12,12 @@
|
|
|
12
12
|
import { spawn } from "node:child_process";
|
|
13
13
|
import { homedir } from "node:os";
|
|
14
14
|
import { dirname, join, resolve } from "node:path";
|
|
15
|
+
import { stripVTControlCharacters } from "node:util";
|
|
15
16
|
import chalk from "chalk";
|
|
16
17
|
import ora from "ora";
|
|
17
18
|
import { buildProxyHealthResponse, createProxyReadinessState, markProxyDrainingForUpdate, markProxyReady, resumeProxyConnections, waitForProxyReadiness, } from "../../lib/proxy/proxyHealth.js";
|
|
18
19
|
import { logger } from "../../lib/utils/logger.js";
|
|
19
|
-
import { sanitizeForLog } from "../../lib/utils/logSanitize.js";
|
|
20
|
+
import { redactUrlsInText, sanitizeForLog, } from "../../lib/utils/logSanitize.js";
|
|
20
21
|
import { withTimeout } from "../../lib/utils/async/withTimeout.js";
|
|
21
22
|
import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
|
|
22
23
|
import { configureProxyKeepAliveDispatcher } from "../../lib/proxy/proxyDispatcher.js";
|
|
@@ -39,9 +40,11 @@ import { fileURLToPath } from "node:url";
|
|
|
39
40
|
import packageJson from "../../../package.json" with { type: "json" };
|
|
40
41
|
const _require = createRequire(import.meta.url);
|
|
41
42
|
const PROXY_VERSION = packageJson.version;
|
|
43
|
+
const PROXY_INTERNAL_ACCOUNT_LABEL = "proxy/internal";
|
|
42
44
|
const PROXY_TELEMETRY_SCRIPT_PATH = fileURLToPath(new URL("../../../scripts/observability/manage-local-openobserve.sh", import.meta.url));
|
|
43
45
|
const PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS = 5_000;
|
|
44
46
|
const LEGACY_STATUS_ACCOUNT_CACHE_TTL_MS = 5_000;
|
|
47
|
+
const PROXY_STATUS_TOKEN_READ_TIMEOUT_MS = 2_000;
|
|
45
48
|
let legacyStatusAccountCache;
|
|
46
49
|
// Allowed drift between a pid's OS-reported start time and the persisted
|
|
47
50
|
// ProxySupervisorState.startTime before processLooksLikeProxySupervisor
|
|
@@ -187,7 +190,7 @@ async function getProcessStartTime(pid) {
|
|
|
187
190
|
* string) recorded when the supervisor we expect at `pid` was launched.
|
|
188
191
|
* When provided, this is cross-checked against `pid`'s actual OS-reported
|
|
189
192
|
* start time (see {@link getProcessStartTime}) — the args match alone
|
|
190
|
-
* (
|
|
193
|
+
* (the `neurolink proxy start` command is present) is not airtight, since an
|
|
191
194
|
* unrelated process (e.g. a shell running this very test suite, or a
|
|
192
195
|
* coincidentally-named script) could match it too. A pid recycled by such
|
|
193
196
|
* a process would have a start time far from the recorded supervisor
|
|
@@ -215,9 +218,9 @@ export async function processLooksLikeProxySupervisor(pid, expectedStartTimeIso)
|
|
|
215
218
|
// "neurolink", an editor with a neurolink file open) would match, and a
|
|
216
219
|
// stale/recycled pid running such a process could then be SIGTERM'd/
|
|
217
220
|
// SIGKILL'd by `ensureSupervisorStoppedBeforeClear` during uninstall.
|
|
218
|
-
// Require the actual proxy-supervisor invocation
|
|
219
|
-
//
|
|
220
|
-
if (
|
|
221
|
+
// Require the actual proxy-supervisor invocation. Commands such as
|
|
222
|
+
// `neurolink proxy status` must never be mistaken for the listener owner.
|
|
223
|
+
if (!/\bneurolink(?:-proxy)?\b.*\bproxy\s+start\b/i.test(args)) {
|
|
221
224
|
return false;
|
|
222
225
|
}
|
|
223
226
|
if (!expectedStartTimeIso) {
|
|
@@ -353,6 +356,15 @@ async function resolveLegacyStatusAccountLabel(storedAnthropicAccountCount) {
|
|
|
353
356
|
};
|
|
354
357
|
return label;
|
|
355
358
|
}
|
|
359
|
+
function deriveAccountAllowance(accountKey, now, allowlist, expirations, cooldowns) {
|
|
360
|
+
const allowed = isAccountAllowed(accountKey, allowlist);
|
|
361
|
+
const expiresAt = expirations.get(accountKey);
|
|
362
|
+
return {
|
|
363
|
+
allowed,
|
|
364
|
+
expired: expiresAt !== undefined && expiresAt <= now,
|
|
365
|
+
cooling: (cooldowns[accountKey]?.coolingUntil ?? 0) > now,
|
|
366
|
+
};
|
|
367
|
+
}
|
|
356
368
|
/**
|
|
357
369
|
* Check if the launchd service is loaded and actively managing the proxy.
|
|
358
370
|
* Returns true if launchctl reports the service as running.
|
|
@@ -1248,7 +1260,13 @@ export async function createProxyStartApp(params) {
|
|
|
1248
1260
|
const recordRuntimeError = async (metadata, status, errorType, errorMessage, options) => {
|
|
1249
1261
|
const clientMessage = options?.clientMessage ?? errorMessage;
|
|
1250
1262
|
const clientErrorType = options?.clientErrorType ?? errorType;
|
|
1251
|
-
recordFinalError(status
|
|
1263
|
+
recordFinalError(status, undefined, undefined, {
|
|
1264
|
+
requestId: metadata.requestId,
|
|
1265
|
+
errorType,
|
|
1266
|
+
errorCode: options?.errorCode,
|
|
1267
|
+
terminalOutcome: "handler_error",
|
|
1268
|
+
message: errorMessage,
|
|
1269
|
+
});
|
|
1252
1270
|
await Promise.all([
|
|
1253
1271
|
logRequest({
|
|
1254
1272
|
timestamp: new Date().toISOString(),
|
|
@@ -1522,21 +1540,45 @@ export async function createProxyStartApp(params) {
|
|
|
1522
1540
|
const activeAccountAllowlist = runtimeConfig
|
|
1523
1541
|
? runtimeConfig.accountAllowlist
|
|
1524
1542
|
: params.accountAllowlist;
|
|
1525
|
-
const {
|
|
1543
|
+
const { getReconciledUsageSnapshot, getUsageStatsPersistenceStatus } = await import("../../lib/proxy/usageStats.js");
|
|
1526
1544
|
const { loadAccountCooldowns } = await import("../../lib/proxy/accountCooldown.js");
|
|
1527
|
-
const
|
|
1545
|
+
const usageSnapshot = await getReconciledUsageSnapshot();
|
|
1546
|
+
const { stats, terminalErrors } = usageSnapshot;
|
|
1547
|
+
const terminalErrorDetailsComparable = usageSnapshot.statsVersion === usageSnapshot.terminalErrorsVersion;
|
|
1548
|
+
const lastTerminalError = terminalErrors.recent.at(-1) ?? null;
|
|
1549
|
+
const terminalErrorDetailsMissing = terminalErrorDetailsComparable
|
|
1550
|
+
? Math.max(0, stats.totalErrors - terminalErrors.totalErrors)
|
|
1551
|
+
: undefined;
|
|
1552
|
+
const terminalErrorDetailsExcess = terminalErrorDetailsComparable
|
|
1553
|
+
? Math.max(0, terminalErrors.totalErrors - stats.totalErrors)
|
|
1554
|
+
: undefined;
|
|
1528
1555
|
const runtimeState = loadProxyState();
|
|
1529
1556
|
const supervisorState = loadProxySupervisorState();
|
|
1530
1557
|
const updateState = loadUpdateState();
|
|
1531
1558
|
const cooldowns = await loadAccountCooldowns();
|
|
1532
1559
|
const storedAccountKeys = new Set();
|
|
1560
|
+
const storedAccountExpirations = new Map();
|
|
1533
1561
|
const disabledAccountKeys = new Set();
|
|
1534
1562
|
let accountInventoryLoaded = false;
|
|
1535
1563
|
try {
|
|
1536
1564
|
const { tokenStore } = await import("../../lib/auth/tokenStore.js");
|
|
1537
|
-
|
|
1538
|
-
|
|
1565
|
+
const storedKeys = await tokenStore.listByPrefix("anthropic:");
|
|
1566
|
+
for (const key of storedKeys) {
|
|
1567
|
+
const normalizedKey = normalizeAnthropicAccountKey(key);
|
|
1568
|
+
storedAccountKeys.add(normalizedKey);
|
|
1539
1569
|
}
|
|
1570
|
+
await Promise.all(storedKeys.map(async (key) => {
|
|
1571
|
+
const normalizedKey = normalizeAnthropicAccountKey(key);
|
|
1572
|
+
try {
|
|
1573
|
+
const tokens = await withTimeout(tokenStore.peekTokens(key), PROXY_STATUS_TOKEN_READ_TIMEOUT_MS, "[proxy] /status token inspection timed out");
|
|
1574
|
+
if (tokens) {
|
|
1575
|
+
storedAccountExpirations.set(normalizedKey, tokens.expiresAt);
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
catch (err) {
|
|
1579
|
+
logger.debug(`[proxy] /status: failed to inspect token metadata for ${normalizedKey}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1580
|
+
}
|
|
1581
|
+
}));
|
|
1540
1582
|
for (const key of await tokenStore.listDisabled()) {
|
|
1541
1583
|
disabledAccountKeys.add(normalizeAnthropicAccountKey(key));
|
|
1542
1584
|
}
|
|
@@ -1556,6 +1598,141 @@ export async function createProxyStartApp(params) {
|
|
|
1556
1598
|
});
|
|
1557
1599
|
const primaryAccount = await resolveStatusPrimaryAccount(activeProxyConfig);
|
|
1558
1600
|
const activeUpdaterPid = supervisorState?.updaterPid ?? runtimeState?.updaterPid;
|
|
1601
|
+
const accountRows = Object.values(stats.accounts).map((account) => {
|
|
1602
|
+
const normalizedKey = normalizeAnthropicAccountKey(account.label);
|
|
1603
|
+
const isLegacyAccount = account.type === "oauth" && account.label === legacyAccountLabel;
|
|
1604
|
+
const accountKey = storedAccountKeys.has(normalizedKey)
|
|
1605
|
+
? normalizedKey
|
|
1606
|
+
: isLegacyAccount
|
|
1607
|
+
? LEGACY_ANTHROPIC_ACCOUNT_KEY
|
|
1608
|
+
: account.label === "env"
|
|
1609
|
+
? ENV_ANTHROPIC_ACCOUNT_KEY
|
|
1610
|
+
: normalizedKey;
|
|
1611
|
+
const isStored = storedAccountKeys.has(accountKey) || isLegacyAccount;
|
|
1612
|
+
const { allowed, expired, cooling } = deriveAccountAllowance(accountKey, now, activeAccountAllowlist, storedAccountExpirations, cooldowns);
|
|
1613
|
+
const accountStatus = account.type === "internal"
|
|
1614
|
+
? "internal"
|
|
1615
|
+
: disabledAccountKeys.has(accountKey)
|
|
1616
|
+
? "disabled"
|
|
1617
|
+
: expired
|
|
1618
|
+
? "expired"
|
|
1619
|
+
: !allowed
|
|
1620
|
+
? "excluded"
|
|
1621
|
+
: accountInventoryLoaded &&
|
|
1622
|
+
account.type === "oauth" &&
|
|
1623
|
+
!isStored
|
|
1624
|
+
? "removed"
|
|
1625
|
+
: cooling
|
|
1626
|
+
? "cooling"
|
|
1627
|
+
: "active";
|
|
1628
|
+
return {
|
|
1629
|
+
label: account.label,
|
|
1630
|
+
type: account.type,
|
|
1631
|
+
attempts: account.attemptCount,
|
|
1632
|
+
requests: account.successCount + account.errorCount,
|
|
1633
|
+
success: account.successCount,
|
|
1634
|
+
errors: account.errorCount,
|
|
1635
|
+
attemptErrors: account.attemptErrorCount,
|
|
1636
|
+
rateLimits: account.rateLimitCount,
|
|
1637
|
+
transientRateLimits: account.transientRateLimitCount,
|
|
1638
|
+
quotaRateLimits: account.quotaRateLimitCount,
|
|
1639
|
+
cooling,
|
|
1640
|
+
status: accountStatus,
|
|
1641
|
+
allowed: account.type === "internal" ? undefined : allowed,
|
|
1642
|
+
expired: account.type === "oauth" ? expired : undefined,
|
|
1643
|
+
};
|
|
1644
|
+
});
|
|
1645
|
+
const representedAccountKeys = new Set(accountRows
|
|
1646
|
+
.filter((account) => account.type === "oauth")
|
|
1647
|
+
.map((account) => normalizeAnthropicAccountKey(account.label)));
|
|
1648
|
+
for (const accountKey of storedAccountKeys) {
|
|
1649
|
+
if (representedAccountKeys.has(accountKey)) {
|
|
1650
|
+
continue;
|
|
1651
|
+
}
|
|
1652
|
+
const { allowed, expired, cooling } = deriveAccountAllowance(accountKey, now, activeAccountAllowlist, storedAccountExpirations, cooldowns);
|
|
1653
|
+
accountRows.push({
|
|
1654
|
+
label: accountKey.slice("anthropic:".length),
|
|
1655
|
+
type: "oauth",
|
|
1656
|
+
attempts: 0,
|
|
1657
|
+
requests: 0,
|
|
1658
|
+
success: 0,
|
|
1659
|
+
errors: 0,
|
|
1660
|
+
attemptErrors: 0,
|
|
1661
|
+
rateLimits: 0,
|
|
1662
|
+
transientRateLimits: 0,
|
|
1663
|
+
quotaRateLimits: 0,
|
|
1664
|
+
cooling,
|
|
1665
|
+
status: disabledAccountKeys.has(accountKey)
|
|
1666
|
+
? "disabled"
|
|
1667
|
+
: expired
|
|
1668
|
+
? "expired"
|
|
1669
|
+
: !allowed
|
|
1670
|
+
? "excluded"
|
|
1671
|
+
: cooling
|
|
1672
|
+
? "cooling"
|
|
1673
|
+
: "active",
|
|
1674
|
+
allowed,
|
|
1675
|
+
expired,
|
|
1676
|
+
});
|
|
1677
|
+
}
|
|
1678
|
+
const attributed = accountRows.reduce((total, account) => ({
|
|
1679
|
+
attempts: total.attempts + (account.attempts ?? 0),
|
|
1680
|
+
requests: total.requests + (account.requests ?? 0),
|
|
1681
|
+
success: total.success + (account.success ?? 0),
|
|
1682
|
+
errors: total.errors + (account.errors ?? 0),
|
|
1683
|
+
attemptErrors: total.attemptErrors + (account.attemptErrors ?? 0),
|
|
1684
|
+
rateLimits: total.rateLimits + (account.rateLimits ?? 0),
|
|
1685
|
+
transientRateLimits: total.transientRateLimits + (account.transientRateLimits ?? 0),
|
|
1686
|
+
quotaRateLimits: total.quotaRateLimits + (account.quotaRateLimits ?? 0),
|
|
1687
|
+
}), {
|
|
1688
|
+
attempts: 0,
|
|
1689
|
+
requests: 0,
|
|
1690
|
+
success: 0,
|
|
1691
|
+
errors: 0,
|
|
1692
|
+
attemptErrors: 0,
|
|
1693
|
+
rateLimits: 0,
|
|
1694
|
+
transientRateLimits: 0,
|
|
1695
|
+
quotaRateLimits: 0,
|
|
1696
|
+
});
|
|
1697
|
+
const unattributed = {
|
|
1698
|
+
attempts: Math.max(0, stats.totalAttempts - attributed.attempts),
|
|
1699
|
+
requests: Math.max(0, stats.totalRequests - attributed.requests),
|
|
1700
|
+
success: Math.max(0, stats.totalSuccess - attributed.success),
|
|
1701
|
+
errors: Math.max(0, stats.totalErrors - attributed.errors),
|
|
1702
|
+
attemptErrors: Math.max(0, stats.totalAttemptErrors - attributed.attemptErrors),
|
|
1703
|
+
rateLimits: Math.max(0, stats.totalRateLimits - attributed.rateLimits),
|
|
1704
|
+
transientRateLimits: Math.max(0, stats.totalTransientRateLimits - attributed.transientRateLimits),
|
|
1705
|
+
quotaRateLimits: Math.max(0, stats.totalQuotaRateLimits - attributed.quotaRateLimits),
|
|
1706
|
+
};
|
|
1707
|
+
if (Object.values(unattributed).some((count) => count > 0)) {
|
|
1708
|
+
const internalRow = accountRows.find((account) => account.label === PROXY_INTERNAL_ACCOUNT_LABEL);
|
|
1709
|
+
if (internalRow) {
|
|
1710
|
+
internalRow.attempts =
|
|
1711
|
+
(internalRow.attempts ?? 0) + unattributed.attempts;
|
|
1712
|
+
internalRow.requests =
|
|
1713
|
+
(internalRow.requests ?? 0) + unattributed.requests;
|
|
1714
|
+
internalRow.success = (internalRow.success ?? 0) + unattributed.success;
|
|
1715
|
+
internalRow.errors = (internalRow.errors ?? 0) + unattributed.errors;
|
|
1716
|
+
internalRow.attemptErrors =
|
|
1717
|
+
(internalRow.attemptErrors ?? 0) + unattributed.attemptErrors;
|
|
1718
|
+
internalRow.rateLimits =
|
|
1719
|
+
(internalRow.rateLimits ?? 0) + unattributed.rateLimits;
|
|
1720
|
+
internalRow.transientRateLimits =
|
|
1721
|
+
(internalRow.transientRateLimits ?? 0) +
|
|
1722
|
+
unattributed.transientRateLimits;
|
|
1723
|
+
internalRow.quotaRateLimits =
|
|
1724
|
+
(internalRow.quotaRateLimits ?? 0) + unattributed.quotaRateLimits;
|
|
1725
|
+
}
|
|
1726
|
+
else {
|
|
1727
|
+
accountRows.push({
|
|
1728
|
+
label: PROXY_INTERNAL_ACCOUNT_LABEL,
|
|
1729
|
+
type: "internal",
|
|
1730
|
+
...unattributed,
|
|
1731
|
+
cooling: false,
|
|
1732
|
+
status: "internal",
|
|
1733
|
+
});
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1559
1736
|
return c.json({
|
|
1560
1737
|
status: "running",
|
|
1561
1738
|
ready: health.ready,
|
|
@@ -1578,42 +1755,12 @@ export async function createProxyStartApp(params) {
|
|
|
1578
1755
|
totalRateLimits: stats.totalRateLimits,
|
|
1579
1756
|
totalTransientRateLimits: stats.totalTransientRateLimits,
|
|
1580
1757
|
totalQuotaRateLimits: stats.totalQuotaRateLimits,
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
? LEGACY_ANTHROPIC_ACCOUNT_KEY
|
|
1588
|
-
: account.label === "env"
|
|
1589
|
-
? ENV_ANTHROPIC_ACCOUNT_KEY
|
|
1590
|
-
: normalizedKey;
|
|
1591
|
-
const isStored = storedAccountKeys.has(accountKey) || isLegacyAccount;
|
|
1592
|
-
const cooling = (cooldowns[accountKey]?.coolingUntil ?? 0) > now;
|
|
1593
|
-
const accountStatus = disabledAccountKeys.has(accountKey)
|
|
1594
|
-
? "disabled"
|
|
1595
|
-
: !isAccountAllowed(accountKey, activeAccountAllowlist)
|
|
1596
|
-
? "excluded"
|
|
1597
|
-
: accountInventoryLoaded && account.type === "oauth" && !isStored
|
|
1598
|
-
? "removed"
|
|
1599
|
-
: cooling
|
|
1600
|
-
? "cooling"
|
|
1601
|
-
: "active";
|
|
1602
|
-
return {
|
|
1603
|
-
label: account.label,
|
|
1604
|
-
type: account.type,
|
|
1605
|
-
attempts: account.attemptCount,
|
|
1606
|
-
requests: account.successCount + account.errorCount,
|
|
1607
|
-
success: account.successCount,
|
|
1608
|
-
errors: account.errorCount,
|
|
1609
|
-
attemptErrors: account.attemptErrorCount,
|
|
1610
|
-
rateLimits: account.rateLimitCount,
|
|
1611
|
-
transientRateLimits: account.transientRateLimitCount,
|
|
1612
|
-
quotaRateLimits: account.quotaRateLimitCount,
|
|
1613
|
-
cooling,
|
|
1614
|
-
status: accountStatus,
|
|
1615
|
-
};
|
|
1616
|
-
}),
|
|
1758
|
+
terminalErrors,
|
|
1759
|
+
lastTerminalError,
|
|
1760
|
+
terminalErrorDetailsComparable,
|
|
1761
|
+
terminalErrorDetailsMissing,
|
|
1762
|
+
terminalErrorDetailsExcess,
|
|
1763
|
+
accounts: accountRows,
|
|
1617
1764
|
primaryAccount,
|
|
1618
1765
|
persistence: getUsageStatsPersistenceStatus(),
|
|
1619
1766
|
},
|
|
@@ -1930,9 +2077,11 @@ function registerProxyShutdownHandlers(params) {
|
|
|
1930
2077
|
}
|
|
1931
2078
|
}
|
|
1932
2079
|
const usageStatsFlush = import("../../lib/proxy/usageStats.js").then(({ flushUsageStats }) => flushUsageStats());
|
|
1933
|
-
const
|
|
2080
|
+
const requestLogsFlush = import("../../lib/proxy/requestLogger.js").then(({ flushRequestLogs }) => flushRequestLogs());
|
|
2081
|
+
const [usageStatsFlushResult, lifecycleFlushResult, requestLogsFlushResult,] = await Promise.allSettled([
|
|
1934
2082
|
withTimeout(usageStatsFlush, PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS, "Timed out flushing proxy usage statistics during shutdown"),
|
|
1935
2083
|
withTimeout(flushProxyLifecycleEvents(), PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS, "Timed out flushing proxy lifecycle metadata during shutdown"),
|
|
2084
|
+
withTimeout(requestLogsFlush, PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS, "Timed out flushing proxy request logs during shutdown"),
|
|
1936
2085
|
]);
|
|
1937
2086
|
if (usageStatsFlushResult.status === "rejected") {
|
|
1938
2087
|
const error = usageStatsFlushResult.reason;
|
|
@@ -1942,6 +2091,10 @@ function registerProxyShutdownHandlers(params) {
|
|
|
1942
2091
|
const error = lifecycleFlushResult.reason;
|
|
1943
2092
|
logger.debug(`[proxy] lifecycle metadata flush failed during shutdown: ${error instanceof Error ? error.message : String(error)}`);
|
|
1944
2093
|
}
|
|
2094
|
+
if (requestLogsFlushResult.status === "rejected") {
|
|
2095
|
+
const error = requestLogsFlushResult.reason;
|
|
2096
|
+
logger.warn(`[proxy] request log flush failed during shutdown: ${error instanceof Error ? error.message : String(error)}`);
|
|
2097
|
+
}
|
|
1945
2098
|
try {
|
|
1946
2099
|
const { flushOpenTelemetry, shutdownOpenTelemetry } = await import("../../lib/services/server/ai/observability/instrumentation.js");
|
|
1947
2100
|
await flushOpenTelemetry();
|
|
@@ -2136,8 +2289,27 @@ async function startProxyRuntime(params) {
|
|
|
2136
2289
|
let stopRuntimeConfig;
|
|
2137
2290
|
if (params.runtimeConfigStore) {
|
|
2138
2291
|
const runtimeConfigStore = params.runtimeConfigStore;
|
|
2139
|
-
const unsubscribeReload = runtimeConfigStore.subscribeReload(() => {
|
|
2140
|
-
|
|
2292
|
+
const unsubscribeReload = runtimeConfigStore.subscribeReload((result) => {
|
|
2293
|
+
const snapshot = runtimeConfigStore.getSnapshot();
|
|
2294
|
+
persistRuntimeConfig(snapshot);
|
|
2295
|
+
if (socketWorker &&
|
|
2296
|
+
result.applied &&
|
|
2297
|
+
result.changed &&
|
|
2298
|
+
result.environmentChanged &&
|
|
2299
|
+
process.connected &&
|
|
2300
|
+
process.send) {
|
|
2301
|
+
try {
|
|
2302
|
+
process.send({
|
|
2303
|
+
type: "proxy-worker:replacement-requested",
|
|
2304
|
+
generation: getProxyWorkerGeneration(),
|
|
2305
|
+
pid: process.pid,
|
|
2306
|
+
reason: "environment",
|
|
2307
|
+
});
|
|
2308
|
+
}
|
|
2309
|
+
catch (error) {
|
|
2310
|
+
logger.warn(`[proxy] failed to request environment replacement: ${error instanceof Error ? error.message : String(error)}`);
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2141
2313
|
});
|
|
2142
2314
|
const reloadOnSighup = () => {
|
|
2143
2315
|
void runtimeConfigStore.reload("sighup");
|
|
@@ -2372,6 +2544,12 @@ async function startProxyCommandHandler(argv) {
|
|
|
2372
2544
|
else if (usageStatsPersistence.lastRecoveryAt) {
|
|
2373
2545
|
logger.warn(`[proxy] recovered corrupt usage statistics state at ${new Date(usageStatsPersistence.lastRecoveryAt).toISOString()}`);
|
|
2374
2546
|
}
|
|
2547
|
+
if (usageStatsPersistence.terminalErrorsLastError) {
|
|
2548
|
+
logger.warn(`[proxy] terminal-error persistence unavailable: ${usageStatsPersistence.terminalErrorsLastError}`);
|
|
2549
|
+
}
|
|
2550
|
+
else if (usageStatsPersistence.terminalErrorsLastRecoveryAt) {
|
|
2551
|
+
logger.warn(`[proxy] recovered corrupt terminal-error state at ${new Date(usageStatsPersistence.terminalErrorsLastRecoveryAt).toISOString()}`);
|
|
2552
|
+
}
|
|
2375
2553
|
const baseEnv = { ...process.env };
|
|
2376
2554
|
const envResolution = resolveProxyEnvFile({
|
|
2377
2555
|
explicitEnvFile: argv.envFile,
|
|
@@ -2524,8 +2702,17 @@ export const proxyStartCommand = {
|
|
|
2524
2702
|
// =============================================================================
|
|
2525
2703
|
// STATUS DISPLAY HELPERS
|
|
2526
2704
|
// =============================================================================
|
|
2705
|
+
export function sanitizeProxyStatusTerminalErrorMessage(message) {
|
|
2706
|
+
return stripVTControlCharacters(sanitizeForLog(redactUrlsInText(message), 700))
|
|
2707
|
+
.replace(/\s+/g, " ")
|
|
2708
|
+
.trim()
|
|
2709
|
+
.slice(0, 200);
|
|
2710
|
+
}
|
|
2527
2711
|
function printStatusStats(stats) {
|
|
2528
2712
|
console.info(`\n Stats:`);
|
|
2713
|
+
if (stats.startedAt !== undefined) {
|
|
2714
|
+
console.info(` Since: ${new Date(stats.startedAt).toISOString()}`);
|
|
2715
|
+
}
|
|
2529
2716
|
if (stats.totalAttempts !== undefined) {
|
|
2530
2717
|
console.info(` Attempts: ${stats.totalAttempts}`);
|
|
2531
2718
|
}
|
|
@@ -2538,6 +2725,32 @@ function printStatusStats(stats) {
|
|
|
2538
2725
|
stats.totalQuotaRateLimits !== undefined
|
|
2539
2726
|
? ` (${stats.totalTransientRateLimits} transient, ${stats.totalQuotaRateLimits} quota)`
|
|
2540
2727
|
: ""));
|
|
2728
|
+
if (stats.terminalErrors) {
|
|
2729
|
+
const causes = Object.entries(stats.terminalErrors.counts)
|
|
2730
|
+
.filter(([, count]) => count > 0)
|
|
2731
|
+
.map(([category, count]) => `${category}=${count}`)
|
|
2732
|
+
.join(", ");
|
|
2733
|
+
console.info(` Error details: ${stats.terminalErrors.totalErrors}/${stats.totalErrors}` +
|
|
2734
|
+
(causes ? ` (${causes})` : ""));
|
|
2735
|
+
console.info(` Detail since: ${new Date(stats.terminalErrors.startedAt).toISOString()}`);
|
|
2736
|
+
if ((stats.terminalErrorDetailsMissing ?? 0) > 0 ||
|
|
2737
|
+
(stats.terminalErrorDetailsExcess ?? 0) > 0) {
|
|
2738
|
+
console.info(` Detail gap: ${stats.terminalErrorDetailsMissing ?? 0} missing, ${stats.terminalErrorDetailsExcess ?? 0} awaiting counter reconciliation`);
|
|
2739
|
+
}
|
|
2740
|
+
else if (stats.terminalErrorDetailsComparable === false) {
|
|
2741
|
+
console.info(" Detail gap: unavailable during snapshot reconciliation");
|
|
2742
|
+
}
|
|
2743
|
+
const lastError = stats.lastTerminalError ?? stats.terminalErrors.recent.at(-1);
|
|
2744
|
+
if (lastError) {
|
|
2745
|
+
const cause = lastError.errorType ?? lastError.category;
|
|
2746
|
+
const code = lastError.errorCode ? `/${lastError.errorCode}` : "";
|
|
2747
|
+
const account = lastError.account ? ` account=${lastError.account}` : "";
|
|
2748
|
+
console.info(` Last error: ${new Date(lastError.at).toISOString()} ${cause}${code} status=${lastError.status}${account}`);
|
|
2749
|
+
if (lastError.message) {
|
|
2750
|
+
console.info(` Last cause: ${sanitizeProxyStatusTerminalErrorMessage(lastError.message)}`);
|
|
2751
|
+
}
|
|
2752
|
+
}
|
|
2753
|
+
}
|
|
2541
2754
|
if (stats.accounts?.length) {
|
|
2542
2755
|
console.info(`\n Accounts:`);
|
|
2543
2756
|
const headers = [
|
|
@@ -2556,7 +2769,17 @@ function printStatusStats(stats) {
|
|
|
2556
2769
|
String(account.success ?? 0),
|
|
2557
2770
|
String(account.errors ?? 0),
|
|
2558
2771
|
String(account.rateLimits ?? 0),
|
|
2559
|
-
|
|
2772
|
+
(() => {
|
|
2773
|
+
const status = account.status ?? (account.cooling ? "cooling" : "active");
|
|
2774
|
+
const states = [status];
|
|
2775
|
+
if (account.expired && status !== "expired") {
|
|
2776
|
+
states.push("expired");
|
|
2777
|
+
}
|
|
2778
|
+
if (account.allowed === false && status !== "excluded") {
|
|
2779
|
+
states.push("excluded");
|
|
2780
|
+
}
|
|
2781
|
+
return states.join(", ");
|
|
2782
|
+
})(),
|
|
2560
2783
|
]);
|
|
2561
2784
|
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => row[index].length)));
|
|
2562
2785
|
const numericColumns = new Set([2, 3, 4, 5]);
|
|
@@ -2646,20 +2869,23 @@ export const proxyStatusCommand = {
|
|
|
2646
2869
|
supervisorState.rolling.active?.pid === state?.pid)
|
|
2647
2870
|
: true);
|
|
2648
2871
|
if ((state || supervisorState) && (servingWorker || supervisorRunning)) {
|
|
2649
|
-
const
|
|
2650
|
-
const
|
|
2872
|
+
const servingState = servingWorker ? state : null;
|
|
2873
|
+
const activeHost = servingState?.host ?? supervisorState?.host ?? null;
|
|
2874
|
+
const activePort = servingState?.port ?? supervisorState?.port ?? null;
|
|
2651
2875
|
status.running = true;
|
|
2652
2876
|
status.pid = servingWorker && state ? state.pid : null;
|
|
2653
2877
|
status.port = activePort;
|
|
2654
2878
|
status.host = activeHost;
|
|
2655
|
-
status.mode =
|
|
2656
|
-
?
|
|
2879
|
+
status.mode = servingState
|
|
2880
|
+
? servingState.passthrough
|
|
2657
2881
|
? "passthrough"
|
|
2658
2882
|
: "full"
|
|
2659
2883
|
: null;
|
|
2660
|
-
status.strategy =
|
|
2884
|
+
status.strategy = servingState?.strategy ?? null;
|
|
2661
2885
|
status.startTime =
|
|
2662
|
-
|
|
2886
|
+
(supervisorRunning ? supervisorState?.startTime : null) ??
|
|
2887
|
+
servingState?.startTime ??
|
|
2888
|
+
null;
|
|
2663
2889
|
status.uptime = status.startTime
|
|
2664
2890
|
? Date.now() - new Date(status.startTime).getTime()
|
|
2665
2891
|
: null;
|
|
@@ -2667,17 +2893,18 @@ export const proxyStatusCommand = {
|
|
|
2667
2893
|
activeHost && activePort
|
|
2668
2894
|
? `http://${activeHost === "0.0.0.0" ? "localhost" : activeHost}:${activePort}`
|
|
2669
2895
|
: null;
|
|
2670
|
-
status.envFile =
|
|
2671
|
-
status.fallbackChain =
|
|
2672
|
-
status.accountAllowlist =
|
|
2673
|
-
status.configGeneration =
|
|
2674
|
-
status.configLoadedAt =
|
|
2675
|
-
status.lastConfigReloadError =
|
|
2896
|
+
status.envFile = servingState?.envFile ?? null;
|
|
2897
|
+
status.fallbackChain = servingState?.fallbackChain ?? null;
|
|
2898
|
+
status.accountAllowlist = servingState?.accountAllowlist ?? null;
|
|
2899
|
+
status.configGeneration = servingState?.configGeneration ?? null;
|
|
2900
|
+
status.configLoadedAt = servingState?.configLoadedAt ?? null;
|
|
2901
|
+
status.lastConfigReloadError =
|
|
2902
|
+
servingState?.lastConfigReloadError ?? null;
|
|
2676
2903
|
status.supervisorPid = supervisorPid ?? null;
|
|
2677
2904
|
status.supervisorRunning = supervisorRunning;
|
|
2678
2905
|
status.rolling = supervisorState?.rolling ?? null;
|
|
2679
2906
|
status.updaterPid =
|
|
2680
|
-
supervisorState?.updaterPid ??
|
|
2907
|
+
supervisorState?.updaterPid ?? servingState?.updaterPid ?? null;
|
|
2681
2908
|
status.updaterRunning = status.updaterPid
|
|
2682
2909
|
? isProcessRunning(status.updaterPid)
|
|
2683
2910
|
: false;
|
|
@@ -11,7 +11,7 @@ function printAnalysis(report) {
|
|
|
11
11
|
logger.always(chalk.gray("=".repeat(50)));
|
|
12
12
|
logger.always(` Since: ${chalk.cyan(report.since)}`);
|
|
13
13
|
logger.always(` Logs: ${chalk.cyan(report.logsDir)}`);
|
|
14
|
-
logger.always(` Files: ${report.files.requests} request, ${report.files.attempts} attempt, ${report.files.lifecycle} lifecycle`);
|
|
14
|
+
logger.always(` Files: ${report.files.requests} request, ${report.files.attempts} attempt, ${report.files.lifecycle} lifecycle, ${report.files.debug} debug`);
|
|
15
15
|
logger.always("");
|
|
16
16
|
logger.always(chalk.bold(" Reliability"));
|
|
17
17
|
logger.always(report.coverage.finalRequests
|
|
@@ -68,6 +68,15 @@ function printAnalysis(report) {
|
|
|
68
68
|
logger.always("");
|
|
69
69
|
logger.always(chalk.bold(" Data Quality"));
|
|
70
70
|
logger.always(` ${report.dataQuality.linesRead} lines scanned, ${report.dataQuality.malformedLines} malformed, ${report.dataQuality.unsupportedLifecycleLines} unsupported lifecycle, ${report.dataQuality.lifecycleSequenceGaps} sequence gaps, ${report.dataQuality.lifecycleSequenceDuplicates} duplicates`);
|
|
71
|
+
for (const [stream, range] of Object.entries(report.dataQuality.streams)) {
|
|
72
|
+
if (range.observedFrom) {
|
|
73
|
+
logger.always(` ${stream}: ${range.observedFrom} to ${range.observedTo}${range.startsAtOrBeforeRequestedWindow ? "" : chalk.yellow(" (starts after requested window)")}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const artifacts = report.dataQuality.bodyArtifacts;
|
|
77
|
+
if (artifacts.capturesIndexed > 0) {
|
|
78
|
+
logger.always(` Body artifacts: ${artifacts.artifactsPresent}/${artifacts.artifactsReferenced} present, ${artifacts.artifactsMissing} missing, ${artifacts.writeFailures} write failures, ${artifacts.invalidPaths} invalid paths, ${artifacts.truncatedCaptures} truncated captures`);
|
|
79
|
+
}
|
|
71
80
|
if (report.accounts.length > 0) {
|
|
72
81
|
logger.always("");
|
|
73
82
|
logger.always(chalk.bold(" Accounts"));
|
|
@@ -96,6 +96,8 @@ export declare class TokenStore {
|
|
|
96
96
|
* @throws TokenStoreError if reading fails (other than file not found)
|
|
97
97
|
*/
|
|
98
98
|
loadTokens(provider: string): Promise<StoredOAuthTokens | null>;
|
|
99
|
+
/** Reads tokens without updating access metadata or rewriting the store. */
|
|
100
|
+
peekTokens(provider: string): Promise<StoredOAuthTokens | null>;
|
|
99
101
|
/**
|
|
100
102
|
* Internal load without mutex — callers must already hold the mutex.
|
|
101
103
|
*/
|
|
@@ -179,6 +181,11 @@ export declare class TokenStore {
|
|
|
179
181
|
* @param reason - Optional human-readable reason (e.g., "refresh_failed")
|
|
180
182
|
*/
|
|
181
183
|
markDisabled(provider: string, reason?: string): Promise<void>;
|
|
184
|
+
/**
|
|
185
|
+
* Disables an account only when the persisted credentials still match the
|
|
186
|
+
* request that observed the authentication failure.
|
|
187
|
+
*/
|
|
188
|
+
markDisabledIfCurrent(provider: string, expectedTokens: Pick<StoredOAuthTokens, "accessToken" | "refreshToken" | "expiresAt">, reason?: string): Promise<boolean>;
|
|
182
189
|
/**
|
|
183
190
|
* Re-enables a previously disabled provider (persisted to disk).
|
|
184
191
|
*
|
|
@@ -213,6 +213,22 @@ export class TokenStore {
|
|
|
213
213
|
return result;
|
|
214
214
|
});
|
|
215
215
|
}
|
|
216
|
+
/** Reads tokens without updating access metadata or rewriting the store. */
|
|
217
|
+
async peekTokens(provider) {
|
|
218
|
+
return this._mutex.runExclusive(async () => {
|
|
219
|
+
try {
|
|
220
|
+
const storageData = await this.loadStorageData();
|
|
221
|
+
const tokens = storageData.providers[provider]?.tokens;
|
|
222
|
+
return tokens ? { ...tokens } : null;
|
|
223
|
+
}
|
|
224
|
+
catch (error) {
|
|
225
|
+
if (error instanceof TokenStoreError && error.code === "NOT_FOUND") {
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
throw error;
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
}
|
|
216
232
|
/**
|
|
217
233
|
* Internal load without mutex — callers must already hold the mutex.
|
|
218
234
|
*/
|
|
@@ -539,6 +555,44 @@ export class TokenStore {
|
|
|
539
555
|
});
|
|
540
556
|
});
|
|
541
557
|
}
|
|
558
|
+
/**
|
|
559
|
+
* Disables an account only when the persisted credentials still match the
|
|
560
|
+
* request that observed the authentication failure.
|
|
561
|
+
*/
|
|
562
|
+
async markDisabledIfCurrent(provider, expectedTokens, reason) {
|
|
563
|
+
return this._mutex.runExclusive(async () => {
|
|
564
|
+
let storageData;
|
|
565
|
+
try {
|
|
566
|
+
storageData = await this.loadStorageData();
|
|
567
|
+
}
|
|
568
|
+
catch (error) {
|
|
569
|
+
if (error instanceof TokenStoreError && error.code === "NOT_FOUND") {
|
|
570
|
+
return false;
|
|
571
|
+
}
|
|
572
|
+
throw error;
|
|
573
|
+
}
|
|
574
|
+
const providerData = storageData.providers[provider];
|
|
575
|
+
if (!providerData) {
|
|
576
|
+
return false;
|
|
577
|
+
}
|
|
578
|
+
const current = providerData.tokens;
|
|
579
|
+
if (current.accessToken !== expectedTokens.accessToken ||
|
|
580
|
+
current.refreshToken !== expectedTokens.refreshToken ||
|
|
581
|
+
current.expiresAt !== expectedTokens.expiresAt) {
|
|
582
|
+
return false;
|
|
583
|
+
}
|
|
584
|
+
providerData.disabled = true;
|
|
585
|
+
providerData.disabledAt = Date.now();
|
|
586
|
+
providerData.disabledReason = reason;
|
|
587
|
+
storageData.lastModified = Date.now();
|
|
588
|
+
await this.saveStorageData(storageData);
|
|
589
|
+
logger.info("Provider marked as disabled", {
|
|
590
|
+
provider,
|
|
591
|
+
reason: reason ?? "unspecified",
|
|
592
|
+
});
|
|
593
|
+
return true;
|
|
594
|
+
});
|
|
595
|
+
}
|
|
542
596
|
/**
|
|
543
597
|
* Re-enables a previously disabled provider (persisted to disk).
|
|
544
598
|
*
|