@juspay/neurolink 10.4.2 → 10.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/auth/tokenStore.d.ts +7 -0
  3. package/dist/auth/tokenStore.js +54 -0
  4. package/dist/browser/neurolink.min.js +377 -380
  5. package/dist/cli/commands/proxy.d.ts +2 -1
  6. package/dist/cli/commands/proxy.js +290 -63
  7. package/dist/cli/commands/proxyAnalyze.js +10 -1
  8. package/dist/lib/auth/tokenStore.d.ts +7 -0
  9. package/dist/lib/auth/tokenStore.js +54 -0
  10. package/dist/lib/proxy/proxyAnalysis.js +136 -12
  11. package/dist/lib/proxy/proxyTranslationEngine.d.ts +1 -0
  12. package/dist/lib/proxy/proxyTranslationEngine.js +56 -12
  13. package/dist/lib/proxy/rawStreamCapture.js +47 -1
  14. package/dist/lib/proxy/requestLogger.d.ts +2 -0
  15. package/dist/lib/proxy/requestLogger.js +72 -13
  16. package/dist/lib/proxy/rollingProxyServer.js +79 -8
  17. package/dist/lib/proxy/rollingWorkerProcess.js +20 -15
  18. package/dist/lib/proxy/rollingWorkerProtocol.js +3 -0
  19. package/dist/lib/proxy/rollingWorkerSupervisor.d.ts +1 -0
  20. package/dist/lib/proxy/rollingWorkerSupervisor.js +31 -11
  21. package/dist/lib/proxy/runtimeConfig.d.ts +1 -0
  22. package/dist/lib/proxy/runtimeConfig.js +20 -5
  23. package/dist/lib/proxy/socketWorkerRuntime.js +41 -13
  24. package/dist/lib/proxy/sseInterceptor.js +47 -1
  25. package/dist/lib/proxy/tokenRefresh.d.ts +6 -0
  26. package/dist/lib/proxy/tokenRefresh.js +70 -15
  27. package/dist/lib/proxy/usageStats.d.ts +25 -3
  28. package/dist/lib/proxy/usageStats.js +546 -55
  29. package/dist/lib/server/routes/claudeProxyRoutes.d.ts +2 -0
  30. package/dist/lib/server/routes/claudeProxyRoutes.js +261 -297
  31. package/dist/lib/types/proxy.d.ts +90 -1
  32. package/dist/proxy/proxyAnalysis.js +136 -12
  33. package/dist/proxy/proxyTranslationEngine.d.ts +1 -0
  34. package/dist/proxy/proxyTranslationEngine.js +56 -12
  35. package/dist/proxy/rawStreamCapture.js +47 -1
  36. package/dist/proxy/requestLogger.d.ts +2 -0
  37. package/dist/proxy/requestLogger.js +72 -13
  38. package/dist/proxy/rollingProxyServer.js +79 -8
  39. package/dist/proxy/rollingWorkerProcess.js +20 -15
  40. package/dist/proxy/rollingWorkerProtocol.js +3 -0
  41. package/dist/proxy/rollingWorkerSupervisor.d.ts +1 -0
  42. package/dist/proxy/rollingWorkerSupervisor.js +31 -11
  43. package/dist/proxy/runtimeConfig.d.ts +1 -0
  44. package/dist/proxy/runtimeConfig.js +20 -5
  45. package/dist/proxy/socketWorkerRuntime.js +41 -13
  46. package/dist/proxy/sseInterceptor.js +47 -1
  47. package/dist/proxy/tokenRefresh.d.ts +6 -0
  48. package/dist/proxy/tokenRefresh.js +70 -15
  49. package/dist/proxy/usageStats.d.ts +25 -3
  50. package/dist/proxy/usageStats.js +546 -55
  51. package/dist/server/routes/claudeProxyRoutes.d.ts +2 -0
  52. package/dist/server/routes/claudeProxyRoutes.js +261 -297
  53. package/dist/types/proxy.d.ts +90 -1
  54. package/package.json +1 -1
@@ -22,7 +22,7 @@ import { ProxyRuntimeConfigStore } from "../../lib/proxy/runtimeConfig.js";
22
22
  * string) recorded when the supervisor we expect at `pid` was launched.
23
23
  * When provided, this is cross-checked against `pid`'s actual OS-reported
24
24
  * start time (see {@link getProcessStartTime}) — the args match alone
25
- * ("neurolink" + "proxy" both present) is not airtight, since an
25
+ * (the `neurolink proxy start` command is present) is not airtight, since an
26
26
  * unrelated process (e.g. a shell running this very test suite, or a
27
27
  * coincidentally-named script) could match it too. A pid recycled by such
28
28
  * a process would have a start time far from the recorded supervisor
@@ -57,6 +57,7 @@ export declare function createProxyStartApp(params: {
57
57
  readiness: ProxyReadinessState;
58
58
  }>;
59
59
  export declare const proxyStartCommand: CommandModule<object, ProxyStartArgs>;
60
+ export declare function sanitizeProxyStatusTerminalErrorMessage(message: string): string;
60
61
  export declare const proxyStatusCommand: CommandModule<object, ProxyStatusArgs>;
61
62
  export declare const proxyTelemetryCommand: CommandModule<object, ProxyTelemetryArgs>;
62
63
  export declare const proxyGuardCommand: CommandModule<object, ProxyGuardArgs>;
@@ -12,11 +12,12 @@
12
12
  import { spawn } from "node:child_process";
13
13
  import { homedir } from "node:os";
14
14
  import { dirname, join, resolve } from "node:path";
15
+ import { stripVTControlCharacters } from "node:util";
15
16
  import chalk from "chalk";
16
17
  import ora from "ora";
17
18
  import { buildProxyHealthResponse, createProxyReadinessState, markProxyDrainingForUpdate, markProxyReady, resumeProxyConnections, waitForProxyReadiness, } from "../../lib/proxy/proxyHealth.js";
18
19
  import { logger } from "../../lib/utils/logger.js";
19
- import { sanitizeForLog } from "../../lib/utils/logSanitize.js";
20
+ import { redactUrlsInText, sanitizeForLog, } from "../../lib/utils/logSanitize.js";
20
21
  import { withTimeout } from "../../lib/utils/async/withTimeout.js";
21
22
  import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
22
23
  import { configureProxyKeepAliveDispatcher } from "../../lib/proxy/proxyDispatcher.js";
@@ -39,9 +40,11 @@ import { fileURLToPath } from "node:url";
39
40
  import packageJson from "../../../package.json" with { type: "json" };
40
41
  const _require = createRequire(import.meta.url);
41
42
  const PROXY_VERSION = packageJson.version;
43
+ const PROXY_INTERNAL_ACCOUNT_LABEL = "proxy/internal";
42
44
  const PROXY_TELEMETRY_SCRIPT_PATH = fileURLToPath(new URL("../../../scripts/observability/manage-local-openobserve.sh", import.meta.url));
43
45
  const PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS = 5_000;
44
46
  const LEGACY_STATUS_ACCOUNT_CACHE_TTL_MS = 5_000;
47
+ const PROXY_STATUS_TOKEN_READ_TIMEOUT_MS = 2_000;
45
48
  let legacyStatusAccountCache;
46
49
  // Allowed drift between a pid's OS-reported start time and the persisted
47
50
  // ProxySupervisorState.startTime before processLooksLikeProxySupervisor
@@ -187,7 +190,7 @@ async function getProcessStartTime(pid) {
187
190
  * string) recorded when the supervisor we expect at `pid` was launched.
188
191
  * When provided, this is cross-checked against `pid`'s actual OS-reported
189
192
  * start time (see {@link getProcessStartTime}) — the args match alone
190
- * ("neurolink" + "proxy" both present) is not airtight, since an
193
+ * (the `neurolink proxy start` command is present) is not airtight, since an
191
194
  * unrelated process (e.g. a shell running this very test suite, or a
192
195
  * coincidentally-named script) could match it too. A pid recycled by such
193
196
  * a process would have a start time far from the recorded supervisor
@@ -215,9 +218,9 @@ export async function processLooksLikeProxySupervisor(pid, expectedStartTimeIso)
215
218
  // "neurolink", an editor with a neurolink file open) would match, and a
216
219
  // stale/recycled pid running such a process could then be SIGTERM'd/
217
220
  // SIGKILL'd by `ensureSupervisorStoppedBeforeClear` during uninstall.
218
- // Require the actual proxy-supervisor invocation both "neurolink" AND
219
- // the "proxy" subcommand present in args.
220
- if (!(/neurolink/i.test(args) && /\bproxy\b/i.test(args))) {
221
+ // Require the actual proxy-supervisor invocation. Commands such as
222
+ // `neurolink proxy status` must never be mistaken for the listener owner.
223
+ if (!/\bneurolink(?:-proxy)?\b.*\bproxy\s+start\b/i.test(args)) {
221
224
  return false;
222
225
  }
223
226
  if (!expectedStartTimeIso) {
@@ -353,6 +356,15 @@ async function resolveLegacyStatusAccountLabel(storedAnthropicAccountCount) {
353
356
  };
354
357
  return label;
355
358
  }
359
+ function deriveAccountAllowance(accountKey, now, allowlist, expirations, cooldowns) {
360
+ const allowed = isAccountAllowed(accountKey, allowlist);
361
+ const expiresAt = expirations.get(accountKey);
362
+ return {
363
+ allowed,
364
+ expired: expiresAt !== undefined && expiresAt <= now,
365
+ cooling: (cooldowns[accountKey]?.coolingUntil ?? 0) > now,
366
+ };
367
+ }
356
368
  /**
357
369
  * Check if the launchd service is loaded and actively managing the proxy.
358
370
  * Returns true if launchctl reports the service as running.
@@ -1248,7 +1260,13 @@ export async function createProxyStartApp(params) {
1248
1260
  const recordRuntimeError = async (metadata, status, errorType, errorMessage, options) => {
1249
1261
  const clientMessage = options?.clientMessage ?? errorMessage;
1250
1262
  const clientErrorType = options?.clientErrorType ?? errorType;
1251
- recordFinalError(status);
1263
+ recordFinalError(status, undefined, undefined, {
1264
+ requestId: metadata.requestId,
1265
+ errorType,
1266
+ errorCode: options?.errorCode,
1267
+ terminalOutcome: "handler_error",
1268
+ message: errorMessage,
1269
+ });
1252
1270
  await Promise.all([
1253
1271
  logRequest({
1254
1272
  timestamp: new Date().toISOString(),
@@ -1522,21 +1540,45 @@ export async function createProxyStartApp(params) {
1522
1540
  const activeAccountAllowlist = runtimeConfig
1523
1541
  ? runtimeConfig.accountAllowlist
1524
1542
  : params.accountAllowlist;
1525
- const { getReconciledStats, getUsageStatsPersistenceStatus } = await import("../../lib/proxy/usageStats.js");
1543
+ const { getReconciledUsageSnapshot, getUsageStatsPersistenceStatus } = await import("../../lib/proxy/usageStats.js");
1526
1544
  const { loadAccountCooldowns } = await import("../../lib/proxy/accountCooldown.js");
1527
- const stats = await getReconciledStats();
1545
+ const usageSnapshot = await getReconciledUsageSnapshot();
1546
+ const { stats, terminalErrors } = usageSnapshot;
1547
+ const terminalErrorDetailsComparable = usageSnapshot.statsVersion === usageSnapshot.terminalErrorsVersion;
1548
+ const lastTerminalError = terminalErrors.recent.at(-1) ?? null;
1549
+ const terminalErrorDetailsMissing = terminalErrorDetailsComparable
1550
+ ? Math.max(0, stats.totalErrors - terminalErrors.totalErrors)
1551
+ : undefined;
1552
+ const terminalErrorDetailsExcess = terminalErrorDetailsComparable
1553
+ ? Math.max(0, terminalErrors.totalErrors - stats.totalErrors)
1554
+ : undefined;
1528
1555
  const runtimeState = loadProxyState();
1529
1556
  const supervisorState = loadProxySupervisorState();
1530
1557
  const updateState = loadUpdateState();
1531
1558
  const cooldowns = await loadAccountCooldowns();
1532
1559
  const storedAccountKeys = new Set();
1560
+ const storedAccountExpirations = new Map();
1533
1561
  const disabledAccountKeys = new Set();
1534
1562
  let accountInventoryLoaded = false;
1535
1563
  try {
1536
1564
  const { tokenStore } = await import("../../lib/auth/tokenStore.js");
1537
- for (const key of await tokenStore.listByPrefix("anthropic:")) {
1538
- storedAccountKeys.add(normalizeAnthropicAccountKey(key));
1565
+ const storedKeys = await tokenStore.listByPrefix("anthropic:");
1566
+ for (const key of storedKeys) {
1567
+ const normalizedKey = normalizeAnthropicAccountKey(key);
1568
+ storedAccountKeys.add(normalizedKey);
1539
1569
  }
1570
+ await Promise.all(storedKeys.map(async (key) => {
1571
+ const normalizedKey = normalizeAnthropicAccountKey(key);
1572
+ try {
1573
+ const tokens = await withTimeout(tokenStore.peekTokens(key), PROXY_STATUS_TOKEN_READ_TIMEOUT_MS, "[proxy] /status token inspection timed out");
1574
+ if (tokens) {
1575
+ storedAccountExpirations.set(normalizedKey, tokens.expiresAt);
1576
+ }
1577
+ }
1578
+ catch (err) {
1579
+ logger.debug(`[proxy] /status: failed to inspect token metadata for ${normalizedKey}: ${err instanceof Error ? err.message : String(err)}`);
1580
+ }
1581
+ }));
1540
1582
  for (const key of await tokenStore.listDisabled()) {
1541
1583
  disabledAccountKeys.add(normalizeAnthropicAccountKey(key));
1542
1584
  }
@@ -1556,6 +1598,141 @@ export async function createProxyStartApp(params) {
1556
1598
  });
1557
1599
  const primaryAccount = await resolveStatusPrimaryAccount(activeProxyConfig);
1558
1600
  const activeUpdaterPid = supervisorState?.updaterPid ?? runtimeState?.updaterPid;
1601
+ const accountRows = Object.values(stats.accounts).map((account) => {
1602
+ const normalizedKey = normalizeAnthropicAccountKey(account.label);
1603
+ const isLegacyAccount = account.type === "oauth" && account.label === legacyAccountLabel;
1604
+ const accountKey = storedAccountKeys.has(normalizedKey)
1605
+ ? normalizedKey
1606
+ : isLegacyAccount
1607
+ ? LEGACY_ANTHROPIC_ACCOUNT_KEY
1608
+ : account.label === "env"
1609
+ ? ENV_ANTHROPIC_ACCOUNT_KEY
1610
+ : normalizedKey;
1611
+ const isStored = storedAccountKeys.has(accountKey) || isLegacyAccount;
1612
+ const { allowed, expired, cooling } = deriveAccountAllowance(accountKey, now, activeAccountAllowlist, storedAccountExpirations, cooldowns);
1613
+ const accountStatus = account.type === "internal"
1614
+ ? "internal"
1615
+ : disabledAccountKeys.has(accountKey)
1616
+ ? "disabled"
1617
+ : expired
1618
+ ? "expired"
1619
+ : !allowed
1620
+ ? "excluded"
1621
+ : accountInventoryLoaded &&
1622
+ account.type === "oauth" &&
1623
+ !isStored
1624
+ ? "removed"
1625
+ : cooling
1626
+ ? "cooling"
1627
+ : "active";
1628
+ return {
1629
+ label: account.label,
1630
+ type: account.type,
1631
+ attempts: account.attemptCount,
1632
+ requests: account.successCount + account.errorCount,
1633
+ success: account.successCount,
1634
+ errors: account.errorCount,
1635
+ attemptErrors: account.attemptErrorCount,
1636
+ rateLimits: account.rateLimitCount,
1637
+ transientRateLimits: account.transientRateLimitCount,
1638
+ quotaRateLimits: account.quotaRateLimitCount,
1639
+ cooling,
1640
+ status: accountStatus,
1641
+ allowed: account.type === "internal" ? undefined : allowed,
1642
+ expired: account.type === "oauth" ? expired : undefined,
1643
+ };
1644
+ });
1645
+ const representedAccountKeys = new Set(accountRows
1646
+ .filter((account) => account.type === "oauth")
1647
+ .map((account) => normalizeAnthropicAccountKey(account.label)));
1648
+ for (const accountKey of storedAccountKeys) {
1649
+ if (representedAccountKeys.has(accountKey)) {
1650
+ continue;
1651
+ }
1652
+ const { allowed, expired, cooling } = deriveAccountAllowance(accountKey, now, activeAccountAllowlist, storedAccountExpirations, cooldowns);
1653
+ accountRows.push({
1654
+ label: accountKey.slice("anthropic:".length),
1655
+ type: "oauth",
1656
+ attempts: 0,
1657
+ requests: 0,
1658
+ success: 0,
1659
+ errors: 0,
1660
+ attemptErrors: 0,
1661
+ rateLimits: 0,
1662
+ transientRateLimits: 0,
1663
+ quotaRateLimits: 0,
1664
+ cooling,
1665
+ status: disabledAccountKeys.has(accountKey)
1666
+ ? "disabled"
1667
+ : expired
1668
+ ? "expired"
1669
+ : !allowed
1670
+ ? "excluded"
1671
+ : cooling
1672
+ ? "cooling"
1673
+ : "active",
1674
+ allowed,
1675
+ expired,
1676
+ });
1677
+ }
1678
+ const attributed = accountRows.reduce((total, account) => ({
1679
+ attempts: total.attempts + (account.attempts ?? 0),
1680
+ requests: total.requests + (account.requests ?? 0),
1681
+ success: total.success + (account.success ?? 0),
1682
+ errors: total.errors + (account.errors ?? 0),
1683
+ attemptErrors: total.attemptErrors + (account.attemptErrors ?? 0),
1684
+ rateLimits: total.rateLimits + (account.rateLimits ?? 0),
1685
+ transientRateLimits: total.transientRateLimits + (account.transientRateLimits ?? 0),
1686
+ quotaRateLimits: total.quotaRateLimits + (account.quotaRateLimits ?? 0),
1687
+ }), {
1688
+ attempts: 0,
1689
+ requests: 0,
1690
+ success: 0,
1691
+ errors: 0,
1692
+ attemptErrors: 0,
1693
+ rateLimits: 0,
1694
+ transientRateLimits: 0,
1695
+ quotaRateLimits: 0,
1696
+ });
1697
+ const unattributed = {
1698
+ attempts: Math.max(0, stats.totalAttempts - attributed.attempts),
1699
+ requests: Math.max(0, stats.totalRequests - attributed.requests),
1700
+ success: Math.max(0, stats.totalSuccess - attributed.success),
1701
+ errors: Math.max(0, stats.totalErrors - attributed.errors),
1702
+ attemptErrors: Math.max(0, stats.totalAttemptErrors - attributed.attemptErrors),
1703
+ rateLimits: Math.max(0, stats.totalRateLimits - attributed.rateLimits),
1704
+ transientRateLimits: Math.max(0, stats.totalTransientRateLimits - attributed.transientRateLimits),
1705
+ quotaRateLimits: Math.max(0, stats.totalQuotaRateLimits - attributed.quotaRateLimits),
1706
+ };
1707
+ if (Object.values(unattributed).some((count) => count > 0)) {
1708
+ const internalRow = accountRows.find((account) => account.label === PROXY_INTERNAL_ACCOUNT_LABEL);
1709
+ if (internalRow) {
1710
+ internalRow.attempts =
1711
+ (internalRow.attempts ?? 0) + unattributed.attempts;
1712
+ internalRow.requests =
1713
+ (internalRow.requests ?? 0) + unattributed.requests;
1714
+ internalRow.success = (internalRow.success ?? 0) + unattributed.success;
1715
+ internalRow.errors = (internalRow.errors ?? 0) + unattributed.errors;
1716
+ internalRow.attemptErrors =
1717
+ (internalRow.attemptErrors ?? 0) + unattributed.attemptErrors;
1718
+ internalRow.rateLimits =
1719
+ (internalRow.rateLimits ?? 0) + unattributed.rateLimits;
1720
+ internalRow.transientRateLimits =
1721
+ (internalRow.transientRateLimits ?? 0) +
1722
+ unattributed.transientRateLimits;
1723
+ internalRow.quotaRateLimits =
1724
+ (internalRow.quotaRateLimits ?? 0) + unattributed.quotaRateLimits;
1725
+ }
1726
+ else {
1727
+ accountRows.push({
1728
+ label: PROXY_INTERNAL_ACCOUNT_LABEL,
1729
+ type: "internal",
1730
+ ...unattributed,
1731
+ cooling: false,
1732
+ status: "internal",
1733
+ });
1734
+ }
1735
+ }
1559
1736
  return c.json({
1560
1737
  status: "running",
1561
1738
  ready: health.ready,
@@ -1578,42 +1755,12 @@ export async function createProxyStartApp(params) {
1578
1755
  totalRateLimits: stats.totalRateLimits,
1579
1756
  totalTransientRateLimits: stats.totalTransientRateLimits,
1580
1757
  totalQuotaRateLimits: stats.totalQuotaRateLimits,
1581
- accounts: Object.values(stats.accounts).map((account) => {
1582
- const normalizedKey = normalizeAnthropicAccountKey(account.label);
1583
- const isLegacyAccount = account.type === "oauth" && account.label === legacyAccountLabel;
1584
- const accountKey = storedAccountKeys.has(normalizedKey)
1585
- ? normalizedKey
1586
- : isLegacyAccount
1587
- ? LEGACY_ANTHROPIC_ACCOUNT_KEY
1588
- : account.label === "env"
1589
- ? ENV_ANTHROPIC_ACCOUNT_KEY
1590
- : normalizedKey;
1591
- const isStored = storedAccountKeys.has(accountKey) || isLegacyAccount;
1592
- const cooling = (cooldowns[accountKey]?.coolingUntil ?? 0) > now;
1593
- const accountStatus = disabledAccountKeys.has(accountKey)
1594
- ? "disabled"
1595
- : !isAccountAllowed(accountKey, activeAccountAllowlist)
1596
- ? "excluded"
1597
- : accountInventoryLoaded && account.type === "oauth" && !isStored
1598
- ? "removed"
1599
- : cooling
1600
- ? "cooling"
1601
- : "active";
1602
- return {
1603
- label: account.label,
1604
- type: account.type,
1605
- attempts: account.attemptCount,
1606
- requests: account.successCount + account.errorCount,
1607
- success: account.successCount,
1608
- errors: account.errorCount,
1609
- attemptErrors: account.attemptErrorCount,
1610
- rateLimits: account.rateLimitCount,
1611
- transientRateLimits: account.transientRateLimitCount,
1612
- quotaRateLimits: account.quotaRateLimitCount,
1613
- cooling,
1614
- status: accountStatus,
1615
- };
1616
- }),
1758
+ terminalErrors,
1759
+ lastTerminalError,
1760
+ terminalErrorDetailsComparable,
1761
+ terminalErrorDetailsMissing,
1762
+ terminalErrorDetailsExcess,
1763
+ accounts: accountRows,
1617
1764
  primaryAccount,
1618
1765
  persistence: getUsageStatsPersistenceStatus(),
1619
1766
  },
@@ -1930,9 +2077,11 @@ function registerProxyShutdownHandlers(params) {
1930
2077
  }
1931
2078
  }
1932
2079
  const usageStatsFlush = import("../../lib/proxy/usageStats.js").then(({ flushUsageStats }) => flushUsageStats());
1933
- const [usageStatsFlushResult, lifecycleFlushResult] = await Promise.allSettled([
2080
+ const requestLogsFlush = import("../../lib/proxy/requestLogger.js").then(({ flushRequestLogs }) => flushRequestLogs());
2081
+ const [usageStatsFlushResult, lifecycleFlushResult, requestLogsFlushResult,] = await Promise.allSettled([
1934
2082
  withTimeout(usageStatsFlush, PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS, "Timed out flushing proxy usage statistics during shutdown"),
1935
2083
  withTimeout(flushProxyLifecycleEvents(), PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS, "Timed out flushing proxy lifecycle metadata during shutdown"),
2084
+ withTimeout(requestLogsFlush, PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS, "Timed out flushing proxy request logs during shutdown"),
1936
2085
  ]);
1937
2086
  if (usageStatsFlushResult.status === "rejected") {
1938
2087
  const error = usageStatsFlushResult.reason;
@@ -1942,6 +2091,10 @@ function registerProxyShutdownHandlers(params) {
1942
2091
  const error = lifecycleFlushResult.reason;
1943
2092
  logger.debug(`[proxy] lifecycle metadata flush failed during shutdown: ${error instanceof Error ? error.message : String(error)}`);
1944
2093
  }
2094
+ if (requestLogsFlushResult.status === "rejected") {
2095
+ const error = requestLogsFlushResult.reason;
2096
+ logger.warn(`[proxy] request log flush failed during shutdown: ${error instanceof Error ? error.message : String(error)}`);
2097
+ }
1945
2098
  try {
1946
2099
  const { flushOpenTelemetry, shutdownOpenTelemetry } = await import("../../lib/services/server/ai/observability/instrumentation.js");
1947
2100
  await flushOpenTelemetry();
@@ -2136,8 +2289,27 @@ async function startProxyRuntime(params) {
2136
2289
  let stopRuntimeConfig;
2137
2290
  if (params.runtimeConfigStore) {
2138
2291
  const runtimeConfigStore = params.runtimeConfigStore;
2139
- const unsubscribeReload = runtimeConfigStore.subscribeReload(() => {
2140
- persistRuntimeConfig(runtimeConfigStore.getSnapshot());
2292
+ const unsubscribeReload = runtimeConfigStore.subscribeReload((result) => {
2293
+ const snapshot = runtimeConfigStore.getSnapshot();
2294
+ persistRuntimeConfig(snapshot);
2295
+ if (socketWorker &&
2296
+ result.applied &&
2297
+ result.changed &&
2298
+ result.environmentChanged &&
2299
+ process.connected &&
2300
+ process.send) {
2301
+ try {
2302
+ process.send({
2303
+ type: "proxy-worker:replacement-requested",
2304
+ generation: getProxyWorkerGeneration(),
2305
+ pid: process.pid,
2306
+ reason: "environment",
2307
+ });
2308
+ }
2309
+ catch (error) {
2310
+ logger.warn(`[proxy] failed to request environment replacement: ${error instanceof Error ? error.message : String(error)}`);
2311
+ }
2312
+ }
2141
2313
  });
2142
2314
  const reloadOnSighup = () => {
2143
2315
  void runtimeConfigStore.reload("sighup");
@@ -2372,6 +2544,12 @@ async function startProxyCommandHandler(argv) {
2372
2544
  else if (usageStatsPersistence.lastRecoveryAt) {
2373
2545
  logger.warn(`[proxy] recovered corrupt usage statistics state at ${new Date(usageStatsPersistence.lastRecoveryAt).toISOString()}`);
2374
2546
  }
2547
+ if (usageStatsPersistence.terminalErrorsLastError) {
2548
+ logger.warn(`[proxy] terminal-error persistence unavailable: ${usageStatsPersistence.terminalErrorsLastError}`);
2549
+ }
2550
+ else if (usageStatsPersistence.terminalErrorsLastRecoveryAt) {
2551
+ logger.warn(`[proxy] recovered corrupt terminal-error state at ${new Date(usageStatsPersistence.terminalErrorsLastRecoveryAt).toISOString()}`);
2552
+ }
2375
2553
  const baseEnv = { ...process.env };
2376
2554
  const envResolution = resolveProxyEnvFile({
2377
2555
  explicitEnvFile: argv.envFile,
@@ -2524,8 +2702,17 @@ export const proxyStartCommand = {
2524
2702
  // =============================================================================
2525
2703
  // STATUS DISPLAY HELPERS
2526
2704
  // =============================================================================
2705
+ export function sanitizeProxyStatusTerminalErrorMessage(message) {
2706
+ return stripVTControlCharacters(sanitizeForLog(redactUrlsInText(message), 700))
2707
+ .replace(/\s+/g, " ")
2708
+ .trim()
2709
+ .slice(0, 200);
2710
+ }
2527
2711
  function printStatusStats(stats) {
2528
2712
  console.info(`\n Stats:`);
2713
+ if (stats.startedAt !== undefined) {
2714
+ console.info(` Since: ${new Date(stats.startedAt).toISOString()}`);
2715
+ }
2529
2716
  if (stats.totalAttempts !== undefined) {
2530
2717
  console.info(` Attempts: ${stats.totalAttempts}`);
2531
2718
  }
@@ -2538,6 +2725,32 @@ function printStatusStats(stats) {
2538
2725
  stats.totalQuotaRateLimits !== undefined
2539
2726
  ? ` (${stats.totalTransientRateLimits} transient, ${stats.totalQuotaRateLimits} quota)`
2540
2727
  : ""));
2728
+ if (stats.terminalErrors) {
2729
+ const causes = Object.entries(stats.terminalErrors.counts)
2730
+ .filter(([, count]) => count > 0)
2731
+ .map(([category, count]) => `${category}=${count}`)
2732
+ .join(", ");
2733
+ console.info(` Error details: ${stats.terminalErrors.totalErrors}/${stats.totalErrors}` +
2734
+ (causes ? ` (${causes})` : ""));
2735
+ console.info(` Detail since: ${new Date(stats.terminalErrors.startedAt).toISOString()}`);
2736
+ if ((stats.terminalErrorDetailsMissing ?? 0) > 0 ||
2737
+ (stats.terminalErrorDetailsExcess ?? 0) > 0) {
2738
+ console.info(` Detail gap: ${stats.terminalErrorDetailsMissing ?? 0} missing, ${stats.terminalErrorDetailsExcess ?? 0} awaiting counter reconciliation`);
2739
+ }
2740
+ else if (stats.terminalErrorDetailsComparable === false) {
2741
+ console.info(" Detail gap: unavailable during snapshot reconciliation");
2742
+ }
2743
+ const lastError = stats.lastTerminalError ?? stats.terminalErrors.recent.at(-1);
2744
+ if (lastError) {
2745
+ const cause = lastError.errorType ?? lastError.category;
2746
+ const code = lastError.errorCode ? `/${lastError.errorCode}` : "";
2747
+ const account = lastError.account ? ` account=${lastError.account}` : "";
2748
+ console.info(` Last error: ${new Date(lastError.at).toISOString()} ${cause}${code} status=${lastError.status}${account}`);
2749
+ if (lastError.message) {
2750
+ console.info(` Last cause: ${sanitizeProxyStatusTerminalErrorMessage(lastError.message)}`);
2751
+ }
2752
+ }
2753
+ }
2541
2754
  if (stats.accounts?.length) {
2542
2755
  console.info(`\n Accounts:`);
2543
2756
  const headers = [
@@ -2556,7 +2769,17 @@ function printStatusStats(stats) {
2556
2769
  String(account.success ?? 0),
2557
2770
  String(account.errors ?? 0),
2558
2771
  String(account.rateLimits ?? 0),
2559
- account.status ?? (account.cooling ? "cooling" : "active"),
2772
+ (() => {
2773
+ const status = account.status ?? (account.cooling ? "cooling" : "active");
2774
+ const states = [status];
2775
+ if (account.expired && status !== "expired") {
2776
+ states.push("expired");
2777
+ }
2778
+ if (account.allowed === false && status !== "excluded") {
2779
+ states.push("excluded");
2780
+ }
2781
+ return states.join(", ");
2782
+ })(),
2560
2783
  ]);
2561
2784
  const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => row[index].length)));
2562
2785
  const numericColumns = new Set([2, 3, 4, 5]);
@@ -2646,20 +2869,23 @@ export const proxyStatusCommand = {
2646
2869
  supervisorState.rolling.active?.pid === state?.pid)
2647
2870
  : true);
2648
2871
  if ((state || supervisorState) && (servingWorker || supervisorRunning)) {
2649
- const activeHost = state?.host ?? supervisorState?.host ?? null;
2650
- const activePort = state?.port ?? supervisorState?.port ?? null;
2872
+ const servingState = servingWorker ? state : null;
2873
+ const activeHost = servingState?.host ?? supervisorState?.host ?? null;
2874
+ const activePort = servingState?.port ?? supervisorState?.port ?? null;
2651
2875
  status.running = true;
2652
2876
  status.pid = servingWorker && state ? state.pid : null;
2653
2877
  status.port = activePort;
2654
2878
  status.host = activeHost;
2655
- status.mode = state
2656
- ? state.passthrough
2879
+ status.mode = servingState
2880
+ ? servingState.passthrough
2657
2881
  ? "passthrough"
2658
2882
  : "full"
2659
2883
  : null;
2660
- status.strategy = state?.strategy ?? null;
2884
+ status.strategy = servingState?.strategy ?? null;
2661
2885
  status.startTime =
2662
- state?.startTime ?? supervisorState?.startTime ?? null;
2886
+ (supervisorRunning ? supervisorState?.startTime : null) ??
2887
+ servingState?.startTime ??
2888
+ null;
2663
2889
  status.uptime = status.startTime
2664
2890
  ? Date.now() - new Date(status.startTime).getTime()
2665
2891
  : null;
@@ -2667,17 +2893,18 @@ export const proxyStatusCommand = {
2667
2893
  activeHost && activePort
2668
2894
  ? `http://${activeHost === "0.0.0.0" ? "localhost" : activeHost}:${activePort}`
2669
2895
  : null;
2670
- status.envFile = state?.envFile ?? null;
2671
- status.fallbackChain = state?.fallbackChain ?? null;
2672
- status.accountAllowlist = state?.accountAllowlist ?? null;
2673
- status.configGeneration = state?.configGeneration ?? null;
2674
- status.configLoadedAt = state?.configLoadedAt ?? null;
2675
- status.lastConfigReloadError = state?.lastConfigReloadError ?? null;
2896
+ status.envFile = servingState?.envFile ?? null;
2897
+ status.fallbackChain = servingState?.fallbackChain ?? null;
2898
+ status.accountAllowlist = servingState?.accountAllowlist ?? null;
2899
+ status.configGeneration = servingState?.configGeneration ?? null;
2900
+ status.configLoadedAt = servingState?.configLoadedAt ?? null;
2901
+ status.lastConfigReloadError =
2902
+ servingState?.lastConfigReloadError ?? null;
2676
2903
  status.supervisorPid = supervisorPid ?? null;
2677
2904
  status.supervisorRunning = supervisorRunning;
2678
2905
  status.rolling = supervisorState?.rolling ?? null;
2679
2906
  status.updaterPid =
2680
- supervisorState?.updaterPid ?? state?.updaterPid ?? null;
2907
+ supervisorState?.updaterPid ?? servingState?.updaterPid ?? null;
2681
2908
  status.updaterRunning = status.updaterPid
2682
2909
  ? isProcessRunning(status.updaterPid)
2683
2910
  : false;
@@ -11,7 +11,7 @@ function printAnalysis(report) {
11
11
  logger.always(chalk.gray("=".repeat(50)));
12
12
  logger.always(` Since: ${chalk.cyan(report.since)}`);
13
13
  logger.always(` Logs: ${chalk.cyan(report.logsDir)}`);
14
- logger.always(` Files: ${report.files.requests} request, ${report.files.attempts} attempt, ${report.files.lifecycle} lifecycle`);
14
+ logger.always(` Files: ${report.files.requests} request, ${report.files.attempts} attempt, ${report.files.lifecycle} lifecycle, ${report.files.debug} debug`);
15
15
  logger.always("");
16
16
  logger.always(chalk.bold(" Reliability"));
17
17
  logger.always(report.coverage.finalRequests
@@ -68,6 +68,15 @@ function printAnalysis(report) {
68
68
  logger.always("");
69
69
  logger.always(chalk.bold(" Data Quality"));
70
70
  logger.always(` ${report.dataQuality.linesRead} lines scanned, ${report.dataQuality.malformedLines} malformed, ${report.dataQuality.unsupportedLifecycleLines} unsupported lifecycle, ${report.dataQuality.lifecycleSequenceGaps} sequence gaps, ${report.dataQuality.lifecycleSequenceDuplicates} duplicates`);
71
+ for (const [stream, range] of Object.entries(report.dataQuality.streams)) {
72
+ if (range.observedFrom) {
73
+ logger.always(` ${stream}: ${range.observedFrom} to ${range.observedTo}${range.startsAtOrBeforeRequestedWindow ? "" : chalk.yellow(" (starts after requested window)")}`);
74
+ }
75
+ }
76
+ const artifacts = report.dataQuality.bodyArtifacts;
77
+ if (artifacts.capturesIndexed > 0) {
78
+ logger.always(` Body artifacts: ${artifacts.artifactsPresent}/${artifacts.artifactsReferenced} present, ${artifacts.artifactsMissing} missing, ${artifacts.writeFailures} write failures, ${artifacts.invalidPaths} invalid paths, ${artifacts.truncatedCaptures} truncated captures`);
79
+ }
71
80
  if (report.accounts.length > 0) {
72
81
  logger.always("");
73
82
  logger.always(chalk.bold(" Accounts"));
@@ -96,6 +96,8 @@ export declare class TokenStore {
96
96
  * @throws TokenStoreError if reading fails (other than file not found)
97
97
  */
98
98
  loadTokens(provider: string): Promise<StoredOAuthTokens | null>;
99
+ /** Reads tokens without updating access metadata or rewriting the store. */
100
+ peekTokens(provider: string): Promise<StoredOAuthTokens | null>;
99
101
  /**
100
102
  * Internal load without mutex — callers must already hold the mutex.
101
103
  */
@@ -179,6 +181,11 @@ export declare class TokenStore {
179
181
  * @param reason - Optional human-readable reason (e.g., "refresh_failed")
180
182
  */
181
183
  markDisabled(provider: string, reason?: string): Promise<void>;
184
+ /**
185
+ * Disables an account only when the persisted credentials still match the
186
+ * request that observed the authentication failure.
187
+ */
188
+ markDisabledIfCurrent(provider: string, expectedTokens: Pick<StoredOAuthTokens, "accessToken" | "refreshToken" | "expiresAt">, reason?: string): Promise<boolean>;
182
189
  /**
183
190
  * Re-enables a previously disabled provider (persisted to disk).
184
191
  *
@@ -213,6 +213,22 @@ export class TokenStore {
213
213
  return result;
214
214
  });
215
215
  }
216
+ /** Reads tokens without updating access metadata or rewriting the store. */
217
+ async peekTokens(provider) {
218
+ return this._mutex.runExclusive(async () => {
219
+ try {
220
+ const storageData = await this.loadStorageData();
221
+ const tokens = storageData.providers[provider]?.tokens;
222
+ return tokens ? { ...tokens } : null;
223
+ }
224
+ catch (error) {
225
+ if (error instanceof TokenStoreError && error.code === "NOT_FOUND") {
226
+ return null;
227
+ }
228
+ throw error;
229
+ }
230
+ });
231
+ }
216
232
  /**
217
233
  * Internal load without mutex — callers must already hold the mutex.
218
234
  */
@@ -539,6 +555,44 @@ export class TokenStore {
539
555
  });
540
556
  });
541
557
  }
558
+ /**
559
+ * Disables an account only when the persisted credentials still match the
560
+ * request that observed the authentication failure.
561
+ */
562
+ async markDisabledIfCurrent(provider, expectedTokens, reason) {
563
+ return this._mutex.runExclusive(async () => {
564
+ let storageData;
565
+ try {
566
+ storageData = await this.loadStorageData();
567
+ }
568
+ catch (error) {
569
+ if (error instanceof TokenStoreError && error.code === "NOT_FOUND") {
570
+ return false;
571
+ }
572
+ throw error;
573
+ }
574
+ const providerData = storageData.providers[provider];
575
+ if (!providerData) {
576
+ return false;
577
+ }
578
+ const current = providerData.tokens;
579
+ if (current.accessToken !== expectedTokens.accessToken ||
580
+ current.refreshToken !== expectedTokens.refreshToken ||
581
+ current.expiresAt !== expectedTokens.expiresAt) {
582
+ return false;
583
+ }
584
+ providerData.disabled = true;
585
+ providerData.disabledAt = Date.now();
586
+ providerData.disabledReason = reason;
587
+ storageData.lastModified = Date.now();
588
+ await this.saveStorageData(storageData);
589
+ logger.info("Provider marked as disabled", {
590
+ provider,
591
+ reason: reason ?? "unspecified",
592
+ });
593
+ return true;
594
+ });
595
+ }
542
596
  /**
543
597
  * Re-enables a previously disabled provider (persisted to disk).
544
598
  *