@juspay/neurolink 9.93.0 → 9.93.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/browser/neurolink.min.js +379 -382
- package/dist/cli/commands/proxy.d.ts +3 -2
- package/dist/cli/commands/proxy.js +209 -48
- package/dist/cli/commands/proxyAnalyze.d.ts +3 -0
- package/dist/cli/commands/proxyAnalyze.js +147 -0
- package/dist/cli/parser.js +3 -1
- package/dist/lib/proxy/proxyAnalysis.d.ts +3 -0
- package/dist/lib/proxy/proxyAnalysis.js +454 -0
- package/dist/lib/proxy/proxyHealth.d.ts +4 -0
- package/dist/lib/proxy/proxyHealth.js +21 -0
- package/dist/lib/proxy/sseInterceptor.d.ts +9 -1
- package/dist/lib/proxy/sseInterceptor.js +6 -5
- package/dist/lib/proxy/streamOutcome.d.ts +3 -1
- package/dist/lib/proxy/streamOutcome.js +77 -0
- package/dist/lib/proxy/updateCoordinator.d.ts +7 -0
- package/dist/lib/proxy/updateCoordinator.js +60 -0
- package/dist/lib/proxy/updateState.d.ts +4 -0
- package/dist/lib/proxy/updateState.js +51 -0
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +1 -0
- package/dist/lib/server/routes/claudeProxyRoutes.js +114 -17
- package/dist/lib/types/cli.d.ts +7 -0
- package/dist/lib/types/proxy.d.ts +187 -0
- package/dist/proxy/proxyAnalysis.d.ts +3 -0
- package/dist/proxy/proxyAnalysis.js +453 -0
- package/dist/proxy/proxyHealth.d.ts +4 -0
- package/dist/proxy/proxyHealth.js +21 -0
- package/dist/proxy/sseInterceptor.d.ts +9 -1
- package/dist/proxy/sseInterceptor.js +6 -5
- package/dist/proxy/streamOutcome.d.ts +3 -1
- package/dist/proxy/streamOutcome.js +77 -0
- package/dist/proxy/updateCoordinator.d.ts +7 -0
- package/dist/proxy/updateCoordinator.js +59 -0
- package/dist/proxy/updateState.d.ts +4 -0
- package/dist/proxy/updateState.js +51 -0
- package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
- package/dist/server/routes/claudeProxyRoutes.js +114 -17
- package/dist/types/cli.d.ts +7 -0
- package/dist/types/proxy.d.ts +187 -0
- package/package.json +4 -1
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import type { CommandModule } from "yargs";
|
|
13
13
|
import type { Hono } from "hono";
|
|
14
|
-
import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxyStatusArgs, ProxyTelemetryArgs } from "../../lib/types/index.js";
|
|
14
|
+
import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../lib/types/index.js";
|
|
15
15
|
import { ProxyRuntimeConfigStore } from "../../lib/proxy/runtimeConfig.js";
|
|
16
16
|
export declare function mapClaudeErrorTypeToStatus(errorType?: string): number;
|
|
17
17
|
export declare function createProxyStartApp(params: {
|
|
@@ -25,9 +25,10 @@ export declare function createProxyStartApp(params: {
|
|
|
25
25
|
primaryAccountKey: string | undefined;
|
|
26
26
|
accountAllowlist: AccountAllowlist | undefined;
|
|
27
27
|
runtimeConfigStore?: ProxyRuntimeConfigStore;
|
|
28
|
+
updateControlToken?: string;
|
|
28
29
|
}): Promise<{
|
|
29
30
|
app: Hono<import("hono/types").BlankEnv, import("hono/types").BlankSchema, "/">;
|
|
30
|
-
readiness:
|
|
31
|
+
readiness: ProxyReadinessState;
|
|
31
32
|
}>;
|
|
32
33
|
export declare const proxyStartCommand: CommandModule<object, ProxyStartArgs>;
|
|
33
34
|
export declare const proxyStatusCommand: CommandModule<object, ProxyStatusArgs>;
|
|
@@ -14,7 +14,7 @@ import { homedir } from "node:os";
|
|
|
14
14
|
import { dirname, join, resolve } from "node:path";
|
|
15
15
|
import chalk from "chalk";
|
|
16
16
|
import ora from "ora";
|
|
17
|
-
import { buildProxyHealthResponse, createProxyReadinessState, markProxyReady, waitForProxyReadiness, } from "../../lib/proxy/proxyHealth.js";
|
|
17
|
+
import { buildProxyHealthResponse, createProxyReadinessState, markProxyDrainingForUpdate, markProxyReady, resumeProxyConnections, waitForProxyReadiness, } from "../../lib/proxy/proxyHealth.js";
|
|
18
18
|
import { logger } from "../../lib/utils/logger.js";
|
|
19
19
|
import { withTimeout } from "../../lib/utils/async/withTimeout.js";
|
|
20
20
|
import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
|
|
@@ -25,7 +25,8 @@ import { beginProxyRequest, getProxyActivitySnapshot, trackProxyResponse, } from
|
|
|
25
25
|
import { flushProxyLifecycleEvents, getProxyLifecycleLoggerSnapshot, hashProxyLifecycleSessionId, logProxyLifecycleEvent, } from "../../lib/proxy/proxyLifecycle.js";
|
|
26
26
|
import { describeInstallFailure, getGlobalInstallArgs, resolveGlobalInstaller, validateInstalledVersion, } from "../../lib/proxy/globalInstaller.js";
|
|
27
27
|
import { startUpdaterWorkerSupervisor } from "../../lib/proxy/updaterSupervisor.js";
|
|
28
|
-
import {
|
|
28
|
+
import { waitForProxyUpdateWindow } from "../../lib/proxy/updateCoordinator.js";
|
|
29
|
+
import { abandonPendingUpdate, clearUpdateDeferral, isVersionSuppressed, loadUpdateState, recordCheck, recordSuccessfulUpdate, recordUpdateDeferred, recordUpdateFailure, recordUpdateInstalled, suppressVersion, } from "../../lib/proxy/updateState.js";
|
|
29
30
|
import { loadProxyEnvFile, resolveProxyEnvFile, } from "../../lib/proxy/proxyEnv.js";
|
|
30
31
|
import { createRequire } from "node:module";
|
|
31
32
|
import { fileURLToPath } from "node:url";
|
|
@@ -34,6 +35,8 @@ const _require = createRequire(import.meta.url);
|
|
|
34
35
|
const PROXY_VERSION = packageJson.version;
|
|
35
36
|
const PROXY_TELEMETRY_SCRIPT_PATH = fileURLToPath(new URL("../../../scripts/observability/manage-local-openobserve.sh", import.meta.url));
|
|
36
37
|
const PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS = 5_000;
|
|
38
|
+
const PROXY_UPDATE_CONTROL_TOKEN = process.env.NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN?.trim() ||
|
|
39
|
+
crypto.randomUUID();
|
|
37
40
|
// =============================================================================
|
|
38
41
|
// STATE MANAGEMENT
|
|
39
42
|
// =============================================================================
|
|
@@ -370,16 +373,52 @@ async function getProxyRuntimeActivity(host, port, timeoutMs = 3_000) {
|
|
|
370
373
|
return null;
|
|
371
374
|
}
|
|
372
375
|
}
|
|
373
|
-
function
|
|
374
|
-
|
|
375
|
-
|
|
376
|
+
async function setProxyUpdateDrain(host, port, draining) {
|
|
377
|
+
const confirmState = async () => {
|
|
378
|
+
try {
|
|
379
|
+
const response = await fetch(`http://${host}:${port}/status`, {
|
|
380
|
+
signal: AbortSignal.timeout(3_000),
|
|
381
|
+
});
|
|
382
|
+
if (!response.ok) {
|
|
383
|
+
return false;
|
|
384
|
+
}
|
|
385
|
+
const payload = (await response.json());
|
|
386
|
+
return payload.health?.drainingForUpdate === draining;
|
|
387
|
+
}
|
|
388
|
+
catch {
|
|
389
|
+
return false;
|
|
390
|
+
}
|
|
391
|
+
};
|
|
392
|
+
try {
|
|
393
|
+
const response = await fetch(`http://${host}:${port}/internal/update-control`, {
|
|
394
|
+
method: "POST",
|
|
395
|
+
headers: {
|
|
396
|
+
"content-type": "application/json",
|
|
397
|
+
"x-neurolink-update-token": PROXY_UPDATE_CONTROL_TOKEN,
|
|
398
|
+
},
|
|
399
|
+
body: JSON.stringify({ action: draining ? "drain" : "resume" }),
|
|
400
|
+
signal: AbortSignal.timeout(5_000),
|
|
401
|
+
});
|
|
402
|
+
if (!response.ok) {
|
|
403
|
+
return confirmState();
|
|
404
|
+
}
|
|
405
|
+
const payload = (await response.json());
|
|
406
|
+
return payload.draining === draining;
|
|
376
407
|
}
|
|
377
|
-
|
|
378
|
-
return
|
|
408
|
+
catch {
|
|
409
|
+
return confirmState();
|
|
379
410
|
}
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
411
|
+
}
|
|
412
|
+
async function resumeProxyUpdateDrain(host, port, attempts = 3) {
|
|
413
|
+
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
414
|
+
if (await setProxyUpdateDrain(host, port, false)) {
|
|
415
|
+
return true;
|
|
416
|
+
}
|
|
417
|
+
if (attempt < attempts) {
|
|
418
|
+
await sleep(1_000);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
return false;
|
|
383
422
|
}
|
|
384
423
|
// ---------------------------------------------------------------------------
|
|
385
424
|
// Stable entrypoint for launchd
|
|
@@ -556,7 +595,10 @@ function spawnProxyUpdater(host, port, parentPid) {
|
|
|
556
595
|
const child = spawn(process.execPath, args, {
|
|
557
596
|
detached: true,
|
|
558
597
|
stdio: ["ignore", logFd, logFd],
|
|
559
|
-
env:
|
|
598
|
+
env: {
|
|
599
|
+
...process.env,
|
|
600
|
+
NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN: PROXY_UPDATE_CONTROL_TOKEN,
|
|
601
|
+
},
|
|
560
602
|
});
|
|
561
603
|
child.once("error", (error) => {
|
|
562
604
|
logger.always(`[proxy] updater worker error pid=${child.pid ?? "unknown"}: ${error.message}`);
|
|
@@ -727,7 +769,7 @@ async function createProxyNeurolinkRuntime(logsDir) {
|
|
|
727
769
|
cleanupLogs(7, 500);
|
|
728
770
|
return { neurolink, cleanupLogs };
|
|
729
771
|
}
|
|
730
|
-
function registerProxyRequestTracking(app, requestMetadata) {
|
|
772
|
+
function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
731
773
|
app.use("/v1/*", async (c, next) => {
|
|
732
774
|
const startedMonotonicMs = performance.now();
|
|
733
775
|
const contentLengthHeader = c.req.raw.headers.get("content-length");
|
|
@@ -747,9 +789,12 @@ function registerProxyRequestTracking(app, requestMetadata) {
|
|
|
747
789
|
model: "-",
|
|
748
790
|
stream: false,
|
|
749
791
|
toolCount: 0,
|
|
792
|
+
rejectForUpdate: readiness.drainingForUpdate,
|
|
750
793
|
};
|
|
751
794
|
requestMetadata.set(c.req.raw, metadata);
|
|
752
|
-
const finishActivity =
|
|
795
|
+
const finishActivity = metadata.rejectForUpdate
|
|
796
|
+
? () => undefined
|
|
797
|
+
: beginProxyRequest();
|
|
753
798
|
const finish = () => {
|
|
754
799
|
finishActivity();
|
|
755
800
|
requestMetadata.delete(c.req.raw);
|
|
@@ -924,7 +969,34 @@ export async function createProxyStartApp(params) {
|
|
|
924
969
|
},
|
|
925
970
|
}, 502);
|
|
926
971
|
});
|
|
927
|
-
|
|
972
|
+
app.post("/internal/update-control", async (c) => {
|
|
973
|
+
const suppliedToken = c.req.header("x-neurolink-update-token");
|
|
974
|
+
const expectedToken = params.updateControlToken ?? PROXY_UPDATE_CONTROL_TOKEN;
|
|
975
|
+
if (!suppliedToken || suppliedToken !== expectedToken) {
|
|
976
|
+
return c.json({ error: "not_found" }, 404);
|
|
977
|
+
}
|
|
978
|
+
const payload = await c.req
|
|
979
|
+
.json()
|
|
980
|
+
.catch(() => ({ action: undefined }));
|
|
981
|
+
if (payload.action === "drain") {
|
|
982
|
+
if (!markProxyDrainingForUpdate(readiness)) {
|
|
983
|
+
return c.json({ error: "proxy_not_ready" }, 409);
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
else if (payload.action === "resume") {
|
|
987
|
+
if (!resumeProxyConnections(readiness)) {
|
|
988
|
+
return c.json({ error: "proxy_not_ready" }, 409);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
else {
|
|
992
|
+
return c.json({ error: "invalid_action" }, 400);
|
|
993
|
+
}
|
|
994
|
+
return c.json({
|
|
995
|
+
draining: readiness.drainingForUpdate,
|
|
996
|
+
acceptingConnections: readiness.acceptingConnections,
|
|
997
|
+
});
|
|
998
|
+
});
|
|
999
|
+
registerProxyRequestTracking(app, requestMetadata, readiness);
|
|
928
1000
|
const runtimeConfigStore = params.runtimeConfigStore;
|
|
929
1001
|
const runtimeConfigProvider = runtimeConfigStore
|
|
930
1002
|
? () => runtimeConfigStore.getSnapshot()
|
|
@@ -940,6 +1012,24 @@ export async function createProxyStartApp(params) {
|
|
|
940
1012
|
for (const route of allProxyRoutes) {
|
|
941
1013
|
const method = route.method.toLowerCase();
|
|
942
1014
|
app[method](route.path, async (c) => {
|
|
1015
|
+
const metadata = requestMetadata.get(c.req.raw);
|
|
1016
|
+
if (metadata?.rejectForUpdate) {
|
|
1017
|
+
if (metadata) {
|
|
1018
|
+
metadata.terminalErrorType = "proxy_draining";
|
|
1019
|
+
await recordRuntimeError(metadata, 503, "proxy_draining", "Proxy is draining for an automatic update", {
|
|
1020
|
+
clientMessage: "Proxy is restarting; retry shortly",
|
|
1021
|
+
clientErrorType: "overloaded_error",
|
|
1022
|
+
});
|
|
1023
|
+
}
|
|
1024
|
+
c.header("Retry-After", "2");
|
|
1025
|
+
return c.json({
|
|
1026
|
+
type: "error",
|
|
1027
|
+
error: {
|
|
1028
|
+
type: "overloaded_error",
|
|
1029
|
+
message: "Proxy is restarting; retry shortly",
|
|
1030
|
+
},
|
|
1031
|
+
}, 503);
|
|
1032
|
+
}
|
|
943
1033
|
const emptyBody = {};
|
|
944
1034
|
let body;
|
|
945
1035
|
let rawBody;
|
|
@@ -970,7 +1060,6 @@ export async function createProxyStartApp(params) {
|
|
|
970
1060
|
const toolCount = Array.isArray(bodyRec?.tools)
|
|
971
1061
|
? bodyRec.tools.length
|
|
972
1062
|
: 0;
|
|
973
|
-
const metadata = requestMetadata.get(c.req.raw);
|
|
974
1063
|
if (metadata) {
|
|
975
1064
|
metadata.model = String(model);
|
|
976
1065
|
metadata.stream = stream === "stream";
|
|
@@ -1175,6 +1264,7 @@ export async function createProxyStartApp(params) {
|
|
|
1175
1264
|
liveVersion: PROXY_VERSION,
|
|
1176
1265
|
latestVersion: updateState?.lastCheckVersion || null,
|
|
1177
1266
|
pendingRestartVersion: updateState?.pendingRestartVersion ?? null,
|
|
1267
|
+
deferredUpdate: updateState?.deferredUpdate ?? null,
|
|
1178
1268
|
lastCheckAt: updateState?.lastCheckAt ?? null,
|
|
1179
1269
|
lastUpdateAt: updateState?.lastUpdateAt ?? null,
|
|
1180
1270
|
lastUpdateVersion: updateState?.lastUpdateVersion ?? null,
|
|
@@ -1978,6 +2068,7 @@ export const proxyStatusCommand = {
|
|
|
1978
2068
|
updaterRunning: false,
|
|
1979
2069
|
latestVersion: updateState?.lastCheckVersion || null,
|
|
1980
2070
|
pendingRestartVersion: updateState?.pendingRestartVersion ?? null,
|
|
2071
|
+
deferredUpdate: updateState?.deferredUpdate ?? null,
|
|
1981
2072
|
lastUpdateFailure: updateState?.lastFailure ?? null,
|
|
1982
2073
|
};
|
|
1983
2074
|
if (state && isProcessRunning(state.pid)) {
|
|
@@ -2057,6 +2148,12 @@ export const proxyStatusCommand = {
|
|
|
2057
2148
|
if (status.pendingRestartVersion) {
|
|
2058
2149
|
logger.always(` ${chalk.bold("Pending:")} ${chalk.yellow(`v${status.pendingRestartVersion} installed; restart pending`)}`);
|
|
2059
2150
|
}
|
|
2151
|
+
if (status.deferredUpdate) {
|
|
2152
|
+
const active = status.deferredUpdate.activeRequests === null
|
|
2153
|
+
? "unknown activity"
|
|
2154
|
+
: `${status.deferredUpdate.activeRequests} active request(s)`;
|
|
2155
|
+
logger.always(` ${chalk.bold("Deferred:")} ${chalk.yellow(`v${status.deferredUpdate.version} ${status.deferredUpdate.reason} (${active})`)}`);
|
|
2156
|
+
}
|
|
2060
2157
|
if (status.latestVersion) {
|
|
2061
2158
|
logger.always(` ${chalk.bold("Latest:")} ${chalk.cyan(`v${status.latestVersion}`)}`);
|
|
2062
2159
|
}
|
|
@@ -2239,6 +2336,10 @@ export const proxyGuardCommand = {
|
|
|
2239
2336
|
// foreground fail-open guard remains cleanup-only.
|
|
2240
2337
|
const UPDATE_CHECK_INTERVAL_MS = 2 * 60 * 60 * 1000; // 2 hours
|
|
2241
2338
|
const QUIET_THRESHOLD_MS = 120 * 1000; // 2 minutes of silence
|
|
2339
|
+
const NATURAL_WINDOW_WAIT_MS = 10 * 60 * 1000; // prefer no admission pause
|
|
2340
|
+
const UPDATE_DRAIN_TIMEOUT_MS = 30 * 60 * 1000; // preserve long streams
|
|
2341
|
+
const UPDATE_RETRY_DELAY_MS = 5 * 60 * 1000;
|
|
2342
|
+
const UPDATE_ACTIVITY_POLL_MS = 10 * 1000;
|
|
2242
2343
|
const UPDATE_TIMEOUT_MS = 30 * 1000; // 30 seconds to come healthy
|
|
2243
2344
|
// Get running version from /health endpoint (with timeout to avoid hanging)
|
|
2244
2345
|
let runningVersion = PROXY_VERSION; // fallback
|
|
@@ -2261,6 +2362,7 @@ export const proxyGuardCommand = {
|
|
|
2261
2362
|
let guardStopping = false;
|
|
2262
2363
|
let updateCheckTimeout;
|
|
2263
2364
|
let updateCheckInterval;
|
|
2365
|
+
let updateRetryTimeout;
|
|
2264
2366
|
const stopUpdateChecks = () => {
|
|
2265
2367
|
guardStopping = true;
|
|
2266
2368
|
if (updateCheckTimeout) {
|
|
@@ -2271,6 +2373,10 @@ export const proxyGuardCommand = {
|
|
|
2271
2373
|
clearInterval(updateCheckInterval);
|
|
2272
2374
|
updateCheckInterval = undefined;
|
|
2273
2375
|
}
|
|
2376
|
+
if (updateRetryTimeout) {
|
|
2377
|
+
clearTimeout(updateRetryTimeout);
|
|
2378
|
+
updateRetryTimeout = undefined;
|
|
2379
|
+
}
|
|
2274
2380
|
};
|
|
2275
2381
|
/** Keep state-write failures observable without terminating the updater. */
|
|
2276
2382
|
const persistUpdaterState = (operation, action) => {
|
|
@@ -2289,41 +2395,93 @@ export const proxyGuardCommand = {
|
|
|
2289
2395
|
}
|
|
2290
2396
|
updateInProgress = true;
|
|
2291
2397
|
let updateVersion = runningVersion;
|
|
2398
|
+
let drainActive = false;
|
|
2292
2399
|
try {
|
|
2293
2400
|
// Lazy-load update modules so they're only imported at check time
|
|
2294
2401
|
const { checkForUpdate } = await import("../../lib/proxy/updateChecker.js");
|
|
2295
2402
|
// 1. Check for update
|
|
2296
|
-
|
|
2403
|
+
let result = await checkForUpdate(runningVersion);
|
|
2297
2404
|
updateVersion = result.latestVersion;
|
|
2298
2405
|
persistUpdaterState("record update check", () => recordCheck(result.latestVersion));
|
|
2299
2406
|
if (!result.updateAvailable) {
|
|
2407
|
+
persistUpdaterState("clear update deferral", () => clearUpdateDeferral());
|
|
2300
2408
|
return;
|
|
2301
2409
|
}
|
|
2302
|
-
const
|
|
2303
|
-
if (isVersionSuppressed(result.latestVersion) &&
|
|
2410
|
+
const initiallyPendingRestart = loadUpdateState()?.pendingRestartVersion === result.latestVersion;
|
|
2411
|
+
if (isVersionSuppressed(result.latestVersion) &&
|
|
2412
|
+
!initiallyPendingRestart) {
|
|
2304
2413
|
logger.debug(`[guard] version ${result.latestVersion} is suppressed, skipping`);
|
|
2305
2414
|
return;
|
|
2306
2415
|
}
|
|
2307
2416
|
logger.always(`[updater] update available: ${runningVersion} → ${result.latestVersion}`);
|
|
2308
|
-
// 2.
|
|
2309
|
-
// a
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2417
|
+
// 2. Prefer a naturally quiet period. Sustained traffic is handled by
|
|
2418
|
+
// a bounded graceful drain so update checks cannot wait forever.
|
|
2419
|
+
let lastDeferralSignature = "";
|
|
2420
|
+
const waitForWindow = async (quietWaitMs) => {
|
|
2421
|
+
const window = await waitForProxyUpdateWindow({
|
|
2422
|
+
quietThresholdMs: QUIET_THRESHOLD_MS,
|
|
2423
|
+
quietWaitMs,
|
|
2424
|
+
drainWaitMs: UPDATE_DRAIN_TIMEOUT_MS,
|
|
2425
|
+
pollIntervalMs: UPDATE_ACTIVITY_POLL_MS,
|
|
2426
|
+
getActivity: () => getProxyRuntimeActivity(host, port),
|
|
2427
|
+
setDraining: async (draining) => {
|
|
2428
|
+
const changed = await setProxyUpdateDrain(host, port, draining);
|
|
2429
|
+
if (changed) {
|
|
2430
|
+
logger.always(`[updater] ${draining ? "draining new inference requests" : "inference admission resumed"}`);
|
|
2431
|
+
}
|
|
2432
|
+
return changed;
|
|
2433
|
+
},
|
|
2434
|
+
isStopping: () => guardStopping,
|
|
2435
|
+
isParentAlive: () => getProcessStatus(parentPid) !== "not_running",
|
|
2436
|
+
onPhase: (phase, activity) => {
|
|
2437
|
+
const reason = phase === "waiting_for_quiet" && !activity
|
|
2438
|
+
? "activity_unavailable"
|
|
2439
|
+
: phase;
|
|
2440
|
+
const signature = `${reason}:${activity?.activeRequests ?? "unknown"}`;
|
|
2441
|
+
if (signature !== lastDeferralSignature) {
|
|
2442
|
+
lastDeferralSignature = signature;
|
|
2443
|
+
persistUpdaterState("record update deferral", () => recordUpdateDeferred(result.latestVersion, reason, activity?.activeRequests ?? null));
|
|
2444
|
+
}
|
|
2445
|
+
logger.debug(`[updater] ${reason} (${activity?.activeRequests ?? "unknown"} active requests)`);
|
|
2446
|
+
},
|
|
2447
|
+
});
|
|
2448
|
+
drainActive = drainActive || window.draining;
|
|
2449
|
+
return window;
|
|
2450
|
+
};
|
|
2451
|
+
let updateWindow = await waitForWindow(NATURAL_WINDOW_WAIT_MS);
|
|
2452
|
+
if (!updateWindow.ready) {
|
|
2453
|
+
if (updateWindow.reason === "drain_failed") {
|
|
2454
|
+
persistUpdaterState("record unavailable update drain", () => recordUpdateDeferred(result.latestVersion, "drain_unavailable", null));
|
|
2315
2455
|
}
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2456
|
+
scheduleUpdateRetry();
|
|
2457
|
+
return;
|
|
2458
|
+
}
|
|
2459
|
+
// Close the small race between observing zero activity and package
|
|
2460
|
+
// mutation by holding admission closed through install and restart.
|
|
2461
|
+
if (!drainActive) {
|
|
2462
|
+
updateWindow = await waitForWindow(0);
|
|
2463
|
+
if (!updateWindow.ready) {
|
|
2464
|
+
scheduleUpdateRetry();
|
|
2465
|
+
return;
|
|
2320
2466
|
}
|
|
2321
|
-
logger.debug(`[updater] waiting for idle traffic (${activity?.activeRequests ?? "unknown"} active requests)`);
|
|
2322
|
-
await new Promise((r) => setTimeout(r, quietPollMs));
|
|
2323
2467
|
}
|
|
2324
|
-
|
|
2468
|
+
// Refresh after waiting so a long deferral can never install a stale
|
|
2469
|
+
// target while a newer release is already available.
|
|
2470
|
+
const refreshedResult = await checkForUpdate(runningVersion);
|
|
2471
|
+
result = refreshedResult;
|
|
2472
|
+
updateVersion = result.latestVersion;
|
|
2473
|
+
persistUpdaterState("record refreshed update check", () => recordCheck(result.latestVersion));
|
|
2474
|
+
if (!result.updateAvailable) {
|
|
2475
|
+
persistUpdaterState("clear update deferral", () => clearUpdateDeferral());
|
|
2325
2476
|
return;
|
|
2326
2477
|
}
|
|
2478
|
+
const pendingRestart = loadUpdateState()?.pendingRestartVersion === result.latestVersion;
|
|
2479
|
+
if (isVersionSuppressed(result.latestVersion) && !pendingRestart) {
|
|
2480
|
+
logger.debug(`[guard] refreshed version ${result.latestVersion} is suppressed, skipping`);
|
|
2481
|
+
return;
|
|
2482
|
+
}
|
|
2483
|
+
persistUpdaterState("clear update deferral", () => clearUpdateDeferral());
|
|
2484
|
+
logger.always(`[updater] safe update window acquired for v${result.latestVersion}`);
|
|
2327
2485
|
// 3. Install update (validate version string before passing to shell)
|
|
2328
2486
|
if (!/^\d+\.\d+\.\d+$/.test(result.latestVersion)) {
|
|
2329
2487
|
const message = `invalid version format: ${result.latestVersion}`;
|
|
@@ -2396,22 +2554,8 @@ export const proxyGuardCommand = {
|
|
|
2396
2554
|
if (guardStopping || getProcessStatus(parentPid) === "not_running") {
|
|
2397
2555
|
return;
|
|
2398
2556
|
}
|
|
2399
|
-
//
|
|
2400
|
-
//
|
|
2401
|
-
while (!guardStopping) {
|
|
2402
|
-
if (getProcessStatus(parentPid) === "not_running") {
|
|
2403
|
-
return;
|
|
2404
|
-
}
|
|
2405
|
-
const activity = await getProxyRuntimeActivity(host, port);
|
|
2406
|
-
if (activity && isSafeUpdateWindow(activity, QUIET_THRESHOLD_MS)) {
|
|
2407
|
-
break;
|
|
2408
|
-
}
|
|
2409
|
-
logger.debug(`[updater] update installed; deferring restart (${activity?.activeRequests ?? "unknown"} active requests)`);
|
|
2410
|
-
await sleep(10_000);
|
|
2411
|
-
}
|
|
2412
|
-
if (guardStopping) {
|
|
2413
|
-
return;
|
|
2414
|
-
}
|
|
2557
|
+
// Admission has remained closed since active requests reached zero, so
|
|
2558
|
+
// restart cannot interrupt an accepted request or long-lived stream.
|
|
2415
2559
|
// Signal the health loop to not exit when it detects
|
|
2416
2560
|
// the parent PID is gone — we're intentionally restarting.
|
|
2417
2561
|
updateRestartInProgress = true;
|
|
@@ -2432,6 +2576,7 @@ export const proxyGuardCommand = {
|
|
|
2432
2576
|
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "restart", msg));
|
|
2433
2577
|
return;
|
|
2434
2578
|
}
|
|
2579
|
+
drainActive = false;
|
|
2435
2580
|
// 5. Wait for healthy restart
|
|
2436
2581
|
let healthy = false;
|
|
2437
2582
|
const restartStart = Date.now();
|
|
@@ -2473,9 +2618,25 @@ export const proxyGuardCommand = {
|
|
|
2473
2618
|
persistUpdaterState("record update failure", () => recordUpdateFailure(updateVersion, "check", message));
|
|
2474
2619
|
}
|
|
2475
2620
|
finally {
|
|
2621
|
+
if (drainActive && !updateRestartInProgress) {
|
|
2622
|
+
const resumed = await resumeProxyUpdateDrain(host, port);
|
|
2623
|
+
if (!resumed && getProcessStatus(parentPid) !== "not_running") {
|
|
2624
|
+
logger.always(`[updater] WARNING: failed to resume inference admission after deferred update`);
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2476
2627
|
updateInProgress = false;
|
|
2477
2628
|
}
|
|
2478
2629
|
};
|
|
2630
|
+
const scheduleUpdateRetry = () => {
|
|
2631
|
+
if (guardStopping || updateRetryTimeout) {
|
|
2632
|
+
return;
|
|
2633
|
+
}
|
|
2634
|
+
updateRetryTimeout = setTimeout(() => {
|
|
2635
|
+
updateRetryTimeout = undefined;
|
|
2636
|
+
void runUpdateCheck();
|
|
2637
|
+
}, UPDATE_RETRY_DELAY_MS);
|
|
2638
|
+
updateRetryTimeout.unref?.();
|
|
2639
|
+
};
|
|
2479
2640
|
// Run first check after a short delay, then on interval
|
|
2480
2641
|
if (canAutoUpdate) {
|
|
2481
2642
|
updateCheckTimeout = setTimeout(runUpdateCheck, 30_000);
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import chalk from "chalk";
|
|
2
|
+
import { analyzeProxyLogs } from "../../lib/proxy/proxyAnalysis.js";
|
|
3
|
+
import { logger } from "../../lib/utils/logger.js";
|
|
4
|
+
function formatLatency(label, summary) {
|
|
5
|
+
const value = (amount) => amount === null ? "-" : amount.toFixed(1);
|
|
6
|
+
return `${label.padEnd(22)} ${String(summary.count).padStart(7)} ${value(summary.p50).padStart(9)} ${value(summary.p95).padStart(9)} ${value(summary.p99).padStart(9)} ${value(summary.max).padStart(9)}`;
|
|
7
|
+
}
|
|
8
|
+
function printAnalysis(report) {
|
|
9
|
+
logger.always("");
|
|
10
|
+
logger.always(chalk.bold.cyan("NeuroLink Proxy Analysis"));
|
|
11
|
+
logger.always(chalk.gray("=".repeat(50)));
|
|
12
|
+
logger.always(` Since: ${chalk.cyan(report.since)}`);
|
|
13
|
+
logger.always(` Logs: ${chalk.cyan(report.logsDir)}`);
|
|
14
|
+
logger.always(` Files: ${report.files.requests} request, ${report.files.attempts} attempt, ${report.files.lifecycle} lifecycle`);
|
|
15
|
+
logger.always("");
|
|
16
|
+
logger.always(chalk.bold(" Reliability"));
|
|
17
|
+
logger.always(report.coverage.finalRequests
|
|
18
|
+
? ` Completed: ${report.requests.completed} (${chalk.green(`${report.requests.success} success`)}, ${chalk.red(`${report.requests.errors} errors`)})`
|
|
19
|
+
: chalk.yellow(" Completed: unavailable (no final request logs)"));
|
|
20
|
+
logger.always(report.coverage.attempts && report.coverage.finalRequests
|
|
21
|
+
? ` Recovered after retry: ${report.requests.recoveredAfterRetry}`
|
|
22
|
+
: chalk.yellow(" Recovered after retry: unavailable"));
|
|
23
|
+
if (report.requests.errors > 0) {
|
|
24
|
+
logger.always(` Final error types: ${JSON.stringify(report.requests.errorTypes)}`);
|
|
25
|
+
}
|
|
26
|
+
if (report.coverage.attempts) {
|
|
27
|
+
logger.always(` Attempts: ${report.attempts.total}, ${report.attempts.errors} errors${report.attempts.errors > 0 ? ` ${JSON.stringify(report.attempts.errorTypes)}` : ""}`);
|
|
28
|
+
}
|
|
29
|
+
logger.always(report.coverage.lifecycle
|
|
30
|
+
? ` Lifecycle: ${report.lifecycle.accepted} accepted, ${report.lifecycle.terminal} terminal, ${report.lifecycle.unsettled} unsettled`
|
|
31
|
+
: chalk.yellow(" Lifecycle: unavailable (no lifecycle metadata)"));
|
|
32
|
+
if (report.coverage.attempts) {
|
|
33
|
+
const finalRateLimits = report.coverage.finalRequests
|
|
34
|
+
? `${report.requests.finalRateLimits} final`
|
|
35
|
+
: "final unavailable";
|
|
36
|
+
logger.always(` Rate limits: ${report.rateLimits.attemptRateLimits} attempts (${report.rateLimits.quota} quota, ${report.rateLimits.transient} transient, ${report.rateLimits.unclassified} unclassified), ${finalRateLimits}`);
|
|
37
|
+
}
|
|
38
|
+
else if (report.coverage.finalRequests) {
|
|
39
|
+
logger.always(chalk.yellow(` Rate limits: attempt classification unavailable; ${report.requests.finalRateLimits} final 429 response(s)`));
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
logger.always(chalk.yellow(" Rate limits: unavailable"));
|
|
43
|
+
}
|
|
44
|
+
logger.always("");
|
|
45
|
+
logger.always(chalk.bold(" Latency (ms)"));
|
|
46
|
+
const latencyHeader = `${"METRIC".padEnd(22)} ${"COUNT".padStart(7)} ${"P50".padStart(9)} ${"P95".padStart(9)} ${"P99".padStart(9)} ${"MAX".padStart(9)}`;
|
|
47
|
+
logger.always(` ${chalk.gray(latencyHeader)}`);
|
|
48
|
+
logger.always(` ${chalk.gray("-".repeat(latencyHeader.length))}`);
|
|
49
|
+
for (const [label, summary] of [
|
|
50
|
+
["Response headers", report.latencyMs.headers],
|
|
51
|
+
["First chunk", report.latencyMs.firstChunk],
|
|
52
|
+
["Terminal", report.latencyMs.terminal],
|
|
53
|
+
["Final request log", report.latencyMs.finalRequest],
|
|
54
|
+
["Account attempt", report.latencyMs.attempt],
|
|
55
|
+
["Single-attempt delta", report.latencyMs.singleAttemptDelta],
|
|
56
|
+
]) {
|
|
57
|
+
logger.always(` ${formatLatency(label, summary)}`);
|
|
58
|
+
}
|
|
59
|
+
logger.always("");
|
|
60
|
+
logger.always(chalk.bold(" Cache"));
|
|
61
|
+
if (report.coverage.cacheUsage) {
|
|
62
|
+
logger.always(` Usage records: ${report.cache.requestsWithUsage}, cache-read requests: ${report.cache.requestsWithCacheRead}, hit rate: ${report.cache.requestHitRate === null ? "-" : `${(report.cache.requestHitRate * 100).toFixed(1)}%`}`);
|
|
63
|
+
logger.always(` Tokens: ${report.cache.cacheReadTokens} read, ${report.cache.cacheCreationTokens} created, ${report.cache.inputTokens} input`);
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
66
|
+
logger.always(chalk.yellow(" Cache usage: unavailable"));
|
|
67
|
+
}
|
|
68
|
+
logger.always("");
|
|
69
|
+
logger.always(chalk.bold(" Data Quality"));
|
|
70
|
+
logger.always(` ${report.dataQuality.linesRead} lines scanned, ${report.dataQuality.malformedLines} malformed, ${report.dataQuality.unsupportedLifecycleLines} unsupported lifecycle, ${report.dataQuality.lifecycleSequenceGaps} sequence gaps, ${report.dataQuality.lifecycleSequenceDuplicates} duplicates`);
|
|
71
|
+
if (report.accounts.length > 0) {
|
|
72
|
+
logger.always("");
|
|
73
|
+
logger.always(chalk.bold(" Accounts"));
|
|
74
|
+
const header = `${"ACCOUNT".padEnd(30)} ${"AUTH".padEnd(8)} ${"ATTEMPT".padStart(8)} ${"ATT ERR".padStart(7)} ${"FINAL".padStart(7)} ${"FIN ERR".padStart(7)} ${"QUOTA".padStart(7)} ${"TRANS".padStart(7)} ${"UNKNOWN".padStart(8)}`;
|
|
75
|
+
logger.always(` ${chalk.gray(header)}`);
|
|
76
|
+
logger.always(` ${chalk.gray("-".repeat(header.length))}`);
|
|
77
|
+
for (const account of report.accounts) {
|
|
78
|
+
const attempt = report.coverage.attempts ? String(account.attempts) : "-";
|
|
79
|
+
const attemptErrors = report.coverage.attempts
|
|
80
|
+
? String(account.attemptErrors)
|
|
81
|
+
: "-";
|
|
82
|
+
const quota = report.coverage.attempts
|
|
83
|
+
? String(account.quotaRateLimits)
|
|
84
|
+
: "-";
|
|
85
|
+
const transient = report.coverage.attempts
|
|
86
|
+
? String(account.transientRateLimits)
|
|
87
|
+
: "-";
|
|
88
|
+
const unknown = report.coverage.attempts
|
|
89
|
+
? String(account.unclassifiedRateLimits)
|
|
90
|
+
: "-";
|
|
91
|
+
const finalRequests = report.coverage.finalRequests
|
|
92
|
+
? String(account.finalRequests)
|
|
93
|
+
: "-";
|
|
94
|
+
const finalErrors = report.coverage.finalRequests
|
|
95
|
+
? String(account.finalErrors)
|
|
96
|
+
: "-";
|
|
97
|
+
logger.always(` ${account.account.slice(0, 30).padEnd(30)} ${account.accountType.slice(0, 8).padEnd(8)} ${attempt.padStart(8)} ${attemptErrors.padStart(7)} ${finalRequests.padStart(7)} ${finalErrors.padStart(7)} ${quota.padStart(7)} ${transient.padStart(7)} ${unknown.padStart(8)}`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
logger.always("");
|
|
101
|
+
}
|
|
102
|
+
export const proxyAnalyzeCommand = {
|
|
103
|
+
command: "analyze",
|
|
104
|
+
describe: "Analyze local proxy reliability and latency logs",
|
|
105
|
+
builder: (yargs) => yargs
|
|
106
|
+
.option("logs-dir", {
|
|
107
|
+
type: "string",
|
|
108
|
+
alias: "logsDir",
|
|
109
|
+
description: "Proxy JSONL log directory",
|
|
110
|
+
})
|
|
111
|
+
.option("since", {
|
|
112
|
+
type: "string",
|
|
113
|
+
default: "24h",
|
|
114
|
+
description: "ISO timestamp or lookback such as 6h, 1d, or 1w",
|
|
115
|
+
})
|
|
116
|
+
.option("format", {
|
|
117
|
+
type: "string",
|
|
118
|
+
choices: ["text", "json"],
|
|
119
|
+
default: "text",
|
|
120
|
+
description: "Output format",
|
|
121
|
+
})
|
|
122
|
+
.option("quiet", {
|
|
123
|
+
type: "boolean",
|
|
124
|
+
alias: "q",
|
|
125
|
+
default: false,
|
|
126
|
+
description: "Suppress non-essential output",
|
|
127
|
+
})
|
|
128
|
+
.example("neurolink proxy analyze --since 24h", "Analyze the last 24 hours of proxy logs"),
|
|
129
|
+
handler: async (argv) => {
|
|
130
|
+
try {
|
|
131
|
+
const report = await analyzeProxyLogs({
|
|
132
|
+
logsDir: argv.logsDir,
|
|
133
|
+
since: argv.since,
|
|
134
|
+
});
|
|
135
|
+
if (argv.format === "json") {
|
|
136
|
+
logger.always(JSON.stringify(report, null, 2));
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
printAnalysis(report);
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
logger.error(chalk.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
143
|
+
process.exitCode = 1;
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
//# sourceMappingURL=proxyAnalyze.js.map
|
package/dist/cli/parser.js
CHANGED
|
@@ -14,6 +14,7 @@ import { ragCommand } from "./commands/rag.js";
|
|
|
14
14
|
import { ObservabilityCommandFactory } from "./commands/observability.js";
|
|
15
15
|
import { TelemetryCommandFactory } from "./commands/telemetry.js";
|
|
16
16
|
import { proxyStartCommand, proxyStatusCommand, proxyTelemetryCommand, proxySetupCommand, proxyGuardCommand, proxyInstallCommand, proxyUninstallCommand, } from "./commands/proxy.js";
|
|
17
|
+
import { proxyAnalyzeCommand } from "./commands/proxyAnalyze.js";
|
|
17
18
|
import { EvaluateCommandFactory } from "./commands/evaluate.js";
|
|
18
19
|
import { TaskCommandFactory } from "./commands/task.js";
|
|
19
20
|
import { AutoresearchCommandFactory } from "./commands/autoresearch.js";
|
|
@@ -200,12 +201,13 @@ export function initializeCliParser() {
|
|
|
200
201
|
builder: (yargs) => yargs
|
|
201
202
|
.command(proxyStartCommand)
|
|
202
203
|
.command(proxyStatusCommand)
|
|
204
|
+
.command(proxyAnalyzeCommand)
|
|
203
205
|
.command(proxyTelemetryCommand)
|
|
204
206
|
.command(proxySetupCommand)
|
|
205
207
|
.command(proxyGuardCommand)
|
|
206
208
|
.command(proxyInstallCommand)
|
|
207
209
|
.command(proxyUninstallCommand)
|
|
208
|
-
.demandCommand(1, "Please specify a proxy subcommand: start, status, telemetry <setup|start|stop|status|logs|import-dashboard>, setup, guard, install, or uninstall"),
|
|
210
|
+
.demandCommand(1, "Please specify a proxy subcommand: start, status, analyze, telemetry <setup|start|stop|status|logs|import-dashboard>, setup, guard, install, or uninstall"),
|
|
209
211
|
handler: () => { },
|
|
210
212
|
})
|
|
211
213
|
// Evaluate Command Group - Using EvaluateCommandFactory
|