@juspay/neurolink 12.14.5 → 12.14.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +412 -412
- package/dist/cli/commands/proxy.d.ts +1 -1
- package/dist/cli/commands/proxy.js +123 -14
- package/dist/cli/commands/proxyRestart.d.ts +3 -0
- package/dist/cli/commands/proxyRestart.js +88 -0
- package/dist/cli/parser.js +3 -1
- package/dist/proxy/bodyCaptureWorker.js +26 -1
- package/dist/proxy/codexUsage.js +83 -2
- package/dist/proxy/otelLogSink.d.ts +13 -0
- package/dist/proxy/otelLogSink.js +62 -1
- package/dist/proxy/proxyTraceContext.d.ts +21 -0
- package/dist/proxy/proxyTraceContext.js +47 -0
- package/dist/proxy/proxyTracer.d.ts +9 -5
- package/dist/proxy/proxyTracer.js +81 -2
- package/dist/proxy/requestLogger.js +24 -18
- package/dist/proxy/restartControl.d.ts +12 -0
- package/dist/proxy/restartControl.js +283 -0
- package/dist/proxy/rollingWorkerSupervisor.js +8 -0
- package/dist/server/routes/codexProxyRoutes.js +414 -332
- package/dist/types/cli.d.ts +7 -1
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/proxy.d.ts +119 -3
- package/dist/types/proxyRestart.d.ts +49 -0
- package/dist/types/proxyRestart.js +1 -0
- package/docs-site/static/search-index.json +1 -1
- package/package.json +2 -1
- package/scripts/observability/check-proxy-telemetry.mjs +29 -241
- package/scripts/observability/proxy-telemetry-backend.mjs +202 -0
- package/scripts/observability/proxy-telemetry-check.mjs +544 -0
- package/scripts/observability/query-proxy-history.mjs +30 -19
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import type { CommandModule } from "yargs";
|
|
13
13
|
import type { Hono } from "hono";
|
|
14
|
-
import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyHealthProbe, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxySupervisorState, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../types/index.js";
|
|
14
|
+
import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyHealthProbe, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxyRestartSupervisorState as ProxySupervisorState, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../types/index.js";
|
|
15
15
|
import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
|
|
16
16
|
/**
|
|
17
17
|
* Drop a supervisor `version` that is not a string.
|
|
@@ -12,6 +12,7 @@ import { applyAllClients, restoreAllClients, } from "../proxy-clients/registry.j
|
|
|
12
12
|
import { resolveProxyConfigPath } from "../../proxy/proxyConfig.js";
|
|
13
13
|
import { redactUrlsInText, sanitizeForLog, } from "../../utils/logSanitize.js";
|
|
14
14
|
import { withTimeout } from "../../utils/async/withTimeout.js";
|
|
15
|
+
import { startProxyHttpTrace } from "../../proxy/proxyTracer.js";
|
|
15
16
|
import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
|
|
16
17
|
import { configureProxyKeepAliveDispatcher } from "../../proxy/proxyDispatcher.js";
|
|
17
18
|
import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
|
|
@@ -24,6 +25,7 @@ import { describeInstallFailure, getGlobalInstallArgs, isTransientInstallFailure
|
|
|
24
25
|
import { startUpdaterWorkerSupervisor } from "../../proxy/updaterSupervisor.js";
|
|
25
26
|
import { openProxyWorkerLog } from "../../proxy/workerLog.js";
|
|
26
27
|
import { startRollingProxyServer } from "../../proxy/rollingProxyServer.js";
|
|
28
|
+
import { startProxyRestartControl } from "../../proxy/restartControl.js";
|
|
27
29
|
import { isProxyAuxiliaryRequest } from "../../proxy/proxyRequestKind.js";
|
|
28
30
|
import { spawnProxySocketWorker } from "../../proxy/rollingWorkerProcess.js";
|
|
29
31
|
import { PROXY_ROLLING_SUPERVISOR_ENV, PROXY_SOCKET_WORKER_ENV, } from "../../proxy/rollingWorkerProtocol.js";
|
|
@@ -836,13 +838,36 @@ function spawnProxyUpdater(host, port, parentPid, rollingSupervisor = false) {
|
|
|
836
838
|
workerLog.close();
|
|
837
839
|
}
|
|
838
840
|
}
|
|
839
|
-
async function runProxyTelemetryManager(command) {
|
|
841
|
+
async function runProxyTelemetryManager(command, argv) {
|
|
840
842
|
const { existsSync } = await import("fs");
|
|
841
843
|
if (!existsSync(PROXY_TELEMETRY_SCRIPT_PATH)) {
|
|
842
844
|
throw new Error("Proxy telemetry helper files were not found in this installation. Reinstall NeuroLink with observability assets included.");
|
|
843
845
|
}
|
|
844
846
|
await new Promise((resolve, reject) => {
|
|
845
|
-
const
|
|
847
|
+
const queryCommand = command === "doctor" || command === "query";
|
|
848
|
+
const queryArgs = [];
|
|
849
|
+
if (queryCommand && argv) {
|
|
850
|
+
for (const [key, option] of [
|
|
851
|
+
["since", "--since"],
|
|
852
|
+
["until", "--until"],
|
|
853
|
+
["kind", "--kind"],
|
|
854
|
+
["format", "--format"],
|
|
855
|
+
["maxRows", "--max-rows"],
|
|
856
|
+
["proxyUrl", "--proxy-url"],
|
|
857
|
+
]) {
|
|
858
|
+
if (argv[key] !== undefined) {
|
|
859
|
+
queryArgs.push(option, String(argv[key]));
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
const child = spawn(queryCommand ? process.execPath : "bash", queryCommand
|
|
864
|
+
? [
|
|
865
|
+
join(dirname(PROXY_TELEMETRY_SCRIPT_PATH), command === "doctor"
|
|
866
|
+
? "check-proxy-telemetry.mjs"
|
|
867
|
+
: "query-proxy-history.mjs"),
|
|
868
|
+
...queryArgs,
|
|
869
|
+
]
|
|
870
|
+
: [PROXY_TELEMETRY_SCRIPT_PATH, command], {
|
|
846
871
|
stdio: "inherit",
|
|
847
872
|
env: process.env,
|
|
848
873
|
});
|
|
@@ -1077,6 +1102,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1077
1102
|
rejectForUpdate: readiness.drainingForUpdate,
|
|
1078
1103
|
};
|
|
1079
1104
|
requestMetadata.set(c.req.raw, metadata);
|
|
1105
|
+
const httpTrace = startProxyHttpTrace(metadata, Object.fromEntries(c.req.raw.headers));
|
|
1080
1106
|
const stopObservingFinalLog = observeProxyFinalLog(metadata.requestId, (entry) => {
|
|
1081
1107
|
metadata.terminalResult = entry;
|
|
1082
1108
|
}, (entry) => {
|
|
@@ -1086,6 +1112,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1086
1112
|
? () => undefined
|
|
1087
1113
|
: beginProxyRequest();
|
|
1088
1114
|
const finish = () => {
|
|
1115
|
+
httpTrace.end(metadata.terminalResult?.responseStatus ?? c.res.status, metadata.terminalResult?.terminalOutcome ?? "unknown", metadata.terminalErrorType);
|
|
1089
1116
|
stopObservingFinalLog();
|
|
1090
1117
|
finishActivity();
|
|
1091
1118
|
// Borrowed traffic holds a concurrency slot for the lifetime of the
|
|
@@ -1098,7 +1125,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1098
1125
|
// them at acceptance instead of publishing misleading placeholder values;
|
|
1099
1126
|
// subsequent events carry the parsed metadata under the same request ID.
|
|
1100
1127
|
try {
|
|
1101
|
-
await persistProxyLifecycleAcceptance({
|
|
1128
|
+
await httpTrace.run(() => persistProxyLifecycleAcceptance({
|
|
1102
1129
|
requestId: metadata.requestId,
|
|
1103
1130
|
method: metadata.method,
|
|
1104
1131
|
path: metadata.path,
|
|
@@ -1106,8 +1133,8 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1106
1133
|
requestBytes,
|
|
1107
1134
|
elapsedMs: 0,
|
|
1108
1135
|
monotonicMs: startedMonotonicMs,
|
|
1109
|
-
});
|
|
1110
|
-
await next
|
|
1136
|
+
}));
|
|
1137
|
+
await httpTrace.run(next);
|
|
1111
1138
|
const responseStatus = c.res.status;
|
|
1112
1139
|
logProxyLifecycleEvent({
|
|
1113
1140
|
event: "response_headers",
|
|
@@ -1168,12 +1195,12 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1168
1195
|
let accountingTimedOut = false;
|
|
1169
1196
|
let accountingFailed = false;
|
|
1170
1197
|
try {
|
|
1171
|
-
accountingFailed = await notifyRouteTerminal({
|
|
1198
|
+
accountingFailed = await httpTrace.run(() => notifyRouteTerminal({
|
|
1172
1199
|
outcome,
|
|
1173
1200
|
error,
|
|
1174
1201
|
observedBodyBytes,
|
|
1175
1202
|
responseChunks,
|
|
1176
|
-
});
|
|
1203
|
+
}));
|
|
1177
1204
|
}
|
|
1178
1205
|
catch {
|
|
1179
1206
|
accountingTimedOut = true;
|
|
@@ -1230,6 +1257,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1230
1257
|
errorCode: final?.errorCode ?? metadata.terminalErrorCode,
|
|
1231
1258
|
});
|
|
1232
1259
|
stopObservingFinalLog();
|
|
1260
|
+
httpTrace.end(final?.responseStatus ?? responseStatus, terminalOutcome, final?.errorType ?? metadata.terminalErrorType);
|
|
1233
1261
|
},
|
|
1234
1262
|
});
|
|
1235
1263
|
}
|
|
@@ -1258,6 +1286,9 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1258
1286
|
errorType: error instanceof Error ? error.name : "unknown_error",
|
|
1259
1287
|
errorCode: getProxyRuntimeErrorCode(error),
|
|
1260
1288
|
});
|
|
1289
|
+
httpTrace.end(getProxyRuntimeErrorCode(error) === "PROXY_TELEMETRY_UNAVAILABLE"
|
|
1290
|
+
? 503
|
|
1291
|
+
: 502, "handler_error", error instanceof Error ? error.name : "unknown_error");
|
|
1261
1292
|
throw error;
|
|
1262
1293
|
}
|
|
1263
1294
|
};
|
|
@@ -1330,8 +1361,9 @@ export async function createProxyStartApp(params) {
|
|
|
1330
1361
|
accountKey: attempt?.accountKey,
|
|
1331
1362
|
provider: attempt?.provider,
|
|
1332
1363
|
transportScope: attempt?.transportScope,
|
|
1333
|
-
traceId: attempt?.traceId,
|
|
1334
|
-
spanId: attempt?.spanId,
|
|
1364
|
+
traceId: attempt?.traceId ?? metadata.traceId,
|
|
1365
|
+
spanId: attempt?.spanId ?? metadata.spanId,
|
|
1366
|
+
traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
|
|
1335
1367
|
responseStatus: status,
|
|
1336
1368
|
responseTimeMs: Date.now() - metadata.startedAt,
|
|
1337
1369
|
errorType,
|
|
@@ -1341,6 +1373,9 @@ export async function createProxyStartApp(params) {
|
|
|
1341
1373
|
logBodyCapture({
|
|
1342
1374
|
timestamp: new Date().toISOString(),
|
|
1343
1375
|
requestId: metadata.requestId,
|
|
1376
|
+
traceId: attempt?.traceId ?? metadata.traceId,
|
|
1377
|
+
spanId: attempt?.spanId ?? metadata.spanId,
|
|
1378
|
+
traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
|
|
1344
1379
|
model: metadata.model,
|
|
1345
1380
|
stream: metadata.stream,
|
|
1346
1381
|
phase: "client_response",
|
|
@@ -1400,6 +1435,15 @@ export async function createProxyStartApp(params) {
|
|
|
1400
1435
|
.json()
|
|
1401
1436
|
.catch(() => ({ action: undefined }));
|
|
1402
1437
|
if (payload.action === "drain") {
|
|
1438
|
+
// A rolling worker must keep admitting until the supervisor has a ready
|
|
1439
|
+
// replacement. The legacy global drain can otherwise strand the listener
|
|
1440
|
+
// behind maintenance responses when its caller stalls or disappears.
|
|
1441
|
+
if (isProxySocketWorkerProcess()) {
|
|
1442
|
+
return c.json({
|
|
1443
|
+
error: "rolling_restart_required",
|
|
1444
|
+
message: "Use neurolink proxy restart; global update drain is disabled for rolling workers.",
|
|
1445
|
+
}, 409);
|
|
1446
|
+
}
|
|
1403
1447
|
if (!markProxyDrainingForUpdate(readiness)) {
|
|
1404
1448
|
return c.json({ error: "proxy_not_ready" }, 409);
|
|
1405
1449
|
}
|
|
@@ -2777,6 +2821,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2777
2821
|
filePrefix: "proxy-supervisor",
|
|
2778
2822
|
});
|
|
2779
2823
|
let currentUpdaterPid;
|
|
2824
|
+
let restartControl;
|
|
2780
2825
|
const rollingServer = await startRollingProxyServer({
|
|
2781
2826
|
onEvent: (event) => logProxyLifecycleEvent({
|
|
2782
2827
|
event: "supervisor_event",
|
|
@@ -2807,6 +2852,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2807
2852
|
version: PROXY_VERSION,
|
|
2808
2853
|
updaterPid: currentUpdaterPid,
|
|
2809
2854
|
rolling: snapshot,
|
|
2855
|
+
restartControl: restartControl?.identity,
|
|
2810
2856
|
});
|
|
2811
2857
|
},
|
|
2812
2858
|
log: (message) => logger.always(message),
|
|
@@ -2843,13 +2889,46 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2843
2889
|
};
|
|
2844
2890
|
process.on("SIGUSR2", activatePendingUpdate);
|
|
2845
2891
|
const readinessHost = host === "0.0.0.0" ? "127.0.0.1" : host;
|
|
2892
|
+
try {
|
|
2893
|
+
restartControl = await startProxyRestartControl({
|
|
2894
|
+
stateDir: join(homedir(), ".neurolink"),
|
|
2895
|
+
server: rollingServer,
|
|
2896
|
+
log: (message) => logger.warn(message),
|
|
2897
|
+
isUpdatePending: () => !!rollingReplacement || !!loadUpdateState()?.pendingRestartVersion,
|
|
2898
|
+
getInstalledVersion: async () => {
|
|
2899
|
+
const { execFile } = await import("node:child_process");
|
|
2900
|
+
const { promisify } = await import("node:util");
|
|
2901
|
+
const { stdout } = await promisify(execFile)(TRAMPOLINE_PATH, ["--version"], { timeout: 5_000, maxBuffer: 65_536 });
|
|
2902
|
+
return stdout.trim();
|
|
2903
|
+
},
|
|
2904
|
+
getStatus: async () => {
|
|
2905
|
+
const probeHost = readinessHost === "::" ? "::1" : readinessHost;
|
|
2906
|
+
const response = await fetch(`http://${probeHost.includes(":") ? `[${probeHost}]` : probeHost}:${rollingServer.address.port}/status`, {
|
|
2907
|
+
headers: { connection: "close" },
|
|
2908
|
+
signal: AbortSignal.timeout(3_000),
|
|
2909
|
+
});
|
|
2910
|
+
if (!response.ok) {
|
|
2911
|
+
throw new Error("Serving status is unavailable.");
|
|
2912
|
+
}
|
|
2913
|
+
return response.json();
|
|
2914
|
+
},
|
|
2915
|
+
});
|
|
2916
|
+
}
|
|
2917
|
+
catch (error) {
|
|
2918
|
+
// A control-plane setup failure must not stop an otherwise serving proxy.
|
|
2919
|
+
logger.warn(`[proxy-supervisor] safe restart control unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
2920
|
+
}
|
|
2846
2921
|
const updatePersistedUpdaterPid = (updaterPid) => {
|
|
2847
2922
|
currentUpdaterPid = updaterPid;
|
|
2848
2923
|
const state = loadProxySupervisorState();
|
|
2849
2924
|
if (!state || state.pid !== process.pid) {
|
|
2850
2925
|
return;
|
|
2851
2926
|
}
|
|
2852
|
-
saveProxySupervisorState({
|
|
2927
|
+
saveProxySupervisorState({
|
|
2928
|
+
...state,
|
|
2929
|
+
updaterPid,
|
|
2930
|
+
restartControl: restartControl?.identity,
|
|
2931
|
+
});
|
|
2853
2932
|
};
|
|
2854
2933
|
const updaterSupervisor = startUpdaterWorkerSupervisor({
|
|
2855
2934
|
spawnWorker: () => spawnProxyUpdater(readinessHost, rollingServer.address.port, process.pid, true),
|
|
@@ -2872,6 +2951,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2872
2951
|
stopping = true;
|
|
2873
2952
|
logger.always(`[proxy-supervisor] shutting down (${signal})`);
|
|
2874
2953
|
process.off("SIGUSR2", activatePendingUpdate);
|
|
2954
|
+
await restartControl?.close();
|
|
2875
2955
|
updaterSupervisor.stop();
|
|
2876
2956
|
await rollingServer.close();
|
|
2877
2957
|
await flushProxyLifecycleEvents().catch((error) => logger.warn(String(error)));
|
|
@@ -3507,6 +3587,8 @@ export const proxyStatusCommand = {
|
|
|
3507
3587
|
// PROXY TELEMETRY COMMAND
|
|
3508
3588
|
// =============================================================================
|
|
3509
3589
|
const PROXY_TELEMETRY_ACTIONS = [
|
|
3590
|
+
"doctor",
|
|
3591
|
+
"query",
|
|
3510
3592
|
"setup",
|
|
3511
3593
|
"start",
|
|
3512
3594
|
"stop",
|
|
@@ -3516,12 +3598,37 @@ const PROXY_TELEMETRY_ACTIONS = [
|
|
|
3516
3598
|
];
|
|
3517
3599
|
export const proxyTelemetryCommand = {
|
|
3518
3600
|
command: "telemetry <action>",
|
|
3519
|
-
describe: "
|
|
3601
|
+
describe: "Query and verify stored OTel telemetry or manage the local observability stack",
|
|
3520
3602
|
builder: (yargs) => yargs
|
|
3521
3603
|
.positional("action", {
|
|
3522
3604
|
type: "string",
|
|
3523
3605
|
choices: [...PROXY_TELEMETRY_ACTIONS],
|
|
3524
|
-
describe: "Telemetry action: setup, start, stop, status, logs, or import-dashboard",
|
|
3606
|
+
describe: "Telemetry action: doctor, query, setup, start, stop, status, logs, or import-dashboard",
|
|
3607
|
+
})
|
|
3608
|
+
.option("since", {
|
|
3609
|
+
type: "string",
|
|
3610
|
+
description: "Inclusive ISO timestamp for stored telemetry",
|
|
3611
|
+
})
|
|
3612
|
+
.option("until", {
|
|
3613
|
+
type: "string",
|
|
3614
|
+
description: "Exclusive ISO timestamp for stored telemetry",
|
|
3615
|
+
})
|
|
3616
|
+
.option("kind", {
|
|
3617
|
+
type: "string",
|
|
3618
|
+
description: "Metadata kind for query, such as request_final or telemetry_delivery",
|
|
3619
|
+
})
|
|
3620
|
+
.option("format", {
|
|
3621
|
+
type: "string",
|
|
3622
|
+
choices: ["json", "text"],
|
|
3623
|
+
description: "Doctor report format; query always returns JSON",
|
|
3624
|
+
})
|
|
3625
|
+
.option("max-rows", {
|
|
3626
|
+
type: "number",
|
|
3627
|
+
description: "Explicit bounded history record limit",
|
|
3628
|
+
})
|
|
3629
|
+
.option("proxy-url", {
|
|
3630
|
+
type: "string",
|
|
3631
|
+
description: "Proxy endpoint used by the read-only doctor",
|
|
3525
3632
|
})
|
|
3526
3633
|
.option("quiet", {
|
|
3527
3634
|
type: "boolean",
|
|
@@ -3529,19 +3636,21 @@ export const proxyTelemetryCommand = {
|
|
|
3529
3636
|
default: false,
|
|
3530
3637
|
description: "Suppress the local CLI spinner and delegate directly",
|
|
3531
3638
|
})
|
|
3639
|
+
.example("neurolink proxy telemetry doctor --format json", "Verify field coverage, stored captures, trace correlation, and delivery evidence")
|
|
3640
|
+
.example("neurolink proxy telemetry query --since 2026-09-15T00:00:00Z --kind request_final", "Read stored OTLP metadata without scanning proxy log files")
|
|
3532
3641
|
.example("neurolink proxy telemetry setup", "Start OpenObserve, start the OTEL collector, and import the dashboard")
|
|
3533
3642
|
.example("neurolink proxy telemetry start", "Start the local proxy telemetry stack without re-importing the dashboard")
|
|
3534
3643
|
.example("neurolink proxy telemetry stop", "Stop the local OpenObserve and OTEL collector containers"),
|
|
3535
3644
|
handler: async (argv) => {
|
|
3536
3645
|
const action = argv.action;
|
|
3537
|
-
const spinner = argv.quiet
|
|
3646
|
+
const spinner = argv.quiet || action === "doctor" || action === "query"
|
|
3538
3647
|
? null
|
|
3539
3648
|
: ora(`Running proxy telemetry ${action}...`).start();
|
|
3540
3649
|
try {
|
|
3541
3650
|
if (spinner) {
|
|
3542
3651
|
spinner.stop();
|
|
3543
3652
|
}
|
|
3544
|
-
await runProxyTelemetryManager(action);
|
|
3653
|
+
await runProxyTelemetryManager(action, argv);
|
|
3545
3654
|
if (spinner) {
|
|
3546
3655
|
spinner.succeed(`proxy telemetry ${action} completed`);
|
|
3547
3656
|
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { readFile, lstat } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { resolveProxyPaths } from "../../proxy/proxyPaths.js";
|
|
5
|
+
import { requestProxyRestart } from "../../proxy/restartControl.js";
|
|
6
|
+
const supervisorSchema = z.object({
|
|
7
|
+
pid: z.number().int().positive(),
|
|
8
|
+
restartControl: z.object({
|
|
9
|
+
protocol: z.literal(1),
|
|
10
|
+
socketPath: z.string(),
|
|
11
|
+
instanceId: z.string().uuid(),
|
|
12
|
+
}),
|
|
13
|
+
});
|
|
14
|
+
export const proxyRestartCommand = {
|
|
15
|
+
command: "restart",
|
|
16
|
+
describe: "Replace the serving worker without stopping the proxy listener",
|
|
17
|
+
builder: (yargs) => yargs
|
|
18
|
+
.option("check", {
|
|
19
|
+
type: "boolean",
|
|
20
|
+
default: false,
|
|
21
|
+
description: "Check restart readiness without changing the running proxy",
|
|
22
|
+
})
|
|
23
|
+
.option("dev", {
|
|
24
|
+
type: "boolean",
|
|
25
|
+
default: false,
|
|
26
|
+
description: "Use the isolated proxy state in .neurolink-dev/",
|
|
27
|
+
})
|
|
28
|
+
.option("format", {
|
|
29
|
+
type: "string",
|
|
30
|
+
choices: ["text", "json"],
|
|
31
|
+
default: "text",
|
|
32
|
+
})
|
|
33
|
+
.example("neurolink proxy restart --check", "Verify the running supervisor supports safe worker replacement")
|
|
34
|
+
.example("neurolink proxy restart", "Start a replacement, verify it, then let existing streams finish"),
|
|
35
|
+
handler: async (argv) => {
|
|
36
|
+
try {
|
|
37
|
+
const { stateDir } = resolveProxyPaths(argv.dev);
|
|
38
|
+
const saved = supervisorSchema.safeParse(JSON.parse(await readFile(join(stateDir, "proxy-supervisor-state.json"), "utf8")));
|
|
39
|
+
if (!saved.success) {
|
|
40
|
+
throw new Error("This running supervisor does not advertise safe restart support. It must first be upgraded through a separately planned service activation.");
|
|
41
|
+
}
|
|
42
|
+
const { pid, restartControl } = saved.data;
|
|
43
|
+
if (restartControl.socketPath !==
|
|
44
|
+
join(stateDir, `restart-${pid}-${restartControl.instanceId}.sock`)) {
|
|
45
|
+
throw new Error("Restart control path does not match the recorded supervisor.");
|
|
46
|
+
}
|
|
47
|
+
const socket = await lstat(restartControl.socketPath);
|
|
48
|
+
if (!socket.isSocket() ||
|
|
49
|
+
(socket.mode & 0o077) !== 0 ||
|
|
50
|
+
(process.getuid && socket.uid !== process.getuid())) {
|
|
51
|
+
throw new Error("Restart control socket ownership or permissions are invalid.");
|
|
52
|
+
}
|
|
53
|
+
const result = await requestProxyRestart(restartControl, argv.check);
|
|
54
|
+
if (result.supervisorPid !== pid) {
|
|
55
|
+
throw new Error("Restart control identity changed; outcome is unverified.");
|
|
56
|
+
}
|
|
57
|
+
if (argv.format === "json") {
|
|
58
|
+
console.info(JSON.stringify(result, null, 2));
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
console.info(result.message);
|
|
62
|
+
console.info(`Supervisor ${result.supervisorPid}; worker ${result.workerPid ?? "unknown"}; version ${result.version ?? "unknown"}; previous workers finishing requests: ${result.drainingWorkers}.`);
|
|
63
|
+
console.info("This operation preserves the supervisor and launchd settings. It does not reinstall the service.");
|
|
64
|
+
}
|
|
65
|
+
if (!result.ok) {
|
|
66
|
+
process.exitCode = 1;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
const message = error instanceof Error
|
|
71
|
+
? error.message
|
|
72
|
+
: "Restart control is unavailable.";
|
|
73
|
+
if (argv.format === "json") {
|
|
74
|
+
const result = {
|
|
75
|
+
ok: false,
|
|
76
|
+
phase: "unverified",
|
|
77
|
+
message,
|
|
78
|
+
};
|
|
79
|
+
console.info(JSON.stringify(result));
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
console.error(message);
|
|
83
|
+
}
|
|
84
|
+
process.exitCode = 1;
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
//# sourceMappingURL=proxyRestart.js.map
|
package/dist/cli/parser.js
CHANGED
|
@@ -23,6 +23,7 @@ import { ObservabilityCommandFactory } from "./commands/observability.js";
|
|
|
23
23
|
import { TelemetryCommandFactory } from "./commands/telemetry.js";
|
|
24
24
|
import { proxyStartCommand, proxyStatusCommand, proxyTelemetryCommand, proxySetupCommand, proxyGuardCommand, proxyInstallCommand, proxyUninstallCommand, } from "./commands/proxy.js";
|
|
25
25
|
import { proxyAnalyzeCommand } from "./commands/proxyAnalyze.js";
|
|
26
|
+
import { proxyRestartCommand } from "./commands/proxyRestart.js";
|
|
26
27
|
import { proxyShareCommand } from "./commands/proxyShare.js";
|
|
27
28
|
import { proxyPeerCommand } from "./commands/proxyPeer.js";
|
|
28
29
|
import { proxyExposeCommand } from "./commands/proxyExpose.js";
|
|
@@ -218,6 +219,7 @@ export function initializeCliParser() {
|
|
|
218
219
|
builder: (yargs) => yargs
|
|
219
220
|
.command(proxyStartCommand)
|
|
220
221
|
.command(proxyStatusCommand)
|
|
222
|
+
.command(proxyRestartCommand)
|
|
221
223
|
.command(proxyShareCommand)
|
|
222
224
|
.command(proxyPeerCommand)
|
|
223
225
|
.command(proxyExposeCommand)
|
|
@@ -228,7 +230,7 @@ export function initializeCliParser() {
|
|
|
228
230
|
.command(proxyGuardCommand)
|
|
229
231
|
.command(proxyInstallCommand)
|
|
230
232
|
.command(proxyUninstallCommand)
|
|
231
|
-
.demandCommand(1, "Please specify a proxy subcommand: start, status, share <create|provision|url|list|status|pause|resume|revoke|topup|set|link|rotate|level|note|notes|receipts|delete>, peer <add|request|sync|receipts|net|redeem|list|status|test|remove|pause|resume|set>, expose, analyze, replay <export|compare>, telemetry <setup|start|stop|status|logs|import-dashboard>, setup, guard, install, or uninstall"),
|
|
233
|
+
.demandCommand(1, "Please specify a proxy subcommand: start, status, restart, share <create|provision|url|list|status|pause|resume|revoke|topup|set|link|rotate|level|note|notes|receipts|delete>, peer <add|request|sync|receipts|net|redeem|list|status|test|remove|pause|resume|set>, expose, analyze, replay <export|compare>, telemetry <setup|start|stop|status|logs|import-dashboard>, setup, guard, install, or uninstall"),
|
|
232
234
|
handler: () => { },
|
|
233
235
|
})
|
|
234
236
|
// Evaluate Command Group - Using EvaluateCommandFactory
|
|
@@ -18,6 +18,8 @@ const snapshot = {
|
|
|
18
18
|
pendingBytes: 0,
|
|
19
19
|
maxPending: MAX_PENDING,
|
|
20
20
|
maxPendingBytes: MAX_PENDING_BYTES,
|
|
21
|
+
highWaterPending: 0,
|
|
22
|
+
highWaterBytes: 0,
|
|
21
23
|
rejectionReasons: {},
|
|
22
24
|
};
|
|
23
25
|
const pending = new Map();
|
|
@@ -161,9 +163,27 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
161
163
|
? "body_worker_backoff"
|
|
162
164
|
: "body_capture_queue_full";
|
|
163
165
|
snapshot.lastError = error;
|
|
166
|
+
snapshot.lastRejectedAt = new Date().toISOString();
|
|
164
167
|
snapshot.rejectionReasons[error] =
|
|
165
168
|
(snapshot.rejectionReasons[error] ?? 0) + 1;
|
|
166
|
-
return consume({
|
|
169
|
+
return consume({
|
|
170
|
+
error,
|
|
171
|
+
stored: { bodyWriteFailed: true },
|
|
172
|
+
admission: {
|
|
173
|
+
limitingResource: bytes > MAX_ENTRY_BYTES
|
|
174
|
+
? "entry"
|
|
175
|
+
: Date.now() < retryAfter
|
|
176
|
+
? "worker"
|
|
177
|
+
: snapshot.pending >= MAX_PENDING
|
|
178
|
+
? "captures"
|
|
179
|
+
: "bytes",
|
|
180
|
+
estimatedBytes: Number.isFinite(bytes) ? bytes : undefined,
|
|
181
|
+
pending: snapshot.pending,
|
|
182
|
+
pendingBytes: snapshot.pendingBytes,
|
|
183
|
+
maxPending: MAX_PENDING,
|
|
184
|
+
maxPendingBytes: MAX_PENDING_BYTES,
|
|
185
|
+
},
|
|
186
|
+
});
|
|
167
187
|
}
|
|
168
188
|
let current;
|
|
169
189
|
try {
|
|
@@ -207,7 +227,9 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
207
227
|
},
|
|
208
228
|
});
|
|
209
229
|
snapshot.pending += 1;
|
|
230
|
+
snapshot.highWaterPending = Math.max(snapshot.highWaterPending, snapshot.pending);
|
|
210
231
|
snapshot.pendingBytes += bytes;
|
|
232
|
+
snapshot.highWaterBytes = Math.max(snapshot.highWaterBytes, snapshot.pendingBytes);
|
|
211
233
|
current.ref();
|
|
212
234
|
try {
|
|
213
235
|
current.postMessage({ id, entry, logDir, queuedAt: Date.now() });
|
|
@@ -248,6 +270,9 @@ export const __bodyCaptureWorkerTestHooks = {
|
|
|
248
270
|
pending: 0,
|
|
249
271
|
pendingBytes: 0,
|
|
250
272
|
lastError: undefined,
|
|
273
|
+
lastRejectedAt: undefined,
|
|
274
|
+
highWaterPending: 0,
|
|
275
|
+
highWaterBytes: 0,
|
|
251
276
|
rejectionReasons: {},
|
|
252
277
|
});
|
|
253
278
|
},
|
package/dist/proxy/codexUsage.js
CHANGED
|
@@ -45,6 +45,34 @@ import { sanitizeForLog } from "../utils/logSanitize.js";
|
|
|
45
45
|
const nonNegativeInt = (value) => typeof value === "number" && Number.isFinite(value) && value > 0
|
|
46
46
|
? Math.floor(value)
|
|
47
47
|
: 0;
|
|
48
|
+
function usefulOutputItem(item) {
|
|
49
|
+
if (!item || typeof item !== "object") {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
if ("type" in item) {
|
|
53
|
+
if (item.type === "function_call") {
|
|
54
|
+
return ("arguments" in item &&
|
|
55
|
+
typeof item.arguments === "string" &&
|
|
56
|
+
item.arguments.length > 0);
|
|
57
|
+
}
|
|
58
|
+
if (item.type === "custom_tool_call") {
|
|
59
|
+
return ("input" in item &&
|
|
60
|
+
typeof item.input === "string" &&
|
|
61
|
+
item.input.length > 0);
|
|
62
|
+
}
|
|
63
|
+
if (item.type === "output_text") {
|
|
64
|
+
return ("text" in item && typeof item.text === "string" && item.text.length > 0);
|
|
65
|
+
}
|
|
66
|
+
if (item.type === "refusal") {
|
|
67
|
+
return ("refusal" in item &&
|
|
68
|
+
typeof item.refusal === "string" &&
|
|
69
|
+
item.refusal.length > 0);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return ("content" in item &&
|
|
73
|
+
Array.isArray(item.content) &&
|
|
74
|
+
item.content.some(usefulOutputItem));
|
|
75
|
+
}
|
|
48
76
|
/**
|
|
49
77
|
* Pull usage out of one parsed SSE `data:` payload.
|
|
50
78
|
*
|
|
@@ -125,7 +153,7 @@ const CAPTURE_LIMIT_BYTES = 256 * 1024;
|
|
|
125
153
|
*/
|
|
126
154
|
function createCaptureSink() {
|
|
127
155
|
const target = process.env.NEUROLINK_PROXY_CODEX_CAPTURE;
|
|
128
|
-
if (!target) {
|
|
156
|
+
if (!target || process.env.NEUROLINK_PROXY_LOG_SINK === "otel") {
|
|
129
157
|
return null;
|
|
130
158
|
}
|
|
131
159
|
let written = 0;
|
|
@@ -167,9 +195,13 @@ export function createCodexUsageTap() {
|
|
|
167
195
|
let totalBytes = 0;
|
|
168
196
|
const inspectEvidence = (events) => {
|
|
169
197
|
for (const frame of events) {
|
|
198
|
+
if (frame.data.trim() === "[DONE]") {
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
170
201
|
try {
|
|
171
202
|
const event = JSON.parse(frame.data);
|
|
172
203
|
if (!event || typeof event !== "object") {
|
|
204
|
+
evidence.observationIncomplete = true;
|
|
173
205
|
continue;
|
|
174
206
|
}
|
|
175
207
|
const seen = extractCodexUsage(event);
|
|
@@ -178,10 +210,53 @@ export function createCodexUsageTap() {
|
|
|
178
210
|
}
|
|
179
211
|
const type = event.type ?? frame.event;
|
|
180
212
|
if ((type === "response.output_text.delta" ||
|
|
213
|
+
type === "response.refusal.delta" ||
|
|
181
214
|
type === "response.function_call_arguments.delta") &&
|
|
182
215
|
typeof event.delta === "string" &&
|
|
183
216
|
event.delta.length > 0) {
|
|
184
|
-
evidence.firstUsefulOutputAt
|
|
217
|
+
if (evidence.firstUsefulOutputAt === undefined) {
|
|
218
|
+
evidence.firstUsefulOutputAt = Date.now();
|
|
219
|
+
evidence.firstUsefulOutputEvent = String(type);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
// Some clients/providers emit complete output items without deltas.
|
|
223
|
+
// Reasoning/control events are not useful client output. Recognize
|
|
224
|
+
// actual tool calls and text only, without inventing an earlier time.
|
|
225
|
+
const usefulItem = usefulOutputItem(event.item);
|
|
226
|
+
const completedResponse = event.response;
|
|
227
|
+
const usefulCompletion = type === "response.completed" &&
|
|
228
|
+
completedResponse &&
|
|
229
|
+
typeof completedResponse === "object" &&
|
|
230
|
+
"output" in completedResponse &&
|
|
231
|
+
Array.isArray(completedResponse.output) &&
|
|
232
|
+
completedResponse.output.some(usefulOutputItem);
|
|
233
|
+
const usefulDone = (type === "response.output_text.done" &&
|
|
234
|
+
typeof event.text === "string" &&
|
|
235
|
+
event.text.length > 0) ||
|
|
236
|
+
(type === "response.refusal.done" &&
|
|
237
|
+
typeof event.refusal === "string" &&
|
|
238
|
+
event.refusal.length > 0) ||
|
|
239
|
+
(type === "response.function_call_arguments.done" &&
|
|
240
|
+
typeof event.arguments === "string" &&
|
|
241
|
+
event.arguments.length > 0) ||
|
|
242
|
+
(type === "response.custom_tool_call_input.done" &&
|
|
243
|
+
typeof event.input === "string" &&
|
|
244
|
+
event.input.length > 0) ||
|
|
245
|
+
((type === "response.content_part.added" ||
|
|
246
|
+
type === "response.content_part.done") &&
|
|
247
|
+
usefulOutputItem(event.part));
|
|
248
|
+
const usefulToolDelta = type === "response.custom_tool_call_input.delta" &&
|
|
249
|
+
typeof event.delta === "string" &&
|
|
250
|
+
event.delta.length > 0;
|
|
251
|
+
if (evidence.firstUsefulOutputAt === undefined &&
|
|
252
|
+
(usefulDone ||
|
|
253
|
+
usefulCompletion ||
|
|
254
|
+
usefulToolDelta ||
|
|
255
|
+
((type === "response.output_item.added" ||
|
|
256
|
+
type === "response.output_item.done") &&
|
|
257
|
+
usefulItem))) {
|
|
258
|
+
evidence.firstUsefulOutputAt = Date.now();
|
|
259
|
+
evidence.firstUsefulOutputEvent = String(type);
|
|
185
260
|
}
|
|
186
261
|
if (type === "response.completed") {
|
|
187
262
|
evidence.completed = true;
|
|
@@ -222,6 +297,7 @@ export function createCodexUsageTap() {
|
|
|
222
297
|
}
|
|
223
298
|
catch {
|
|
224
299
|
// Unknown frames cannot establish successful completion.
|
|
300
|
+
evidence.observationIncomplete = true;
|
|
225
301
|
}
|
|
226
302
|
}
|
|
227
303
|
};
|
|
@@ -268,16 +344,21 @@ export function createCodexUsageTap() {
|
|
|
268
344
|
carry = remainder;
|
|
269
345
|
inspectEvidence(events);
|
|
270
346
|
if (carry.length > CARRY_LIMIT_CHARS) {
|
|
347
|
+
evidence.observationIncomplete = true;
|
|
271
348
|
carry = carry.slice(-2);
|
|
272
349
|
discardingEvent = true;
|
|
273
350
|
}
|
|
274
351
|
}
|
|
275
352
|
catch {
|
|
276
353
|
// Telemetry must never break the relay.
|
|
354
|
+
evidence.observationIncomplete = true;
|
|
277
355
|
}
|
|
278
356
|
},
|
|
279
357
|
flush() {
|
|
280
358
|
// An event without its dispatch delimiter is incomplete on the wire.
|
|
359
|
+
if (discardingEvent || carry.trim()) {
|
|
360
|
+
evidence.observationIncomplete = true;
|
|
361
|
+
}
|
|
281
362
|
settle(latest);
|
|
282
363
|
},
|
|
283
364
|
/**
|
|
@@ -34,6 +34,18 @@ export declare function getProxyOtelLogSnapshot(): {
|
|
|
34
34
|
maxPendingBytes: number;
|
|
35
35
|
};
|
|
36
36
|
queues: {
|
|
37
|
+
recentFailures: {
|
|
38
|
+
records: {
|
|
39
|
+
eventId: string;
|
|
40
|
+
kind?: string;
|
|
41
|
+
requestId?: string;
|
|
42
|
+
captureId?: string;
|
|
43
|
+
}[];
|
|
44
|
+
id: string;
|
|
45
|
+
at: string;
|
|
46
|
+
reason: "export_unconfirmed" | "queue_full";
|
|
47
|
+
error?: string;
|
|
48
|
+
}[];
|
|
37
49
|
attempted: number;
|
|
38
50
|
submitted: number;
|
|
39
51
|
transportAcknowledged: number;
|
|
@@ -43,6 +55,7 @@ export declare function getProxyOtelLogSnapshot(): {
|
|
|
43
55
|
lastAcknowledgedAt: string | undefined;
|
|
44
56
|
lastFailureAt: string | undefined;
|
|
45
57
|
highWaterOutstanding: number;
|
|
58
|
+
failureHistoryEvicted: number;
|
|
46
59
|
kind: "metadata" | "bodies";
|
|
47
60
|
capacity: number;
|
|
48
61
|
}[];
|