@juspay/neurolink 12.20.0 → 12.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -6
- package/dist/browser/neurolink.min.js +400 -400
- package/dist/cli/commands/proxy.js +171 -25
- package/dist/cli/commands/telemetry.d.ts +4 -1
- package/dist/cli/commands/telemetry.js +113 -0
- package/dist/cli/utils/nativeTelemetry.d.ts +19 -0
- package/dist/cli/utils/nativeTelemetry.js +322 -0
- package/dist/proxy/bodyCapturePolicy.d.ts +6 -0
- package/dist/proxy/bodyCapturePolicy.js +132 -0
- package/dist/proxy/codexFallback.d.ts +27 -9
- package/dist/proxy/codexFallback.js +84 -13
- package/dist/proxy/openaiFormat.js +8 -1
- package/dist/proxy/otelLogSink.d.ts +11 -1
- package/dist/proxy/otelLogSink.js +174 -27
- package/dist/proxy/otlpLogTransport.d.ts +4 -2
- package/dist/proxy/otlpLogTransport.js +22 -7
- package/dist/proxy/requestLogger.js +11 -3
- package/dist/proxy/restartControl.js +18 -3
- package/dist/proxy/rollingProxyServer.js +48 -1
- package/dist/proxy/rollingWorkerProcess.js +87 -10
- package/dist/proxy/rollingWorkerProtocol.js +4 -2
- package/dist/proxy/rollingWorkerSupervisor.js +47 -10
- package/dist/proxy/socketWorkerRuntime.js +90 -4
- package/dist/proxy/updateChecker.d.ts +3 -3
- package/dist/proxy/updateChecker.js +16 -9
- package/dist/proxy/updateCoordinator.d.ts +10 -1
- package/dist/proxy/updateCoordinator.js +68 -1
- package/dist/proxy/updateState.d.ts +2 -0
- package/dist/proxy/updateState.js +15 -0
- package/dist/proxy/vertexAnthropicFallback.d.ts +17 -1
- package/dist/proxy/vertexAnthropicFallback.js +25 -1
- package/dist/server/routes/claudeProxyRoutes.d.ts +17 -0
- package/dist/server/routes/claudeProxyRoutes.js +53 -5
- package/dist/types/cli.d.ts +18 -0
- package/dist/types/providers.d.ts +8 -0
- package/dist/types/proxy.d.ts +117 -7
- package/dist/utils/anthropicCacheBreakpoints.d.ts +22 -1
- package/dist/utils/anthropicCacheBreakpoints.js +113 -0
- package/dist/utils/pricing.js +69 -2
- package/docs-site/static/search-index.json +1 -1
- package/package.json +1 -1
- package/scripts/observability/NATIVE-DURABLE.md +68 -0
- package/scripts/observability/otel-collector.proxy-native-durable.yaml +2 -2
- package/scripts/observability/proxy-telemetry-check.mjs +135 -50
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { initializeProxyOtelLogs, routeProxyConsoleToOtel,
|
|
1
|
+
import { initializeProxyOtelLogs, routeProxyConsoleToOtel, shutdownProxyOtelLogs, isProxyOtelOnly, withProxyOtelLogShutdown, } from "../../proxy/otelLogSink.js";
|
|
2
2
|
import { registerCliShutdownOwner } from "../lifecycle.js";
|
|
3
3
|
import { writeFileAtomic } from "../proxy-clients/snapshot.js";
|
|
4
4
|
import { spawn } from "node:child_process";
|
|
@@ -35,8 +35,8 @@ import { isProxyAuxiliaryRequest } from "../../proxy/proxyRequestKind.js";
|
|
|
35
35
|
import { spawnProxySocketWorker } from "../../proxy/rollingWorkerProcess.js";
|
|
36
36
|
import { PROXY_ROLLING_SUPERVISOR_ENV, PROXY_SOCKET_WORKER_ENV, } from "../../proxy/rollingWorkerProtocol.js";
|
|
37
37
|
import { attachSocketWorkerProcess } from "../../proxy/socketWorkerRuntime.js";
|
|
38
|
-
import { parseProxyRuntimeActivity, isProxyUpdateOwnerCurrent, shouldRefreshStaleSupervisor, waitForProxyUpdateWindow, } from "../../proxy/updateCoordinator.js";
|
|
39
|
-
import { abandonPendingUpdate, clearUpdateDeferral, isVersionSuppressed, loadUpdateState, recordCheck, recordSuccessfulUpdate, recordUpdateDeferred, recordUpdateFailure, recordUpdateInstalled, suppressVersion, } from "../../proxy/updateState.js";
|
|
38
|
+
import { isSameProxyServingWorker, parseProxyServingWorkerIdentity, parseProxyRuntimeActivity, isProxyUpdateOwnerCurrent, shouldRefreshStaleSupervisor, waitForProxyUpdateWindow, } from "../../proxy/updateCoordinator.js";
|
|
39
|
+
import { abandonPendingUpdate, clearUpdateDeferral, isVersionSuppressed, loadUpdateState, recordCheck, recordCheckFailure, recordSuccessfulUpdate, recordUpdateDeferred, recordUpdateFailure, recordUpdateInstalled, suppressVersion, } from "../../proxy/updateState.js";
|
|
40
40
|
import { loadProxyEnvFile, resolveProxyEnvFile, } from "../../proxy/proxyEnv.js";
|
|
41
41
|
import { createRequire } from "node:module";
|
|
42
42
|
import { fileURLToPath } from "node:url";
|
|
@@ -542,7 +542,7 @@ async function fetchProxyHealthVersion(host, port) {
|
|
|
542
542
|
}
|
|
543
543
|
}
|
|
544
544
|
/** Read both serving generations so an updated worker cannot hide a stale parent. */
|
|
545
|
-
async function fetchProxyRuntimeVersions(host, port) {
|
|
545
|
+
async function fetchProxyRuntimeVersions(host, port, rollingSupervisor = false) {
|
|
546
546
|
try {
|
|
547
547
|
const response = await fetch(`http://${host}:${port}/status`, {
|
|
548
548
|
headers: { connection: "close" },
|
|
@@ -552,10 +552,12 @@ async function fetchProxyRuntimeVersions(host, port) {
|
|
|
552
552
|
return {};
|
|
553
553
|
}
|
|
554
554
|
const data = (await response.json());
|
|
555
|
+
const workerIdentity = parseProxyServingWorkerIdentity(data, rollingSupervisor);
|
|
555
556
|
return {
|
|
556
557
|
...(typeof data.version === "string"
|
|
557
558
|
? { workerVersion: data.version }
|
|
558
559
|
: {}),
|
|
560
|
+
...(workerIdentity ? { workerIdentity } : {}),
|
|
559
561
|
...(typeof data.autoUpdate?.supervisorVersion === "string"
|
|
560
562
|
? { supervisorVersion: data.autoUpdate.supervisorVersion }
|
|
561
563
|
: {}),
|
|
@@ -574,11 +576,17 @@ async function fetchProxyRuntimeVersions(host, port) {
|
|
|
574
576
|
* signal SIGUSR2) and wait until the supervisor actually serves that version
|
|
575
577
|
* again. Returns false if it never reports the running version healthy in time.
|
|
576
578
|
*/
|
|
577
|
-
async function activateRollbackVersion(host, port, parentPid, runningVersion, timeoutMs) {
|
|
579
|
+
async function activateRollbackVersion(host, port, parentPid, runningVersion, timeoutMs, isCurrentOwner) {
|
|
580
|
+
if (!isCurrentOwner()) {
|
|
581
|
+
return false;
|
|
582
|
+
}
|
|
578
583
|
recordUpdateInstalled(runningVersion);
|
|
579
584
|
try {
|
|
580
585
|
const start = Date.now();
|
|
581
586
|
while (Date.now() - start < timeoutMs) {
|
|
587
|
+
if (!isCurrentOwner()) {
|
|
588
|
+
return false;
|
|
589
|
+
}
|
|
582
590
|
try {
|
|
583
591
|
process.kill(parentPid, "SIGUSR2");
|
|
584
592
|
}
|
|
@@ -626,7 +634,7 @@ async function setProxyUpdateDrain(host, port, draining, supervisorRefresh = fal
|
|
|
626
634
|
body: JSON.stringify({
|
|
627
635
|
action: draining
|
|
628
636
|
? supervisorRefresh
|
|
629
|
-
? "
|
|
637
|
+
? "fence_for_supervisor_refresh"
|
|
630
638
|
: "drain"
|
|
631
639
|
: "resume",
|
|
632
640
|
}),
|
|
@@ -1555,21 +1563,31 @@ export async function createProxyStartApp(params) {
|
|
|
1555
1563
|
.json()
|
|
1556
1564
|
.catch(() => ({ action: undefined }));
|
|
1557
1565
|
if (payload.action === "drain" ||
|
|
1558
|
-
payload.action === "drain_for_supervisor_refresh"
|
|
1566
|
+
payload.action === "drain_for_supervisor_refresh" ||
|
|
1567
|
+
payload.action === "fence_for_supervisor_refresh") {
|
|
1559
1568
|
// A rolling worker must keep admitting until the supervisor has a ready
|
|
1560
1569
|
// replacement. The legacy global drain can otherwise strand the listener
|
|
1561
1570
|
// behind maintenance responses when its caller stalls or disappears.
|
|
1562
1571
|
if (isProxySocketWorkerProcess() &&
|
|
1563
|
-
payload.action !== "drain_for_supervisor_refresh"
|
|
1572
|
+
payload.action !== "drain_for_supervisor_refresh" &&
|
|
1573
|
+
payload.action !== "fence_for_supervisor_refresh") {
|
|
1564
1574
|
return c.json({
|
|
1565
1575
|
error: "rolling_restart_required",
|
|
1566
1576
|
message: "Use neurolink proxy restart; global update drain is disabled for rolling workers.",
|
|
1567
1577
|
}, 409);
|
|
1568
1578
|
}
|
|
1579
|
+
// No await between the activity check and fence: an HTTP request cannot
|
|
1580
|
+
// enter this worker between the idle observation and admission closure.
|
|
1581
|
+
// The updater separately verifies all supervisor generations are settled.
|
|
1582
|
+
if (payload.action === "fence_for_supervisor_refresh" &&
|
|
1583
|
+
getProxyActivitySnapshot().activeRequests !== 0) {
|
|
1584
|
+
return c.json({ error: "proxy_busy", draining: false }, 409);
|
|
1585
|
+
}
|
|
1569
1586
|
if (!markProxyDrainingForUpdate(readiness)) {
|
|
1570
1587
|
return c.json({ error: "proxy_not_ready" }, 409);
|
|
1571
1588
|
}
|
|
1572
|
-
if (payload.action === "drain_for_supervisor_refresh"
|
|
1589
|
+
if (payload.action === "drain_for_supervisor_refresh" ||
|
|
1590
|
+
payload.action === "fence_for_supervisor_refresh") {
|
|
1573
1591
|
// A crashed updater must not strand the current rolling worker in
|
|
1574
1592
|
// maintenance mode. A successful launchd refresh replaces this process.
|
|
1575
1593
|
scheduleSupervisorRefreshResume(90 * 1000);
|
|
@@ -2243,6 +2261,8 @@ export async function createProxyStartApp(params) {
|
|
|
2243
2261
|
: "restart",
|
|
2244
2262
|
deferredUpdate: updateState?.deferredUpdate ?? null,
|
|
2245
2263
|
lastCheckAt: updateState?.lastCheckAt ?? null,
|
|
2264
|
+
lastCheckAttemptAt: updateState?.lastCheckAttemptAt ?? null,
|
|
2265
|
+
lastCheckError: updateState?.lastCheckError ?? null,
|
|
2246
2266
|
lastUpdateAt: updateState?.lastUpdateAt ?? null,
|
|
2247
2267
|
lastUpdateVersion: updateState?.lastUpdateVersion ?? null,
|
|
2248
2268
|
lastFailure: updateState?.lastFailure
|
|
@@ -2583,7 +2603,6 @@ function registerProxyShutdownHandlers(params) {
|
|
|
2583
2603
|
try {
|
|
2584
2604
|
const { flushOpenTelemetry, shutdownOpenTelemetry } = await import("../../services/server/ai/observability/instrumentation.js");
|
|
2585
2605
|
await withShutdownDeadline((async () => {
|
|
2586
|
-
await flushProxyOtelLogs();
|
|
2587
2606
|
await shutdownProxyOtelLogs();
|
|
2588
2607
|
await flushOpenTelemetry();
|
|
2589
2608
|
await shutdownOpenTelemetry();
|
|
@@ -2990,7 +3009,6 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2990
3009
|
if (otelInitialized) {
|
|
2991
3010
|
await cleanup(async () => {
|
|
2992
3011
|
await flushProxyLifecycleEvents().catch(reportCleanupFailure);
|
|
2993
|
-
await flushProxyOtelLogs().catch(reportCleanupFailure);
|
|
2994
3012
|
await shutdownProxyOtelLogs();
|
|
2995
3013
|
}, PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS, "supervisor telemetry shutdown deadline exceeded");
|
|
2996
3014
|
}
|
|
@@ -3585,6 +3603,8 @@ export const proxyStatusCommand = {
|
|
|
3585
3603
|
latestVersion: updateState?.lastCheckVersion || null,
|
|
3586
3604
|
lastDetectedVersion: updateState?.lastCheckVersion || null,
|
|
3587
3605
|
lastCheckAt: updateState?.lastCheckAt ?? null,
|
|
3606
|
+
lastCheckAttemptAt: updateState?.lastCheckAttemptAt ?? null,
|
|
3607
|
+
lastCheckError: updateState?.lastCheckError ?? null,
|
|
3588
3608
|
installedVersion: updateState?.installedVersion ??
|
|
3589
3609
|
updateState?.lastUpdateVersion ??
|
|
3590
3610
|
null,
|
|
@@ -4048,16 +4068,68 @@ export const proxyGuardCommand = {
|
|
|
4048
4068
|
updateInProgress = true;
|
|
4049
4069
|
let updateVersion = runningVersion;
|
|
4050
4070
|
let drainActive = false;
|
|
4051
|
-
let previousPackage
|
|
4052
|
-
if (previousPackage?.version !== runningVersion) {
|
|
4053
|
-
previousPackage = readProxyPackageSelection(PROXY_PACKAGES_DIR, true);
|
|
4054
|
-
}
|
|
4071
|
+
let previousPackage;
|
|
4055
4072
|
let candidatePackage;
|
|
4073
|
+
let servingWorkerIdentity;
|
|
4074
|
+
let activatedWorkerIdentity;
|
|
4075
|
+
const readPersistedServingIdentity = () => {
|
|
4076
|
+
if (!servingWorkerIdentity) {
|
|
4077
|
+
return null;
|
|
4078
|
+
}
|
|
4079
|
+
if (rollingSupervisor) {
|
|
4080
|
+
const active = loadProxySupervisorState()?.rolling?.active;
|
|
4081
|
+
return active
|
|
4082
|
+
? {
|
|
4083
|
+
version: active.version,
|
|
4084
|
+
pid: active.pid,
|
|
4085
|
+
generation: active.generation,
|
|
4086
|
+
}
|
|
4087
|
+
: null;
|
|
4088
|
+
}
|
|
4089
|
+
const state = loadProxyState();
|
|
4090
|
+
return state?.pid
|
|
4091
|
+
? {
|
|
4092
|
+
version: servingWorkerIdentity.version,
|
|
4093
|
+
pid: state.pid,
|
|
4094
|
+
generation: null,
|
|
4095
|
+
}
|
|
4096
|
+
: null;
|
|
4097
|
+
};
|
|
4098
|
+
const isCurrentUpdateAttemptOwner = () => {
|
|
4099
|
+
if (!isCurrentUpdateOwner() || !servingWorkerIdentity) {
|
|
4100
|
+
return false;
|
|
4101
|
+
}
|
|
4102
|
+
const current = readPersistedServingIdentity();
|
|
4103
|
+
return (isSameProxyServingWorker(servingWorkerIdentity, current) ||
|
|
4104
|
+
(activatedWorkerIdentity !== undefined &&
|
|
4105
|
+
isSameProxyServingWorker(activatedWorkerIdentity, current)));
|
|
4106
|
+
};
|
|
4056
4107
|
try {
|
|
4108
|
+
// A rolling worker may have changed since this updater started. Never
|
|
4109
|
+
// use startup identity to choose rollback or downgrade a serving worker.
|
|
4110
|
+
const runtimeVersions = await fetchProxyRuntimeVersions(host, port, rollingSupervisor);
|
|
4111
|
+
servingWorkerIdentity = runtimeVersions.workerIdentity;
|
|
4112
|
+
const observedVersion = servingWorkerIdentity?.version;
|
|
4113
|
+
if (!observedVersion) {
|
|
4114
|
+
persistUpdaterState("record unavailable runtime identity", () => recordCheckFailure("Serving worker version could not be verified"));
|
|
4115
|
+
scheduleUpdateRetry("runtime version probe incomplete", runningVersion);
|
|
4116
|
+
return;
|
|
4117
|
+
}
|
|
4118
|
+
runningVersion = observedVersion;
|
|
4119
|
+
updateVersion = runningVersion;
|
|
4120
|
+
previousPackage = readProxyPackageSelection(PROXY_PACKAGES_DIR);
|
|
4121
|
+
if (previousPackage?.version !== runningVersion) {
|
|
4122
|
+
previousPackage = readProxyPackageSelection(PROXY_PACKAGES_DIR, true);
|
|
4123
|
+
}
|
|
4057
4124
|
// Lazy-load update modules so they're only imported at check time
|
|
4058
4125
|
const { checkForUpdate } = await import("../../proxy/updateChecker.js");
|
|
4059
4126
|
// 1. Check for update
|
|
4060
4127
|
let result = await checkForUpdate(runningVersion);
|
|
4128
|
+
if (result.checkSucceeded === false) {
|
|
4129
|
+
persistUpdaterState("record failed update check", () => recordCheckFailure(result.checkError ?? "Registry check failed"));
|
|
4130
|
+
scheduleUpdateRetry("registry check failed", runningVersion);
|
|
4131
|
+
return;
|
|
4132
|
+
}
|
|
4061
4133
|
updateVersion = result.latestVersion;
|
|
4062
4134
|
persistUpdaterState("record update check", () => recordCheck(result.latestVersion));
|
|
4063
4135
|
let supervisorRefreshOnly = false;
|
|
@@ -4084,6 +4156,10 @@ export const proxyGuardCommand = {
|
|
|
4084
4156
|
persistUpdaterState("clear update deferral", () => clearUpdateDeferral());
|
|
4085
4157
|
return;
|
|
4086
4158
|
}
|
|
4159
|
+
// A stale registry mirror can report a lower version. Reconcile the
|
|
4160
|
+
// supervisor to the observed worker, never to that older registry row.
|
|
4161
|
+
result = { ...result, latestVersion: runningVersion };
|
|
4162
|
+
updateVersion = runningVersion;
|
|
4087
4163
|
logger.always(`[updater] worker v${runningVersion} is current; reconciling stale supervisor v${activeVersions.supervisorVersion}`);
|
|
4088
4164
|
}
|
|
4089
4165
|
const initiallyPendingRestart = loadUpdateState()?.pendingRestartVersion === result.latestVersion;
|
|
@@ -4100,7 +4176,20 @@ export const proxyGuardCommand = {
|
|
|
4100
4176
|
// Recheck the target before staging because discovery can be delayed.
|
|
4101
4177
|
let pendingRestart = initiallyPendingRestart;
|
|
4102
4178
|
if (!supervisorRefreshOnly) {
|
|
4179
|
+
const refreshedRuntime = await fetchProxyRuntimeVersions(host, port, rollingSupervisor);
|
|
4180
|
+
if (!servingWorkerIdentity ||
|
|
4181
|
+
!isSameProxyServingWorker(servingWorkerIdentity, refreshedRuntime.workerIdentity)) {
|
|
4182
|
+
// Another updater/manual rotation won while registry discovery was
|
|
4183
|
+
// pending. Restart discovery using that worker on the next tick.
|
|
4184
|
+
scheduleUpdateRetry("serving worker changed during discovery", runningVersion);
|
|
4185
|
+
return;
|
|
4186
|
+
}
|
|
4103
4187
|
const refreshedResult = await checkForUpdate(runningVersion);
|
|
4188
|
+
if (refreshedResult.checkSucceeded === false) {
|
|
4189
|
+
persistUpdaterState("record failed refreshed update check", () => recordCheckFailure(refreshedResult.checkError ?? "Registry check failed"));
|
|
4190
|
+
scheduleUpdateRetry("refreshed registry check failed", runningVersion);
|
|
4191
|
+
return;
|
|
4192
|
+
}
|
|
4104
4193
|
result = refreshedResult;
|
|
4105
4194
|
updateVersion = result.latestVersion;
|
|
4106
4195
|
persistUpdaterState("record refreshed update check", () => recordCheck(result.latestVersion));
|
|
@@ -4185,6 +4274,12 @@ export const proxyGuardCommand = {
|
|
|
4185
4274
|
if (!isCurrentUpdateOwner()) {
|
|
4186
4275
|
return;
|
|
4187
4276
|
}
|
|
4277
|
+
const publicationRuntime = await fetchProxyRuntimeVersions(host, port, rollingSupervisor);
|
|
4278
|
+
if (!servingWorkerIdentity ||
|
|
4279
|
+
!isSameProxyServingWorker(servingWorkerIdentity, publicationRuntime.workerIdentity)) {
|
|
4280
|
+
scheduleUpdateRetry("serving worker changed during staging", result.latestVersion);
|
|
4281
|
+
return;
|
|
4282
|
+
}
|
|
4188
4283
|
// 4. Refresh and validate the stable trampoline. The plist already
|
|
4189
4284
|
// points at this path, so it must not be unloaded or rewritten here.
|
|
4190
4285
|
if (!supervisorRefreshOnly) {
|
|
@@ -4200,9 +4295,9 @@ export const proxyGuardCommand = {
|
|
|
4200
4295
|
}
|
|
4201
4296
|
if (previousPackage?.version === runningVersion &&
|
|
4202
4297
|
!pendingRestart) {
|
|
4203
|
-
selectProxyPackage(PROXY_PACKAGES_DIR, previousPackage,
|
|
4298
|
+
selectProxyPackage(PROXY_PACKAGES_DIR, previousPackage, isCurrentUpdateAttemptOwner);
|
|
4204
4299
|
}
|
|
4205
|
-
writeTrampoline(candidatePackage,
|
|
4300
|
+
writeTrampoline(candidatePackage, isCurrentUpdateAttemptOwner);
|
|
4206
4301
|
const validation = await validateInstalledVersion({
|
|
4207
4302
|
binPath: TRAMPOLINE_PATH,
|
|
4208
4303
|
expectedVersion: result.latestVersion,
|
|
@@ -4216,7 +4311,7 @@ export const proxyGuardCommand = {
|
|
|
4216
4311
|
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "validation", message));
|
|
4217
4312
|
persistUpdaterState("abandon invalid pending update", () => abandonPendingUpdate(result.latestVersion));
|
|
4218
4313
|
if (previousPackage) {
|
|
4219
|
-
writeTrampoline(previousPackage,
|
|
4314
|
+
writeTrampoline(previousPackage, isCurrentUpdateAttemptOwner);
|
|
4220
4315
|
}
|
|
4221
4316
|
scheduleUpdateRetry("transient candidate validation failure", result.latestVersion);
|
|
4222
4317
|
return;
|
|
@@ -4235,7 +4330,7 @@ export const proxyGuardCommand = {
|
|
|
4235
4330
|
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "validation", message));
|
|
4236
4331
|
persistUpdaterState("abandon invalid pending update", () => abandonPendingUpdate(result.latestVersion));
|
|
4237
4332
|
if (previousPackage) {
|
|
4238
|
-
writeTrampoline(previousPackage,
|
|
4333
|
+
writeTrampoline(previousPackage, isCurrentUpdateAttemptOwner);
|
|
4239
4334
|
}
|
|
4240
4335
|
scheduleUpdateRetry("candidate validation failed; restaging permitted", result.latestVersion);
|
|
4241
4336
|
return;
|
|
@@ -4297,16 +4392,28 @@ export const proxyGuardCommand = {
|
|
|
4297
4392
|
if (!isCurrentUpdateOwner()) {
|
|
4298
4393
|
return;
|
|
4299
4394
|
}
|
|
4395
|
+
const activationRuntime = await fetchProxyRuntimeVersions(host, port, rollingSupervisor);
|
|
4396
|
+
if (!servingWorkerIdentity ||
|
|
4397
|
+
!isSameProxyServingWorker(servingWorkerIdentity, activationRuntime.workerIdentity)) {
|
|
4398
|
+
scheduleUpdateRetry("serving worker changed before activation", result.latestVersion);
|
|
4399
|
+
return;
|
|
4400
|
+
}
|
|
4300
4401
|
updateRestartInProgress = true;
|
|
4301
4402
|
if (rollingSupervisor) {
|
|
4302
|
-
const activeVersions = await fetchProxyRuntimeVersions(host, port);
|
|
4403
|
+
const activeVersions = await fetchProxyRuntimeVersions(host, port, true);
|
|
4404
|
+
if (!servingWorkerIdentity ||
|
|
4405
|
+
!isSameProxyServingWorker(servingWorkerIdentity, activeVersions.workerIdentity)) {
|
|
4406
|
+
updateRestartInProgress = false;
|
|
4407
|
+
scheduleUpdateRetry("serving worker changed at activation boundary", result.latestVersion);
|
|
4408
|
+
return;
|
|
4409
|
+
}
|
|
4303
4410
|
if (activeVersions.workerVersion === result.latestVersion) {
|
|
4304
4411
|
logger.always(`[updater] worker v${result.latestVersion} is already active; resuming supervisor refresh without another worker replacement`);
|
|
4305
4412
|
}
|
|
4306
4413
|
else {
|
|
4307
4414
|
logger.always(`[updater] requesting rolling activation of v${result.latestVersion}`);
|
|
4308
4415
|
try {
|
|
4309
|
-
if (!
|
|
4416
|
+
if (!isCurrentUpdateAttemptOwner()) {
|
|
4310
4417
|
updateRestartInProgress = false;
|
|
4311
4418
|
return;
|
|
4312
4419
|
}
|
|
@@ -4358,6 +4465,14 @@ export const proxyGuardCommand = {
|
|
|
4358
4465
|
if (resp.ok) {
|
|
4359
4466
|
const data = (await resp.json());
|
|
4360
4467
|
if (data.version === result.latestVersion) {
|
|
4468
|
+
if (rollingSupervisor && servingWorkerIdentity) {
|
|
4469
|
+
const observedRuntime = await fetchProxyRuntimeVersions(host, port, true);
|
|
4470
|
+
if (observedRuntime.workerIdentity?.version ===
|
|
4471
|
+
result.latestVersion &&
|
|
4472
|
+
!isSameProxyServingWorker(servingWorkerIdentity, observedRuntime.workerIdentity)) {
|
|
4473
|
+
activatedWorkerIdentity ??= observedRuntime.workerIdentity;
|
|
4474
|
+
}
|
|
4475
|
+
}
|
|
4361
4476
|
healthy = true;
|
|
4362
4477
|
break;
|
|
4363
4478
|
}
|
|
@@ -4367,6 +4482,13 @@ export const proxyGuardCommand = {
|
|
|
4367
4482
|
/* retry */
|
|
4368
4483
|
}
|
|
4369
4484
|
if (rollingSupervisor) {
|
|
4485
|
+
const observedRuntime = await fetchProxyRuntimeVersions(host, port, true);
|
|
4486
|
+
if (observedRuntime.workerIdentity?.version ===
|
|
4487
|
+
result.latestVersion &&
|
|
4488
|
+
servingWorkerIdentity &&
|
|
4489
|
+
!isSameProxyServingWorker(servingWorkerIdentity, observedRuntime.workerIdentity)) {
|
|
4490
|
+
activatedWorkerIdentity ??= observedRuntime.workerIdentity;
|
|
4491
|
+
}
|
|
4370
4492
|
rollingFailure = await getRollingActivationFailure(host, port, result.latestVersion);
|
|
4371
4493
|
if (rollingFailure) {
|
|
4372
4494
|
break;
|
|
@@ -4380,6 +4502,7 @@ export const proxyGuardCommand = {
|
|
|
4380
4502
|
let lastRefreshDeferral = "";
|
|
4381
4503
|
const waitForSupervisorRefresh = async (quietWaitMs) => {
|
|
4382
4504
|
const window = await waitForProxyUpdateWindow({
|
|
4505
|
+
allowBusyDrain: false,
|
|
4383
4506
|
quietThresholdMs: QUIET_THRESHOLD_MS,
|
|
4384
4507
|
quietWaitMs,
|
|
4385
4508
|
drainWaitMs: UPDATE_DRAIN_TIMEOUT_MS,
|
|
@@ -4416,7 +4539,9 @@ export const proxyGuardCommand = {
|
|
|
4416
4539
|
logger.always(`[updater] supervisor refresh deferred: ${refreshWindow.reason ?? "safe window unavailable"}`);
|
|
4417
4540
|
persistUpdaterState("record supervisor refresh deferral", () => recordUpdateDeferred(result.latestVersion, refreshWindow.reason === "drain_timeout"
|
|
4418
4541
|
? "drain_timeout"
|
|
4419
|
-
: "
|
|
4542
|
+
: refreshWindow.reason === "busy"
|
|
4543
|
+
? "waiting_for_quiet"
|
|
4544
|
+
: "drain_unavailable", null));
|
|
4420
4545
|
updateRestartInProgress = false;
|
|
4421
4546
|
scheduleUpdateRetry("supervisor refresh window unavailable", result.latestVersion);
|
|
4422
4547
|
return;
|
|
@@ -4425,7 +4550,21 @@ export const proxyGuardCommand = {
|
|
|
4425
4550
|
updateRestartInProgress = false;
|
|
4426
4551
|
return;
|
|
4427
4552
|
}
|
|
4553
|
+
const refreshIdentity = await fetchProxyRuntimeVersions(host, port, true);
|
|
4554
|
+
const allowedRefreshIdentity = servingWorkerIdentity &&
|
|
4555
|
+
(isSameProxyServingWorker(servingWorkerIdentity, refreshIdentity.workerIdentity) ||
|
|
4556
|
+
(activatedWorkerIdentity !== undefined &&
|
|
4557
|
+
isSameProxyServingWorker(activatedWorkerIdentity, refreshIdentity.workerIdentity)));
|
|
4558
|
+
if (!allowedRefreshIdentity) {
|
|
4559
|
+
updateRestartInProgress = false;
|
|
4560
|
+
scheduleUpdateRetry("serving worker changed before supervisor refresh", result.latestVersion);
|
|
4561
|
+
return;
|
|
4562
|
+
}
|
|
4428
4563
|
logger.always(`[updater] refreshing launchd supervisor at v${result.latestVersion}`);
|
|
4564
|
+
if (!isCurrentUpdateAttemptOwner()) {
|
|
4565
|
+
updateRestartInProgress = false;
|
|
4566
|
+
return;
|
|
4567
|
+
}
|
|
4429
4568
|
const uid = process.getuid?.() ?? 501;
|
|
4430
4569
|
try {
|
|
4431
4570
|
execFileSync("launchctl", ["kickstart", "-k", `gui/${uid}/${PLIST_LABEL}`], { timeout: 10_000, stdio: "pipe" });
|
|
@@ -4473,7 +4612,6 @@ export const proxyGuardCommand = {
|
|
|
4473
4612
|
updateRetryAttempts = 0;
|
|
4474
4613
|
updateRetryVersion = null;
|
|
4475
4614
|
// The replacement proxy starts a worker running the new version.
|
|
4476
|
-
await flushProxyOtelLogs().catch(() => undefined);
|
|
4477
4615
|
await shutdownProxyOtelLogs().catch(() => undefined);
|
|
4478
4616
|
process.exit(0);
|
|
4479
4617
|
}
|
|
@@ -4488,12 +4626,20 @@ export const proxyGuardCommand = {
|
|
|
4488
4626
|
return;
|
|
4489
4627
|
}
|
|
4490
4628
|
try {
|
|
4629
|
+
const rollbackRuntime = await fetchProxyRuntimeVersions(host, port, true);
|
|
4630
|
+
const rollbackStillOwned = servingWorkerIdentity &&
|
|
4631
|
+
(isSameProxyServingWorker(servingWorkerIdentity, rollbackRuntime.workerIdentity) ||
|
|
4632
|
+
(activatedWorkerIdentity !== undefined &&
|
|
4633
|
+
isSameProxyServingWorker(activatedWorkerIdentity, rollbackRuntime.workerIdentity)));
|
|
4634
|
+
if (!rollbackStillOwned) {
|
|
4635
|
+
throw new Error("Serving worker identity changed; stale updater refused rollback");
|
|
4636
|
+
}
|
|
4491
4637
|
if (!previousPackage ||
|
|
4492
4638
|
previousPackage.version !== runningVersion) {
|
|
4493
4639
|
throw new Error("Previous validated package selection is unavailable");
|
|
4494
4640
|
}
|
|
4495
4641
|
logger.always(`[updater] restoring immutable package selection v${runningVersion}`);
|
|
4496
|
-
writeTrampoline(previousPackage,
|
|
4642
|
+
writeTrampoline(previousPackage, isCurrentUpdateAttemptOwner);
|
|
4497
4643
|
const rollbackValidation = await validateInstalledVersion({
|
|
4498
4644
|
binPath: TRAMPOLINE_PATH,
|
|
4499
4645
|
expectedVersion: runningVersion,
|
|
@@ -4511,7 +4657,7 @@ export const proxyGuardCommand = {
|
|
|
4511
4657
|
updateRestartInProgress = false;
|
|
4512
4658
|
return;
|
|
4513
4659
|
}
|
|
4514
|
-
const rollbackActivated = await activateRollbackVersion(host, port, parentPid, runningVersion, UPDATE_TIMEOUT_MS);
|
|
4660
|
+
const rollbackActivated = await activateRollbackVersion(host, port, parentPid, runningVersion, UPDATE_TIMEOUT_MS, isCurrentUpdateAttemptOwner);
|
|
4515
4661
|
if (!rollbackActivated) {
|
|
4516
4662
|
throw new Error(`supervisor did not report v${runningVersion} healthy within ${UPDATE_TIMEOUT_MS}ms after rollback`);
|
|
4517
4663
|
}
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* - stats: Show token/cost stats
|
|
11
11
|
*/
|
|
12
12
|
import type { CommandModule } from "yargs";
|
|
13
|
-
import type { TelemetryStatusArgs as StatusArgs, TelemetryConfigureArgs as ConfigureArgs, TelemetryListExportersArgs as ListExportersArgs, TelemetryFlushArgs as FlushArgs, TelemetryStatsArgs as StatsArgs } from "../../types/index.js";
|
|
13
|
+
import type { TelemetryStatusArgs as StatusArgs, TelemetryConfigureArgs as ConfigureArgs, TelemetryListExportersArgs as ListExportersArgs, TelemetryFlushArgs as FlushArgs, TelemetryStatsArgs as StatsArgs, TelemetryNativePrepareArgs, TelemetryNativeValidateArgs } from "../../types/index.js";
|
|
14
14
|
/**
|
|
15
15
|
* Telemetry Command Factory
|
|
16
16
|
*/
|
|
@@ -19,6 +19,9 @@ export declare class TelemetryCommandFactory {
|
|
|
19
19
|
* Create the telemetry command group
|
|
20
20
|
*/
|
|
21
21
|
static createTelemetryCommands(): CommandModule<object, object>;
|
|
22
|
+
/** Stage a native migration without modifying an installed service. */
|
|
23
|
+
static createNativePrepareCommand(): CommandModule<object, TelemetryNativePrepareArgs>;
|
|
24
|
+
static createNativeValidateCommand(): CommandModule<object, TelemetryNativeValidateArgs>;
|
|
22
25
|
/**
|
|
23
26
|
* Create the status subcommand
|
|
24
27
|
*/
|
|
@@ -16,6 +16,7 @@ import { NeuroLink } from "../../neurolink.js";
|
|
|
16
16
|
import { flushOpenTelemetry } from "../../services/server/ai/observability/instrumentation.js";
|
|
17
17
|
import { formatRow, formatCost } from "../utils/formatters.js";
|
|
18
18
|
import { redactUrlCredentials } from "../../utils/logSanitize.js";
|
|
19
|
+
import { prepareNativeTelemetry, validateNativeTelemetry, } from "../utils/nativeTelemetry.js";
|
|
19
20
|
/**
|
|
20
21
|
* Available exporter names
|
|
21
22
|
*/
|
|
@@ -49,6 +50,8 @@ export class TelemetryCommandFactory {
|
|
|
49
50
|
.command(TelemetryCommandFactory.createListExportersCommand())
|
|
50
51
|
.command(TelemetryCommandFactory.createFlushCommand())
|
|
51
52
|
.command(TelemetryCommandFactory.createStatsCommand())
|
|
53
|
+
.command(TelemetryCommandFactory.createNativePrepareCommand())
|
|
54
|
+
.command(TelemetryCommandFactory.createNativeValidateCommand())
|
|
52
55
|
.demandCommand(1, "Please specify a subcommand")
|
|
53
56
|
.strict();
|
|
54
57
|
},
|
|
@@ -57,6 +60,116 @@ export class TelemetryCommandFactory {
|
|
|
57
60
|
},
|
|
58
61
|
};
|
|
59
62
|
}
|
|
63
|
+
/** Stage a native migration without modifying an installed service. */
|
|
64
|
+
static createNativePrepareCommand() {
|
|
65
|
+
return {
|
|
66
|
+
command: "native-prepare",
|
|
67
|
+
describe: "Stage private durable/split collector files; never activate a service",
|
|
68
|
+
builder: (yargs) => yargs
|
|
69
|
+
.option("collectorConfig", {
|
|
70
|
+
alias: "collector-config",
|
|
71
|
+
type: "string",
|
|
72
|
+
demandOption: true,
|
|
73
|
+
describe: "Existing native collector YAML (read only)",
|
|
74
|
+
})
|
|
75
|
+
.option("output", {
|
|
76
|
+
type: "string",
|
|
77
|
+
demandOption: true,
|
|
78
|
+
describe: "New absolute staging directory; existing paths are refused",
|
|
79
|
+
})
|
|
80
|
+
.option("queueDirectory", {
|
|
81
|
+
alias: "queue-directory",
|
|
82
|
+
type: "string",
|
|
83
|
+
demandOption: true,
|
|
84
|
+
describe: "Planned private durable queue directory (not created)",
|
|
85
|
+
})
|
|
86
|
+
.option("compactionDirectory", {
|
|
87
|
+
alias: "compaction-directory",
|
|
88
|
+
type: "string",
|
|
89
|
+
demandOption: true,
|
|
90
|
+
describe: "Planned separate private compaction directory (not created)",
|
|
91
|
+
})
|
|
92
|
+
.option("bodyPort", {
|
|
93
|
+
alias: "body-port",
|
|
94
|
+
type: "number",
|
|
95
|
+
default: 14319,
|
|
96
|
+
})
|
|
97
|
+
.option("bodyStream", {
|
|
98
|
+
alias: "body-stream",
|
|
99
|
+
type: "string",
|
|
100
|
+
describe: "Separate body stream; defaults to <existing metadata stream>_bodies",
|
|
101
|
+
})
|
|
102
|
+
.option("metadataQueueMib", {
|
|
103
|
+
alias: "metadata-queue-mib",
|
|
104
|
+
type: "number",
|
|
105
|
+
default: 32,
|
|
106
|
+
describe: "Payload limit per metadata signal (three queues)",
|
|
107
|
+
})
|
|
108
|
+
.option("bodyQueueMib", {
|
|
109
|
+
alias: "body-queue-mib",
|
|
110
|
+
type: "number",
|
|
111
|
+
default: 256,
|
|
112
|
+
})
|
|
113
|
+
.option("diskQuotaMib", {
|
|
114
|
+
alias: "disk-quota-mib",
|
|
115
|
+
type: "number",
|
|
116
|
+
default: 1024,
|
|
117
|
+
describe: "Operator-managed volume quota to review; not applied by this command",
|
|
118
|
+
})
|
|
119
|
+
.option("metadataRetentionDays", {
|
|
120
|
+
alias: "metadata-retention-days",
|
|
121
|
+
type: "number",
|
|
122
|
+
default: 14,
|
|
123
|
+
describe: "Planned backend retention; not applied",
|
|
124
|
+
})
|
|
125
|
+
.option("bodyRetentionDays", {
|
|
126
|
+
alias: "body-retention-days",
|
|
127
|
+
type: "number",
|
|
128
|
+
default: 3,
|
|
129
|
+
describe: "Planned backend body retention; not applied",
|
|
130
|
+
}),
|
|
131
|
+
handler: async (args) => {
|
|
132
|
+
try {
|
|
133
|
+
logger.always(JSON.stringify(await prepareNativeTelemetry(args), null, 2));
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
logger.error(error instanceof Error
|
|
137
|
+
? error.message
|
|
138
|
+
: "Native telemetry preparation failed");
|
|
139
|
+
process.exitCode = 1;
|
|
140
|
+
}
|
|
141
|
+
},
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
static createNativeValidateCommand() {
|
|
145
|
+
return {
|
|
146
|
+
command: "native-validate",
|
|
147
|
+
describe: "Validate staged profile using collector >= 0.160; never start or restart a service",
|
|
148
|
+
builder: (yargs) => yargs
|
|
149
|
+
.option("directory", {
|
|
150
|
+
type: "string",
|
|
151
|
+
demandOption: true,
|
|
152
|
+
describe: "Private directory produced by native-prepare",
|
|
153
|
+
})
|
|
154
|
+
.option("collectorBin", {
|
|
155
|
+
alias: "collector-bin",
|
|
156
|
+
type: "string",
|
|
157
|
+
demandOption: true,
|
|
158
|
+
describe: "Absolute path to trusted otelcol-contrib executable",
|
|
159
|
+
}),
|
|
160
|
+
handler: async (args) => {
|
|
161
|
+
try {
|
|
162
|
+
logger.always(JSON.stringify(await validateNativeTelemetry(args), null, 2));
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
logger.error(error instanceof Error
|
|
166
|
+
? error.message
|
|
167
|
+
: "Native telemetry validation failed");
|
|
168
|
+
process.exitCode = 1;
|
|
169
|
+
}
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
}
|
|
60
173
|
/**
|
|
61
174
|
* Create the status subcommand
|
|
62
175
|
*/
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { TelemetryNativePrepareArgs, TelemetryNativeValidateArgs } from "../../types/index.js";
|
|
2
|
+
/** Produce new private files only. Existing config, services and queues are untouched. */
|
|
3
|
+
export declare function prepareNativeTelemetry(args: TelemetryNativePrepareArgs): Promise<{
|
|
4
|
+
directory: string;
|
|
5
|
+
prepared: boolean;
|
|
6
|
+
activated: boolean;
|
|
7
|
+
metadataStream: string;
|
|
8
|
+
bodyStream: string;
|
|
9
|
+
pending: string[];
|
|
10
|
+
}>;
|
|
11
|
+
/** Runs only the collector's static validator, with test-owned temporary storage. */
|
|
12
|
+
export declare function validateNativeTelemetry(args: TelemetryNativeValidateArgs): Promise<{
|
|
13
|
+
directory: string;
|
|
14
|
+
valid: boolean;
|
|
15
|
+
collectorVersion: string;
|
|
16
|
+
activated: boolean;
|
|
17
|
+
runtimeVerified: boolean;
|
|
18
|
+
pending: string[];
|
|
19
|
+
}>;
|