@juspay/neurolink 12.14.13 → 12.14.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +309 -309
- package/dist/cli/commands/proxy.js +171 -17
- package/dist/providers/anthropic/client.js +13 -1
- package/dist/proxy/otelLogSink.js +121 -37
- package/dist/proxy/proxyLifecycle.js +4 -0
- package/dist/types/proxy.d.ts +4 -0
- package/docs-site/static/search-index.json +1 -1
- package/package.json +2 -1
- package/scripts/observability/docker-compose.proxy-observability.yaml +4 -0
- package/scripts/observability/otel-collector.proxy-observability.yaml +19 -1
- package/scripts/observability/proxy-observability.env.example +4 -0
- package/scripts/observability/proxy-telemetry-check.mjs +145 -28
|
@@ -544,6 +544,33 @@ async function fetchProxyHealthVersion(host, port) {
|
|
|
544
544
|
return undefined;
|
|
545
545
|
}
|
|
546
546
|
}
|
|
547
|
+
/** Read both serving generations so an updated worker cannot hide a stale parent. */
|
|
548
|
+
async function fetchProxyRuntimeVersions(host, port) {
|
|
549
|
+
try {
|
|
550
|
+
const response = await fetch(`http://${host}:${port}/status`, {
|
|
551
|
+
headers: { connection: "close" },
|
|
552
|
+
signal: AbortSignal.timeout(3_000),
|
|
553
|
+
});
|
|
554
|
+
if (!response.ok) {
|
|
555
|
+
return {};
|
|
556
|
+
}
|
|
557
|
+
const data = (await response.json());
|
|
558
|
+
return {
|
|
559
|
+
...(typeof data.version === "string"
|
|
560
|
+
? { workerVersion: data.version }
|
|
561
|
+
: {}),
|
|
562
|
+
...(typeof data.autoUpdate?.supervisorVersion === "string"
|
|
563
|
+
? { supervisorVersion: data.autoUpdate.supervisorVersion }
|
|
564
|
+
: {}),
|
|
565
|
+
...(typeof data.autoUpdate?.supervisorPid === "number"
|
|
566
|
+
? { supervisorPid: data.autoUpdate.supervisorPid }
|
|
567
|
+
: {}),
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
catch {
|
|
571
|
+
return {};
|
|
572
|
+
}
|
|
573
|
+
}
|
|
547
574
|
/**
|
|
548
575
|
* After a failed rolling update, re-point the live supervisor at the reinstalled
|
|
549
576
|
* running version (mirroring the forward path: record the pending target, then
|
|
@@ -576,7 +603,7 @@ async function activateRollbackVersion(host, port, parentPid, runningVersion, ti
|
|
|
576
603
|
abandonPendingUpdate(runningVersion);
|
|
577
604
|
}
|
|
578
605
|
}
|
|
579
|
-
async function setProxyUpdateDrain(host, port, draining) {
|
|
606
|
+
async function setProxyUpdateDrain(host, port, draining, supervisorRefresh = false) {
|
|
580
607
|
const confirmState = async () => {
|
|
581
608
|
try {
|
|
582
609
|
const response = await fetch(`http://${host}:${port}/status`, {
|
|
@@ -599,7 +626,13 @@ async function setProxyUpdateDrain(host, port, draining) {
|
|
|
599
626
|
"content-type": "application/json",
|
|
600
627
|
"x-neurolink-update-token": PROXY_UPDATE_CONTROL_TOKEN,
|
|
601
628
|
},
|
|
602
|
-
body: JSON.stringify({
|
|
629
|
+
body: JSON.stringify({
|
|
630
|
+
action: draining
|
|
631
|
+
? supervisorRefresh
|
|
632
|
+
? "drain_for_supervisor_refresh"
|
|
633
|
+
: "drain"
|
|
634
|
+
: "resume",
|
|
635
|
+
}),
|
|
603
636
|
signal: AbortSignal.timeout(5_000),
|
|
604
637
|
});
|
|
605
638
|
if (!response.ok) {
|
|
@@ -1426,6 +1459,22 @@ export async function createProxyStartApp(params) {
|
|
|
1426
1459
|
},
|
|
1427
1460
|
}, status);
|
|
1428
1461
|
});
|
|
1462
|
+
let supervisorRefreshDrainLease;
|
|
1463
|
+
const scheduleSupervisorRefreshResume = (delayMs) => {
|
|
1464
|
+
clearTimeout(supervisorRefreshDrainLease);
|
|
1465
|
+
const resumeWhenReady = () => {
|
|
1466
|
+
if (resumeProxyConnections(readiness)) {
|
|
1467
|
+
supervisorRefreshDrainLease = undefined;
|
|
1468
|
+
return;
|
|
1469
|
+
}
|
|
1470
|
+
// Readiness can be transiently false during worker replacement. Keep a
|
|
1471
|
+
// recovery lease until admission has actually reopened.
|
|
1472
|
+
supervisorRefreshDrainLease = setTimeout(resumeWhenReady, 30_000);
|
|
1473
|
+
supervisorRefreshDrainLease.unref?.();
|
|
1474
|
+
};
|
|
1475
|
+
supervisorRefreshDrainLease = setTimeout(resumeWhenReady, delayMs);
|
|
1476
|
+
supervisorRefreshDrainLease.unref?.();
|
|
1477
|
+
};
|
|
1429
1478
|
app.post("/internal/update-control", async (c) => {
|
|
1430
1479
|
const suppliedToken = c.req.header("x-neurolink-update-token");
|
|
1431
1480
|
const expectedToken = params.updateControlToken ?? PROXY_UPDATE_CONTROL_TOKEN;
|
|
@@ -1435,11 +1484,13 @@ export async function createProxyStartApp(params) {
|
|
|
1435
1484
|
const payload = await c.req
|
|
1436
1485
|
.json()
|
|
1437
1486
|
.catch(() => ({ action: undefined }));
|
|
1438
|
-
if (payload.action === "drain"
|
|
1487
|
+
if (payload.action === "drain" ||
|
|
1488
|
+
payload.action === "drain_for_supervisor_refresh") {
|
|
1439
1489
|
// A rolling worker must keep admitting until the supervisor has a ready
|
|
1440
1490
|
// replacement. The legacy global drain can otherwise strand the listener
|
|
1441
1491
|
// behind maintenance responses when its caller stalls or disappears.
|
|
1442
|
-
if (isProxySocketWorkerProcess()
|
|
1492
|
+
if (isProxySocketWorkerProcess() &&
|
|
1493
|
+
payload.action !== "drain_for_supervisor_refresh") {
|
|
1443
1494
|
return c.json({
|
|
1444
1495
|
error: "rolling_restart_required",
|
|
1445
1496
|
message: "Use neurolink proxy restart; global update drain is disabled for rolling workers.",
|
|
@@ -1448,11 +1499,18 @@ export async function createProxyStartApp(params) {
|
|
|
1448
1499
|
if (!markProxyDrainingForUpdate(readiness)) {
|
|
1449
1500
|
return c.json({ error: "proxy_not_ready" }, 409);
|
|
1450
1501
|
}
|
|
1502
|
+
if (payload.action === "drain_for_supervisor_refresh") {
|
|
1503
|
+
// A crashed updater must not strand the current rolling worker in
|
|
1504
|
+
// maintenance mode. A successful launchd refresh replaces this process.
|
|
1505
|
+
scheduleSupervisorRefreshResume(35 * 60 * 1000);
|
|
1506
|
+
}
|
|
1451
1507
|
}
|
|
1452
1508
|
else if (payload.action === "resume") {
|
|
1453
1509
|
if (!resumeProxyConnections(readiness)) {
|
|
1454
1510
|
return c.json({ error: "proxy_not_ready" }, 409);
|
|
1455
1511
|
}
|
|
1512
|
+
clearTimeout(supervisorRefreshDrainLease);
|
|
1513
|
+
supervisorRefreshDrainLease = undefined;
|
|
1456
1514
|
}
|
|
1457
1515
|
else {
|
|
1458
1516
|
return c.json({ error: "invalid_action" }, 400);
|
|
@@ -3955,8 +4013,6 @@ export const proxyGuardCommand = {
|
|
|
3955
4013
|
return;
|
|
3956
4014
|
}
|
|
3957
4015
|
persistUpdaterState("record installed update", () => recordUpdateInstalled(result.latestVersion));
|
|
3958
|
-
updateRetryAttempts = 0;
|
|
3959
|
-
updateRetryVersion = null;
|
|
3960
4016
|
logger.always(`[updater] trampoline validated at v${validation.version} after ${validation.attempts} attempt(s)`);
|
|
3961
4017
|
}
|
|
3962
4018
|
catch (trampolineError) {
|
|
@@ -3973,18 +4029,24 @@ export const proxyGuardCommand = {
|
|
|
3973
4029
|
}
|
|
3974
4030
|
updateRestartInProgress = true;
|
|
3975
4031
|
if (rollingSupervisor) {
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
|
|
4032
|
+
const activeVersions = await fetchProxyRuntimeVersions(host, port);
|
|
4033
|
+
if (activeVersions.workerVersion === result.latestVersion) {
|
|
4034
|
+
logger.always(`[updater] worker v${result.latestVersion} is already active; resuming supervisor refresh without another worker replacement`);
|
|
3979
4035
|
}
|
|
3980
|
-
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
|
|
3984
|
-
|
|
3985
|
-
|
|
3986
|
-
|
|
3987
|
-
|
|
4036
|
+
else {
|
|
4037
|
+
logger.always(`[updater] requesting rolling activation of v${result.latestVersion}`);
|
|
4038
|
+
try {
|
|
4039
|
+
process.kill(parentPid, "SIGUSR2");
|
|
4040
|
+
}
|
|
4041
|
+
catch (activationError) {
|
|
4042
|
+
updateRestartInProgress = false;
|
|
4043
|
+
const message = activationError instanceof Error
|
|
4044
|
+
? activationError.message
|
|
4045
|
+
: String(activationError);
|
|
4046
|
+
logger.always(`[updater] WARNING: rolling activation request failed: ${message}`);
|
|
4047
|
+
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "restart", message));
|
|
4048
|
+
return;
|
|
4049
|
+
}
|
|
3988
4050
|
}
|
|
3989
4051
|
}
|
|
3990
4052
|
else {
|
|
@@ -4037,9 +4099,101 @@ export const proxyGuardCommand = {
|
|
|
4037
4099
|
}
|
|
4038
4100
|
}
|
|
4039
4101
|
}
|
|
4102
|
+
if (healthy && rollingSupervisor) {
|
|
4103
|
+
const beforeRefresh = await fetchProxyRuntimeVersions(host, port);
|
|
4104
|
+
if (beforeRefresh.supervisorVersion !== result.latestVersion) {
|
|
4105
|
+
logger.always(`[updater] worker v${result.latestVersion} is healthy; waiting for a safe supervisor refresh (supervisor v${beforeRefresh.supervisorVersion ?? "unknown"})`);
|
|
4106
|
+
let lastRefreshDeferral = "";
|
|
4107
|
+
const waitForSupervisorRefresh = async (quietWaitMs) => {
|
|
4108
|
+
const window = await waitForProxyUpdateWindow({
|
|
4109
|
+
quietThresholdMs: QUIET_THRESHOLD_MS,
|
|
4110
|
+
quietWaitMs,
|
|
4111
|
+
drainWaitMs: UPDATE_DRAIN_TIMEOUT_MS,
|
|
4112
|
+
pollIntervalMs: UPDATE_ACTIVITY_POLL_MS,
|
|
4113
|
+
getActivity: () => getProxyRuntimeActivity(host, port),
|
|
4114
|
+
setDraining: async (draining) => {
|
|
4115
|
+
const changed = await setProxyUpdateDrain(host, port, draining, true);
|
|
4116
|
+
if (changed) {
|
|
4117
|
+
logger.always(`[updater] ${draining ? "draining new inference requests for supervisor refresh" : "inference admission resumed"}`);
|
|
4118
|
+
}
|
|
4119
|
+
return changed;
|
|
4120
|
+
},
|
|
4121
|
+
isStopping: () => guardStopping,
|
|
4122
|
+
isParentAlive: () => getProcessStatus(parentPid) !== "not_running",
|
|
4123
|
+
onPhase: (phase, activity) => {
|
|
4124
|
+
const signature = `${phase}:${activity?.activeRequests ?? "unknown"}`;
|
|
4125
|
+
if (signature !== lastRefreshDeferral) {
|
|
4126
|
+
lastRefreshDeferral = signature;
|
|
4127
|
+
persistUpdaterState("record supervisor refresh deferral", () => recordUpdateDeferred(result.latestVersion, phase, activity?.activeRequests ?? null));
|
|
4128
|
+
}
|
|
4129
|
+
logger.debug(`[updater] supervisor refresh ${phase} (${activity?.activeRequests ?? "unknown"} active requests)`);
|
|
4130
|
+
},
|
|
4131
|
+
});
|
|
4132
|
+
drainActive = drainActive || window.draining;
|
|
4133
|
+
return window;
|
|
4134
|
+
};
|
|
4135
|
+
let refreshWindow = await waitForSupervisorRefresh(NATURAL_WINDOW_WAIT_MS);
|
|
4136
|
+
// A naturally quiet result still needs a short admission fence to
|
|
4137
|
+
// close the race between the last activity poll and launchd.
|
|
4138
|
+
if (refreshWindow.ready && !refreshWindow.draining) {
|
|
4139
|
+
refreshWindow = await waitForSupervisorRefresh(0);
|
|
4140
|
+
}
|
|
4141
|
+
if (!refreshWindow.ready) {
|
|
4142
|
+
logger.always(`[updater] supervisor refresh deferred: ${refreshWindow.reason ?? "safe window unavailable"}`);
|
|
4143
|
+
persistUpdaterState("record supervisor refresh deferral", () => recordUpdateDeferred(result.latestVersion, refreshWindow.reason === "drain_timeout"
|
|
4144
|
+
? "drain_timeout"
|
|
4145
|
+
: "drain_unavailable", null));
|
|
4146
|
+
updateRestartInProgress = false;
|
|
4147
|
+
scheduleUpdateRetry("supervisor refresh window unavailable", result.latestVersion);
|
|
4148
|
+
return;
|
|
4149
|
+
}
|
|
4150
|
+
logger.always(`[updater] refreshing launchd supervisor at v${result.latestVersion}`);
|
|
4151
|
+
const uid = process.getuid?.() ?? 501;
|
|
4152
|
+
try {
|
|
4153
|
+
execFileSync("launchctl", ["kickstart", "-k", `gui/${uid}/${PLIST_LABEL}`], { timeout: 10_000, stdio: "pipe" });
|
|
4154
|
+
// The drained worker belongs to the replaced service. The new
|
|
4155
|
+
// supervisor starts with admission open.
|
|
4156
|
+
drainActive = false;
|
|
4157
|
+
}
|
|
4158
|
+
catch (refreshError) {
|
|
4159
|
+
const message = refreshError instanceof Error
|
|
4160
|
+
? refreshError.message
|
|
4161
|
+
: String(refreshError);
|
|
4162
|
+
logger.always(`[updater] WARNING: supervisor refresh failed: ${message}`);
|
|
4163
|
+
persistUpdaterState("record supervisor refresh failure", () => recordUpdateFailure(result.latestVersion, "restart", `supervisor refresh failed: ${message}`));
|
|
4164
|
+
updateRestartInProgress = false;
|
|
4165
|
+
scheduleUpdateRetry("supervisor refresh failed", result.latestVersion);
|
|
4166
|
+
return;
|
|
4167
|
+
}
|
|
4168
|
+
const refreshStartedAt = Date.now();
|
|
4169
|
+
const previousSupervisorPid = beforeRefresh.supervisorPid ?? parentPid;
|
|
4170
|
+
healthy = false;
|
|
4171
|
+
while (Date.now() - refreshStartedAt < 120_000) {
|
|
4172
|
+
await sleep(2_000);
|
|
4173
|
+
const versions = await fetchProxyRuntimeVersions(host, port);
|
|
4174
|
+
if (versions.workerVersion === result.latestVersion &&
|
|
4175
|
+
versions.supervisorVersion === result.latestVersion &&
|
|
4176
|
+
versions.supervisorPid !== undefined &&
|
|
4177
|
+
versions.supervisorPid !== previousSupervisorPid) {
|
|
4178
|
+
healthy = true;
|
|
4179
|
+
break;
|
|
4180
|
+
}
|
|
4181
|
+
}
|
|
4182
|
+
if (!healthy) {
|
|
4183
|
+
const message = `supervisor did not report v${result.latestVersion} with a new process within 120000ms`;
|
|
4184
|
+
logger.always(`[updater] WARNING: ${message}`);
|
|
4185
|
+
persistUpdaterState("record supervisor refresh failure", () => recordUpdateFailure(result.latestVersion, "health", message));
|
|
4186
|
+
updateRestartInProgress = false;
|
|
4187
|
+
scheduleUpdateRetry("supervisor refresh health unavailable", result.latestVersion);
|
|
4188
|
+
return;
|
|
4189
|
+
}
|
|
4190
|
+
}
|
|
4191
|
+
}
|
|
4040
4192
|
if (healthy) {
|
|
4041
4193
|
logger.always(`[updater] update successful: now running ${result.latestVersion}`);
|
|
4042
4194
|
persistUpdaterState("record successful update", () => recordSuccessfulUpdate(result.latestVersion));
|
|
4195
|
+
updateRetryAttempts = 0;
|
|
4196
|
+
updateRetryVersion = null;
|
|
4043
4197
|
// The replacement proxy starts a worker running the new version.
|
|
4044
4198
|
await flushProxyOtelLogs().catch(() => undefined);
|
|
4045
4199
|
await shutdownProxyOtelLogs().catch(() => undefined);
|
|
@@ -1317,7 +1317,19 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1317
1317
|
// stop_reason "tool_use", but no tool call is surfaced, so
|
|
1318
1318
|
// reporting "tool-calls" would misread as a step-capped turn.
|
|
1319
1319
|
// `raw` still carries the provider's verbatim stop_reason.
|
|
1320
|
-
|
|
1320
|
+
//
|
|
1321
|
+
// Only "tool-calls" is substituted, never every reason. The turn
|
|
1322
|
+
// can also end at stop_reason "max_tokens" — the model was still
|
|
1323
|
+
// writing the final_result arguments when the output ceiling hit,
|
|
1324
|
+
// so the structured payload is cut mid-JSON. Reporting that as
|
|
1325
|
+
// "stop" told the caller the response was complete: neurolink.ts's
|
|
1326
|
+
// `finishReason === "length"` check never fired, so `jsonTruncated`
|
|
1327
|
+
// stayed unset and no truncation warning was logged, leaving a
|
|
1328
|
+
// partial object indistinguishable from a whole one. This mirrors
|
|
1329
|
+
// the streaming path, which already narrows the same substitution
|
|
1330
|
+
// to a "tool-calls" reason.
|
|
1331
|
+
unified: finalResultText !== undefined &&
|
|
1332
|
+
mapAnthropicStopReason(response.stop_reason) === "tool-calls"
|
|
1321
1333
|
? "stop"
|
|
1322
1334
|
: mapAnthropicStopReason(response.stop_reason),
|
|
1323
1335
|
raw: response.stop_reason ?? "stop",
|
|
@@ -14,8 +14,13 @@ let provider;
|
|
|
14
14
|
let restoreConsole;
|
|
15
15
|
const queues = [];
|
|
16
16
|
const bodyPublications = new Map();
|
|
17
|
+
const bodyPublicationDeadlines = new WeakMap();
|
|
17
18
|
let bodyPublicationChain = Promise.resolve();
|
|
18
19
|
let shuttingDown = false;
|
|
20
|
+
const BODY_OTLP_CHUNK_SIZE = 128 * 1024;
|
|
21
|
+
const OTLP_EXPORT_TIMEOUT_MS = 30_000;
|
|
22
|
+
const OTLP_EXPORT_CALLBACK_DEADLINE_MS = OTLP_EXPORT_TIMEOUT_MS + 1_000;
|
|
23
|
+
const BODY_PUBLICATION_ACTIVE_DEADLINE_MS = 20_000;
|
|
19
24
|
const bodyDelivery = {
|
|
20
25
|
attempted: 0,
|
|
21
26
|
transportAcknowledged: 0,
|
|
@@ -94,14 +99,17 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
94
99
|
}
|
|
95
100
|
});
|
|
96
101
|
};
|
|
97
|
-
const transport = new OTLPLogExporter({
|
|
102
|
+
const transport = new OTLPLogExporter({
|
|
103
|
+
url,
|
|
104
|
+
timeoutMillis: OTLP_EXPORT_TIMEOUT_MS,
|
|
105
|
+
});
|
|
98
106
|
const exporter = {
|
|
99
107
|
export(records, callback) {
|
|
100
108
|
let settled = false;
|
|
101
109
|
const deadline = setTimeout(() => settle({
|
|
102
110
|
code: ExportResultCode.FAILED,
|
|
103
111
|
error: new Error("OTLP export callback deadline exceeded"),
|
|
104
|
-
}),
|
|
112
|
+
}), OTLP_EXPORT_CALLBACK_DEADLINE_MS);
|
|
105
113
|
deadline.unref();
|
|
106
114
|
const settle = (result) => {
|
|
107
115
|
if (settled) {
|
|
@@ -125,7 +133,9 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
125
133
|
const id = record.attributes?.["body.capture_id"];
|
|
126
134
|
const publication = typeof id === "string" ? bodyPublications.get(id) : undefined;
|
|
127
135
|
if (publication) {
|
|
128
|
-
|
|
136
|
+
const activeDeadline = bodyPublicationDeadlines.get(publication);
|
|
137
|
+
if (result.code === ExportResultCode.SUCCESS &&
|
|
138
|
+
(activeDeadline === undefined || Date.now() < activeDeadline)) {
|
|
129
139
|
publication.acknowledged++;
|
|
130
140
|
}
|
|
131
141
|
else {
|
|
@@ -155,7 +165,7 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
155
165
|
maxQueueSize: capacity,
|
|
156
166
|
maxExportBatchSize: 64,
|
|
157
167
|
scheduledDelayMillis: 1000,
|
|
158
|
-
exportTimeoutMillis:
|
|
168
|
+
exportTimeoutMillis: OTLP_EXPORT_CALLBACK_DEADLINE_MS,
|
|
159
169
|
});
|
|
160
170
|
const processor = {
|
|
161
171
|
onEmit(record) {
|
|
@@ -247,21 +257,25 @@ export async function publishProxyOtelBody(captureId, body, emit) {
|
|
|
247
257
|
emitted: 0,
|
|
248
258
|
};
|
|
249
259
|
bodyPublications.set(captureId, progress);
|
|
250
|
-
|
|
260
|
+
let deferredRelease;
|
|
251
261
|
const operation = bodyPublicationChain.then(async () => {
|
|
252
262
|
const queue = queues.find((candidate) => candidate.kind === "bodies");
|
|
253
|
-
if (!queue
|
|
263
|
+
if (!queue) {
|
|
254
264
|
return {
|
|
255
265
|
status: "rejected",
|
|
256
266
|
acknowledgedChunks: 0,
|
|
257
267
|
unconfirmedChunks: 0,
|
|
258
268
|
droppedChunks: 0,
|
|
259
|
-
reason:
|
|
260
|
-
? "body_exporter_unavailable"
|
|
261
|
-
: "body_publication_deadline",
|
|
269
|
+
reason: "body_exporter_unavailable",
|
|
262
270
|
};
|
|
263
271
|
}
|
|
264
|
-
|
|
272
|
+
// A capture owns bounded publication memory while waiting behind earlier
|
|
273
|
+
// captures. Start its active deadline only when it reaches the head of
|
|
274
|
+
// the chain; healthy queued captures must not expire solely because an
|
|
275
|
+
// earlier export used its transport timeout.
|
|
276
|
+
const deadline = Date.now() + BODY_PUBLICATION_ACTIVE_DEADLINE_MS;
|
|
277
|
+
bodyPublicationDeadlines.set(progress, deadline);
|
|
278
|
+
const chunks = splitUtf8StringByBytes(body, BODY_OTLP_CHUNK_SIZE);
|
|
265
279
|
const awaitSettlement = () => new Promise((resolve) => {
|
|
266
280
|
const check = () => {
|
|
267
281
|
if (progress.acknowledged + progress.unconfirmed + progress.dropped >=
|
|
@@ -273,6 +287,60 @@ export async function publishProxyOtelBody(captureId, body, emit) {
|
|
|
273
287
|
progress.notify = check;
|
|
274
288
|
check();
|
|
275
289
|
});
|
|
290
|
+
const resultFromProgress = (snapshot, reason) => {
|
|
291
|
+
const pending = Math.max(0, snapshot.emitted -
|
|
292
|
+
snapshot.acknowledged -
|
|
293
|
+
snapshot.unconfirmed -
|
|
294
|
+
snapshot.dropped);
|
|
295
|
+
const unconfirmedChunks = snapshot.unconfirmed + pending;
|
|
296
|
+
const status = snapshot.emitted === 0 && chunks.length > 0
|
|
297
|
+
? "rejected"
|
|
298
|
+
: snapshot.dropped || snapshot.emitted !== chunks.length
|
|
299
|
+
? "partial"
|
|
300
|
+
: unconfirmedChunks
|
|
301
|
+
? "export_unconfirmed"
|
|
302
|
+
: "transport_acknowledged";
|
|
303
|
+
return {
|
|
304
|
+
status,
|
|
305
|
+
expectedChunks: chunks.length,
|
|
306
|
+
acknowledgedChunks: snapshot.acknowledged,
|
|
307
|
+
unconfirmedChunks,
|
|
308
|
+
droppedChunks: snapshot.dropped,
|
|
309
|
+
notSubmittedChunks: chunks.length - snapshot.emitted,
|
|
310
|
+
...(reason ? { reason } : {}),
|
|
311
|
+
};
|
|
312
|
+
};
|
|
313
|
+
const settleCurrentBatch = async () => {
|
|
314
|
+
const currentProgress = () => ({
|
|
315
|
+
acknowledged: progress.acknowledged,
|
|
316
|
+
unconfirmed: progress.unconfirmed,
|
|
317
|
+
dropped: progress.dropped,
|
|
318
|
+
emitted: progress.emitted,
|
|
319
|
+
});
|
|
320
|
+
const settlement = queue.processor.forceFlush().then(awaitSettlement);
|
|
321
|
+
const remainingMs = deadline - Date.now();
|
|
322
|
+
if (remainingMs <= 0) {
|
|
323
|
+
deferredRelease = settlement.catch(() => undefined);
|
|
324
|
+
return resultFromProgress(currentProgress(), "body_publication_deadline");
|
|
325
|
+
}
|
|
326
|
+
let timer;
|
|
327
|
+
const completed = await Promise.race([
|
|
328
|
+
settlement.then(() => true),
|
|
329
|
+
new Promise((resolve) => {
|
|
330
|
+
timer = setTimeout(() => resolve(false), remainingMs);
|
|
331
|
+
timer.unref?.();
|
|
332
|
+
}),
|
|
333
|
+
]).finally(() => clearTimeout(timer));
|
|
334
|
+
if (!completed || Date.now() >= deadline) {
|
|
335
|
+
if (!completed) {
|
|
336
|
+
// The exporter callback owns the submitted records until it settles.
|
|
337
|
+
// Keep the capture identity and memory reservation alive meanwhile.
|
|
338
|
+
deferredRelease = settlement.catch(() => undefined);
|
|
339
|
+
}
|
|
340
|
+
return resultFromProgress(currentProgress(), "body_publication_deadline");
|
|
341
|
+
}
|
|
342
|
+
return undefined;
|
|
343
|
+
};
|
|
276
344
|
let reason;
|
|
277
345
|
try {
|
|
278
346
|
for (let offset = 0; offset < chunks.length; offset += 64) {
|
|
@@ -283,16 +351,18 @@ export async function publishProxyOtelBody(captureId, body, emit) {
|
|
|
283
351
|
// Ordinary body records from an external logger may share this queue.
|
|
284
352
|
// Wait for them before admitting any part of this batch.
|
|
285
353
|
while (queue.state.outstanding > queue.capacity - 64) {
|
|
286
|
-
await
|
|
287
|
-
if (
|
|
288
|
-
|
|
354
|
+
const deadlineResult = await settleCurrentBatch();
|
|
355
|
+
if (deadlineResult) {
|
|
356
|
+
return deadlineResult;
|
|
289
357
|
}
|
|
290
358
|
}
|
|
291
359
|
for (let i = offset; i < Math.min(offset + 64, chunks.length); i++) {
|
|
292
360
|
emit(chunks[i], i, chunks.length);
|
|
293
361
|
}
|
|
294
|
-
await
|
|
295
|
-
|
|
362
|
+
const deadlineResult = await settleCurrentBatch();
|
|
363
|
+
if (deadlineResult) {
|
|
364
|
+
return deadlineResult;
|
|
365
|
+
}
|
|
296
366
|
if (progress.unconfirmed || progress.dropped) {
|
|
297
367
|
reason = "body_export_unconfirmed";
|
|
298
368
|
break;
|
|
@@ -306,26 +376,32 @@ export async function publishProxyOtelBody(captureId, body, emit) {
|
|
|
306
376
|
error.message === "body_publication_deadline"
|
|
307
377
|
? "body_publication_deadline"
|
|
308
378
|
: "body_publication_failed";
|
|
309
|
-
//
|
|
310
|
-
|
|
311
|
-
|
|
379
|
+
// Do not let error cleanup hold the serial publication chain past this
|
|
380
|
+
// capture's deadline. Exporter callbacks still own submitted chunks, so
|
|
381
|
+
// retain the capture identity and memory reservation until they settle.
|
|
382
|
+
const settlement = queue.processor
|
|
383
|
+
.forceFlush()
|
|
384
|
+
.catch(() => undefined)
|
|
385
|
+
.then(awaitSettlement);
|
|
386
|
+
const remainingMs = deadline - Date.now();
|
|
387
|
+
if (remainingMs <= 0) {
|
|
388
|
+
deferredRelease = settlement;
|
|
389
|
+
}
|
|
390
|
+
else {
|
|
391
|
+
let timer;
|
|
392
|
+
const completed = await Promise.race([
|
|
393
|
+
settlement.then(() => true),
|
|
394
|
+
new Promise((resolve) => {
|
|
395
|
+
timer = setTimeout(() => resolve(false), remainingMs);
|
|
396
|
+
timer.unref?.();
|
|
397
|
+
}),
|
|
398
|
+
]).finally(() => clearTimeout(timer));
|
|
399
|
+
if (!completed) {
|
|
400
|
+
deferredRelease = settlement;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
312
403
|
}
|
|
313
|
-
|
|
314
|
-
? "rejected"
|
|
315
|
-
: progress.dropped || progress.emitted !== chunks.length
|
|
316
|
-
? "partial"
|
|
317
|
-
: progress.unconfirmed
|
|
318
|
-
? "export_unconfirmed"
|
|
319
|
-
: "transport_acknowledged";
|
|
320
|
-
return {
|
|
321
|
-
status,
|
|
322
|
-
expectedChunks: chunks.length,
|
|
323
|
-
acknowledgedChunks: progress.acknowledged,
|
|
324
|
-
unconfirmedChunks: progress.unconfirmed,
|
|
325
|
-
droppedChunks: progress.dropped,
|
|
326
|
-
notSubmittedChunks: chunks.length - progress.emitted,
|
|
327
|
-
...(reason ? { reason } : {}),
|
|
328
|
-
};
|
|
404
|
+
return resultFromProgress(progress, reason);
|
|
329
405
|
});
|
|
330
406
|
bodyPublicationChain = operation.then(() => undefined, () => undefined);
|
|
331
407
|
try {
|
|
@@ -345,9 +421,17 @@ export async function publishProxyOtelBody(captureId, body, emit) {
|
|
|
345
421
|
return result;
|
|
346
422
|
}
|
|
347
423
|
finally {
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
424
|
+
const release = () => {
|
|
425
|
+
bodyPublications.delete(captureId);
|
|
426
|
+
bodyDelivery.pending--;
|
|
427
|
+
bodyDelivery.pendingBytes -= bytes;
|
|
428
|
+
};
|
|
429
|
+
if (deferredRelease) {
|
|
430
|
+
void deferredRelease.finally(release);
|
|
431
|
+
}
|
|
432
|
+
else {
|
|
433
|
+
release();
|
|
434
|
+
}
|
|
351
435
|
}
|
|
352
436
|
}
|
|
353
437
|
/** Initialize a log-only provider in every proxy process, including the supervisor. */
|
|
@@ -30,6 +30,7 @@ let processInstanceId = randomUUID();
|
|
|
30
30
|
let sessionHashKey = randomBytes(32);
|
|
31
31
|
let nextSequence = 1;
|
|
32
32
|
let attempted = 0;
|
|
33
|
+
let otelSubmitted = 0;
|
|
33
34
|
let enqueued = 0;
|
|
34
35
|
let written = 0;
|
|
35
36
|
let dropped = 0;
|
|
@@ -428,6 +429,7 @@ function enqueueLifecycleEvent(input, onPersisted) {
|
|
|
428
429
|
};
|
|
429
430
|
if (isProxyOtelOnly()) {
|
|
430
431
|
emitProxyOtelEvent(filePrefix === "proxy-supervisor" ? "supervisor" : "lifecycle", record);
|
|
432
|
+
otelSubmitted += 1;
|
|
431
433
|
return;
|
|
432
434
|
}
|
|
433
435
|
queue.push({
|
|
@@ -496,6 +498,7 @@ export function getProxyLifecycleLoggerSnapshot() {
|
|
|
496
498
|
processInstanceId,
|
|
497
499
|
nextSequence,
|
|
498
500
|
attempted,
|
|
501
|
+
otelSubmitted,
|
|
499
502
|
enqueued,
|
|
500
503
|
written,
|
|
501
504
|
dropped,
|
|
@@ -530,6 +533,7 @@ export function resetProxyLifecycleLoggerForTests() {
|
|
|
530
533
|
sessionHashKey = randomBytes(32);
|
|
531
534
|
nextSequence = 1;
|
|
532
535
|
attempted = 0;
|
|
536
|
+
otelSubmitted = 0;
|
|
533
537
|
enqueued = 0;
|
|
534
538
|
written = 0;
|
|
535
539
|
dropped = 0;
|
package/dist/types/proxy.d.ts
CHANGED
|
@@ -744,6 +744,8 @@ export type ProxyTelemetryCheck = {
|
|
|
744
744
|
/** Small stored metadata used by the doctor; bodies are queried separately. */
|
|
745
745
|
export type ProxyTelemetryStoredRecord = Partial<RequestLogEntry> & {
|
|
746
746
|
recordedAtMicroseconds?: number;
|
|
747
|
+
id?: string;
|
|
748
|
+
at?: string;
|
|
747
749
|
captureId?: string;
|
|
748
750
|
bodySha256?: string;
|
|
749
751
|
redactedBodyBytes?: number;
|
|
@@ -1824,6 +1826,8 @@ export type ProxyLifecycleLoggerSnapshot = {
|
|
|
1824
1826
|
processInstanceId: string;
|
|
1825
1827
|
nextSequence: number;
|
|
1826
1828
|
attempted: number;
|
|
1829
|
+
/** Records delegated to the OTel sink; delivery is reported by its queues. */
|
|
1830
|
+
otelSubmitted: number;
|
|
1827
1831
|
enqueued: number;
|
|
1828
1832
|
written: number;
|
|
1829
1833
|
dropped: number;
|