@camstack/addon-pipeline 1.2.214 → 1.2.215
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/detection-pipeline/index.js +1 -2
- package/dist/detection-pipeline/index.mjs +1 -2
- package/dist/pipeline-runner/index.js +3 -2
- package/dist/pipeline-runner/index.mjs +2 -1
- package/dist/recorder/index.js +10 -7
- package/dist/recorder/index.mjs +10 -7
- package/dist/session-decode/decode-worker-child.js +3 -135
- package/dist/session-decode/decode-worker-child.mjs +3 -135
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CbBa4iV-.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-ZIja2AOe.mjs} +3 -3
- package/dist/stream-broker/{hostInit-CagSUJH7.mjs → hostInit-DpatMS4g.mjs} +3 -3
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/event-loop-stall-monitor-DXLNBMkY.mjs +0 -138
- package/dist/event-loop-stall-monitor-DaUBcb13.js +0 -149
|
@@ -6,7 +6,6 @@ const require_dist = require("../dist-DoHcimdu.js");
|
|
|
6
6
|
const require_node = require("../node-6uF72wri.js");
|
|
7
7
|
const require_default_detection_model = require("../default-detection-model-f0PImZnh.js");
|
|
8
8
|
const require_lazy_sharp = require("../lazy-sharp-BeeQGCUu.js");
|
|
9
|
-
const require_event_loop_stall_monitor = require("../event-loop-stall-monitor-DaUBcb13.js");
|
|
10
9
|
const require_process_memory = require("../process-memory-BRdyl5fE.js");
|
|
11
10
|
let node_os = require("node:os");
|
|
12
11
|
node_os = require_dist.__toESM(node_os);
|
|
@@ -10211,7 +10210,7 @@ var DetectionPipelineAddon = class extends require_dist.BaseAddon {
|
|
|
10211
10210
|
async onInitialize() {
|
|
10212
10211
|
const modelsDir = await this.resolveModelsDir();
|
|
10213
10212
|
if (!this.ctx.settings) throw new Error("DetectionPipelineAddon: ctx.settings not available");
|
|
10214
|
-
this.stopStallMonitor =
|
|
10213
|
+
this.stopStallMonitor = (0, _camstack_system_addon_utils.startEventLoopStallMonitor)(this.ctx.logger);
|
|
10215
10214
|
this.pythonAddonDir = resolveAddonPythonDir();
|
|
10216
10215
|
const py = await ensurePythonReady(this.ctx.deps, this.ctx.logger);
|
|
10217
10216
|
if (py.ok && py.pythonPath) this.pythonPath = py.pythonPath;
|
|
@@ -3,7 +3,6 @@ import { $t as parseJsonUnknown, B as defaultDeviceFor$1, Dt as resolvePoolMemor
|
|
|
3
3
|
import { o as readProcessCost } from "../node-NtM7__8q.mjs";
|
|
4
4
|
import { a as ALL_PIPELINE_STEPS, c as getStep, d as landmarkPrecisionVerdict, l as getStepDefinition, n as resolveDefaultDetectionModel, o as ALL_STEPS, r as localFrameRegistry, s as getDefaultModelForFormat, u as resolveModelForFormat } from "../default-detection-model-C2vYSX3O.mjs";
|
|
5
5
|
import { t as getSharp } from "../lazy-sharp-DXsqwpph.mjs";
|
|
6
|
-
import { n as startEventLoopStallMonitor } from "../event-loop-stall-monitor-DXLNBMkY.mjs";
|
|
7
6
|
import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-CddpCDWL.mjs";
|
|
8
7
|
import * as os from "node:os";
|
|
9
8
|
import * as fs from "node:fs";
|
|
@@ -11,7 +10,7 @@ import * as path$1 from "node:path";
|
|
|
11
10
|
import { dirname, join } from "node:path";
|
|
12
11
|
import { spawn } from "node:child_process";
|
|
13
12
|
import sharp from "sharp";
|
|
14
|
-
import { deleteModelFromDisk, ensureModel, isModelDownloaded } from "@camstack/system/addon-utils";
|
|
13
|
+
import { deleteModelFromDisk, ensureModel, isModelDownloaded, startEventLoopStallMonitor } from "@camstack/system/addon-utils";
|
|
15
14
|
//#region src/detection-pipeline/runtimes.ts
|
|
16
15
|
var KNOWN_PLATFORMS = [
|
|
17
16
|
"darwin",
|
|
@@ -9,7 +9,6 @@ const require_worker_protocol = require("../worker-protocol-BWO2WNjw.js");
|
|
|
9
9
|
const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
|
|
10
10
|
const require_lazy_sharp = require("../lazy-sharp-BeeQGCUu.js");
|
|
11
11
|
const require_restream_intent = require("../restream-intent-B1Difcnq.js");
|
|
12
|
-
const require_event_loop_stall_monitor = require("../event-loop-stall-monitor-DaUBcb13.js");
|
|
13
12
|
const require_remote_restream = require("../remote-restream-CO36Sr30.js");
|
|
14
13
|
let node_os = require("node:os");
|
|
15
14
|
node_os = require_dist.__toESM(node_os);
|
|
@@ -18,6 +17,7 @@ let sharp = require("sharp");
|
|
|
18
17
|
sharp = require_dist.__toESM(sharp);
|
|
19
18
|
let node_url = require("node:url");
|
|
20
19
|
let node_perf_hooks = require("node:perf_hooks");
|
|
20
|
+
let _camstack_system_addon_utils = require("@camstack/system/addon-utils");
|
|
21
21
|
//#region src/session-decode/fork-decode-worker.ts
|
|
22
22
|
/** Absolute path to the built decode-worker child entry (a vite build entry). */
|
|
23
23
|
var CHILD_ENTRY_PATH = (0, node_url.fileURLToPath)(new URL("../session-decode/decode-worker-child.js", require("url").pathToFileURL(__filename).href));
|
|
@@ -2451,7 +2451,7 @@ var EMPTY_SAMPLE = {
|
|
|
2451
2451
|
*/
|
|
2452
2452
|
function applyStallTick(sample, lateByMs, gcWindow, reportThresholdMs) {
|
|
2453
2453
|
if (lateByMs < reportThresholdMs) return sample;
|
|
2454
|
-
const verdict =
|
|
2454
|
+
const verdict = (0, _camstack_system_addon_utils.attributeStall)(lateByMs, Math.round(gcWindow.totalMs), gcWindow.count, Math.round(gcWindow.longestMs));
|
|
2455
2455
|
return {
|
|
2456
2456
|
count: sample.count + 1,
|
|
2457
2457
|
maxMs: Math.max(sample.maxMs, lateByMs),
|
|
@@ -9887,6 +9887,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
9887
9887
|
frame: result,
|
|
9888
9888
|
nodeId: this.nodeId,
|
|
9889
9889
|
frameHandle: handle,
|
|
9890
|
+
emittedAt: Date.now(),
|
|
9890
9891
|
...typeof capturedAt === "number" && capturedAt > 0 ? { capturedAt } : {},
|
|
9891
9892
|
...detailSteps && detailSteps.length > 0 ? { detailSteps } : {}
|
|
9892
9893
|
};
|
|
@@ -5,13 +5,13 @@ import { a as nativeLeaseSettingEnv, t as isWorkerReply } from "../worker-protoc
|
|
|
5
5
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
6
6
|
import { t as getSharp } from "../lazy-sharp-DXsqwpph.mjs";
|
|
7
7
|
import { a as preRollRequestOf, o as withDetectionIntent, t as DETECTION_PRE_ROLL_REQUEST_MS } from "../restream-intent-DWe2kInY.mjs";
|
|
8
|
-
import { t as attributeStall } from "../event-loop-stall-monitor-DXLNBMkY.mjs";
|
|
9
8
|
import { n as profileForStreamId, t as isRemoteRestream } from "../remote-restream-BeHi78PZ.mjs";
|
|
10
9
|
import * as os from "node:os";
|
|
11
10
|
import { fork } from "node:child_process";
|
|
12
11
|
import sharp from "sharp";
|
|
13
12
|
import { fileURLToPath } from "node:url";
|
|
14
13
|
import { PerformanceObserver } from "node:perf_hooks";
|
|
14
|
+
import { attributeStall } from "@camstack/system/addon-utils";
|
|
15
15
|
//#region src/session-decode/fork-decode-worker.ts
|
|
16
16
|
/** Absolute path to the built decode-worker child entry (a vite build entry). */
|
|
17
17
|
var CHILD_ENTRY_PATH = fileURLToPath(new URL("../session-decode/decode-worker-child.js", import.meta.url));
|
|
@@ -9880,6 +9880,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
9880
9880
|
frame: result,
|
|
9881
9881
|
nodeId: this.nodeId,
|
|
9882
9882
|
frameHandle: handle,
|
|
9883
|
+
emittedAt: Date.now(),
|
|
9883
9884
|
...typeof capturedAt === "number" && capturedAt > 0 ? { capturedAt } : {},
|
|
9884
9885
|
...detailSteps && detailSteps.length > 0 ? { detailSteps } : {}
|
|
9885
9886
|
};
|
package/dist/recorder/index.js
CHANGED
|
@@ -6,7 +6,6 @@ const require_dist = require("../dist-DoHcimdu.js");
|
|
|
6
6
|
const require_node = require("../node-6uF72wri.js");
|
|
7
7
|
const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
|
|
8
8
|
const require_restream_intent = require("../restream-intent-B1Difcnq.js");
|
|
9
|
-
const require_event_loop_stall_monitor = require("../event-loop-stall-monitor-DaUBcb13.js");
|
|
10
9
|
const require_retire_root_keys = require("../retire-root-keys-Csb8z2f3.js");
|
|
11
10
|
let node_crypto = require("node:crypto");
|
|
12
11
|
let node_fs = require("node:fs");
|
|
@@ -16983,11 +16982,15 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
16983
16982
|
/**
|
|
16984
16983
|
* Stops the GC-attributing stall monitor armed in {@link onInitialize}.
|
|
16985
16984
|
*
|
|
16986
|
-
* REUSED,
|
|
16987
|
-
*
|
|
16988
|
-
*
|
|
16989
|
-
* that
|
|
16990
|
-
*
|
|
16985
|
+
* REUSED, never re-implemented. It used to live next door in this bundle;
|
|
16986
|
+
* on 2026-09-09 it moved to `@camstack/system/addon-utils` (the subpath, never
|
|
16987
|
+
* the barrel — that one drags the kernel into the runner) because two more
|
|
16988
|
+
* processes that do NOT share this bundle needed it — `pipeline-analytics` and
|
|
16989
|
+
* hub-main — and a per-bundle copy is how two diagnostics end up disagreeing
|
|
16990
|
+
* about the same freeze. The cost of the move is a slower ship chain: this
|
|
16991
|
+
* import is host-resolved, so a `camstack deploy` of this addon onto a hub
|
|
16992
|
+
* whose server closure predates the export fails to load. Deploy order is
|
|
16993
|
+
* closure first, addons second. See D417.
|
|
16991
16994
|
*/
|
|
16992
16995
|
stopStallMonitor = null;
|
|
16993
16996
|
/**
|
|
@@ -17188,7 +17191,7 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
17188
17191
|
} });
|
|
17189
17192
|
}
|
|
17190
17193
|
async onInitialize() {
|
|
17191
|
-
this.stopStallMonitor =
|
|
17194
|
+
this.stopStallMonitor = (0, _camstack_system_addon_utils.startEventLoopStallMonitor)(this.ctx.logger);
|
|
17192
17195
|
const raw = this.ctx.kernel.localNodeId ?? this.ctx.id;
|
|
17193
17196
|
this.nodeId = raw.includes("/") ? raw.split("/")[0] : raw;
|
|
17194
17197
|
const configuredNode = this.config.recordingNodeId;
|
package/dist/recorder/index.mjs
CHANGED
|
@@ -2,14 +2,13 @@ import { At as storageEvictableCapability, Ct as recordingSignalCapability, D as
|
|
|
2
2
|
import { a as physicalRootOf, i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-NtM7__8q.mjs";
|
|
3
3
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
4
4
|
import { c as PREBUFFER_MAX_MS, r as RECORDING_PRE_ROLL_REQUEST_MS, s as withRecordingIntent } from "../restream-intent-DWe2kInY.mjs";
|
|
5
|
-
import { n as startEventLoopStallMonitor } from "../event-loop-stall-monitor-DXLNBMkY.mjs";
|
|
6
5
|
import { n as RowMapStore, t as retireRootKeys } from "../retire-root-keys-DZvlso2l.mjs";
|
|
7
6
|
import { randomUUID } from "node:crypto";
|
|
8
7
|
import { createReadStream, promises, realpathSync } from "node:fs";
|
|
9
8
|
import path from "node:path";
|
|
10
9
|
import { spawn } from "node:child_process";
|
|
11
10
|
import { open } from "node:fs/promises";
|
|
12
|
-
import { contentTypeFor, createFileDataPlaneHandler, parseRangeHeader } from "@camstack/system/addon-utils";
|
|
11
|
+
import { contentTypeFor, createFileDataPlaneHandler, parseRangeHeader, startEventLoopStallMonitor } from "@camstack/system/addon-utils";
|
|
13
12
|
//#region src/recorder/event-capture.ts
|
|
14
13
|
/**
|
|
15
14
|
* La sottoscrizione motion del recorder: bus → `onTrigger(deviceId, atMs)`.
|
|
@@ -16977,11 +16976,15 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
16977
16976
|
/**
|
|
16978
16977
|
* Stops the GC-attributing stall monitor armed in {@link onInitialize}.
|
|
16979
16978
|
*
|
|
16980
|
-
* REUSED,
|
|
16981
|
-
*
|
|
16982
|
-
*
|
|
16983
|
-
* that
|
|
16984
|
-
*
|
|
16979
|
+
* REUSED, never re-implemented. It used to live next door in this bundle;
|
|
16980
|
+
* on 2026-09-09 it moved to `@camstack/system/addon-utils` (the subpath, never
|
|
16981
|
+
* the barrel — that one drags the kernel into the runner) because two more
|
|
16982
|
+
* processes that do NOT share this bundle needed it — `pipeline-analytics` and
|
|
16983
|
+
* hub-main — and a per-bundle copy is how two diagnostics end up disagreeing
|
|
16984
|
+
* about the same freeze. The cost of the move is a slower ship chain: this
|
|
16985
|
+
* import is host-resolved, so a `camstack deploy` of this addon onto a hub
|
|
16986
|
+
* whose server closure predates the export fails to load. Deploy order is
|
|
16987
|
+
* closure first, addons second. See D417.
|
|
16985
16988
|
*/
|
|
16986
16989
|
stopStallMonitor = null;
|
|
16987
16990
|
/**
|
|
@@ -361,115 +361,6 @@ function mediaWallClockOf(anchor, ptsMs, nowWall) {
|
|
|
361
361
|
return shotAt > nowWall ? nowWall : shotAt;
|
|
362
362
|
}
|
|
363
363
|
//#endregion
|
|
364
|
-
//#region src/session-decode/pre-roll-burst.ts
|
|
365
|
-
var ADMITTED = { admitted: true };
|
|
366
|
-
/**
|
|
367
|
-
* Choose the media stride for one burst.
|
|
368
|
-
*
|
|
369
|
-
* `plannedFrames` is what the stride actually yields, so a caller can log the
|
|
370
|
-
* intent next to the outcome — a budget that is never reached and a budget that
|
|
371
|
-
* is always hit are different problems and used to read the same.
|
|
372
|
-
*/
|
|
373
|
-
function planPreRollBurst(input) {
|
|
374
|
-
const span = Number.isFinite(input.spanMs) && input.spanMs > 0 ? input.spanMs : 0;
|
|
375
|
-
const fps = Number.isFinite(input.detectionFps) && input.detectionFps > 0 ? input.detectionFps : 1;
|
|
376
|
-
const maxFrames = Number.isFinite(input.maxFrames) && input.maxFrames > 0 ? input.maxFrames : 0;
|
|
377
|
-
if (span === 0 || maxFrames === 0) return {
|
|
378
|
-
strideMs: Number.POSITIVE_INFINITY,
|
|
379
|
-
plannedFrames: 0,
|
|
380
|
-
subsampled: false
|
|
381
|
-
};
|
|
382
|
-
const cadenceMs = 1e3 / fps;
|
|
383
|
-
const budgetStrideMs = span / maxFrames;
|
|
384
|
-
const subsampled = budgetStrideMs > cadenceMs;
|
|
385
|
-
const strideMs = subsampled ? budgetStrideMs : cadenceMs;
|
|
386
|
-
return {
|
|
387
|
-
strideMs,
|
|
388
|
-
plannedFrames: Math.min(maxFrames, Math.max(1, Math.floor(span / strideMs))),
|
|
389
|
-
subsampled
|
|
390
|
-
};
|
|
391
|
-
}
|
|
392
|
-
/**
|
|
393
|
-
* The live, per-session budget. Stateful by necessity — a stride is a decision
|
|
394
|
-
* about the PREVIOUS admission — and deliberately free of any clock of its own:
|
|
395
|
-
* both times are passed in, so a test can drive a catch-up burst without
|
|
396
|
-
* pretending 4 s of media takes 4 s to decode.
|
|
397
|
-
*/
|
|
398
|
-
var PreRollBurst = class {
|
|
399
|
-
budget;
|
|
400
|
-
plan;
|
|
401
|
-
startedAtWall;
|
|
402
|
-
liveEdgeWall;
|
|
403
|
-
lastAdmittedMediaWall = null;
|
|
404
|
-
offered = 0;
|
|
405
|
-
inferred = 0;
|
|
406
|
-
done = false;
|
|
407
|
-
drops = {
|
|
408
|
-
stride: 0,
|
|
409
|
-
"budget-frames": 0,
|
|
410
|
-
"budget-wall-ms": 0,
|
|
411
|
-
"live-edge": 0
|
|
412
|
-
};
|
|
413
|
-
constructor(input) {
|
|
414
|
-
this.budget = input.budget;
|
|
415
|
-
this.plan = input.plan;
|
|
416
|
-
this.startedAtWall = input.startedAtWall;
|
|
417
|
-
this.liveEdgeWall = input.liveEdgeWall;
|
|
418
|
-
}
|
|
419
|
-
/** True once the burst has ended — by budget, or by reaching the live edge. */
|
|
420
|
-
get finished() {
|
|
421
|
-
return this.done;
|
|
422
|
-
}
|
|
423
|
-
/**
|
|
424
|
-
* Offer one decoded history frame.
|
|
425
|
-
*
|
|
426
|
-
* `mediaWall` is the frame's OWN moment (see `media-clock.ts`); `nowWall` is
|
|
427
|
-
* the wall clock of the offer. The two are different by construction during
|
|
428
|
-
* catch-up, which is the whole reason this class exists.
|
|
429
|
-
*/
|
|
430
|
-
offer(mediaWall, nowWall) {
|
|
431
|
-
this.offered++;
|
|
432
|
-
if (this.done) return this.drop("budget-frames");
|
|
433
|
-
if (mediaWall >= this.liveEdgeWall) {
|
|
434
|
-
this.done = true;
|
|
435
|
-
return this.drop("live-edge");
|
|
436
|
-
}
|
|
437
|
-
if (nowWall - this.startedAtWall >= this.budget.maxWallMs) {
|
|
438
|
-
this.done = true;
|
|
439
|
-
return this.drop("budget-wall-ms");
|
|
440
|
-
}
|
|
441
|
-
if (this.inferred >= this.budget.maxFrames) {
|
|
442
|
-
this.done = true;
|
|
443
|
-
return this.drop("budget-frames");
|
|
444
|
-
}
|
|
445
|
-
if (this.lastAdmittedMediaWall !== null && mediaWall - this.lastAdmittedMediaWall < this.plan.strideMs) return this.drop("stride");
|
|
446
|
-
this.lastAdmittedMediaWall = mediaWall;
|
|
447
|
-
this.inferred++;
|
|
448
|
-
return ADMITTED;
|
|
449
|
-
}
|
|
450
|
-
/** End the burst without offering a frame (session torn down, decode ended). */
|
|
451
|
-
finish() {
|
|
452
|
-
this.done = true;
|
|
453
|
-
}
|
|
454
|
-
report() {
|
|
455
|
-
return {
|
|
456
|
-
offered: this.offered,
|
|
457
|
-
inferred: this.inferred,
|
|
458
|
-
dropped: this.offered - this.inferred,
|
|
459
|
-
droppedByReason: { ...this.drops },
|
|
460
|
-
strideMs: Math.round(this.plan.strideMs),
|
|
461
|
-
subsampled: this.plan.subsampled
|
|
462
|
-
};
|
|
463
|
-
}
|
|
464
|
-
drop(reason) {
|
|
465
|
-
this.drops[reason]++;
|
|
466
|
-
return {
|
|
467
|
-
admitted: false,
|
|
468
|
-
reason
|
|
469
|
-
};
|
|
470
|
-
}
|
|
471
|
-
};
|
|
472
|
-
//#endregion
|
|
473
364
|
//#region src/session-decode/lease-activity-gate.ts
|
|
474
365
|
/**
|
|
475
366
|
* Demand-driven gate for the native-frame lease capture (see
|
|
@@ -1674,18 +1565,6 @@ var REDIAL_MS = 3e3;
|
|
|
1674
1565
|
/** How often the worker emits its `framesDecoded/framesSkipped/deliveredFps` line. */
|
|
1675
1566
|
var METRICS_INTERVAL_MS = 1e4;
|
|
1676
1567
|
/**
|
|
1677
|
-
* Wall-clock ceiling on one pre-roll catch-up burst.
|
|
1678
|
-
*
|
|
1679
|
-
* A burst is a race against the live edge it is trying to get ahead of. Two
|
|
1680
|
-
* seconds in, the decoder has either caught up (the burst ended on its own) or
|
|
1681
|
-
* something is wrong — a cold GPU, a node under load, a camera whose GOP is
|
|
1682
|
-
* longer than its history — and continuing to spend inference on footage that
|
|
1683
|
-
* is now three or four seconds stale buys an operator nothing while a live
|
|
1684
|
-
* frame waits. The frame budget bounds the WORK; this bounds the WINDOW in
|
|
1685
|
-
* which that work is still worth doing.
|
|
1686
|
-
*/
|
|
1687
|
-
var PRE_ROLL_BURST_MAX_WALL_MS = 2e3;
|
|
1688
|
-
/**
|
|
1689
1568
|
* Hard cap on the native-frame retention ring (native-res crop feature). Kept
|
|
1690
1569
|
* DELIBERATELY TINY — its only job is to cover the detection→crop latency
|
|
1691
1570
|
* (typically 1-4 frames), NOT the full shm ring depth. On the HW path every
|
|
@@ -3347,24 +3226,13 @@ var DecodeWorkerChild = class {
|
|
|
3347
3226
|
const ptsMs = this.frameMediaPtsMs(frame);
|
|
3348
3227
|
if (ptsMs === null) return now;
|
|
3349
3228
|
if (!this.mediaClock) {
|
|
3229
|
+
const servedPreRollMs = 0;
|
|
3230
|
+
if (this.preRollMs > 0) this.emitStderr(`[decode] pre-roll requested ${this.preRollMs}ms but the SERVED span does not reach this worker — burst suppressed, clock anchored at now (D411 follow-up)`);
|
|
3350
3231
|
this.mediaClock = anchorMediaClock({
|
|
3351
3232
|
firstPtsMs: ptsMs,
|
|
3352
|
-
preRollMs:
|
|
3233
|
+
preRollMs: servedPreRollMs,
|
|
3353
3234
|
nowWall: now
|
|
3354
3235
|
});
|
|
3355
|
-
if (this.preRollMs > 0 && this.preRollMaxFrames > 0) this.preRollBurst = new PreRollBurst({
|
|
3356
|
-
budget: {
|
|
3357
|
-
maxFrames: this.preRollMaxFrames,
|
|
3358
|
-
maxWallMs: PRE_ROLL_BURST_MAX_WALL_MS
|
|
3359
|
-
},
|
|
3360
|
-
plan: planPreRollBurst({
|
|
3361
|
-
spanMs: this.preRollMs,
|
|
3362
|
-
detectionFps: this.minIntervalMs > 0 ? 1e3 / this.minIntervalMs : 1,
|
|
3363
|
-
maxFrames: this.preRollMaxFrames
|
|
3364
|
-
}),
|
|
3365
|
-
startedAtWall: now,
|
|
3366
|
-
liveEdgeWall: now
|
|
3367
|
-
});
|
|
3368
3236
|
}
|
|
3369
3237
|
return mediaWallClockOf(this.mediaClock, ptsMs, now);
|
|
3370
3238
|
}
|
|
@@ -360,115 +360,6 @@ function mediaWallClockOf(anchor, ptsMs, nowWall) {
|
|
|
360
360
|
return shotAt > nowWall ? nowWall : shotAt;
|
|
361
361
|
}
|
|
362
362
|
//#endregion
|
|
363
|
-
//#region src/session-decode/pre-roll-burst.ts
|
|
364
|
-
var ADMITTED = { admitted: true };
|
|
365
|
-
/**
|
|
366
|
-
* Choose the media stride for one burst.
|
|
367
|
-
*
|
|
368
|
-
* `plannedFrames` is what the stride actually yields, so a caller can log the
|
|
369
|
-
* intent next to the outcome — a budget that is never reached and a budget that
|
|
370
|
-
* is always hit are different problems and used to read the same.
|
|
371
|
-
*/
|
|
372
|
-
function planPreRollBurst(input) {
|
|
373
|
-
const span = Number.isFinite(input.spanMs) && input.spanMs > 0 ? input.spanMs : 0;
|
|
374
|
-
const fps = Number.isFinite(input.detectionFps) && input.detectionFps > 0 ? input.detectionFps : 1;
|
|
375
|
-
const maxFrames = Number.isFinite(input.maxFrames) && input.maxFrames > 0 ? input.maxFrames : 0;
|
|
376
|
-
if (span === 0 || maxFrames === 0) return {
|
|
377
|
-
strideMs: Number.POSITIVE_INFINITY,
|
|
378
|
-
plannedFrames: 0,
|
|
379
|
-
subsampled: false
|
|
380
|
-
};
|
|
381
|
-
const cadenceMs = 1e3 / fps;
|
|
382
|
-
const budgetStrideMs = span / maxFrames;
|
|
383
|
-
const subsampled = budgetStrideMs > cadenceMs;
|
|
384
|
-
const strideMs = subsampled ? budgetStrideMs : cadenceMs;
|
|
385
|
-
return {
|
|
386
|
-
strideMs,
|
|
387
|
-
plannedFrames: Math.min(maxFrames, Math.max(1, Math.floor(span / strideMs))),
|
|
388
|
-
subsampled
|
|
389
|
-
};
|
|
390
|
-
}
|
|
391
|
-
/**
|
|
392
|
-
* The live, per-session budget. Stateful by necessity — a stride is a decision
|
|
393
|
-
* about the PREVIOUS admission — and deliberately free of any clock of its own:
|
|
394
|
-
* both times are passed in, so a test can drive a catch-up burst without
|
|
395
|
-
* pretending 4 s of media takes 4 s to decode.
|
|
396
|
-
*/
|
|
397
|
-
var PreRollBurst = class {
|
|
398
|
-
budget;
|
|
399
|
-
plan;
|
|
400
|
-
startedAtWall;
|
|
401
|
-
liveEdgeWall;
|
|
402
|
-
lastAdmittedMediaWall = null;
|
|
403
|
-
offered = 0;
|
|
404
|
-
inferred = 0;
|
|
405
|
-
done = false;
|
|
406
|
-
drops = {
|
|
407
|
-
stride: 0,
|
|
408
|
-
"budget-frames": 0,
|
|
409
|
-
"budget-wall-ms": 0,
|
|
410
|
-
"live-edge": 0
|
|
411
|
-
};
|
|
412
|
-
constructor(input) {
|
|
413
|
-
this.budget = input.budget;
|
|
414
|
-
this.plan = input.plan;
|
|
415
|
-
this.startedAtWall = input.startedAtWall;
|
|
416
|
-
this.liveEdgeWall = input.liveEdgeWall;
|
|
417
|
-
}
|
|
418
|
-
/** True once the burst has ended — by budget, or by reaching the live edge. */
|
|
419
|
-
get finished() {
|
|
420
|
-
return this.done;
|
|
421
|
-
}
|
|
422
|
-
/**
|
|
423
|
-
* Offer one decoded history frame.
|
|
424
|
-
*
|
|
425
|
-
* `mediaWall` is the frame's OWN moment (see `media-clock.ts`); `nowWall` is
|
|
426
|
-
* the wall clock of the offer. The two are different by construction during
|
|
427
|
-
* catch-up, which is the whole reason this class exists.
|
|
428
|
-
*/
|
|
429
|
-
offer(mediaWall, nowWall) {
|
|
430
|
-
this.offered++;
|
|
431
|
-
if (this.done) return this.drop("budget-frames");
|
|
432
|
-
if (mediaWall >= this.liveEdgeWall) {
|
|
433
|
-
this.done = true;
|
|
434
|
-
return this.drop("live-edge");
|
|
435
|
-
}
|
|
436
|
-
if (nowWall - this.startedAtWall >= this.budget.maxWallMs) {
|
|
437
|
-
this.done = true;
|
|
438
|
-
return this.drop("budget-wall-ms");
|
|
439
|
-
}
|
|
440
|
-
if (this.inferred >= this.budget.maxFrames) {
|
|
441
|
-
this.done = true;
|
|
442
|
-
return this.drop("budget-frames");
|
|
443
|
-
}
|
|
444
|
-
if (this.lastAdmittedMediaWall !== null && mediaWall - this.lastAdmittedMediaWall < this.plan.strideMs) return this.drop("stride");
|
|
445
|
-
this.lastAdmittedMediaWall = mediaWall;
|
|
446
|
-
this.inferred++;
|
|
447
|
-
return ADMITTED;
|
|
448
|
-
}
|
|
449
|
-
/** End the burst without offering a frame (session torn down, decode ended). */
|
|
450
|
-
finish() {
|
|
451
|
-
this.done = true;
|
|
452
|
-
}
|
|
453
|
-
report() {
|
|
454
|
-
return {
|
|
455
|
-
offered: this.offered,
|
|
456
|
-
inferred: this.inferred,
|
|
457
|
-
dropped: this.offered - this.inferred,
|
|
458
|
-
droppedByReason: { ...this.drops },
|
|
459
|
-
strideMs: Math.round(this.plan.strideMs),
|
|
460
|
-
subsampled: this.plan.subsampled
|
|
461
|
-
};
|
|
462
|
-
}
|
|
463
|
-
drop(reason) {
|
|
464
|
-
this.drops[reason]++;
|
|
465
|
-
return {
|
|
466
|
-
admitted: false,
|
|
467
|
-
reason
|
|
468
|
-
};
|
|
469
|
-
}
|
|
470
|
-
};
|
|
471
|
-
//#endregion
|
|
472
363
|
//#region src/session-decode/lease-activity-gate.ts
|
|
473
364
|
/**
|
|
474
365
|
* Demand-driven gate for the native-frame lease capture (see
|
|
@@ -1673,18 +1564,6 @@ var REDIAL_MS = 3e3;
|
|
|
1673
1564
|
/** How often the worker emits its `framesDecoded/framesSkipped/deliveredFps` line. */
|
|
1674
1565
|
var METRICS_INTERVAL_MS = 1e4;
|
|
1675
1566
|
/**
|
|
1676
|
-
* Wall-clock ceiling on one pre-roll catch-up burst.
|
|
1677
|
-
*
|
|
1678
|
-
* A burst is a race against the live edge it is trying to get ahead of. Two
|
|
1679
|
-
* seconds in, the decoder has either caught up (the burst ended on its own) or
|
|
1680
|
-
* something is wrong — a cold GPU, a node under load, a camera whose GOP is
|
|
1681
|
-
* longer than its history — and continuing to spend inference on footage that
|
|
1682
|
-
* is now three or four seconds stale buys an operator nothing while a live
|
|
1683
|
-
* frame waits. The frame budget bounds the WORK; this bounds the WINDOW in
|
|
1684
|
-
* which that work is still worth doing.
|
|
1685
|
-
*/
|
|
1686
|
-
var PRE_ROLL_BURST_MAX_WALL_MS = 2e3;
|
|
1687
|
-
/**
|
|
1688
1567
|
* Hard cap on the native-frame retention ring (native-res crop feature). Kept
|
|
1689
1568
|
* DELIBERATELY TINY — its only job is to cover the detection→crop latency
|
|
1690
1569
|
* (typically 1-4 frames), NOT the full shm ring depth. On the HW path every
|
|
@@ -3346,24 +3225,13 @@ var DecodeWorkerChild = class {
|
|
|
3346
3225
|
const ptsMs = this.frameMediaPtsMs(frame);
|
|
3347
3226
|
if (ptsMs === null) return now;
|
|
3348
3227
|
if (!this.mediaClock) {
|
|
3228
|
+
const servedPreRollMs = 0;
|
|
3229
|
+
if (this.preRollMs > 0) this.emitStderr(`[decode] pre-roll requested ${this.preRollMs}ms but the SERVED span does not reach this worker — burst suppressed, clock anchored at now (D411 follow-up)`);
|
|
3349
3230
|
this.mediaClock = anchorMediaClock({
|
|
3350
3231
|
firstPtsMs: ptsMs,
|
|
3351
|
-
preRollMs:
|
|
3232
|
+
preRollMs: servedPreRollMs,
|
|
3352
3233
|
nowWall: now
|
|
3353
3234
|
});
|
|
3354
|
-
if (this.preRollMs > 0 && this.preRollMaxFrames > 0) this.preRollBurst = new PreRollBurst({
|
|
3355
|
-
budget: {
|
|
3356
|
-
maxFrames: this.preRollMaxFrames,
|
|
3357
|
-
maxWallMs: PRE_ROLL_BURST_MAX_WALL_MS
|
|
3358
|
-
},
|
|
3359
|
-
plan: planPreRollBurst({
|
|
3360
|
-
spanMs: this.preRollMs,
|
|
3361
|
-
detectionFps: this.minIntervalMs > 0 ? 1e3 / this.minIntervalMs : 1,
|
|
3362
|
-
maxFrames: this.preRollMaxFrames
|
|
3363
|
-
}),
|
|
3364
|
-
startedAtWall: now,
|
|
3365
|
-
liveEdgeWall: now
|
|
3366
|
-
});
|
|
3367
3235
|
}
|
|
3368
3236
|
return mediaWallClockOf(this.mediaClock, ptsMs, now);
|
|
3369
3237
|
}
|
|
@@ -3,7 +3,7 @@ import "./dist-CYZr2fwk.mjs";
|
|
|
3
3
|
var e = {
|
|
4
4
|
"@camstack/sdk": {
|
|
5
5
|
name: "@camstack/sdk",
|
|
6
|
-
version: "1.2.
|
|
6
|
+
version: "1.2.93",
|
|
7
7
|
scope: ["default"],
|
|
8
8
|
loaded: !1,
|
|
9
9
|
from: "addon_stream_broker_widgets",
|
|
@@ -18,7 +18,7 @@ var e = {
|
|
|
18
18
|
},
|
|
19
19
|
"@camstack/types": {
|
|
20
20
|
name: "@camstack/types",
|
|
21
|
-
version: "1.2.
|
|
21
|
+
version: "1.2.173",
|
|
22
22
|
scope: ["default"],
|
|
23
23
|
loaded: !1,
|
|
24
24
|
from: "addon_stream_broker_widgets",
|
|
@@ -33,7 +33,7 @@ var e = {
|
|
|
33
33
|
},
|
|
34
34
|
"@camstack/ui-library": {
|
|
35
35
|
name: "@camstack/ui-library",
|
|
36
|
-
version: "1.2.
|
|
36
|
+
version: "1.2.141",
|
|
37
37
|
scope: ["default"],
|
|
38
38
|
loaded: !1,
|
|
39
39
|
from: "addon_stream_broker_widgets",
|
|
@@ -36,7 +36,7 @@ async function r() {
|
|
|
36
36
|
}
|
|
37
37
|
},
|
|
38
38
|
"@camstack/types": {
|
|
39
|
-
version: "1.2.
|
|
39
|
+
version: "1.2.173",
|
|
40
40
|
scope: "default",
|
|
41
41
|
shareConfig: {
|
|
42
42
|
singleton: !0,
|
|
@@ -45,7 +45,7 @@ async function r() {
|
|
|
45
45
|
}
|
|
46
46
|
},
|
|
47
47
|
"@camstack/sdk": {
|
|
48
|
-
version: "1.2.
|
|
48
|
+
version: "1.2.93",
|
|
49
49
|
scope: "default",
|
|
50
50
|
shareConfig: {
|
|
51
51
|
singleton: !0,
|
|
@@ -81,7 +81,7 @@ async function r() {
|
|
|
81
81
|
}
|
|
82
82
|
},
|
|
83
83
|
"@camstack/ui-library": {
|
|
84
|
-
version: "1.2.
|
|
84
|
+
version: "1.2.141",
|
|
85
85
|
scope: "default",
|
|
86
86
|
shareConfig: {
|
|
87
87
|
singleton: !0,
|
|
@@ -30,7 +30,7 @@ async function d(e) {
|
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
async function f() {
|
|
33
|
-
return l ||= d(() => import("./_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-
|
|
33
|
+
return l ||= d(() => import("./_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-ZIja2AOe.mjs")).catch((e) => {
|
|
34
34
|
throw l = void 0, e;
|
|
35
35
|
}), l;
|
|
36
36
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@camstack/addon-pipeline",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.215",
|
|
4
4
|
"description": "Pipeline bundle — runner, detection, motion, audio + stream broker. Multi-entry npm package shipping pipeline addons under a single bundle.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"camstack",
|
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
import { PerformanceObserver } from "node:perf_hooks";
|
|
2
|
-
//#region src/detection-pipeline/diagnostics/event-loop-stall-monitor.ts
|
|
3
|
-
/**
|
|
4
|
-
* event-loop-stall-monitor — names the thing that freezes the detection process.
|
|
5
|
-
*
|
|
6
|
-
* ## What this exists to settle
|
|
7
|
-
*
|
|
8
|
-
* On 2026-08-01/02 inference across the whole fleet stalled in bursts: 759
|
|
9
|
-
* calls in 90 minutes crossed the runner's 20s deadline, p50 41s and max 80s,
|
|
10
|
-
* and EVERY one of them eventually resolved. Six explanations were killed with
|
|
11
|
-
* telemetry alone:
|
|
12
|
-
*
|
|
13
|
-
* - not a slow GPU or a wedged pool — the work completes, just enormously late;
|
|
14
|
-
* - not CPU starvation — hosts sat at 8-48% during the freezes;
|
|
15
|
-
* - not swap — `pswpin/pswpout` are flat zero on every host;
|
|
16
|
-
* - not a logging artifact — Loki's timestamp trails the in-process one by
|
|
17
|
-
* 0.1-1.0s at p95, so the gaps are real, not buffered delivery;
|
|
18
|
-
* - not the v1.2.30 release — the stalls predate it by a day;
|
|
19
|
-
* - not only the idle-pool recompile — that is real and hub-side, but the
|
|
20
|
-
* agent stalls with no eviction and no compile at all.
|
|
21
|
-
*
|
|
22
|
-
* What IS established: the 10s stats timer — which touches no inference code —
|
|
23
|
-
* stops firing for 22-88s and resumes at the same instant the abandoned calls
|
|
24
|
-
* settle. The event loop itself freezes. From outside the process, GC and a
|
|
25
|
-
* synchronous block are indistinguishable, and guessing between them is how
|
|
26
|
-
* this investigation would go wrong. So measure it.
|
|
27
|
-
*
|
|
28
|
-
* ## How
|
|
29
|
-
*
|
|
30
|
-
* A 1s interval whose LATENESS is the freeze: a tick that arrives 40s late means
|
|
31
|
-
* the loop was blocked ~39s, whatever blocked it. In the same window the GC
|
|
32
|
-
* observer accumulates pause time. The two numbers together answer the question
|
|
33
|
-
* outright:
|
|
34
|
-
*
|
|
35
|
-
* - `gcMs` ≈ `stallMs` → garbage collection. Look at heap growth.
|
|
36
|
-
* - `gcMs` ≈ 0 → something SYNCHRONOUS blocked the loop, and GC is innocent.
|
|
37
|
-
*
|
|
38
|
-
* Heap and RSS ride along because if it is GC, the next question is always
|
|
39
|
-
* "growing since when?".
|
|
40
|
-
*
|
|
41
|
-
* Deliberately not a histogram (`monitorEventLoopDelay`): that reports a
|
|
42
|
-
* distribution, and the question here is about ONE outlier and what coincided
|
|
43
|
-
* with it.
|
|
44
|
-
*/
|
|
45
|
-
/** Tick period. Small enough to bound attribution error, large enough to cost
|
|
46
|
-
* nothing on a process already doing frame work. */
|
|
47
|
-
var STALL_TICK_MS = 1e3;
|
|
48
|
-
/**
|
|
49
|
-
* Report a freeze only past this. The runner's own inference deadline is 20s
|
|
50
|
-
* and the observed freezes run 22-88s, so this is far below the signal while
|
|
51
|
-
* staying clear of ordinary scheduling jitter (a busy frame path routinely
|
|
52
|
-
* overruns a 1s timer by tens of ms).
|
|
53
|
-
*/
|
|
54
|
-
var STALL_REPORT_MS = 2e3;
|
|
55
|
-
/**
|
|
56
|
-
* Attribute a freeze. Pure, because this is the ONE judgement the whole module
|
|
57
|
-
* exists to make and it must be testable without freezing a real event loop.
|
|
58
|
-
*
|
|
59
|
-
* `gc` when garbage collection accounts for most of the stall; a
|
|
60
|
-
* `synchronous-block` when it accounts for almost none — that verdict points
|
|
61
|
-
* away from the heap entirely and at whatever ran without yielding.
|
|
62
|
-
*/
|
|
63
|
-
function attributeStall(stallMs, gcMs, gcCount, longestGcMs) {
|
|
64
|
-
const share = stallMs > 0 ? gcMs / stallMs : 0;
|
|
65
|
-
return {
|
|
66
|
-
stallMs,
|
|
67
|
-
gcMs,
|
|
68
|
-
gcCount,
|
|
69
|
-
longestGcMs,
|
|
70
|
-
cause: share >= .7 ? "gc" : share <= .2 ? "synchronous-block" : "mixed"
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* Start watching. Returns a stop function; safe to call twice.
|
|
75
|
-
*
|
|
76
|
-
* Never throws into the caller: a diagnostic that can take the detection
|
|
77
|
-
* process down is worse than no diagnostic. `PerformanceObserver` for `gc` is
|
|
78
|
-
* unavailable in some embedded runtimes, and that must degrade to
|
|
79
|
-
* stall-detection-without-attribution rather than a boot failure.
|
|
80
|
-
*/
|
|
81
|
-
function startEventLoopStallMonitor(logger, reportThresholdMs = STALL_REPORT_MS) {
|
|
82
|
-
let gc = {
|
|
83
|
-
totalMs: 0,
|
|
84
|
-
count: 0,
|
|
85
|
-
longestMs: 0
|
|
86
|
-
};
|
|
87
|
-
let observer = null;
|
|
88
|
-
try {
|
|
89
|
-
observer = new PerformanceObserver((list) => {
|
|
90
|
-
for (const entry of list.getEntries()) {
|
|
91
|
-
gc.totalMs += entry.duration;
|
|
92
|
-
gc.count += 1;
|
|
93
|
-
if (entry.duration > gc.longestMs) gc.longestMs = entry.duration;
|
|
94
|
-
}
|
|
95
|
-
});
|
|
96
|
-
observer.observe({ entryTypes: ["gc"] });
|
|
97
|
-
} catch {
|
|
98
|
-
observer = null;
|
|
99
|
-
}
|
|
100
|
-
let expected = Date.now() + STALL_TICK_MS;
|
|
101
|
-
const timer = setInterval(() => {
|
|
102
|
-
const now = Date.now();
|
|
103
|
-
const lateBy = now - expected;
|
|
104
|
-
const window = gc;
|
|
105
|
-
gc = {
|
|
106
|
-
totalMs: 0,
|
|
107
|
-
count: 0,
|
|
108
|
-
longestMs: 0
|
|
109
|
-
};
|
|
110
|
-
expected = now + STALL_TICK_MS;
|
|
111
|
-
if (lateBy < reportThresholdMs) return;
|
|
112
|
-
const verdict = attributeStall(lateBy, Math.round(window.totalMs), window.count, Math.round(window.longestMs));
|
|
113
|
-
const mem = process.memoryUsage();
|
|
114
|
-
logger.warn("event loop stalled", { meta: {
|
|
115
|
-
stallMs: verdict.stallMs,
|
|
116
|
-
cause: verdict.cause,
|
|
117
|
-
gcMs: verdict.gcMs,
|
|
118
|
-
gcCount: verdict.gcCount,
|
|
119
|
-
longestGcMs: verdict.longestGcMs,
|
|
120
|
-
gcObserved: observer !== null,
|
|
121
|
-
heapUsedMb: Math.round(mem.heapUsed / 1e6),
|
|
122
|
-
heapTotalMb: Math.round(mem.heapTotal / 1e6),
|
|
123
|
-
rssMb: Math.round(mem.rss / 1e6),
|
|
124
|
-
externalMb: Math.round(mem.external / 1e6),
|
|
125
|
-
arrayBuffersMb: Math.round(mem.arrayBuffers / 1e6)
|
|
126
|
-
} });
|
|
127
|
-
}, STALL_TICK_MS);
|
|
128
|
-
timer.unref?.();
|
|
129
|
-
let stopped = false;
|
|
130
|
-
return () => {
|
|
131
|
-
if (stopped) return;
|
|
132
|
-
stopped = true;
|
|
133
|
-
clearInterval(timer);
|
|
134
|
-
observer?.disconnect();
|
|
135
|
-
};
|
|
136
|
-
}
|
|
137
|
-
//#endregion
|
|
138
|
-
export { startEventLoopStallMonitor as n, attributeStall as t };
|
|
@@ -1,149 +0,0 @@
|
|
|
1
|
-
let node_perf_hooks = require("node:perf_hooks");
|
|
2
|
-
//#region src/detection-pipeline/diagnostics/event-loop-stall-monitor.ts
|
|
3
|
-
/**
|
|
4
|
-
* event-loop-stall-monitor — names the thing that freezes the detection process.
|
|
5
|
-
*
|
|
6
|
-
* ## What this exists to settle
|
|
7
|
-
*
|
|
8
|
-
* On 2026-08-01/02 inference across the whole fleet stalled in bursts: 759
|
|
9
|
-
* calls in 90 minutes crossed the runner's 20s deadline, p50 41s and max 80s,
|
|
10
|
-
* and EVERY one of them eventually resolved. Six explanations were killed with
|
|
11
|
-
* telemetry alone:
|
|
12
|
-
*
|
|
13
|
-
* - not a slow GPU or a wedged pool — the work completes, just enormously late;
|
|
14
|
-
* - not CPU starvation — hosts sat at 8-48% during the freezes;
|
|
15
|
-
* - not swap — `pswpin/pswpout` are flat zero on every host;
|
|
16
|
-
* - not a logging artifact — Loki's timestamp trails the in-process one by
|
|
17
|
-
* 0.1-1.0s at p95, so the gaps are real, not buffered delivery;
|
|
18
|
-
* - not the v1.2.30 release — the stalls predate it by a day;
|
|
19
|
-
* - not only the idle-pool recompile — that is real and hub-side, but the
|
|
20
|
-
* agent stalls with no eviction and no compile at all.
|
|
21
|
-
*
|
|
22
|
-
* What IS established: the 10s stats timer — which touches no inference code —
|
|
23
|
-
* stops firing for 22-88s and resumes at the same instant the abandoned calls
|
|
24
|
-
* settle. The event loop itself freezes. From outside the process, GC and a
|
|
25
|
-
* synchronous block are indistinguishable, and guessing between them is how
|
|
26
|
-
* this investigation would go wrong. So measure it.
|
|
27
|
-
*
|
|
28
|
-
* ## How
|
|
29
|
-
*
|
|
30
|
-
* A 1s interval whose LATENESS is the freeze: a tick that arrives 40s late means
|
|
31
|
-
* the loop was blocked ~39s, whatever blocked it. In the same window the GC
|
|
32
|
-
* observer accumulates pause time. The two numbers together answer the question
|
|
33
|
-
* outright:
|
|
34
|
-
*
|
|
35
|
-
* - `gcMs` ≈ `stallMs` → garbage collection. Look at heap growth.
|
|
36
|
-
* - `gcMs` ≈ 0 → something SYNCHRONOUS blocked the loop, and GC is innocent.
|
|
37
|
-
*
|
|
38
|
-
* Heap and RSS ride along because if it is GC, the next question is always
|
|
39
|
-
* "growing since when?".
|
|
40
|
-
*
|
|
41
|
-
* Deliberately not a histogram (`monitorEventLoopDelay`): that reports a
|
|
42
|
-
* distribution, and the question here is about ONE outlier and what coincided
|
|
43
|
-
* with it.
|
|
44
|
-
*/
|
|
45
|
-
/** Tick period. Small enough to bound attribution error, large enough to cost
|
|
46
|
-
* nothing on a process already doing frame work. */
|
|
47
|
-
var STALL_TICK_MS = 1e3;
|
|
48
|
-
/**
|
|
49
|
-
* Report a freeze only past this. The runner's own inference deadline is 20s
|
|
50
|
-
* and the observed freezes run 22-88s, so this is far below the signal while
|
|
51
|
-
* staying clear of ordinary scheduling jitter (a busy frame path routinely
|
|
52
|
-
* overruns a 1s timer by tens of ms).
|
|
53
|
-
*/
|
|
54
|
-
var STALL_REPORT_MS = 2e3;
|
|
55
|
-
/**
|
|
56
|
-
* Attribute a freeze. Pure, because this is the ONE judgement the whole module
|
|
57
|
-
* exists to make and it must be testable without freezing a real event loop.
|
|
58
|
-
*
|
|
59
|
-
* `gc` when garbage collection accounts for most of the stall; a
|
|
60
|
-
* `synchronous-block` when it accounts for almost none — that verdict points
|
|
61
|
-
* away from the heap entirely and at whatever ran without yielding.
|
|
62
|
-
*/
|
|
63
|
-
function attributeStall(stallMs, gcMs, gcCount, longestGcMs) {
|
|
64
|
-
const share = stallMs > 0 ? gcMs / stallMs : 0;
|
|
65
|
-
return {
|
|
66
|
-
stallMs,
|
|
67
|
-
gcMs,
|
|
68
|
-
gcCount,
|
|
69
|
-
longestGcMs,
|
|
70
|
-
cause: share >= .7 ? "gc" : share <= .2 ? "synchronous-block" : "mixed"
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* Start watching. Returns a stop function; safe to call twice.
|
|
75
|
-
*
|
|
76
|
-
* Never throws into the caller: a diagnostic that can take the detection
|
|
77
|
-
* process down is worse than no diagnostic. `PerformanceObserver` for `gc` is
|
|
78
|
-
* unavailable in some embedded runtimes, and that must degrade to
|
|
79
|
-
* stall-detection-without-attribution rather than a boot failure.
|
|
80
|
-
*/
|
|
81
|
-
function startEventLoopStallMonitor(logger, reportThresholdMs = STALL_REPORT_MS) {
|
|
82
|
-
let gc = {
|
|
83
|
-
totalMs: 0,
|
|
84
|
-
count: 0,
|
|
85
|
-
longestMs: 0
|
|
86
|
-
};
|
|
87
|
-
let observer = null;
|
|
88
|
-
try {
|
|
89
|
-
observer = new node_perf_hooks.PerformanceObserver((list) => {
|
|
90
|
-
for (const entry of list.getEntries()) {
|
|
91
|
-
gc.totalMs += entry.duration;
|
|
92
|
-
gc.count += 1;
|
|
93
|
-
if (entry.duration > gc.longestMs) gc.longestMs = entry.duration;
|
|
94
|
-
}
|
|
95
|
-
});
|
|
96
|
-
observer.observe({ entryTypes: ["gc"] });
|
|
97
|
-
} catch {
|
|
98
|
-
observer = null;
|
|
99
|
-
}
|
|
100
|
-
let expected = Date.now() + STALL_TICK_MS;
|
|
101
|
-
const timer = setInterval(() => {
|
|
102
|
-
const now = Date.now();
|
|
103
|
-
const lateBy = now - expected;
|
|
104
|
-
const window = gc;
|
|
105
|
-
gc = {
|
|
106
|
-
totalMs: 0,
|
|
107
|
-
count: 0,
|
|
108
|
-
longestMs: 0
|
|
109
|
-
};
|
|
110
|
-
expected = now + STALL_TICK_MS;
|
|
111
|
-
if (lateBy < reportThresholdMs) return;
|
|
112
|
-
const verdict = attributeStall(lateBy, Math.round(window.totalMs), window.count, Math.round(window.longestMs));
|
|
113
|
-
const mem = process.memoryUsage();
|
|
114
|
-
logger.warn("event loop stalled", { meta: {
|
|
115
|
-
stallMs: verdict.stallMs,
|
|
116
|
-
cause: verdict.cause,
|
|
117
|
-
gcMs: verdict.gcMs,
|
|
118
|
-
gcCount: verdict.gcCount,
|
|
119
|
-
longestGcMs: verdict.longestGcMs,
|
|
120
|
-
gcObserved: observer !== null,
|
|
121
|
-
heapUsedMb: Math.round(mem.heapUsed / 1e6),
|
|
122
|
-
heapTotalMb: Math.round(mem.heapTotal / 1e6),
|
|
123
|
-
rssMb: Math.round(mem.rss / 1e6),
|
|
124
|
-
externalMb: Math.round(mem.external / 1e6),
|
|
125
|
-
arrayBuffersMb: Math.round(mem.arrayBuffers / 1e6)
|
|
126
|
-
} });
|
|
127
|
-
}, STALL_TICK_MS);
|
|
128
|
-
timer.unref?.();
|
|
129
|
-
let stopped = false;
|
|
130
|
-
return () => {
|
|
131
|
-
if (stopped) return;
|
|
132
|
-
stopped = true;
|
|
133
|
-
clearInterval(timer);
|
|
134
|
-
observer?.disconnect();
|
|
135
|
-
};
|
|
136
|
-
}
|
|
137
|
-
//#endregion
|
|
138
|
-
Object.defineProperty(exports, "attributeStall", {
|
|
139
|
-
enumerable: true,
|
|
140
|
-
get: function() {
|
|
141
|
-
return attributeStall;
|
|
142
|
-
}
|
|
143
|
-
});
|
|
144
|
-
Object.defineProperty(exports, "startEventLoopStallMonitor", {
|
|
145
|
-
enumerable: true,
|
|
146
|
-
get: function() {
|
|
147
|
-
return startEventLoopStallMonitor;
|
|
148
|
-
}
|
|
149
|
-
});
|