@camstack/addon-pipeline 1.2.145 → 1.2.148
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio-analyzer/index.js +2 -2
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/detection-pipeline/index.js +36 -33
- package/dist/detection-pipeline/index.mjs +8 -5
- package/dist/{dist-DML3mpn9.js → dist-DLy_uz82.js} +121 -13
- package/dist/{dist-cZopp8Xm.mjs → dist-DjplinYP.mjs} +121 -13
- package/dist/event-loop-stall-monitor-DXLNBMkY.mjs +138 -0
- package/dist/event-loop-stall-monitor-DaUBcb13.js +149 -0
- package/dist/{lazy-sharp-BzXIjfN8.js → lazy-sharp-BI4LsfMM.js} +1 -1
- package/dist/{event-loop-stall-monitor-DH_vFYWu.mjs → local-frame-registry-77FL9ua_.mjs} +2 -139
- package/dist/{event-loop-stall-monitor-DaV4Zdsz.js → local-frame-registry-Dr_U_xCG.js} +1 -150
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/{node-OvVqe9eh.js → node-BHFLhfkh.js} +1 -1
- package/dist/{node-Br6p9KvD.mjs → node-D7uxWSXD.mjs} +1 -1
- package/dist/pipeline-runner/index.js +14 -13
- package/dist/pipeline-runner/index.mjs +5 -4
- package/dist/{process-memory-DX25Z60Q.mjs → process-memory-B2tApNua.mjs} +1 -1
- package/dist/{process-memory-NWKrKfmi.js → process-memory-M8WiR3qj.js} +1 -1
- package/dist/recorder/index.js +633 -381
- package/dist/recorder/index.mjs +633 -381
- package/dist/{segment-demux-js-DGmFQ1Zj.mjs → segment-demux-js-3niBZUpJ.mjs} +1 -1
- package/dist/{segment-demux-js-CerfMroc.js → segment-demux-js-Cc_If7C_.js} +1 -1
- package/dist/session-decode/decode-worker-child.js +2 -2
- package/dist/session-decode/decode-worker-child.mjs +1 -1
- package/dist/stream-broker/_stub.js +2 -2
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-DQNgiTjz.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-QfvsYvjB.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DXrU11Bz.mjs +26 -0
- package/dist/stream-broker/{_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-bjYJYwAq.mjs → _virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CgIcw5Wl.mjs} +1 -1
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-NShcJE25.mjs → hostInit-Dd2CEvDK.mjs} +3 -3
- package/dist/stream-broker/index.js +3 -3
- package/dist/stream-broker/index.mjs +3 -3
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-CUCQ1IPR.mjs → worker-protocol-Cvln9GmS.mjs} +1 -1
- package/dist/{worker-protocol-Dt3zWCQY.js → worker-protocol-basMNc_q.js} +1 -1
- package/package.json +1 -1
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-D4-thZGg.mjs +0 -26
|
@@ -9351,18 +9351,61 @@ var RelocateFootageInputSchema = object({
|
|
|
9351
9351
|
* `RecordingConfig.enabled` or camera wrapper bindings. */
|
|
9352
9352
|
var StorageMigrationLeaseInputSchema = object({ leaseId: string().min(1) });
|
|
9353
9353
|
var StorageMigrationFootageMoveInputSchema = RelocateFootageInputSchema.extend({ leaseId: string().min(1) });
|
|
9354
|
+
/**
|
|
9355
|
+
* What a `relocateMedia` pass DOES. One engine, three passes — never a second
|
|
9356
|
+
* mover (the engine already walks both collections with a timestamp cursor and
|
|
9357
|
+
* already has a stamp-without-copy path).
|
|
9358
|
+
*
|
|
9359
|
+
* - `move` — the default and the historical behaviour: event-media and
|
|
9360
|
+
* retrain blobs move to `toLocationId` and their rows are
|
|
9361
|
+
* stamped. The enrolled gallery is skipped (D197).
|
|
9362
|
+
* - `seal` — ROWS ONLY, no bytes. Every row whose `locationId` is NULL is
|
|
9363
|
+
* stamped with `toLocationId`. `toLocationId` here is the id the
|
|
9364
|
+
* bytes ALREADY sit on — today's `eventMedia` default — because
|
|
9365
|
+
* a NULL row means "wherever `eventMedia` points *now*", and the
|
|
9366
|
+
* instant a repoint moves that pointer the row reads from the
|
|
9367
|
+
* new disk while its bytes are on the old one.
|
|
9368
|
+
* - `gallery` — the inverse selection of `move`: ONLY the retention-exempt
|
|
9369
|
+
* (enrolled-gallery) rows, which `move` deliberately skips.
|
|
9370
|
+
* `galleryMedia` is `cardinality: 'single'`, so this pass can
|
|
9371
|
+
* never run beside a live second location: it is stop-the-world
|
|
9372
|
+
* by construction, which is acceptable only because the gallery
|
|
9373
|
+
* is a few KB per enrolled sample.
|
|
9374
|
+
*/
|
|
9375
|
+
var MediaRelocateModeSchema = _enum([
|
|
9376
|
+
"move",
|
|
9377
|
+
"seal",
|
|
9378
|
+
"gallery"
|
|
9379
|
+
]);
|
|
9354
9380
|
var RelocateMediaInputSchema = object({
|
|
9355
9381
|
toLocationId: string(),
|
|
9356
|
-
throttleMbps: number().min(1).max(1e3).optional()
|
|
9382
|
+
throttleMbps: number().min(1).max(1e3).optional(),
|
|
9383
|
+
/** Omitted = `move`, the pre-existing behaviour. */
|
|
9384
|
+
mode: MediaRelocateModeSchema.optional()
|
|
9385
|
+
});
|
|
9386
|
+
/** How many rows still carry NO `locationId` — the population a repoint would
|
|
9387
|
+
* silently re-aim at a disk that does not hold their bytes. Zero is the only
|
|
9388
|
+
* value that permits a non-blocking `eventMedia` cutover. */
|
|
9389
|
+
var UnstampedEventMediaCountSchema = object({
|
|
9390
|
+
media: number().int().nonnegative(),
|
|
9391
|
+
retrainFrames: number().int().nonnegative(),
|
|
9392
|
+
total: number().int().nonnegative()
|
|
9357
9393
|
});
|
|
9358
9394
|
var StorageMigrationMediaMoveInputSchema = RelocateMediaInputSchema.extend({ leaseId: string().min(1) });
|
|
9359
|
-
/** The independently selectable logical storage classes
|
|
9360
|
-
*
|
|
9361
|
-
*
|
|
9395
|
+
/** The independently selectable logical storage classes — every class
|
|
9396
|
+
* `storage.listLocationDeclarations` reports, so an operator never meets a
|
|
9397
|
+
* Zod enum error where they should meet an explanation.
|
|
9398
|
+
*
|
|
9399
|
+
* `recordings` encompasses the high and mid segment profiles; `recordingsLow`
|
|
9400
|
+
* is low segments; `eventMedia` is post-analysis blobs; `galleryMedia` is the
|
|
9401
|
+
* enrolled gallery; `backups` is the system backup archive. The last two have
|
|
9402
|
+
* their own rules — see {@link StorageMigrationFindingCodeSchema}. */
|
|
9362
9403
|
var StorageMigrationClassSchema = _enum([
|
|
9363
9404
|
"recordings",
|
|
9364
9405
|
"recordingsLow",
|
|
9365
|
-
"eventMedia"
|
|
9406
|
+
"eventMedia",
|
|
9407
|
+
"backups",
|
|
9408
|
+
"galleryMedia"
|
|
9366
9409
|
]);
|
|
9367
9410
|
/** A destination is always an existing, fully-qualified location id. The
|
|
9368
9411
|
* migration API intentionally never changes a source location's `basePath`:
|
|
@@ -9370,20 +9413,56 @@ var StorageMigrationClassSchema = _enum([
|
|
|
9370
9413
|
var StorageMigrationDestinationsSchema = object({
|
|
9371
9414
|
recordings: string().min(1).optional(),
|
|
9372
9415
|
recordingsLow: string().min(1).optional(),
|
|
9373
|
-
eventMedia: string().min(1).optional()
|
|
9416
|
+
eventMedia: string().min(1).optional(),
|
|
9417
|
+
backups: string().min(1).optional(),
|
|
9418
|
+
galleryMedia: string().min(1).optional()
|
|
9374
9419
|
}).refine((value) => Object.keys(value).length > 0, { message: "select at least one storage class" });
|
|
9420
|
+
/**
|
|
9421
|
+
* How a migration sequences the cutover against the byte move.
|
|
9422
|
+
*
|
|
9423
|
+
* - `blocking` — the historical order: pause, move every byte, repoint,
|
|
9424
|
+
* resume. Recording is stopped for the whole move. Right
|
|
9425
|
+
* for a small or a cold class, and the only legal mode for
|
|
9426
|
+
* a `cardinality: 'single'` class.
|
|
9427
|
+
* - `nonBlocking` — repoint FIRST, drain behind: seal, pause, repoint,
|
|
9428
|
+
* refresh, resume, then move the past with everything
|
|
9429
|
+
* running. The pause is three bounded instants (a detach +
|
|
9430
|
+
* attach round, a write-gate drain, a lease) instead of one
|
|
9431
|
+
* bounded by bytes. 1.09 TB at 7–14 MB/s is thirty hours of
|
|
9432
|
+
* stopped recording under `blocking`; the same move is
|
|
9433
|
+
* seconds of stopped recording under `nonBlocking`.
|
|
9434
|
+
*
|
|
9435
|
+
* The mode is on the JOB, not only on the input, because `status` is where an
|
|
9436
|
+
* operator finds out which one is running.
|
|
9437
|
+
*/
|
|
9438
|
+
var StorageMigrationModeSchema = _enum(["blocking", "nonBlocking"]);
|
|
9375
9439
|
/** Shared input for planning and starting an orchestrated storage migration. */
|
|
9376
9440
|
var StorageMigrationInputSchema = object({
|
|
9377
9441
|
destinations: StorageMigrationDestinationsSchema,
|
|
9378
|
-
throttleMbps: number().min(1).max(1e3).optional()
|
|
9442
|
+
throttleMbps: number().min(1).max(1e3).optional(),
|
|
9443
|
+
/** Omitted = `blocking`, which stays the default. */
|
|
9444
|
+
mode: StorageMigrationModeSchema.optional()
|
|
9379
9445
|
});
|
|
9380
|
-
/**
|
|
9381
|
-
*
|
|
9382
|
-
*
|
|
9446
|
+
/**
|
|
9447
|
+
* The durable coordinator state machine.
|
|
9448
|
+
*
|
|
9449
|
+
* `blocking`:
|
|
9450
|
+
* planning → pausing → moving → verifying → repointing → refreshing → resuming → done
|
|
9451
|
+
*
|
|
9452
|
+
* `nonBlocking`:
|
|
9453
|
+
* planning → sealing → pausing → repointing → refreshing → resuming → draining → verifying → done
|
|
9454
|
+
*
|
|
9455
|
+
* Same phases, different order plus two new ones — not a second mover.
|
|
9456
|
+
* `sealing` closes the `eventMedia` NULL-row hole BEFORE anything is paused;
|
|
9457
|
+
* `draining` runs the same movers UNLEASED, after every writer is back up.
|
|
9458
|
+
* `repointing` is still the only phase that changes a default location.
|
|
9459
|
+
*/
|
|
9383
9460
|
var StorageMigrationPhaseSchema = _enum([
|
|
9384
9461
|
"planning",
|
|
9462
|
+
"sealing",
|
|
9385
9463
|
"pausing",
|
|
9386
9464
|
"moving",
|
|
9465
|
+
"draining",
|
|
9387
9466
|
"verifying",
|
|
9388
9467
|
"repointing",
|
|
9389
9468
|
"refreshing",
|
|
@@ -9408,6 +9487,9 @@ var StorageMigrationMoveSchema = object({
|
|
|
9408
9487
|
var StorageMigrationJobSchema = object({
|
|
9409
9488
|
jobId: string(),
|
|
9410
9489
|
phase: StorageMigrationPhaseSchema,
|
|
9490
|
+
/** Which order this job is running. `status` is the only place an operator
|
|
9491
|
+
* can tell a seconds-long cutover from a thirty-hour one. */
|
|
9492
|
+
mode: StorageMigrationModeSchema,
|
|
9411
9493
|
destinations: StorageMigrationDestinationsSchema,
|
|
9412
9494
|
throttleMbps: number(),
|
|
9413
9495
|
moves: array(StorageMigrationMoveSchema),
|
|
@@ -9420,13 +9502,30 @@ var StorageMigrationJobSchema = object({
|
|
|
9420
9502
|
finishedAt: number().nullable(),
|
|
9421
9503
|
error: string().nullable()
|
|
9422
9504
|
});
|
|
9505
|
+
var StorageMigrationFindingSchema = object({
|
|
9506
|
+
code: _enum([
|
|
9507
|
+
"sharesDeviceWithSource",
|
|
9508
|
+
"deviceIdentityUnknown",
|
|
9509
|
+
"unstampedEventMediaRows",
|
|
9510
|
+
"blockingOnly",
|
|
9511
|
+
"noMover"
|
|
9512
|
+
]),
|
|
9513
|
+
storageClass: StorageMigrationClassSchema,
|
|
9514
|
+
/** Human-readable, already carrying the ids and counts. */
|
|
9515
|
+
message: string()
|
|
9516
|
+
});
|
|
9423
9517
|
var StorageMigrationPlanSchema = object({
|
|
9424
9518
|
destinations: StorageMigrationDestinationsSchema,
|
|
9519
|
+
/** The mode this plan was built for. A plan is only valid for its mode: the
|
|
9520
|
+
* `eventMedia` seal gate and the single-cardinality refusal both depend on
|
|
9521
|
+
* it. */
|
|
9522
|
+
mode: StorageMigrationModeSchema,
|
|
9425
9523
|
moves: array(object({
|
|
9426
9524
|
storageClass: StorageMigrationClassSchema,
|
|
9427
9525
|
fromLocationId: string(),
|
|
9428
9526
|
toLocationId: string()
|
|
9429
|
-
}))
|
|
9527
|
+
})),
|
|
9528
|
+
findings: array(StorageMigrationFindingSchema)
|
|
9430
9529
|
});
|
|
9431
9530
|
/**
|
|
9432
9531
|
* `StorageLocationType` — an addon-declared id that identifies the *kind* of
|
|
@@ -21099,7 +21198,7 @@ DeviceType.Camera, method(object({ deviceId: number() }), array(TrackSchema).rea
|
|
|
21099
21198
|
}), method(RelocateMediaInputSchema, object({ jobId: string() }), {
|
|
21100
21199
|
kind: "mutation",
|
|
21101
21200
|
auth: "admin"
|
|
21102
|
-
}), method(object({}), array(RelocateJobSchema).readonly(), {
|
|
21201
|
+
}), method(object({}), UnstampedEventMediaCountSchema, { auth: "admin" }), method(object({}), array(RelocateJobSchema).readonly(), {
|
|
21103
21202
|
kind: "query",
|
|
21104
21203
|
auth: "admin"
|
|
21105
21204
|
}), method(object({ jobId: string() }), object({ cancelled: boolean() }), {
|
|
@@ -23148,7 +23247,10 @@ method(object({
|
|
|
23148
23247
|
}), StorageLocationSchema, {
|
|
23149
23248
|
kind: "mutation",
|
|
23150
23249
|
auth: "admin"
|
|
23151
|
-
}), method(object({
|
|
23250
|
+
}), method(object({
|
|
23251
|
+
id: string(),
|
|
23252
|
+
force: boolean().optional()
|
|
23253
|
+
}), _void(), {
|
|
23152
23254
|
kind: "mutation",
|
|
23153
23255
|
auth: "admin"
|
|
23154
23256
|
}), method(object({ id: string() }), object({
|
|
@@ -34108,6 +34210,12 @@ Object.freeze({
|
|
|
34108
34210
|
addonId: null,
|
|
34109
34211
|
access: "create"
|
|
34110
34212
|
},
|
|
34213
|
+
"pipelineAnalytics.countUnstampedEventMedia": {
|
|
34214
|
+
capName: "pipeline-analytics",
|
|
34215
|
+
capScope: "device",
|
|
34216
|
+
addonId: null,
|
|
34217
|
+
access: "view"
|
|
34218
|
+
},
|
|
34111
34219
|
"pipelineAnalytics.deleteDeviceEvents": {
|
|
34112
34220
|
capName: "pipeline-analytics",
|
|
34113
34221
|
capScope: "device",
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { PerformanceObserver } from "node:perf_hooks";
|
|
2
|
+
//#region src/detection-pipeline/diagnostics/event-loop-stall-monitor.ts
|
|
3
|
+
/**
|
|
4
|
+
* event-loop-stall-monitor — names the thing that freezes the detection process.
|
|
5
|
+
*
|
|
6
|
+
* ## What this exists to settle
|
|
7
|
+
*
|
|
8
|
+
* On 2026-08-01/02 inference across the whole fleet stalled in bursts: 759
|
|
9
|
+
* calls in 90 minutes crossed the runner's 20s deadline, p50 41s and max 80s,
|
|
10
|
+
* and EVERY one of them eventually resolved. Six explanations were killed with
|
|
11
|
+
* telemetry alone:
|
|
12
|
+
*
|
|
13
|
+
* - not a slow GPU or a wedged pool — the work completes, just enormously late;
|
|
14
|
+
* - not CPU starvation — hosts sat at 8-48% during the freezes;
|
|
15
|
+
* - not swap — `pswpin/pswpout` are flat zero on every host;
|
|
16
|
+
* - not a logging artifact — Loki's timestamp trails the in-process one by
|
|
17
|
+
* 0.1-1.0s at p95, so the gaps are real, not buffered delivery;
|
|
18
|
+
* - not the v1.2.30 release — the stalls predate it by a day;
|
|
19
|
+
* - not only the idle-pool recompile — that is real and hub-side, but the
|
|
20
|
+
* agent stalls with no eviction and no compile at all.
|
|
21
|
+
*
|
|
22
|
+
* What IS established: the 10s stats timer — which touches no inference code —
|
|
23
|
+
* stops firing for 22-88s and resumes at the same instant the abandoned calls
|
|
24
|
+
* settle. The event loop itself freezes. From outside the process, GC and a
|
|
25
|
+
* synchronous block are indistinguishable, and guessing between them is how
|
|
26
|
+
* this investigation would go wrong. So measure it.
|
|
27
|
+
*
|
|
28
|
+
* ## How
|
|
29
|
+
*
|
|
30
|
+
* A 1s interval whose LATENESS is the freeze: a tick that arrives 40s late means
|
|
31
|
+
* the loop was blocked ~39s, whatever blocked it. In the same window the GC
|
|
32
|
+
* observer accumulates pause time. The two numbers together answer the question
|
|
33
|
+
* outright:
|
|
34
|
+
*
|
|
35
|
+
* - `gcMs` ≈ `stallMs` → garbage collection. Look at heap growth.
|
|
36
|
+
* - `gcMs` ≈ 0 → something SYNCHRONOUS blocked the loop, and GC is innocent.
|
|
37
|
+
*
|
|
38
|
+
* Heap and RSS ride along because if it is GC, the next question is always
|
|
39
|
+
* "growing since when?".
|
|
40
|
+
*
|
|
41
|
+
* Deliberately not a histogram (`monitorEventLoopDelay`): that reports a
|
|
42
|
+
* distribution, and the question here is about ONE outlier and what coincided
|
|
43
|
+
* with it.
|
|
44
|
+
*/
|
|
45
|
+
/** Tick period. Small enough to bound attribution error, large enough to cost
|
|
46
|
+
* nothing on a process already doing frame work. */
|
|
47
|
+
var STALL_TICK_MS = 1e3;
|
|
48
|
+
/**
|
|
49
|
+
* Report a freeze only past this. The runner's own inference deadline is 20s
|
|
50
|
+
* and the observed freezes run 22-88s, so this is far below the signal while
|
|
51
|
+
* staying clear of ordinary scheduling jitter (a busy frame path routinely
|
|
52
|
+
* overruns a 1s timer by tens of ms).
|
|
53
|
+
*/
|
|
54
|
+
var STALL_REPORT_MS = 2e3;
|
|
55
|
+
/**
|
|
56
|
+
* Attribute a freeze. Pure, because this is the ONE judgement the whole module
|
|
57
|
+
* exists to make and it must be testable without freezing a real event loop.
|
|
58
|
+
*
|
|
59
|
+
* `gc` when garbage collection accounts for most of the stall; a
|
|
60
|
+
* `synchronous-block` when it accounts for almost none — that verdict points
|
|
61
|
+
* away from the heap entirely and at whatever ran without yielding.
|
|
62
|
+
*/
|
|
63
|
+
function attributeStall(stallMs, gcMs, gcCount, longestGcMs) {
|
|
64
|
+
const share = stallMs > 0 ? gcMs / stallMs : 0;
|
|
65
|
+
return {
|
|
66
|
+
stallMs,
|
|
67
|
+
gcMs,
|
|
68
|
+
gcCount,
|
|
69
|
+
longestGcMs,
|
|
70
|
+
cause: share >= .7 ? "gc" : share <= .2 ? "synchronous-block" : "mixed"
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Start watching. Returns a stop function; safe to call twice.
|
|
75
|
+
*
|
|
76
|
+
* Never throws into the caller: a diagnostic that can take the detection
|
|
77
|
+
* process down is worse than no diagnostic. `PerformanceObserver` for `gc` is
|
|
78
|
+
* unavailable in some embedded runtimes, and that must degrade to
|
|
79
|
+
* stall-detection-without-attribution rather than a boot failure.
|
|
80
|
+
*/
|
|
81
|
+
function startEventLoopStallMonitor(logger, reportThresholdMs = STALL_REPORT_MS) {
|
|
82
|
+
let gc = {
|
|
83
|
+
totalMs: 0,
|
|
84
|
+
count: 0,
|
|
85
|
+
longestMs: 0
|
|
86
|
+
};
|
|
87
|
+
let observer = null;
|
|
88
|
+
try {
|
|
89
|
+
observer = new PerformanceObserver((list) => {
|
|
90
|
+
for (const entry of list.getEntries()) {
|
|
91
|
+
gc.totalMs += entry.duration;
|
|
92
|
+
gc.count += 1;
|
|
93
|
+
if (entry.duration > gc.longestMs) gc.longestMs = entry.duration;
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
observer.observe({ entryTypes: ["gc"] });
|
|
97
|
+
} catch {
|
|
98
|
+
observer = null;
|
|
99
|
+
}
|
|
100
|
+
let expected = Date.now() + STALL_TICK_MS;
|
|
101
|
+
const timer = setInterval(() => {
|
|
102
|
+
const now = Date.now();
|
|
103
|
+
const lateBy = now - expected;
|
|
104
|
+
const window = gc;
|
|
105
|
+
gc = {
|
|
106
|
+
totalMs: 0,
|
|
107
|
+
count: 0,
|
|
108
|
+
longestMs: 0
|
|
109
|
+
};
|
|
110
|
+
expected = now + STALL_TICK_MS;
|
|
111
|
+
if (lateBy < reportThresholdMs) return;
|
|
112
|
+
const verdict = attributeStall(lateBy, Math.round(window.totalMs), window.count, Math.round(window.longestMs));
|
|
113
|
+
const mem = process.memoryUsage();
|
|
114
|
+
logger.warn("event loop stalled", { meta: {
|
|
115
|
+
stallMs: verdict.stallMs,
|
|
116
|
+
cause: verdict.cause,
|
|
117
|
+
gcMs: verdict.gcMs,
|
|
118
|
+
gcCount: verdict.gcCount,
|
|
119
|
+
longestGcMs: verdict.longestGcMs,
|
|
120
|
+
gcObserved: observer !== null,
|
|
121
|
+
heapUsedMb: Math.round(mem.heapUsed / 1e6),
|
|
122
|
+
heapTotalMb: Math.round(mem.heapTotal / 1e6),
|
|
123
|
+
rssMb: Math.round(mem.rss / 1e6),
|
|
124
|
+
externalMb: Math.round(mem.external / 1e6),
|
|
125
|
+
arrayBuffersMb: Math.round(mem.arrayBuffers / 1e6)
|
|
126
|
+
} });
|
|
127
|
+
}, STALL_TICK_MS);
|
|
128
|
+
timer.unref?.();
|
|
129
|
+
let stopped = false;
|
|
130
|
+
return () => {
|
|
131
|
+
if (stopped) return;
|
|
132
|
+
stopped = true;
|
|
133
|
+
clearInterval(timer);
|
|
134
|
+
observer?.disconnect();
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
//#endregion
|
|
138
|
+
export { startEventLoopStallMonitor as n, attributeStall as t };
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
let node_perf_hooks = require("node:perf_hooks");
|
|
2
|
+
//#region src/detection-pipeline/diagnostics/event-loop-stall-monitor.ts
|
|
3
|
+
/**
|
|
4
|
+
* event-loop-stall-monitor — names the thing that freezes the detection process.
|
|
5
|
+
*
|
|
6
|
+
* ## What this exists to settle
|
|
7
|
+
*
|
|
8
|
+
* On 2026-08-01/02 inference across the whole fleet stalled in bursts: 759
|
|
9
|
+
* calls in 90 minutes crossed the runner's 20s deadline, p50 41s and max 80s,
|
|
10
|
+
* and EVERY one of them eventually resolved. Six explanations were killed with
|
|
11
|
+
* telemetry alone:
|
|
12
|
+
*
|
|
13
|
+
* - not a slow GPU or a wedged pool — the work completes, just enormously late;
|
|
14
|
+
* - not CPU starvation — hosts sat at 8-48% during the freezes;
|
|
15
|
+
* - not swap — `pswpin/pswpout` are flat zero on every host;
|
|
16
|
+
* - not a logging artifact — Loki's timestamp trails the in-process one by
|
|
17
|
+
* 0.1-1.0s at p95, so the gaps are real, not buffered delivery;
|
|
18
|
+
* - not the v1.2.30 release — the stalls predate it by a day;
|
|
19
|
+
* - not only the idle-pool recompile — that is real and hub-side, but the
|
|
20
|
+
* agent stalls with no eviction and no compile at all.
|
|
21
|
+
*
|
|
22
|
+
* What IS established: the 10s stats timer — which touches no inference code —
|
|
23
|
+
* stops firing for 22-88s and resumes at the same instant the abandoned calls
|
|
24
|
+
* settle. The event loop itself freezes. From outside the process, GC and a
|
|
25
|
+
* synchronous block are indistinguishable, and guessing between them is how
|
|
26
|
+
* this investigation would go wrong. So measure it.
|
|
27
|
+
*
|
|
28
|
+
* ## How
|
|
29
|
+
*
|
|
30
|
+
* A 1s interval whose LATENESS is the freeze: a tick that arrives 40s late means
|
|
31
|
+
* the loop was blocked ~39s, whatever blocked it. In the same window the GC
|
|
32
|
+
* observer accumulates pause time. The two numbers together answer the question
|
|
33
|
+
* outright:
|
|
34
|
+
*
|
|
35
|
+
* - `gcMs` ≈ `stallMs` → garbage collection. Look at heap growth.
|
|
36
|
+
* - `gcMs` ≈ 0 → something SYNCHRONOUS blocked the loop, and GC is innocent.
|
|
37
|
+
*
|
|
38
|
+
* Heap and RSS ride along because if it is GC, the next question is always
|
|
39
|
+
* "growing since when?".
|
|
40
|
+
*
|
|
41
|
+
* Deliberately not a histogram (`monitorEventLoopDelay`): that reports a
|
|
42
|
+
* distribution, and the question here is about ONE outlier and what coincided
|
|
43
|
+
* with it.
|
|
44
|
+
*/
|
|
45
|
+
/** Tick period. Small enough to bound attribution error, large enough to cost
|
|
46
|
+
* nothing on a process already doing frame work. */
|
|
47
|
+
var STALL_TICK_MS = 1e3;
|
|
48
|
+
/**
|
|
49
|
+
* Report a freeze only past this. The runner's own inference deadline is 20s
|
|
50
|
+
* and the observed freezes run 22-88s, so this is far below the signal while
|
|
51
|
+
* staying clear of ordinary scheduling jitter (a busy frame path routinely
|
|
52
|
+
* overruns a 1s timer by tens of ms).
|
|
53
|
+
*/
|
|
54
|
+
var STALL_REPORT_MS = 2e3;
|
|
55
|
+
/**
|
|
56
|
+
* Attribute a freeze. Pure, because this is the ONE judgement the whole module
|
|
57
|
+
* exists to make and it must be testable without freezing a real event loop.
|
|
58
|
+
*
|
|
59
|
+
* `gc` when garbage collection accounts for most of the stall; a
|
|
60
|
+
* `synchronous-block` when it accounts for almost none — that verdict points
|
|
61
|
+
* away from the heap entirely and at whatever ran without yielding.
|
|
62
|
+
*/
|
|
63
|
+
function attributeStall(stallMs, gcMs, gcCount, longestGcMs) {
|
|
64
|
+
const share = stallMs > 0 ? gcMs / stallMs : 0;
|
|
65
|
+
return {
|
|
66
|
+
stallMs,
|
|
67
|
+
gcMs,
|
|
68
|
+
gcCount,
|
|
69
|
+
longestGcMs,
|
|
70
|
+
cause: share >= .7 ? "gc" : share <= .2 ? "synchronous-block" : "mixed"
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Start watching. Returns a stop function; safe to call twice.
|
|
75
|
+
*
|
|
76
|
+
* Never throws into the caller: a diagnostic that can take the detection
|
|
77
|
+
* process down is worse than no diagnostic. `PerformanceObserver` for `gc` is
|
|
78
|
+
* unavailable in some embedded runtimes, and that must degrade to
|
|
79
|
+
* stall-detection-without-attribution rather than a boot failure.
|
|
80
|
+
*/
|
|
81
|
+
function startEventLoopStallMonitor(logger, reportThresholdMs = STALL_REPORT_MS) {
|
|
82
|
+
let gc = {
|
|
83
|
+
totalMs: 0,
|
|
84
|
+
count: 0,
|
|
85
|
+
longestMs: 0
|
|
86
|
+
};
|
|
87
|
+
let observer = null;
|
|
88
|
+
try {
|
|
89
|
+
observer = new node_perf_hooks.PerformanceObserver((list) => {
|
|
90
|
+
for (const entry of list.getEntries()) {
|
|
91
|
+
gc.totalMs += entry.duration;
|
|
92
|
+
gc.count += 1;
|
|
93
|
+
if (entry.duration > gc.longestMs) gc.longestMs = entry.duration;
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
observer.observe({ entryTypes: ["gc"] });
|
|
97
|
+
} catch {
|
|
98
|
+
observer = null;
|
|
99
|
+
}
|
|
100
|
+
let expected = Date.now() + STALL_TICK_MS;
|
|
101
|
+
const timer = setInterval(() => {
|
|
102
|
+
const now = Date.now();
|
|
103
|
+
const lateBy = now - expected;
|
|
104
|
+
const window = gc;
|
|
105
|
+
gc = {
|
|
106
|
+
totalMs: 0,
|
|
107
|
+
count: 0,
|
|
108
|
+
longestMs: 0
|
|
109
|
+
};
|
|
110
|
+
expected = now + STALL_TICK_MS;
|
|
111
|
+
if (lateBy < reportThresholdMs) return;
|
|
112
|
+
const verdict = attributeStall(lateBy, Math.round(window.totalMs), window.count, Math.round(window.longestMs));
|
|
113
|
+
const mem = process.memoryUsage();
|
|
114
|
+
logger.warn("event loop stalled", { meta: {
|
|
115
|
+
stallMs: verdict.stallMs,
|
|
116
|
+
cause: verdict.cause,
|
|
117
|
+
gcMs: verdict.gcMs,
|
|
118
|
+
gcCount: verdict.gcCount,
|
|
119
|
+
longestGcMs: verdict.longestGcMs,
|
|
120
|
+
gcObserved: observer !== null,
|
|
121
|
+
heapUsedMb: Math.round(mem.heapUsed / 1e6),
|
|
122
|
+
heapTotalMb: Math.round(mem.heapTotal / 1e6),
|
|
123
|
+
rssMb: Math.round(mem.rss / 1e6),
|
|
124
|
+
externalMb: Math.round(mem.external / 1e6),
|
|
125
|
+
arrayBuffersMb: Math.round(mem.arrayBuffers / 1e6)
|
|
126
|
+
} });
|
|
127
|
+
}, STALL_TICK_MS);
|
|
128
|
+
timer.unref?.();
|
|
129
|
+
let stopped = false;
|
|
130
|
+
return () => {
|
|
131
|
+
if (stopped) return;
|
|
132
|
+
stopped = true;
|
|
133
|
+
clearInterval(timer);
|
|
134
|
+
observer?.disconnect();
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
//#endregion
|
|
138
|
+
Object.defineProperty(exports, "attributeStall", {
|
|
139
|
+
enumerable: true,
|
|
140
|
+
get: function() {
|
|
141
|
+
return attributeStall;
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
Object.defineProperty(exports, "startEventLoopStallMonitor", {
|
|
145
|
+
enumerable: true,
|
|
146
|
+
get: function() {
|
|
147
|
+
return startEventLoopStallMonitor;
|
|
148
|
+
}
|
|
149
|
+
});
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { M as audioKindId, Z as hfModelUrl, a as COCO_80_LABELS, o as COCO_TO_MACRO, r as AUDIO_MACRO_LABELS } from "./dist-
|
|
1
|
+
import { M as audioKindId, Z as hfModelUrl, a as COCO_80_LABELS, o as COCO_TO_MACRO, r as AUDIO_MACRO_LABELS } from "./dist-DjplinYP.mjs";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
-
import { PerformanceObserver } from "node:perf_hooks";
|
|
4
3
|
//#region src/detection-pipeline/registry/model-catalogs.ts
|
|
5
4
|
var HF_REPO = "camstack/camstack-models";
|
|
6
5
|
var HF_SCRYPTED = "scrypted/plugin-models";
|
|
@@ -2111,140 +2110,4 @@ var localFrameRegistry = new LocalFrameRegistry({
|
|
|
2111
2110
|
maxBytes: envPositiveInt("CAMSTACK_LOCAL_FRAME_BUDGET_MB", DEFAULT_MAX_BYTES / 1048576) * 1048576
|
|
2112
2111
|
});
|
|
2113
2112
|
//#endregion
|
|
2114
|
-
|
|
2115
|
-
/**
|
|
2116
|
-
* event-loop-stall-monitor — names the thing that freezes the detection process.
|
|
2117
|
-
*
|
|
2118
|
-
* ## What this exists to settle
|
|
2119
|
-
*
|
|
2120
|
-
* On 2026-08-01/02 inference across the whole fleet stalled in bursts: 759
|
|
2121
|
-
* calls in 90 minutes crossed the runner's 20s deadline, p50 41s and max 80s,
|
|
2122
|
-
* and EVERY one of them eventually resolved. Six explanations were killed with
|
|
2123
|
-
* telemetry alone:
|
|
2124
|
-
*
|
|
2125
|
-
* - not a slow GPU or a wedged pool — the work completes, just enormously late;
|
|
2126
|
-
* - not CPU starvation — hosts sat at 8-48% during the freezes;
|
|
2127
|
-
* - not swap — `pswpin/pswpout` are flat zero on every host;
|
|
2128
|
-
* - not a logging artifact — Loki's timestamp trails the in-process one by
|
|
2129
|
-
* 0.1-1.0s at p95, so the gaps are real, not buffered delivery;
|
|
2130
|
-
* - not the v1.2.30 release — the stalls predate it by a day;
|
|
2131
|
-
* - not only the idle-pool recompile — that is real and hub-side, but the
|
|
2132
|
-
* agent stalls with no eviction and no compile at all.
|
|
2133
|
-
*
|
|
2134
|
-
* What IS established: the 10s stats timer — which touches no inference code —
|
|
2135
|
-
* stops firing for 22-88s and resumes at the same instant the abandoned calls
|
|
2136
|
-
* settle. The event loop itself freezes. From outside the process, GC and a
|
|
2137
|
-
* synchronous block are indistinguishable, and guessing between them is how
|
|
2138
|
-
* this investigation would go wrong. So measure it.
|
|
2139
|
-
*
|
|
2140
|
-
* ## How
|
|
2141
|
-
*
|
|
2142
|
-
* A 1s interval whose LATENESS is the freeze: a tick that arrives 40s late means
|
|
2143
|
-
* the loop was blocked ~39s, whatever blocked it. In the same window the GC
|
|
2144
|
-
* observer accumulates pause time. The two numbers together answer the question
|
|
2145
|
-
* outright:
|
|
2146
|
-
*
|
|
2147
|
-
* - `gcMs` ≈ `stallMs` → garbage collection. Look at heap growth.
|
|
2148
|
-
* - `gcMs` ≈ 0 → something SYNCHRONOUS blocked the loop, and GC is innocent.
|
|
2149
|
-
*
|
|
2150
|
-
* Heap and RSS ride along because if it is GC, the next question is always
|
|
2151
|
-
* "growing since when?".
|
|
2152
|
-
*
|
|
2153
|
-
* Deliberately not a histogram (`monitorEventLoopDelay`): that reports a
|
|
2154
|
-
* distribution, and the question here is about ONE outlier and what coincided
|
|
2155
|
-
* with it.
|
|
2156
|
-
*/
|
|
2157
|
-
/** Tick period. Small enough to bound attribution error, large enough to cost
|
|
2158
|
-
* nothing on a process already doing frame work. */
|
|
2159
|
-
var STALL_TICK_MS = 1e3;
|
|
2160
|
-
/**
|
|
2161
|
-
* Report a freeze only past this. The runner's own inference deadline is 20s
|
|
2162
|
-
* and the observed freezes run 22-88s, so this is far below the signal while
|
|
2163
|
-
* staying clear of ordinary scheduling jitter (a busy frame path routinely
|
|
2164
|
-
* overruns a 1s timer by tens of ms).
|
|
2165
|
-
*/
|
|
2166
|
-
var STALL_REPORT_MS = 2e3;
|
|
2167
|
-
/**
|
|
2168
|
-
* Attribute a freeze. Pure, because this is the ONE judgement the whole module
|
|
2169
|
-
* exists to make and it must be testable without freezing a real event loop.
|
|
2170
|
-
*
|
|
2171
|
-
* `gc` when garbage collection accounts for most of the stall; a
|
|
2172
|
-
* `synchronous-block` when it accounts for almost none — that verdict points
|
|
2173
|
-
* away from the heap entirely and at whatever ran without yielding.
|
|
2174
|
-
*/
|
|
2175
|
-
function attributeStall(stallMs, gcMs, gcCount, longestGcMs) {
|
|
2176
|
-
const share = stallMs > 0 ? gcMs / stallMs : 0;
|
|
2177
|
-
return {
|
|
2178
|
-
stallMs,
|
|
2179
|
-
gcMs,
|
|
2180
|
-
gcCount,
|
|
2181
|
-
longestGcMs,
|
|
2182
|
-
cause: share >= .7 ? "gc" : share <= .2 ? "synchronous-block" : "mixed"
|
|
2183
|
-
};
|
|
2184
|
-
}
|
|
2185
|
-
/**
|
|
2186
|
-
* Start watching. Returns a stop function; safe to call twice.
|
|
2187
|
-
*
|
|
2188
|
-
* Never throws into the caller: a diagnostic that can take the detection
|
|
2189
|
-
* process down is worse than no diagnostic. `PerformanceObserver` for `gc` is
|
|
2190
|
-
* unavailable in some embedded runtimes, and that must degrade to
|
|
2191
|
-
* stall-detection-without-attribution rather than a boot failure.
|
|
2192
|
-
*/
|
|
2193
|
-
function startEventLoopStallMonitor(logger, reportThresholdMs = STALL_REPORT_MS) {
|
|
2194
|
-
let gc = {
|
|
2195
|
-
totalMs: 0,
|
|
2196
|
-
count: 0,
|
|
2197
|
-
longestMs: 0
|
|
2198
|
-
};
|
|
2199
|
-
let observer = null;
|
|
2200
|
-
try {
|
|
2201
|
-
observer = new PerformanceObserver((list) => {
|
|
2202
|
-
for (const entry of list.getEntries()) {
|
|
2203
|
-
gc.totalMs += entry.duration;
|
|
2204
|
-
gc.count += 1;
|
|
2205
|
-
if (entry.duration > gc.longestMs) gc.longestMs = entry.duration;
|
|
2206
|
-
}
|
|
2207
|
-
});
|
|
2208
|
-
observer.observe({ entryTypes: ["gc"] });
|
|
2209
|
-
} catch {
|
|
2210
|
-
observer = null;
|
|
2211
|
-
}
|
|
2212
|
-
let expected = Date.now() + STALL_TICK_MS;
|
|
2213
|
-
const timer = setInterval(() => {
|
|
2214
|
-
const now = Date.now();
|
|
2215
|
-
const lateBy = now - expected;
|
|
2216
|
-
const window = gc;
|
|
2217
|
-
gc = {
|
|
2218
|
-
totalMs: 0,
|
|
2219
|
-
count: 0,
|
|
2220
|
-
longestMs: 0
|
|
2221
|
-
};
|
|
2222
|
-
expected = now + STALL_TICK_MS;
|
|
2223
|
-
if (lateBy < reportThresholdMs) return;
|
|
2224
|
-
const verdict = attributeStall(lateBy, Math.round(window.totalMs), window.count, Math.round(window.longestMs));
|
|
2225
|
-
const mem = process.memoryUsage();
|
|
2226
|
-
logger.warn("event loop stalled", { meta: {
|
|
2227
|
-
stallMs: verdict.stallMs,
|
|
2228
|
-
cause: verdict.cause,
|
|
2229
|
-
gcMs: verdict.gcMs,
|
|
2230
|
-
gcCount: verdict.gcCount,
|
|
2231
|
-
longestGcMs: verdict.longestGcMs,
|
|
2232
|
-
gcObserved: observer !== null,
|
|
2233
|
-
heapUsedMb: Math.round(mem.heapUsed / 1e6),
|
|
2234
|
-
heapTotalMb: Math.round(mem.heapTotal / 1e6),
|
|
2235
|
-
rssMb: Math.round(mem.rss / 1e6),
|
|
2236
|
-
externalMb: Math.round(mem.external / 1e6),
|
|
2237
|
-
arrayBuffersMb: Math.round(mem.arrayBuffers / 1e6)
|
|
2238
|
-
} });
|
|
2239
|
-
}, STALL_TICK_MS);
|
|
2240
|
-
timer.unref?.();
|
|
2241
|
-
let stopped = false;
|
|
2242
|
-
return () => {
|
|
2243
|
-
if (stopped) return;
|
|
2244
|
-
stopped = true;
|
|
2245
|
-
clearInterval(timer);
|
|
2246
|
-
observer?.disconnect();
|
|
2247
|
-
};
|
|
2248
|
-
}
|
|
2249
|
-
//#endregion
|
|
2250
|
-
export { ALL_PIPELINE_STEPS as a, getDefaultModelForFormatFromDef as c, resolveModelForFormat as d, landmarkPrecisionVerdict as f, resolveFrameViewGeometry as i, getStep as l, startEventLoopStallMonitor as n, ALL_STEPS as o, localFrameRegistry as r, getDefaultModelForFormat as s, attributeStall as t, getStepDefinition as u };
|
|
2113
|
+
export { getDefaultModelForFormat as a, getStepDefinition as c, ALL_STEPS as i, resolveModelForFormat as l, resolveFrameViewGeometry as n, getDefaultModelForFormatFromDef as o, ALL_PIPELINE_STEPS as r, getStep as s, localFrameRegistry as t, landmarkPrecisionVerdict as u };
|