@camstack/addon-pipeline 1.2.94 → 1.2.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-DnUCCZVx.js → addon-utils-B6usm-YO.js} +1 -1
- package/dist/audio-analyzer/index.js +58 -4
- package/dist/audio-analyzer/index.mjs +56 -2
- package/dist/detection-pipeline/index.js +194 -6
- package/dist/detection-pipeline/index.mjs +191 -3
- package/dist/{dist-XbrYiMV3.mjs → dist-LIWfUNC_.mjs} +324 -2
- package/dist/{dist-BbRv3bM2.js → dist-w9QWDvWn.js} +341 -1
- package/dist/{event-loop-stall-monitor-CFWrOZ6G.mjs → event-loop-stall-monitor-BDC8qURn.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-D9hqbc68.js → event-loop-stall-monitor-DndI0S2s.js} +1 -1
- package/dist/{lazy-sharp-U0EtN7_C.js → lazy-sharp-BVqBydNU.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +4 -4
- package/dist/pipeline-runner/index.mjs +3 -3
- package/dist/process-memory-C6zWIgos.js +78 -0
- package/dist/process-memory-k45KJaKH.mjs +66 -0
- package/dist/recorder/index.js +423 -47
- package/dist/recorder/index.mjs +422 -46
- package/dist/session-decode/decode-worker-child.js +2 -2
- package/dist/session-decode/decode-worker-child.mjs +1 -1
- package/dist/stream-broker/_stub.js +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BNOPhQ-y.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CJWMZMhQ.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-D2WOTeYi.mjs +26 -0
- package/dist/stream-broker/{hostInit-DCV2-mSi.mjs → hostInit-CGIo3d4K.mjs} +3 -3
- package/dist/stream-broker/index.js +25 -7
- package/dist/stream-broker/index.mjs +25 -7
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-D0ewr67U.mjs → worker-protocol-DTe7Ntat.mjs} +1 -1
- package/dist/{worker-protocol-CNC-ZcU3.js → worker-protocol-DqrXmX0g.js} +1 -1
- package/package.json +12 -7
- package/python/inference_pool.py +178 -3
- package/python/test_inference_pool_backpressure.py +7 -1
- package/python/test_inference_pool_memstats.py +146 -0
- package/dist/node-topology-platform-BkR_k6WT.mjs +0 -15
- package/dist/node-topology-platform-CFZ7F4xW.js +0 -20
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DMyOrQHq.mjs +0 -26
package/dist/recorder/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as recordingExportCapability, C as RecordingConfigSchema, Ct as nodePin, Et as selectAssignedProfileSlots, Ft as object, It as record, L as deriveRecordingMode, Lt as string, Pt as number, Q as recordingCapability, S as RECORDING_EXPORT_MAX_READ_BYTES, _t as DeviceType, ct as errMsg, f as EVENT_PAD_MS, it as storageEvictableCapability, kt as array, l as DEFAULT_EVENTS_BAND_BUFFER_SEC, m as ExportRecordSchema, mt as BaseAddon, nt as resolveRecordingProfiles, v as OpsLogEntrySchema, yt as hydrateSchema, zt as EventCategory } from "../dist-LIWfUNC_.mjs";
|
|
2
2
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
3
3
|
import { n as createFileDataPlaneHandler, s as parseRangeHeader, t as contentTypeFor } from "../addon-utils-A2S9D7pu.mjs";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
@@ -456,6 +456,51 @@ var MfraTableStore = class {
|
|
|
456
456
|
}
|
|
457
457
|
}
|
|
458
458
|
};
|
|
459
|
+
var PendingGate = class {
|
|
460
|
+
deps;
|
|
461
|
+
inFlight = 0;
|
|
462
|
+
droppedSinceReport = 0;
|
|
463
|
+
lastReportAt = Number.NEGATIVE_INFINITY;
|
|
464
|
+
constructor(deps) {
|
|
465
|
+
this.deps = deps;
|
|
466
|
+
}
|
|
467
|
+
/** Tasks currently in flight. */
|
|
468
|
+
get pending() {
|
|
469
|
+
return this.inFlight;
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* Admit `task` unless the lane is saturated.
|
|
473
|
+
*
|
|
474
|
+
* Returns `false` when the task was REFUSED — the caller must log the drop
|
|
475
|
+
* (with its `deviceId` tag when the work is about a device); a branch that
|
|
476
|
+
* drops work silently reads as "never happened".
|
|
477
|
+
*/
|
|
478
|
+
run(task) {
|
|
479
|
+
if (this.inFlight >= this.deps.limit) {
|
|
480
|
+
this.droppedSinceReport += 1;
|
|
481
|
+
const now = this.deps.now?.() ?? Date.now();
|
|
482
|
+
const intervalMs = this.deps.reportIntervalMs ?? 3e4;
|
|
483
|
+
if (now - this.lastReportAt >= intervalMs) {
|
|
484
|
+
this.lastReportAt = now;
|
|
485
|
+
const dropped = this.droppedSinceReport;
|
|
486
|
+
this.droppedSinceReport = 0;
|
|
487
|
+
this.deps.onSaturated({
|
|
488
|
+
label: this.deps.label,
|
|
489
|
+
dropped,
|
|
490
|
+
pending: this.inFlight
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
return false;
|
|
494
|
+
}
|
|
495
|
+
this.inFlight += 1;
|
|
496
|
+
Promise.resolve().then(task).catch((error) => {
|
|
497
|
+
this.deps.onTaskError?.(error);
|
|
498
|
+
}).finally(() => {
|
|
499
|
+
this.inFlight -= 1;
|
|
500
|
+
});
|
|
501
|
+
return true;
|
|
502
|
+
}
|
|
503
|
+
};
|
|
459
504
|
//#endregion
|
|
460
505
|
//#region src/recorder/merge-ranges.ts
|
|
461
506
|
function mergeRanges(segments, gapToleranceMs) {
|
|
@@ -515,12 +560,33 @@ function* hourBuckets(fromMs, toMs) {
|
|
|
515
560
|
const first = Math.floor(fromMs / HOUR_MS$5) * HOUR_MS$5;
|
|
516
561
|
for (let h = first; h < toMs; h += HOUR_MS$5) yield h;
|
|
517
562
|
}
|
|
563
|
+
/**
|
|
564
|
+
* Index of the first element whose `startMs` is GREATER than `startMs` (upper
|
|
565
|
+
* bound). Inserting there keeps equal-start rows in arrival order — exactly
|
|
566
|
+
* what a stable full sort of the backing map (insertion-ordered) produces, so
|
|
567
|
+
* an incrementally-maintained view is indistinguishable from a rebuilt one.
|
|
568
|
+
*/
|
|
569
|
+
function upperBoundByStart(rows, startMs) {
|
|
570
|
+
let lo = 0;
|
|
571
|
+
let hi = rows.length;
|
|
572
|
+
while (lo < hi) {
|
|
573
|
+
const mid = lo + hi >> 1;
|
|
574
|
+
if (rows[mid].startMs <= startMs) lo = mid + 1;
|
|
575
|
+
else hi = mid;
|
|
576
|
+
}
|
|
577
|
+
return lo;
|
|
578
|
+
}
|
|
518
579
|
var RecordingIndex = class {
|
|
519
580
|
byDevice = /* @__PURE__ */ new Map();
|
|
520
|
-
/** Start-sorted
|
|
521
|
-
*
|
|
522
|
-
*
|
|
581
|
+
/** Start-sorted views. Built lazily on first read (one copy+sort), then kept
|
|
582
|
+
* LIVE by `addSegment`'s ordered insert — rows arrive roughly ascending, so
|
|
583
|
+
* the insert is near-appending. Bulk mutations (hydrate, eviction, a re-set
|
|
584
|
+
* of an existing path) still invalidate; the steady-state finalize never
|
|
585
|
+
* does. Interactive locate/range calls reuse these instead of
|
|
586
|
+
* filtering+sorting the complete archive on every seek and scrub miss. */
|
|
523
587
|
sortedByDevice = /* @__PURE__ */ new Map();
|
|
588
|
+
fullRebuilds = 0;
|
|
589
|
+
incrementalInserts = 0;
|
|
524
590
|
/** Hour buckets this device has been walked for; `ALL_HOURS` after a full
|
|
525
591
|
* hydrate. Empty = nothing has been looked at, which is NOT the same as
|
|
526
592
|
* "there is nothing" — see {@link hydrationOf}. */
|
|
@@ -547,10 +613,34 @@ var RecordingIndex = class {
|
|
|
547
613
|
if (cached) return cached;
|
|
548
614
|
const m = this.byDevice.get(deviceId);
|
|
549
615
|
if (!m) return [];
|
|
616
|
+
this.fullRebuilds += 1;
|
|
550
617
|
const sorted = [...m.values()].filter((s) => profile == null || s.profile === profile).toSorted((a, b) => a.startMs - b.startMs);
|
|
551
618
|
deviceCache.set(key, sorted);
|
|
552
619
|
return sorted;
|
|
553
620
|
}
|
|
621
|
+
/** See {@link SortedViewStats}. Telemetry + the guard the churn test asserts on. */
|
|
622
|
+
sortedViewStats() {
|
|
623
|
+
return {
|
|
624
|
+
fullRebuilds: this.fullRebuilds,
|
|
625
|
+
incrementalInserts: this.incrementalInserts
|
|
626
|
+
};
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* Ordered insert of ONE NEW row into every live view it belongs to (the
|
|
630
|
+
* all-profiles view and its own profile's view). Views not yet materialised
|
|
631
|
+
* stay unmaterialised — their first read pays the one build. PRECONDITION:
|
|
632
|
+
* the row's path was NOT already in the device map; a replace must
|
|
633
|
+
* invalidate instead, because the views hold the old row object.
|
|
634
|
+
*/
|
|
635
|
+
insertIntoSortedViews(s) {
|
|
636
|
+
const deviceCache = this.sortedByDevice.get(s.deviceId);
|
|
637
|
+
if (!deviceCache) return;
|
|
638
|
+
for (const [key, rows] of deviceCache) {
|
|
639
|
+
if (key !== "" && key !== s.profile) continue;
|
|
640
|
+
rows.splice(upperBoundByStart(rows, s.startMs), 0, s);
|
|
641
|
+
this.incrementalInserts += 1;
|
|
642
|
+
}
|
|
643
|
+
}
|
|
554
644
|
/**
|
|
555
645
|
* Replace a device's segments FOR ONE LOCATION from a `storage.list` result (paths relative to that
|
|
556
646
|
* location root). Rows on other locations are preserved. Non-segment paths and other devices are ignored.
|
|
@@ -631,10 +721,26 @@ var RecordingIndex = class {
|
|
|
631
721
|
this.invalidateSorted(deviceId);
|
|
632
722
|
this.markHydrated(deviceId, hourStartMs, hourStartMs + HOUR_MS$5);
|
|
633
723
|
}
|
|
634
|
-
/**
|
|
724
|
+
/**
|
|
725
|
+
* Insert/replace one finalized segment (idempotent by path).
|
|
726
|
+
*
|
|
727
|
+
* THE hot mutation: every profile-writer lands here every `segmentSeconds`,
|
|
728
|
+
* ~3.4 times a second fleet-wide. A NEW path is an ordered insert into the
|
|
729
|
+
* live sorted views (rows arrive roughly ascending, so it is near-appending
|
|
730
|
+
* — O(log n) search + a short memmove). Only the rare RE-SET of an existing
|
|
731
|
+
* path (watcher replay after a playlist reset, or a relocate that changes
|
|
732
|
+
* `locationId`) invalidates: the views hold the old row object, and a
|
|
733
|
+
* one-off rebuild is the simple correct answer for a case that is not hot.
|
|
734
|
+
*/
|
|
635
735
|
addSegment(s) {
|
|
636
|
-
this.mapFor(s.deviceId)
|
|
637
|
-
|
|
736
|
+
const m = this.mapFor(s.deviceId);
|
|
737
|
+
const replacing = m.has(s.path);
|
|
738
|
+
m.set(s.path, s);
|
|
739
|
+
if (replacing) {
|
|
740
|
+
this.invalidateSorted(s.deviceId);
|
|
741
|
+
return;
|
|
742
|
+
}
|
|
743
|
+
this.insertIntoSortedViews(s);
|
|
638
744
|
}
|
|
639
745
|
/** Remove segments by relative path, routing each path to the correct device map via its encoded deviceId. */
|
|
640
746
|
removeSegments(paths) {
|
|
@@ -2900,6 +3006,67 @@ async function listExportsFor(state, deviceId) {
|
|
|
2900
3006
|
return [...(await readExports(state)).values()].filter((r) => deviceId === void 0 || r.deviceId === deviceId).toSorted((a, b) => b.createdAt - a.createdAt);
|
|
2901
3007
|
}
|
|
2902
3008
|
//#endregion
|
|
3009
|
+
//#region src/recorder/addon/spawn-failure.ts
|
|
3010
|
+
/**
|
|
3011
|
+
* How the recorder reads a `child_process` `'error'` — ONE owner of the
|
|
3012
|
+
* transient-vs-permanent question.
|
|
3013
|
+
*
|
|
3014
|
+
* ── Why this file exists ───────────────────────────────────────────────────
|
|
3015
|
+
* In Node a child that cannot be spawned emits `'error'`, and an `'error'`
|
|
3016
|
+
* with no listener is re-thrown as an UNCAUGHT EXCEPTION. On 2026-08-18 three
|
|
3017
|
+
* cameras were unplugged, every writer for them re-dialled at once, the host
|
|
3018
|
+
* momentarily could not fork (`spawn ffmpeg EAGAIN`), and the `recorder`
|
|
3019
|
+
* runner died three times inside nine seconds. The D6 crash circuit-breaker
|
|
3020
|
+
* did its job and stopped respawning — leaving the whole fleet with no
|
|
3021
|
+
* recording for three hours and fifteen minutes while `/health` returned 200.
|
|
3022
|
+
*
|
|
3023
|
+
* Every ffmpeg spawn site in the recorder therefore attaches an `'error'`
|
|
3024
|
+
* listener, and every one of them classifies the failure HERE rather than
|
|
3025
|
+
* inventing its own list.
|
|
3026
|
+
*/
|
|
3027
|
+
/**
|
|
3028
|
+
* Spawn failures that RETRYING CANNOT FIX.
|
|
3029
|
+
*
|
|
3030
|
+
* The EAGAIN-vs-ENOENT split, stated once:
|
|
3031
|
+
*
|
|
3032
|
+
* - EAGAIN / EMFILE / ENOMEM are the host saying "not right now". The
|
|
3033
|
+
* resource comes back, often within one backoff. They take the ordinary
|
|
3034
|
+
* restart-with-backoff path, whose own comment already states the policy
|
|
3035
|
+
* for this shape: "only RAPID back-to-back failures count toward giving
|
|
3036
|
+
* up, so a brief blip never permanently stops a camera's recording".
|
|
3037
|
+
* - ENOENT / EACCES are the host saying "there is no ffmpeg here you may
|
|
3038
|
+
* run". Nothing about the next ten attempts differs from this one, so the
|
|
3039
|
+
* caller stops immediately and surfaces it at ERROR level: burning a
|
|
3040
|
+
* ten-attempt budget on a missing binary only delays recovery and buries
|
|
3041
|
+
* the single actionable line under ten identical warnings.
|
|
3042
|
+
*
|
|
3043
|
+
* EPERM is deliberately absent: it is reported for transient sandbox/cgroup
|
|
3044
|
+
* refusals as well as for real permission faults, and treating an ambiguous
|
|
3045
|
+
* code as permanent stops a camera that would have recovered.
|
|
3046
|
+
*/
|
|
3047
|
+
var PERMANENT_SPAWN_CODES = new Set(["ENOENT", "EACCES"]);
|
|
3048
|
+
/**
|
|
3049
|
+
* Read the `'error'` payload without a cast.
|
|
3050
|
+
*
|
|
3051
|
+
* An unknown code is treated as TRANSIENT: the cost of retrying something
|
|
3052
|
+
* permanent is a bounded restart budget, while the cost of giving up on
|
|
3053
|
+
* something transient is a camera that stops recording until an operator
|
|
3054
|
+
* notices.
|
|
3055
|
+
*/
|
|
3056
|
+
function readSpawnFailure(raw) {
|
|
3057
|
+
const code = errnoCode(raw);
|
|
3058
|
+
return {
|
|
3059
|
+
code,
|
|
3060
|
+
message: raw instanceof Error ? raw.message : String(raw),
|
|
3061
|
+
permanent: code !== void 0 && PERMANENT_SPAWN_CODES.has(code)
|
|
3062
|
+
};
|
|
3063
|
+
}
|
|
3064
|
+
function errnoCode(raw) {
|
|
3065
|
+
if (typeof raw !== "object" || raw === null || !("code" in raw)) return void 0;
|
|
3066
|
+
const code = raw.code;
|
|
3067
|
+
return typeof code === "string" ? code : void 0;
|
|
3068
|
+
}
|
|
3069
|
+
//#endregion
|
|
2903
3070
|
//#region src/recorder/addon/export-engine.ts
|
|
2904
3071
|
/** Stderr lines retained for the failure LOG. ffmpeg prints one line per failed
|
|
2905
3072
|
* segment before the decisive "Error opening input" summary, so a short tail can
|
|
@@ -3069,7 +3236,7 @@ var ExportEngine = class {
|
|
|
3069
3236
|
outPath,
|
|
3070
3237
|
options
|
|
3071
3238
|
});
|
|
3072
|
-
const ran = await this.runFfmpeg(id, args, {
|
|
3239
|
+
const ran = await this.runFfmpeg(id, rec.deviceId, args, {
|
|
3073
3240
|
totalSec: expectedOutSeconds(rec),
|
|
3074
3241
|
floor: 0,
|
|
3075
3242
|
ceil: 99
|
|
@@ -3089,7 +3256,7 @@ var ExportEngine = class {
|
|
|
3089
3256
|
inputPlaylist: playlist,
|
|
3090
3257
|
outPath: concatPath
|
|
3091
3258
|
});
|
|
3092
|
-
const concatRan = await this.runFfmpeg(id, concatArgs, {
|
|
3259
|
+
const concatRan = await this.runFfmpeg(id, rec.deviceId, concatArgs, {
|
|
3093
3260
|
totalSec: Math.max(.001, (rec.toMs - rec.fromMs) / 1e3),
|
|
3094
3261
|
floor: 0,
|
|
3095
3262
|
ceil: 49
|
|
@@ -3103,7 +3270,7 @@ var ExportEngine = class {
|
|
|
3103
3270
|
outPath,
|
|
3104
3271
|
options
|
|
3105
3272
|
});
|
|
3106
|
-
const selectRan = await this.runFfmpeg(id, selectArgs, {
|
|
3273
|
+
const selectRan = await this.runFfmpeg(id, rec.deviceId, selectArgs, {
|
|
3107
3274
|
totalSec: expectedOutSeconds(rec),
|
|
3108
3275
|
floor: 50,
|
|
3109
3276
|
ceil: 99
|
|
@@ -3115,7 +3282,7 @@ var ExportEngine = class {
|
|
|
3115
3282
|
} catch {}
|
|
3116
3283
|
}
|
|
3117
3284
|
}
|
|
3118
|
-
runFfmpeg(id, args, progress) {
|
|
3285
|
+
runFfmpeg(id, deviceId, args, progress) {
|
|
3119
3286
|
this.deps.logger.debug("export ffmpeg argv", { meta: {
|
|
3120
3287
|
exportId: id,
|
|
3121
3288
|
args
|
|
@@ -3152,6 +3319,24 @@ var ExportEngine = class {
|
|
|
3152
3319
|
stderrTail
|
|
3153
3320
|
});
|
|
3154
3321
|
});
|
|
3322
|
+
proc.on("error", (raw) => {
|
|
3323
|
+
this.activeProc = null;
|
|
3324
|
+
const failure = readSpawnFailure(raw);
|
|
3325
|
+
this.deps.logger.error("export ffmpeg could not be spawned", {
|
|
3326
|
+
tags: { deviceId },
|
|
3327
|
+
meta: {
|
|
3328
|
+
exportId: id,
|
|
3329
|
+
errorCode: failure.code,
|
|
3330
|
+
error: failure.message,
|
|
3331
|
+
permanent: failure.permanent
|
|
3332
|
+
}
|
|
3333
|
+
});
|
|
3334
|
+
stderrTail.push(`ffmpeg spawn failed: ${failure.message}` + (failure.code === void 0 ? "" : ` (${failure.code})`));
|
|
3335
|
+
resolve({
|
|
3336
|
+
code: null,
|
|
3337
|
+
stderrTail
|
|
3338
|
+
});
|
|
3339
|
+
});
|
|
3155
3340
|
});
|
|
3156
3341
|
}
|
|
3157
3342
|
async finish(id, code, outPath, stderrTail) {
|
|
@@ -6129,7 +6314,9 @@ var ReadinessRestore = class {
|
|
|
6129
6314
|
this.deps = deps;
|
|
6130
6315
|
}
|
|
6131
6316
|
get concurrency() {
|
|
6132
|
-
|
|
6317
|
+
const declared = this.deps.concurrency;
|
|
6318
|
+
const value = typeof declared === "function" ? declared() : declared;
|
|
6319
|
+
return Math.max(1, value ?? 8);
|
|
6133
6320
|
}
|
|
6134
6321
|
/** Seed the pending set, subscribe to readiness, and drain once immediately
|
|
6135
6322
|
* (covers the already-ready case). Returns when the first drain settles. */
|
|
@@ -6188,6 +6375,70 @@ var ReadinessRestore = class {
|
|
|
6188
6375
|
}
|
|
6189
6376
|
}
|
|
6190
6377
|
};
|
|
6378
|
+
/** Adaptive bound for the recorder's attach parallelism. */
|
|
6379
|
+
var SpawnPressureGovernor = class {
|
|
6380
|
+
deps;
|
|
6381
|
+
limit;
|
|
6382
|
+
min;
|
|
6383
|
+
recoverAfterMs;
|
|
6384
|
+
/** When the limit last MOVED — the clock both directions are measured from. */
|
|
6385
|
+
lastChangeAt;
|
|
6386
|
+
underPressure = false;
|
|
6387
|
+
constructor(deps) {
|
|
6388
|
+
this.deps = deps;
|
|
6389
|
+
this.min = Math.max(1, deps.minConcurrency ?? 1);
|
|
6390
|
+
this.limit = Math.max(this.min, deps.maxConcurrency);
|
|
6391
|
+
this.recoverAfterMs = deps.recoverAfterMs ?? 3e4;
|
|
6392
|
+
this.lastChangeAt = this.now();
|
|
6393
|
+
}
|
|
6394
|
+
/**
|
|
6395
|
+
* How many attaches may overlap right now.
|
|
6396
|
+
*
|
|
6397
|
+
* Reading is what drives recovery: there is no timer to leak, no interval to
|
|
6398
|
+
* lose to a runner respawn, and a recorder that stops attaching entirely
|
|
6399
|
+
* simply keeps its reduced bound until it tries again — which is correct.
|
|
6400
|
+
*/
|
|
6401
|
+
get concurrency() {
|
|
6402
|
+
const max = Math.max(this.min, this.deps.maxConcurrency);
|
|
6403
|
+
if (this.limit >= max) return max;
|
|
6404
|
+
const elapsed = this.now() - this.lastChangeAt;
|
|
6405
|
+
if (elapsed < this.recoverAfterMs) return this.limit;
|
|
6406
|
+
const lanes = Math.floor(elapsed / this.recoverAfterMs);
|
|
6407
|
+
this.limit = Math.min(max, this.limit + lanes);
|
|
6408
|
+
this.lastChangeAt += lanes * this.recoverAfterMs;
|
|
6409
|
+
if (this.limit >= max && this.underPressure) {
|
|
6410
|
+
this.underPressure = false;
|
|
6411
|
+
this.deps.logger?.info("recorder: spawn pressure cleared — attach concurrency restored", { meta: { concurrency: this.limit } });
|
|
6412
|
+
} else this.deps.logger?.info("recorder: recovering attach concurrency", { meta: {
|
|
6413
|
+
concurrency: this.limit,
|
|
6414
|
+
max
|
|
6415
|
+
} });
|
|
6416
|
+
return this.limit;
|
|
6417
|
+
}
|
|
6418
|
+
/**
|
|
6419
|
+
* A transient spawn failure happened on `deviceId`'s writer.
|
|
6420
|
+
*
|
|
6421
|
+
* Halves the CURRENT limit, not the max: two waves of pressure inside one
|
|
6422
|
+
* recovery window must compound, or a host under sustained load is asked for
|
|
6423
|
+
* the same too-large batch over and over.
|
|
6424
|
+
*/
|
|
6425
|
+
report(deviceId) {
|
|
6426
|
+
const before = this.limit;
|
|
6427
|
+
this.limit = Math.max(this.min, Math.floor(this.limit / 2));
|
|
6428
|
+
this.lastChangeAt = this.now();
|
|
6429
|
+
this.underPressure = true;
|
|
6430
|
+
this.deps.logger?.warn("recorder: spawn pressure — reducing attach concurrency", {
|
|
6431
|
+
tags: { deviceId },
|
|
6432
|
+
meta: {
|
|
6433
|
+
from: before,
|
|
6434
|
+
to: this.limit
|
|
6435
|
+
}
|
|
6436
|
+
});
|
|
6437
|
+
}
|
|
6438
|
+
now() {
|
|
6439
|
+
return this.deps.now?.() ?? Date.now();
|
|
6440
|
+
}
|
|
6441
|
+
};
|
|
6191
6442
|
//#endregion
|
|
6192
6443
|
//#region src/recorder/addon/segment-watcher.ts
|
|
6193
6444
|
/**
|
|
@@ -6573,36 +6824,87 @@ var SegmentWriter = class {
|
|
|
6573
6824
|
if (stderrTail.length > STDERR_TAIL_LINES) stderrTail.shift();
|
|
6574
6825
|
}
|
|
6575
6826
|
});
|
|
6827
|
+
let settled = false;
|
|
6828
|
+
const terminate = (termination) => {
|
|
6829
|
+
if (settled) return;
|
|
6830
|
+
settled = true;
|
|
6831
|
+
this.onTermination(termination, stderrTail);
|
|
6832
|
+
};
|
|
6576
6833
|
proc.on("exit", (code) => {
|
|
6577
|
-
|
|
6578
|
-
|
|
6579
|
-
|
|
6580
|
-
|
|
6581
|
-
|
|
6582
|
-
|
|
6583
|
-
|
|
6584
|
-
|
|
6834
|
+
terminate({
|
|
6835
|
+
reason: "exit",
|
|
6836
|
+
exitCode: typeof code === "number" ? code : null
|
|
6837
|
+
});
|
|
6838
|
+
});
|
|
6839
|
+
proc.on("error", (raw) => {
|
|
6840
|
+
const failure = readSpawnFailure(raw);
|
|
6841
|
+
terminate({
|
|
6842
|
+
reason: "spawn-error",
|
|
6843
|
+
errorCode: failure.code,
|
|
6844
|
+
message: failure.message,
|
|
6845
|
+
permanent: failure.permanent
|
|
6846
|
+
});
|
|
6847
|
+
});
|
|
6848
|
+
}
|
|
6849
|
+
/**
|
|
6850
|
+
* The ONE restart policy, reached from both `exit` and `error`.
|
|
6851
|
+
*
|
|
6852
|
+
* `deps.logger` is the device-scoped child the controller binds
|
|
6853
|
+
* (`logger.withTags({ deviceId })`), so every line below carries
|
|
6854
|
+
* `tags: { deviceId }` — "why is 617 worse than 615" is always asked per
|
|
6855
|
+
* camera, and a writer that stops recording one camera must be greppable by
|
|
6856
|
+
* that camera.
|
|
6857
|
+
*/
|
|
6858
|
+
onTermination(termination, stderrTail) {
|
|
6859
|
+
this.resolveExit?.();
|
|
6860
|
+
this.resolveExit = null;
|
|
6861
|
+
this.proc = null;
|
|
6862
|
+
if (this.stopped) return;
|
|
6863
|
+
const ranMs = Date.now() - this.startedAt;
|
|
6864
|
+
if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
|
|
6865
|
+
if (termination.reason === "spawn-error") {
|
|
6866
|
+
this.deps.logger.warn("SegmentWriter ffmpeg spawn failed", { meta: {
|
|
6867
|
+
outDir: this.cfg.outDir,
|
|
6868
|
+
errorCode: termination.errorCode,
|
|
6869
|
+
error: termination.message,
|
|
6870
|
+
permanent: termination.permanent
|
|
6871
|
+
} });
|
|
6872
|
+
if (termination.permanent) {
|
|
6873
|
+
this.deps.logger.error("SegmentWriter giving up: ffmpeg cannot be executed on this node", { meta: {
|
|
6585
6874
|
outDir: this.cfg.outDir,
|
|
6586
|
-
|
|
6587
|
-
|
|
6875
|
+
errorCode: termination.errorCode,
|
|
6876
|
+
error: termination.message
|
|
6588
6877
|
} });
|
|
6589
6878
|
this.stopped = true;
|
|
6590
6879
|
this.deps.onGaveUp?.();
|
|
6591
6880
|
return;
|
|
6592
6881
|
}
|
|
6593
|
-
this.
|
|
6594
|
-
|
|
6595
|
-
|
|
6882
|
+
this.deps.onResourcePressure?.();
|
|
6883
|
+
}
|
|
6884
|
+
if (this.restarts >= MAX_RESTARTS) {
|
|
6885
|
+
this.deps.logger.warn("SegmentWriter giving up after max restarts", { meta: {
|
|
6596
6886
|
outDir: this.cfg.outDir,
|
|
6597
|
-
|
|
6598
|
-
|
|
6599
|
-
|
|
6887
|
+
reason: termination.reason,
|
|
6888
|
+
code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
|
|
6889
|
+
stderrTail: stderrTail.join(" | ")
|
|
6600
6890
|
} });
|
|
6601
|
-
this.
|
|
6602
|
-
|
|
6603
|
-
|
|
6604
|
-
|
|
6605
|
-
|
|
6891
|
+
this.stopped = true;
|
|
6892
|
+
this.deps.onGaveUp?.();
|
|
6893
|
+
return;
|
|
6894
|
+
}
|
|
6895
|
+
this.restarts++;
|
|
6896
|
+
const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
|
|
6897
|
+
this.deps.logger.info("SegmentWriter restarting", { meta: {
|
|
6898
|
+
outDir: this.cfg.outDir,
|
|
6899
|
+
attempt: this.restarts,
|
|
6900
|
+
delayMs,
|
|
6901
|
+
ranMs,
|
|
6902
|
+
reason: termination.reason
|
|
6903
|
+
} });
|
|
6904
|
+
this.restartTimer = setTimeout(() => {
|
|
6905
|
+
this.restartTimer = null;
|
|
6906
|
+
this.start();
|
|
6907
|
+
}, delayMs);
|
|
6606
6908
|
}
|
|
6607
6909
|
stop() {
|
|
6608
6910
|
this.stopAndWait();
|
|
@@ -6894,8 +7196,20 @@ var RecordingController = class {
|
|
|
6894
7196
|
/** A reversible maintenance lease. Unlike `stopped`, it never changes
|
|
6895
7197
|
* persisted recording intent and `resume()` restarts normal convergence. */
|
|
6896
7198
|
paused = false;
|
|
7199
|
+
/**
|
|
7200
|
+
* The ADAPTIVE attach bound. A transient `spawn ffmpeg` failure (EAGAIN) is
|
|
7201
|
+
* the host saying it cannot fork right now, and the answer is to attach
|
|
7202
|
+
* FEWER cameras at once — not to serialise them, which `e2fa17a2b` removed
|
|
7203
|
+
* for good reason. Every bounded-parallel attach in this file reads it.
|
|
7204
|
+
*/
|
|
7205
|
+
spawnPressure;
|
|
6897
7206
|
constructor(deps) {
|
|
6898
7207
|
this.deps = deps;
|
|
7208
|
+
this.spawnPressure = new SpawnPressureGovernor({
|
|
7209
|
+
maxConcurrency: 8,
|
|
7210
|
+
now: () => this.now(),
|
|
7211
|
+
logger: deps.logger
|
|
7212
|
+
});
|
|
6899
7213
|
this.wakeups = new DeviceWakeups({
|
|
6900
7214
|
logger: deps.logger,
|
|
6901
7215
|
now: () => this.now(),
|
|
@@ -6963,6 +7277,7 @@ var RecordingController = class {
|
|
|
6963
7277
|
}
|
|
6964
7278
|
this.restore = new ReadinessRestore({
|
|
6965
7279
|
subscribeReady: (handler) => this.deps.onBrokerReady(handler),
|
|
7280
|
+
concurrency: () => this.spawnPressure.concurrency,
|
|
6966
7281
|
attempt: async (deviceId) => {
|
|
6967
7282
|
await this.evaluateDevice(deviceId);
|
|
6968
7283
|
},
|
|
@@ -7043,7 +7358,7 @@ var RecordingController = class {
|
|
|
7043
7358
|
*/
|
|
7044
7359
|
async reconcile() {
|
|
7045
7360
|
if (this.stopped) return;
|
|
7046
|
-
await runBounded(await this.trackedDeviceIds(),
|
|
7361
|
+
await runBounded(await this.trackedDeviceIds(), this.spawnPressure.concurrency, async (id) => {
|
|
7047
7362
|
try {
|
|
7048
7363
|
await this.evaluateDevice(id);
|
|
7049
7364
|
} catch (err) {
|
|
@@ -7402,7 +7717,8 @@ var RecordingController = class {
|
|
|
7402
7717
|
logger: deviceLog,
|
|
7403
7718
|
onGaveUp: () => {
|
|
7404
7719
|
this.recoverWriterGaveUp(deviceId);
|
|
7405
|
-
}
|
|
7720
|
+
},
|
|
7721
|
+
onResourcePressure: () => this.spawnPressure.report(deviceId)
|
|
7406
7722
|
});
|
|
7407
7723
|
const writerStartedMs = this.now();
|
|
7408
7724
|
writer.start();
|
|
@@ -8877,6 +9193,23 @@ var PLAYBACK_PREFIX = "playback";
|
|
|
8877
9193
|
/** Data-plane prefix for on-demand single stills →
|
|
8878
9194
|
* `/addon/recorder/still/<deviceId>/<epochSec>`. */
|
|
8879
9195
|
var STILL_PREFIX = "still";
|
|
9196
|
+
/**
|
|
9197
|
+
* Bound on pending hour-ledger writes (one per finalized segment, an RPC to
|
|
9198
|
+
* the hub store). Steady state completes in milliseconds and holds ~0–2; under
|
|
9199
|
+
* a store stall the fleet-wide finalize rate (~3.4/s) fills 256 in ~75s of
|
|
9200
|
+
* grace before refusals begin. Each pending write holds one hour-row copy
|
|
9201
|
+
* (≤360 paths ≈ ~25KB), so the lane's worst-case heap is single-digit MB —
|
|
9202
|
+
* a dropped write is one silent re-seed from the archive walk (D148).
|
|
9203
|
+
*/
|
|
9204
|
+
var LEDGER_GATE_LIMIT = 256;
|
|
9205
|
+
/**
|
|
9206
|
+
* Bound on pending mfra tail reads (one per finalized RECENT segment; each is
|
|
9207
|
+
* an open + two reads on the same 4-thread libuv pool the live writers use,
|
|
9208
|
+
* holding read buffers while pending). 64 bounds both the held memory and the
|
|
9209
|
+
* pool contention; a dropped capture costs that segment's client one extra
|
|
9210
|
+
* round trip to the file's tail on the first seek.
|
|
9211
|
+
*/
|
|
9212
|
+
var MFRA_GATE_LIMIT = 64;
|
|
8880
9213
|
/** Data-plane prefix for export MP4 downloads → `/addon/recorder/exports/<id>.mp4`. */
|
|
8881
9214
|
var EXPORTS_PREFIX = "exports";
|
|
8882
9215
|
/** How often the export janitor sweeps expired exports. */
|
|
@@ -8953,6 +9286,25 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
8953
9286
|
/** Sync-sample tables captured at finalize, served by the directory so a
|
|
8954
9287
|
* playback seek needs no mfra tail fetch. RAM-only, TTL-bounded. */
|
|
8955
9288
|
mfraTables = new MfraTableStore();
|
|
9289
|
+
/** Bound on the per-finalize fire-and-forget lanes (item 3, 2026-08-18): a
|
|
9290
|
+
* disk/store stall must throttle this work, never balloon live heap — the
|
|
9291
|
+
* unbounded pending set is what fed the two `Ineffective mark-compacts`
|
|
9292
|
+
* fatals. Saturation is reported loudly but throttled; every individual
|
|
9293
|
+
* drop is logged with its deviceId at the call site. */
|
|
9294
|
+
ledgerWriteGate = new PendingGate({
|
|
9295
|
+
label: "hour-ledger",
|
|
9296
|
+
limit: LEDGER_GATE_LIMIT,
|
|
9297
|
+
onSaturated: (report) => {
|
|
9298
|
+
this.reportGateSaturated(report);
|
|
9299
|
+
}
|
|
9300
|
+
});
|
|
9301
|
+
mfraCaptureGate = new PendingGate({
|
|
9302
|
+
label: "mfra-capture",
|
|
9303
|
+
limit: MFRA_GATE_LIMIT,
|
|
9304
|
+
onSaturated: (report) => {
|
|
9305
|
+
this.reportGateSaturated(report);
|
|
9306
|
+
}
|
|
9307
|
+
});
|
|
8956
9308
|
/** Continuous + events band write-path controller (ffmpeg writers + watchers). */
|
|
8957
9309
|
controller = null;
|
|
8958
9310
|
/** Per-device motion/audio markers for playback (B3 events feed the controller). */
|
|
@@ -8990,6 +9342,20 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
8990
9342
|
constructor() {
|
|
8991
9343
|
super({ ...DEFAULT_CONFIG });
|
|
8992
9344
|
}
|
|
9345
|
+
/**
|
|
9346
|
+
* One throttled line per saturated lane per report interval. Lane-level (no
|
|
9347
|
+
* deviceId — the lane aggregates every camera); the per-drop debug lines at
|
|
9348
|
+
* the call sites carry the tag. WARN because a saturated lane means the
|
|
9349
|
+
* store or the disk is stalling under live finalize load — the line that
|
|
9350
|
+
* explains the missing ledger hours and mfra misses an operator will see.
|
|
9351
|
+
*/
|
|
9352
|
+
reportGateSaturated(report) {
|
|
9353
|
+
this.ctx.logger.warn("recorder: fire-and-forget lane saturated — dropping over the bound", { meta: {
|
|
9354
|
+
lane: report.label,
|
|
9355
|
+
dropped: report.dropped,
|
|
9356
|
+
pending: report.pending
|
|
9357
|
+
} });
|
|
9358
|
+
}
|
|
8993
9359
|
async onInitialize() {
|
|
8994
9360
|
const raw = this.ctx.kernel.localNodeId ?? this.ctx.id;
|
|
8995
9361
|
this.nodeId = raw.includes("/") ? raw.split("/")[0] : raw;
|
|
@@ -9053,18 +9419,28 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
9053
9419
|
this.segmentHours?.dropSegments(rows);
|
|
9054
9420
|
},
|
|
9055
9421
|
onIndexed: (row, absPath) => {
|
|
9056
|
-
this.segmentHours
|
|
9422
|
+
const ledger = this.segmentHours;
|
|
9423
|
+
if (ledger && !this.ledgerWriteGate.run(() => ledger.recordSegment(row))) this.ctx.logger.debug("recorder: hour-ledger write dropped — lane saturated; the archive walk reconciles this hour", {
|
|
9424
|
+
tags: { deviceId: row.deviceId },
|
|
9425
|
+
meta: { path: row.path }
|
|
9426
|
+
});
|
|
9057
9427
|
if (!isMfraTableServable(row.startMs, Date.now())) return;
|
|
9058
|
-
|
|
9059
|
-
|
|
9060
|
-
|
|
9061
|
-
|
|
9062
|
-
|
|
9063
|
-
|
|
9064
|
-
|
|
9065
|
-
|
|
9066
|
-
|
|
9067
|
-
|
|
9428
|
+
if (!this.mfraCaptureGate.run(async () => {
|
|
9429
|
+
try {
|
|
9430
|
+
const table = await readMfraTable(absPath, row.bytes);
|
|
9431
|
+
this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
|
|
9432
|
+
} catch (err) {
|
|
9433
|
+
this.ctx.logger.debug("mfra capture failed", {
|
|
9434
|
+
tags: { deviceId: row.deviceId },
|
|
9435
|
+
meta: {
|
|
9436
|
+
path: row.path,
|
|
9437
|
+
error: errMsg(err)
|
|
9438
|
+
}
|
|
9439
|
+
});
|
|
9440
|
+
}
|
|
9441
|
+
})) this.ctx.logger.debug("recorder: mfra capture dropped — lane saturated; first seek pays one tail fetch", {
|
|
9442
|
+
tags: { deviceId: row.deviceId },
|
|
9443
|
+
meta: { path: row.path }
|
|
9068
9444
|
});
|
|
9069
9445
|
},
|
|
9070
9446
|
removeDirIfEmpty: async (locationId, relDir) => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_worker_protocol = require("../worker-protocol-
|
|
3
|
-
const require_lazy_sharp = require("../lazy-sharp-
|
|
2
|
+
const require_worker_protocol = require("../worker-protocol-DqrXmX0g.js");
|
|
3
|
+
const require_lazy_sharp = require("../lazy-sharp-BVqBydNU.js");
|
|
4
4
|
//#region src/session-decode/color-conversion.ts
|
|
5
5
|
/**
|
|
6
6
|
* Explicit YUV→RGB colorspace/range resolution for the session-decode worker.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { i as formatNativeLeaseKnobs, n as isWorkerRequest, o as resolveNativeLeaseKnobs, r as logLevelForLine } from "../worker-protocol-
|
|
1
|
+
import { i as formatNativeLeaseKnobs, n as isWorkerRequest, o as resolveNativeLeaseKnobs, r as logLevelForLine } from "../worker-protocol-DTe7Ntat.mjs";
|
|
2
2
|
import { n as setSharpWarnSink, r as hostExternalEntryUrls, t as getSharp } from "../lazy-sharp-6oymT_yf.mjs";
|
|
3
3
|
//#region src/session-decode/color-conversion.ts
|
|
4
4
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { a as e, c as t, d as n, f as r, i, l as a, n as o, o as s, p as c, r as l, s as u, t as d, u as f } from "./_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DsIi4G5Q.mjs";
|
|
2
2
|
import { a as p, i as m, n as h, o as g, r as _, t as v } from "./_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.js-C9j-2lBe.mjs";
|
|
3
3
|
import { n as y, r as b, t as x } from "./_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.js-XO0-Pyu6.mjs";
|
|
4
|
-
import { t as S } from "./_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-
|
|
4
|
+
import { t as S } from "./_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-D2WOTeYi.mjs";
|
|
5
5
|
import { n as C, t as w } from "./_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.js-BO7TIbJV.mjs";
|
|
6
6
|
//#region ../../node_modules/lucide-react/dist/esm/shared/src/utils.js
|
|
7
7
|
var T = (e) => e.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase(), E = (e) => e.replace(/^([A-Z])|[\s-_]+(\w)/g, (e, t, n) => n ? n.toUpperCase() : t.toLowerCase()), D = (e) => {
|