@trillboards/edge-platform-rtsp 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -0
- package/dist/RTSPAudioProvider.d.ts +9 -11
- package/dist/RTSPAudioProvider.d.ts.map +1 -1
- package/dist/RTSPAudioProvider.js +10 -67
- package/dist/RTSPAudioProvider.js.map +1 -1
- package/dist/RTSPCameraProvider.d.ts +11 -0
- package/dist/RTSPCameraProvider.d.ts.map +1 -1
- package/dist/RTSPCameraProvider.js +13 -3
- package/dist/RTSPCameraProvider.js.map +1 -1
- package/dist/RTSPStreamCameraProvider.d.ts +615 -0
- package/dist/RTSPStreamCameraProvider.d.ts.map +1 -0
- package/dist/RTSPStreamCameraProvider.js +1357 -0
- package/dist/RTSPStreamCameraProvider.js.map +1 -0
- package/dist/ffmpegStderr.d.ts +53 -0
- package/dist/ffmpegStderr.d.ts.map +1 -0
- package/dist/ffmpegStderr.js +80 -0
- package/dist/ffmpegStderr.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +20 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,1357 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.RTSPStreamCameraProvider = exports.streamSessionStatsFor = exports.__resetStreamAdmissionForTests = exports.FrameRing = exports.MjpegFrameSplitter = exports.jpegFrameEnd = exports.JPEG_MALFORMED = exports.JPEG_NEED_MORE = exports.DEFAULT_MAX_STREAM_SESSIONS_PER_SOURCE = exports.RtspStreamStaleError = exports.RtspStreamAdmissionError = void 0;
|
|
27
|
+
const child_process_1 = require("child_process");
|
|
28
|
+
const fs = __importStar(require("fs"));
|
|
29
|
+
const os = __importStar(require("os"));
|
|
30
|
+
const path = __importStar(require("path"));
|
|
31
|
+
const RTSPCameraProvider_1 = require("./RTSPCameraProvider");
|
|
32
|
+
const ffmpegStderr_1 = require("./ffmpegStderr");
|
|
33
|
+
/**
|
|
34
|
+
* RTSPStreamCameraProvider — video frames off an RTSP camera/NVR channel as a
|
|
35
|
+
* LONG-LIVED ffmpeg session feeding a bounded in-memory ring, so that a LOOK
|
|
36
|
+
* becomes a memory read instead of an RTSP handshake.
|
|
37
|
+
*
|
|
38
|
+
* ── THE PROBLEM THIS EXISTS TO REMOVE, AND IT IS ONE MEASUREMENT ───────────
|
|
39
|
+
*
|
|
40
|
+
* `RTSPCameraProvider.buildCaptureArgs` spawns ONE ffmpeg per grab with
|
|
41
|
+
* `-frames:v 1`, so every look pays a process start, a full RTSP
|
|
42
|
+
* DESCRIBE/SETUP/PLAY handshake, and a wait for the camera's next keyframe
|
|
43
|
+
* before anything can be decoded. Measured through that provider against
|
|
44
|
+
* mediamtx re-streaming a real CCTV clip at the documented degraded profile
|
|
45
|
+
* (640x360 / 399 kbps): p50 **1,986-2,025 ms** per grab at a 1-2 s keyframe
|
|
46
|
+
* interval, 4,005 ms at 4 s. The control — the SAME ffmpeg args against a local
|
|
47
|
+
* file, so spawn + decode + scale + pad + JPEG encode with no stream — is
|
|
48
|
+
* **20-30 ms**. So ~99% of a look is session SETUP, not imaging.
|
|
49
|
+
*
|
|
50
|
+
* That is the whole catch-rate problem. An event is catchable only if a look
|
|
51
|
+
* lands inside it, so catch rate is bounded by `E[min(d/T,1)]` for a look
|
|
52
|
+
* cadence `T`; the Brief group (median ~2.4 s, heavy sub-second tail) needs
|
|
53
|
+
* **0.75 s** to clear the 90% target, which is 2.7x BELOW what one grab
|
|
54
|
+
* physically costs. `connectorCadence.ts` says so in as many words: "Closing
|
|
55
|
+
* that gap needs a persistent RTSP session — one long-lived ffmpeg per feed
|
|
56
|
+
* instead of one per grab — which is an architecture change, not a number."
|
|
57
|
+
* This is that change.
|
|
58
|
+
*
|
|
59
|
+
* Measured through THIS provider against the same rig: the handshake is paid
|
|
60
|
+
* ONCE at open, and thereafter frames arrive every ~250 ms (`streamFps` 4) and
|
|
61
|
+
* `getFrame()` returns in **microseconds** because it reads the newest frame
|
|
62
|
+
* out of memory. See `scripts/measure-persistent-capture.mjs` for the driver
|
|
63
|
+
* and the committed numbers.
|
|
64
|
+
*
|
|
65
|
+
* ── PROCESS COUNT GOES DOWN, NOT UP. SAY IT OUT LOUD ───────────────────────
|
|
66
|
+
*
|
|
67
|
+
* A reviewer reads "persistent session" as "more load on the recorder". It is
|
|
68
|
+
* the opposite, and the incident this package is written around is the reason.
|
|
69
|
+
* A 32-channel NVR once accumulated **28 simultaneous ffmpeg children** because
|
|
70
|
+
* one-shot pulls started outliving the cadence, and the recorder answered with
|
|
71
|
+
* rotating timeouts, CSeq mismatches and cameras flapping into "needs
|
|
72
|
+
* attention" — which is why `CaptureAdmission` caps at 2 concurrent pulls per
|
|
73
|
+
* physical recorder
|
|
74
|
+
* (.claude/lessons/2026-08-17-rtsp-session-admission-and-frame-integrity.md).
|
|
75
|
+
*
|
|
76
|
+
* Compare the two designs on the SAME 16-feed recorder:
|
|
77
|
+
*
|
|
78
|
+
* per-grab (today) 16 feeds x 1 look / 16 s = 1.0 RTSP session SETUP per
|
|
79
|
+
* second, forever — 3,600 DESCRIBE/SETUP/PLAY/TEARDOWN
|
|
80
|
+
* cycles per hour and 3,600 process spawns per hour, with
|
|
81
|
+
* 2 concurrent at any instant and an unbounded pile-up
|
|
82
|
+
* under load (the incident).
|
|
83
|
+
*
|
|
84
|
+
* persistent 16 session setups TOTAL, then nothing. 16 processes,
|
|
85
|
+
* bounded by construction, and 16 concurrent substream
|
|
86
|
+
* readers — which is exactly what the recorder's OWN
|
|
87
|
+
* client app opens for a 16-up grid view, i.e. its designed
|
|
88
|
+
* workload.
|
|
89
|
+
*
|
|
90
|
+
* So the churn goes to zero and the process count per hour falls by ~225x. The
|
|
91
|
+
* one number that goes UP is steady-state concurrent sessions (2 -> N), and
|
|
92
|
+
* that is why the persistent lane is bounded per recorder and degrades rather
|
|
93
|
+
* than fails — see ADMISSION below.
|
|
94
|
+
*
|
|
95
|
+
* That bound used to be a guess. It was validated against mediamtx, which
|
|
96
|
+
* serves any number of readers and therefore cannot refuse anything, so the
|
|
97
|
+
* only property that actually matters here was untestable on the rig. Measured
|
|
98
|
+
* 2026-08-19 against two real implementations, the ceiling ranges from **4**
|
|
99
|
+
* (Happytime ONVIF/RTSP Server, a fixed pool with no config key) to **>=160**
|
|
100
|
+
* (Nx Media Server 6.1.2.42921, never refused). It is a property of the
|
|
101
|
+
* recorder, so the lane LEARNS it — see `DEFAULT_MAX_STREAM_SESSIONS_PER_SOURCE`
|
|
102
|
+
* and `StreamSessionAdmission.learnCeiling`.
|
|
103
|
+
*
|
|
104
|
+
* ── ADMISSION: A SECOND LANE, AND IT REFUSES RATHER THAN QUEUES ────────────
|
|
105
|
+
*
|
|
106
|
+
* A long-lived session cannot be admitted from `CaptureAdmission`: a permanent
|
|
107
|
+
* holder of a 2-slot pool is a 50% permanent tax on frame capture for that
|
|
108
|
+
* recorder, and two of them deadlock it entirely. So persistent sessions are
|
|
109
|
+
* admitted from `streamAdmissionBySource`, keyed by the SAME
|
|
110
|
+
* `captureSourceKey(url)` — host:port, credentials and channel path
|
|
111
|
+
* deliberately excluded so 32 channels of one NVR share one budget — counted in
|
|
112
|
+
* their OWN pool with its own cap. Exactly the split `RTSPAudioProvider` uses,
|
|
113
|
+
* for exactly the same reason.
|
|
114
|
+
*
|
|
115
|
+
* The lane never queues. A queued video session would be admitted minutes later
|
|
116
|
+
* against a scene nobody is asking about any more, and a feed waiting on a slot
|
|
117
|
+
* is a feed that is not sensing.
|
|
118
|
+
*
|
|
119
|
+
* ── DEGRADE, NEVER FAIL. THIS IS THE PROPERTY THAT LETS THIS SHIP ──────────
|
|
120
|
+
*
|
|
121
|
+
* When the persistent session cannot be had — the recorder's lane is full, or
|
|
122
|
+
* the recorder refuses the session, or ffmpeg cannot hold it open — this
|
|
123
|
+
* provider does NOT throw. It falls back to an internal `RTSPCameraProvider`
|
|
124
|
+
* and runs the per-grab path, which is precisely what the connector does today.
|
|
125
|
+
* So no camera that works today can be broken by this change; the worst case is
|
|
126
|
+
* that it is no faster than it was.
|
|
127
|
+
*
|
|
128
|
+
* The degrade is a REPORTED STATE, never a hidden branch: `getStats().mode` is
|
|
129
|
+
* `'persistent' | 'per_grab'`, `degradedReason` says which of the three causes
|
|
130
|
+
* fired, and it is logged at warn. A silent downgrade would be the same defect
|
|
131
|
+
* as a silent drop — it would move the problem somewhere nobody looks.
|
|
132
|
+
*
|
|
133
|
+
* ── A DEAD STREAM MUST BE LOUD, AND A PERSISTENT SESSION MAKES THAT HARDER ─
|
|
134
|
+
*
|
|
135
|
+
* The connector's recurring defect is a signal that cannot report its own
|
|
136
|
+
* failure: a recorder that accepts a connection and then stops sending frames
|
|
137
|
+
* is ~50 minutes of beacon silence rendered as "connector not running". A
|
|
138
|
+
* persistent session makes that MORE likely, because the process is still
|
|
139
|
+
* alive and the socket is still open. Three things make it loud instead:
|
|
140
|
+
*
|
|
141
|
+
* 1. `getFrame()` NEVER returns a frame it has already returned. It waits up
|
|
142
|
+
* to `frameWaitMs` for a genuinely new one and then throws
|
|
143
|
+
* `RtspStreamStaleError` carrying the age. Handing back a stale frame is
|
|
144
|
+
* the failure mode, not the fallback: it would keep stamping the camera's
|
|
145
|
+
* `lastFrameAtMs`, and a frozen camera would report as healthy forever.
|
|
146
|
+
* 2. The error message is worded so `classifyRtspFailureClass` reads it as
|
|
147
|
+
* `no_frames` — the recorder answered, no video arrived — and NOT as a
|
|
148
|
+
* reachability class, which would wrongly implicate the LAN grant. The
|
|
149
|
+
* ffmpeg stderr summary is deliberately kept OUT of that message and put
|
|
150
|
+
* in `getStats().lastErrorMessage`, so the classification is deterministic
|
|
151
|
+
* instead of depending on which warning ffmpeg happened to print.
|
|
152
|
+
* 3. `getStats()` DERIVES `frameAgeMs`, `running`, `mode` and the counters
|
|
153
|
+
* live on every call. There is no cached capability boolean. On the
|
|
154
|
+
* Android fleet `cap_audio_classification` is false FLEET-WIDE, including
|
|
155
|
+
* on devices shipping live classifications, because a flag was latched at
|
|
156
|
+
* boot from a value that was still "probing" and nothing ever re-wrote it
|
|
157
|
+
* (docs/runbooks/audio-sensing-substrate.md).
|
|
158
|
+
*
|
|
159
|
+
* A CLEAN EXIT IS A FAULT HERE. `code === 0` means success on the one-shot
|
|
160
|
+
* path; on a persistent session it means the stream ended, which is never what
|
|
161
|
+
* we asked for. Measured against the rig: killing the publisher made ffmpeg
|
|
162
|
+
* exit 0 within 26 ms, and freezing it (session held open, no RTP) made ffmpeg
|
|
163
|
+
* exit 0 after ~10 s with "Failed reading RTSP data: End of file". Both are
|
|
164
|
+
* restarts here, both are counted, and neither is ever silent.
|
|
165
|
+
*
|
|
166
|
+
* ── BOUNDED MEMORY, ENFORCED ON ADMISSION ─────────────────────────────────
|
|
167
|
+
*
|
|
168
|
+
* `VlmCandidateBuffer` gets this right and this copies its discipline: the
|
|
169
|
+
* ceiling is checked on the way IN, in exact bytes, and every eviction is
|
|
170
|
+
* COUNTED. A ring that grows under back-pressure is an OOM on a 7 GB box.
|
|
171
|
+
*
|
|
172
|
+
* There are TWO bounds because there are two places bytes accumulate, and the
|
|
173
|
+
* second one is the easy one to miss: the ring holds complete frames, but the
|
|
174
|
+
* SPLITTER holds a partially-received one. A stream that emits an SOI and then
|
|
175
|
+
* never an EOI — a corrupt feed, a mid-frame protocol change — would grow that
|
|
176
|
+
* accumulator without limit. `maxFrameBytes` bounds it, and an overflow resyncs
|
|
177
|
+
* and is counted as `oversizeFrames`.
|
|
178
|
+
*
|
|
179
|
+
* ── JPEG ONLY, ON PURPOSE ─────────────────────────────────────────────────
|
|
180
|
+
*
|
|
181
|
+
* `RTSPCameraProvider` supports `outputFormat: 'rgb24'` for the ONNX face
|
|
182
|
+
* zoo. A persistent rgb24 stream is 640x480x3 = 921,600 bytes per frame, i.e.
|
|
183
|
+
* 3.7 MB/s per camera at 4 fps and ~59 MB/s across 16 feeds, through pipes and
|
|
184
|
+
* into a ring. The same stream as MJPEG is ~0.12 MB/s per camera (measured:
|
|
185
|
+
* 29.6 KB/frame at q:v 11 on the degraded profile). So this provider is JPEG
|
|
186
|
+
* only and the rgb24 zoo keeps the one-shot path, which is also the right split
|
|
187
|
+
* by product: the zoo is the DOOH audience rail, and the look-cadence problem
|
|
188
|
+
* is the CCTV sensing rail.
|
|
189
|
+
*
|
|
190
|
+
* ── RATE LIMITING WITHOUT FABRICATION ─────────────────────────────────────
|
|
191
|
+
*
|
|
192
|
+
* The output is rate-limited with the `fps` filter rather than by an output
|
|
193
|
+
* `-r`, and the difference matters for constraint 3. The `fps` filter is driven
|
|
194
|
+
* BY ARRIVING INPUT FRAMES: when no frames arrive it emits nothing. Verified
|
|
195
|
+
* against the rig — freezing the source produced zero further output, so a
|
|
196
|
+
* frozen camera cannot manufacture "fresh" frames and defeat the staleness
|
|
197
|
+
* signal. Decode cost is unaffected by the filter (ffmpeg must decode the
|
|
198
|
+
* stream to produce any frame at all); what `streamFps` bounds is scale, pad,
|
|
199
|
+
* JPEG encode and pipe traffic.
|
|
200
|
+
*
|
|
201
|
+
* ── SECRET HYGIENE, INHERITED ─────────────────────────────────────────────
|
|
202
|
+
*
|
|
203
|
+
* RTSP URLs carry passwords. Exactly as the other two providers do, the URL is
|
|
204
|
+
* written to a 0600 ffconcat file and only the temp path reaches argv, so a
|
|
205
|
+
* password never appears in `ps` output; stderr is scrubbed through
|
|
206
|
+
* `sanitizeStreamUrlForLog` before it can reach a log line.
|
|
207
|
+
*/
|
|
208
|
+
/** The recorder's persistent-session lane is full. Degrade, never queue. */
|
|
209
|
+
class RtspStreamAdmissionError extends Error {
|
|
210
|
+
constructor(sourceKey, limit) {
|
|
211
|
+
super(`RTSP stream session refused: recorder ${sourceKey} already runs ${limit} persistent session(s).`);
|
|
212
|
+
this.code = 'RTSP_STREAM_SESSION_LIMIT';
|
|
213
|
+
this.name = 'RtspStreamAdmissionError';
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
exports.RtspStreamAdmissionError = RtspStreamAdmissionError;
|
|
217
|
+
/**
|
|
218
|
+
* No NEW frame arrived within the deadline.
|
|
219
|
+
*
|
|
220
|
+
* The wording is load bearing. `classifyRtspFailureClass` tests, in order,
|
|
221
|
+
* auth -> nvr-busy -> unreachable -> timeout -> decode -> no-frames, so this
|
|
222
|
+
* message must avoid every earlier class's vocabulary ("timeout", "timed out",
|
|
223
|
+
* "refused", "corrupt", "eof") and must contain "no frame". A stale persistent
|
|
224
|
+
* session is not a reachability failure — the recorder answered and the socket
|
|
225
|
+
* is up — and misclassifying it as one would send the health surface to
|
|
226
|
+
* `local_network_denied` and blame the operator's macOS LAN grant for a camera
|
|
227
|
+
* that is plainly reachable.
|
|
228
|
+
*/
|
|
229
|
+
class RtspStreamStaleError extends Error {
|
|
230
|
+
constructor(url, ageMs, restarts) {
|
|
231
|
+
super(`RTSP stream produced no frame for ${Math.round(ageMs)}ms ` +
|
|
232
|
+
`(${restarts} session restart(s)): ${(0, RTSPCameraProvider_1.sanitizeStreamUrlForLog)(url)}`);
|
|
233
|
+
this.code = 'RTSP_STREAM_STALE';
|
|
234
|
+
this.name = 'RtspStreamStaleError';
|
|
235
|
+
this.ageMs = ageMs;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
exports.RtspStreamStaleError = RtspStreamStaleError;
|
|
239
|
+
const DEFAULT_STREAM_FPS = 4;
|
|
240
|
+
const DEFAULT_TIMEOUT_US = 12000000;
|
|
241
|
+
/**
|
|
242
|
+
* Recorder slots the persistent lane leaves free so the PER-GRAB fallback can
|
|
243
|
+
* still pull. Not a new number: it is the transient lane's own concurrency
|
|
244
|
+
* bound, imported so the two lanes cannot disagree.
|
|
245
|
+
*/
|
|
246
|
+
const STREAM_SESSION_RESERVE_FOR_PER_GRAB = RTSPCameraProvider_1.DEFAULT_MAX_CONCURRENT_CAPTURES_PER_SOURCE;
|
|
247
|
+
/**
|
|
248
|
+
* Failures that name a cause belonging to ONE FEED, and therefore say nothing
|
|
249
|
+
* about how many sessions the recorder can hold.
|
|
250
|
+
*
|
|
251
|
+
* ── WHY THIS IS A NEGATIVE TEST AND NOT A POSITIVE ONE ────────────────────
|
|
252
|
+
*
|
|
253
|
+
* There is no positive on-the-wire signal for "this recorder is full". Measured
|
|
254
|
+
* 2026-08-20 against Happytime at its ceiling: it does not answer RTSP 453 or
|
|
255
|
+
* 455, it CLOSES THE TCP CONNECTION with no status line at all, in 0-1 ms — a
|
|
256
|
+
* bare hang-up that is byte-for-byte what a dead stream looks like. So a rule
|
|
257
|
+
* of the form "learn only from a recognised capacity refusal" would learn from
|
|
258
|
+
* nothing at all on the one recorder we can prove has a capacity.
|
|
259
|
+
*
|
|
260
|
+
* What CAN be recognised is the opposite: a recorder that answers with a
|
|
261
|
+
* per-feed diagnosis has told us the fault is that feed's. Bad credentials and
|
|
262
|
+
* a bad channel path both do this, and both are exactly the misconfiguration
|
|
263
|
+
* that must never reach a SHARED ceiling — one mistyped camera would otherwise
|
|
264
|
+
* demote every healthy feed on the same recorder to per-grab for the life of
|
|
265
|
+
* the process, because the learned ceiling only ever moves down.
|
|
266
|
+
*
|
|
267
|
+
* Deliberately mirrors `AUTH_FAILURE_RE` and the not-found half of
|
|
268
|
+
* `UNREACHABLE_FAILURE_RE` in `CloudSensingManager.classifyRtspFailureClass`.
|
|
269
|
+
* Restated rather than imported ON PURPOSE: `edge-sdk` depends on this package,
|
|
270
|
+
* so importing the classifier here would be a dependency cycle. Kept in the
|
|
271
|
+
* same vocabulary so the two cannot drift into disagreeing about what an auth
|
|
272
|
+
* failure looks like.
|
|
273
|
+
*/
|
|
274
|
+
const PER_FEED_FAILURE_RE = /(\b(?:401|403|404)\b|unauthori[sz]ed|forbidden|authentication failed|auth failed|invalid (user|username|password)|wrong password|bad credentials|access denied|not found)/i;
|
|
275
|
+
/**
|
|
276
|
+
* Our POLICY ceiling on persistent sessions to one recorder — never a claim
|
|
277
|
+
* about what the recorder can hold.
|
|
278
|
+
*
|
|
279
|
+
* ── THIS USED TO BE A STATED JUDGEMENT AND IT WAS WRONG IN BOTH DIRECTIONS ─
|
|
280
|
+
*
|
|
281
|
+
* It read: "A STATED JUDGEMENT, not a measurement… mediamtx serves any number
|
|
282
|
+
* of readers, so the rig cannot find a real NVR's ceiling." That was honest
|
|
283
|
+
* about its own ignorance and still shipped a number into the one place the
|
|
284
|
+
* ignorance mattered. Measured 2026-08-19 against two real implementations
|
|
285
|
+
* with `scripts/measure-rtsp-session-ceiling.mjs` (a raw RTSP client, so the
|
|
286
|
+
* refusal is graded by which request was refused and how, not by an ffmpeg
|
|
287
|
+
* exit code):
|
|
288
|
+
*
|
|
289
|
+
* Happytime ONVIF/RTSP Server V8.1 4 sessions, then the server
|
|
290
|
+
* CLOSES THE TCP CONNECTION with
|
|
291
|
+
* no RTSP status at all (0-1 ms).
|
|
292
|
+
* Its own log: "rtsp_listen_rx,
|
|
293
|
+
* rua_get_idle_rua return NULL".
|
|
294
|
+
* A fixed pool, with no config key.
|
|
295
|
+
* Nx Media Server 6.1.2.42921 >=160 sessions, never refused.
|
|
296
|
+
* mediamtx unbounded (the original validation target,
|
|
297
|
+
* which is why none of this was
|
|
298
|
+
* visible).
|
|
299
|
+
*
|
|
300
|
+
* 16 is 4x ABOVE the first and 10x BELOW the second, so no constant is the
|
|
301
|
+
* answer: the ceiling is a property of the recorder. Above it the damage is not
|
|
302
|
+
* merely "some feeds stay on the slow path" — measured on Happytime with the
|
|
303
|
+
* cap at 4, the fifth feed degraded to per-grab exactly as designed and then
|
|
304
|
+
* produced NO FRAMES AT ALL, because the recorder was full of our own
|
|
305
|
+
* persistent sessions and the fallback needs a session too.
|
|
306
|
+
*
|
|
307
|
+
* So the real limit is LEARNED per recorder — see
|
|
308
|
+
* `StreamSessionAdmission.learnCeiling`, which records what the recorder did
|
|
309
|
+
* the first time it refused and sheds our own over-quota sessions down to it,
|
|
310
|
+
* keeping `STREAM_SESSION_RESERVE_FOR_PER_GRAB` slots free for the fallback.
|
|
311
|
+
* This constant is only where the search STARTS, and the two directions are not
|
|
312
|
+
* symmetric: starting too low costs nothing but the latency win on some feeds
|
|
313
|
+
* (they run the per-grab path the connector ran before this architecture),
|
|
314
|
+
* while starting too high costs a round trip per feed before the lane learns.
|
|
315
|
+
* 16 is kept because it is above every deployment this connector is sized for
|
|
316
|
+
* (`connectorCadence.ts` puts a host at seven or fewer feeds) and Nx proves a
|
|
317
|
+
* real NVR is nowhere near bothered by it.
|
|
318
|
+
*/
|
|
319
|
+
exports.DEFAULT_MAX_STREAM_SESSIONS_PER_SOURCE = 16;
|
|
320
|
+
/**
|
|
321
|
+
* Threads ffmpeg may use for this session.
|
|
322
|
+
*
|
|
323
|
+
* ONE, and it is faster as well as smaller — which is the opposite of the
|
|
324
|
+
* instinct. Measured against the rig (640x360 / 399 kbps substream, 25 s,
|
|
325
|
+
* one feed), default threading versus `-threads 1`:
|
|
326
|
+
*
|
|
327
|
+
* default RSS 89.6 MB first frame 3,156 ms 88 frames
|
|
328
|
+
* threads 1 RSS 26.8 MB first frame 1,690 ms 97 frames
|
|
329
|
+
*
|
|
330
|
+
* RSS falls 3.3x and the first frame arrives 1.5 s sooner, because frame-level
|
|
331
|
+
* threading buys parallelism by holding a PIPELINE of frames in flight — which
|
|
332
|
+
* costs memory per thread and adds a multi-frame delay before anything comes
|
|
333
|
+
* out. Neither is worth having here: a 640x360 substream decodes far faster
|
|
334
|
+
* than real time on one thread, and latency is the entire product of this file.
|
|
335
|
+
*
|
|
336
|
+
* It is also the difference between a 16-feed connector costing ~1.4 GB of
|
|
337
|
+
* ffmpeg RSS and costing ~0.4 GB, on a box the runbooks size at 7 GB.
|
|
338
|
+
*
|
|
339
|
+
* A main-stream (1080p+) feed is a different measurement and this is a config
|
|
340
|
+
* knob for that reason; the connector runs substreams.
|
|
341
|
+
*/
|
|
342
|
+
const DEFAULT_DECODER_THREADS = 1;
|
|
343
|
+
/** Complete frames held. The newest is what a look reads; the rest are slack. */
|
|
344
|
+
const DEFAULT_RING_MAX_FRAMES = 4;
|
|
345
|
+
/** Total bytes of complete frames held. ~34x the measured 29.6 KB frame x 4. */
|
|
346
|
+
const DEFAULT_RING_MAX_BYTES = 4 * 1024 * 1024;
|
|
347
|
+
/** Ceiling on ONE partially-received frame, so a missing EOI cannot grow. */
|
|
348
|
+
const DEFAULT_MAX_FRAME_BYTES = 2 * 1024 * 1024;
|
|
349
|
+
/** Session-start attempts before degrading. Lower than the per-grab path's 4
|
|
350
|
+
* because the per-grab provider behind this one does its own validation. */
|
|
351
|
+
const DEFAULT_OPEN_VALIDATION_ATTEMPTS = 2;
|
|
352
|
+
const DEFAULT_RESTART_DELAY_MS = 1000;
|
|
353
|
+
const DEFAULT_MAX_RESTART_DELAY_MS = 60000;
|
|
354
|
+
/** Multiple of the frame interval `getFrame` waits before calling it stale. */
|
|
355
|
+
const STALE_DEADLINE_FRAME_MULTIPLE = 6;
|
|
356
|
+
/** Floor on that deadline, so a fast `streamFps` cannot make it hair-trigger. */
|
|
357
|
+
const MIN_STALE_DEADLINE_MS = 3000;
|
|
358
|
+
// ---------------------------------------------------------------------------
|
|
359
|
+
// MJPEG framing
|
|
360
|
+
// ---------------------------------------------------------------------------
|
|
361
|
+
/** A complete JPEG ends at this offset; -1 = need more bytes; -2 = malformed. */
|
|
362
|
+
exports.JPEG_NEED_MORE = -1;
|
|
363
|
+
exports.JPEG_MALFORMED = -2;
|
|
364
|
+
/**
|
|
365
|
+
* Offset just past the EOI of the JPEG starting at `start`, by walking JPEG
|
|
366
|
+
* segments — NOT by scanning for the `FF D9` byte pair.
|
|
367
|
+
*
|
|
368
|
+
* A plain byte scan is wrong and the failure is silent. Marker segments carry
|
|
369
|
+
* arbitrary payloads (a DQT's 64 quantisation values, a DHT's code lengths, an
|
|
370
|
+
* APPn's contents), any of which may contain `FF D9`, and a scanner that found
|
|
371
|
+
* one would cut the frame short and hand a truncated JPEG downstream — which
|
|
372
|
+
* `RTSPCameraProvider` already refuses on the one-shot path precisely because a
|
|
373
|
+
* truncated JPEG renders as grey and becomes alert evidence.
|
|
374
|
+
*
|
|
375
|
+
* So: step over each segment by its declared length until SOS, then scan the
|
|
376
|
+
* entropy-coded data honouring JPEG byte stuffing (`FF 00` is a literal `FF`,
|
|
377
|
+
* `FF D0`-`FF D7` are restart markers, `FF FF` is fill) until a real marker
|
|
378
|
+
* appears. In ffmpeg's baseline MJPEG output that marker is EOI.
|
|
379
|
+
*/
|
|
380
|
+
function jpegFrameEnd(buf, start = 0) {
|
|
381
|
+
if (buf.length < start + 4)
|
|
382
|
+
return exports.JPEG_NEED_MORE;
|
|
383
|
+
if (buf[start] !== 0xff || buf[start + 1] !== 0xd8)
|
|
384
|
+
return exports.JPEG_MALFORMED;
|
|
385
|
+
let i = start + 2;
|
|
386
|
+
while (i + 1 < buf.length) {
|
|
387
|
+
if (buf[i] !== 0xff)
|
|
388
|
+
return exports.JPEG_MALFORMED;
|
|
389
|
+
// Fill bytes: any number of 0xFF may precede a marker code.
|
|
390
|
+
while (i + 1 < buf.length && buf[i + 1] === 0xff)
|
|
391
|
+
i += 1;
|
|
392
|
+
if (i + 1 >= buf.length)
|
|
393
|
+
return exports.JPEG_NEED_MORE;
|
|
394
|
+
const marker = buf[i + 1];
|
|
395
|
+
if (marker === 0xd9)
|
|
396
|
+
return i + 2; // EOI
|
|
397
|
+
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
|
|
398
|
+
i += 2; // standalone marker, no payload
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
if (i + 3 >= buf.length)
|
|
402
|
+
return exports.JPEG_NEED_MORE;
|
|
403
|
+
const len = buf.readUInt16BE(i + 2);
|
|
404
|
+
if (len < 2)
|
|
405
|
+
return exports.JPEG_MALFORMED;
|
|
406
|
+
if (marker === 0xda) {
|
|
407
|
+
// SOS — entropy-coded data follows the header, unlengthed.
|
|
408
|
+
let j = i + 2 + len;
|
|
409
|
+
while (j + 1 < buf.length) {
|
|
410
|
+
if (buf[j] !== 0xff) {
|
|
411
|
+
j += 1;
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
414
|
+
const next = buf[j + 1];
|
|
415
|
+
if (next === 0xff) {
|
|
416
|
+
j += 1; // fill
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
if (next === 0x00 || (next >= 0xd0 && next <= 0xd7)) {
|
|
420
|
+
j += 2; // stuffed FF, or a restart marker
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
423
|
+
break; // a real marker — resume the segment walk here
|
|
424
|
+
}
|
|
425
|
+
if (j + 1 >= buf.length)
|
|
426
|
+
return exports.JPEG_NEED_MORE;
|
|
427
|
+
i = j;
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
i = i + 2 + len;
|
|
431
|
+
}
|
|
432
|
+
return exports.JPEG_NEED_MORE;
|
|
433
|
+
}
|
|
434
|
+
exports.jpegFrameEnd = jpegFrameEnd;
|
|
435
|
+
/**
|
|
436
|
+
* Cuts a byte stream of concatenated JPEGs into whole frames, with a HARD
|
|
437
|
+
* ceiling on the partially-received one.
|
|
438
|
+
*
|
|
439
|
+
* The ceiling is the bound people forget: the ring is bounded, but the thing
|
|
440
|
+
* feeding it is an accumulator, and an accumulator with no cap is the same leak
|
|
441
|
+
* one layer up. On overflow the accumulator is dropped and the splitter resyncs
|
|
442
|
+
* to the next SOI, and the event is COUNTED rather than swallowed.
|
|
443
|
+
*/
|
|
444
|
+
class MjpegFrameSplitter {
|
|
445
|
+
constructor(maxFrameBytes) {
|
|
446
|
+
this.maxFrameBytes = maxFrameBytes;
|
|
447
|
+
this.acc = Buffer.alloc(0);
|
|
448
|
+
this.oversize = 0;
|
|
449
|
+
this.resyncs = 0;
|
|
450
|
+
}
|
|
451
|
+
getOversizeFrames() {
|
|
452
|
+
return this.oversize;
|
|
453
|
+
}
|
|
454
|
+
getResyncs() {
|
|
455
|
+
return this.resyncs;
|
|
456
|
+
}
|
|
457
|
+
getPendingBytes() {
|
|
458
|
+
return this.acc.length;
|
|
459
|
+
}
|
|
460
|
+
reset() {
|
|
461
|
+
this.acc = Buffer.alloc(0);
|
|
462
|
+
}
|
|
463
|
+
/** Append a chunk and return every complete frame it completed, in order. */
|
|
464
|
+
push(chunk) {
|
|
465
|
+
this.acc = this.acc.length === 0 ? chunk : Buffer.concat([this.acc, chunk]);
|
|
466
|
+
const out = [];
|
|
467
|
+
for (;;) {
|
|
468
|
+
const end = jpegFrameEnd(this.acc, 0);
|
|
469
|
+
if (end === exports.JPEG_NEED_MORE) {
|
|
470
|
+
if (this.acc.length > this.maxFrameBytes) {
|
|
471
|
+
// Enforced on ADMISSION: the accumulator never exceeds its ceiling
|
|
472
|
+
// and is then reconciled — it is dropped the moment it would.
|
|
473
|
+
this.oversize += 1;
|
|
474
|
+
this.resyncToNextSoi(1);
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
break;
|
|
478
|
+
}
|
|
479
|
+
if (end === exports.JPEG_MALFORMED) {
|
|
480
|
+
this.resyncs += 1;
|
|
481
|
+
if (!this.resyncToNextSoi(1))
|
|
482
|
+
break;
|
|
483
|
+
continue;
|
|
484
|
+
}
|
|
485
|
+
out.push(this.acc.subarray(0, end));
|
|
486
|
+
this.acc = this.acc.subarray(end);
|
|
487
|
+
if (this.acc.length === 0)
|
|
488
|
+
break;
|
|
489
|
+
}
|
|
490
|
+
return out;
|
|
491
|
+
}
|
|
492
|
+
/** Drop bytes up to the next SOI. Returns false when there is no next SOI. */
|
|
493
|
+
resyncToNextSoi(from) {
|
|
494
|
+
for (let i = from; i + 1 < this.acc.length; i += 1) {
|
|
495
|
+
if (this.acc[i] === 0xff && this.acc[i + 1] === 0xd8) {
|
|
496
|
+
this.acc = this.acc.subarray(i);
|
|
497
|
+
return true;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
// Keep the last byte: an SOI may straddle the chunk boundary.
|
|
501
|
+
this.acc = this.acc.length > 0 ? this.acc.subarray(this.acc.length - 1) : this.acc;
|
|
502
|
+
return false;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
exports.MjpegFrameSplitter = MjpegFrameSplitter;
|
|
506
|
+
/**
|
|
507
|
+
* Bounded newest-first frame ring. Byte- and count-bounded, enforced when a
|
|
508
|
+
* frame is ADMITTED, and every eviction counted — `VlmCandidateBuffer`'s
|
|
509
|
+
* discipline, for the same reason: a silent eviction moves a lost frame from
|
|
510
|
+
* one invisible place to another.
|
|
511
|
+
*/
|
|
512
|
+
class FrameRing {
|
|
513
|
+
constructor(maxFrames, maxBytes) {
|
|
514
|
+
this.maxFrames = maxFrames;
|
|
515
|
+
this.maxBytes = maxBytes;
|
|
516
|
+
this.frames = [];
|
|
517
|
+
this.bytes = 0;
|
|
518
|
+
this.evicted = 0;
|
|
519
|
+
this.rejected = 0;
|
|
520
|
+
this.seq = 0;
|
|
521
|
+
}
|
|
522
|
+
/** Admit a frame. Returns its sequence number, or null if it cannot fit. */
|
|
523
|
+
push(bytes, atMs) {
|
|
524
|
+
if (bytes.length > this.maxBytes) {
|
|
525
|
+
// A single frame larger than the whole budget can never be admitted, and
|
|
526
|
+
// evicting the ring to make room for it would trade every frame we have
|
|
527
|
+
// for one we cannot keep either.
|
|
528
|
+
this.rejected += 1;
|
|
529
|
+
return null;
|
|
530
|
+
}
|
|
531
|
+
while (this.frames.length > 0 &&
|
|
532
|
+
(this.frames.length + 1 > this.maxFrames || this.bytes + bytes.length > this.maxBytes)) {
|
|
533
|
+
const oldest = this.frames.shift();
|
|
534
|
+
this.bytes -= oldest.bytes.length;
|
|
535
|
+
this.evicted += 1;
|
|
536
|
+
}
|
|
537
|
+
this.seq += 1;
|
|
538
|
+
this.frames.push({ bytes, atMs, seq: this.seq });
|
|
539
|
+
this.bytes += bytes.length;
|
|
540
|
+
return this.seq;
|
|
541
|
+
}
|
|
542
|
+
newest() {
|
|
543
|
+
return this.frames.length ? this.frames[this.frames.length - 1] : null;
|
|
544
|
+
}
|
|
545
|
+
getBytes() {
|
|
546
|
+
return this.bytes;
|
|
547
|
+
}
|
|
548
|
+
getFrames() {
|
|
549
|
+
return this.frames.length;
|
|
550
|
+
}
|
|
551
|
+
getEvicted() {
|
|
552
|
+
return this.evicted;
|
|
553
|
+
}
|
|
554
|
+
getRejected() {
|
|
555
|
+
return this.rejected;
|
|
556
|
+
}
|
|
557
|
+
clear() {
|
|
558
|
+
this.frames = [];
|
|
559
|
+
this.bytes = 0;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
exports.FrameRing = FrameRing;
|
|
563
|
+
// ---------------------------------------------------------------------------
|
|
564
|
+
// Per-recorder admission for LONG-LIVED video sessions
|
|
565
|
+
// ---------------------------------------------------------------------------
|
|
566
|
+
/**
|
|
567
|
+
* Deliberately a different pool from `CaptureAdmission` AND from
|
|
568
|
+
* `AudioSessionAdmission`: same key, three separate budgets. `tryAcquire` only
|
|
569
|
+
* — this lane never queues.
|
|
570
|
+
*
|
|
571
|
+
* ── THE LIMIT IS LEARNED FROM THE RECORDER, NOT DECLARED ──────────────────
|
|
572
|
+
*
|
|
573
|
+
* `maxStreamSessionsPerSource` is a POLICY CEILING on our own occupancy. The
|
|
574
|
+
* recorder's ceiling is a property of the recorder, it is not discoverable
|
|
575
|
+
* before you reach it, and it is not close to constant across recorders —
|
|
576
|
+
* measured 2026-08-19 with `scripts/measure-rtsp-session-ceiling.mjs`:
|
|
577
|
+
*
|
|
578
|
+
* Happytime ONVIF/RTSP Server V8.1 4 (fixed RUA pool; no config key)
|
|
579
|
+
* Nx Media Server 6.1.2.42921 >=160 (never refused)
|
|
580
|
+
* mediamtx unbounded (which is why the original
|
|
581
|
+
* validation could not see any
|
|
582
|
+
* of this)
|
|
583
|
+
*
|
|
584
|
+
* So `learnCeiling` records what the recorder actually did the first time it
|
|
585
|
+
* refused, and every later feed is refused by US — instantly, with no recorder
|
|
586
|
+
* traffic — instead of by the recorder.
|
|
587
|
+
*/
|
|
588
|
+
class StreamSessionAdmission {
|
|
589
|
+
constructor(limit) {
|
|
590
|
+
this.active = 0;
|
|
591
|
+
this.learnedCeiling = null;
|
|
592
|
+
/** Live holders, oldest first. Shedding pops the NEWEST — see `learnCeiling`. */
|
|
593
|
+
this.holders = [];
|
|
594
|
+
this.limit = limit;
|
|
595
|
+
}
|
|
596
|
+
/** Tighten only, matching the other two lanes: the most cautious caller wins. */
|
|
597
|
+
tightenLimit(limit) {
|
|
598
|
+
this.limit = Math.min(this.limit, limit);
|
|
599
|
+
}
|
|
600
|
+
getLimit() {
|
|
601
|
+
return this.limit;
|
|
602
|
+
}
|
|
603
|
+
getActive() {
|
|
604
|
+
return this.active;
|
|
605
|
+
}
|
|
606
|
+
/** The recorder's own ceiling, once it has refused once. Null until then. */
|
|
607
|
+
getLearnedCeiling() {
|
|
608
|
+
return this.learnedCeiling;
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* A persistent session could NOT be established while `observedHeld` were
|
|
612
|
+
* already held against this recorder. That is the recorder's ceiling, so
|
|
613
|
+
* stop guessing at it — and leave the transient lane room to work.
|
|
614
|
+
*
|
|
615
|
+
* ── WHY A RESERVE, AND WHY THIS SIZE ──────────────────────────────────
|
|
616
|
+
*
|
|
617
|
+
* The degrade path is `RTSPCameraProvider`, and a per-grab pull needs an
|
|
618
|
+
* RTSP session of its own. If the persistent lane holds every slot the
|
|
619
|
+
* recorder has, the fallback has nowhere to go: measured on Happytime
|
|
620
|
+
* (ceiling 4) with a cap of 4, the fifth feed degraded to per-grab exactly
|
|
621
|
+
* as designed and then returned NO FRAMES AT ALL, because the recorder was
|
|
622
|
+
* full of our own persistent sessions. Before this architecture that feed
|
|
623
|
+
* worked — transient pulls are serialised by `CaptureAdmission`, so N feeds
|
|
624
|
+
* shared 2 slots. Leaving the reserve is what keeps "degrade, never fail"
|
|
625
|
+
* true on a small recorder.
|
|
626
|
+
*
|
|
627
|
+
* The reserve is `DEFAULT_MAX_CONCURRENT_CAPTURES_PER_SOURCE` — the SAME
|
|
628
|
+
* number that bounds the transient lane, imported rather than restated, so
|
|
629
|
+
* the two lanes can never disagree about how much room the fallback needs.
|
|
630
|
+
*
|
|
631
|
+
* Measured after this change, Happytime + 6 feeds: 2 persistent, 4 per-grab,
|
|
632
|
+
* 6 of 6 producing complete decodable frames.
|
|
633
|
+
*/
|
|
634
|
+
learnCeiling(observedHeld, failureMessage = '') {
|
|
635
|
+
// A recorder that named a cause belonging to THIS FEED told us the fault is
|
|
636
|
+
// this feed's, not the box's. Bad credentials and a bad channel path are the
|
|
637
|
+
// misconfigurations that would otherwise reach a SHARED ceiling and demote
|
|
638
|
+
// every healthy camera on the same recorder — see `PER_FEED_FAILURE_RE`.
|
|
639
|
+
if (PER_FEED_FAILURE_RE.test(failureMessage))
|
|
640
|
+
return;
|
|
641
|
+
const ceiling = Math.max(0, Math.trunc(observedHeld));
|
|
642
|
+
// A refusal with NOTHING of ours held is not a ceiling.
|
|
643
|
+
//
|
|
644
|
+
// It is a feed that could not open at all — wrong credentials, a dead
|
|
645
|
+
// stream, a recorder still booting, a network blip — and every one of those
|
|
646
|
+
// is transient in a way a session pool is not. Learning from it would be
|
|
647
|
+
// permanent in a way the fault is not: `learnedCeiling` only ever goes DOWN
|
|
648
|
+
// (Math.min) and `tightenLimit` only ever goes DOWN, so a single failed
|
|
649
|
+
// first open would pin the lane at 1 persistent session for that recorder
|
|
650
|
+
// for the life of the process — turning one transient blip on a healthy
|
|
651
|
+
// 16-channel NVR into a permanent 16x capture-latency regression that
|
|
652
|
+
// nothing re-arms.
|
|
653
|
+
//
|
|
654
|
+
// Contention is the thing being measured, so it takes at least one held
|
|
655
|
+
// session to measure it. Below that, say nothing and let the ordinary
|
|
656
|
+
// degrade + reconnect path handle the feed.
|
|
657
|
+
if (ceiling < 1)
|
|
658
|
+
return;
|
|
659
|
+
this.learnedCeiling =
|
|
660
|
+
this.learnedCeiling === null ? ceiling : Math.min(this.learnedCeiling, ceiling);
|
|
661
|
+
this.tightenLimit(Math.max(1, this.learnedCeiling - STREAM_SESSION_RESERVE_FOR_PER_GRAB));
|
|
662
|
+
this.shedOverQuota();
|
|
663
|
+
}
|
|
664
|
+
/**
|
|
665
|
+
* Bring occupancy down to the limit by asking the NEWEST holders to let go.
|
|
666
|
+
*
|
|
667
|
+
* Newest-first because the oldest sessions are the ones the recorder has
|
|
668
|
+
* already proven it will serve, and because a feed that has been streaming
|
|
669
|
+
* for an hour is the worst one to interrupt. Shedding is how a learned
|
|
670
|
+
* ceiling reaches sessions that were admitted before we knew it — without it
|
|
671
|
+
* the reserve would be a number in a variable and the recorder would stay
|
|
672
|
+
* full until every feed happened to reconnect.
|
|
673
|
+
*/
|
|
674
|
+
shedOverQuota() {
|
|
675
|
+
while (this.active > this.limit && this.holders.length > 0) {
|
|
676
|
+
const holder = this.holders.pop();
|
|
677
|
+
// `shed` releases through the same closure `tryAcquire` handed out, so
|
|
678
|
+
// `active` cannot drift no matter which side initiates the release.
|
|
679
|
+
try {
|
|
680
|
+
holder.shed();
|
|
681
|
+
}
|
|
682
|
+
catch {
|
|
683
|
+
/* a provider that cannot shed must not wedge the pool */
|
|
684
|
+
}
|
|
685
|
+
holder.release();
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
tryAcquire(shed = () => undefined) {
|
|
689
|
+
if (this.active >= this.limit)
|
|
690
|
+
return null;
|
|
691
|
+
this.active += 1;
|
|
692
|
+
let released = false;
|
|
693
|
+
const entry = {
|
|
694
|
+
shed,
|
|
695
|
+
release: () => {
|
|
696
|
+
if (released)
|
|
697
|
+
return;
|
|
698
|
+
released = true;
|
|
699
|
+
this.active = Math.max(0, this.active - 1);
|
|
700
|
+
const i = this.holders.indexOf(entry);
|
|
701
|
+
if (i >= 0)
|
|
702
|
+
this.holders.splice(i, 1);
|
|
703
|
+
},
|
|
704
|
+
};
|
|
705
|
+
this.holders.push(entry);
|
|
706
|
+
return entry.release;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
const streamAdmissionBySource = new Map();
|
|
710
|
+
function streamAdmissionFor(streamUrl, limit) {
|
|
711
|
+
const key = (0, RTSPCameraProvider_1.captureSourceKey)(streamUrl);
|
|
712
|
+
if (!key)
|
|
713
|
+
return null;
|
|
714
|
+
const existing = streamAdmissionBySource.get(key);
|
|
715
|
+
if (existing) {
|
|
716
|
+
existing.tightenLimit(limit);
|
|
717
|
+
return existing;
|
|
718
|
+
}
|
|
719
|
+
const admission = new StreamSessionAdmission(limit);
|
|
720
|
+
streamAdmissionBySource.set(key, admission);
|
|
721
|
+
return admission;
|
|
722
|
+
}
|
|
723
|
+
/** Test seam: forget every recorder's persistent-session budget. */
|
|
724
|
+
function __resetStreamAdmissionForTests() {
|
|
725
|
+
streamAdmissionBySource.clear();
|
|
726
|
+
}
|
|
727
|
+
exports.__resetStreamAdmissionForTests = __resetStreamAdmissionForTests;
|
|
728
|
+
/** Live persistent-session occupancy for a recorder, for diagnostics. */
|
|
729
|
+
function streamSessionStatsFor(streamUrl) {
|
|
730
|
+
const key = (0, RTSPCameraProvider_1.captureSourceKey)(streamUrl);
|
|
731
|
+
if (!key)
|
|
732
|
+
return null;
|
|
733
|
+
const admission = streamAdmissionBySource.get(key);
|
|
734
|
+
if (!admission)
|
|
735
|
+
return null;
|
|
736
|
+
return {
|
|
737
|
+
key,
|
|
738
|
+
active: admission.getActive(),
|
|
739
|
+
limit: admission.getLimit(),
|
|
740
|
+
learnedCeiling: admission.getLearnedCeiling(),
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
exports.streamSessionStatsFor = streamSessionStatsFor;
|
|
744
|
+
/**
|
|
745
|
+
* One instance per feed. Implements the SAME `CameraProvider` interface as
|
|
746
|
+
* `RTSPCameraProvider`, so every consumer is unchanged — but `getFrame()` reads
|
|
747
|
+
* the newest frame out of the ring the long-lived session is filling, rather
|
|
748
|
+
* than spawning a process and handshaking a stream per call.
|
|
749
|
+
*/
|
|
750
|
+
class RTSPStreamCameraProvider {
|
|
751
|
+
constructor(config) {
|
|
752
|
+
this.opened = false;
|
|
753
|
+
this.mode = 'persistent';
|
|
754
|
+
this.degradedReason = null;
|
|
755
|
+
this.fallback = null;
|
|
756
|
+
/**
|
|
757
|
+
* Why the per-grab fallback could not open, when it could not.
|
|
758
|
+
*
|
|
759
|
+
* Kept because the alternative is what shipped: a feed whose fallback failed
|
|
760
|
+
* answered `getFrame()` with "Camera not opened. Call open() first." — a
|
|
761
|
+
* PROGRAMMING error message for a RECORDER condition, thrown from
|
|
762
|
+
* `RTSPCameraProvider` after this provider had already been told, in words,
|
|
763
|
+
* that the recorder refused. Measured on Happytime at its 4-session ceiling:
|
|
764
|
+
* `open()` carried "Failed reading RTSP data: End of file" and `getFrame()`
|
|
765
|
+
* threw the not-opened message, so the one classifiable fact was discarded
|
|
766
|
+
* one frame later.
|
|
767
|
+
*/
|
|
768
|
+
this.fallbackOpenError = null;
|
|
769
|
+
this.proc = null;
|
|
770
|
+
/** This recorder's persistent-session lane, for the learned ceiling. */
|
|
771
|
+
this.admission = null;
|
|
772
|
+
this.releaseAdmission = null;
|
|
773
|
+
this.restartTimer = null;
|
|
774
|
+
this.cleanupInput = null;
|
|
775
|
+
this.startingUp = false;
|
|
776
|
+
this.sessionStarts = 0;
|
|
777
|
+
this.restarts = 0;
|
|
778
|
+
this.framesDecoded = 0;
|
|
779
|
+
this.framesServed = 0;
|
|
780
|
+
this.staleReads = 0;
|
|
781
|
+
this.lastFrameAtMs = null;
|
|
782
|
+
this.lastServedSeq = 0;
|
|
783
|
+
this.lastErrorMessage = null;
|
|
784
|
+
/** Resolvers waiting for the next frame. Woken by `admitFrame`. */
|
|
785
|
+
this.frameWaiters = [];
|
|
786
|
+
if (!config?.rtspUrl)
|
|
787
|
+
throw new Error('rtspUrl is required for RTSPStreamCameraProvider');
|
|
788
|
+
this.rtspUrl = config.rtspUrl;
|
|
789
|
+
this.width = config.width ?? 640;
|
|
790
|
+
this.height = config.height ?? 480;
|
|
791
|
+
this.crop = config.crop;
|
|
792
|
+
this.transport = config.transport ?? 'tcp';
|
|
793
|
+
this.timeoutUs = config.timeoutUs ?? DEFAULT_TIMEOUT_US;
|
|
794
|
+
this.ffmpegPath = config.ffmpegPath ?? 'ffmpeg';
|
|
795
|
+
this.ffmpegMajorVersion = config.ffmpegMajorVersion ?? 6;
|
|
796
|
+
this.jpegQuality = config.jpegQuality ?? 70;
|
|
797
|
+
this.streamFps = Math.max(0.1, Number(config.streamFps ?? DEFAULT_STREAM_FPS));
|
|
798
|
+
this.frameIntervalMs = 1000 / this.streamFps;
|
|
799
|
+
this.decoderThreads = Math.max(1, Math.trunc(config.decoderThreads ?? DEFAULT_DECODER_THREADS));
|
|
800
|
+
this.maxStreamSessionsPerSource = Math.max(1, Math.trunc(config.maxStreamSessionsPerSource ?? exports.DEFAULT_MAX_STREAM_SESSIONS_PER_SOURCE));
|
|
801
|
+
this.frameWaitMs = Math.max(1, Math.trunc(config.frameWaitMs ??
|
|
802
|
+
Math.max(MIN_STALE_DEADLINE_MS, STALE_DEADLINE_FRAME_MULTIPLE * this.frameIntervalMs)));
|
|
803
|
+
this.openValidationAttempts = Math.max(1, Math.trunc(config.openValidationAttempts ?? DEFAULT_OPEN_VALIDATION_ATTEMPTS));
|
|
804
|
+
this.restartDelayMs = Math.max(0, config.restartDelayMs ?? DEFAULT_RESTART_DELAY_MS);
|
|
805
|
+
this.maxRestartDelayMs = Math.max(this.restartDelayMs, config.maxRestartDelayMs ?? DEFAULT_MAX_RESTART_DELAY_MS);
|
|
806
|
+
this.currentRestartDelay = this.restartDelayMs;
|
|
807
|
+
this.persistentRequested = config.persistent !== false;
|
|
808
|
+
this.spawnFn = config.spawnFn ?? child_process_1.spawn;
|
|
809
|
+
this.now = config.now ?? Date.now;
|
|
810
|
+
this.onLog = config.onLog;
|
|
811
|
+
this.ring = new FrameRing(Math.max(1, Math.trunc(config.ringMaxFrames ?? DEFAULT_RING_MAX_FRAMES)), Math.max(1, Math.trunc(config.ringMaxBytes ?? DEFAULT_RING_MAX_BYTES)));
|
|
812
|
+
this.splitter = new MjpegFrameSplitter(Math.max(1, Math.trunc(config.maxFrameBytes ?? DEFAULT_MAX_FRAME_BYTES)));
|
|
813
|
+
const fallbackConfig = {
|
|
814
|
+
...(config.fallbackConfig ?? {}),
|
|
815
|
+
rtspUrl: this.rtspUrl,
|
|
816
|
+
width: this.width,
|
|
817
|
+
height: this.height,
|
|
818
|
+
crop: this.crop,
|
|
819
|
+
transport: this.transport,
|
|
820
|
+
timeoutUs: this.timeoutUs,
|
|
821
|
+
ffmpegPath: this.ffmpegPath,
|
|
822
|
+
outputFormat: 'jpeg',
|
|
823
|
+
jpegQuality: this.jpegQuality,
|
|
824
|
+
...(config.streamUrlResolver ? { streamUrlResolver: config.streamUrlResolver } : {}),
|
|
825
|
+
};
|
|
826
|
+
const make = config.fallbackFactory ?? ((c) => new RTSPCameraProvider_1.RTSPCameraProvider(c));
|
|
827
|
+
this.buildFallback = () => make(fallbackConfig);
|
|
828
|
+
}
|
|
829
|
+
// ── CameraProvider ───────────────────────────────────────────────────────
|
|
830
|
+
/**
|
|
831
|
+
* Take a recorder session slot and start streaming. NEVER throws because the
|
|
832
|
+
* persistent path was unavailable — it degrades to the per-grab provider and
|
|
833
|
+
* says so. It throws only when the per-grab path ALSO fails, which is exactly
|
|
834
|
+
* what the connector does today, so no working camera can be broken here.
|
|
835
|
+
*/
|
|
836
|
+
async open(_deviceId) {
|
|
837
|
+
if (this.opened)
|
|
838
|
+
return;
|
|
839
|
+
// A previous open() degraded and left a per-grab provider behind. Re-opening
|
|
840
|
+
// is the manager's reconnect path, and it is also the ONLY chance to retake
|
|
841
|
+
// a persistent session from a recorder that has since rebooted, freed a
|
|
842
|
+
// slot, or come back on the network. Retrying is therefore the re-arm — a
|
|
843
|
+
// provider that degraded once and could never come back would be the Ember
|
|
844
|
+
// permanent-retirement shape (see handleSessionEnd) at the capture layer.
|
|
845
|
+
// Drop the old provider rather than leaking one per reconnect.
|
|
846
|
+
if (this.fallback) {
|
|
847
|
+
const previous = this.fallback;
|
|
848
|
+
this.fallback = null;
|
|
849
|
+
this.fallbackOpenError = null;
|
|
850
|
+
try {
|
|
851
|
+
await previous.close();
|
|
852
|
+
}
|
|
853
|
+
catch {
|
|
854
|
+
/* a closing fallback must not be able to block a re-open */
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
if (!this.persistentRequested) {
|
|
858
|
+
await this.degradeTo('disabled');
|
|
859
|
+
return;
|
|
860
|
+
}
|
|
861
|
+
const admission = streamAdmissionFor(this.rtspUrl, this.maxStreamSessionsPerSource);
|
|
862
|
+
this.admission = admission;
|
|
863
|
+
if (admission) {
|
|
864
|
+
// The shed callback is how a LEARNED ceiling reaches a session that was
|
|
865
|
+
// admitted before we knew it. It must not throw and must not await: the
|
|
866
|
+
// pool calls it while it is reconciling occupancy.
|
|
867
|
+
const release = admission.tryAcquire(() => this.shedPersistentSession());
|
|
868
|
+
if (!release) {
|
|
869
|
+
this.lastErrorMessage = new RtspStreamAdmissionError((0, RTSPCameraProvider_1.captureSourceKey)(this.rtspUrl) ?? 'unknown', admission.getLimit()).message;
|
|
870
|
+
await this.degradeTo('session_limit');
|
|
871
|
+
return;
|
|
872
|
+
}
|
|
873
|
+
this.releaseAdmission = release;
|
|
874
|
+
}
|
|
875
|
+
this.opened = true;
|
|
876
|
+
this.mode = 'persistent';
|
|
877
|
+
this.degradedReason = null;
|
|
878
|
+
this.currentRestartDelay = this.restartDelayMs;
|
|
879
|
+
this.startingUp = true;
|
|
880
|
+
let lastErr;
|
|
881
|
+
try {
|
|
882
|
+
for (let attempt = 1; attempt <= this.openValidationAttempts; attempt += 1) {
|
|
883
|
+
try {
|
|
884
|
+
await this.startSession();
|
|
885
|
+
return;
|
|
886
|
+
}
|
|
887
|
+
catch (err) {
|
|
888
|
+
lastErr = err;
|
|
889
|
+
this.stopProcess();
|
|
890
|
+
if (attempt < this.openValidationAttempts && this.restartDelayMs > 0) {
|
|
891
|
+
await new Promise((resolve) => setTimeout(resolve, this.restartDelayMs));
|
|
892
|
+
}
|
|
893
|
+
if (!this.opened)
|
|
894
|
+
break; // closed underneath us
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
finally {
|
|
899
|
+
this.startingUp = false;
|
|
900
|
+
}
|
|
901
|
+
// The persistent session could not be held. Give the recorder slot straight
|
|
902
|
+
// back so a feed that CAN hold one is not denied by a feed that cannot, and
|
|
903
|
+
// run the path the connector runs today.
|
|
904
|
+
this.opened = false;
|
|
905
|
+
this.stopProcess();
|
|
906
|
+
this.releaseSlot();
|
|
907
|
+
this.lastErrorMessage = (0, RTSPCameraProvider_1.sanitizeStreamUrlForLog)(lastErr instanceof Error ? lastErr.message : String(lastErr ?? 'stream session failed'));
|
|
908
|
+
// THIS is the recorder telling us its ceiling, and it is the only moment it
|
|
909
|
+
// ever will. Whatever is still held after our own slot went back is what
|
|
910
|
+
// this recorder actually serves; record it so no later feed has to find out
|
|
911
|
+
// the same way, and so the fallback keeps room to pull. Read AFTER
|
|
912
|
+
// `releaseSlot` on purpose — including our own failed slot would inflate
|
|
913
|
+
// the ceiling by one every time.
|
|
914
|
+
// The message is passed so the pool can REFUSE to learn from a failure the
|
|
915
|
+
// recorder attributed to this feed. Without it a single mistyped password
|
|
916
|
+
// teaches every other camera on the box a ceiling that does not exist.
|
|
917
|
+
if (admission)
|
|
918
|
+
admission.learnCeiling(admission.getActive(), this.lastErrorMessage ?? '');
|
|
919
|
+
await this.degradeTo('session_unavailable');
|
|
920
|
+
}
|
|
921
|
+
/**
|
|
922
|
+
* Drop this feed's persistent session because the recorder's learned ceiling
|
|
923
|
+
* turned out to be lower than the lane believed.
|
|
924
|
+
*
|
|
925
|
+
* Synchronous and non-throwing by contract — the admission pool calls this
|
|
926
|
+
* while it is reconciling occupancy, and an exception or an await here would
|
|
927
|
+
* leave the pool's accounting mid-flight. The transition itself is the one
|
|
928
|
+
* this provider already implements everywhere else: stop the session, report
|
|
929
|
+
* `per_grab` / `session_limit`, and build the fallback. The fallback's own
|
|
930
|
+
* `open()` is deliberately left to run on its promise; `getFrame()` reports
|
|
931
|
+
* its failure rather than the generic not-opened message.
|
|
932
|
+
*/
|
|
933
|
+
shedPersistentSession() {
|
|
934
|
+
if (!this.opened || this.fallback)
|
|
935
|
+
return;
|
|
936
|
+
this.opened = false;
|
|
937
|
+
if (this.restartTimer) {
|
|
938
|
+
clearTimeout(this.restartTimer);
|
|
939
|
+
this.restartTimer = null;
|
|
940
|
+
}
|
|
941
|
+
this.stopProcess();
|
|
942
|
+
this.ring.clear();
|
|
943
|
+
this.splitter.reset();
|
|
944
|
+
this.wakeWaiters();
|
|
945
|
+
this.lastErrorMessage = new RtspStreamAdmissionError((0, RTSPCameraProvider_1.captureSourceKey)(this.rtspUrl) ?? 'unknown', this.admission?.getLimit() ?? this.maxStreamSessionsPerSource).message;
|
|
946
|
+
void this.degradeTo('session_limit').catch((err) => {
|
|
947
|
+
this.fallbackOpenError = (0, RTSPCameraProvider_1.sanitizeStreamUrlForLog)(err instanceof Error ? err.message : String(err));
|
|
948
|
+
});
|
|
949
|
+
}
|
|
950
|
+
/**
|
|
951
|
+
* The newest frame this feed has produced AND NOT YET SERVED.
|
|
952
|
+
*
|
|
953
|
+
* On the persistent path this is a memory read — microseconds — when a fresh
|
|
954
|
+
* frame is in hand, which at `streamFps` 4 against a 500 ms look cadence is
|
|
955
|
+
* every time. When it is not, the call waits up to `frameWaitMs` and is woken
|
|
956
|
+
* by the next arrival; a wait that expires is a STALE STREAM and throws.
|
|
957
|
+
*
|
|
958
|
+
* Never returning a repeat is the whole loudness guarantee. Handing back the
|
|
959
|
+
* same frame would report a frozen camera as healthy forever, and would also
|
|
960
|
+
* feed the motion detector a constant — which is the documented `file://`
|
|
961
|
+
* fixture trap (`-frames:v 1` with no `-ss` returns frame 0 forever), where
|
|
962
|
+
* the gate calibrates against a constant and settles into `no_motion`
|
|
963
|
+
* permanently.
|
|
964
|
+
*/
|
|
965
|
+
async getFrame() {
|
|
966
|
+
if (this.fallback) {
|
|
967
|
+
// A fallback that never opened cannot answer, and the reason it could not
|
|
968
|
+
// is the one thing worth saying. Re-throwing the recorder's own words
|
|
969
|
+
// keeps `classifyRtspFailureClass` looking at evidence instead of at a
|
|
970
|
+
// message about how this API is called.
|
|
971
|
+
if (this.fallbackOpenError !== null) {
|
|
972
|
+
throw new Error(`RTSP per-grab fallback is not open for ${(0, RTSPCameraProvider_1.sanitizeStreamUrlForLog)(this.rtspUrl)}: ` +
|
|
973
|
+
this.fallbackOpenError);
|
|
974
|
+
}
|
|
975
|
+
return this.fallback.getFrame();
|
|
976
|
+
}
|
|
977
|
+
if (!this.opened)
|
|
978
|
+
throw new Error('Camera not opened. Call open() first.');
|
|
979
|
+
const fresh = this.takeFreshFrame();
|
|
980
|
+
if (fresh)
|
|
981
|
+
return fresh;
|
|
982
|
+
const arrived = await this.waitForFrame(this.frameWaitMs);
|
|
983
|
+
if (arrived) {
|
|
984
|
+
const next = this.takeFreshFrame();
|
|
985
|
+
if (next)
|
|
986
|
+
return next;
|
|
987
|
+
}
|
|
988
|
+
this.staleReads += 1;
|
|
989
|
+
const newest = this.ring.newest();
|
|
990
|
+
const ageMs = newest ? this.now() - newest.atMs : this.frameWaitMs;
|
|
991
|
+
throw new RtspStreamStaleError(this.rtspUrl, ageMs, this.restarts);
|
|
992
|
+
}
|
|
993
|
+
getResolution() {
|
|
994
|
+
return { width: this.width, height: this.height };
|
|
995
|
+
}
|
|
996
|
+
getFps() {
|
|
997
|
+
return this.streamFps;
|
|
998
|
+
}
|
|
999
|
+
isOpen() {
|
|
1000
|
+
if (this.fallback)
|
|
1001
|
+
return this.fallback.isOpen();
|
|
1002
|
+
return this.opened;
|
|
1003
|
+
}
|
|
1004
|
+
async close() {
|
|
1005
|
+
this.opened = false;
|
|
1006
|
+
if (this.restartTimer) {
|
|
1007
|
+
clearTimeout(this.restartTimer);
|
|
1008
|
+
this.restartTimer = null;
|
|
1009
|
+
}
|
|
1010
|
+
this.stopProcess();
|
|
1011
|
+
this.releaseSlot();
|
|
1012
|
+
this.ring.clear();
|
|
1013
|
+
this.splitter.reset();
|
|
1014
|
+
this.wakeWaiters();
|
|
1015
|
+
if (this.fallback) {
|
|
1016
|
+
await this.fallback.close();
|
|
1017
|
+
this.fallback = null;
|
|
1018
|
+
}
|
|
1019
|
+
this.fallbackOpenError = null;
|
|
1020
|
+
this.admission = null;
|
|
1021
|
+
}
|
|
1022
|
+
/**
|
|
1023
|
+
* Every field DERIVED on each call. There is no cached capability boolean
|
|
1024
|
+
* here and one must never be added — see the header on
|
|
1025
|
+
* `cap_audio_classification`.
|
|
1026
|
+
*/
|
|
1027
|
+
getStats() {
|
|
1028
|
+
const newest = this.ring.newest();
|
|
1029
|
+
return {
|
|
1030
|
+
opened: this.isOpen(),
|
|
1031
|
+
mode: this.mode,
|
|
1032
|
+
degradedReason: this.degradedReason,
|
|
1033
|
+
running: this.proc !== null,
|
|
1034
|
+
bufferedFrames: this.ring.getFrames(),
|
|
1035
|
+
bufferedBytes: this.ring.getBytes(),
|
|
1036
|
+
pendingBytes: this.splitter.getPendingBytes(),
|
|
1037
|
+
sessionStarts: this.sessionStarts,
|
|
1038
|
+
restarts: this.restarts,
|
|
1039
|
+
framesDecoded: this.framesDecoded,
|
|
1040
|
+
framesServed: this.framesServed,
|
|
1041
|
+
framesEvicted: this.ring.getEvicted(),
|
|
1042
|
+
framesRejected: this.ring.getRejected(),
|
|
1043
|
+
oversizeFrames: this.splitter.getOversizeFrames(),
|
|
1044
|
+
resyncs: this.splitter.getResyncs(),
|
|
1045
|
+
staleReads: this.staleReads,
|
|
1046
|
+
lastFrameAtMs: this.lastFrameAtMs,
|
|
1047
|
+
frameAgeMs: newest ? this.now() - newest.atMs : null,
|
|
1048
|
+
lastErrorMessage: this.lastErrorMessage,
|
|
1049
|
+
};
|
|
1050
|
+
}
|
|
1051
|
+
// ── internals ────────────────────────────────────────────────────────────
|
|
1052
|
+
async degradeTo(reason) {
|
|
1053
|
+
this.mode = 'per_grab';
|
|
1054
|
+
this.degradedReason = reason;
|
|
1055
|
+
if (reason !== 'disabled') {
|
|
1056
|
+
this.log('warn', `RTSP persistent session unavailable (${reason}); falling back to per-grab capture` +
|
|
1057
|
+
(this.lastErrorMessage ? `: ${this.lastErrorMessage}` : ''));
|
|
1058
|
+
}
|
|
1059
|
+
const fallback = this.buildFallback();
|
|
1060
|
+
// Assign BEFORE open() so a rejection still leaves the provider describing
|
|
1061
|
+
// the path it is actually on. A throw here is the per-grab path failing,
|
|
1062
|
+
// which is exactly what a connector sees today.
|
|
1063
|
+
this.fallback = fallback;
|
|
1064
|
+
this.fallbackOpenError = null;
|
|
1065
|
+
try {
|
|
1066
|
+
await fallback.open();
|
|
1067
|
+
}
|
|
1068
|
+
catch (err) {
|
|
1069
|
+
// KEEP the reason. It is the only description of what the recorder did,
|
|
1070
|
+
// and without it `getFrame()` falls back to a message about how the
|
|
1071
|
+
// caller used the API — see `fallbackOpenError`.
|
|
1072
|
+
this.fallbackOpenError = (0, RTSPCameraProvider_1.sanitizeStreamUrlForLog)(err instanceof Error ? err.message : String(err));
|
|
1073
|
+
this.lastErrorMessage = this.fallbackOpenError;
|
|
1074
|
+
throw err;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
releaseSlot() {
|
|
1078
|
+
if (this.releaseAdmission) {
|
|
1079
|
+
this.releaseAdmission();
|
|
1080
|
+
this.releaseAdmission = null;
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
/** The newest frame if it has not been served yet, else null. */
|
|
1084
|
+
takeFreshFrame() {
|
|
1085
|
+
const newest = this.ring.newest();
|
|
1086
|
+
if (!newest || newest.seq <= this.lastServedSeq)
|
|
1087
|
+
return null;
|
|
1088
|
+
this.lastServedSeq = newest.seq;
|
|
1089
|
+
this.framesServed += 1;
|
|
1090
|
+
return newest.bytes;
|
|
1091
|
+
}
|
|
1092
|
+
waitForFrame(timeoutMs) {
|
|
1093
|
+
return new Promise((resolve) => {
|
|
1094
|
+
let settled = false;
|
|
1095
|
+
const done = (ok) => {
|
|
1096
|
+
if (settled)
|
|
1097
|
+
return;
|
|
1098
|
+
settled = true;
|
|
1099
|
+
this.frameWaiters = this.frameWaiters.filter((w) => w !== wake);
|
|
1100
|
+
clearTimeout(timer);
|
|
1101
|
+
resolve(ok);
|
|
1102
|
+
};
|
|
1103
|
+
const wake = () => done(true);
|
|
1104
|
+
const timer = setTimeout(() => done(false), timeoutMs);
|
|
1105
|
+
if (typeof timer.unref === 'function')
|
|
1106
|
+
timer.unref();
|
|
1107
|
+
this.frameWaiters.push(wake);
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
1110
|
+
wakeWaiters() {
|
|
1111
|
+
const waiters = this.frameWaiters;
|
|
1112
|
+
this.frameWaiters = [];
|
|
1113
|
+
for (const w of waiters) {
|
|
1114
|
+
try {
|
|
1115
|
+
w();
|
|
1116
|
+
}
|
|
1117
|
+
catch {
|
|
1118
|
+
/* a waiter cannot break the session */
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
admitFrame(bytes) {
|
|
1123
|
+
const at = this.now();
|
|
1124
|
+
const seq = this.ring.push(bytes, at);
|
|
1125
|
+
if (seq === null)
|
|
1126
|
+
return; // counted as rejected inside the ring
|
|
1127
|
+
this.framesDecoded += 1;
|
|
1128
|
+
this.lastFrameAtMs = at;
|
|
1129
|
+
this.wakeWaiters();
|
|
1130
|
+
}
|
|
1131
|
+
/**
|
|
1132
|
+
* ffmpeg args for a CONTINUOUS MJPEG pull. The URL goes into a 0600 ffconcat
|
|
1133
|
+
* file exactly as the other two providers do, so it never reaches argv.
|
|
1134
|
+
*/
|
|
1135
|
+
buildInputArgs() {
|
|
1136
|
+
const isRtsp = /^rtsps?:\/\//i.test(this.rtspUrl);
|
|
1137
|
+
if (!isRtsp) {
|
|
1138
|
+
return { args: ['-i', this.rtspUrl], cleanup: () => undefined };
|
|
1139
|
+
}
|
|
1140
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trillboards-rtsp-stream-'));
|
|
1141
|
+
const file = path.join(dir, 'input.ffconcat');
|
|
1142
|
+
const timeoutOpt = this.ffmpegMajorVersion >= 5 ? 'timeout' : 'stimeout';
|
|
1143
|
+
fs.writeFileSync(file, [
|
|
1144
|
+
'ffconcat version 1.0',
|
|
1145
|
+
`file '${(0, RTSPCameraProvider_1.escapeFfconcatValue)(this.rtspUrl)}'`,
|
|
1146
|
+
`option rtsp_transport ${this.transport}`,
|
|
1147
|
+
`option ${timeoutOpt} ${this.timeoutUs}`,
|
|
1148
|
+
'',
|
|
1149
|
+
].join('\n'), { mode: 0o600 });
|
|
1150
|
+
try {
|
|
1151
|
+
fs.chmodSync(file, 0o600);
|
|
1152
|
+
}
|
|
1153
|
+
catch {
|
|
1154
|
+
/* best-effort on platforms with ACL semantics */
|
|
1155
|
+
}
|
|
1156
|
+
return {
|
|
1157
|
+
args: [
|
|
1158
|
+
'-protocol_whitelist',
|
|
1159
|
+
'file,crypto,tcp,tls,rtp,rtsp,udp,http,https',
|
|
1160
|
+
'-safe',
|
|
1161
|
+
'0',
|
|
1162
|
+
'-f',
|
|
1163
|
+
'concat',
|
|
1164
|
+
'-i',
|
|
1165
|
+
file,
|
|
1166
|
+
],
|
|
1167
|
+
cleanup: () => {
|
|
1168
|
+
try {
|
|
1169
|
+
fs.unlinkSync(file);
|
|
1170
|
+
}
|
|
1171
|
+
catch {
|
|
1172
|
+
/* already gone */
|
|
1173
|
+
}
|
|
1174
|
+
try {
|
|
1175
|
+
fs.rmdirSync(dir);
|
|
1176
|
+
}
|
|
1177
|
+
catch {
|
|
1178
|
+
/* best-effort */
|
|
1179
|
+
}
|
|
1180
|
+
},
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
1183
|
+
buildArgs() {
|
|
1184
|
+
const input = this.buildInputArgs();
|
|
1185
|
+
return {
|
|
1186
|
+
args: [
|
|
1187
|
+
'-nostdin',
|
|
1188
|
+
// Before the input, so it binds to the DECODER. See
|
|
1189
|
+
// DEFAULT_DECODER_THREADS: one thread is both smaller and faster here.
|
|
1190
|
+
'-threads',
|
|
1191
|
+
String(this.decoderThreads),
|
|
1192
|
+
...input.args,
|
|
1193
|
+
'-an',
|
|
1194
|
+
// crop (mosaic tile) FIRST, then the aspect-preserving scale+pad — the
|
|
1195
|
+
// SAME carve math as the one-shot path, imported rather than restated,
|
|
1196
|
+
// so a persistent feed and a per-grab feed can never disagree about
|
|
1197
|
+
// what a frame of this camera looks like.
|
|
1198
|
+
//
|
|
1199
|
+
// `fps` LAST and deliberately not an output `-r`: the filter is driven
|
|
1200
|
+
// by arriving input frames, so a frozen source emits nothing instead of
|
|
1201
|
+
// duplicating the last frame into a lie about freshness.
|
|
1202
|
+
'-vf',
|
|
1203
|
+
`${(0, RTSPCameraProvider_1.buildVideoFilter)(this.width, this.height, this.crop)},fps=${this.streamFps}`,
|
|
1204
|
+
'-f',
|
|
1205
|
+
'mjpeg',
|
|
1206
|
+
'-q:v',
|
|
1207
|
+
String((0, RTSPCameraProvider_1.mjpegQv)(this.jpegQuality)),
|
|
1208
|
+
// …and again after the output codec, so it binds to the ENCODER too.
|
|
1209
|
+
'-threads',
|
|
1210
|
+
String(this.decoderThreads),
|
|
1211
|
+
'pipe:1',
|
|
1212
|
+
],
|
|
1213
|
+
cleanup: input.cleanup,
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1216
|
+
/**
|
|
1217
|
+
* Start one ffmpeg session. Resolves on the FIRST decoded frame — which is
|
|
1218
|
+
* the only evidence that matters, because a spawned process that never
|
|
1219
|
+
* produces an image is precisely the failure this provider exists to make
|
|
1220
|
+
* loud. Rejects when the session ends before that.
|
|
1221
|
+
*/
|
|
1222
|
+
startSession() {
|
|
1223
|
+
return new Promise((resolve, reject) => {
|
|
1224
|
+
const { args, cleanup } = this.buildArgs();
|
|
1225
|
+
this.cleanupInput = cleanup;
|
|
1226
|
+
this.sessionStarts += 1;
|
|
1227
|
+
this.splitter.reset();
|
|
1228
|
+
let proc;
|
|
1229
|
+
try {
|
|
1230
|
+
proc = this.spawnFn(this.ffmpegPath, args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
1231
|
+
}
|
|
1232
|
+
catch (err) {
|
|
1233
|
+
cleanup();
|
|
1234
|
+
this.cleanupInput = null;
|
|
1235
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
1236
|
+
return;
|
|
1237
|
+
}
|
|
1238
|
+
this.proc = proc;
|
|
1239
|
+
const stderrBuffer = new ffmpegStderr_1.BoundedSessionStderr();
|
|
1240
|
+
let settled = false;
|
|
1241
|
+
const settle = (fn) => {
|
|
1242
|
+
if (settled)
|
|
1243
|
+
return;
|
|
1244
|
+
settled = true;
|
|
1245
|
+
fn();
|
|
1246
|
+
};
|
|
1247
|
+
proc.stdout?.on('data', (chunk) => {
|
|
1248
|
+
for (const frame of this.splitter.push(chunk)) {
|
|
1249
|
+
this.admitFrame(frame);
|
|
1250
|
+
// A decoded frame proves the session is healthy — reset the backoff.
|
|
1251
|
+
this.currentRestartDelay = this.restartDelayMs;
|
|
1252
|
+
settle(resolve);
|
|
1253
|
+
}
|
|
1254
|
+
});
|
|
1255
|
+
proc.stderr?.on('data', (chunk) => {
|
|
1256
|
+
stderrBuffer.append(chunk.toString());
|
|
1257
|
+
});
|
|
1258
|
+
proc.on('error', (err) => {
|
|
1259
|
+
this.lastErrorMessage = (0, RTSPCameraProvider_1.sanitizeStreamUrlForLog)(err.message);
|
|
1260
|
+
settle(() => reject(err));
|
|
1261
|
+
this.handleSessionEnd(stderrBuffer.text());
|
|
1262
|
+
});
|
|
1263
|
+
proc.on('close', (code) => {
|
|
1264
|
+
// A CLEAN EXIT IS A FAULT HERE. `code === 0` means success on the
|
|
1265
|
+
// one-shot path; on a session we asked to run forever it means the
|
|
1266
|
+
// stream ended. Measured against the rig: killing the publisher exits 0
|
|
1267
|
+
// in 26 ms and freezing it exits 0 after ~10 s with "Failed reading
|
|
1268
|
+
// RTSP data: End of file". Both are restarts, never successes.
|
|
1269
|
+
const stderr = stderrBuffer.text();
|
|
1270
|
+
const summary = (0, RTSPCameraProvider_1.summarizeFfmpegStderr)(stderr);
|
|
1271
|
+
this.lastErrorMessage = summary || `ffmpeg stream session exited with code ${code}`;
|
|
1272
|
+
settle(() => reject(new Error(`RTSP stream session ended during startup: ${this.lastErrorMessage}`)));
|
|
1273
|
+
this.handleSessionEnd(stderr);
|
|
1274
|
+
});
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
1277
|
+
stopProcess() {
|
|
1278
|
+
const proc = this.proc;
|
|
1279
|
+
this.proc = null;
|
|
1280
|
+
if (proc) {
|
|
1281
|
+
try {
|
|
1282
|
+
proc.kill('SIGTERM');
|
|
1283
|
+
}
|
|
1284
|
+
catch {
|
|
1285
|
+
/* already gone */
|
|
1286
|
+
}
|
|
1287
|
+
// Bounded escalation: an ffmpeg wedged on a half-open RTSP socket ignores
|
|
1288
|
+
// SIGTERM, and a connector that leaks one child per restart is the
|
|
1289
|
+
// 28-children incident wearing a different hat.
|
|
1290
|
+
const timer = setTimeout(() => {
|
|
1291
|
+
try {
|
|
1292
|
+
proc.kill('SIGKILL');
|
|
1293
|
+
}
|
|
1294
|
+
catch {
|
|
1295
|
+
/* already gone */
|
|
1296
|
+
}
|
|
1297
|
+
}, 2000);
|
|
1298
|
+
if (typeof timer.unref === 'function')
|
|
1299
|
+
timer.unref();
|
|
1300
|
+
}
|
|
1301
|
+
if (this.cleanupInput) {
|
|
1302
|
+
this.cleanupInput();
|
|
1303
|
+
this.cleanupInput = null;
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
/**
|
|
1307
|
+
* A session ended. While the provider is open this ALWAYS schedules a restart
|
|
1308
|
+
* with capped exponential backoff, and it is never allowed to give up. On the
|
|
1309
|
+
* Ember fleet `shouldGiveUp` retired a camera permanently after a fixed 411 s
|
|
1310
|
+
* and NOTHING re-armed it — `onCameraStartFailed` was telemetry-only, no
|
|
1311
|
+
* config push restarted the analyzer, and `setMaxRetries` had zero call sites
|
|
1312
|
+
* — so 7,359 escalations became permanent silence
|
|
1313
|
+
* (docs/runbooks/ember-camera-health-taxonomy.md). The cap is what keeps a
|
|
1314
|
+
* never-give-up retry from being a hammer.
|
|
1315
|
+
*
|
|
1316
|
+
* Waiters are woken on the way out so a `getFrame()` blocked on a session
|
|
1317
|
+
* that just died reaches its stale error rather than sitting out the full
|
|
1318
|
+
* deadline for a stream that is provably not coming back.
|
|
1319
|
+
*/
|
|
1320
|
+
handleSessionEnd(stderr) {
|
|
1321
|
+
if (this.cleanupInput) {
|
|
1322
|
+
this.cleanupInput();
|
|
1323
|
+
this.cleanupInput = null;
|
|
1324
|
+
}
|
|
1325
|
+
this.proc = null;
|
|
1326
|
+
// Wake anyone blocked in getFrame(): the session that would have fed them
|
|
1327
|
+
// is gone, so they should reach their stale error now rather than sit out a
|
|
1328
|
+
// full deadline waiting for a stream that is provably not coming back.
|
|
1329
|
+
this.wakeWaiters();
|
|
1330
|
+
if (!this.opened)
|
|
1331
|
+
return;
|
|
1332
|
+
if (this.startingUp)
|
|
1333
|
+
return; // open() owns the retry loop during startup
|
|
1334
|
+
if (this.restartTimer)
|
|
1335
|
+
return;
|
|
1336
|
+
const delay = this.currentRestartDelay;
|
|
1337
|
+
this.currentRestartDelay = Math.min(this.maxRestartDelayMs, Math.max(1, this.currentRestartDelay * 2));
|
|
1338
|
+
this.restarts += 1;
|
|
1339
|
+
this.log('warn', `RTSP stream session ended, restarting in ${delay}ms (restart #${this.restarts}): ` +
|
|
1340
|
+
(0, RTSPCameraProvider_1.summarizeFfmpegStderr)(stderr, 200));
|
|
1341
|
+
this.restartTimer = setTimeout(() => {
|
|
1342
|
+
this.restartTimer = null;
|
|
1343
|
+
if (!this.opened)
|
|
1344
|
+
return;
|
|
1345
|
+
this.startSession().catch((err) => {
|
|
1346
|
+
this.lastErrorMessage = (0, RTSPCameraProvider_1.sanitizeStreamUrlForLog)(err?.message ?? String(err));
|
|
1347
|
+
});
|
|
1348
|
+
}, delay);
|
|
1349
|
+
if (typeof this.restartTimer.unref === 'function')
|
|
1350
|
+
this.restartTimer.unref();
|
|
1351
|
+
}
|
|
1352
|
+
log(level, message) {
|
|
1353
|
+
this.onLog?.(level, (0, RTSPCameraProvider_1.sanitizeStreamUrlForLog)(message));
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
exports.RTSPStreamCameraProvider = RTSPStreamCameraProvider;
|
|
1357
|
+
//# sourceMappingURL=RTSPStreamCameraProvider.js.map
|