@synapse-sdk/media 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/camera.d.ts +3 -4
- package/dist/camera.js +10 -8
- package/dist/canvas.d.ts +2 -3
- package/dist/decoder.d.ts +2 -2
- package/dist/decoder.js +3 -3
- package/dist/encoder.js +6 -8
- package/dist/index.d.ts +3 -3
- package/dist/index.js +6 -6
- package/dist/react.d.ts +2 -11
- package/dist/react.js +27 -23
- package/dist/reinsert.js +1 -1
- package/dist/sync.js +19 -4
- package/dist/video.d.ts +38 -115
- package/dist/video.js +153 -231
- package/package.json +4 -4
- package/dist/annexb.d.ts +0 -93
- package/dist/annexb.js +0 -227
package/dist/video.js
CHANGED
|
@@ -1,250 +1,172 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
1
|
+
// Video in one call each way (PLAN v0.25 §2), the browser spelling of the Rust client's `video.rs`:
|
|
2
|
+
// `videoPublisher(node, topic)` takes `VideoFrame`s (WebCodecs H.264 encode + publish in the
|
|
3
|
+
// caller's `publish`), `videoSubscriber(node, topic)` gives decoded `VideoFrame`s. Underneath they
|
|
4
|
+
// are the two-step path anyone may still take: an `H264Encoder` feeding a publisher on a
|
|
5
|
+
// `video/h264` topic, and a subscription feeding an `H264Decoder` (its `DecoderSync` policy decides
|
|
6
|
+
// where decoding starts and when to ask for an IDR).
|
|
7
|
+
//
|
|
8
|
+
// The publisher serves `<topic>/idr`: the next picture is an IDR with SPS/PPS. The subscriber calls
|
|
9
|
+
// it when its decoder's policy says `request_idr` (one call in flight at a time).
|
|
10
|
+
import { SynapseError, defineTopic } from "@synapse-sdk/core";
|
|
11
|
+
import { pumpTrack } from "./camera.js";
|
|
12
|
+
import { H264Decoder } from "./decoder.js";
|
|
13
|
+
import { H264Encoder } from "./encoder.js";
|
|
14
|
+
const H264 = "video/h264";
|
|
15
|
+
/** The key of a client publisher handle's encoder-frame path (`@synapse-sdk/client` `PUBLISH_MEDIA_FRAME`): flags and `STAGES` as the encoder made them. */
|
|
16
|
+
const PUBLISH_MEDIA_FRAME = Symbol.for("@synapse-sdk/client:publishMediaFrame");
|
|
17
|
+
/** The `video/h264` contract of `topic`: a name gets one; a contract must already be `video/h264`. */
|
|
18
|
+
function videoContract(topic) {
|
|
19
|
+
if (typeof topic === "string")
|
|
20
|
+
return defineTopic({ name: topic, contentType: H264 });
|
|
21
|
+
if (topic.contentType !== H264)
|
|
22
|
+
throw new SynapseError("invalid_input", `topic ${topic.name}: a video publisher needs contentType ${H264} (it has ${topic.contentType})`);
|
|
23
|
+
return topic;
|
|
21
24
|
}
|
|
22
25
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* re-insertion at recovery points, plus the `<topic>/idr` service that forces the next frame to be
|
|
26
|
-
* an IDR. `VideoPublisher.open()` resolves once the runtime admitted the topic (and the service).
|
|
26
|
+
* A video publisher on `topic` (a name, or a `video/h264` topic: datagrams with FEC unless the
|
|
27
|
+
* contract says otherwise). Resolves once the topic and its `<topic>/idr` service are admitted.
|
|
27
28
|
*/
|
|
28
|
-
export
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
#sent = 0;
|
|
37
|
-
#idrCalls = 0;
|
|
38
|
-
constructor(node, topic, publisher, registration, encoder) {
|
|
39
|
-
this.node = node;
|
|
40
|
-
this.topic = topic;
|
|
41
|
-
this.publisher = publisher;
|
|
42
|
-
this.registration = registration;
|
|
43
|
-
this.encoder = encoder;
|
|
44
|
-
this.#detach = encoder.attachSink(f => this.#onFrame(f));
|
|
29
|
+
export async function videoPublisher(node, topic, opts = {}) {
|
|
30
|
+
const contract = videoContract(topic);
|
|
31
|
+
const publisher = await node.publisher(contract);
|
|
32
|
+
let encoder;
|
|
33
|
+
let requests = 0;
|
|
34
|
+
let idr;
|
|
35
|
+
try {
|
|
36
|
+
idr = await node.serve(`${contract.name}/idr`, () => { requests++; encoder?.forceIdr(); return { ok: true, requests }; }, { minRole: "observer", description: "force the next picture to be an IDR with SPS/PPS" });
|
|
45
37
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
let reg;
|
|
50
|
-
if (opts.idrService !== false) {
|
|
51
|
-
let self;
|
|
52
|
-
try {
|
|
53
|
-
reg = await node.serve(idrServiceName(contract.name), () => { if (self)
|
|
54
|
-
self.#idrCalls++; encoder.forceIdr(); return { ok: true }; }, { minRole: "observer", description: `force the next ${contract.name} frame to be an IDR` });
|
|
55
|
-
}
|
|
56
|
-
catch (e) {
|
|
57
|
-
publisher.close();
|
|
58
|
-
throw e;
|
|
59
|
-
}
|
|
60
|
-
self = new VideoPublisher(node, contract, publisher, reg, encoder);
|
|
61
|
-
return self;
|
|
62
|
-
}
|
|
63
|
-
return new VideoPublisher(node, contract, publisher, undefined, encoder);
|
|
38
|
+
catch (e) {
|
|
39
|
+
publisher.close();
|
|
40
|
+
throw e;
|
|
64
41
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
42
|
+
let closed = false;
|
|
43
|
+
const send = publisher;
|
|
44
|
+
const publish = (frame, tUs) => {
|
|
45
|
+
if (closed)
|
|
46
|
+
throw new SynapseError("not_connected", "video publisher closed");
|
|
47
|
+
if (encoder && (encoder.cfg.width !== frame.displayWidth || encoder.cfg.height !== frame.displayHeight)) {
|
|
48
|
+
// TODO(video, D 2026-10-08): one publisher per video topic is assumed, not enforced;
|
|
49
|
+
// resolution and settings do not change while a publisher lives. Codify later.
|
|
50
|
+
throw new SynapseError("invalid_input", "frame size changed; one publisher, one size");
|
|
71
51
|
}
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
}
|
|
84
|
-
const freshWindow = () => ({ t0: performance.now(), frames: 0, bytes: 0, e2e: 0, e2eN: 0, enc: 0, encN: 0, send: 0, sendN: 0, net: 0, netN: 0, dec: 0, decN: 0, eps: 0, epsN: 0 });
|
|
85
|
-
/** A received envelope on a raw media topic as a `MediaFrame` (the header's flags and `STAGES`). */
|
|
86
|
-
export function mediaFrameOf(env) {
|
|
87
|
-
const f = { data: env.data, tUs: env.tUs, key: (env.flags & 1) !== 0, config: (env.flags & 32) !== 0, recovery: (env.flags & 64) !== 0 };
|
|
88
|
-
for (const t of env.tlvs)
|
|
89
|
-
if (t.kind === "stages") {
|
|
90
|
-
f.stages = { encodedUs: Number(t.encodedUs), sentUs: Number(t.sentUs) };
|
|
91
|
-
break;
|
|
52
|
+
if (!encoder) {
|
|
53
|
+
encoder = new H264Encoder({ width: frame.displayWidth, height: frame.displayHeight, ...opts, onError: e => node.warn(`video ${contract.name}: encoder: ${e instanceof Error ? e.message : String(e)}`) });
|
|
54
|
+
// flags and `STAGES` as the encoder made them; `sentUs` 0 is stamped at the write
|
|
55
|
+
encoder.attachSink(f => {
|
|
56
|
+
try {
|
|
57
|
+
send[PUBLISH_MEDIA_FRAME](f);
|
|
58
|
+
}
|
|
59
|
+
catch (e) {
|
|
60
|
+
node.warn(`video ${contract.name}: publish failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
61
|
+
}
|
|
62
|
+
});
|
|
92
63
|
}
|
|
93
|
-
|
|
64
|
+
encoder.encode(frame, tUs === undefined ? {} : { tUs });
|
|
65
|
+
};
|
|
66
|
+
return {
|
|
67
|
+
publish,
|
|
68
|
+
pumpTrack: (track, o = {}) => pumpTrack(track, publish, o),
|
|
69
|
+
close() {
|
|
70
|
+
if (closed)
|
|
71
|
+
return;
|
|
72
|
+
closed = true;
|
|
73
|
+
encoder?.close();
|
|
74
|
+
idr.unregister();
|
|
75
|
+
publisher.close();
|
|
76
|
+
},
|
|
77
|
+
get encoder() { return encoder; },
|
|
78
|
+
};
|
|
94
79
|
}
|
|
95
80
|
/**
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
81
|
+
* Decoded pictures of the video topic `topic` (by name; the established descriptor says what it
|
|
82
|
+
* carries). Every frame is fed to the decoder in order and every picture it gives is delivered.
|
|
83
|
+
* Frames before a start point are dropped by the decoder's policy, which asks the publisher for an
|
|
84
|
+
* IDR when stuck or waiting. Throws `not_supported` where WebCodecs is missing.
|
|
100
85
|
*/
|
|
101
|
-
export
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
sub;
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
#closed = false;
|
|
116
|
-
constructor(node, topic, decoder, opts = {}) {
|
|
117
|
-
this.node = node;
|
|
118
|
-
this.decoder = decoder;
|
|
119
|
-
this.opts = opts;
|
|
120
|
-
this.topic = rawContract(topic, "VideoSubscriber");
|
|
121
|
-
const prevIdr = decoder.onRequestIdr;
|
|
122
|
-
const onIdr = () => { prevIdr?.(); this.#requestIdr("stuck"); };
|
|
123
|
-
decoder.onRequestIdr = onIdr;
|
|
124
|
-
const prevOut = decoder.onOutput;
|
|
125
|
-
const onOut = (meta) => { prevOut?.(meta); this.#onOutput(meta); };
|
|
126
|
-
decoder.onOutput = onOut;
|
|
127
|
-
// `close()` puts the decoder's previous callbacks back when ours are still the ones installed.
|
|
128
|
-
this.#restore = () => {
|
|
129
|
-
if (decoder.onRequestIdr === onIdr)
|
|
130
|
-
decoder.onRequestIdr = prevIdr;
|
|
131
|
-
if (decoder.onOutput === onOut)
|
|
132
|
-
decoder.onOutput = prevOut;
|
|
133
|
-
};
|
|
134
|
-
// Every frame reaches the decoder (queueDepth 0): the decoder's own queue sheds, in picture order.
|
|
135
|
-
this.sub = node.subscribe(this.topic, { priority: opts.priority ?? 2, queueDepth: 0 });
|
|
136
|
-
void this.#read();
|
|
137
|
-
}
|
|
138
|
-
async #read() {
|
|
139
|
-
for (;;) {
|
|
140
|
-
const batch = await this.sub.next();
|
|
141
|
-
if (!batch || this.#closed)
|
|
142
|
-
return;
|
|
143
|
-
for (const env of batch)
|
|
144
|
-
this.#onEnvelope(env);
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
#onEnvelope(env) {
|
|
148
|
-
if (this.#closed)
|
|
86
|
+
export function videoSubscriber(node, topic, opts = {}) {
|
|
87
|
+
const name = typeof topic === "string" ? topic : topic.name;
|
|
88
|
+
const idrService = `${name}/idr`;
|
|
89
|
+
const decoder = new H264Decoder();
|
|
90
|
+
// every frame reaches the decoder (a P picture references the one before it)
|
|
91
|
+
const sub = node.subscribe(name, { queueDepth: 0 });
|
|
92
|
+
const pending = [];
|
|
93
|
+
let wake;
|
|
94
|
+
let ended = false;
|
|
95
|
+
const signal = () => { const w = wake; wake = undefined; w?.(); };
|
|
96
|
+
let idrInFlight = false;
|
|
97
|
+
// `<topic>/idr` unless a call is still in flight; a failure is the policy's business (it asks again after another stuck window).
|
|
98
|
+
decoder.onRequestIdr = () => {
|
|
99
|
+
if (idrInFlight || ended)
|
|
149
100
|
return;
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
this.#win.bytes += f.data.length;
|
|
154
|
-
const recvUs = nowUs();
|
|
155
|
-
// Stamps within the publisher's clock need no conversion; those compared with ours go through `toLocalUs` (SDK §4.4).
|
|
156
|
-
if (f.stages) {
|
|
157
|
-
if (f.stages.encodedUs > 0 && f.tUs > 0) {
|
|
158
|
-
this.#win.enc += (f.stages.encodedUs - f.tUs) / 1000;
|
|
159
|
-
this.#win.encN++;
|
|
160
|
-
}
|
|
161
|
-
if (f.stages.sentUs > 0 && f.stages.encodedUs > 0) {
|
|
162
|
-
this.#win.send += (f.stages.sentUs - f.stages.encodedUs) / 1000;
|
|
163
|
-
this.#win.sendN++;
|
|
164
|
-
}
|
|
165
|
-
if (f.stages.sentUs > 0) {
|
|
166
|
-
const sent = this.node.toLocalUs(f.stages.sentUs, env.dom);
|
|
167
|
-
if (sent) {
|
|
168
|
-
this.#win.net += (recvUs - sent.localUs) / 1000;
|
|
169
|
-
this.#win.netN++;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
if (f.tUs > 0) {
|
|
174
|
-
const capture = this.node.toLocalUs(f.tUs, env.dom);
|
|
175
|
-
if (capture) {
|
|
176
|
-
this.#captureLocal.set(f, capture);
|
|
177
|
-
this.#win.eps += capture.epsUs / 1000;
|
|
178
|
-
this.#win.epsN++;
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
const droppedBefore = this.decoder.sync.stats.dropped;
|
|
182
|
-
this.decoder.push(f);
|
|
183
|
-
if (this.decoder.sync.stats.dropped > droppedBefore) {
|
|
184
|
-
// Waiting for a start point: a browser publisher emits IDRs only on demand, so ask (first frame, then every stuck window).
|
|
185
|
-
this.#waitingDropped++;
|
|
186
|
-
if (this.#waitingDropped === 1 || this.#waitingDropped % this.decoder.sync.cfg.stuckFrames === 0)
|
|
187
|
-
this.#requestIdr("waiting");
|
|
188
|
-
}
|
|
189
|
-
else
|
|
190
|
-
this.#waitingDropped = 0;
|
|
191
|
-
}
|
|
192
|
-
#onOutput(meta) {
|
|
193
|
-
this.#win.dec += meta.decodeMs;
|
|
194
|
-
this.#win.decN++;
|
|
195
|
-
const capture = this.#captureLocal.get(meta.frame);
|
|
196
|
-
if (capture) {
|
|
197
|
-
this.#win.e2e += (nowUs() - capture.localUs) / 1000;
|
|
198
|
-
this.#win.e2eN++;
|
|
101
|
+
let run;
|
|
102
|
+
try {
|
|
103
|
+
run = node.call(idrService, {}, { deadlineMs: 1000 });
|
|
199
104
|
}
|
|
200
|
-
|
|
201
|
-
#requestIdr(why) {
|
|
202
|
-
if (this.opts.requestIdr === false || this.#closed)
|
|
105
|
+
catch {
|
|
203
106
|
return;
|
|
204
|
-
|
|
205
|
-
|
|
107
|
+
} // no session: the next window asks again
|
|
108
|
+
idrInFlight = true;
|
|
109
|
+
run.result.then(() => { }, () => { }).finally(() => { idrInFlight = false; });
|
|
110
|
+
};
|
|
111
|
+
decoder.attachSink((vf, meta) => {
|
|
112
|
+
if (ended) {
|
|
113
|
+
vf.close();
|
|
206
114
|
return;
|
|
207
|
-
|
|
208
|
-
|
|
115
|
+
}
|
|
116
|
+
if (!opts.onFrame) {
|
|
117
|
+
pending.push(vf);
|
|
118
|
+
signal();
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
const env = meta.frame;
|
|
122
|
+
const m = "seq" in env ? env : { data: env.data, tUs: env.tUs, seq: 0, publisher: "", dom: "", flags: 0, type: 0, tlvs: [], key: env.key, config: env.config, recovery: env.recovery };
|
|
209
123
|
try {
|
|
210
|
-
|
|
124
|
+
opts.onFrame(vf, m);
|
|
211
125
|
}
|
|
212
|
-
|
|
213
|
-
|
|
126
|
+
finally {
|
|
127
|
+
vf.close();
|
|
214
128
|
}
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
stats() {
|
|
219
|
-
const w = this.#win;
|
|
220
|
-
const dt = (performance.now() - w.t0) / 1000;
|
|
221
|
-
const avg = (sum, n) => (n ? sum / n : undefined);
|
|
222
|
-
const d = this.decoder.stats();
|
|
223
|
-
const superseded = this.sub.stats.superseded;
|
|
224
|
-
const s = {
|
|
225
|
-
fps: dt > 0 ? (d.output - this.#lastOutput) / dt : 0, kbps: dt > 0 ? (w.bytes * 8) / dt / 1000 : 0,
|
|
226
|
-
received: this.#received, decoded: d.output, dropped: d.dropped + superseded, superseded, resets: d.resets, idrRequests: this.#idrRequests,
|
|
227
|
-
queue: d.queue, width: d.width, height: d.height, state: d.state,
|
|
228
|
-
latency: { e2eMs: avg(w.e2e, w.e2eN), encMs: avg(w.enc, w.encN), sendMs: avg(w.send, w.sendN), netMs: avg(w.net, w.netN), decodeMs: avg(w.dec, w.decN), epsMs: avg(w.eps, w.epsN) },
|
|
229
|
-
};
|
|
230
|
-
this.#lastOutput = d.output;
|
|
231
|
-
this.#win = freshWindow();
|
|
232
|
-
return s;
|
|
233
|
-
}
|
|
234
|
-
#lastOutput = 0;
|
|
235
|
-
/** IDR calls that failed (no publisher serving `<topic>/idr`, or timed out). */
|
|
236
|
-
get idrFailed() { return this.#idrFailed; }
|
|
237
|
-
/**
|
|
238
|
-
* Stops the subscription and gives the decoder back its previous callbacks (the decoder stays
|
|
239
|
-
* yours to close). Close subscribers sharing one decoder newest first: a subscriber whose
|
|
240
|
-
* callbacks another one wrapped since cannot take them back, so the newer one still calls them.
|
|
241
|
-
*/
|
|
242
|
-
close() {
|
|
243
|
-
if (this.#closed)
|
|
129
|
+
});
|
|
130
|
+
const close = () => {
|
|
131
|
+
if (ended)
|
|
244
132
|
return;
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
133
|
+
ended = true;
|
|
134
|
+
sub.unsubscribe();
|
|
135
|
+
decoder.close();
|
|
136
|
+
for (const vf of pending.splice(0))
|
|
137
|
+
vf.close();
|
|
138
|
+
signal();
|
|
139
|
+
};
|
|
140
|
+
void (async () => {
|
|
141
|
+
for (;;) {
|
|
142
|
+
const batch = await sub.next();
|
|
143
|
+
if (!batch || ended)
|
|
144
|
+
break;
|
|
145
|
+
// an envelope is the decoder's `MediaFrame` (data, tUs, key, config, recovery): the picture's meta is the envelope itself
|
|
146
|
+
for (const env of batch)
|
|
147
|
+
if (env.data instanceof Uint8Array)
|
|
148
|
+
decoder.push(env);
|
|
149
|
+
}
|
|
150
|
+
close();
|
|
151
|
+
})();
|
|
152
|
+
return {
|
|
153
|
+
close,
|
|
154
|
+
async *[Symbol.asyncIterator]() {
|
|
155
|
+
try {
|
|
156
|
+
for (;;) {
|
|
157
|
+
const vf = pending.shift();
|
|
158
|
+
if (vf) {
|
|
159
|
+
yield vf;
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
if (ended)
|
|
163
|
+
return;
|
|
164
|
+
await new Promise(r => { wake = r; });
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
close();
|
|
169
|
+
}
|
|
170
|
+
},
|
|
171
|
+
};
|
|
250
172
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@synapse-sdk/media",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.3",
|
|
4
4
|
"description": "Synapse SDK media: H.264 video in the browser — encoder, decoder, canvas sink, video publisher/subscriber and the React useVideo hook.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -18,11 +18,11 @@
|
|
|
18
18
|
"dist"
|
|
19
19
|
],
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@synapse-sdk/core": "0.3.
|
|
22
|
-
"@synapse-sdk/client": "0.3.
|
|
21
|
+
"@synapse-sdk/core": "0.3.3",
|
|
22
|
+
"@synapse-sdk/client": "0.3.3"
|
|
23
23
|
},
|
|
24
24
|
"peerDependencies": {
|
|
25
|
-
"@synapse-sdk/react": "0.3.
|
|
25
|
+
"@synapse-sdk/react": "0.3.3",
|
|
26
26
|
"react": ">=18"
|
|
27
27
|
},
|
|
28
28
|
"peerDependenciesMeta": {
|
package/dist/annexb.d.ts
DELETED
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
import type { MediaFrame } from "@synapse-sdk/core";
|
|
2
|
-
/** NAL unit types this package cares about (H.264 Table 7-1). */
|
|
3
|
-
export declare const NAL_TYPE: {
|
|
4
|
-
readonly SLICE: 1;
|
|
5
|
-
readonly SLICE_DPA: 2;
|
|
6
|
-
readonly SLICE_DPB: 3;
|
|
7
|
-
readonly SLICE_DPC: 4;
|
|
8
|
-
readonly IDR: 5;
|
|
9
|
-
readonly SEI: 6;
|
|
10
|
-
readonly SPS: 7;
|
|
11
|
-
readonly PPS: 8;
|
|
12
|
-
readonly AUD: 9;
|
|
13
|
-
};
|
|
14
|
-
/** SEI payload type of a recovery point (H.264 D.1.8). */
|
|
15
|
-
export declare const SEI_RECOVERY_POINT = 6;
|
|
16
|
-
/** One NAL unit inside an Annex-B buffer. */
|
|
17
|
-
export interface Nal {
|
|
18
|
-
/** Offset of the start code in the buffer. */
|
|
19
|
-
offset: number;
|
|
20
|
-
/** 3 or 4 (bytes of `00 00 01` / `00 00 00 01`). */
|
|
21
|
-
startCodeLen: number;
|
|
22
|
-
/** The NAL unit bytes after the start code: header byte, then the payload (still escaped). A view, not a copy. */
|
|
23
|
-
bytes: Uint8Array;
|
|
24
|
-
}
|
|
25
|
-
/** `nal_unit_type` (low 5 bits of the header byte); 0 for an empty NAL. */
|
|
26
|
-
export declare function nalType(n: Nal): number;
|
|
27
|
-
/** Payload after the 1-byte NAL header (escaped RBSP). */
|
|
28
|
-
export declare function nalPayload(n: Nal): Uint8Array;
|
|
29
|
-
/** A VCL NAL type: a coded slice (types 1-5). */
|
|
30
|
-
export declare function isVclType(t: number): boolean;
|
|
31
|
-
export declare function nalIsVcl(n: Nal): boolean;
|
|
32
|
-
/** Start code plus NAL unit, as it sits in the stream (a view). */
|
|
33
|
-
export declare function nalWithStartCode(buf: Uint8Array, n: Nal): Uint8Array;
|
|
34
|
-
/** Next Annex-B start code (`00 00 01` or `00 00 00 01`) at or after `from`: `[offset, len]`. */
|
|
35
|
-
export declare function findStartCode(buf: Uint8Array, from: number): [offset: number, len: number] | undefined;
|
|
36
|
-
/**
|
|
37
|
-
* The NAL units of an Annex-B buffer, in order. Bytes before the first start code are skipped;
|
|
38
|
-
* trailing zero bytes (`trailing_zero_8bits`) stay attached to the NAL before them, which is
|
|
39
|
-
* harmless for type/flag decisions and keeps the bytes of a walked AU identical to the input.
|
|
40
|
-
*/
|
|
41
|
-
export declare function nalUnits(buf: Uint8Array): Nal[];
|
|
42
|
-
/** The SPS + PPS NAL units of an access unit (with their start codes, in stream order), if it carries any. Defined ⇒ the AU deserves the `CONFIG` flag. */
|
|
43
|
-
export declare function auParamSets(au: Uint8Array): Uint8Array | undefined;
|
|
44
|
-
/**
|
|
45
|
-
* The sync points an Annex-B access unit offers.
|
|
46
|
-
* - `idr`: an IDR slice (NAL type 5) is present → instantaneous start point (`KEY`).
|
|
47
|
-
* - `recovery`: a recovery point SEI (NAL 6, payload type 6) is present → gradual start point,
|
|
48
|
-
* clean one refresh period later (`RECOVERY`; x264 and nvenc both emit it at each intra-refresh sweep).
|
|
49
|
-
* Decided from the bitstream rather than the encoder's keyframe flag, which libx264 raises for
|
|
50
|
-
* both kinds and nvenc only for IDRs.
|
|
51
|
-
*/
|
|
52
|
-
export declare function auSyncPoints(au: Uint8Array): {
|
|
53
|
-
idr: boolean;
|
|
54
|
-
recovery: boolean;
|
|
55
|
-
};
|
|
56
|
-
/** An IDR is present: the AU is a keyframe in the `KEY` sense. */
|
|
57
|
-
export declare function isKeyframe(au: Uint8Array): boolean;
|
|
58
|
-
/** The AU carries at least one coded slice (otherwise it is parameter sets / SEI only). */
|
|
59
|
-
export declare function hasVcl(au: Uint8Array): boolean;
|
|
60
|
-
/** Whether an SEI NAL payload (after the NAL header byte) contains a recovery point message (payload type 6). */
|
|
61
|
-
export declare function seiHasRecoveryPoint(payload: Uint8Array): boolean;
|
|
62
|
-
/** The payload types of the SEI messages in an SEI NAL payload (D.1.1), in order. Emulation-prevention bytes are removed first. */
|
|
63
|
-
export declare function seiPayloadTypes(payload: Uint8Array): number[];
|
|
64
|
-
/** Strip H.264 emulation-prevention bytes (the `03` in `00 00 03`), giving the RBSP. */
|
|
65
|
-
export declare function rbspUnescape(nal: Uint8Array): Uint8Array;
|
|
66
|
-
/** The SPS of an AU (NAL bytes without start code), if present. */
|
|
67
|
-
export declare function findSps(au: Uint8Array): Uint8Array | undefined;
|
|
68
|
-
/** The WebCodecs / RFC 6381 codec string for an AU that carries an SPS: `avc1.` + profile_idc + constraint flags + level_idc as hex (e.g. `avc1.42c01e`). */
|
|
69
|
-
export declare function codecStringFromSps(au: Uint8Array): string | undefined;
|
|
70
|
-
/** `parts` joined into one new buffer (a single part is copied too, so the result never aliases the input). */
|
|
71
|
-
export declare function concatBytes(parts: readonly Uint8Array[]): Uint8Array;
|
|
72
|
-
/**
|
|
73
|
-
* Splits a continuous Annex-B byte stream (a file, an encoder pipe) into access units. Rule
|
|
74
|
-
* (dylithium's reader loop): an AU ends with its coded slice; the next NAL of any kind starts a
|
|
75
|
-
* new AU, so non-VCL NALs (SPS, PPS, SEI, AUD) are attached to the picture that follows them.
|
|
76
|
-
* Sufficient for one-slice-per-picture streams (x264 `slices=1`, nvenc, WebCodecs), which is what
|
|
77
|
-
* the encoder options enforce.
|
|
78
|
-
*/
|
|
79
|
-
export declare class AuSplitter {
|
|
80
|
-
#private;
|
|
81
|
-
/** Feeds bytes; returns every access unit completed by them (in order). */
|
|
82
|
-
push(bytes: Uint8Array): Uint8Array[];
|
|
83
|
-
/** End of stream: the last NAL (whose end no start code marks) closes the last AU, or starts one more if the current AU already has its slice. */
|
|
84
|
-
finish(): Uint8Array[];
|
|
85
|
-
}
|
|
86
|
-
/** Splits a whole Annex-B buffer into access units (see {@link AuSplitter}). */
|
|
87
|
-
export declare function splitAccessUnits(stream: Uint8Array): Uint8Array[];
|
|
88
|
-
/**
|
|
89
|
-
* A `MediaFrame` for an H.264 Annex-B access unit with `key`/`config`/`recovery` derived from the
|
|
90
|
-
* NAL units (IDR present / SPS+PPS present / recovery-point SEI present), so the header flags
|
|
91
|
-
* always agree with the bytes (R27 §2 publisher rule). `tUs` is the capture time (0 = stamp at publish).
|
|
92
|
-
*/
|
|
93
|
-
export declare function mediaFrameFromAnnexb(au: Uint8Array, tUs?: number): MediaFrame;
|