@reactor-team/js-sdk 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -0
- package/dist/index.cjs +296 -25
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +98 -4
- package/dist/index.d.ts +98 -4
- package/dist/index.js +296 -25
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +5 -2
- package/dist/wasm/reactor_wasm.d.ts +61 -1
- package/dist/wasm/reactor_wasm.js +117 -7
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +6 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -232,6 +232,58 @@ await reactor.step();
|
|
|
232
232
|
`MessageTooLargeError` when the request is too large to send. The frames pushed for a step
|
|
233
233
|
belong to it: when its request cannot be sent, the step rejects and its frames are dropped
|
|
234
234
|
with it, not kept for the next step.
|
|
235
|
+
- A step's result travels in one data-channel message, whose limit is the smaller of the
|
|
236
|
+
client's `maxMessageSize` option (default 64 MiB) and the runtime's, which is 64 MiB unless
|
|
237
|
+
the model raises it in its `reactor.yaml` (`runtime.webrtc.max_message_size`). Raise
|
|
238
|
+
`maxMessageSize` only for a model whose steps need it: a message is held whole in the tab's
|
|
239
|
+
memory while it arrives.
|
|
240
|
+
|
|
241
|
+
### Encoded video in a step
|
|
242
|
+
|
|
243
|
+
A step can carry video already encoded as H.264, its bytes as they are: much smaller than raw
|
|
244
|
+
RGB. The runtime decodes a track's encoded frames before the step runs, so the model reads RGB
|
|
245
|
+
frames exactly as if they had been pushed raw. Give the frames of one step in decode order,
|
|
246
|
+
starting at a keyframe. A WebCodecs `VideoEncoder`'s chunks go in as they are; configure it for
|
|
247
|
+
Annex B, as its default `avc` format is length-prefixed:
|
|
248
|
+
|
|
249
|
+
```ts
|
|
250
|
+
import type { EncodedFrame } from "@reactor-team/js-sdk";
|
|
251
|
+
|
|
252
|
+
let gop: EncodedFrame[] = [];
|
|
253
|
+
const encoder = new VideoEncoder({
|
|
254
|
+
output: (chunk) => {
|
|
255
|
+
if (chunk.type === "key") gop = [];
|
|
256
|
+
gop.push({ codec: "h264", data: chunk, width: 1280, height: 720 });
|
|
257
|
+
},
|
|
258
|
+
error: console.error,
|
|
259
|
+
});
|
|
260
|
+
encoder.configure({ codec: "avc1.42001f", width: 1280, height: 720, avc: { format: "annexb" } });
|
|
261
|
+
|
|
262
|
+
// ... encode frames, then send the ones from the last keyframe with the step:
|
|
263
|
+
const result = await reactor.step({ media: { camera: gop } });
|
|
264
|
+
|
|
265
|
+
// The same, pushed ahead of the step:
|
|
266
|
+
for (const frame of gop) await camera.pushEncodedFrame(frame);
|
|
267
|
+
await reactor.step();
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
- An `EncodedFrame` is `{ codec: "h264", data, width, height, metadata?, captureTimeUs? }`:
|
|
271
|
+
`data` one access unit as a `Uint8Array`, an `ArrayBuffer` or an `EncodedVideoChunk` (its
|
|
272
|
+
bytes are copied out when pushed), `width` and `height` the frame it decodes to. `codec` is
|
|
273
|
+
what tells it from a raw frame in `media`.
|
|
274
|
+
- `media` maps input track names to frames: `EncodedFrame`s, or anything `pushFrame()` takes.
|
|
275
|
+
They go after the frames pushed to the same track since the previous step, and only once
|
|
276
|
+
`state` and `commands` are known to be well formed and the session is ready, so a step
|
|
277
|
+
refused for those leaves the buffer as it was. Each track must be published first. A bad
|
|
278
|
+
track or frame anywhere in `media` fails the step before any of it is pushed, leaving the
|
|
279
|
+
buffer as it was too. A session that leaves ready once the media is in has the step refused
|
|
280
|
+
and keeps the frames for the next one, as pushed frames are. A push or step called after
|
|
281
|
+
`step()` waits for it to take the buffer, so its frames go to the next step, not this one.
|
|
282
|
+
- `pushEncodedFrame()` rejects with an `InvalidStateError` outside a stepped session, on an
|
|
283
|
+
audio or recvonly track, a track not published on this connection, empty data or a zero
|
|
284
|
+
dimension. A step whose frames do not decode, or do not start at a keyframe, rejects with a
|
|
285
|
+
`StepRefusedError` naming the frame in `entry`; a runtime that predates encoded frames
|
|
286
|
+
refuses them the same way.
|
|
235
287
|
|
|
236
288
|
## Recording and clips
|
|
237
289
|
|
package/dist/index.cjs
CHANGED
|
@@ -330,6 +330,52 @@ var Emitter = class {
|
|
|
330
330
|
}
|
|
331
331
|
};
|
|
332
332
|
|
|
333
|
+
// src/internal/encoded-frames.ts
|
|
334
|
+
var CODECS = /* @__PURE__ */ new Set(["h264"]);
|
|
335
|
+
function isEncodedFrame(frame) {
|
|
336
|
+
return typeof frame === "object" && frame !== null && "codec" in frame;
|
|
337
|
+
}
|
|
338
|
+
function isChunk(data) {
|
|
339
|
+
return !(data instanceof Uint8Array) && !(data instanceof ArrayBuffer) && typeof data.copyTo === "function";
|
|
340
|
+
}
|
|
341
|
+
function toEncodedBytes(frame, operation, trackName) {
|
|
342
|
+
const refuse = (why) => new InvalidStateError(`${operation}('${trackName}'): ${why}`, { operation });
|
|
343
|
+
if (typeof frame !== "object" || frame === null) {
|
|
344
|
+
throw refuse("expected an EncodedFrame, { codec, data, width, height }");
|
|
345
|
+
}
|
|
346
|
+
const { codec, data, width, height, metadata, captureTimeUs } = frame;
|
|
347
|
+
if (!CODECS.has(codec)) {
|
|
348
|
+
throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264"`);
|
|
349
|
+
}
|
|
350
|
+
let bytes;
|
|
351
|
+
if (data instanceof Uint8Array) {
|
|
352
|
+
bytes = data;
|
|
353
|
+
} else if (data instanceof ArrayBuffer) {
|
|
354
|
+
bytes = new Uint8Array(data);
|
|
355
|
+
} else if (data && isChunk(data)) {
|
|
356
|
+
bytes = new Uint8Array(data.byteLength);
|
|
357
|
+
data.copyTo(bytes);
|
|
358
|
+
} else {
|
|
359
|
+
throw refuse("data must be a Uint8Array, an ArrayBuffer or an EncodedVideoChunk");
|
|
360
|
+
}
|
|
361
|
+
if (bytes.byteLength === 0) {
|
|
362
|
+
throw refuse("an encoded frame needs data");
|
|
363
|
+
}
|
|
364
|
+
if (!isDimension(width) || !isDimension(height)) {
|
|
365
|
+
throw refuse(`width and height are the decoded frame's, got ${String(width)}x${String(height)}`);
|
|
366
|
+
}
|
|
367
|
+
if (metadata != null && !(metadata instanceof Uint8Array)) {
|
|
368
|
+
throw refuse("metadata must be a Uint8Array");
|
|
369
|
+
}
|
|
370
|
+
if (captureTimeUs != null && !Number.isFinite(captureTimeUs)) {
|
|
371
|
+
throw refuse("captureTimeUs must be a finite number");
|
|
372
|
+
}
|
|
373
|
+
return { codec, data: bytes, width, height, metadata: metadata ?? null, captureTimeUs: captureTimeUs ?? null };
|
|
374
|
+
}
|
|
375
|
+
function isDimension(value) {
|
|
376
|
+
return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 4294967295;
|
|
377
|
+
}
|
|
378
|
+
|
|
333
379
|
// src/file-ref.ts
|
|
334
380
|
var FileRef = class {
|
|
335
381
|
constructor(uploadId, name, mimeType, size) {
|
|
@@ -540,6 +586,67 @@ function toPublicClip(clip) {
|
|
|
540
586
|
};
|
|
541
587
|
}
|
|
542
588
|
|
|
589
|
+
// src/internal/step-media.ts
|
|
590
|
+
function holdStepMedia(media) {
|
|
591
|
+
if (typeof media !== "object" || media === null || Array.isArray(media)) {
|
|
592
|
+
throw new InvalidStateError("step(): media must map track names to arrays of frames", {
|
|
593
|
+
operation: "step"
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
const held = [];
|
|
597
|
+
try {
|
|
598
|
+
for (const [trackName, frames] of Object.entries(media)) {
|
|
599
|
+
if (!Array.isArray(frames)) {
|
|
600
|
+
throw new InvalidStateError(`step(): media['${trackName}'] must be an array of frames`, {
|
|
601
|
+
operation: "step"
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
for (const frame of frames) {
|
|
605
|
+
held.push(holdStepFrame(trackName, frame));
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
} catch (cause) {
|
|
609
|
+
for (const frame of held) {
|
|
610
|
+
frame.release();
|
|
611
|
+
}
|
|
612
|
+
throw cause;
|
|
613
|
+
}
|
|
614
|
+
return held;
|
|
615
|
+
}
|
|
616
|
+
function holdStepFrame(track, frame) {
|
|
617
|
+
if (isEncodedFrame(frame)) {
|
|
618
|
+
const operation = "pushEncodedFrame";
|
|
619
|
+
return { track, kind: "encoded", operation, encoded: toEncodedBytes(frame, operation, track), release: () => {
|
|
620
|
+
} };
|
|
621
|
+
}
|
|
622
|
+
if (typeof frame !== "object" || frame === null) {
|
|
623
|
+
throw new InvalidStateError(`pushFrame('${track}'): expected a frame`, { operation: "pushFrame" });
|
|
624
|
+
}
|
|
625
|
+
const held = holdFrame(frame);
|
|
626
|
+
return { track, kind: "raw", operation: "pushFrame", frame: held.frame, release: held.release };
|
|
627
|
+
}
|
|
628
|
+
async function readStepFrame(frame) {
|
|
629
|
+
const { track } = frame;
|
|
630
|
+
if (frame.kind === "encoded") {
|
|
631
|
+
return { track, kind: "encoded", operation: frame.operation, encoded: frame.encoded };
|
|
632
|
+
}
|
|
633
|
+
if (isAudioFrame(frame.frame)) {
|
|
634
|
+
return { track, kind: "audio", operation: frame.operation, audio: frame.frame };
|
|
635
|
+
}
|
|
636
|
+
return { track, kind: "video", operation: frame.operation, pixels: await toPixels(frame.frame) };
|
|
637
|
+
}
|
|
638
|
+
function checkStepFrame(client, frame) {
|
|
639
|
+
if (frame.kind === "encoded") {
|
|
640
|
+
const { codec, data, width, height, captureTimeUs } = frame.encoded;
|
|
641
|
+
client.checkEncodedVideoFrame(frame.track, codec, data.byteLength, width, height, captureTimeUs);
|
|
642
|
+
} else if (frame.kind === "video") {
|
|
643
|
+
const { data, width, height, format } = frame.pixels;
|
|
644
|
+
client.checkVideoFrame(frame.track, data.byteLength, width, height, format);
|
|
645
|
+
} else {
|
|
646
|
+
client.checkAudioFrame(frame.track, frame.audio.samples, frame.audio.channels ?? 1);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
|
|
543
650
|
// src/internal/stats.ts
|
|
544
651
|
var LOCAL_STATS_INTERVAL_MS = 2e3;
|
|
545
652
|
var RUNTIME_REPORT_INTERVAL_MS = 5e3;
|
|
@@ -867,7 +974,7 @@ function describeCause(cause) {
|
|
|
867
974
|
// package.json
|
|
868
975
|
var package_default = {
|
|
869
976
|
name: "@reactor-team/js-sdk",
|
|
870
|
-
version: "3.
|
|
977
|
+
version: "3.4.0",
|
|
871
978
|
description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
|
|
872
979
|
license: "Apache-2.0",
|
|
873
980
|
repository: {
|
|
@@ -1389,9 +1496,10 @@ var Reactor = class {
|
|
|
1389
1496
|
* is finalized on "ready" — see `handleStatusChanged()`. */
|
|
1390
1497
|
connectStartTime;
|
|
1391
1498
|
waitingStartTime;
|
|
1392
|
-
/** The tail of every `pushFrame()` so far, settled
|
|
1393
|
-
* reach the binding in call order however long each
|
|
1394
|
-
* read, and `step()` waits for the ones called
|
|
1499
|
+
/** The tail of every `pushFrame()`/`pushEncodedFrame()` so far, settled
|
|
1500
|
+
* either way — pushes reach the binding in call order however long each
|
|
1501
|
+
* one's pixels take to read, and `step()` waits for the ones called
|
|
1502
|
+
* before it. */
|
|
1395
1503
|
pendingPushes = Promise.resolve();
|
|
1396
1504
|
emitter = new Emitter();
|
|
1397
1505
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
@@ -1732,7 +1840,8 @@ var Reactor = class {
|
|
|
1732
1840
|
}, "publishTrack");
|
|
1733
1841
|
return {
|
|
1734
1842
|
name,
|
|
1735
|
-
pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
|
|
1843
|
+
pushFrame: (frame, options) => this.pushTo(name, publication, frame, options),
|
|
1844
|
+
pushEncodedFrame: (frame) => this.pushEncodedTo(name, publication, frame)
|
|
1736
1845
|
};
|
|
1737
1846
|
} catch (cause) {
|
|
1738
1847
|
throw this.captureError(cause);
|
|
@@ -1860,6 +1969,20 @@ var Reactor = class {
|
|
|
1860
1969
|
* when its request cannot be sent, they are dropped with it rather than
|
|
1861
1970
|
* kept for the next step.
|
|
1862
1971
|
*
|
|
1972
|
+
* `media` gives the step frames of its own, by input track name, pushed
|
|
1973
|
+
* after the ones already pushed to that track: what `pushFrame()` takes,
|
|
1974
|
+
* or an `EncodedFrame` — H.264 the runtime decodes before the step runs,
|
|
1975
|
+
* so the model reads RGB frames as for a raw push. They are pushed only
|
|
1976
|
+
* once `state` and `commands` are well formed and the session is ready, so
|
|
1977
|
+
* a step refused for those leaves the frames buffered as they were. They go
|
|
1978
|
+
* all or nothing: a bad track or frame anywhere in `media` rejects the step
|
|
1979
|
+
* before any of it is pushed or sent, leaving the buffer as it was. A
|
|
1980
|
+
* session that leaves 'ready' once the media is in has the step refused and
|
|
1981
|
+
* keeps the frames for the next one, as pushed frames are.
|
|
1982
|
+
* A track's encoded frames of one step go in decode order, starting at a
|
|
1983
|
+
* keyframe; a step whose frames do not decode rejects with a
|
|
1984
|
+
* `StepRefusedError`.
|
|
1985
|
+
*
|
|
1863
1986
|
* ```ts
|
|
1864
1987
|
* const camera = await reactor.publishTrack('camera');
|
|
1865
1988
|
* await camera.pushFrame(imageData);
|
|
@@ -1873,14 +1996,31 @@ var Reactor = class {
|
|
|
1873
1996
|
this.assertNotDisposed();
|
|
1874
1997
|
try {
|
|
1875
1998
|
const state = rewriteNestedFileRefs(options.state ?? null);
|
|
1876
|
-
const commands = (options.commands ?? [])
|
|
1877
|
-
|
|
1878
|
-
);
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
return client.step(state, commands);
|
|
1999
|
+
const commands = rewriteNestedFileRefs(options.commands ?? []);
|
|
2000
|
+
const previous = this.pendingPushes;
|
|
2001
|
+
const before = options.media !== void 0 ? this.pushStepMedia(options.media, state, commands) : previous;
|
|
2002
|
+
let taken;
|
|
2003
|
+
const bufferTaken = new Promise((resolve) => {
|
|
2004
|
+
taken = resolve;
|
|
1883
2005
|
});
|
|
2006
|
+
const stepping = before.then(
|
|
2007
|
+
() => (
|
|
2008
|
+
// Excluded from connect()/disconnect()/reconnect() like sendCommand():
|
|
2009
|
+
// a step can take minutes, and must neither be freed out from under
|
|
2010
|
+
// nor block unrelated reads.
|
|
2011
|
+
this.withReaderAccess(async () => {
|
|
2012
|
+
const client = await this.getOrCreateClient();
|
|
2013
|
+
try {
|
|
2014
|
+
return client.step(state, commands);
|
|
2015
|
+
} finally {
|
|
2016
|
+
taken();
|
|
2017
|
+
}
|
|
2018
|
+
})
|
|
2019
|
+
)
|
|
2020
|
+
);
|
|
2021
|
+
stepping.catch(() => previous.then(taken));
|
|
2022
|
+
this.pendingPushes = bufferTaken;
|
|
2023
|
+
return await stepping;
|
|
1884
2024
|
} catch (cause) {
|
|
1885
2025
|
throw this.captureError(cause);
|
|
1886
2026
|
}
|
|
@@ -1939,19 +2079,7 @@ var Reactor = class {
|
|
|
1939
2079
|
held.release();
|
|
1940
2080
|
}
|
|
1941
2081
|
await this.withReaderAccess(async () => {
|
|
1942
|
-
|
|
1943
|
-
throw new InvalidStateError(
|
|
1944
|
-
`pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
|
|
1945
|
-
{ operation: "pushFrame" }
|
|
1946
|
-
);
|
|
1947
|
-
}
|
|
1948
|
-
const current = this.publishedInputs.get(trackName);
|
|
1949
|
-
if (current === void 0 || publication !== void 0 && publication !== current) {
|
|
1950
|
-
throw new InvalidStateError(
|
|
1951
|
-
publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
|
|
1952
|
-
{ operation: "pushFrame" }
|
|
1953
|
-
);
|
|
1954
|
-
}
|
|
2082
|
+
this.checkPushTarget("pushFrame", trackName, publication, epoch);
|
|
1955
2083
|
const client = await this.getOrCreateClient();
|
|
1956
2084
|
if (pixels) {
|
|
1957
2085
|
client.pushVideoFrame(
|
|
@@ -1975,6 +2103,149 @@ var Reactor = class {
|
|
|
1975
2103
|
throw this.captureError(cause);
|
|
1976
2104
|
}
|
|
1977
2105
|
}
|
|
2106
|
+
/**
|
|
2107
|
+
* Buffers one encoded video frame for the next `step()` on the sendonly
|
|
2108
|
+
* video track `trackName`, its bytes as they are: one H.264 access unit
|
|
2109
|
+
* (Annex B), with the `width` and `height` it decodes to. The
|
|
2110
|
+
* `PublishedTrack` that `publishTrack()` resolves with does the same
|
|
2111
|
+
* through `track.pushEncodedFrame(frame)`.
|
|
2112
|
+
*
|
|
2113
|
+
* The runtime decodes a track's encoded frames of one step as one stream,
|
|
2114
|
+
* before the step runs, and the model reads the RGB frames as if they had
|
|
2115
|
+
* been pushed raw. So push them in decode order, starting at a keyframe; a
|
|
2116
|
+
* step whose frames do not decode rejects with a `StepRefusedError`, as
|
|
2117
|
+
* does a runtime that predates encoded frames. Encoded and raw pushes keep
|
|
2118
|
+
* their call order, and a `step()` waits for every push called before it.
|
|
2119
|
+
*
|
|
2120
|
+
* ```ts
|
|
2121
|
+
* const encoder = new VideoEncoder({
|
|
2122
|
+
* output: (chunk) => void camera.pushEncodedFrame({ codec: 'h264', data: chunk, width, height }),
|
|
2123
|
+
* error: console.error,
|
|
2124
|
+
* });
|
|
2125
|
+
* encoder.configure({ codec: 'avc1.42001f', width, height, avc: { format: 'annexb' } });
|
|
2126
|
+
* ```
|
|
2127
|
+
*
|
|
2128
|
+
* Rejects with an `InvalidStateError` outside a stepped session, for a
|
|
2129
|
+
* track not published on this connection, for an audio or recvonly track,
|
|
2130
|
+
* for empty data, a zero dimension or a codec other than `"h264"`.
|
|
2131
|
+
*/
|
|
2132
|
+
async pushEncodedFrame(trackName, frame) {
|
|
2133
|
+
return this.pushEncodedTo(trackName, void 0, frame);
|
|
2134
|
+
}
|
|
2135
|
+
/** `pushEncodedFrame()`, for the track's current publication or, from a
|
|
2136
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
2137
|
+
async pushEncodedTo(trackName, publication, frame) {
|
|
2138
|
+
const operation = "pushEncodedFrame";
|
|
2139
|
+
this.assertNotDisposed();
|
|
2140
|
+
const epoch = this.connectionEpoch;
|
|
2141
|
+
let encoded;
|
|
2142
|
+
try {
|
|
2143
|
+
encoded = toEncodedBytes(frame, operation, trackName);
|
|
2144
|
+
} catch (cause) {
|
|
2145
|
+
throw this.captureError(cause);
|
|
2146
|
+
}
|
|
2147
|
+
const push = this.pendingPushes.then(
|
|
2148
|
+
() => this.withReaderAccess(async () => {
|
|
2149
|
+
if (!this.isStepped()) {
|
|
2150
|
+
throw new InvalidStateError(
|
|
2151
|
+
`${operation}('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
|
|
2152
|
+
{ operation }
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
this.checkPushTarget(operation, trackName, publication, epoch);
|
|
2156
|
+
const client = await this.getOrCreateClient();
|
|
2157
|
+
const declared = client.tracks().find((track) => track.name === trackName);
|
|
2158
|
+
if (declared && (declared.kind !== "video" || declared.direction !== "sendonly")) {
|
|
2159
|
+
throw new InvalidStateError(
|
|
2160
|
+
`${operation}('${trackName}'): '${trackName}' is a ${declared.direction} ${declared.kind} track; only a sendonly video track takes encoded frames.`,
|
|
2161
|
+
{ operation }
|
|
2162
|
+
);
|
|
2163
|
+
}
|
|
2164
|
+
client.pushEncodedVideoFrame(
|
|
2165
|
+
trackName,
|
|
2166
|
+
encoded.codec,
|
|
2167
|
+
encoded.data,
|
|
2168
|
+
encoded.width,
|
|
2169
|
+
encoded.height,
|
|
2170
|
+
encoded.metadata,
|
|
2171
|
+
encoded.captureTimeUs
|
|
2172
|
+
);
|
|
2173
|
+
})
|
|
2174
|
+
);
|
|
2175
|
+
this.pendingPushes = push.catch(() => void 0);
|
|
2176
|
+
try {
|
|
2177
|
+
await push;
|
|
2178
|
+
} catch (cause) {
|
|
2179
|
+
throw this.captureError(cause);
|
|
2180
|
+
}
|
|
2181
|
+
}
|
|
2182
|
+
/** Refuses a push whose connection ended since it was called, or whose
|
|
2183
|
+
* track is not (or no longer) published on it. */
|
|
2184
|
+
checkPushTarget(operation, trackName, publication, epoch) {
|
|
2185
|
+
if (epoch !== this.connectionEpoch) {
|
|
2186
|
+
throw new InvalidStateError(
|
|
2187
|
+
`${operation}('${trackName}'): the connection it was pushed to ended while its frame was read`,
|
|
2188
|
+
{ operation }
|
|
2189
|
+
);
|
|
2190
|
+
}
|
|
2191
|
+
const current = this.publishedInputs.get(trackName);
|
|
2192
|
+
if (current === void 0 || publication !== void 0 && publication !== current) {
|
|
2193
|
+
throw new InvalidStateError(
|
|
2194
|
+
publication === void 0 ? `${operation}('${trackName}') needs publishTrack('${trackName}') first.` : `${operation}('${trackName}'): this track was unpublished, or its connection ended.`,
|
|
2195
|
+
{ operation }
|
|
2196
|
+
);
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
/** Pushes `step()`'s `media`, track by track and in order, each frame as
|
|
2200
|
+
* `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
|
|
2201
|
+
* checks the step's `state` and `commands`, then every frame, before the
|
|
2202
|
+
* first is pushed, so a refused step, or a bad track or frame anywhere,
|
|
2203
|
+
* leaves the buffer as it was. */
|
|
2204
|
+
async pushStepMedia(media, state, commands) {
|
|
2205
|
+
const epoch = this.connectionEpoch;
|
|
2206
|
+
const held = holdStepMedia(media);
|
|
2207
|
+
const push = this.pendingPushes.then(async () => {
|
|
2208
|
+
const ready = [];
|
|
2209
|
+
try {
|
|
2210
|
+
for (const frame of held) {
|
|
2211
|
+
ready.push(await readStepFrame(frame));
|
|
2212
|
+
}
|
|
2213
|
+
} finally {
|
|
2214
|
+
for (const frame of held) {
|
|
2215
|
+
frame.release();
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2218
|
+
await this.withReaderAccess(async () => {
|
|
2219
|
+
const client = await this.getOrCreateClient();
|
|
2220
|
+
client.checkStep(state, commands);
|
|
2221
|
+
for (const frame of ready) {
|
|
2222
|
+
this.checkPushTarget(frame.operation, frame.track, void 0, epoch);
|
|
2223
|
+
checkStepFrame(client, frame);
|
|
2224
|
+
}
|
|
2225
|
+
for (const frame of ready) {
|
|
2226
|
+
if (frame.kind === "encoded") {
|
|
2227
|
+
client.pushEncodedVideoFrame(
|
|
2228
|
+
frame.track,
|
|
2229
|
+
frame.encoded.codec,
|
|
2230
|
+
frame.encoded.data,
|
|
2231
|
+
frame.encoded.width,
|
|
2232
|
+
frame.encoded.height,
|
|
2233
|
+
frame.encoded.metadata,
|
|
2234
|
+
frame.encoded.captureTimeUs
|
|
2235
|
+
);
|
|
2236
|
+
} else if (frame.kind === "video") {
|
|
2237
|
+
const { data, width, height, format } = frame.pixels;
|
|
2238
|
+
client.pushVideoFrame(frame.track, data, width, height, format, null, null);
|
|
2239
|
+
} else {
|
|
2240
|
+
const { samples, sampleRate, channels } = frame.audio;
|
|
2241
|
+
client.pushAudioFrame(frame.track, samples, sampleRate, channels ?? 1);
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
});
|
|
2245
|
+
});
|
|
2246
|
+
this.pendingPushes = push.catch(() => void 0);
|
|
2247
|
+
await push;
|
|
2248
|
+
}
|
|
1978
2249
|
/** Drops every frame buffered for the next step, pushes called before it
|
|
1979
2250
|
* included: it waits its turn behind them, as a `step()` does. */
|
|
1980
2251
|
async clearStepInputs() {
|