@reactor-team/js-sdk 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -21
- package/dist/index.cjs +742 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +182 -66
- package/dist/index.d.ts +182 -66
- package/dist/index.js +742 -45
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +36 -12
- package/dist/wasm/reactor_wasm.d.ts +93 -20
- package/dist/wasm/reactor_wasm.js +114 -22
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +6 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -331,7 +331,7 @@ var Emitter = class {
|
|
|
331
331
|
};
|
|
332
332
|
|
|
333
333
|
// src/internal/encoded-frames.ts
|
|
334
|
-
var CODECS = /* @__PURE__ */ new Set(["h264"]);
|
|
334
|
+
var CODECS = /* @__PURE__ */ new Set(["h264", "vp8", "vp9", "av1"]);
|
|
335
335
|
function isEncodedFrame(frame) {
|
|
336
336
|
return typeof frame === "object" && frame !== null && "codec" in frame;
|
|
337
337
|
}
|
|
@@ -343,9 +343,12 @@ function toEncodedBytes(frame, operation, trackName) {
|
|
|
343
343
|
if (typeof frame !== "object" || frame === null) {
|
|
344
344
|
throw refuse("expected an EncodedFrame, { codec, data, width, height }");
|
|
345
345
|
}
|
|
346
|
-
const { codec, data, width, height, metadata, captureTimeUs } = frame;
|
|
346
|
+
const { codec, data, width, height, metadata, captureTimeUs, keyframe } = frame;
|
|
347
347
|
if (!CODECS.has(codec)) {
|
|
348
|
-
throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264"`);
|
|
348
|
+
throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264", "vp8", "vp9" or "av1"`);
|
|
349
|
+
}
|
|
350
|
+
if (keyframe != null && typeof keyframe !== "boolean") {
|
|
351
|
+
throw refuse("keyframe must be a boolean");
|
|
349
352
|
}
|
|
350
353
|
let bytes;
|
|
351
354
|
if (data instanceof Uint8Array) {
|
|
@@ -370,7 +373,15 @@ function toEncodedBytes(frame, operation, trackName) {
|
|
|
370
373
|
if (captureTimeUs != null && !Number.isFinite(captureTimeUs)) {
|
|
371
374
|
throw refuse("captureTimeUs must be a finite number");
|
|
372
375
|
}
|
|
373
|
-
return {
|
|
376
|
+
return {
|
|
377
|
+
codec,
|
|
378
|
+
data: bytes,
|
|
379
|
+
width,
|
|
380
|
+
height,
|
|
381
|
+
metadata: metadata ?? null,
|
|
382
|
+
captureTimeUs: captureTimeUs ?? null,
|
|
383
|
+
keyframe: keyframe ?? (isChunk(data) && data.type === "key")
|
|
384
|
+
};
|
|
374
385
|
}
|
|
375
386
|
function isDimension(value) {
|
|
376
387
|
return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 4294967295;
|
|
@@ -647,6 +658,513 @@ function checkStepFrame(client, frame) {
|
|
|
647
658
|
}
|
|
648
659
|
}
|
|
649
660
|
|
|
661
|
+
// src/internal/bt601.ts
|
|
662
|
+
var BT601_LIMITED = {
|
|
663
|
+
matrix: "smpte170m",
|
|
664
|
+
primaries: "smpte170m",
|
|
665
|
+
transfer: "smpte170m",
|
|
666
|
+
fullRange: false
|
|
667
|
+
};
|
|
668
|
+
function clamp(value) {
|
|
669
|
+
return value < 0 ? 0 : value > 255 ? 255 : value;
|
|
670
|
+
}
|
|
671
|
+
function rgbToI420(pixels) {
|
|
672
|
+
const { width, height, data } = pixels;
|
|
673
|
+
const step = pixels.format === "rgb" ? 3 : 4;
|
|
674
|
+
const [ri, bi] = pixels.format === "bgra" ? [2, 0] : [0, 2];
|
|
675
|
+
const codedWidth = width + (width & 1);
|
|
676
|
+
const codedHeight = height + (height & 1);
|
|
677
|
+
const chromaWidth = codedWidth >> 1;
|
|
678
|
+
const chromaHeight = codedHeight >> 1;
|
|
679
|
+
const ySize = codedWidth * codedHeight;
|
|
680
|
+
const out = new Uint8Array(ySize + 2 * chromaWidth * chromaHeight);
|
|
681
|
+
const uOffset = ySize;
|
|
682
|
+
const vOffset = ySize + chromaWidth * chromaHeight;
|
|
683
|
+
const at = (x, y) => (Math.min(y, height - 1) * width + Math.min(x, width - 1)) * step;
|
|
684
|
+
for (let y = 0; y < codedHeight; y++) {
|
|
685
|
+
for (let x = 0; x < codedWidth; x++) {
|
|
686
|
+
const i = at(x, y);
|
|
687
|
+
const r = data[i + ri];
|
|
688
|
+
const g = data[i + 1];
|
|
689
|
+
const b = data[i + bi];
|
|
690
|
+
out[y * codedWidth + x] = 66 * r + 129 * g + 25 * b + 4224 >> 8 & 255;
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
for (let cy = 0; cy < chromaHeight; cy++) {
|
|
694
|
+
for (let cx = 0; cx < chromaWidth; cx++) {
|
|
695
|
+
let r = 0;
|
|
696
|
+
let g = 0;
|
|
697
|
+
let b = 0;
|
|
698
|
+
for (const [dx, dy] of [
|
|
699
|
+
[0, 0],
|
|
700
|
+
[1, 0],
|
|
701
|
+
[0, 1],
|
|
702
|
+
[1, 1]
|
|
703
|
+
]) {
|
|
704
|
+
const i = at(2 * cx + dx, 2 * cy + dy);
|
|
705
|
+
r += data[i + ri];
|
|
706
|
+
g += data[i + 1];
|
|
707
|
+
b += data[i + bi];
|
|
708
|
+
}
|
|
709
|
+
r = r + 2 >> 2;
|
|
710
|
+
g = g + 2 >> 2;
|
|
711
|
+
b = b + 2 >> 2;
|
|
712
|
+
out[uOffset + cy * chromaWidth + cx] = 112 * b - 74 * g - 38 * r + 32896 >> 8;
|
|
713
|
+
out[vOffset + cy * chromaWidth + cx] = 112 * r - 94 * g - 18 * b + 32896 >> 8;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
return { data: out, width, height, codedWidth, codedHeight };
|
|
717
|
+
}
|
|
718
|
+
function yuvToRgb(data, { format, width, height, planes }) {
|
|
719
|
+
const [yPlane, uPlane, vPlane] = planes;
|
|
720
|
+
if (yPlane === void 0 || uPlane === void 0 || format === "I420" && vPlane === void 0) {
|
|
721
|
+
throw new Error(`a ${format} picture needs ${format === "I420" ? 3 : 2} planes, got ${planes.length}`);
|
|
722
|
+
}
|
|
723
|
+
const rgb = new Uint8Array(width * height * 3);
|
|
724
|
+
for (let y = 0; y < height; y++) {
|
|
725
|
+
const cy = y >> 1;
|
|
726
|
+
for (let x = 0; x < width; x++) {
|
|
727
|
+
const cx = x >> 1;
|
|
728
|
+
const luma = data[yPlane.offset + y * yPlane.stride + x];
|
|
729
|
+
let u;
|
|
730
|
+
let v;
|
|
731
|
+
if (format === "NV12") {
|
|
732
|
+
u = data[uPlane.offset + cy * uPlane.stride + 2 * cx];
|
|
733
|
+
v = data[uPlane.offset + cy * uPlane.stride + 2 * cx + 1];
|
|
734
|
+
} else {
|
|
735
|
+
u = data[uPlane.offset + cy * uPlane.stride + cx];
|
|
736
|
+
v = data[vPlane.offset + cy * vPlane.stride + cx];
|
|
737
|
+
}
|
|
738
|
+
const c = 1.164 * (luma - 16);
|
|
739
|
+
const d = u - 128;
|
|
740
|
+
const e = v - 128;
|
|
741
|
+
const j = (y * width + x) * 3;
|
|
742
|
+
rgb[j] = clamp(Math.round(c + 1.596 * e));
|
|
743
|
+
rgb[j + 1] = clamp(Math.round(c - 0.391 * d - 0.813 * e));
|
|
744
|
+
rgb[j + 2] = clamp(Math.round(c + 2.018 * d));
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
return rgb;
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
// src/internal/step-codecs.ts
|
|
751
|
+
var STEP_CODECS = ["h264", "vp9", "av1", "vp8"];
|
|
752
|
+
var DECODE_TIMEOUT_MS = 1e4;
|
|
753
|
+
function codecString(codec) {
|
|
754
|
+
switch (codec) {
|
|
755
|
+
case "h264":
|
|
756
|
+
return "avc1.42e033";
|
|
757
|
+
case "vp8":
|
|
758
|
+
return "vp8";
|
|
759
|
+
case "vp9":
|
|
760
|
+
return "vp09.00.51.08";
|
|
761
|
+
case "av1":
|
|
762
|
+
return "av01.0.13M.08";
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
function encoderConfig(codec, width, height) {
|
|
766
|
+
return {
|
|
767
|
+
codec: codecString(codec),
|
|
768
|
+
width,
|
|
769
|
+
height,
|
|
770
|
+
// Rate-controlled, as a real-time encoder is: enough for a still image
|
|
771
|
+
// to come back close, small enough to keep a step's message small.
|
|
772
|
+
bitrate: Math.min(2e7, Math.max(3e5, width * height * 4)),
|
|
773
|
+
framerate: 30,
|
|
774
|
+
latencyMode: "realtime",
|
|
775
|
+
...codec === "h264" ? { avc: { format: "annexb" } } : {}
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
function decoderConfig(codec) {
|
|
779
|
+
return { codec: codecString(codec), optimizeForLatency: true };
|
|
780
|
+
}
|
|
781
|
+
async function probeStepCodecs(api = globalThis) {
|
|
782
|
+
const supported = async (probe) => {
|
|
783
|
+
const results = await Promise.all(STEP_CODECS.map((codec) => probe(codec).catch(() => false)));
|
|
784
|
+
return STEP_CODECS.filter((_, i) => results[i]);
|
|
785
|
+
};
|
|
786
|
+
const { VideoEncoder: Encoder, VideoDecoder: Decoder, VideoFrame: Frame, EncodedVideoChunk: Chunk } = api;
|
|
787
|
+
const [encode, decode] = await Promise.all([
|
|
788
|
+
Encoder && Frame ? supported(async (codec) => (await Encoder.isConfigSupported(encoderConfig(codec, 1280, 720))).supported === true) : [],
|
|
789
|
+
Decoder && Chunk ? supported(async (codec) => (await Decoder.isConfigSupported(decoderConfig(codec))).supported === true) : []
|
|
790
|
+
]);
|
|
791
|
+
return { encode, decode };
|
|
792
|
+
}
|
|
793
|
+
var TrackEncoder = class {
|
|
794
|
+
constructor(api, codec, width, height) {
|
|
795
|
+
this.api = api;
|
|
796
|
+
this.codec = codec;
|
|
797
|
+
this.width = width;
|
|
798
|
+
this.height = height;
|
|
799
|
+
this.encoder = new api.VideoEncoder({
|
|
800
|
+
output: (chunk) => {
|
|
801
|
+
const data = new Uint8Array(chunk.byteLength);
|
|
802
|
+
chunk.copyTo(data);
|
|
803
|
+
this.outputs.push({ data, keyframe: chunk.type === "key" });
|
|
804
|
+
},
|
|
805
|
+
error: (error) => {
|
|
806
|
+
this.failure = error instanceof Error ? error : new Error(String(error));
|
|
807
|
+
}
|
|
808
|
+
});
|
|
809
|
+
try {
|
|
810
|
+
this.encoder.configure(encoderConfig(codec, width, height));
|
|
811
|
+
} catch (cause) {
|
|
812
|
+
this.close();
|
|
813
|
+
throw cause;
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
api;
|
|
817
|
+
codec;
|
|
818
|
+
width;
|
|
819
|
+
height;
|
|
820
|
+
encoder;
|
|
821
|
+
outputs = [];
|
|
822
|
+
failure;
|
|
823
|
+
started = false;
|
|
824
|
+
timestampUs = 0;
|
|
825
|
+
/** Encode `pixels` as the stream's next frame, a keyframe when `keyframe`
|
|
826
|
+
* or when it is the stream's first, and hand back what the encoder made
|
|
827
|
+
* of it: every frame is flushed out before the next goes in. */
|
|
828
|
+
async encode(pixels, keyframe) {
|
|
829
|
+
const i420 = rgbToI420(pixels);
|
|
830
|
+
const frame = new this.api.VideoFrame(i420.data, {
|
|
831
|
+
format: "I420",
|
|
832
|
+
codedWidth: i420.codedWidth,
|
|
833
|
+
codedHeight: i420.codedHeight,
|
|
834
|
+
visibleRect: { x: 0, y: 0, width: i420.width, height: i420.height },
|
|
835
|
+
colorSpace: BT601_LIMITED,
|
|
836
|
+
timestamp: this.timestampUs
|
|
837
|
+
});
|
|
838
|
+
this.timestampUs += 33333;
|
|
839
|
+
try {
|
|
840
|
+
this.encoder.encode(frame, { keyFrame: keyframe || !this.started });
|
|
841
|
+
} finally {
|
|
842
|
+
frame.close();
|
|
843
|
+
}
|
|
844
|
+
this.started = true;
|
|
845
|
+
await this.encoder.flush();
|
|
846
|
+
if (this.failure !== void 0) {
|
|
847
|
+
throw this.failure;
|
|
848
|
+
}
|
|
849
|
+
return this.outputs.splice(0);
|
|
850
|
+
}
|
|
851
|
+
close() {
|
|
852
|
+
if (this.encoder.state !== "closed") {
|
|
853
|
+
this.encoder.close();
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
};
|
|
857
|
+
var InputEncoders = class {
|
|
858
|
+
constructor(api = globalThis) {
|
|
859
|
+
this.api = api;
|
|
860
|
+
}
|
|
861
|
+
api;
|
|
862
|
+
encoders = /* @__PURE__ */ new Map();
|
|
863
|
+
/** Whether this browser can encode at all. */
|
|
864
|
+
get available() {
|
|
865
|
+
return this.api.VideoEncoder !== void 0 && this.api.VideoFrame !== void 0;
|
|
866
|
+
}
|
|
867
|
+
/** Encode `pixels` as `track`'s next frame in `codec`. A track whose codec
|
|
868
|
+
* or size changed starts a new stream, at a keyframe. On a failure the
|
|
869
|
+
* track's encoder is dropped, so its next frame is a keyframe, and the
|
|
870
|
+
* error is thrown for the caller to send the frame raw instead. */
|
|
871
|
+
async encode(track, codec, pixels, keyframe) {
|
|
872
|
+
const { VideoEncoder: Encoder, VideoFrame: Frame } = this.api;
|
|
873
|
+
if (!Encoder || !Frame) {
|
|
874
|
+
throw new Error("this browser has no WebCodecs VideoEncoder");
|
|
875
|
+
}
|
|
876
|
+
let encoder = this.encoders.get(track);
|
|
877
|
+
if (encoder && (encoder.codec !== codec || encoder.width !== pixels.width || encoder.height !== pixels.height)) {
|
|
878
|
+
encoder.close();
|
|
879
|
+
encoder = void 0;
|
|
880
|
+
}
|
|
881
|
+
try {
|
|
882
|
+
if (!encoder) {
|
|
883
|
+
encoder = new TrackEncoder({ VideoEncoder: Encoder, VideoFrame: Frame }, codec, pixels.width, pixels.height);
|
|
884
|
+
this.encoders.set(track, encoder);
|
|
885
|
+
}
|
|
886
|
+
return await encoder.encode(pixels, keyframe);
|
|
887
|
+
} catch (cause) {
|
|
888
|
+
this.drop(track);
|
|
889
|
+
throw cause;
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
/** Drop `track`'s encoder: its next frame starts a new stream. */
|
|
893
|
+
drop(track) {
|
|
894
|
+
this.encoders.get(track)?.close();
|
|
895
|
+
this.encoders.delete(track);
|
|
896
|
+
}
|
|
897
|
+
reset() {
|
|
898
|
+
for (const track of [...this.encoders.keys()]) {
|
|
899
|
+
this.drop(track);
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
};
|
|
903
|
+
var TrackDecoder = class {
|
|
904
|
+
constructor(api, codec) {
|
|
905
|
+
this.codec = codec;
|
|
906
|
+
this.Chunk = api.EncodedVideoChunk;
|
|
907
|
+
this.decoder = new api.VideoDecoder({
|
|
908
|
+
output: (frame) => this.deliver(frame),
|
|
909
|
+
error: (error) => this.fail(error)
|
|
910
|
+
});
|
|
911
|
+
try {
|
|
912
|
+
this.decoder.configure(decoderConfig(codec));
|
|
913
|
+
} catch (cause) {
|
|
914
|
+
this.close();
|
|
915
|
+
throw cause;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
codec;
|
|
919
|
+
decoder;
|
|
920
|
+
Chunk;
|
|
921
|
+
/** Resolvers of the frames handed in and not yet handed back, by the
|
|
922
|
+
* timestamp each chunk was given, in the order they were handed in. */
|
|
923
|
+
pending = /* @__PURE__ */ new Map();
|
|
924
|
+
/** The next chunk's timestamp: a count, so each output names the one chunk
|
|
925
|
+
* it was made of, whatever capture time the frame carried. */
|
|
926
|
+
nextTimestamp = 0;
|
|
927
|
+
failed = false;
|
|
928
|
+
/** Hand `frame` to the chunk it was made of, by its timestamp. A step's
|
|
929
|
+
* stream is decoded in order, with no frame reordered, so a chunk handed
|
|
930
|
+
* in before that one with no picture yet made none (a frame that is not
|
|
931
|
+
* shown): it resolves with none rather than wait for the timeout, and no
|
|
932
|
+
* picture is ever matched to a chunk it was not made of. */
|
|
933
|
+
deliver(frame) {
|
|
934
|
+
const entry = this.pending.get(frame.timestamp);
|
|
935
|
+
if (!entry) {
|
|
936
|
+
frame.close();
|
|
937
|
+
return;
|
|
938
|
+
}
|
|
939
|
+
for (const [timestamp, earlier] of this.pending) {
|
|
940
|
+
if (timestamp === frame.timestamp) {
|
|
941
|
+
break;
|
|
942
|
+
}
|
|
943
|
+
this.pending.delete(timestamp);
|
|
944
|
+
earlier.resolve(void 0);
|
|
945
|
+
}
|
|
946
|
+
this.pending.delete(frame.timestamp);
|
|
947
|
+
entry.resolve(frame);
|
|
948
|
+
}
|
|
949
|
+
/** Hand a chunk of `data` to the decoder now, and resolve with the frame it
|
|
950
|
+
* makes of it, or `undefined` for one it shows none of. Called
|
|
951
|
+
* synchronously for every frame of a result, so the decoder takes them in
|
|
952
|
+
* the order results arrive. */
|
|
953
|
+
decode(type, data) {
|
|
954
|
+
const timestamp = this.nextTimestamp++;
|
|
955
|
+
const decoded = new Promise((resolve, reject) => {
|
|
956
|
+
this.pending.set(timestamp, { resolve, reject });
|
|
957
|
+
});
|
|
958
|
+
try {
|
|
959
|
+
this.decoder.decode(new this.Chunk({ type, timestamp, data }));
|
|
960
|
+
} catch (cause) {
|
|
961
|
+
this.fail(cause);
|
|
962
|
+
}
|
|
963
|
+
return withTimeout(decoded, DECODE_TIMEOUT_MS, () => {
|
|
964
|
+
this.fail(new Error(`the ${this.codec} decoder handed back no frame in ${DECODE_TIMEOUT_MS} ms`));
|
|
965
|
+
});
|
|
966
|
+
}
|
|
967
|
+
/** The decoder lost its state, or is given up: every frame still in it
|
|
968
|
+
* fails, so no step waits on it. */
|
|
969
|
+
fail(cause) {
|
|
970
|
+
this.failed = true;
|
|
971
|
+
const pending = [...this.pending.values()];
|
|
972
|
+
this.pending.clear();
|
|
973
|
+
for (const next of pending) {
|
|
974
|
+
next.reject(cause);
|
|
975
|
+
}
|
|
976
|
+
this.close();
|
|
977
|
+
}
|
|
978
|
+
close() {
|
|
979
|
+
if (this.decoder.state !== "closed") {
|
|
980
|
+
this.decoder.close();
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
};
|
|
984
|
+
function withTimeout(promise, ms, onTimeout) {
|
|
985
|
+
let timer;
|
|
986
|
+
const timeout = new Promise((_, reject) => {
|
|
987
|
+
timer = setTimeout(() => {
|
|
988
|
+
onTimeout();
|
|
989
|
+
reject(new Error(`timed out after ${ms} ms`));
|
|
990
|
+
}, ms);
|
|
991
|
+
});
|
|
992
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
993
|
+
}
|
|
994
|
+
var OutputDecoders = class {
|
|
995
|
+
constructor(api = globalThis) {
|
|
996
|
+
this.api = api;
|
|
997
|
+
}
|
|
998
|
+
api;
|
|
999
|
+
decoders = /* @__PURE__ */ new Map();
|
|
1000
|
+
awaitingKeyframe = /* @__PURE__ */ new Set();
|
|
1001
|
+
lost = [];
|
|
1002
|
+
/** Whether a track lost its decoder and waits for a keyframe. */
|
|
1003
|
+
needsKeyframe() {
|
|
1004
|
+
return this.awaitingKeyframe.size > 0;
|
|
1005
|
+
}
|
|
1006
|
+
/** The tracks that lost their stream since the last call, each once per
|
|
1007
|
+
* loss however many frames it then discards: the caller restarts each
|
|
1008
|
+
* one's stream. */
|
|
1009
|
+
takeLostTracks() {
|
|
1010
|
+
const lost = this.lost;
|
|
1011
|
+
this.lost = [];
|
|
1012
|
+
return lost;
|
|
1013
|
+
}
|
|
1014
|
+
/** A step gave up after its request went out: its result, if it still
|
|
1015
|
+
* comes, is never fed here, so every stream lost frames. Drop the
|
|
1016
|
+
* decoders, and report each of their tracks lost. */
|
|
1017
|
+
abandon() {
|
|
1018
|
+
for (const [track, decoder] of this.decoders) {
|
|
1019
|
+
decoder.fail(new DecodeError(`step(): track '${track}' lost its stream: a step gave up mid-stream`, { operation: "step" }));
|
|
1020
|
+
this.lose(track, false);
|
|
1021
|
+
}
|
|
1022
|
+
this.decoders.clear();
|
|
1023
|
+
}
|
|
1024
|
+
/** Make `track` wait for a keyframe, and report it lost when it was not
|
|
1025
|
+
* waiting already, or when `again` says this is a new loss. */
|
|
1026
|
+
lose(track, again) {
|
|
1027
|
+
if (!this.awaitingKeyframe.has(track) || again) {
|
|
1028
|
+
this.lost.push(track);
|
|
1029
|
+
}
|
|
1030
|
+
this.awaitingKeyframe.add(track);
|
|
1031
|
+
}
|
|
1032
|
+
/** `result` with every encoded frame decoded to RGB `(height, width, 3)`
|
|
1033
|
+
* `uint8`, as a raw frame comes. Call it as the result arrives: each
|
|
1034
|
+
* encoded frame goes into its track's decoder before this returns, so
|
|
1035
|
+
* results that arrive one after another are decoded in that order.
|
|
1036
|
+
*
|
|
1037
|
+
* Every frame of every track is fed to its decoder even after one fails,
|
|
1038
|
+
* so the other streams keep up; the first failure rejects, with a
|
|
1039
|
+
* `DecodeError`, rather than resolving a partial step. */
|
|
1040
|
+
decodeResult(result) {
|
|
1041
|
+
if (result.tracks.every((track) => track.frames.every((frame) => !isEncoded(frame)))) {
|
|
1042
|
+
return Promise.resolve(result);
|
|
1043
|
+
}
|
|
1044
|
+
let firstError;
|
|
1045
|
+
const tracks = result.tracks.map((track) => {
|
|
1046
|
+
const frames = track.frames.map(
|
|
1047
|
+
(frame) => this.decodeFrame(track.track, frame).catch((cause) => {
|
|
1048
|
+
firstError ??= cause;
|
|
1049
|
+
return void 0;
|
|
1050
|
+
})
|
|
1051
|
+
);
|
|
1052
|
+
return Promise.all(frames).then((decoded) => ({
|
|
1053
|
+
...track,
|
|
1054
|
+
frames: decoded.filter((frame) => frame !== void 0)
|
|
1055
|
+
}));
|
|
1056
|
+
});
|
|
1057
|
+
return Promise.all(tracks).then((decoded) => {
|
|
1058
|
+
if (firstError !== void 0) {
|
|
1059
|
+
throw toDecodeError(firstError);
|
|
1060
|
+
}
|
|
1061
|
+
return { ...result, tracks: decoded };
|
|
1062
|
+
});
|
|
1063
|
+
}
|
|
1064
|
+
/** Decode one frame of `track`; a raw one passes as it is, and one its
|
|
1065
|
+
* decoder shows no picture of is `undefined`. Synchronous up to the
|
|
1066
|
+
* decoder taking the frame. */
|
|
1067
|
+
decodeFrame(track, frame) {
|
|
1068
|
+
if (!isEncoded(frame)) {
|
|
1069
|
+
return Promise.resolve(frame);
|
|
1070
|
+
}
|
|
1071
|
+
const codec = frame.encoding;
|
|
1072
|
+
const { VideoDecoder: Decoder, EncodedVideoChunk: Chunk } = this.api;
|
|
1073
|
+
if (!Decoder || !Chunk) {
|
|
1074
|
+
return Promise.reject(new DecodeError(`step(): '${track}' sent ${codec} frames and this browser has no WebCodecs VideoDecoder`, { operation: "step" }));
|
|
1075
|
+
}
|
|
1076
|
+
let decoder = this.decoders.get(track);
|
|
1077
|
+
if (decoder && (decoder.failed || decoder.codec !== codec)) {
|
|
1078
|
+
decoder.fail(new DecodeError(`step(): track '${track}' changed codec to ${codec}`, { operation: "step" }));
|
|
1079
|
+
this.decoders.delete(track);
|
|
1080
|
+
decoder = void 0;
|
|
1081
|
+
}
|
|
1082
|
+
if (!decoder) {
|
|
1083
|
+
if (!frame.keyframe) {
|
|
1084
|
+
this.lose(track, false);
|
|
1085
|
+
return Promise.reject(
|
|
1086
|
+
new DecodeError(
|
|
1087
|
+
`step(): track '${track}' has no decoder state for a ${codec} frame that is not a keyframe; waiting for a keyframe`,
|
|
1088
|
+
{ operation: "step" }
|
|
1089
|
+
)
|
|
1090
|
+
);
|
|
1091
|
+
}
|
|
1092
|
+
try {
|
|
1093
|
+
decoder = new TrackDecoder({ VideoDecoder: Decoder, EncodedVideoChunk: Chunk }, codec);
|
|
1094
|
+
} catch (cause) {
|
|
1095
|
+
return Promise.reject(toDecodeError(cause));
|
|
1096
|
+
}
|
|
1097
|
+
this.decoders.set(track, decoder);
|
|
1098
|
+
}
|
|
1099
|
+
const live = decoder;
|
|
1100
|
+
return live.decode(frame.keyframe ? "key" : "delta", frame.data).then(
|
|
1101
|
+
async (picture) => {
|
|
1102
|
+
if (frame.keyframe) {
|
|
1103
|
+
this.awaitingKeyframe.delete(track);
|
|
1104
|
+
}
|
|
1105
|
+
return picture === void 0 ? void 0 : { ...frame, ...await rgbOf(picture) };
|
|
1106
|
+
},
|
|
1107
|
+
(cause) => {
|
|
1108
|
+
this.lose(track, frame.keyframe === true);
|
|
1109
|
+
if (this.decoders.get(track) === live) {
|
|
1110
|
+
this.decoders.delete(track);
|
|
1111
|
+
}
|
|
1112
|
+
throw cause;
|
|
1113
|
+
}
|
|
1114
|
+
);
|
|
1115
|
+
}
|
|
1116
|
+
reset() {
|
|
1117
|
+
for (const [track, decoder] of this.decoders) {
|
|
1118
|
+
decoder.fail(new DecodeError(`step(): track '${track}' lost its stream: its connection closed`, { operation: "step" }));
|
|
1119
|
+
}
|
|
1120
|
+
this.decoders.clear();
|
|
1121
|
+
this.awaitingKeyframe.clear();
|
|
1122
|
+
this.lost = [];
|
|
1123
|
+
}
|
|
1124
|
+
};
|
|
1125
|
+
function isEncoded(frame) {
|
|
1126
|
+
return frame.encoding !== "raw" && frame.encoding !== "unspecified";
|
|
1127
|
+
}
|
|
1128
|
+
function toDecodeError(cause) {
|
|
1129
|
+
if (cause instanceof DecodeError) {
|
|
1130
|
+
return cause;
|
|
1131
|
+
}
|
|
1132
|
+
const message = cause instanceof Error ? cause.message : JSON.stringify(cause);
|
|
1133
|
+
return new DecodeError(`step(): ${message}`, { operation: "step", cause });
|
|
1134
|
+
}
|
|
1135
|
+
async function rgbOf(picture) {
|
|
1136
|
+
let pixels;
|
|
1137
|
+
try {
|
|
1138
|
+
const yuv = await yuvOf(picture);
|
|
1139
|
+
if (yuv !== void 0) {
|
|
1140
|
+
return yuv;
|
|
1141
|
+
}
|
|
1142
|
+
pixels = await toPixels(picture);
|
|
1143
|
+
} finally {
|
|
1144
|
+
picture.close();
|
|
1145
|
+
}
|
|
1146
|
+
const { width, height } = pixels;
|
|
1147
|
+
const rgb = new Uint8Array(width * height * 3);
|
|
1148
|
+
const [r, b] = pixels.format === "bgra" ? [2, 0] : [0, 2];
|
|
1149
|
+
for (let i = 0, j = 0; j < rgb.length; i += 4, j += 3) {
|
|
1150
|
+
rgb[j] = pixels.data[i + r];
|
|
1151
|
+
rgb[j + 1] = pixels.data[i + 1];
|
|
1152
|
+
rgb[j + 2] = pixels.data[i + b];
|
|
1153
|
+
}
|
|
1154
|
+
return { dtype: "uint8", shape: [height, width, 3], data: rgb };
|
|
1155
|
+
}
|
|
1156
|
+
async function yuvOf(picture) {
|
|
1157
|
+
const format = picture.format === "I420A" ? "I420" : picture.format;
|
|
1158
|
+
if (format !== "I420" && format !== "NV12") {
|
|
1159
|
+
return void 0;
|
|
1160
|
+
}
|
|
1161
|
+
const width = picture.visibleRect?.width ?? picture.codedWidth;
|
|
1162
|
+
const height = picture.visibleRect?.height ?? picture.codedHeight;
|
|
1163
|
+
const data = new Uint8Array(picture.allocationSize());
|
|
1164
|
+
const planes = await picture.copyTo(data);
|
|
1165
|
+
return { dtype: "uint8", shape: [height, width, 3], data: yuvToRgb(data, { format, width, height, planes }) };
|
|
1166
|
+
}
|
|
1167
|
+
|
|
650
1168
|
// src/internal/stats.ts
|
|
651
1169
|
var LOCAL_STATS_INTERVAL_MS = 2e3;
|
|
652
1170
|
var RUNTIME_REPORT_INTERVAL_MS = 5e3;
|
|
@@ -974,7 +1492,7 @@ function describeCause(cause) {
|
|
|
974
1492
|
// package.json
|
|
975
1493
|
var package_default = {
|
|
976
1494
|
name: "@reactor-team/js-sdk",
|
|
977
|
-
version: "3.
|
|
1495
|
+
version: "3.5.0",
|
|
978
1496
|
description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
|
|
979
1497
|
license: "Apache-2.0",
|
|
980
1498
|
repository: {
|
|
@@ -1501,6 +2019,17 @@ var Reactor = class {
|
|
|
1501
2019
|
* one's pixels take to read, and `step()` waits for the ones called
|
|
1502
2020
|
* before it. */
|
|
1503
2021
|
pendingPushes = Promise.resolve();
|
|
2022
|
+
/** Stepped: every `setCodecPreference()` so far, keyed by track (`null`
|
|
2023
|
+
* for the session's list), replayed into a client created later. */
|
|
2024
|
+
codecPreferences = /* @__PURE__ */ new Map();
|
|
2025
|
+
/** Stepped: what this browser's WebCodecs can encode and decode, probed
|
|
2026
|
+
* once, on the first stepped connect. */
|
|
2027
|
+
localStepCodecs;
|
|
2028
|
+
/** Stepped: the input tracks' encoders and the output tracks' decoders of
|
|
2029
|
+
* the current connection, dropped as it ends so the next one's streams
|
|
2030
|
+
* start at keyframes. */
|
|
2031
|
+
inputEncoders = new InputEncoders();
|
|
2032
|
+
outputDecoders = new OutputDecoders();
|
|
1504
2033
|
emitter = new Emitter();
|
|
1505
2034
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
1506
2035
|
* disconnect()/[Symbol.dispose]) against each other. Calling into the
|
|
@@ -1608,6 +2137,7 @@ var Reactor = class {
|
|
|
1608
2137
|
await this.queue.push(
|
|
1609
2138
|
() => this.withWriterAccess(async () => {
|
|
1610
2139
|
const client = await this.getOrCreateClient();
|
|
2140
|
+
await this.offerStepCodecs(client);
|
|
1611
2141
|
await client.connect(options);
|
|
1612
2142
|
}),
|
|
1613
2143
|
"connect"
|
|
@@ -1662,6 +2192,7 @@ var Reactor = class {
|
|
|
1662
2192
|
await this.queue.push(
|
|
1663
2193
|
() => this.withWriterAccess(async () => {
|
|
1664
2194
|
const client = await this.getOrCreateClient();
|
|
2195
|
+
await this.offerStepCodecs(client);
|
|
1665
2196
|
await client.reconnect(options);
|
|
1666
2197
|
}),
|
|
1667
2198
|
"reconnect"
|
|
@@ -1937,10 +2468,10 @@ var Reactor = class {
|
|
|
1937
2468
|
* Runs one step of a `generation: "stepped"` session and resolves with
|
|
1938
2469
|
* what it produced, per output track.
|
|
1939
2470
|
*
|
|
1940
|
-
* The request carries `state
|
|
1941
|
-
*
|
|
1942
|
-
* as the model's `set_<key>` command,
|
|
1943
|
-
* and then generates. They are separate calls, not a transaction: one that
|
|
2471
|
+
* The request carries `state` and `commands`; the frames pushed before it
|
|
2472
|
+
* have already reached the runtime (see `pushFrame()`). The runtime applies
|
|
2473
|
+
* them in that order — each `state` key as the model's `set_<key>` command,
|
|
2474
|
+
* then `commands` — and then generates. They are separate calls, not a transaction: one that
|
|
1944
2475
|
* fails does not undo the ones before it. All of it is optional:
|
|
1945
2476
|
* `step()` with nothing set and nothing pushed runs on the model's current
|
|
1946
2477
|
* state, which commands sent outside a step (`sendCommand()`) also change.
|
|
@@ -1951,7 +2482,9 @@ var Reactor = class {
|
|
|
1951
2482
|
* A step that ran but that the model reported as failed still resolves,
|
|
1952
2483
|
* with `result.error` set. `result.files` lists the files the model kept
|
|
1953
2484
|
* with the step (name, content type, size); their bytes are served over
|
|
1954
|
-
* HTTP from the session's step results, not carried here.
|
|
2485
|
+
* HTTP from the session's step results, not carried here. `result.data`
|
|
2486
|
+
* holds what the model returned besides its media, as opaque bytes (a dict
|
|
2487
|
+
* arrives encoded as JSON); it is absent when the model returned none.
|
|
1955
2488
|
*
|
|
1956
2489
|
* Rejects with a `StepRefusedError` when the runtime or the model turns
|
|
1957
2490
|
* the step down (the runtime checks a whole request first, so nothing it
|
|
@@ -1965,7 +2498,7 @@ var Reactor = class {
|
|
|
1965
2498
|
* message, `MessageTooLargeError` when the request itself is too large
|
|
1966
2499
|
* (nothing was sent), `InvalidStateError` outside a ready stepped
|
|
1967
2500
|
* session, or whatever else failed the step (`DisconnectedError`,
|
|
1968
|
-
* `RequestTimeoutError`, ...). The frames
|
|
2501
|
+
* `RequestTimeoutError`, ...). The frames buffered for a step belong to it:
|
|
1969
2502
|
* when its request cannot be sent, they are dropped with it rather than
|
|
1970
2503
|
* kept for the next step.
|
|
1971
2504
|
*
|
|
@@ -1979,9 +2512,19 @@ var Reactor = class {
|
|
|
1979
2512
|
* before any of it is pushed or sent, leaving the buffer as it was. A
|
|
1980
2513
|
* session that leaves 'ready' once the media is in has the step refused and
|
|
1981
2514
|
* keeps the frames for the next one, as pushed frames are.
|
|
1982
|
-
* A track's encoded frames
|
|
1983
|
-
*
|
|
1984
|
-
* `StepRefusedError
|
|
2515
|
+
* A track's encoded frames go in decode order, starting at a keyframe; a
|
|
2516
|
+
* step whose buffered frames do not decode rejects with a
|
|
2517
|
+
* `StepRefusedError` (see `pushEncodedFrame()` for frames already sent).
|
|
2518
|
+
*
|
|
2519
|
+
* Video step frames travel encoded both ways when this browser and the
|
|
2520
|
+
* runtime share a codec (H.264 first by default; see
|
|
2521
|
+
* `setCodecPreference()`), on WebCodecs: the frames pushed on an input
|
|
2522
|
+
* track are encoded, and an output track's frames are decoded, so a result
|
|
2523
|
+
* still holds RGB frames whose `encoding` names the codec they travelled
|
|
2524
|
+
* in. Each track is one stream for the connection. When one of its
|
|
2525
|
+
* decoders lost its stream, the step rejects with a `DecodeError`, the SDK
|
|
2526
|
+
* pauses and resumes that track so the runtime starts it again at a
|
|
2527
|
+
* keyframe, and the track's frames fail until that keyframe arrives.
|
|
1985
2528
|
*
|
|
1986
2529
|
* ```ts
|
|
1987
2530
|
* const camera = await reactor.publishTrack('camera');
|
|
@@ -2011,7 +2554,15 @@ var Reactor = class {
|
|
|
2011
2554
|
this.withReaderAccess(async () => {
|
|
2012
2555
|
const client = await this.getOrCreateClient();
|
|
2013
2556
|
try {
|
|
2014
|
-
|
|
2557
|
+
const decoders = this.outputDecoders;
|
|
2558
|
+
return client.step(state, commands).then(
|
|
2559
|
+
(result) => decoders.decodeResult(result).finally(() => this.restartLostTracks(client, decoders)),
|
|
2560
|
+
(cause) => {
|
|
2561
|
+
decoders.abandon();
|
|
2562
|
+
this.restartLostTracks(client, decoders);
|
|
2563
|
+
throw cause;
|
|
2564
|
+
}
|
|
2565
|
+
);
|
|
2015
2566
|
} finally {
|
|
2016
2567
|
taken();
|
|
2017
2568
|
}
|
|
@@ -2026,11 +2577,15 @@ var Reactor = class {
|
|
|
2026
2577
|
}
|
|
2027
2578
|
}
|
|
2028
2579
|
/**
|
|
2029
|
-
*
|
|
2030
|
-
*
|
|
2031
|
-
*
|
|
2032
|
-
*
|
|
2033
|
-
*
|
|
2580
|
+
* Pushes one input frame on the sendonly track `trackName` of a stepped
|
|
2581
|
+
* session. Once the session is ready, against a runtime that takes pushed
|
|
2582
|
+
* frames, the frame is sent to the runtime at once over the data channel,
|
|
2583
|
+
* ahead of any `step()` after it; one the channel cannot take is dropped
|
|
2584
|
+
* and raised as an error event. A frame pushed before the session is ready,
|
|
2585
|
+
* or to an older runtime, is buffered and sent with the next push or
|
|
2586
|
+
* `step()`, per track in push order. The `PublishedTrack` that
|
|
2587
|
+
* `publishTrack()` resolves with does the same through
|
|
2588
|
+
* `track.pushFrame(frame)`.
|
|
2034
2589
|
*
|
|
2035
2590
|
* Video: a `VideoFrame` (cloned when called, so the caller may close it as
|
|
2036
2591
|
* soon as this returns; it stays the caller's), an
|
|
@@ -2041,9 +2596,15 @@ var Reactor = class {
|
|
|
2041
2596
|
* Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
|
|
2042
2597
|
* PCM at the rate the model's track declares.
|
|
2043
2598
|
*
|
|
2599
|
+
* A video frame travels encoded when the connection negotiated a codec for
|
|
2600
|
+
* the track (see `setCodecPreference()`): it is encoded here, with
|
|
2601
|
+
* WebCodecs, as the next frame of the track's stream, and the model still
|
|
2602
|
+
* reads RGB. A track's first frame on a connection is a keyframe, and so
|
|
2603
|
+
* is the next one after the runtime could not decode its frames.
|
|
2604
|
+
*
|
|
2044
2605
|
* Pushes keep their call order, and a `step()` waits for every push called
|
|
2045
2606
|
* before it. A refused push rejects: await it, or `.catch()` it in a frame
|
|
2046
|
-
* loop, or every bad frame is an unhandled rejection. What
|
|
2607
|
+
* loop, or every bad frame is an unhandled rejection. What is buffered for
|
|
2047
2608
|
* the next step is at most one message's worth of frames: pushing faster
|
|
2048
2609
|
* than steps run drops the oldest frames, across tracks. Rejects with
|
|
2049
2610
|
* an `InvalidStateError` outside a stepped session, for a track not
|
|
@@ -2082,14 +2643,13 @@ var Reactor = class {
|
|
|
2082
2643
|
this.checkPushTarget("pushFrame", trackName, publication, epoch);
|
|
2083
2644
|
const client = await this.getOrCreateClient();
|
|
2084
2645
|
if (pixels) {
|
|
2085
|
-
|
|
2646
|
+
await this.pushPixels(
|
|
2647
|
+
client,
|
|
2086
2648
|
trackName,
|
|
2087
|
-
pixels
|
|
2088
|
-
pixels.width,
|
|
2089
|
-
pixels.height,
|
|
2090
|
-
pixels.format,
|
|
2649
|
+
pixels,
|
|
2091
2650
|
options.metadata ?? null,
|
|
2092
|
-
options.captureTimeUs ?? null
|
|
2651
|
+
options.captureTimeUs ?? null,
|
|
2652
|
+
() => this.checkPushTarget("pushFrame", trackName, publication, epoch)
|
|
2093
2653
|
);
|
|
2094
2654
|
} else if (isAudioFrame(frame)) {
|
|
2095
2655
|
client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
|
|
@@ -2104,18 +2664,22 @@ var Reactor = class {
|
|
|
2104
2664
|
}
|
|
2105
2665
|
}
|
|
2106
2666
|
/**
|
|
2107
|
-
*
|
|
2108
|
-
* video track `trackName`, its bytes as they are: one
|
|
2109
|
-
* (Annex B),
|
|
2110
|
-
* `
|
|
2111
|
-
*
|
|
2667
|
+
* Pushes one encoded video frame, sent or buffered as `pushFrame()` does,
|
|
2668
|
+
* on the sendonly video track `trackName`, its bytes as they are: one
|
|
2669
|
+
* H.264 access unit (Annex B), one VP8 or VP9 frame, or one AV1 temporal
|
|
2670
|
+
* unit, with the `width` and `height` it decodes to. The `PublishedTrack`
|
|
2671
|
+
* that `publishTrack()` resolves with does the same through
|
|
2672
|
+
* `track.pushEncodedFrame(frame)`.
|
|
2112
2673
|
*
|
|
2113
|
-
* The runtime decodes a track's encoded frames
|
|
2114
|
-
*
|
|
2115
|
-
*
|
|
2116
|
-
*
|
|
2117
|
-
*
|
|
2118
|
-
*
|
|
2674
|
+
* The runtime decodes a track's encoded frames as one stream, as they
|
|
2675
|
+
* arrive, and the model reads the RGB frames as if they had been pushed
|
|
2676
|
+
* raw. So push them in decode order, starting at a keyframe (`keyframe`, or
|
|
2677
|
+
* a chunk's `type`). A pushed frame that does not decode raises an
|
|
2678
|
+
* `input_decode_failed` error event naming the track, whose next frame
|
|
2679
|
+
* must then be a keyframe; a step whose buffered frames do not decode
|
|
2680
|
+
* rejects with a `StepRefusedError`, as does a runtime that predates
|
|
2681
|
+
* encoded frames. Encoded and raw pushes keep their call order, and a
|
|
2682
|
+
* `step()` waits for every push called before it.
|
|
2119
2683
|
*
|
|
2120
2684
|
* ```ts
|
|
2121
2685
|
* const encoder = new VideoEncoder({
|
|
@@ -2127,7 +2691,7 @@ var Reactor = class {
|
|
|
2127
2691
|
*
|
|
2128
2692
|
* Rejects with an `InvalidStateError` outside a stepped session, for a
|
|
2129
2693
|
* track not published on this connection, for an audio or recvonly track,
|
|
2130
|
-
* for empty data, a zero dimension or
|
|
2694
|
+
* for empty data, a zero dimension or an unknown codec.
|
|
2131
2695
|
*/
|
|
2132
2696
|
async pushEncodedFrame(trackName, frame) {
|
|
2133
2697
|
return this.pushEncodedTo(trackName, void 0, frame);
|
|
@@ -2168,7 +2732,8 @@ var Reactor = class {
|
|
|
2168
2732
|
encoded.width,
|
|
2169
2733
|
encoded.height,
|
|
2170
2734
|
encoded.metadata,
|
|
2171
|
-
encoded.captureTimeUs
|
|
2735
|
+
encoded.captureTimeUs,
|
|
2736
|
+
encoded.keyframe
|
|
2172
2737
|
);
|
|
2173
2738
|
})
|
|
2174
2739
|
);
|
|
@@ -2196,6 +2761,61 @@ var Reactor = class {
|
|
|
2196
2761
|
);
|
|
2197
2762
|
}
|
|
2198
2763
|
}
|
|
2764
|
+
/** Hands `pixels` to the binding for `track`: encoded first, on the
|
|
2765
|
+
* track's encoder, when the connection negotiated a codec for the track's
|
|
2766
|
+
* step frames and this browser can encode; raw otherwise, and when the
|
|
2767
|
+
* encoder fails (its next frame is then a keyframe). Checked by the
|
|
2768
|
+
* binding before anything is encoded, so a bad frame is refused either
|
|
2769
|
+
* way. `stillTarget` throws when the push's connection or publication is
|
|
2770
|
+
* gone: asked again once the encode is awaited, since a reconnect can
|
|
2771
|
+
* start meanwhile, and neither the encoded frame nor its raw fallback may
|
|
2772
|
+
* then reach the next session. */
|
|
2773
|
+
async pushPixels(client, track, pixels, metadata, captureTimeUs, stillTarget) {
|
|
2774
|
+
const { data, width, height, format } = pixels;
|
|
2775
|
+
const encoding = this.inputEncoders.available ? client.stepFrameEncodings()[track] : void 0;
|
|
2776
|
+
if (encoding !== void 0 && encoding !== "raw") {
|
|
2777
|
+
client.checkVideoFrame(track, data.byteLength, width, height, format);
|
|
2778
|
+
const keyframe = client.takeInputKeyframeRequest(track);
|
|
2779
|
+
let encoded;
|
|
2780
|
+
try {
|
|
2781
|
+
encoded = await this.inputEncoders.encode(track, encoding, pixels, keyframe);
|
|
2782
|
+
} catch (cause) {
|
|
2783
|
+
console.warn(`[Reactor] pushFrame('${track}'): ${encoding} encoding failed, sending the frame raw:`, cause);
|
|
2784
|
+
}
|
|
2785
|
+
stillTarget();
|
|
2786
|
+
if (encoded !== void 0) {
|
|
2787
|
+
for (const frame of encoded) {
|
|
2788
|
+
client.pushEncodedVideoFrame(track, encoding, frame.data, width, height, metadata, captureTimeUs, frame.keyframe);
|
|
2789
|
+
}
|
|
2790
|
+
return;
|
|
2791
|
+
}
|
|
2792
|
+
}
|
|
2793
|
+
client.pushVideoFrame(track, data, width, height, format, metadata, captureTimeUs);
|
|
2794
|
+
}
|
|
2795
|
+
/** Gives the binding the codecs this browser can carry step frames in,
|
|
2796
|
+
* probed once, so the next offer can negotiate them. A streamed session
|
|
2797
|
+
* carries no step frames. */
|
|
2798
|
+
/** Restart each output track whose decoder lost its stream: a pause and
|
|
2799
|
+
* a resume of that track, so the runtime starts it again at a keyframe. A
|
|
2800
|
+
* restart that cannot be sent is left to the next connection, whose
|
|
2801
|
+
* streams start at keyframes anyway. */
|
|
2802
|
+
restartLostTracks(client, decoders) {
|
|
2803
|
+
for (const track of decoders.takeLostTracks()) {
|
|
2804
|
+
try {
|
|
2805
|
+
client.restartTrackStream(track);
|
|
2806
|
+
} catch (cause) {
|
|
2807
|
+
console.warn(`step(): could not restart track '${track}':`, cause);
|
|
2808
|
+
}
|
|
2809
|
+
}
|
|
2810
|
+
}
|
|
2811
|
+
async offerStepCodecs(client) {
|
|
2812
|
+
if (!this.isStepped()) {
|
|
2813
|
+
return;
|
|
2814
|
+
}
|
|
2815
|
+
this.localStepCodecs ??= probeStepCodecs();
|
|
2816
|
+
const { encode, decode } = await this.localStepCodecs;
|
|
2817
|
+
client.setLocalStepCodecs(encode, decode);
|
|
2818
|
+
}
|
|
2199
2819
|
/** Pushes `step()`'s `media`, track by track and in order, each frame as
|
|
2200
2820
|
* `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
|
|
2201
2821
|
* checks the step's `state` and `commands`, then every frame, before the
|
|
@@ -2231,11 +2851,18 @@ var Reactor = class {
|
|
|
2231
2851
|
frame.encoded.width,
|
|
2232
2852
|
frame.encoded.height,
|
|
2233
2853
|
frame.encoded.metadata,
|
|
2234
|
-
frame.encoded.captureTimeUs
|
|
2854
|
+
frame.encoded.captureTimeUs,
|
|
2855
|
+
frame.encoded.keyframe
|
|
2235
2856
|
);
|
|
2236
2857
|
} else if (frame.kind === "video") {
|
|
2237
|
-
|
|
2238
|
-
|
|
2858
|
+
await this.pushPixels(
|
|
2859
|
+
client,
|
|
2860
|
+
frame.track,
|
|
2861
|
+
frame.pixels,
|
|
2862
|
+
null,
|
|
2863
|
+
null,
|
|
2864
|
+
() => this.checkPushTarget(frame.operation, frame.track, void 0, epoch)
|
|
2865
|
+
);
|
|
2239
2866
|
} else {
|
|
2240
2867
|
const { samples, sampleRate, channels } = frame.audio;
|
|
2241
2868
|
client.pushAudioFrame(frame.track, samples, sampleRate, channels ?? 1);
|
|
@@ -2246,8 +2873,63 @@ var Reactor = class {
|
|
|
2246
2873
|
this.pendingPushes = push.catch(() => void 0);
|
|
2247
2874
|
await push;
|
|
2248
2875
|
}
|
|
2249
|
-
/**
|
|
2250
|
-
*
|
|
2876
|
+
/**
|
|
2877
|
+
* Chooses the codecs a stepped session carries its video step frames in,
|
|
2878
|
+
* most preferred first, from `"h264"`, `"vp9"`, `"av1"` and `"vp8"`.
|
|
2879
|
+
* `null` goes back to the default, H.264 first. An empty list for the
|
|
2880
|
+
* session turns encoding off, so every frame travels raw:
|
|
2881
|
+
*
|
|
2882
|
+
* ```ts
|
|
2883
|
+
* reactor.setCodecPreference(['vp9', 'h264']); // VP9 when both ends have it
|
|
2884
|
+
* reactor.setCodecPreference([]); // raw frames
|
|
2885
|
+
* reactor.setCodecPreference(['av1'], { track: 'main_video' });
|
|
2886
|
+
* ```
|
|
2887
|
+
*
|
|
2888
|
+
* With `track`, the list applies to that track only, in place of the
|
|
2889
|
+
* session's; an empty one there goes back to the session's, since one
|
|
2890
|
+
* track cannot turn encoding off alone. The list is an order, not a
|
|
2891
|
+
* limit: the codecs it leaves out follow it, so a runtime without the
|
|
2892
|
+
* preferred ones takes another. Of those, the ones this browser's
|
|
2893
|
+
* WebCodecs can encode (input tracks) or decode (output tracks) are
|
|
2894
|
+
* offered, and the runtime takes per track the first it shares; a track
|
|
2895
|
+
* they share no codec for stays raw.
|
|
2896
|
+
*
|
|
2897
|
+
* Whatever the codec, `step()` results and the frames `pushFrame()` takes
|
|
2898
|
+
* stay RGB: the encoding is between this client and the runtime. Callable
|
|
2899
|
+
* before `connect()` or during a session; it takes effect on the next
|
|
2900
|
+
* connection or reconnect, and the live connection keeps what it
|
|
2901
|
+
* negotiated (see `getStepFrameEncodings()`). A streamed session ignores
|
|
2902
|
+
* it. Throws an `InvalidStateError` on a name that is not a codec.
|
|
2903
|
+
*/
|
|
2904
|
+
setCodecPreference(codecs, options = {}) {
|
|
2905
|
+
this.assertNotDisposed();
|
|
2906
|
+
const operation = "setCodecPreference";
|
|
2907
|
+
const track = options.track ?? null;
|
|
2908
|
+
if (codecs !== null && (!Array.isArray(codecs) || codecs.some((codec) => !STEP_CODECS.includes(codec)))) {
|
|
2909
|
+
throw this.captureError(
|
|
2910
|
+
new InvalidStateError(
|
|
2911
|
+
`${operation}: expected null or an array of "h264", "vp9", "av1" and "vp8", got ${JSON.stringify(codecs)}`,
|
|
2912
|
+
{ operation }
|
|
2913
|
+
)
|
|
2914
|
+
);
|
|
2915
|
+
}
|
|
2916
|
+
if (track !== null && typeof track !== "string") {
|
|
2917
|
+
throw this.captureError(new InvalidStateError(`${operation}: track must be a string`, { operation }));
|
|
2918
|
+
}
|
|
2919
|
+
const list = codecs === null ? null : [...codecs];
|
|
2920
|
+
this.codecPreferences.set(track, list);
|
|
2921
|
+
this.client?.setCodecPreference(list, track);
|
|
2922
|
+
}
|
|
2923
|
+
/** How each track's step frames travel on this connection, as its offer
|
|
2924
|
+
* and answer negotiated them: `"raw"`, or the codec of an encoded video
|
|
2925
|
+
* track. Empty outside a connected stepped session; a track left out is
|
|
2926
|
+
* raw. */
|
|
2927
|
+
getStepFrameEncodings() {
|
|
2928
|
+
return this.client?.stepFrameEncodings() ?? {};
|
|
2929
|
+
}
|
|
2930
|
+
/** Drops every frame still buffered for the next step, pushes called before
|
|
2931
|
+
* it included: it waits its turn behind them, as a `step()` does. Frames
|
|
2932
|
+
* already sent to the runtime are not recalled. */
|
|
2251
2933
|
async clearStepInputs() {
|
|
2252
2934
|
this.assertNotDisposed();
|
|
2253
2935
|
const clear = this.pendingPushes.then(
|
|
@@ -2438,6 +3120,9 @@ var Reactor = class {
|
|
|
2438
3120
|
throw new Error("Reactor was disposed while connecting.");
|
|
2439
3121
|
}
|
|
2440
3122
|
const client = new WasmReactorClient(this.clientOptions, this.jwt);
|
|
3123
|
+
for (const [track, codecs] of this.codecPreferences) {
|
|
3124
|
+
client.setCodecPreference(codecs, track);
|
|
3125
|
+
}
|
|
2441
3126
|
client.onStatusChanged((status) => {
|
|
2442
3127
|
this.emitter.emit("statusChanged", status);
|
|
2443
3128
|
this.handleStatusChanged(client, status);
|
|
@@ -2526,6 +3211,11 @@ var Reactor = class {
|
|
|
2526
3211
|
* → "ready" status sequence (see `ReactorStatus`), and starts/stops stats
|
|
2527
3212
|
* polling around the "ready" window. */
|
|
2528
3213
|
handleStatusChanged(client, status) {
|
|
3214
|
+
if (status === "connecting" || status === "disconnected") {
|
|
3215
|
+
this.resetStepStreams();
|
|
3216
|
+
this.connectionEpoch += 1;
|
|
3217
|
+
this.publishedInputs.clear();
|
|
3218
|
+
}
|
|
2529
3219
|
switch (status) {
|
|
2530
3220
|
case "connecting":
|
|
2531
3221
|
this.connectStartTime = performance.now();
|
|
@@ -2617,6 +3307,7 @@ var Reactor = class {
|
|
|
2617
3307
|
* leave `getCapabilities()` returning the previous session's stale
|
|
2618
3308
|
* tracks/commands until a new `capabilitiesReceived` lands. */
|
|
2619
3309
|
resetConnectionState() {
|
|
3310
|
+
this.resetStepStreams();
|
|
2620
3311
|
this.connectionEpoch += 1;
|
|
2621
3312
|
this.publishedInputs.clear();
|
|
2622
3313
|
this.stopStatsPolling();
|
|
@@ -2625,6 +3316,12 @@ var Reactor = class {
|
|
|
2625
3316
|
this.waitingStartTime = void 0;
|
|
2626
3317
|
this.capabilities = void 0;
|
|
2627
3318
|
}
|
|
3319
|
+
/** Drops every step frame encoder and decoder: each track's stream lived
|
|
3320
|
+
* with the connection that ended, and the next one's start at keyframes. */
|
|
3321
|
+
resetStepStreams() {
|
|
3322
|
+
this.inputEncoders.reset();
|
|
3323
|
+
this.outputDecoders.reset();
|
|
3324
|
+
}
|
|
2628
3325
|
/** Wraps `cause` and records it as the most recent failure, for a call
|
|
2629
3326
|
* that's about to throw rather than emit — see `getLastError()`'s doc
|
|
2630
3327
|
* comment. */
|