@reactor-team/js-sdk 3.3.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -330,6 +330,63 @@ var Emitter = class {
330
330
  }
331
331
  };
332
332
 
333
+ // src/internal/encoded-frames.ts
334
+ var CODECS = /* @__PURE__ */ new Set(["h264", "vp8", "vp9", "av1"]);
335
+ function isEncodedFrame(frame) {
336
+ return typeof frame === "object" && frame !== null && "codec" in frame;
337
+ }
338
+ function isChunk(data) {
339
+ return !(data instanceof Uint8Array) && !(data instanceof ArrayBuffer) && typeof data.copyTo === "function";
340
+ }
341
+ function toEncodedBytes(frame, operation, trackName) {
342
+ const refuse = (why) => new InvalidStateError(`${operation}('${trackName}'): ${why}`, { operation });
343
+ if (typeof frame !== "object" || frame === null) {
344
+ throw refuse("expected an EncodedFrame, { codec, data, width, height }");
345
+ }
346
+ const { codec, data, width, height, metadata, captureTimeUs, keyframe } = frame;
347
+ if (!CODECS.has(codec)) {
348
+ throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264", "vp8", "vp9" or "av1"`);
349
+ }
350
+ if (keyframe != null && typeof keyframe !== "boolean") {
351
+ throw refuse("keyframe must be a boolean");
352
+ }
353
+ let bytes;
354
+ if (data instanceof Uint8Array) {
355
+ bytes = data;
356
+ } else if (data instanceof ArrayBuffer) {
357
+ bytes = new Uint8Array(data);
358
+ } else if (data && isChunk(data)) {
359
+ bytes = new Uint8Array(data.byteLength);
360
+ data.copyTo(bytes);
361
+ } else {
362
+ throw refuse("data must be a Uint8Array, an ArrayBuffer or an EncodedVideoChunk");
363
+ }
364
+ if (bytes.byteLength === 0) {
365
+ throw refuse("an encoded frame needs data");
366
+ }
367
+ if (!isDimension(width) || !isDimension(height)) {
368
+ throw refuse(`width and height are the decoded frame's, got ${String(width)}x${String(height)}`);
369
+ }
370
+ if (metadata != null && !(metadata instanceof Uint8Array)) {
371
+ throw refuse("metadata must be a Uint8Array");
372
+ }
373
+ if (captureTimeUs != null && !Number.isFinite(captureTimeUs)) {
374
+ throw refuse("captureTimeUs must be a finite number");
375
+ }
376
+ return {
377
+ codec,
378
+ data: bytes,
379
+ width,
380
+ height,
381
+ metadata: metadata ?? null,
382
+ captureTimeUs: captureTimeUs ?? null,
383
+ keyframe: keyframe ?? (isChunk(data) && data.type === "key")
384
+ };
385
+ }
386
+ function isDimension(value) {
387
+ return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 4294967295;
388
+ }
389
+
333
390
  // src/file-ref.ts
334
391
  var FileRef = class {
335
392
  constructor(uploadId, name, mimeType, size) {
@@ -540,6 +597,574 @@ function toPublicClip(clip) {
540
597
  };
541
598
  }
542
599
 
600
+ // src/internal/step-media.ts
601
+ function holdStepMedia(media) {
602
+ if (typeof media !== "object" || media === null || Array.isArray(media)) {
603
+ throw new InvalidStateError("step(): media must map track names to arrays of frames", {
604
+ operation: "step"
605
+ });
606
+ }
607
+ const held = [];
608
+ try {
609
+ for (const [trackName, frames] of Object.entries(media)) {
610
+ if (!Array.isArray(frames)) {
611
+ throw new InvalidStateError(`step(): media['${trackName}'] must be an array of frames`, {
612
+ operation: "step"
613
+ });
614
+ }
615
+ for (const frame of frames) {
616
+ held.push(holdStepFrame(trackName, frame));
617
+ }
618
+ }
619
+ } catch (cause) {
620
+ for (const frame of held) {
621
+ frame.release();
622
+ }
623
+ throw cause;
624
+ }
625
+ return held;
626
+ }
627
+ function holdStepFrame(track, frame) {
628
+ if (isEncodedFrame(frame)) {
629
+ const operation = "pushEncodedFrame";
630
+ return { track, kind: "encoded", operation, encoded: toEncodedBytes(frame, operation, track), release: () => {
631
+ } };
632
+ }
633
+ if (typeof frame !== "object" || frame === null) {
634
+ throw new InvalidStateError(`pushFrame('${track}'): expected a frame`, { operation: "pushFrame" });
635
+ }
636
+ const held = holdFrame(frame);
637
+ return { track, kind: "raw", operation: "pushFrame", frame: held.frame, release: held.release };
638
+ }
639
+ async function readStepFrame(frame) {
640
+ const { track } = frame;
641
+ if (frame.kind === "encoded") {
642
+ return { track, kind: "encoded", operation: frame.operation, encoded: frame.encoded };
643
+ }
644
+ if (isAudioFrame(frame.frame)) {
645
+ return { track, kind: "audio", operation: frame.operation, audio: frame.frame };
646
+ }
647
+ return { track, kind: "video", operation: frame.operation, pixels: await toPixels(frame.frame) };
648
+ }
649
+ function checkStepFrame(client, frame) {
650
+ if (frame.kind === "encoded") {
651
+ const { codec, data, width, height, captureTimeUs } = frame.encoded;
652
+ client.checkEncodedVideoFrame(frame.track, codec, data.byteLength, width, height, captureTimeUs);
653
+ } else if (frame.kind === "video") {
654
+ const { data, width, height, format } = frame.pixels;
655
+ client.checkVideoFrame(frame.track, data.byteLength, width, height, format);
656
+ } else {
657
+ client.checkAudioFrame(frame.track, frame.audio.samples, frame.audio.channels ?? 1);
658
+ }
659
+ }
660
+
661
+ // src/internal/bt601.ts
662
+ var BT601_LIMITED = {
663
+ matrix: "smpte170m",
664
+ primaries: "smpte170m",
665
+ transfer: "smpte170m",
666
+ fullRange: false
667
+ };
668
+ function clamp(value) {
669
+ return value < 0 ? 0 : value > 255 ? 255 : value;
670
+ }
671
+ function rgbToI420(pixels) {
672
+ const { width, height, data } = pixels;
673
+ const step = pixels.format === "rgb" ? 3 : 4;
674
+ const [ri, bi] = pixels.format === "bgra" ? [2, 0] : [0, 2];
675
+ const codedWidth = width + (width & 1);
676
+ const codedHeight = height + (height & 1);
677
+ const chromaWidth = codedWidth >> 1;
678
+ const chromaHeight = codedHeight >> 1;
679
+ const ySize = codedWidth * codedHeight;
680
+ const out = new Uint8Array(ySize + 2 * chromaWidth * chromaHeight);
681
+ const uOffset = ySize;
682
+ const vOffset = ySize + chromaWidth * chromaHeight;
683
+ const at = (x, y) => (Math.min(y, height - 1) * width + Math.min(x, width - 1)) * step;
684
+ for (let y = 0; y < codedHeight; y++) {
685
+ for (let x = 0; x < codedWidth; x++) {
686
+ const i = at(x, y);
687
+ const r = data[i + ri];
688
+ const g = data[i + 1];
689
+ const b = data[i + bi];
690
+ out[y * codedWidth + x] = 66 * r + 129 * g + 25 * b + 4224 >> 8 & 255;
691
+ }
692
+ }
693
+ for (let cy = 0; cy < chromaHeight; cy++) {
694
+ for (let cx = 0; cx < chromaWidth; cx++) {
695
+ let r = 0;
696
+ let g = 0;
697
+ let b = 0;
698
+ for (const [dx, dy] of [
699
+ [0, 0],
700
+ [1, 0],
701
+ [0, 1],
702
+ [1, 1]
703
+ ]) {
704
+ const i = at(2 * cx + dx, 2 * cy + dy);
705
+ r += data[i + ri];
706
+ g += data[i + 1];
707
+ b += data[i + bi];
708
+ }
709
+ r = r + 2 >> 2;
710
+ g = g + 2 >> 2;
711
+ b = b + 2 >> 2;
712
+ out[uOffset + cy * chromaWidth + cx] = 112 * b - 74 * g - 38 * r + 32896 >> 8;
713
+ out[vOffset + cy * chromaWidth + cx] = 112 * r - 94 * g - 18 * b + 32896 >> 8;
714
+ }
715
+ }
716
+ return { data: out, width, height, codedWidth, codedHeight };
717
+ }
718
+ function yuvToRgb(data, { format, width, height, planes }) {
719
+ const [yPlane, uPlane, vPlane] = planes;
720
+ if (yPlane === void 0 || uPlane === void 0 || format === "I420" && vPlane === void 0) {
721
+ throw new Error(`a ${format} picture needs ${format === "I420" ? 3 : 2} planes, got ${planes.length}`);
722
+ }
723
+ const rgb = new Uint8Array(width * height * 3);
724
+ for (let y = 0; y < height; y++) {
725
+ const cy = y >> 1;
726
+ for (let x = 0; x < width; x++) {
727
+ const cx = x >> 1;
728
+ const luma = data[yPlane.offset + y * yPlane.stride + x];
729
+ let u;
730
+ let v;
731
+ if (format === "NV12") {
732
+ u = data[uPlane.offset + cy * uPlane.stride + 2 * cx];
733
+ v = data[uPlane.offset + cy * uPlane.stride + 2 * cx + 1];
734
+ } else {
735
+ u = data[uPlane.offset + cy * uPlane.stride + cx];
736
+ v = data[vPlane.offset + cy * vPlane.stride + cx];
737
+ }
738
+ const c = 1.164 * (luma - 16);
739
+ const d = u - 128;
740
+ const e = v - 128;
741
+ const j = (y * width + x) * 3;
742
+ rgb[j] = clamp(Math.round(c + 1.596 * e));
743
+ rgb[j + 1] = clamp(Math.round(c - 0.391 * d - 0.813 * e));
744
+ rgb[j + 2] = clamp(Math.round(c + 2.018 * d));
745
+ }
746
+ }
747
+ return rgb;
748
+ }
749
+
750
+ // src/internal/step-codecs.ts
751
+ var STEP_CODECS = ["h264", "vp9", "av1", "vp8"];
752
+ var DECODE_TIMEOUT_MS = 1e4;
753
+ function codecString(codec) {
754
+ switch (codec) {
755
+ case "h264":
756
+ return "avc1.42e033";
757
+ case "vp8":
758
+ return "vp8";
759
+ case "vp9":
760
+ return "vp09.00.51.08";
761
+ case "av1":
762
+ return "av01.0.13M.08";
763
+ }
764
+ }
765
+ function encoderConfig(codec, width, height) {
766
+ return {
767
+ codec: codecString(codec),
768
+ width,
769
+ height,
770
+ // Rate-controlled, as a real-time encoder is: enough for a still image
771
+ // to come back close, small enough to keep a step's message small.
772
+ bitrate: Math.min(2e7, Math.max(3e5, width * height * 4)),
773
+ framerate: 30,
774
+ latencyMode: "realtime",
775
+ ...codec === "h264" ? { avc: { format: "annexb" } } : {}
776
+ };
777
+ }
778
+ function decoderConfig(codec) {
779
+ return { codec: codecString(codec), optimizeForLatency: true };
780
+ }
781
+ async function probeStepCodecs(api = globalThis) {
782
+ const supported = async (probe) => {
783
+ const results = await Promise.all(STEP_CODECS.map((codec) => probe(codec).catch(() => false)));
784
+ return STEP_CODECS.filter((_, i) => results[i]);
785
+ };
786
+ const { VideoEncoder: Encoder, VideoDecoder: Decoder, VideoFrame: Frame, EncodedVideoChunk: Chunk } = api;
787
+ const [encode, decode] = await Promise.all([
788
+ Encoder && Frame ? supported(async (codec) => (await Encoder.isConfigSupported(encoderConfig(codec, 1280, 720))).supported === true) : [],
789
+ Decoder && Chunk ? supported(async (codec) => (await Decoder.isConfigSupported(decoderConfig(codec))).supported === true) : []
790
+ ]);
791
+ return { encode, decode };
792
+ }
793
+ var TrackEncoder = class {
794
+ constructor(api, codec, width, height) {
795
+ this.api = api;
796
+ this.codec = codec;
797
+ this.width = width;
798
+ this.height = height;
799
+ this.encoder = new api.VideoEncoder({
800
+ output: (chunk) => {
801
+ const data = new Uint8Array(chunk.byteLength);
802
+ chunk.copyTo(data);
803
+ this.outputs.push({ data, keyframe: chunk.type === "key" });
804
+ },
805
+ error: (error) => {
806
+ this.failure = error instanceof Error ? error : new Error(String(error));
807
+ }
808
+ });
809
+ try {
810
+ this.encoder.configure(encoderConfig(codec, width, height));
811
+ } catch (cause) {
812
+ this.close();
813
+ throw cause;
814
+ }
815
+ }
816
+ api;
817
+ codec;
818
+ width;
819
+ height;
820
+ encoder;
821
+ outputs = [];
822
+ failure;
823
+ started = false;
824
+ timestampUs = 0;
825
+ /** Encode `pixels` as the stream's next frame, a keyframe when `keyframe`
826
+ * or when it is the stream's first, and hand back what the encoder made
827
+ * of it: every frame is flushed out before the next goes in. */
828
+ async encode(pixels, keyframe) {
829
+ const i420 = rgbToI420(pixels);
830
+ const frame = new this.api.VideoFrame(i420.data, {
831
+ format: "I420",
832
+ codedWidth: i420.codedWidth,
833
+ codedHeight: i420.codedHeight,
834
+ visibleRect: { x: 0, y: 0, width: i420.width, height: i420.height },
835
+ colorSpace: BT601_LIMITED,
836
+ timestamp: this.timestampUs
837
+ });
838
+ this.timestampUs += 33333;
839
+ try {
840
+ this.encoder.encode(frame, { keyFrame: keyframe || !this.started });
841
+ } finally {
842
+ frame.close();
843
+ }
844
+ this.started = true;
845
+ await this.encoder.flush();
846
+ if (this.failure !== void 0) {
847
+ throw this.failure;
848
+ }
849
+ return this.outputs.splice(0);
850
+ }
851
+ close() {
852
+ if (this.encoder.state !== "closed") {
853
+ this.encoder.close();
854
+ }
855
+ }
856
+ };
857
+ var InputEncoders = class {
858
+ constructor(api = globalThis) {
859
+ this.api = api;
860
+ }
861
+ api;
862
+ encoders = /* @__PURE__ */ new Map();
863
+ /** Whether this browser can encode at all. */
864
+ get available() {
865
+ return this.api.VideoEncoder !== void 0 && this.api.VideoFrame !== void 0;
866
+ }
867
+ /** Encode `pixels` as `track`'s next frame in `codec`. A track whose codec
868
+ * or size changed starts a new stream, at a keyframe. On a failure the
869
+ * track's encoder is dropped, so its next frame is a keyframe, and the
870
+ * error is thrown for the caller to send the frame raw instead. */
871
+ async encode(track, codec, pixels, keyframe) {
872
+ const { VideoEncoder: Encoder, VideoFrame: Frame } = this.api;
873
+ if (!Encoder || !Frame) {
874
+ throw new Error("this browser has no WebCodecs VideoEncoder");
875
+ }
876
+ let encoder = this.encoders.get(track);
877
+ if (encoder && (encoder.codec !== codec || encoder.width !== pixels.width || encoder.height !== pixels.height)) {
878
+ encoder.close();
879
+ encoder = void 0;
880
+ }
881
+ try {
882
+ if (!encoder) {
883
+ encoder = new TrackEncoder({ VideoEncoder: Encoder, VideoFrame: Frame }, codec, pixels.width, pixels.height);
884
+ this.encoders.set(track, encoder);
885
+ }
886
+ return await encoder.encode(pixels, keyframe);
887
+ } catch (cause) {
888
+ this.drop(track);
889
+ throw cause;
890
+ }
891
+ }
892
+ /** Drop `track`'s encoder: its next frame starts a new stream. */
893
+ drop(track) {
894
+ this.encoders.get(track)?.close();
895
+ this.encoders.delete(track);
896
+ }
897
+ reset() {
898
+ for (const track of [...this.encoders.keys()]) {
899
+ this.drop(track);
900
+ }
901
+ }
902
+ };
903
+ var TrackDecoder = class {
904
+ constructor(api, codec) {
905
+ this.codec = codec;
906
+ this.Chunk = api.EncodedVideoChunk;
907
+ this.decoder = new api.VideoDecoder({
908
+ output: (frame) => this.deliver(frame),
909
+ error: (error) => this.fail(error)
910
+ });
911
+ try {
912
+ this.decoder.configure(decoderConfig(codec));
913
+ } catch (cause) {
914
+ this.close();
915
+ throw cause;
916
+ }
917
+ }
918
+ codec;
919
+ decoder;
920
+ Chunk;
921
+ /** Resolvers of the frames handed in and not yet handed back, by the
922
+ * timestamp each chunk was given, in the order they were handed in. */
923
+ pending = /* @__PURE__ */ new Map();
924
+ /** The next chunk's timestamp: a count, so each output names the one chunk
925
+ * it was made of, whatever capture time the frame carried. */
926
+ nextTimestamp = 0;
927
+ failed = false;
928
+ /** Hand `frame` to the chunk it was made of, by its timestamp. A step's
929
+ * stream is decoded in order, with no frame reordered, so a chunk handed
930
+ * in before that one with no picture yet made none (a frame that is not
931
+ * shown): it resolves with none rather than wait for the timeout, and no
932
+ * picture is ever matched to a chunk it was not made of. */
933
+ deliver(frame) {
934
+ const entry = this.pending.get(frame.timestamp);
935
+ if (!entry) {
936
+ frame.close();
937
+ return;
938
+ }
939
+ for (const [timestamp, earlier] of this.pending) {
940
+ if (timestamp === frame.timestamp) {
941
+ break;
942
+ }
943
+ this.pending.delete(timestamp);
944
+ earlier.resolve(void 0);
945
+ }
946
+ this.pending.delete(frame.timestamp);
947
+ entry.resolve(frame);
948
+ }
949
+ /** Hand a chunk of `data` to the decoder now, and resolve with the frame it
950
+ * makes of it, or `undefined` for one it shows none of. Called
951
+ * synchronously for every frame of a result, so the decoder takes them in
952
+ * the order results arrive. */
953
+ decode(type, data) {
954
+ const timestamp = this.nextTimestamp++;
955
+ const decoded = new Promise((resolve, reject) => {
956
+ this.pending.set(timestamp, { resolve, reject });
957
+ });
958
+ try {
959
+ this.decoder.decode(new this.Chunk({ type, timestamp, data }));
960
+ } catch (cause) {
961
+ this.fail(cause);
962
+ }
963
+ return withTimeout(decoded, DECODE_TIMEOUT_MS, () => {
964
+ this.fail(new Error(`the ${this.codec} decoder handed back no frame in ${DECODE_TIMEOUT_MS} ms`));
965
+ });
966
+ }
967
+ /** The decoder lost its state, or is given up: every frame still in it
968
+ * fails, so no step waits on it. */
969
+ fail(cause) {
970
+ this.failed = true;
971
+ const pending = [...this.pending.values()];
972
+ this.pending.clear();
973
+ for (const next of pending) {
974
+ next.reject(cause);
975
+ }
976
+ this.close();
977
+ }
978
+ close() {
979
+ if (this.decoder.state !== "closed") {
980
+ this.decoder.close();
981
+ }
982
+ }
983
+ };
984
+ function withTimeout(promise, ms, onTimeout) {
985
+ let timer;
986
+ const timeout = new Promise((_, reject) => {
987
+ timer = setTimeout(() => {
988
+ onTimeout();
989
+ reject(new Error(`timed out after ${ms} ms`));
990
+ }, ms);
991
+ });
992
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
993
+ }
994
+ var OutputDecoders = class {
995
+ constructor(api = globalThis) {
996
+ this.api = api;
997
+ }
998
+ api;
999
+ decoders = /* @__PURE__ */ new Map();
1000
+ awaitingKeyframe = /* @__PURE__ */ new Set();
1001
+ lost = [];
1002
+ /** Whether a track lost its decoder and waits for a keyframe. */
1003
+ needsKeyframe() {
1004
+ return this.awaitingKeyframe.size > 0;
1005
+ }
1006
+ /** The tracks that lost their stream since the last call, each once per
1007
+ * loss however many frames it then discards: the caller restarts each
1008
+ * one's stream. */
1009
+ takeLostTracks() {
1010
+ const lost = this.lost;
1011
+ this.lost = [];
1012
+ return lost;
1013
+ }
1014
+ /** A step gave up after its request went out: its result, if it still
1015
+ * comes, is never fed here, so every stream lost frames. Drop the
1016
+ * decoders, and report each of their tracks lost. */
1017
+ abandon() {
1018
+ for (const [track, decoder] of this.decoders) {
1019
+ decoder.fail(new DecodeError(`step(): track '${track}' lost its stream: a step gave up mid-stream`, { operation: "step" }));
1020
+ this.lose(track, false);
1021
+ }
1022
+ this.decoders.clear();
1023
+ }
1024
+ /** Make `track` wait for a keyframe, and report it lost when it was not
1025
+ * waiting already, or when `again` says this is a new loss. */
1026
+ lose(track, again) {
1027
+ if (!this.awaitingKeyframe.has(track) || again) {
1028
+ this.lost.push(track);
1029
+ }
1030
+ this.awaitingKeyframe.add(track);
1031
+ }
1032
+ /** `result` with every encoded frame decoded to RGB `(height, width, 3)`
1033
+ * `uint8`, as a raw frame comes. Call it as the result arrives: each
1034
+ * encoded frame goes into its track's decoder before this returns, so
1035
+ * results that arrive one after another are decoded in that order.
1036
+ *
1037
+ * Every frame of every track is fed to its decoder even after one fails,
1038
+ * so the other streams keep up; the first failure rejects, with a
1039
+ * `DecodeError`, rather than resolving a partial step. */
1040
+ decodeResult(result) {
1041
+ if (result.tracks.every((track) => track.frames.every((frame) => !isEncoded(frame)))) {
1042
+ return Promise.resolve(result);
1043
+ }
1044
+ let firstError;
1045
+ const tracks = result.tracks.map((track) => {
1046
+ const frames = track.frames.map(
1047
+ (frame) => this.decodeFrame(track.track, frame).catch((cause) => {
1048
+ firstError ??= cause;
1049
+ return void 0;
1050
+ })
1051
+ );
1052
+ return Promise.all(frames).then((decoded) => ({
1053
+ ...track,
1054
+ frames: decoded.filter((frame) => frame !== void 0)
1055
+ }));
1056
+ });
1057
+ return Promise.all(tracks).then((decoded) => {
1058
+ if (firstError !== void 0) {
1059
+ throw toDecodeError(firstError);
1060
+ }
1061
+ return { ...result, tracks: decoded };
1062
+ });
1063
+ }
1064
+ /** Decode one frame of `track`; a raw one passes as it is, and one its
1065
+ * decoder shows no picture of is `undefined`. Synchronous up to the
1066
+ * decoder taking the frame. */
1067
+ decodeFrame(track, frame) {
1068
+ if (!isEncoded(frame)) {
1069
+ return Promise.resolve(frame);
1070
+ }
1071
+ const codec = frame.encoding;
1072
+ const { VideoDecoder: Decoder, EncodedVideoChunk: Chunk } = this.api;
1073
+ if (!Decoder || !Chunk) {
1074
+ return Promise.reject(new DecodeError(`step(): '${track}' sent ${codec} frames and this browser has no WebCodecs VideoDecoder`, { operation: "step" }));
1075
+ }
1076
+ let decoder = this.decoders.get(track);
1077
+ if (decoder && (decoder.failed || decoder.codec !== codec)) {
1078
+ decoder.fail(new DecodeError(`step(): track '${track}' changed codec to ${codec}`, { operation: "step" }));
1079
+ this.decoders.delete(track);
1080
+ decoder = void 0;
1081
+ }
1082
+ if (!decoder) {
1083
+ if (!frame.keyframe) {
1084
+ this.lose(track, false);
1085
+ return Promise.reject(
1086
+ new DecodeError(
1087
+ `step(): track '${track}' has no decoder state for a ${codec} frame that is not a keyframe; waiting for a keyframe`,
1088
+ { operation: "step" }
1089
+ )
1090
+ );
1091
+ }
1092
+ try {
1093
+ decoder = new TrackDecoder({ VideoDecoder: Decoder, EncodedVideoChunk: Chunk }, codec);
1094
+ } catch (cause) {
1095
+ return Promise.reject(toDecodeError(cause));
1096
+ }
1097
+ this.decoders.set(track, decoder);
1098
+ }
1099
+ const live = decoder;
1100
+ return live.decode(frame.keyframe ? "key" : "delta", frame.data).then(
1101
+ async (picture) => {
1102
+ if (frame.keyframe) {
1103
+ this.awaitingKeyframe.delete(track);
1104
+ }
1105
+ return picture === void 0 ? void 0 : { ...frame, ...await rgbOf(picture) };
1106
+ },
1107
+ (cause) => {
1108
+ this.lose(track, frame.keyframe === true);
1109
+ if (this.decoders.get(track) === live) {
1110
+ this.decoders.delete(track);
1111
+ }
1112
+ throw cause;
1113
+ }
1114
+ );
1115
+ }
1116
+ reset() {
1117
+ for (const [track, decoder] of this.decoders) {
1118
+ decoder.fail(new DecodeError(`step(): track '${track}' lost its stream: its connection closed`, { operation: "step" }));
1119
+ }
1120
+ this.decoders.clear();
1121
+ this.awaitingKeyframe.clear();
1122
+ this.lost = [];
1123
+ }
1124
+ };
1125
+ function isEncoded(frame) {
1126
+ return frame.encoding !== "raw" && frame.encoding !== "unspecified";
1127
+ }
1128
+ function toDecodeError(cause) {
1129
+ if (cause instanceof DecodeError) {
1130
+ return cause;
1131
+ }
1132
+ const message = cause instanceof Error ? cause.message : JSON.stringify(cause);
1133
+ return new DecodeError(`step(): ${message}`, { operation: "step", cause });
1134
+ }
1135
+ async function rgbOf(picture) {
1136
+ let pixels;
1137
+ try {
1138
+ const yuv = await yuvOf(picture);
1139
+ if (yuv !== void 0) {
1140
+ return yuv;
1141
+ }
1142
+ pixels = await toPixels(picture);
1143
+ } finally {
1144
+ picture.close();
1145
+ }
1146
+ const { width, height } = pixels;
1147
+ const rgb = new Uint8Array(width * height * 3);
1148
+ const [r, b] = pixels.format === "bgra" ? [2, 0] : [0, 2];
1149
+ for (let i = 0, j = 0; j < rgb.length; i += 4, j += 3) {
1150
+ rgb[j] = pixels.data[i + r];
1151
+ rgb[j + 1] = pixels.data[i + 1];
1152
+ rgb[j + 2] = pixels.data[i + b];
1153
+ }
1154
+ return { dtype: "uint8", shape: [height, width, 3], data: rgb };
1155
+ }
1156
+ async function yuvOf(picture) {
1157
+ const format = picture.format === "I420A" ? "I420" : picture.format;
1158
+ if (format !== "I420" && format !== "NV12") {
1159
+ return void 0;
1160
+ }
1161
+ const width = picture.visibleRect?.width ?? picture.codedWidth;
1162
+ const height = picture.visibleRect?.height ?? picture.codedHeight;
1163
+ const data = new Uint8Array(picture.allocationSize());
1164
+ const planes = await picture.copyTo(data);
1165
+ return { dtype: "uint8", shape: [height, width, 3], data: yuvToRgb(data, { format, width, height, planes }) };
1166
+ }
1167
+
543
1168
  // src/internal/stats.ts
544
1169
  var LOCAL_STATS_INTERVAL_MS = 2e3;
545
1170
  var RUNTIME_REPORT_INTERVAL_MS = 5e3;
@@ -867,7 +1492,7 @@ function describeCause(cause) {
867
1492
  // package.json
868
1493
  var package_default = {
869
1494
  name: "@reactor-team/js-sdk",
870
- version: "3.3.0",
1495
+ version: "3.5.0",
871
1496
  description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
872
1497
  license: "Apache-2.0",
873
1498
  repository: {
@@ -1389,10 +2014,22 @@ var Reactor = class {
1389
2014
  * is finalized on "ready" — see `handleStatusChanged()`. */
1390
2015
  connectStartTime;
1391
2016
  waitingStartTime;
1392
- /** The tail of every `pushFrame()` so far, settled either way — pushes
1393
- * reach the binding in call order however long each one's pixels take to
1394
- * read, and `step()` waits for the ones called before it. */
2017
+ /** The tail of every `pushFrame()`/`pushEncodedFrame()` so far, settled
2018
+ * either way — pushes reach the binding in call order however long each
2019
+ * one's pixels take to read, and `step()` waits for the ones called
2020
+ * before it. */
1395
2021
  pendingPushes = Promise.resolve();
2022
+ /** Stepped: every `setCodecPreference()` so far, keyed by track (`null`
2023
+ * for the session's list), replayed into a client created later. */
2024
+ codecPreferences = /* @__PURE__ */ new Map();
2025
+ /** Stepped: what this browser's WebCodecs can encode and decode, probed
2026
+ * once, on the first stepped connect. */
2027
+ localStepCodecs;
2028
+ /** Stepped: the input tracks' encoders and the output tracks' decoders of
2029
+ * the current connection, dropped as it ends so the next one's streams
2030
+ * start at keyframes. */
2031
+ inputEncoders = new InputEncoders();
2032
+ outputDecoders = new OutputDecoders();
1396
2033
  emitter = new Emitter();
1397
2034
  /** Serializes connect()/reconnect()/disconnect() (and the free() inside
1398
2035
  * disconnect()/[Symbol.dispose]) against each other. Calling into the
@@ -1500,6 +2137,7 @@ var Reactor = class {
1500
2137
  await this.queue.push(
1501
2138
  () => this.withWriterAccess(async () => {
1502
2139
  const client = await this.getOrCreateClient();
2140
+ await this.offerStepCodecs(client);
1503
2141
  await client.connect(options);
1504
2142
  }),
1505
2143
  "connect"
@@ -1554,6 +2192,7 @@ var Reactor = class {
1554
2192
  await this.queue.push(
1555
2193
  () => this.withWriterAccess(async () => {
1556
2194
  const client = await this.getOrCreateClient();
2195
+ await this.offerStepCodecs(client);
1557
2196
  await client.reconnect(options);
1558
2197
  }),
1559
2198
  "reconnect"
@@ -1732,7 +2371,8 @@ var Reactor = class {
1732
2371
  }, "publishTrack");
1733
2372
  return {
1734
2373
  name,
1735
- pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
2374
+ pushFrame: (frame, options) => this.pushTo(name, publication, frame, options),
2375
+ pushEncodedFrame: (frame) => this.pushEncodedTo(name, publication, frame)
1736
2376
  };
1737
2377
  } catch (cause) {
1738
2378
  throw this.captureError(cause);
@@ -1828,10 +2468,10 @@ var Reactor = class {
1828
2468
  * Runs one step of a `generation: "stepped"` session and resolves with
1829
2469
  * what it produced, per output track.
1830
2470
  *
1831
- * The request carries `state`, `commands` and every frame pushed since the
1832
- * previous step. The runtime applies them in that order — each `state` key
1833
- * as the model's `set_<key>` command, then `commands`, then the frames —
1834
- * and then generates. They are separate calls, not a transaction: one that
2471
+ * The request carries `state` and `commands`; the frames pushed before it
2472
+ * have already reached the runtime (see `pushFrame()`). The runtime applies
2473
+ * them in that order — each `state` key as the model's `set_<key>` command,
2474
+ * then `commands` — and then generates. They are separate calls, not a transaction: one that
1835
2475
  * fails does not undo the ones before it. All of it is optional:
1836
2476
  * `step()` with nothing set and nothing pushed runs on the model's current
1837
2477
  * state, which commands sent outside a step (`sendCommand()`) also change.
@@ -1842,7 +2482,9 @@ var Reactor = class {
1842
2482
  * A step that ran but that the model reported as failed still resolves,
1843
2483
  * with `result.error` set. `result.files` lists the files the model kept
1844
2484
  * with the step (name, content type, size); their bytes are served over
1845
- * HTTP from the session's step results, not carried here.
2485
+ * HTTP from the session's step results, not carried here. `result.data`
2486
+ * holds what the model returned besides its media, as opaque bytes (a dict
2487
+ * arrives encoded as JSON); it is absent when the model returned none.
1846
2488
  *
1847
2489
  * Rejects with a `StepRefusedError` when the runtime or the model turns
1848
2490
  * the step down (the runtime checks a whole request first, so nothing it
@@ -1856,10 +2498,34 @@ var Reactor = class {
1856
2498
  * message, `MessageTooLargeError` when the request itself is too large
1857
2499
  * (nothing was sent), `InvalidStateError` outside a ready stepped
1858
2500
  * session, or whatever else failed the step (`DisconnectedError`,
1859
- * `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
2501
+ * `RequestTimeoutError`, ...). The frames buffered for a step belong to it:
1860
2502
  * when its request cannot be sent, they are dropped with it rather than
1861
2503
  * kept for the next step.
1862
2504
  *
2505
+ * `media` gives the step frames of its own, by input track name, pushed
2506
+ * after the ones already pushed to that track: what `pushFrame()` takes,
2507
+ * or an `EncodedFrame` — H.264 the runtime decodes before the step runs,
2508
+ * so the model reads RGB frames as for a raw push. They are pushed only
2509
+ * once `state` and `commands` are well formed and the session is ready, so
2510
+ * a step refused for those leaves the frames buffered as they were. They go
2511
+ * all or nothing: a bad track or frame anywhere in `media` rejects the step
2512
+ * before any of it is pushed or sent, leaving the buffer as it was. A
2513
+ * session that leaves 'ready' once the media is in has the step refused and
2514
+ * keeps the frames for the next one, as pushed frames are.
2515
+ * A track's encoded frames go in decode order, starting at a keyframe; a
2516
+ * step whose buffered frames do not decode rejects with a
2517
+ * `StepRefusedError` (see `pushEncodedFrame()` for frames already sent).
2518
+ *
2519
+ * Video step frames travel encoded both ways when this browser and the
2520
+ * runtime share a codec (H.264 first by default; see
2521
+ * `setCodecPreference()`), on WebCodecs: the frames pushed on an input
2522
+ * track are encoded, and an output track's frames are decoded, so a result
2523
+ * still holds RGB frames whose `encoding` names the codec they travelled
2524
+ * in. Each track is one stream for the connection. When one of its
2525
+ * decoders lost its stream, the step rejects with a `DecodeError`, the SDK
2526
+ * pauses and resumes that track so the runtime starts it again at a
2527
+ * keyframe, and the track's frames fail until that keyframe arrives.
2528
+ *
1863
2529
  * ```ts
1864
2530
  * const camera = await reactor.publishTrack('camera');
1865
2531
  * await camera.pushFrame(imageData);
@@ -1873,24 +2539,53 @@ var Reactor = class {
1873
2539
  this.assertNotDisposed();
1874
2540
  try {
1875
2541
  const state = rewriteNestedFileRefs(options.state ?? null);
1876
- const commands = (options.commands ?? []).map(
1877
- (command) => command.data ? { ...command, data: rewriteNestedFileRefs(command.data) } : command
1878
- );
1879
- await this.pendingPushes;
1880
- return await this.withReaderAccess(async () => {
1881
- const client = await this.getOrCreateClient();
1882
- return client.step(state, commands);
2542
+ const commands = rewriteNestedFileRefs(options.commands ?? []);
2543
+ const previous = this.pendingPushes;
2544
+ const before = options.media !== void 0 ? this.pushStepMedia(options.media, state, commands) : previous;
2545
+ let taken;
2546
+ const bufferTaken = new Promise((resolve) => {
2547
+ taken = resolve;
1883
2548
  });
2549
+ const stepping = before.then(
2550
+ () => (
2551
+ // Excluded from connect()/disconnect()/reconnect() like sendCommand():
2552
+ // a step can take minutes, and must neither be freed out from under
2553
+ // nor block unrelated reads.
2554
+ this.withReaderAccess(async () => {
2555
+ const client = await this.getOrCreateClient();
2556
+ try {
2557
+ const decoders = this.outputDecoders;
2558
+ return client.step(state, commands).then(
2559
+ (result) => decoders.decodeResult(result).finally(() => this.restartLostTracks(client, decoders)),
2560
+ (cause) => {
2561
+ decoders.abandon();
2562
+ this.restartLostTracks(client, decoders);
2563
+ throw cause;
2564
+ }
2565
+ );
2566
+ } finally {
2567
+ taken();
2568
+ }
2569
+ })
2570
+ )
2571
+ );
2572
+ stepping.catch(() => previous.then(taken));
2573
+ this.pendingPushes = bufferTaken;
2574
+ return await stepping;
1884
2575
  } catch (cause) {
1885
2576
  throw this.captureError(cause);
1886
2577
  }
1887
2578
  }
1888
2579
  /**
1889
- * Buffers one input frame for the next `step()` on the sendonly track
1890
- * `trackName`. Nothing is sent until that step, which carries every frame
1891
- * pushed since the one before it, per track in push order. The
1892
- * `PublishedTrack` that `publishTrack()` resolves with does the same
1893
- * through `track.pushFrame(frame)`.
2580
+ * Pushes one input frame on the sendonly track `trackName` of a stepped
2581
+ * session. Once the session is ready, against a runtime that takes pushed
2582
+ * frames, the frame is sent to the runtime at once over the data channel,
2583
+ * ahead of any `step()` after it; one the channel cannot take is dropped
2584
+ * and raised as an error event. A frame pushed before the session is ready,
2585
+ * or to an older runtime, is buffered and sent with the next push or
2586
+ * `step()`, per track in push order. The `PublishedTrack` that
2587
+ * `publishTrack()` resolves with does the same through
2588
+ * `track.pushFrame(frame)`.
1894
2589
  *
1895
2590
  * Video: a `VideoFrame` (cloned when called, so the caller may close it as
1896
2591
  * soon as this returns; it stays the caller's), an
@@ -1901,9 +2596,15 @@ var Reactor = class {
1901
2596
  * Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
1902
2597
  * PCM at the rate the model's track declares.
1903
2598
  *
2599
+ * A video frame travels encoded when the connection negotiated a codec for
2600
+ * the track (see `setCodecPreference()`): it is encoded here, with
2601
+ * WebCodecs, as the next frame of the track's stream, and the model still
2602
+ * reads RGB. A track's first frame on a connection is a keyframe, and so
2603
+ * is the next one after the runtime could not decode its frames.
2604
+ *
1904
2605
  * Pushes keep their call order, and a `step()` waits for every push called
1905
2606
  * before it. A refused push rejects: await it, or `.catch()` it in a frame
1906
- * loop, or every bad frame is an unhandled rejection. What waits for
2607
+ * loop, or every bad frame is an unhandled rejection. What is buffered for
1907
2608
  * the next step is at most one message's worth of frames: pushing faster
1908
2609
  * than steps run drops the oldest frames, across tracks. Rejects with
1909
2610
  * an `InvalidStateError` outside a stepped session, for a track not
@@ -1939,29 +2640,16 @@ var Reactor = class {
1939
2640
  held.release();
1940
2641
  }
1941
2642
  await this.withReaderAccess(async () => {
1942
- if (epoch !== this.connectionEpoch) {
1943
- throw new InvalidStateError(
1944
- `pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
1945
- { operation: "pushFrame" }
1946
- );
1947
- }
1948
- const current = this.publishedInputs.get(trackName);
1949
- if (current === void 0 || publication !== void 0 && publication !== current) {
1950
- throw new InvalidStateError(
1951
- publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
1952
- { operation: "pushFrame" }
1953
- );
1954
- }
2643
+ this.checkPushTarget("pushFrame", trackName, publication, epoch);
1955
2644
  const client = await this.getOrCreateClient();
1956
2645
  if (pixels) {
1957
- client.pushVideoFrame(
2646
+ await this.pushPixels(
2647
+ client,
1958
2648
  trackName,
1959
- pixels.data,
1960
- pixels.width,
1961
- pixels.height,
1962
- pixels.format,
2649
+ pixels,
1963
2650
  options.metadata ?? null,
1964
- options.captureTimeUs ?? null
2651
+ options.captureTimeUs ?? null,
2652
+ () => this.checkPushTarget("pushFrame", trackName, publication, epoch)
1965
2653
  );
1966
2654
  } else if (isAudioFrame(frame)) {
1967
2655
  client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
@@ -1975,8 +2663,273 @@ var Reactor = class {
1975
2663
  throw this.captureError(cause);
1976
2664
  }
1977
2665
  }
1978
- /** Drops every frame buffered for the next step, pushes called before it
1979
- * included: it waits its turn behind them, as a `step()` does. */
2666
+ /**
2667
+ * Pushes one encoded video frame, sent or buffered as `pushFrame()` does,
2668
+ * on the sendonly video track `trackName`, its bytes as they are: one
2669
+ * H.264 access unit (Annex B), one VP8 or VP9 frame, or one AV1 temporal
2670
+ * unit, with the `width` and `height` it decodes to. The `PublishedTrack`
2671
+ * that `publishTrack()` resolves with does the same through
2672
+ * `track.pushEncodedFrame(frame)`.
2673
+ *
2674
+ * The runtime decodes a track's encoded frames as one stream, as they
2675
+ * arrive, and the model reads the RGB frames as if they had been pushed
2676
+ * raw. So push them in decode order, starting at a keyframe (`keyframe`, or
2677
+ * a chunk's `type`). A pushed frame that does not decode raises an
2678
+ * `input_decode_failed` error event naming the track, whose next frame
2679
+ * must then be a keyframe; a step whose buffered frames do not decode
2680
+ * rejects with a `StepRefusedError`, as does a runtime that predates
2681
+ * encoded frames. Encoded and raw pushes keep their call order, and a
2682
+ * `step()` waits for every push called before it.
2683
+ *
2684
+ * ```ts
2685
+ * const encoder = new VideoEncoder({
2686
+ * output: (chunk) => void camera.pushEncodedFrame({ codec: 'h264', data: chunk, width, height }),
2687
+ * error: console.error,
2688
+ * });
2689
+ * encoder.configure({ codec: 'avc1.42001f', width, height, avc: { format: 'annexb' } });
2690
+ * ```
2691
+ *
2692
+ * Rejects with an `InvalidStateError` outside a stepped session, for a
2693
+ * track not published on this connection, for an audio or recvonly track,
2694
+ * for empty data, a zero dimension or an unknown codec.
2695
+ */
2696
+ async pushEncodedFrame(trackName, frame) {
2697
+ return this.pushEncodedTo(trackName, void 0, frame);
2698
+ }
2699
+ /** `pushEncodedFrame()`, for the track's current publication or, from a
2700
+ * `PublishedTrack`, only for the `publication` it came from. */
2701
+ async pushEncodedTo(trackName, publication, frame) {
2702
+ const operation = "pushEncodedFrame";
2703
+ this.assertNotDisposed();
2704
+ const epoch = this.connectionEpoch;
2705
+ let encoded;
2706
+ try {
2707
+ encoded = toEncodedBytes(frame, operation, trackName);
2708
+ } catch (cause) {
2709
+ throw this.captureError(cause);
2710
+ }
2711
+ const push = this.pendingPushes.then(
2712
+ () => this.withReaderAccess(async () => {
2713
+ if (!this.isStepped()) {
2714
+ throw new InvalidStateError(
2715
+ `${operation}('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
2716
+ { operation }
2717
+ );
2718
+ }
2719
+ this.checkPushTarget(operation, trackName, publication, epoch);
2720
+ const client = await this.getOrCreateClient();
2721
+ const declared = client.tracks().find((track) => track.name === trackName);
2722
+ if (declared && (declared.kind !== "video" || declared.direction !== "sendonly")) {
2723
+ throw new InvalidStateError(
2724
+ `${operation}('${trackName}'): '${trackName}' is a ${declared.direction} ${declared.kind} track; only a sendonly video track takes encoded frames.`,
2725
+ { operation }
2726
+ );
2727
+ }
2728
+ client.pushEncodedVideoFrame(
2729
+ trackName,
2730
+ encoded.codec,
2731
+ encoded.data,
2732
+ encoded.width,
2733
+ encoded.height,
2734
+ encoded.metadata,
2735
+ encoded.captureTimeUs,
2736
+ encoded.keyframe
2737
+ );
2738
+ })
2739
+ );
2740
+ this.pendingPushes = push.catch(() => void 0);
2741
+ try {
2742
+ await push;
2743
+ } catch (cause) {
2744
+ throw this.captureError(cause);
2745
+ }
2746
+ }
2747
+ /** Refuses a push whose connection ended since it was called, or whose
2748
+ * track is not (or no longer) published on it. */
2749
+ checkPushTarget(operation, trackName, publication, epoch) {
2750
+ if (epoch !== this.connectionEpoch) {
2751
+ throw new InvalidStateError(
2752
+ `${operation}('${trackName}'): the connection it was pushed to ended while its frame was read`,
2753
+ { operation }
2754
+ );
2755
+ }
2756
+ const current = this.publishedInputs.get(trackName);
2757
+ if (current === void 0 || publication !== void 0 && publication !== current) {
2758
+ throw new InvalidStateError(
2759
+ publication === void 0 ? `${operation}('${trackName}') needs publishTrack('${trackName}') first.` : `${operation}('${trackName}'): this track was unpublished, or its connection ended.`,
2760
+ { operation }
2761
+ );
2762
+ }
2763
+ }
2764
+ /** Hands `pixels` to the binding for `track`: encoded first, on the
2765
+ * track's encoder, when the connection negotiated a codec for the track's
2766
+ * step frames and this browser can encode; raw otherwise, and when the
2767
+ * encoder fails (its next frame is then a keyframe). Checked by the
2768
+ * binding before anything is encoded, so a bad frame is refused either
2769
+ * way. `stillTarget` throws when the push's connection or publication is
2770
+ * gone: asked again once the encode is awaited, since a reconnect can
2771
+ * start meanwhile, and neither the encoded frame nor its raw fallback may
2772
+ * then reach the next session. */
2773
+ async pushPixels(client, track, pixels, metadata, captureTimeUs, stillTarget) {
2774
+ const { data, width, height, format } = pixels;
2775
+ const encoding = this.inputEncoders.available ? client.stepFrameEncodings()[track] : void 0;
2776
+ if (encoding !== void 0 && encoding !== "raw") {
2777
+ client.checkVideoFrame(track, data.byteLength, width, height, format);
2778
+ const keyframe = client.takeInputKeyframeRequest(track);
2779
+ let encoded;
2780
+ try {
2781
+ encoded = await this.inputEncoders.encode(track, encoding, pixels, keyframe);
2782
+ } catch (cause) {
2783
+ console.warn(`[Reactor] pushFrame('${track}'): ${encoding} encoding failed, sending the frame raw:`, cause);
2784
+ }
2785
+ stillTarget();
2786
+ if (encoded !== void 0) {
2787
+ for (const frame of encoded) {
2788
+ client.pushEncodedVideoFrame(track, encoding, frame.data, width, height, metadata, captureTimeUs, frame.keyframe);
2789
+ }
2790
+ return;
2791
+ }
2792
+ }
2793
+ client.pushVideoFrame(track, data, width, height, format, metadata, captureTimeUs);
2794
+ }
2795
+ /** Gives the binding the codecs this browser can carry step frames in,
2796
+ * probed once, so the next offer can negotiate them. A streamed session
2797
+ * carries no step frames. */
2798
+ /** Restart each output track whose decoder lost its stream: a pause and
2799
+ * a resume of that track, so the runtime starts it again at a keyframe. A
2800
+ * restart that cannot be sent is left to the next connection, whose
2801
+ * streams start at keyframes anyway. */
2802
+ restartLostTracks(client, decoders) {
2803
+ for (const track of decoders.takeLostTracks()) {
2804
+ try {
2805
+ client.restartTrackStream(track);
2806
+ } catch (cause) {
2807
+ console.warn(`step(): could not restart track '${track}':`, cause);
2808
+ }
2809
+ }
2810
+ }
2811
+ async offerStepCodecs(client) {
2812
+ if (!this.isStepped()) {
2813
+ return;
2814
+ }
2815
+ this.localStepCodecs ??= probeStepCodecs();
2816
+ const { encode, decode } = await this.localStepCodecs;
2817
+ client.setLocalStepCodecs(encode, decode);
2818
+ }
2819
+ /** Pushes `step()`'s `media`, track by track and in order, each frame as
2820
+ * `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
2821
+ * checks the step's `state` and `commands`, then every frame, before the
2822
+ * first is pushed, so a refused step, or a bad track or frame anywhere,
2823
+ * leaves the buffer as it was. */
2824
+ async pushStepMedia(media, state, commands) {
2825
+ const epoch = this.connectionEpoch;
2826
+ const held = holdStepMedia(media);
2827
+ const push = this.pendingPushes.then(async () => {
2828
+ const ready = [];
2829
+ try {
2830
+ for (const frame of held) {
2831
+ ready.push(await readStepFrame(frame));
2832
+ }
2833
+ } finally {
2834
+ for (const frame of held) {
2835
+ frame.release();
2836
+ }
2837
+ }
2838
+ await this.withReaderAccess(async () => {
2839
+ const client = await this.getOrCreateClient();
2840
+ client.checkStep(state, commands);
2841
+ for (const frame of ready) {
2842
+ this.checkPushTarget(frame.operation, frame.track, void 0, epoch);
2843
+ checkStepFrame(client, frame);
2844
+ }
2845
+ for (const frame of ready) {
2846
+ if (frame.kind === "encoded") {
2847
+ client.pushEncodedVideoFrame(
2848
+ frame.track,
2849
+ frame.encoded.codec,
2850
+ frame.encoded.data,
2851
+ frame.encoded.width,
2852
+ frame.encoded.height,
2853
+ frame.encoded.metadata,
2854
+ frame.encoded.captureTimeUs,
2855
+ frame.encoded.keyframe
2856
+ );
2857
+ } else if (frame.kind === "video") {
2858
+ await this.pushPixels(
2859
+ client,
2860
+ frame.track,
2861
+ frame.pixels,
2862
+ null,
2863
+ null,
2864
+ () => this.checkPushTarget(frame.operation, frame.track, void 0, epoch)
2865
+ );
2866
+ } else {
2867
+ const { samples, sampleRate, channels } = frame.audio;
2868
+ client.pushAudioFrame(frame.track, samples, sampleRate, channels ?? 1);
2869
+ }
2870
+ }
2871
+ });
2872
+ });
2873
+ this.pendingPushes = push.catch(() => void 0);
2874
+ await push;
2875
+ }
2876
+ /**
2877
+ * Chooses the codecs a stepped session carries its video step frames in,
2878
+ * most preferred first, from `"h264"`, `"vp9"`, `"av1"` and `"vp8"`.
2879
+ * `null` goes back to the default, H.264 first. An empty list for the
2880
+ * session turns encoding off, so every frame travels raw:
2881
+ *
2882
+ * ```ts
2883
+ * reactor.setCodecPreference(['vp9', 'h264']); // VP9 when both ends have it
2884
+ * reactor.setCodecPreference([]); // raw frames
2885
+ * reactor.setCodecPreference(['av1'], { track: 'main_video' });
2886
+ * ```
2887
+ *
2888
+ * With `track`, the list applies to that track only, in place of the
2889
+ * session's; an empty one there goes back to the session's, since one
2890
+ * track cannot turn encoding off alone. The list is an order, not a
2891
+ * limit: the codecs it leaves out follow it, so a runtime without the
2892
+ * preferred ones takes another. Of those, the ones this browser's
2893
+ * WebCodecs can encode (input tracks) or decode (output tracks) are
2894
+ * offered, and the runtime takes per track the first it shares; a track
2895
+ * they share no codec for stays raw.
2896
+ *
2897
+ * Whatever the codec, `step()` results and the frames `pushFrame()` takes
2898
+ * stay RGB: the encoding is between this client and the runtime. Callable
2899
+ * before `connect()` or during a session; it takes effect on the next
2900
+ * connection or reconnect, and the live connection keeps what it
2901
+ * negotiated (see `getStepFrameEncodings()`). A streamed session ignores
2902
+ * it. Throws an `InvalidStateError` on a name that is not a codec.
2903
+ */
2904
+ setCodecPreference(codecs, options = {}) {
2905
+ this.assertNotDisposed();
2906
+ const operation = "setCodecPreference";
2907
+ const track = options.track ?? null;
2908
+ if (codecs !== null && (!Array.isArray(codecs) || codecs.some((codec) => !STEP_CODECS.includes(codec)))) {
2909
+ throw this.captureError(
2910
+ new InvalidStateError(
2911
+ `${operation}: expected null or an array of "h264", "vp9", "av1" and "vp8", got ${JSON.stringify(codecs)}`,
2912
+ { operation }
2913
+ )
2914
+ );
2915
+ }
2916
+ if (track !== null && typeof track !== "string") {
2917
+ throw this.captureError(new InvalidStateError(`${operation}: track must be a string`, { operation }));
2918
+ }
2919
+ const list = codecs === null ? null : [...codecs];
2920
+ this.codecPreferences.set(track, list);
2921
+ this.client?.setCodecPreference(list, track);
2922
+ }
2923
+ /** How each track's step frames travel on this connection, as its offer
2924
+ * and answer negotiated them: `"raw"`, or the codec of an encoded video
2925
+ * track. Empty outside a connected stepped session; a track left out is
2926
+ * raw. */
2927
+ getStepFrameEncodings() {
2928
+ return this.client?.stepFrameEncodings() ?? {};
2929
+ }
2930
+ /** Drops every frame still buffered for the next step, pushes called before
2931
+ * it included: it waits its turn behind them, as a `step()` does. Frames
2932
+ * already sent to the runtime are not recalled. */
1980
2933
  async clearStepInputs() {
1981
2934
  this.assertNotDisposed();
1982
2935
  const clear = this.pendingPushes.then(
@@ -2167,6 +3120,9 @@ var Reactor = class {
2167
3120
  throw new Error("Reactor was disposed while connecting.");
2168
3121
  }
2169
3122
  const client = new WasmReactorClient(this.clientOptions, this.jwt);
3123
+ for (const [track, codecs] of this.codecPreferences) {
3124
+ client.setCodecPreference(codecs, track);
3125
+ }
2170
3126
  client.onStatusChanged((status) => {
2171
3127
  this.emitter.emit("statusChanged", status);
2172
3128
  this.handleStatusChanged(client, status);
@@ -2255,6 +3211,11 @@ var Reactor = class {
2255
3211
  * → "ready" status sequence (see `ReactorStatus`), and starts/stops stats
2256
3212
  * polling around the "ready" window. */
2257
3213
  handleStatusChanged(client, status) {
3214
+ if (status === "connecting" || status === "disconnected") {
3215
+ this.resetStepStreams();
3216
+ this.connectionEpoch += 1;
3217
+ this.publishedInputs.clear();
3218
+ }
2258
3219
  switch (status) {
2259
3220
  case "connecting":
2260
3221
  this.connectStartTime = performance.now();
@@ -2346,6 +3307,7 @@ var Reactor = class {
2346
3307
  * leave `getCapabilities()` returning the previous session's stale
2347
3308
  * tracks/commands until a new `capabilitiesReceived` lands. */
2348
3309
  resetConnectionState() {
3310
+ this.resetStepStreams();
2349
3311
  this.connectionEpoch += 1;
2350
3312
  this.publishedInputs.clear();
2351
3313
  this.stopStatsPolling();
@@ -2354,6 +3316,12 @@ var Reactor = class {
2354
3316
  this.waitingStartTime = void 0;
2355
3317
  this.capabilities = void 0;
2356
3318
  }
3319
+ /** Drops every step frame encoder and decoder: each track's stream lived
3320
+ * with the connection that ended, and the next one's start at keyframes. */
3321
+ resetStepStreams() {
3322
+ this.inputEncoders.reset();
3323
+ this.outputDecoders.reset();
3324
+ }
2357
3325
  /** Wraps `cause` and records it as the most recent failure, for a call
2358
3326
  * that's about to throw rather than emit — see `getLastError()`'s doc
2359
3327
  * comment. */