@reactor-team/js-sdk 3.4.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -249,7 +249,7 @@ var Emitter = class {
249
249
  };
250
250
 
251
251
  // src/internal/encoded-frames.ts
252
- var CODECS = /* @__PURE__ */ new Set(["h264"]);
252
+ var CODECS = /* @__PURE__ */ new Set(["h264", "vp8", "vp9", "av1"]);
253
253
  function isEncodedFrame(frame) {
254
254
  return typeof frame === "object" && frame !== null && "codec" in frame;
255
255
  }
@@ -261,9 +261,12 @@ function toEncodedBytes(frame, operation, trackName) {
261
261
  if (typeof frame !== "object" || frame === null) {
262
262
  throw refuse("expected an EncodedFrame, { codec, data, width, height }");
263
263
  }
264
- const { codec, data, width, height, metadata, captureTimeUs } = frame;
264
+ const { codec, data, width, height, metadata, captureTimeUs, keyframe } = frame;
265
265
  if (!CODECS.has(codec)) {
266
- throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264"`);
266
+ throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264", "vp8", "vp9" or "av1"`);
267
+ }
268
+ if (keyframe != null && typeof keyframe !== "boolean") {
269
+ throw refuse("keyframe must be a boolean");
267
270
  }
268
271
  let bytes;
269
272
  if (data instanceof Uint8Array) {
@@ -288,7 +291,15 @@ function toEncodedBytes(frame, operation, trackName) {
288
291
  if (captureTimeUs != null && !Number.isFinite(captureTimeUs)) {
289
292
  throw refuse("captureTimeUs must be a finite number");
290
293
  }
291
- return { codec, data: bytes, width, height, metadata: metadata ?? null, captureTimeUs: captureTimeUs ?? null };
294
+ return {
295
+ codec,
296
+ data: bytes,
297
+ width,
298
+ height,
299
+ metadata: metadata ?? null,
300
+ captureTimeUs: captureTimeUs ?? null,
301
+ keyframe: keyframe ?? (isChunk(data) && data.type === "key")
302
+ };
292
303
  }
293
304
  function isDimension(value) {
294
305
  return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 4294967295;
@@ -565,6 +576,513 @@ function checkStepFrame(client, frame) {
565
576
  }
566
577
  }
567
578
 
579
+ // src/internal/bt601.ts
580
+ var BT601_LIMITED = {
581
+ matrix: "smpte170m",
582
+ primaries: "smpte170m",
583
+ transfer: "smpte170m",
584
+ fullRange: false
585
+ };
586
+ function clamp(value) {
587
+ return value < 0 ? 0 : value > 255 ? 255 : value;
588
+ }
589
+ function rgbToI420(pixels) {
590
+ const { width, height, data } = pixels;
591
+ const step = pixels.format === "rgb" ? 3 : 4;
592
+ const [ri, bi] = pixels.format === "bgra" ? [2, 0] : [0, 2];
593
+ const codedWidth = width + (width & 1);
594
+ const codedHeight = height + (height & 1);
595
+ const chromaWidth = codedWidth >> 1;
596
+ const chromaHeight = codedHeight >> 1;
597
+ const ySize = codedWidth * codedHeight;
598
+ const out = new Uint8Array(ySize + 2 * chromaWidth * chromaHeight);
599
+ const uOffset = ySize;
600
+ const vOffset = ySize + chromaWidth * chromaHeight;
601
+ const at = (x, y) => (Math.min(y, height - 1) * width + Math.min(x, width - 1)) * step;
602
+ for (let y = 0; y < codedHeight; y++) {
603
+ for (let x = 0; x < codedWidth; x++) {
604
+ const i = at(x, y);
605
+ const r = data[i + ri];
606
+ const g = data[i + 1];
607
+ const b = data[i + bi];
608
+ out[y * codedWidth + x] = 66 * r + 129 * g + 25 * b + 4224 >> 8 & 255;
609
+ }
610
+ }
611
+ for (let cy = 0; cy < chromaHeight; cy++) {
612
+ for (let cx = 0; cx < chromaWidth; cx++) {
613
+ let r = 0;
614
+ let g = 0;
615
+ let b = 0;
616
+ for (const [dx, dy] of [
617
+ [0, 0],
618
+ [1, 0],
619
+ [0, 1],
620
+ [1, 1]
621
+ ]) {
622
+ const i = at(2 * cx + dx, 2 * cy + dy);
623
+ r += data[i + ri];
624
+ g += data[i + 1];
625
+ b += data[i + bi];
626
+ }
627
+ r = r + 2 >> 2;
628
+ g = g + 2 >> 2;
629
+ b = b + 2 >> 2;
630
+ out[uOffset + cy * chromaWidth + cx] = 112 * b - 74 * g - 38 * r + 32896 >> 8;
631
+ out[vOffset + cy * chromaWidth + cx] = 112 * r - 94 * g - 18 * b + 32896 >> 8;
632
+ }
633
+ }
634
+ return { data: out, width, height, codedWidth, codedHeight };
635
+ }
636
+ function yuvToRgb(data, { format, width, height, planes }) {
637
+ const [yPlane, uPlane, vPlane] = planes;
638
+ if (yPlane === void 0 || uPlane === void 0 || format === "I420" && vPlane === void 0) {
639
+ throw new Error(`a ${format} picture needs ${format === "I420" ? 3 : 2} planes, got ${planes.length}`);
640
+ }
641
+ const rgb = new Uint8Array(width * height * 3);
642
+ for (let y = 0; y < height; y++) {
643
+ const cy = y >> 1;
644
+ for (let x = 0; x < width; x++) {
645
+ const cx = x >> 1;
646
+ const luma = data[yPlane.offset + y * yPlane.stride + x];
647
+ let u;
648
+ let v;
649
+ if (format === "NV12") {
650
+ u = data[uPlane.offset + cy * uPlane.stride + 2 * cx];
651
+ v = data[uPlane.offset + cy * uPlane.stride + 2 * cx + 1];
652
+ } else {
653
+ u = data[uPlane.offset + cy * uPlane.stride + cx];
654
+ v = data[vPlane.offset + cy * vPlane.stride + cx];
655
+ }
656
+ const c = 1.164 * (luma - 16);
657
+ const d = u - 128;
658
+ const e = v - 128;
659
+ const j = (y * width + x) * 3;
660
+ rgb[j] = clamp(Math.round(c + 1.596 * e));
661
+ rgb[j + 1] = clamp(Math.round(c - 0.391 * d - 0.813 * e));
662
+ rgb[j + 2] = clamp(Math.round(c + 2.018 * d));
663
+ }
664
+ }
665
+ return rgb;
666
+ }
667
+
668
+ // src/internal/step-codecs.ts
669
+ var STEP_CODECS = ["h264", "vp9", "av1", "vp8"];
670
+ var DECODE_TIMEOUT_MS = 1e4;
671
+ function codecString(codec) {
672
+ switch (codec) {
673
+ case "h264":
674
+ return "avc1.42e033";
675
+ case "vp8":
676
+ return "vp8";
677
+ case "vp9":
678
+ return "vp09.00.51.08";
679
+ case "av1":
680
+ return "av01.0.13M.08";
681
+ }
682
+ }
683
+ function encoderConfig(codec, width, height) {
684
+ return {
685
+ codec: codecString(codec),
686
+ width,
687
+ height,
688
+ // Rate-controlled, as a real-time encoder is: enough for a still image
689
+ // to come back close, small enough to keep a step's message small.
690
+ bitrate: Math.min(2e7, Math.max(3e5, width * height * 4)),
691
+ framerate: 30,
692
+ latencyMode: "realtime",
693
+ ...codec === "h264" ? { avc: { format: "annexb" } } : {}
694
+ };
695
+ }
696
+ function decoderConfig(codec) {
697
+ return { codec: codecString(codec), optimizeForLatency: true };
698
+ }
699
+ async function probeStepCodecs(api = globalThis) {
700
+ const supported = async (probe) => {
701
+ const results = await Promise.all(STEP_CODECS.map((codec) => probe(codec).catch(() => false)));
702
+ return STEP_CODECS.filter((_, i) => results[i]);
703
+ };
704
+ const { VideoEncoder: Encoder, VideoDecoder: Decoder, VideoFrame: Frame, EncodedVideoChunk: Chunk } = api;
705
+ const [encode, decode] = await Promise.all([
706
+ Encoder && Frame ? supported(async (codec) => (await Encoder.isConfigSupported(encoderConfig(codec, 1280, 720))).supported === true) : [],
707
+ Decoder && Chunk ? supported(async (codec) => (await Decoder.isConfigSupported(decoderConfig(codec))).supported === true) : []
708
+ ]);
709
+ return { encode, decode };
710
+ }
711
+ var TrackEncoder = class {
712
+ constructor(api, codec, width, height) {
713
+ this.api = api;
714
+ this.codec = codec;
715
+ this.width = width;
716
+ this.height = height;
717
+ this.encoder = new api.VideoEncoder({
718
+ output: (chunk) => {
719
+ const data = new Uint8Array(chunk.byteLength);
720
+ chunk.copyTo(data);
721
+ this.outputs.push({ data, keyframe: chunk.type === "key" });
722
+ },
723
+ error: (error) => {
724
+ this.failure = error instanceof Error ? error : new Error(String(error));
725
+ }
726
+ });
727
+ try {
728
+ this.encoder.configure(encoderConfig(codec, width, height));
729
+ } catch (cause) {
730
+ this.close();
731
+ throw cause;
732
+ }
733
+ }
734
+ api;
735
+ codec;
736
+ width;
737
+ height;
738
+ encoder;
739
+ outputs = [];
740
+ failure;
741
+ started = false;
742
+ timestampUs = 0;
743
+ /** Encode `pixels` as the stream's next frame, a keyframe when `keyframe`
744
+ * or when it is the stream's first, and hand back what the encoder made
745
+ * of it: every frame is flushed out before the next goes in. */
746
+ async encode(pixels, keyframe) {
747
+ const i420 = rgbToI420(pixels);
748
+ const frame = new this.api.VideoFrame(i420.data, {
749
+ format: "I420",
750
+ codedWidth: i420.codedWidth,
751
+ codedHeight: i420.codedHeight,
752
+ visibleRect: { x: 0, y: 0, width: i420.width, height: i420.height },
753
+ colorSpace: BT601_LIMITED,
754
+ timestamp: this.timestampUs
755
+ });
756
+ this.timestampUs += 33333;
757
+ try {
758
+ this.encoder.encode(frame, { keyFrame: keyframe || !this.started });
759
+ } finally {
760
+ frame.close();
761
+ }
762
+ this.started = true;
763
+ await this.encoder.flush();
764
+ if (this.failure !== void 0) {
765
+ throw this.failure;
766
+ }
767
+ return this.outputs.splice(0);
768
+ }
769
+ close() {
770
+ if (this.encoder.state !== "closed") {
771
+ this.encoder.close();
772
+ }
773
+ }
774
+ };
775
+ var InputEncoders = class {
776
+ constructor(api = globalThis) {
777
+ this.api = api;
778
+ }
779
+ api;
780
+ encoders = /* @__PURE__ */ new Map();
781
+ /** Whether this browser can encode at all. */
782
+ get available() {
783
+ return this.api.VideoEncoder !== void 0 && this.api.VideoFrame !== void 0;
784
+ }
785
+ /** Encode `pixels` as `track`'s next frame in `codec`. A track whose codec
786
+ * or size changed starts a new stream, at a keyframe. On a failure the
787
+ * track's encoder is dropped, so its next frame is a keyframe, and the
788
+ * error is thrown for the caller to send the frame raw instead. */
789
+ async encode(track, codec, pixels, keyframe) {
790
+ const { VideoEncoder: Encoder, VideoFrame: Frame } = this.api;
791
+ if (!Encoder || !Frame) {
792
+ throw new Error("this browser has no WebCodecs VideoEncoder");
793
+ }
794
+ let encoder = this.encoders.get(track);
795
+ if (encoder && (encoder.codec !== codec || encoder.width !== pixels.width || encoder.height !== pixels.height)) {
796
+ encoder.close();
797
+ encoder = void 0;
798
+ }
799
+ try {
800
+ if (!encoder) {
801
+ encoder = new TrackEncoder({ VideoEncoder: Encoder, VideoFrame: Frame }, codec, pixels.width, pixels.height);
802
+ this.encoders.set(track, encoder);
803
+ }
804
+ return await encoder.encode(pixels, keyframe);
805
+ } catch (cause) {
806
+ this.drop(track);
807
+ throw cause;
808
+ }
809
+ }
810
+ /** Drop `track`'s encoder: its next frame starts a new stream. */
811
+ drop(track) {
812
+ this.encoders.get(track)?.close();
813
+ this.encoders.delete(track);
814
+ }
815
+ reset() {
816
+ for (const track of [...this.encoders.keys()]) {
817
+ this.drop(track);
818
+ }
819
+ }
820
+ };
821
+ var TrackDecoder = class {
822
+ constructor(api, codec) {
823
+ this.codec = codec;
824
+ this.Chunk = api.EncodedVideoChunk;
825
+ this.decoder = new api.VideoDecoder({
826
+ output: (frame) => this.deliver(frame),
827
+ error: (error) => this.fail(error)
828
+ });
829
+ try {
830
+ this.decoder.configure(decoderConfig(codec));
831
+ } catch (cause) {
832
+ this.close();
833
+ throw cause;
834
+ }
835
+ }
836
+ codec;
837
+ decoder;
838
+ Chunk;
839
+ /** Resolvers of the frames handed in and not yet handed back, by the
840
+ * timestamp each chunk was given, in the order they were handed in. */
841
+ pending = /* @__PURE__ */ new Map();
842
+ /** The next chunk's timestamp: a count, so each output names the one chunk
843
+ * it was made of, whatever capture time the frame carried. */
844
+ nextTimestamp = 0;
845
+ failed = false;
846
+ /** Hand `frame` to the chunk it was made of, by its timestamp. A step's
847
+ * stream is decoded in order, with no frame reordered, so a chunk handed
848
+ * in before that one with no picture yet made none (a frame that is not
849
+ * shown): it resolves with none rather than wait for the timeout, and no
850
+ * picture is ever matched to a chunk it was not made of. */
851
+ deliver(frame) {
852
+ const entry = this.pending.get(frame.timestamp);
853
+ if (!entry) {
854
+ frame.close();
855
+ return;
856
+ }
857
+ for (const [timestamp, earlier] of this.pending) {
858
+ if (timestamp === frame.timestamp) {
859
+ break;
860
+ }
861
+ this.pending.delete(timestamp);
862
+ earlier.resolve(void 0);
863
+ }
864
+ this.pending.delete(frame.timestamp);
865
+ entry.resolve(frame);
866
+ }
867
+ /** Hand a chunk of `data` to the decoder now, and resolve with the frame it
868
+ * makes of it, or `undefined` for one it shows none of. Called
869
+ * synchronously for every frame of a result, so the decoder takes them in
870
+ * the order results arrive. */
871
+ decode(type, data) {
872
+ const timestamp = this.nextTimestamp++;
873
+ const decoded = new Promise((resolve, reject) => {
874
+ this.pending.set(timestamp, { resolve, reject });
875
+ });
876
+ try {
877
+ this.decoder.decode(new this.Chunk({ type, timestamp, data }));
878
+ } catch (cause) {
879
+ this.fail(cause);
880
+ }
881
+ return withTimeout(decoded, DECODE_TIMEOUT_MS, () => {
882
+ this.fail(new Error(`the ${this.codec} decoder handed back no frame in ${DECODE_TIMEOUT_MS} ms`));
883
+ });
884
+ }
885
+ /** The decoder lost its state, or is given up: every frame still in it
886
+ * fails, so no step waits on it. */
887
+ fail(cause) {
888
+ this.failed = true;
889
+ const pending = [...this.pending.values()];
890
+ this.pending.clear();
891
+ for (const next of pending) {
892
+ next.reject(cause);
893
+ }
894
+ this.close();
895
+ }
896
+ close() {
897
+ if (this.decoder.state !== "closed") {
898
+ this.decoder.close();
899
+ }
900
+ }
901
+ };
902
+ function withTimeout(promise, ms, onTimeout) {
903
+ let timer;
904
+ const timeout = new Promise((_, reject) => {
905
+ timer = setTimeout(() => {
906
+ onTimeout();
907
+ reject(new Error(`timed out after ${ms} ms`));
908
+ }, ms);
909
+ });
910
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
911
+ }
912
+ var OutputDecoders = class {
913
+ constructor(api = globalThis) {
914
+ this.api = api;
915
+ }
916
+ api;
917
+ decoders = /* @__PURE__ */ new Map();
918
+ awaitingKeyframe = /* @__PURE__ */ new Set();
919
+ lost = [];
920
+ /** Whether a track lost its decoder and waits for a keyframe. */
921
+ needsKeyframe() {
922
+ return this.awaitingKeyframe.size > 0;
923
+ }
924
+ /** The tracks that lost their stream since the last call, each once per
925
+ * loss however many frames it then discards: the caller restarts each
926
+ * one's stream. */
927
+ takeLostTracks() {
928
+ const lost = this.lost;
929
+ this.lost = [];
930
+ return lost;
931
+ }
932
+ /** A step gave up after its request went out: its result, if it still
933
+ * comes, is never fed here, so every stream lost frames. Drop the
934
+ * decoders, and report each of their tracks lost. */
935
+ abandon() {
936
+ for (const [track, decoder] of this.decoders) {
937
+ decoder.fail(new DecodeError(`step(): track '${track}' lost its stream: a step gave up mid-stream`, { operation: "step" }));
938
+ this.lose(track, false);
939
+ }
940
+ this.decoders.clear();
941
+ }
942
+ /** Make `track` wait for a keyframe, and report it lost when it was not
943
+ * waiting already, or when `again` says this is a new loss. */
944
+ lose(track, again) {
945
+ if (!this.awaitingKeyframe.has(track) || again) {
946
+ this.lost.push(track);
947
+ }
948
+ this.awaitingKeyframe.add(track);
949
+ }
950
+ /** `result` with every encoded frame decoded to RGB `(height, width, 3)`
951
+ * `uint8`, as a raw frame comes. Call it as the result arrives: each
952
+ * encoded frame goes into its track's decoder before this returns, so
953
+ * results that arrive one after another are decoded in that order.
954
+ *
955
+ * Every frame of every track is fed to its decoder even after one fails,
956
+ * so the other streams keep up; the first failure rejects, with a
957
+ * `DecodeError`, rather than resolving a partial step. */
958
+ decodeResult(result) {
959
+ if (result.tracks.every((track) => track.frames.every((frame) => !isEncoded(frame)))) {
960
+ return Promise.resolve(result);
961
+ }
962
+ let firstError;
963
+ const tracks = result.tracks.map((track) => {
964
+ const frames = track.frames.map(
965
+ (frame) => this.decodeFrame(track.track, frame).catch((cause) => {
966
+ firstError ??= cause;
967
+ return void 0;
968
+ })
969
+ );
970
+ return Promise.all(frames).then((decoded) => ({
971
+ ...track,
972
+ frames: decoded.filter((frame) => frame !== void 0)
973
+ }));
974
+ });
975
+ return Promise.all(tracks).then((decoded) => {
976
+ if (firstError !== void 0) {
977
+ throw toDecodeError(firstError);
978
+ }
979
+ return { ...result, tracks: decoded };
980
+ });
981
+ }
982
+ /** Decode one frame of `track`; a raw one passes as it is, and one its
983
+ * decoder shows no picture of is `undefined`. Synchronous up to the
984
+ * decoder taking the frame. */
985
+ decodeFrame(track, frame) {
986
+ if (!isEncoded(frame)) {
987
+ return Promise.resolve(frame);
988
+ }
989
+ const codec = frame.encoding;
990
+ const { VideoDecoder: Decoder, EncodedVideoChunk: Chunk } = this.api;
991
+ if (!Decoder || !Chunk) {
992
+ return Promise.reject(new DecodeError(`step(): '${track}' sent ${codec} frames and this browser has no WebCodecs VideoDecoder`, { operation: "step" }));
993
+ }
994
+ let decoder = this.decoders.get(track);
995
+ if (decoder && (decoder.failed || decoder.codec !== codec)) {
996
+ decoder.fail(new DecodeError(`step(): track '${track}' changed codec to ${codec}`, { operation: "step" }));
997
+ this.decoders.delete(track);
998
+ decoder = void 0;
999
+ }
1000
+ if (!decoder) {
1001
+ if (!frame.keyframe) {
1002
+ this.lose(track, false);
1003
+ return Promise.reject(
1004
+ new DecodeError(
1005
+ `step(): track '${track}' has no decoder state for a ${codec} frame that is not a keyframe; waiting for a keyframe`,
1006
+ { operation: "step" }
1007
+ )
1008
+ );
1009
+ }
1010
+ try {
1011
+ decoder = new TrackDecoder({ VideoDecoder: Decoder, EncodedVideoChunk: Chunk }, codec);
1012
+ } catch (cause) {
1013
+ return Promise.reject(toDecodeError(cause));
1014
+ }
1015
+ this.decoders.set(track, decoder);
1016
+ }
1017
+ const live = decoder;
1018
+ return live.decode(frame.keyframe ? "key" : "delta", frame.data).then(
1019
+ async (picture) => {
1020
+ if (frame.keyframe) {
1021
+ this.awaitingKeyframe.delete(track);
1022
+ }
1023
+ return picture === void 0 ? void 0 : { ...frame, ...await rgbOf(picture) };
1024
+ },
1025
+ (cause) => {
1026
+ this.lose(track, frame.keyframe === true);
1027
+ if (this.decoders.get(track) === live) {
1028
+ this.decoders.delete(track);
1029
+ }
1030
+ throw cause;
1031
+ }
1032
+ );
1033
+ }
1034
+ reset() {
1035
+ for (const [track, decoder] of this.decoders) {
1036
+ decoder.fail(new DecodeError(`step(): track '${track}' lost its stream: its connection closed`, { operation: "step" }));
1037
+ }
1038
+ this.decoders.clear();
1039
+ this.awaitingKeyframe.clear();
1040
+ this.lost = [];
1041
+ }
1042
+ };
1043
+ function isEncoded(frame) {
1044
+ return frame.encoding !== "raw" && frame.encoding !== "unspecified";
1045
+ }
1046
+ function toDecodeError(cause) {
1047
+ if (cause instanceof DecodeError) {
1048
+ return cause;
1049
+ }
1050
+ const message = cause instanceof Error ? cause.message : JSON.stringify(cause);
1051
+ return new DecodeError(`step(): ${message}`, { operation: "step", cause });
1052
+ }
1053
+ async function rgbOf(picture) {
1054
+ let pixels;
1055
+ try {
1056
+ const yuv = await yuvOf(picture);
1057
+ if (yuv !== void 0) {
1058
+ return yuv;
1059
+ }
1060
+ pixels = await toPixels(picture);
1061
+ } finally {
1062
+ picture.close();
1063
+ }
1064
+ const { width, height } = pixels;
1065
+ const rgb = new Uint8Array(width * height * 3);
1066
+ const [r, b] = pixels.format === "bgra" ? [2, 0] : [0, 2];
1067
+ for (let i = 0, j = 0; j < rgb.length; i += 4, j += 3) {
1068
+ rgb[j] = pixels.data[i + r];
1069
+ rgb[j + 1] = pixels.data[i + 1];
1070
+ rgb[j + 2] = pixels.data[i + b];
1071
+ }
1072
+ return { dtype: "uint8", shape: [height, width, 3], data: rgb };
1073
+ }
1074
+ async function yuvOf(picture) {
1075
+ const format = picture.format === "I420A" ? "I420" : picture.format;
1076
+ if (format !== "I420" && format !== "NV12") {
1077
+ return void 0;
1078
+ }
1079
+ const width = picture.visibleRect?.width ?? picture.codedWidth;
1080
+ const height = picture.visibleRect?.height ?? picture.codedHeight;
1081
+ const data = new Uint8Array(picture.allocationSize());
1082
+ const planes = await picture.copyTo(data);
1083
+ return { dtype: "uint8", shape: [height, width, 3], data: yuvToRgb(data, { format, width, height, planes }) };
1084
+ }
1085
+
568
1086
  // src/internal/stats.ts
569
1087
  var LOCAL_STATS_INTERVAL_MS = 2e3;
570
1088
  var RUNTIME_REPORT_INTERVAL_MS = 5e3;
@@ -892,7 +1410,7 @@ function describeCause(cause) {
892
1410
  // package.json
893
1411
  var package_default = {
894
1412
  name: "@reactor-team/js-sdk",
895
- version: "3.4.0",
1413
+ version: "3.5.0",
896
1414
  description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
897
1415
  license: "Apache-2.0",
898
1416
  repository: {
@@ -1419,6 +1937,17 @@ var Reactor = class {
1419
1937
  * one's pixels take to read, and `step()` waits for the ones called
1420
1938
  * before it. */
1421
1939
  pendingPushes = Promise.resolve();
1940
+ /** Stepped: every `setCodecPreference()` so far, keyed by track (`null`
1941
+ * for the session's list), replayed into a client created later. */
1942
+ codecPreferences = /* @__PURE__ */ new Map();
1943
+ /** Stepped: what this browser's WebCodecs can encode and decode, probed
1944
+ * once, on the first stepped connect. */
1945
+ localStepCodecs;
1946
+ /** Stepped: the input tracks' encoders and the output tracks' decoders of
1947
+ * the current connection, dropped as it ends so the next one's streams
1948
+ * start at keyframes. */
1949
+ inputEncoders = new InputEncoders();
1950
+ outputDecoders = new OutputDecoders();
1422
1951
  emitter = new Emitter();
1423
1952
  /** Serializes connect()/reconnect()/disconnect() (and the free() inside
1424
1953
  * disconnect()/[Symbol.dispose]) against each other. Calling into the
@@ -1526,6 +2055,7 @@ var Reactor = class {
1526
2055
  await this.queue.push(
1527
2056
  () => this.withWriterAccess(async () => {
1528
2057
  const client = await this.getOrCreateClient();
2058
+ await this.offerStepCodecs(client);
1529
2059
  await client.connect(options);
1530
2060
  }),
1531
2061
  "connect"
@@ -1580,6 +2110,7 @@ var Reactor = class {
1580
2110
  await this.queue.push(
1581
2111
  () => this.withWriterAccess(async () => {
1582
2112
  const client = await this.getOrCreateClient();
2113
+ await this.offerStepCodecs(client);
1583
2114
  await client.reconnect(options);
1584
2115
  }),
1585
2116
  "reconnect"
@@ -1855,10 +2386,10 @@ var Reactor = class {
1855
2386
  * Runs one step of a `generation: "stepped"` session and resolves with
1856
2387
  * what it produced, per output track.
1857
2388
  *
1858
- * The request carries `state`, `commands` and every frame pushed since the
1859
- * previous step. The runtime applies them in that order — each `state` key
1860
- * as the model's `set_<key>` command, then `commands`, then the frames —
1861
- * and then generates. They are separate calls, not a transaction: one that
2389
+ * The request carries `state` and `commands`; the frames pushed before it
2390
+ * have already reached the runtime (see `pushFrame()`). The runtime applies
2391
+ * them in that order — each `state` key as the model's `set_<key>` command,
2392
+ * then `commands` — and then generates. They are separate calls, not a transaction: one that
1862
2393
  * fails does not undo the ones before it. All of it is optional:
1863
2394
  * `step()` with nothing set and nothing pushed runs on the model's current
1864
2395
  * state, which commands sent outside a step (`sendCommand()`) also change.
@@ -1869,7 +2400,9 @@ var Reactor = class {
1869
2400
  * A step that ran but that the model reported as failed still resolves,
1870
2401
  * with `result.error` set. `result.files` lists the files the model kept
1871
2402
  * with the step (name, content type, size); their bytes are served over
1872
- * HTTP from the session's step results, not carried here.
2403
+ * HTTP from the session's step results, not carried here. `result.data`
2404
+ * holds what the model returned besides its media, as opaque bytes (a dict
2405
+ * arrives encoded as JSON); it is absent when the model returned none.
1873
2406
  *
1874
2407
  * Rejects with a `StepRefusedError` when the runtime or the model turns
1875
2408
  * the step down (the runtime checks a whole request first, so nothing it
@@ -1883,7 +2416,7 @@ var Reactor = class {
1883
2416
  * message, `MessageTooLargeError` when the request itself is too large
1884
2417
  * (nothing was sent), `InvalidStateError` outside a ready stepped
1885
2418
  * session, or whatever else failed the step (`DisconnectedError`,
1886
- * `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
2419
+ * `RequestTimeoutError`, ...). The frames buffered for a step belong to it:
1887
2420
  * when its request cannot be sent, they are dropped with it rather than
1888
2421
  * kept for the next step.
1889
2422
  *
@@ -1897,9 +2430,19 @@ var Reactor = class {
1897
2430
  * before any of it is pushed or sent, leaving the buffer as it was. A
1898
2431
  * session that leaves 'ready' once the media is in has the step refused and
1899
2432
  * keeps the frames for the next one, as pushed frames are.
1900
- * A track's encoded frames of one step go in decode order, starting at a
1901
- * keyframe; a step whose frames do not decode rejects with a
1902
- * `StepRefusedError`.
2433
+ * A track's encoded frames go in decode order, starting at a keyframe; a
2434
+ * step whose buffered frames do not decode rejects with a
2435
+ * `StepRefusedError` (see `pushEncodedFrame()` for frames already sent).
2436
+ *
2437
+ * Video step frames travel encoded both ways when this browser and the
2438
+ * runtime share a codec (H.264 first by default; see
2439
+ * `setCodecPreference()`), on WebCodecs: the frames pushed on an input
2440
+ * track are encoded, and an output track's frames are decoded, so a result
2441
+ * still holds RGB frames whose `encoding` names the codec they travelled
2442
+ * in. Each track is one stream for the connection. When one of its
2443
+ * decoders lost its stream, the step rejects with a `DecodeError`, the SDK
2444
+ * pauses and resumes that track so the runtime starts it again at a
2445
+ * keyframe, and the track's frames fail until that keyframe arrives.
1903
2446
  *
1904
2447
  * ```ts
1905
2448
  * const camera = await reactor.publishTrack('camera');
@@ -1929,7 +2472,15 @@ var Reactor = class {
1929
2472
  this.withReaderAccess(async () => {
1930
2473
  const client = await this.getOrCreateClient();
1931
2474
  try {
1932
- return client.step(state, commands);
2475
+ const decoders = this.outputDecoders;
2476
+ return client.step(state, commands).then(
2477
+ (result) => decoders.decodeResult(result).finally(() => this.restartLostTracks(client, decoders)),
2478
+ (cause) => {
2479
+ decoders.abandon();
2480
+ this.restartLostTracks(client, decoders);
2481
+ throw cause;
2482
+ }
2483
+ );
1933
2484
  } finally {
1934
2485
  taken();
1935
2486
  }
@@ -1944,11 +2495,15 @@ var Reactor = class {
1944
2495
  }
1945
2496
  }
1946
2497
  /**
1947
- * Buffers one input frame for the next `step()` on the sendonly track
1948
- * `trackName`. Nothing is sent until that step, which carries every frame
1949
- * pushed since the one before it, per track in push order. The
1950
- * `PublishedTrack` that `publishTrack()` resolves with does the same
1951
- * through `track.pushFrame(frame)`.
2498
+ * Pushes one input frame on the sendonly track `trackName` of a stepped
2499
+ * session. Once the session is ready, against a runtime that takes pushed
2500
+ * frames, the frame is sent to the runtime at once over the data channel,
2501
+ * ahead of any `step()` after it; one the channel cannot take is dropped
2502
+ * and raised as an error event. A frame pushed before the session is ready,
2503
+ * or to an older runtime, is buffered and sent with the next push or
2504
+ * `step()`, per track in push order. The `PublishedTrack` that
2505
+ * `publishTrack()` resolves with does the same through
2506
+ * `track.pushFrame(frame)`.
1952
2507
  *
1953
2508
  * Video: a `VideoFrame` (cloned when called, so the caller may close it as
1954
2509
  * soon as this returns; it stays the caller's), an
@@ -1959,9 +2514,15 @@ var Reactor = class {
1959
2514
  * Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
1960
2515
  * PCM at the rate the model's track declares.
1961
2516
  *
2517
+ * A video frame travels encoded when the connection negotiated a codec for
2518
+ * the track (see `setCodecPreference()`): it is encoded here, with
2519
+ * WebCodecs, as the next frame of the track's stream, and the model still
2520
+ * reads RGB. A track's first frame on a connection is a keyframe, and so
2521
+ * is the next one after the runtime could not decode its frames.
2522
+ *
1962
2523
  * Pushes keep their call order, and a `step()` waits for every push called
1963
2524
  * before it. A refused push rejects: await it, or `.catch()` it in a frame
1964
- * loop, or every bad frame is an unhandled rejection. What waits for
2525
+ * loop, or every bad frame is an unhandled rejection. What is buffered for
1965
2526
  * the next step is at most one message's worth of frames: pushing faster
1966
2527
  * than steps run drops the oldest frames, across tracks. Rejects with
1967
2528
  * an `InvalidStateError` outside a stepped session, for a track not
@@ -2000,14 +2561,13 @@ var Reactor = class {
2000
2561
  this.checkPushTarget("pushFrame", trackName, publication, epoch);
2001
2562
  const client = await this.getOrCreateClient();
2002
2563
  if (pixels) {
2003
- client.pushVideoFrame(
2564
+ await this.pushPixels(
2565
+ client,
2004
2566
  trackName,
2005
- pixels.data,
2006
- pixels.width,
2007
- pixels.height,
2008
- pixels.format,
2567
+ pixels,
2009
2568
  options.metadata ?? null,
2010
- options.captureTimeUs ?? null
2569
+ options.captureTimeUs ?? null,
2570
+ () => this.checkPushTarget("pushFrame", trackName, publication, epoch)
2011
2571
  );
2012
2572
  } else if (isAudioFrame(frame)) {
2013
2573
  client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
@@ -2022,18 +2582,22 @@ var Reactor = class {
2022
2582
  }
2023
2583
  }
2024
2584
  /**
2025
- * Buffers one encoded video frame for the next `step()` on the sendonly
2026
- * video track `trackName`, its bytes as they are: one H.264 access unit
2027
- * (Annex B), with the `width` and `height` it decodes to. The
2028
- * `PublishedTrack` that `publishTrack()` resolves with does the same
2029
- * through `track.pushEncodedFrame(frame)`.
2585
+ * Pushes one encoded video frame, sent or buffered as `pushFrame()` does,
2586
+ * on the sendonly video track `trackName`, its bytes as they are: one
2587
+ * H.264 access unit (Annex B), one VP8 or VP9 frame, or one AV1 temporal
2588
+ * unit, with the `width` and `height` it decodes to. The `PublishedTrack`
2589
+ * that `publishTrack()` resolves with does the same through
2590
+ * `track.pushEncodedFrame(frame)`.
2030
2591
  *
2031
- * The runtime decodes a track's encoded frames of one step as one stream,
2032
- * before the step runs, and the model reads the RGB frames as if they had
2033
- * been pushed raw. So push them in decode order, starting at a keyframe; a
2034
- * step whose frames do not decode rejects with a `StepRefusedError`, as
2035
- * does a runtime that predates encoded frames. Encoded and raw pushes keep
2036
- * their call order, and a `step()` waits for every push called before it.
2592
+ * The runtime decodes a track's encoded frames as one stream, as they
2593
+ * arrive, and the model reads the RGB frames as if they had been pushed
2594
+ * raw. So push them in decode order, starting at a keyframe (`keyframe`, or
2595
+ * a chunk's `type`). A pushed frame that does not decode raises an
2596
+ * `input_decode_failed` error event naming the track, whose next frame
2597
+ * must then be a keyframe; a step whose buffered frames do not decode
2598
+ * rejects with a `StepRefusedError`, as does a runtime that predates
2599
+ * encoded frames. Encoded and raw pushes keep their call order, and a
2600
+ * `step()` waits for every push called before it.
2037
2601
  *
2038
2602
  * ```ts
2039
2603
  * const encoder = new VideoEncoder({
@@ -2045,7 +2609,7 @@ var Reactor = class {
2045
2609
  *
2046
2610
  * Rejects with an `InvalidStateError` outside a stepped session, for a
2047
2611
  * track not published on this connection, for an audio or recvonly track,
2048
- * for empty data, a zero dimension or a codec other than `"h264"`.
2612
+ * for empty data, a zero dimension or an unknown codec.
2049
2613
  */
2050
2614
  async pushEncodedFrame(trackName, frame) {
2051
2615
  return this.pushEncodedTo(trackName, void 0, frame);
@@ -2086,7 +2650,8 @@ var Reactor = class {
2086
2650
  encoded.width,
2087
2651
  encoded.height,
2088
2652
  encoded.metadata,
2089
- encoded.captureTimeUs
2653
+ encoded.captureTimeUs,
2654
+ encoded.keyframe
2090
2655
  );
2091
2656
  })
2092
2657
  );
@@ -2114,6 +2679,61 @@ var Reactor = class {
2114
2679
  );
2115
2680
  }
2116
2681
  }
2682
+ /** Hands `pixels` to the binding for `track`: encoded first, on the
2683
+ * track's encoder, when the connection negotiated a codec for the track's
2684
+ * step frames and this browser can encode; raw otherwise, and when the
2685
+ * encoder fails (its next frame is then a keyframe). Checked by the
2686
+ * binding before anything is encoded, so a bad frame is refused either
2687
+ * way. `stillTarget` throws when the push's connection or publication is
2688
+ * gone: asked again once the encode is awaited, since a reconnect can
2689
+ * start meanwhile, and neither the encoded frame nor its raw fallback may
2690
+ * then reach the next session. */
2691
+ async pushPixels(client, track, pixels, metadata, captureTimeUs, stillTarget) {
2692
+ const { data, width, height, format } = pixels;
2693
+ const encoding = this.inputEncoders.available ? client.stepFrameEncodings()[track] : void 0;
2694
+ if (encoding !== void 0 && encoding !== "raw") {
2695
+ client.checkVideoFrame(track, data.byteLength, width, height, format);
2696
+ const keyframe = client.takeInputKeyframeRequest(track);
2697
+ let encoded;
2698
+ try {
2699
+ encoded = await this.inputEncoders.encode(track, encoding, pixels, keyframe);
2700
+ } catch (cause) {
2701
+ console.warn(`[Reactor] pushFrame('${track}'): ${encoding} encoding failed, sending the frame raw:`, cause);
2702
+ }
2703
+ stillTarget();
2704
+ if (encoded !== void 0) {
2705
+ for (const frame of encoded) {
2706
+ client.pushEncodedVideoFrame(track, encoding, frame.data, width, height, metadata, captureTimeUs, frame.keyframe);
2707
+ }
2708
+ return;
2709
+ }
2710
+ }
2711
+ client.pushVideoFrame(track, data, width, height, format, metadata, captureTimeUs);
2712
+ }
2713
+ /** Gives the binding the codecs this browser can carry step frames in,
2714
+ * probed once, so the next offer can negotiate them. A streamed session
2715
+ * carries no step frames. */
2716
+ /** Restart each output track whose decoder lost its stream: a pause and
2717
+ * a resume of that track, so the runtime starts it again at a keyframe. A
2718
+ * restart that cannot be sent is left to the next connection, whose
2719
+ * streams start at keyframes anyway. */
2720
+ restartLostTracks(client, decoders) {
2721
+ for (const track of decoders.takeLostTracks()) {
2722
+ try {
2723
+ client.restartTrackStream(track);
2724
+ } catch (cause) {
2725
+ console.warn(`step(): could not restart track '${track}':`, cause);
2726
+ }
2727
+ }
2728
+ }
2729
+ async offerStepCodecs(client) {
2730
+ if (!this.isStepped()) {
2731
+ return;
2732
+ }
2733
+ this.localStepCodecs ??= probeStepCodecs();
2734
+ const { encode, decode } = await this.localStepCodecs;
2735
+ client.setLocalStepCodecs(encode, decode);
2736
+ }
2117
2737
  /** Pushes `step()`'s `media`, track by track and in order, each frame as
2118
2738
  * `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
2119
2739
  * checks the step's `state` and `commands`, then every frame, before the
@@ -2149,11 +2769,18 @@ var Reactor = class {
2149
2769
  frame.encoded.width,
2150
2770
  frame.encoded.height,
2151
2771
  frame.encoded.metadata,
2152
- frame.encoded.captureTimeUs
2772
+ frame.encoded.captureTimeUs,
2773
+ frame.encoded.keyframe
2153
2774
  );
2154
2775
  } else if (frame.kind === "video") {
2155
- const { data, width, height, format } = frame.pixels;
2156
- client.pushVideoFrame(frame.track, data, width, height, format, null, null);
2776
+ await this.pushPixels(
2777
+ client,
2778
+ frame.track,
2779
+ frame.pixels,
2780
+ null,
2781
+ null,
2782
+ () => this.checkPushTarget(frame.operation, frame.track, void 0, epoch)
2783
+ );
2157
2784
  } else {
2158
2785
  const { samples, sampleRate, channels } = frame.audio;
2159
2786
  client.pushAudioFrame(frame.track, samples, sampleRate, channels ?? 1);
@@ -2164,8 +2791,63 @@ var Reactor = class {
2164
2791
  this.pendingPushes = push.catch(() => void 0);
2165
2792
  await push;
2166
2793
  }
2167
- /** Drops every frame buffered for the next step, pushes called before it
2168
- * included: it waits its turn behind them, as a `step()` does. */
2794
+ /**
2795
+ * Chooses the codecs a stepped session carries its video step frames in,
2796
+ * most preferred first, from `"h264"`, `"vp9"`, `"av1"` and `"vp8"`.
2797
+ * `null` goes back to the default, H.264 first. An empty list for the
2798
+ * session turns encoding off, so every frame travels raw:
2799
+ *
2800
+ * ```ts
2801
+ * reactor.setCodecPreference(['vp9', 'h264']); // VP9 when both ends have it
2802
+ * reactor.setCodecPreference([]); // raw frames
2803
+ * reactor.setCodecPreference(['av1'], { track: 'main_video' });
2804
+ * ```
2805
+ *
2806
+ * With `track`, the list applies to that track only, in place of the
2807
+ * session's; an empty one there goes back to the session's, since one
2808
+ * track cannot turn encoding off alone. The list is an order, not a
2809
+ * limit: the codecs it leaves out follow it, so a runtime without the
2810
+ * preferred ones takes another. Of those, the ones this browser's
2811
+ * WebCodecs can encode (input tracks) or decode (output tracks) are
2812
+ * offered, and the runtime takes per track the first it shares; a track
2813
+ * they share no codec for stays raw.
2814
+ *
2815
+ * Whatever the codec, `step()` results and the frames `pushFrame()` takes
2816
+ * stay RGB: the encoding is between this client and the runtime. Callable
2817
+ * before `connect()` or during a session; it takes effect on the next
2818
+ * connection or reconnect, and the live connection keeps what it
2819
+ * negotiated (see `getStepFrameEncodings()`). A streamed session ignores
2820
+ * it. Throws an `InvalidStateError` on a name that is not a codec.
2821
+ */
2822
+ setCodecPreference(codecs, options = {}) {
2823
+ this.assertNotDisposed();
2824
+ const operation = "setCodecPreference";
2825
+ const track = options.track ?? null;
2826
+ if (codecs !== null && (!Array.isArray(codecs) || codecs.some((codec) => !STEP_CODECS.includes(codec)))) {
2827
+ throw this.captureError(
2828
+ new InvalidStateError(
2829
+ `${operation}: expected null or an array of "h264", "vp9", "av1" and "vp8", got ${JSON.stringify(codecs)}`,
2830
+ { operation }
2831
+ )
2832
+ );
2833
+ }
2834
+ if (track !== null && typeof track !== "string") {
2835
+ throw this.captureError(new InvalidStateError(`${operation}: track must be a string`, { operation }));
2836
+ }
2837
+ const list = codecs === null ? null : [...codecs];
2838
+ this.codecPreferences.set(track, list);
2839
+ this.client?.setCodecPreference(list, track);
2840
+ }
2841
+ /** How each track's step frames travel on this connection, as its offer
2842
+ * and answer negotiated them: `"raw"`, or the codec of an encoded video
2843
+ * track. Empty outside a connected stepped session; a track left out is
2844
+ * raw. */
2845
+ getStepFrameEncodings() {
2846
+ return this.client?.stepFrameEncodings() ?? {};
2847
+ }
2848
+ /** Drops every frame still buffered for the next step, pushes called before
2849
+ * it included: it waits its turn behind them, as a `step()` does. Frames
2850
+ * already sent to the runtime are not recalled. */
2169
2851
  async clearStepInputs() {
2170
2852
  this.assertNotDisposed();
2171
2853
  const clear = this.pendingPushes.then(
@@ -2356,6 +3038,9 @@ var Reactor = class {
2356
3038
  throw new Error("Reactor was disposed while connecting.");
2357
3039
  }
2358
3040
  const client = new WasmReactorClient(this.clientOptions, this.jwt);
3041
+ for (const [track, codecs] of this.codecPreferences) {
3042
+ client.setCodecPreference(codecs, track);
3043
+ }
2359
3044
  client.onStatusChanged((status) => {
2360
3045
  this.emitter.emit("statusChanged", status);
2361
3046
  this.handleStatusChanged(client, status);
@@ -2444,6 +3129,11 @@ var Reactor = class {
2444
3129
  * → "ready" status sequence (see `ReactorStatus`), and starts/stops stats
2445
3130
  * polling around the "ready" window. */
2446
3131
  handleStatusChanged(client, status) {
3132
+ if (status === "connecting" || status === "disconnected") {
3133
+ this.resetStepStreams();
3134
+ this.connectionEpoch += 1;
3135
+ this.publishedInputs.clear();
3136
+ }
2447
3137
  switch (status) {
2448
3138
  case "connecting":
2449
3139
  this.connectStartTime = performance.now();
@@ -2535,6 +3225,7 @@ var Reactor = class {
2535
3225
  * leave `getCapabilities()` returning the previous session's stale
2536
3226
  * tracks/commands until a new `capabilitiesReceived` lands. */
2537
3227
  resetConnectionState() {
3228
+ this.resetStepStreams();
2538
3229
  this.connectionEpoch += 1;
2539
3230
  this.publishedInputs.clear();
2540
3231
  this.stopStatsPolling();
@@ -2543,6 +3234,12 @@ var Reactor = class {
2543
3234
  this.waitingStartTime = void 0;
2544
3235
  this.capabilities = void 0;
2545
3236
  }
3237
+ /** Drops every step frame encoder and decoder: each track's stream lived
3238
+ * with the connection that ended, and the next one's start at keyframes. */
3239
+ resetStepStreams() {
3240
+ this.inputEncoders.reset();
3241
+ this.outputDecoders.reset();
3242
+ }
2546
3243
  /** Wraps `cause` and records it as the most recent failure, for a call
2547
3244
  * that's about to throw rather than emit — see `getLastError()`'s doc
2548
3245
  * comment. */