@reactor-team/js-sdk 3.3.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -248,6 +248,52 @@ var Emitter = class {
248
248
  }
249
249
  };
250
250
 
251
+ // src/internal/encoded-frames.ts
252
+ var CODECS = /* @__PURE__ */ new Set(["h264"]);
253
+ function isEncodedFrame(frame) {
254
+ return typeof frame === "object" && frame !== null && "codec" in frame;
255
+ }
256
+ function isChunk(data) {
257
+ return !(data instanceof Uint8Array) && !(data instanceof ArrayBuffer) && typeof data.copyTo === "function";
258
+ }
259
+ function toEncodedBytes(frame, operation, trackName) {
260
+ const refuse = (why) => new InvalidStateError(`${operation}('${trackName}'): ${why}`, { operation });
261
+ if (typeof frame !== "object" || frame === null) {
262
+ throw refuse("expected an EncodedFrame, { codec, data, width, height }");
263
+ }
264
+ const { codec, data, width, height, metadata, captureTimeUs } = frame;
265
+ if (!CODECS.has(codec)) {
266
+ throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264"`);
267
+ }
268
+ let bytes;
269
+ if (data instanceof Uint8Array) {
270
+ bytes = data;
271
+ } else if (data instanceof ArrayBuffer) {
272
+ bytes = new Uint8Array(data);
273
+ } else if (data && isChunk(data)) {
274
+ bytes = new Uint8Array(data.byteLength);
275
+ data.copyTo(bytes);
276
+ } else {
277
+ throw refuse("data must be a Uint8Array, an ArrayBuffer or an EncodedVideoChunk");
278
+ }
279
+ if (bytes.byteLength === 0) {
280
+ throw refuse("an encoded frame needs data");
281
+ }
282
+ if (!isDimension(width) || !isDimension(height)) {
283
+ throw refuse(`width and height are the decoded frame's, got ${String(width)}x${String(height)}`);
284
+ }
285
+ if (metadata != null && !(metadata instanceof Uint8Array)) {
286
+ throw refuse("metadata must be a Uint8Array");
287
+ }
288
+ if (captureTimeUs != null && !Number.isFinite(captureTimeUs)) {
289
+ throw refuse("captureTimeUs must be a finite number");
290
+ }
291
+ return { codec, data: bytes, width, height, metadata: metadata ?? null, captureTimeUs: captureTimeUs ?? null };
292
+ }
293
+ function isDimension(value) {
294
+ return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 4294967295;
295
+ }
296
+
251
297
  // src/file-ref.ts
252
298
  var FileRef = class {
253
299
  constructor(uploadId, name, mimeType, size) {
@@ -458,6 +504,67 @@ function toPublicClip(clip) {
458
504
  };
459
505
  }
460
506
 
507
+ // src/internal/step-media.ts
508
+ function holdStepMedia(media) {
509
+ if (typeof media !== "object" || media === null || Array.isArray(media)) {
510
+ throw new InvalidStateError("step(): media must map track names to arrays of frames", {
511
+ operation: "step"
512
+ });
513
+ }
514
+ const held = [];
515
+ try {
516
+ for (const [trackName, frames] of Object.entries(media)) {
517
+ if (!Array.isArray(frames)) {
518
+ throw new InvalidStateError(`step(): media['${trackName}'] must be an array of frames`, {
519
+ operation: "step"
520
+ });
521
+ }
522
+ for (const frame of frames) {
523
+ held.push(holdStepFrame(trackName, frame));
524
+ }
525
+ }
526
+ } catch (cause) {
527
+ for (const frame of held) {
528
+ frame.release();
529
+ }
530
+ throw cause;
531
+ }
532
+ return held;
533
+ }
534
+ function holdStepFrame(track, frame) {
535
+ if (isEncodedFrame(frame)) {
536
+ const operation = "pushEncodedFrame";
537
+ return { track, kind: "encoded", operation, encoded: toEncodedBytes(frame, operation, track), release: () => {
538
+ } };
539
+ }
540
+ if (typeof frame !== "object" || frame === null) {
541
+ throw new InvalidStateError(`pushFrame('${track}'): expected a frame`, { operation: "pushFrame" });
542
+ }
543
+ const held = holdFrame(frame);
544
+ return { track, kind: "raw", operation: "pushFrame", frame: held.frame, release: held.release };
545
+ }
546
+ async function readStepFrame(frame) {
547
+ const { track } = frame;
548
+ if (frame.kind === "encoded") {
549
+ return { track, kind: "encoded", operation: frame.operation, encoded: frame.encoded };
550
+ }
551
+ if (isAudioFrame(frame.frame)) {
552
+ return { track, kind: "audio", operation: frame.operation, audio: frame.frame };
553
+ }
554
+ return { track, kind: "video", operation: frame.operation, pixels: await toPixels(frame.frame) };
555
+ }
556
+ function checkStepFrame(client, frame) {
557
+ if (frame.kind === "encoded") {
558
+ const { codec, data, width, height, captureTimeUs } = frame.encoded;
559
+ client.checkEncodedVideoFrame(frame.track, codec, data.byteLength, width, height, captureTimeUs);
560
+ } else if (frame.kind === "video") {
561
+ const { data, width, height, format } = frame.pixels;
562
+ client.checkVideoFrame(frame.track, data.byteLength, width, height, format);
563
+ } else {
564
+ client.checkAudioFrame(frame.track, frame.audio.samples, frame.audio.channels ?? 1);
565
+ }
566
+ }
567
+
461
568
  // src/internal/stats.ts
462
569
  var LOCAL_STATS_INTERVAL_MS = 2e3;
463
570
  var RUNTIME_REPORT_INTERVAL_MS = 5e3;
@@ -785,7 +892,7 @@ function describeCause(cause) {
785
892
  // package.json
786
893
  var package_default = {
787
894
  name: "@reactor-team/js-sdk",
788
- version: "3.3.0",
895
+ version: "3.4.0",
789
896
  description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
790
897
  license: "Apache-2.0",
791
898
  repository: {
@@ -1307,9 +1414,10 @@ var Reactor = class {
1307
1414
  * is finalized on "ready" — see `handleStatusChanged()`. */
1308
1415
  connectStartTime;
1309
1416
  waitingStartTime;
1310
- /** The tail of every `pushFrame()` so far, settled either way — pushes
1311
- * reach the binding in call order however long each one's pixels take to
1312
- * read, and `step()` waits for the ones called before it. */
1417
+ /** The tail of every `pushFrame()`/`pushEncodedFrame()` so far, settled
1418
+ * either way — pushes reach the binding in call order however long each
1419
+ * one's pixels take to read, and `step()` waits for the ones called
1420
+ * before it. */
1313
1421
  pendingPushes = Promise.resolve();
1314
1422
  emitter = new Emitter();
1315
1423
  /** Serializes connect()/reconnect()/disconnect() (and the free() inside
@@ -1650,7 +1758,8 @@ var Reactor = class {
1650
1758
  }, "publishTrack");
1651
1759
  return {
1652
1760
  name,
1653
- pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
1761
+ pushFrame: (frame, options) => this.pushTo(name, publication, frame, options),
1762
+ pushEncodedFrame: (frame) => this.pushEncodedTo(name, publication, frame)
1654
1763
  };
1655
1764
  } catch (cause) {
1656
1765
  throw this.captureError(cause);
@@ -1778,6 +1887,20 @@ var Reactor = class {
1778
1887
  * when its request cannot be sent, they are dropped with it rather than
1779
1888
  * kept for the next step.
1780
1889
  *
1890
+ * `media` gives the step frames of its own, by input track name, pushed
1891
+ * after the ones already pushed to that track: what `pushFrame()` takes,
1892
+ * or an `EncodedFrame` — H.264 the runtime decodes before the step runs,
1893
+ * so the model reads RGB frames as for a raw push. They are pushed only
1894
+ * once `state` and `commands` are well formed and the session is ready, so
1895
+ * a step refused for those leaves the frames buffered as they were. They go
1896
+ * all or nothing: a bad track or frame anywhere in `media` rejects the step
1897
+ * before any of it is pushed or sent, leaving the buffer as it was. A
1898
+ * session that leaves 'ready' once the media is in has the step refused and
1899
+ * keeps the frames for the next one, as pushed frames are.
1900
+ * A track's encoded frames of one step go in decode order, starting at a
1901
+ * keyframe; a step whose frames do not decode rejects with a
1902
+ * `StepRefusedError`.
1903
+ *
1781
1904
  * ```ts
1782
1905
  * const camera = await reactor.publishTrack('camera');
1783
1906
  * await camera.pushFrame(imageData);
@@ -1791,14 +1914,31 @@ var Reactor = class {
1791
1914
  this.assertNotDisposed();
1792
1915
  try {
1793
1916
  const state = rewriteNestedFileRefs(options.state ?? null);
1794
- const commands = (options.commands ?? []).map(
1795
- (command) => command.data ? { ...command, data: rewriteNestedFileRefs(command.data) } : command
1796
- );
1797
- await this.pendingPushes;
1798
- return await this.withReaderAccess(async () => {
1799
- const client = await this.getOrCreateClient();
1800
- return client.step(state, commands);
1917
+ const commands = rewriteNestedFileRefs(options.commands ?? []);
1918
+ const previous = this.pendingPushes;
1919
+ const before = options.media !== void 0 ? this.pushStepMedia(options.media, state, commands) : previous;
1920
+ let taken;
1921
+ const bufferTaken = new Promise((resolve) => {
1922
+ taken = resolve;
1801
1923
  });
1924
+ const stepping = before.then(
1925
+ () => (
1926
+ // Excluded from connect()/disconnect()/reconnect() like sendCommand():
1927
+ // a step can take minutes, and must neither be freed out from under
1928
+ // nor block unrelated reads.
1929
+ this.withReaderAccess(async () => {
1930
+ const client = await this.getOrCreateClient();
1931
+ try {
1932
+ return client.step(state, commands);
1933
+ } finally {
1934
+ taken();
1935
+ }
1936
+ })
1937
+ )
1938
+ );
1939
+ stepping.catch(() => previous.then(taken));
1940
+ this.pendingPushes = bufferTaken;
1941
+ return await stepping;
1802
1942
  } catch (cause) {
1803
1943
  throw this.captureError(cause);
1804
1944
  }
@@ -1857,19 +1997,7 @@ var Reactor = class {
1857
1997
  held.release();
1858
1998
  }
1859
1999
  await this.withReaderAccess(async () => {
1860
- if (epoch !== this.connectionEpoch) {
1861
- throw new InvalidStateError(
1862
- `pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
1863
- { operation: "pushFrame" }
1864
- );
1865
- }
1866
- const current = this.publishedInputs.get(trackName);
1867
- if (current === void 0 || publication !== void 0 && publication !== current) {
1868
- throw new InvalidStateError(
1869
- publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
1870
- { operation: "pushFrame" }
1871
- );
1872
- }
2000
+ this.checkPushTarget("pushFrame", trackName, publication, epoch);
1873
2001
  const client = await this.getOrCreateClient();
1874
2002
  if (pixels) {
1875
2003
  client.pushVideoFrame(
@@ -1893,6 +2021,149 @@ var Reactor = class {
1893
2021
  throw this.captureError(cause);
1894
2022
  }
1895
2023
  }
2024
+ /**
2025
+ * Buffers one encoded video frame for the next `step()` on the sendonly
2026
+ * video track `trackName`, its bytes as they are: one H.264 access unit
2027
+ * (Annex B), with the `width` and `height` it decodes to. The
2028
+ * `PublishedTrack` that `publishTrack()` resolves with does the same
2029
+ * through `track.pushEncodedFrame(frame)`.
2030
+ *
2031
+ * The runtime decodes a track's encoded frames of one step as one stream,
2032
+ * before the step runs, and the model reads the RGB frames as if they had
2033
+ * been pushed raw. So push them in decode order, starting at a keyframe; a
2034
+ * step whose frames do not decode rejects with a `StepRefusedError`, as
2035
+ * does a runtime that predates encoded frames. Encoded and raw pushes keep
2036
+ * their call order, and a `step()` waits for every push called before it.
2037
+ *
2038
+ * ```ts
2039
+ * const encoder = new VideoEncoder({
2040
+ * output: (chunk) => void camera.pushEncodedFrame({ codec: 'h264', data: chunk, width, height }),
2041
+ * error: console.error,
2042
+ * });
2043
+ * encoder.configure({ codec: 'avc1.42001f', width, height, avc: { format: 'annexb' } });
2044
+ * ```
2045
+ *
2046
+ * Rejects with an `InvalidStateError` outside a stepped session, for a
2047
+ * track not published on this connection, for an audio or recvonly track,
2048
+ * for empty data, a zero dimension or a codec other than `"h264"`.
2049
+ */
2050
+ async pushEncodedFrame(trackName, frame) {
2051
+ return this.pushEncodedTo(trackName, void 0, frame);
2052
+ }
2053
+ /** `pushEncodedFrame()`, for the track's current publication or, from a
2054
+ * `PublishedTrack`, only for the `publication` it came from. */
2055
+ async pushEncodedTo(trackName, publication, frame) {
2056
+ const operation = "pushEncodedFrame";
2057
+ this.assertNotDisposed();
2058
+ const epoch = this.connectionEpoch;
2059
+ let encoded;
2060
+ try {
2061
+ encoded = toEncodedBytes(frame, operation, trackName);
2062
+ } catch (cause) {
2063
+ throw this.captureError(cause);
2064
+ }
2065
+ const push = this.pendingPushes.then(
2066
+ () => this.withReaderAccess(async () => {
2067
+ if (!this.isStepped()) {
2068
+ throw new InvalidStateError(
2069
+ `${operation}('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
2070
+ { operation }
2071
+ );
2072
+ }
2073
+ this.checkPushTarget(operation, trackName, publication, epoch);
2074
+ const client = await this.getOrCreateClient();
2075
+ const declared = client.tracks().find((track) => track.name === trackName);
2076
+ if (declared && (declared.kind !== "video" || declared.direction !== "sendonly")) {
2077
+ throw new InvalidStateError(
2078
+ `${operation}('${trackName}'): '${trackName}' is a ${declared.direction} ${declared.kind} track; only a sendonly video track takes encoded frames.`,
2079
+ { operation }
2080
+ );
2081
+ }
2082
+ client.pushEncodedVideoFrame(
2083
+ trackName,
2084
+ encoded.codec,
2085
+ encoded.data,
2086
+ encoded.width,
2087
+ encoded.height,
2088
+ encoded.metadata,
2089
+ encoded.captureTimeUs
2090
+ );
2091
+ })
2092
+ );
2093
+ this.pendingPushes = push.catch(() => void 0);
2094
+ try {
2095
+ await push;
2096
+ } catch (cause) {
2097
+ throw this.captureError(cause);
2098
+ }
2099
+ }
2100
+ /** Refuses a push whose connection ended since it was called, or whose
2101
+ * track is not (or no longer) published on it. */
2102
+ checkPushTarget(operation, trackName, publication, epoch) {
2103
+ if (epoch !== this.connectionEpoch) {
2104
+ throw new InvalidStateError(
2105
+ `${operation}('${trackName}'): the connection it was pushed to ended while its frame was read`,
2106
+ { operation }
2107
+ );
2108
+ }
2109
+ const current = this.publishedInputs.get(trackName);
2110
+ if (current === void 0 || publication !== void 0 && publication !== current) {
2111
+ throw new InvalidStateError(
2112
+ publication === void 0 ? `${operation}('${trackName}') needs publishTrack('${trackName}') first.` : `${operation}('${trackName}'): this track was unpublished, or its connection ended.`,
2113
+ { operation }
2114
+ );
2115
+ }
2116
+ }
2117
+ /** Pushes `step()`'s `media`, track by track and in order, each frame as
2118
+ * `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
2119
+ * checks the step's `state` and `commands`, then every frame, before the
2120
+ * first is pushed, so a refused step, or a bad track or frame anywhere,
2121
+ * leaves the buffer as it was. */
2122
+ async pushStepMedia(media, state, commands) {
2123
+ const epoch = this.connectionEpoch;
2124
+ const held = holdStepMedia(media);
2125
+ const push = this.pendingPushes.then(async () => {
2126
+ const ready = [];
2127
+ try {
2128
+ for (const frame of held) {
2129
+ ready.push(await readStepFrame(frame));
2130
+ }
2131
+ } finally {
2132
+ for (const frame of held) {
2133
+ frame.release();
2134
+ }
2135
+ }
2136
+ await this.withReaderAccess(async () => {
2137
+ const client = await this.getOrCreateClient();
2138
+ client.checkStep(state, commands);
2139
+ for (const frame of ready) {
2140
+ this.checkPushTarget(frame.operation, frame.track, void 0, epoch);
2141
+ checkStepFrame(client, frame);
2142
+ }
2143
+ for (const frame of ready) {
2144
+ if (frame.kind === "encoded") {
2145
+ client.pushEncodedVideoFrame(
2146
+ frame.track,
2147
+ frame.encoded.codec,
2148
+ frame.encoded.data,
2149
+ frame.encoded.width,
2150
+ frame.encoded.height,
2151
+ frame.encoded.metadata,
2152
+ frame.encoded.captureTimeUs
2153
+ );
2154
+ } else if (frame.kind === "video") {
2155
+ const { data, width, height, format } = frame.pixels;
2156
+ client.pushVideoFrame(frame.track, data, width, height, format, null, null);
2157
+ } else {
2158
+ const { samples, sampleRate, channels } = frame.audio;
2159
+ client.pushAudioFrame(frame.track, samples, sampleRate, channels ?? 1);
2160
+ }
2161
+ }
2162
+ });
2163
+ });
2164
+ this.pendingPushes = push.catch(() => void 0);
2165
+ await push;
2166
+ }
1896
2167
  /** Drops every frame buffered for the next step, pushes called before it
1897
2168
  * included: it waits its turn behind them, as a `step()` does. */
1898
2169
  async clearStepInputs() {