@reactor-team/js-sdk 3.3.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -232,6 +232,58 @@ await reactor.step();
232
232
  `MessageTooLargeError` when the request is too large to send. The frames pushed for a step
233
233
  belong to it: when its request cannot be sent, the step rejects and its frames are dropped
234
234
  with it, not kept for the next step.
235
+ - A step's result travels in one data-channel message, whose limit is the smaller of the
236
+ client's `maxMessageSize` option (default 64 MiB) and the runtime's, which is 64 MiB unless
237
+ the model raises it in its `reactor.yaml` (`runtime.webrtc.max_message_size`). Raise
238
+ `maxMessageSize` only for a model whose steps need it: a message is held whole in the tab's
239
+ memory while it arrives.
240
+
241
+ ### Encoded video in a step
242
+
243
+ A step can carry video already encoded as H.264, its bytes as they are: much smaller than raw
244
+ RGB. The runtime decodes a track's encoded frames before the step runs, so the model reads RGB
245
+ frames exactly as if they had been pushed raw. Give the frames of one step in decode order,
246
+ starting at a keyframe. A WebCodecs `VideoEncoder`'s chunks go in as they are; configure it for
247
+ Annex B, as its default `avc` format is length-prefixed:
248
+
249
+ ```ts
250
+ import type { EncodedFrame } from "@reactor-team/js-sdk";
251
+
252
+ let gop: EncodedFrame[] = [];
253
+ const encoder = new VideoEncoder({
254
+ output: (chunk) => {
255
+ if (chunk.type === "key") gop = [];
256
+ gop.push({ codec: "h264", data: chunk, width: 1280, height: 720 });
257
+ },
258
+ error: console.error,
259
+ });
260
+ encoder.configure({ codec: "avc1.42001f", width: 1280, height: 720, avc: { format: "annexb" } });
261
+
262
+ // ... encode frames, then send the ones from the last keyframe with the step:
263
+ const result = await reactor.step({ media: { camera: gop } });
264
+
265
+ // The same, pushed ahead of the step:
266
+ for (const frame of gop) await camera.pushEncodedFrame(frame);
267
+ await reactor.step();
268
+ ```
269
+
270
+ - An `EncodedFrame` is `{ codec: "h264", data, width, height, metadata?, captureTimeUs? }`:
271
+ `data` one access unit as a `Uint8Array`, an `ArrayBuffer` or an `EncodedVideoChunk` (its
272
+ bytes are copied out when pushed), `width` and `height` the frame it decodes to. `codec` is
273
+ what tells it from a raw frame in `media`.
274
+ - `media` maps input track names to frames: `EncodedFrame`s, or anything `pushFrame()` takes.
275
+ They go after the frames pushed to the same track since the previous step, and only once
276
+ `state` and `commands` are known to be well formed and the session is ready, so a step
277
+ refused for those leaves the buffer as it was. Each track must be published first. A bad
278
+ track or frame anywhere in `media` fails the step before any of it is pushed, leaving the
279
+ buffer as it was too. A session that leaves ready once the media is in has the step refused
280
+ and keeps the frames for the next one, as pushed frames are. A push or step called after
281
+ `step()` waits for it to take the buffer, so its frames go to the next step, not this one.
282
+ - `pushEncodedFrame()` rejects with an `InvalidStateError` outside a stepped session, on an
283
+ audio or recvonly track, a track not published on this connection, empty data or a zero
284
+ dimension. A step whose frames do not decode, or do not start at a keyframe, rejects with a
285
+ `StepRefusedError` naming the frame in `entry`; a runtime that predates encoded frames
286
+ refuses them the same way.
235
287
 
236
288
  ## Recording and clips
237
289
 
package/dist/index.cjs CHANGED
@@ -330,6 +330,52 @@ var Emitter = class {
330
330
  }
331
331
  };
332
332
 
333
+ // src/internal/encoded-frames.ts
334
+ var CODECS = /* @__PURE__ */ new Set(["h264"]);
335
+ function isEncodedFrame(frame) {
336
+ return typeof frame === "object" && frame !== null && "codec" in frame;
337
+ }
338
+ function isChunk(data) {
339
+ return !(data instanceof Uint8Array) && !(data instanceof ArrayBuffer) && typeof data.copyTo === "function";
340
+ }
341
+ function toEncodedBytes(frame, operation, trackName) {
342
+ const refuse = (why) => new InvalidStateError(`${operation}('${trackName}'): ${why}`, { operation });
343
+ if (typeof frame !== "object" || frame === null) {
344
+ throw refuse("expected an EncodedFrame, { codec, data, width, height }");
345
+ }
346
+ const { codec, data, width, height, metadata, captureTimeUs } = frame;
347
+ if (!CODECS.has(codec)) {
348
+ throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264"`);
349
+ }
350
+ let bytes;
351
+ if (data instanceof Uint8Array) {
352
+ bytes = data;
353
+ } else if (data instanceof ArrayBuffer) {
354
+ bytes = new Uint8Array(data);
355
+ } else if (data && isChunk(data)) {
356
+ bytes = new Uint8Array(data.byteLength);
357
+ data.copyTo(bytes);
358
+ } else {
359
+ throw refuse("data must be a Uint8Array, an ArrayBuffer or an EncodedVideoChunk");
360
+ }
361
+ if (bytes.byteLength === 0) {
362
+ throw refuse("an encoded frame needs data");
363
+ }
364
+ if (!isDimension(width) || !isDimension(height)) {
365
+ throw refuse(`width and height are the decoded frame's, got ${String(width)}x${String(height)}`);
366
+ }
367
+ if (metadata != null && !(metadata instanceof Uint8Array)) {
368
+ throw refuse("metadata must be a Uint8Array");
369
+ }
370
+ if (captureTimeUs != null && !Number.isFinite(captureTimeUs)) {
371
+ throw refuse("captureTimeUs must be a finite number");
372
+ }
373
+ return { codec, data: bytes, width, height, metadata: metadata ?? null, captureTimeUs: captureTimeUs ?? null };
374
+ }
375
+ function isDimension(value) {
376
+ return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 4294967295;
377
+ }
378
+
333
379
  // src/file-ref.ts
334
380
  var FileRef = class {
335
381
  constructor(uploadId, name, mimeType, size) {
@@ -540,6 +586,67 @@ function toPublicClip(clip) {
540
586
  };
541
587
  }
542
588
 
589
+ // src/internal/step-media.ts
590
+ function holdStepMedia(media) {
591
+ if (typeof media !== "object" || media === null || Array.isArray(media)) {
592
+ throw new InvalidStateError("step(): media must map track names to arrays of frames", {
593
+ operation: "step"
594
+ });
595
+ }
596
+ const held = [];
597
+ try {
598
+ for (const [trackName, frames] of Object.entries(media)) {
599
+ if (!Array.isArray(frames)) {
600
+ throw new InvalidStateError(`step(): media['${trackName}'] must be an array of frames`, {
601
+ operation: "step"
602
+ });
603
+ }
604
+ for (const frame of frames) {
605
+ held.push(holdStepFrame(trackName, frame));
606
+ }
607
+ }
608
+ } catch (cause) {
609
+ for (const frame of held) {
610
+ frame.release();
611
+ }
612
+ throw cause;
613
+ }
614
+ return held;
615
+ }
616
+ function holdStepFrame(track, frame) {
617
+ if (isEncodedFrame(frame)) {
618
+ const operation = "pushEncodedFrame";
619
+ return { track, kind: "encoded", operation, encoded: toEncodedBytes(frame, operation, track), release: () => {
620
+ } };
621
+ }
622
+ if (typeof frame !== "object" || frame === null) {
623
+ throw new InvalidStateError(`pushFrame('${track}'): expected a frame`, { operation: "pushFrame" });
624
+ }
625
+ const held = holdFrame(frame);
626
+ return { track, kind: "raw", operation: "pushFrame", frame: held.frame, release: held.release };
627
+ }
628
+ async function readStepFrame(frame) {
629
+ const { track } = frame;
630
+ if (frame.kind === "encoded") {
631
+ return { track, kind: "encoded", operation: frame.operation, encoded: frame.encoded };
632
+ }
633
+ if (isAudioFrame(frame.frame)) {
634
+ return { track, kind: "audio", operation: frame.operation, audio: frame.frame };
635
+ }
636
+ return { track, kind: "video", operation: frame.operation, pixels: await toPixels(frame.frame) };
637
+ }
638
+ function checkStepFrame(client, frame) {
639
+ if (frame.kind === "encoded") {
640
+ const { codec, data, width, height, captureTimeUs } = frame.encoded;
641
+ client.checkEncodedVideoFrame(frame.track, codec, data.byteLength, width, height, captureTimeUs);
642
+ } else if (frame.kind === "video") {
643
+ const { data, width, height, format } = frame.pixels;
644
+ client.checkVideoFrame(frame.track, data.byteLength, width, height, format);
645
+ } else {
646
+ client.checkAudioFrame(frame.track, frame.audio.samples, frame.audio.channels ?? 1);
647
+ }
648
+ }
649
+
543
650
  // src/internal/stats.ts
544
651
  var LOCAL_STATS_INTERVAL_MS = 2e3;
545
652
  var RUNTIME_REPORT_INTERVAL_MS = 5e3;
@@ -867,7 +974,7 @@ function describeCause(cause) {
867
974
  // package.json
868
975
  var package_default = {
869
976
  name: "@reactor-team/js-sdk",
870
- version: "3.3.0",
977
+ version: "3.4.0",
871
978
  description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
872
979
  license: "Apache-2.0",
873
980
  repository: {
@@ -1389,9 +1496,10 @@ var Reactor = class {
1389
1496
  * is finalized on "ready" — see `handleStatusChanged()`. */
1390
1497
  connectStartTime;
1391
1498
  waitingStartTime;
1392
- /** The tail of every `pushFrame()` so far, settled either way — pushes
1393
- * reach the binding in call order however long each one's pixels take to
1394
- * read, and `step()` waits for the ones called before it. */
1499
+ /** The tail of every `pushFrame()`/`pushEncodedFrame()` so far, settled
1500
+ * either way — pushes reach the binding in call order however long each
1501
+ * one's pixels take to read, and `step()` waits for the ones called
1502
+ * before it. */
1395
1503
  pendingPushes = Promise.resolve();
1396
1504
  emitter = new Emitter();
1397
1505
  /** Serializes connect()/reconnect()/disconnect() (and the free() inside
@@ -1732,7 +1840,8 @@ var Reactor = class {
1732
1840
  }, "publishTrack");
1733
1841
  return {
1734
1842
  name,
1735
- pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
1843
+ pushFrame: (frame, options) => this.pushTo(name, publication, frame, options),
1844
+ pushEncodedFrame: (frame) => this.pushEncodedTo(name, publication, frame)
1736
1845
  };
1737
1846
  } catch (cause) {
1738
1847
  throw this.captureError(cause);
@@ -1860,6 +1969,20 @@ var Reactor = class {
1860
1969
  * when its request cannot be sent, they are dropped with it rather than
1861
1970
  * kept for the next step.
1862
1971
  *
1972
+ * `media` gives the step frames of its own, by input track name, pushed
1973
+ * after the ones already pushed to that track: what `pushFrame()` takes,
1974
+ * or an `EncodedFrame` — H.264 the runtime decodes before the step runs,
1975
+ * so the model reads RGB frames as for a raw push. They are pushed only
1976
+ * once `state` and `commands` are well formed and the session is ready, so
1977
+ * a step refused for those leaves the frames buffered as they were. They go
1978
+ * all or nothing: a bad track or frame anywhere in `media` rejects the step
1979
+ * before any of it is pushed or sent, leaving the buffer as it was. A
1980
+ * session that leaves 'ready' once the media is in has the step refused and
1981
+ * keeps the frames for the next one, as pushed frames are.
1982
+ * A track's encoded frames of one step go in decode order, starting at a
1983
+ * keyframe; a step whose frames do not decode rejects with a
1984
+ * `StepRefusedError`.
1985
+ *
1863
1986
  * ```ts
1864
1987
  * const camera = await reactor.publishTrack('camera');
1865
1988
  * await camera.pushFrame(imageData);
@@ -1873,14 +1996,31 @@ var Reactor = class {
1873
1996
  this.assertNotDisposed();
1874
1997
  try {
1875
1998
  const state = rewriteNestedFileRefs(options.state ?? null);
1876
- const commands = (options.commands ?? []).map(
1877
- (command) => command.data ? { ...command, data: rewriteNestedFileRefs(command.data) } : command
1878
- );
1879
- await this.pendingPushes;
1880
- return await this.withReaderAccess(async () => {
1881
- const client = await this.getOrCreateClient();
1882
- return client.step(state, commands);
1999
+ const commands = rewriteNestedFileRefs(options.commands ?? []);
2000
+ const previous = this.pendingPushes;
2001
+ const before = options.media !== void 0 ? this.pushStepMedia(options.media, state, commands) : previous;
2002
+ let taken;
2003
+ const bufferTaken = new Promise((resolve) => {
2004
+ taken = resolve;
1883
2005
  });
2006
+ const stepping = before.then(
2007
+ () => (
2008
+ // Excluded from connect()/disconnect()/reconnect() like sendCommand():
2009
+ // a step can take minutes, and must neither be freed out from under
2010
+ // nor block unrelated reads.
2011
+ this.withReaderAccess(async () => {
2012
+ const client = await this.getOrCreateClient();
2013
+ try {
2014
+ return client.step(state, commands);
2015
+ } finally {
2016
+ taken();
2017
+ }
2018
+ })
2019
+ )
2020
+ );
2021
+ stepping.catch(() => previous.then(taken));
2022
+ this.pendingPushes = bufferTaken;
2023
+ return await stepping;
1884
2024
  } catch (cause) {
1885
2025
  throw this.captureError(cause);
1886
2026
  }
@@ -1939,19 +2079,7 @@ var Reactor = class {
1939
2079
  held.release();
1940
2080
  }
1941
2081
  await this.withReaderAccess(async () => {
1942
- if (epoch !== this.connectionEpoch) {
1943
- throw new InvalidStateError(
1944
- `pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
1945
- { operation: "pushFrame" }
1946
- );
1947
- }
1948
- const current = this.publishedInputs.get(trackName);
1949
- if (current === void 0 || publication !== void 0 && publication !== current) {
1950
- throw new InvalidStateError(
1951
- publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
1952
- { operation: "pushFrame" }
1953
- );
1954
- }
2082
+ this.checkPushTarget("pushFrame", trackName, publication, epoch);
1955
2083
  const client = await this.getOrCreateClient();
1956
2084
  if (pixels) {
1957
2085
  client.pushVideoFrame(
@@ -1975,6 +2103,149 @@ var Reactor = class {
1975
2103
  throw this.captureError(cause);
1976
2104
  }
1977
2105
  }
2106
+ /**
2107
+ * Buffers one encoded video frame for the next `step()` on the sendonly
2108
+ * video track `trackName`, its bytes as they are: one H.264 access unit
2109
+ * (Annex B), with the `width` and `height` it decodes to. The
2110
+ * `PublishedTrack` that `publishTrack()` resolves with does the same
2111
+ * through `track.pushEncodedFrame(frame)`.
2112
+ *
2113
+ * The runtime decodes a track's encoded frames of one step as one stream,
2114
+ * before the step runs, and the model reads the RGB frames as if they had
2115
+ * been pushed raw. So push them in decode order, starting at a keyframe; a
2116
+ * step whose frames do not decode rejects with a `StepRefusedError`, as
2117
+ * does a runtime that predates encoded frames. Encoded and raw pushes keep
2118
+ * their call order, and a `step()` waits for every push called before it.
2119
+ *
2120
+ * ```ts
2121
+ * const encoder = new VideoEncoder({
2122
+ * output: (chunk) => void camera.pushEncodedFrame({ codec: 'h264', data: chunk, width, height }),
2123
+ * error: console.error,
2124
+ * });
2125
+ * encoder.configure({ codec: 'avc1.42001f', width, height, avc: { format: 'annexb' } });
2126
+ * ```
2127
+ *
2128
+ * Rejects with an `InvalidStateError` outside a stepped session, for a
2129
+ * track not published on this connection, for an audio or recvonly track,
2130
+ * for empty data, a zero dimension or a codec other than `"h264"`.
2131
+ */
2132
+ async pushEncodedFrame(trackName, frame) {
2133
+ return this.pushEncodedTo(trackName, void 0, frame);
2134
+ }
2135
+ /** `pushEncodedFrame()`, for the track's current publication or, from a
2136
+ * `PublishedTrack`, only for the `publication` it came from. */
2137
+ async pushEncodedTo(trackName, publication, frame) {
2138
+ const operation = "pushEncodedFrame";
2139
+ this.assertNotDisposed();
2140
+ const epoch = this.connectionEpoch;
2141
+ let encoded;
2142
+ try {
2143
+ encoded = toEncodedBytes(frame, operation, trackName);
2144
+ } catch (cause) {
2145
+ throw this.captureError(cause);
2146
+ }
2147
+ const push = this.pendingPushes.then(
2148
+ () => this.withReaderAccess(async () => {
2149
+ if (!this.isStepped()) {
2150
+ throw new InvalidStateError(
2151
+ `${operation}('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
2152
+ { operation }
2153
+ );
2154
+ }
2155
+ this.checkPushTarget(operation, trackName, publication, epoch);
2156
+ const client = await this.getOrCreateClient();
2157
+ const declared = client.tracks().find((track) => track.name === trackName);
2158
+ if (declared && (declared.kind !== "video" || declared.direction !== "sendonly")) {
2159
+ throw new InvalidStateError(
2160
+ `${operation}('${trackName}'): '${trackName}' is a ${declared.direction} ${declared.kind} track; only a sendonly video track takes encoded frames.`,
2161
+ { operation }
2162
+ );
2163
+ }
2164
+ client.pushEncodedVideoFrame(
2165
+ trackName,
2166
+ encoded.codec,
2167
+ encoded.data,
2168
+ encoded.width,
2169
+ encoded.height,
2170
+ encoded.metadata,
2171
+ encoded.captureTimeUs
2172
+ );
2173
+ })
2174
+ );
2175
+ this.pendingPushes = push.catch(() => void 0);
2176
+ try {
2177
+ await push;
2178
+ } catch (cause) {
2179
+ throw this.captureError(cause);
2180
+ }
2181
+ }
2182
+ /** Refuses a push whose connection ended since it was called, or whose
2183
+ * track is not (or no longer) published on it. */
2184
+ checkPushTarget(operation, trackName, publication, epoch) {
2185
+ if (epoch !== this.connectionEpoch) {
2186
+ throw new InvalidStateError(
2187
+ `${operation}('${trackName}'): the connection it was pushed to ended while its frame was read`,
2188
+ { operation }
2189
+ );
2190
+ }
2191
+ const current = this.publishedInputs.get(trackName);
2192
+ if (current === void 0 || publication !== void 0 && publication !== current) {
2193
+ throw new InvalidStateError(
2194
+ publication === void 0 ? `${operation}('${trackName}') needs publishTrack('${trackName}') first.` : `${operation}('${trackName}'): this track was unpublished, or its connection ended.`,
2195
+ { operation }
2196
+ );
2197
+ }
2198
+ }
2199
+ /** Pushes `step()`'s `media`, track by track and in order, each frame as
2200
+ * `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
2201
+ * checks the step's `state` and `commands`, then every frame, before the
2202
+ * first is pushed, so a refused step, or a bad track or frame anywhere,
2203
+ * leaves the buffer as it was. */
2204
+ async pushStepMedia(media, state, commands) {
2205
+ const epoch = this.connectionEpoch;
2206
+ const held = holdStepMedia(media);
2207
+ const push = this.pendingPushes.then(async () => {
2208
+ const ready = [];
2209
+ try {
2210
+ for (const frame of held) {
2211
+ ready.push(await readStepFrame(frame));
2212
+ }
2213
+ } finally {
2214
+ for (const frame of held) {
2215
+ frame.release();
2216
+ }
2217
+ }
2218
+ await this.withReaderAccess(async () => {
2219
+ const client = await this.getOrCreateClient();
2220
+ client.checkStep(state, commands);
2221
+ for (const frame of ready) {
2222
+ this.checkPushTarget(frame.operation, frame.track, void 0, epoch);
2223
+ checkStepFrame(client, frame);
2224
+ }
2225
+ for (const frame of ready) {
2226
+ if (frame.kind === "encoded") {
2227
+ client.pushEncodedVideoFrame(
2228
+ frame.track,
2229
+ frame.encoded.codec,
2230
+ frame.encoded.data,
2231
+ frame.encoded.width,
2232
+ frame.encoded.height,
2233
+ frame.encoded.metadata,
2234
+ frame.encoded.captureTimeUs
2235
+ );
2236
+ } else if (frame.kind === "video") {
2237
+ const { data, width, height, format } = frame.pixels;
2238
+ client.pushVideoFrame(frame.track, data, width, height, format, null, null);
2239
+ } else {
2240
+ const { samples, sampleRate, channels } = frame.audio;
2241
+ client.pushAudioFrame(frame.track, samples, sampleRate, channels ?? 1);
2242
+ }
2243
+ }
2244
+ });
2245
+ });
2246
+ this.pendingPushes = push.catch(() => void 0);
2247
+ await push;
2248
+ }
1978
2249
  /** Drops every frame buffered for the next step, pushes called before it
1979
2250
  * included: it waits its turn behind them, as a `step()` does. */
1980
2251
  async clearStepInputs() {