@reactor-team/js-sdk 3.2.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -37,9 +37,11 @@ __export(index_exports, {
37
37
  ClipPlayer: () => ClipPlayer,
38
38
  ConflictError: () => ConflictError,
39
39
  DEFAULT_PLAYLIST_POLL_SLACK_MS: () => DEFAULT_PLAYLIST_POLL_SLACK_MS,
40
+ DataChannelChunkingRequiredError: () => DataChannelChunkingRequiredError,
40
41
  DecodeError: () => DecodeError,
41
42
  DisconnectedError: () => DisconnectedError,
42
43
  FileRef: () => FileRef,
44
+ GenerationModeMismatchError: () => GenerationModeMismatchError,
43
45
  InvalidStateError: () => InvalidStateError,
44
46
  MessageTooLargeError: () => MessageTooLargeError,
45
47
  NetworkError: () => NetworkError,
@@ -54,8 +56,14 @@ __export(index_exports, {
54
56
  RequestTimeoutError: () => RequestTimeoutError,
55
57
  ServerError: () => ServerError,
56
58
  SessionTerminalError: () => SessionTerminalError,
59
+ StepCommandFailedError: () => StepCommandFailedError,
60
+ StepNotReadyError: () => StepNotReadyError,
61
+ StepRefusedError: () => StepRefusedError,
62
+ StepResultTooLargeError: () => StepResultTooLargeError,
63
+ StepSessionEndedError: () => StepSessionEndedError,
57
64
  TransportError: () => TransportError,
58
65
  UnauthorizedError: () => UnauthorizedError,
66
+ UnsupportedFrameEncodingError: () => UnsupportedFrameEncodingError,
59
67
  VersionMismatchError: () => VersionMismatchError,
60
68
  WebcamStream: () => WebcamStream,
61
69
  createPlayableManifestUrl: () => createPlayableManifestUrl,
@@ -64,6 +72,7 @@ __export(index_exports, {
64
72
  isFileRef: () => isFileRef,
65
73
  normalizeJwtSource: () => normalizeJwtSource,
66
74
  parsePlaylist: () => parsePlaylist,
75
+ stepFrameToImageData: () => stepFrameToImageData,
67
76
  useClipDownload: () => useClipDownload,
68
77
  useReactor: () => useReactor,
69
78
  useReactorInternalMessage: () => useReactorInternalMessage,
@@ -117,6 +126,14 @@ var ReactorError = class extends Error {
117
126
  operation;
118
127
  /** Backoff hint the platform sent. */
119
128
  retry_after_ms;
129
+ /** For a refused step or a step whose call failed, the part of the step
130
+ * request at fault: `"state.<key>"`, `"commands[<i>]"` or
131
+ * `"media_input[<i>]"`. */
132
+ entry;
133
+ /** For a step whose call failed, the stable code its handler raised (its
134
+ * `CommandError` code, or `internal_error`), to branch on instead of the
135
+ * message. */
136
+ command_code;
120
137
  /** When `reactor-core` reported this failure. For a purely local one (no
121
138
  * payload at all — see {@link ReactorErrorOptions}), this is synthesized
122
139
  * as the construction time instead, since there's nothing to report —
@@ -137,6 +154,8 @@ var ReactorError = class extends Error {
137
154
  this.operation = options.operation;
138
155
  this.retry_after_ms = options.retry_after_ms;
139
156
  this.retryAfter = options.retry_after_ms;
157
+ this.entry = options.entry;
158
+ this.command_code = options.command_code;
140
159
  this.timestamp_ms = options.timestamp_ms ?? Date.now();
141
160
  this.timestamp = this.timestamp_ms;
142
161
  }
@@ -192,6 +211,30 @@ var AbortedError = class extends ReactorError {
192
211
  var RecorderDisabledError = class extends ReactorError {
193
212
  static code = "RECORDER_DISABLED";
194
213
  };
214
+ var GenerationModeMismatchError = class extends ReactorError {
215
+ static code = "GENERATION_MODE_MISMATCH";
216
+ };
217
+ var DataChannelChunkingRequiredError = class extends ReactorError {
218
+ static code = "DATA_CHANNEL_CHUNKING_REQUIRED";
219
+ };
220
+ var StepRefusedError = class extends ReactorError {
221
+ static code = "step_refused";
222
+ };
223
+ var StepNotReadyError = class extends ReactorError {
224
+ static code = "step_not_ready";
225
+ };
226
+ var StepResultTooLargeError = class extends ReactorError {
227
+ static code = "step_result_too_large";
228
+ };
229
+ var StepCommandFailedError = class extends ReactorError {
230
+ static code = "step_command_failed";
231
+ };
232
+ var StepSessionEndedError = class extends ReactorError {
233
+ static code = "step_session_ended";
234
+ };
235
+ var UnsupportedFrameEncodingError = class extends ReactorError {
236
+ static code = "UNSUPPORTED_FRAME_ENCODING";
237
+ };
195
238
  var ERROR_CLASSES = [
196
239
  NetworkError,
197
240
  UnauthorizedError,
@@ -209,7 +252,15 @@ var ERROR_CLASSES = [
209
252
  DisconnectedError,
210
253
  RequestTimeoutError,
211
254
  AbortedError,
212
- RecorderDisabledError
255
+ RecorderDisabledError,
256
+ GenerationModeMismatchError,
257
+ DataChannelChunkingRequiredError,
258
+ StepRefusedError,
259
+ StepNotReadyError,
260
+ StepResultTooLargeError,
261
+ StepCommandFailedError,
262
+ StepSessionEndedError,
263
+ UnsupportedFrameEncodingError
213
264
  ];
214
265
  var BY_CODE = new Map(
215
266
  ERROR_CLASSES.map((errorClass) => [errorClass.code, errorClass])
@@ -231,6 +282,8 @@ function toReactorError(cause) {
231
282
  operation: payload.operation,
232
283
  retry_after_ms: payload.retry_after_ms,
233
284
  timestamp_ms: payload.timestamp_ms,
285
+ entry: payload.entry,
286
+ command_code: payload.command_code,
234
287
  // A thrown JS error (a failed wasm load, say) keeps its stack and its own
235
288
  // `cause` reachable; reactor-core's plain payload objects have neither.
236
289
  cause: cause instanceof Error ? cause : void 0
@@ -368,6 +421,112 @@ function isSerializedAsObject(value) {
368
421
  return typeof value === "object" && value !== null && !ArrayBuffer.isView(value);
369
422
  }
370
423
 
424
+ // src/internal/push-frames.ts
425
+ function isAudioFrame(frame) {
426
+ return "samples" in frame;
427
+ }
428
+ function isVideoFrame(frame) {
429
+ return typeof frame.copyTo === "function" && "displayWidth" in frame;
430
+ }
431
+ function holdFrame(frame) {
432
+ if (!isAudioFrame(frame) && isVideoFrame(frame)) {
433
+ const clone = frame.clone();
434
+ return { frame: clone, release: () => clone.close() };
435
+ }
436
+ return { frame, release: () => {
437
+ } };
438
+ }
439
+ async function toPixels(frame) {
440
+ if (isVideoFrame(frame)) {
441
+ return videoFramePixels(frame);
442
+ }
443
+ return rawPixels(frame);
444
+ }
445
+ function rawPixels(frame) {
446
+ const { data, width, height } = frame;
447
+ const bytes = data instanceof Uint8Array ? data : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
448
+ return { data: bytes, width, height, format: "format" in frame && frame.format || "rgba" };
449
+ }
450
+ var NATIVE_FORMATS = {
451
+ RGBA: "rgba",
452
+ RGBX: "rgba",
453
+ BGRA: "bgra",
454
+ BGRX: "bgra"
455
+ };
456
+ function orientationOf(frame) {
457
+ const { rotation, flip } = frame;
458
+ return { rotation: ((rotation ?? 0) % 360 + 360) % 360, flip: flip ?? false };
459
+ }
460
+ async function videoFramePixels(frame) {
461
+ const rect = frame.visibleRect;
462
+ const width = rect?.width ?? frame.codedWidth;
463
+ const height = rect?.height ?? frame.codedHeight;
464
+ const native = frame.format ? NATIVE_FORMATS[frame.format] : void 0;
465
+ const orientation = orientationOf(frame);
466
+ if (native) {
467
+ const data = new Uint8Array(width * height * 4);
468
+ await frame.copyTo(data, { layout: [{ offset: 0, stride: width * 4 }] });
469
+ return orient({ data, width, height, format: native }, orientation);
470
+ }
471
+ try {
472
+ const data = new Uint8Array(width * height * 4);
473
+ const options = { format: "RGBA", layout: [{ offset: 0, stride: width * 4 }] };
474
+ await frame.copyTo(data, options);
475
+ return orient({ data, width, height, format: "rgba" }, orientation);
476
+ } catch {
477
+ return canvasPixels(frame, width, height, orientation);
478
+ }
479
+ }
480
+ function orient(pixels, { rotation, flip }) {
481
+ if (rotation % 90 !== 0 || rotation === 0 && !flip) {
482
+ return pixels;
483
+ }
484
+ const { width: w, height: h } = pixels;
485
+ const turned = rotation === 90 || rotation === 270;
486
+ const outWidth = turned ? h : w;
487
+ const outHeight = turned ? w : h;
488
+ const bytes = pixels.data.byteOffset % 4 === 0 ? pixels.data : pixels.data.slice();
489
+ const source = new Uint32Array(bytes.buffer, bytes.byteOffset, w * h);
490
+ const data = new Uint8Array(outWidth * outHeight * 4);
491
+ const target = new Uint32Array(data.buffer);
492
+ for (let y = 0; y < h; y++) {
493
+ for (let x = 0; x < w; x++) {
494
+ let tx;
495
+ let ty;
496
+ if (rotation === 90) {
497
+ [tx, ty] = [h - 1 - y, x];
498
+ } else if (rotation === 180) {
499
+ [tx, ty] = [w - 1 - x, h - 1 - y];
500
+ } else if (rotation === 270) {
501
+ [tx, ty] = [y, w - 1 - x];
502
+ } else {
503
+ [tx, ty] = [x, y];
504
+ }
505
+ if (flip) {
506
+ tx = outWidth - 1 - tx;
507
+ }
508
+ target[ty * outWidth + tx] = source[y * w + x];
509
+ }
510
+ }
511
+ return { data, width: outWidth, height: outHeight, format: pixels.format };
512
+ }
513
+ function canvasPixels(frame, visibleWidth, visibleHeight, { rotation }) {
514
+ const turned = rotation === 90 || rotation === 270;
515
+ const width = turned ? visibleHeight : visibleWidth;
516
+ const height = turned ? visibleWidth : visibleHeight;
517
+ if (typeof OffscreenCanvas === "undefined") {
518
+ throw new Error(
519
+ `pushFrame: cannot read a ${frame.format ?? "formatless"} VideoFrame here \u2014 push its pixels as an ImageData or a Uint8Array instead.`
520
+ );
521
+ }
522
+ const context = new OffscreenCanvas(width, height).getContext("2d");
523
+ if (!context) {
524
+ throw new Error("pushFrame: no 2D canvas context to read the VideoFrame through.");
525
+ }
526
+ context.drawImage(frame, 0, 0, width, height);
527
+ return rawPixels(context.getImageData(0, 0, width, height));
528
+ }
529
+
371
530
  // src/internal/recording.ts
372
531
  function toPublicClip(clip) {
373
532
  return {
@@ -430,7 +589,28 @@ var WIRE_CODECS = {
430
589
  };
431
590
  function createTrackStatsExtractor() {
432
591
  const lastBytes = /* @__PURE__ */ new Map();
433
- return (report, tracks, pausedTracks) => {
592
+ const lastStages = /* @__PURE__ */ new Map();
593
+ let pendingBytes = /* @__PURE__ */ new Map();
594
+ let pendingStages = /* @__PURE__ */ new Map();
595
+ const stage = (streamId, name, seconds, frames) => {
596
+ if (seconds === void 0 || frames === void 0) {
597
+ return void 0;
598
+ }
599
+ const key = `${streamId}:${name}`;
600
+ if (!Number.isFinite(seconds)) {
601
+ pendingStages.set(key, void 0);
602
+ return void 0;
603
+ }
604
+ pendingStages.set(key, { seconds, frames });
605
+ const before = lastStages.get(key);
606
+ if (before === void 0 || frames <= before.frames || seconds < before.seconds) {
607
+ return void 0;
608
+ }
609
+ return { name, totalMs: (seconds - before.seconds) * 1e3, frames: frames - before.frames };
610
+ };
611
+ const extract = (report, tracks, pausedTracks) => {
612
+ pendingBytes = /* @__PURE__ */ new Map();
613
+ pendingStages = /* @__PURE__ */ new Map();
434
614
  const reportWithLookup = report;
435
615
  const stats = [];
436
616
  const seenStreams = /* @__PURE__ */ new Set();
@@ -469,7 +649,7 @@ function createTrackStatsExtractor() {
469
649
  if (previous !== void 0 && stat.timestamp > previous.timestamp && bytes >= previous.bytes) {
470
650
  put("bitrate_bps", Math.round((bytes - previous.bytes) * 8 * 1e3 / (stat.timestamp - previous.timestamp)));
471
651
  }
472
- lastBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
652
+ pendingBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
473
653
  }
474
654
  const keyframeRequests = stat.firCount !== void 0 || stat.pliCount !== void 0 ? (stat.firCount ?? 0) + (stat.pliCount ?? 0) : void 0;
475
655
  if (inbound) {
@@ -504,6 +684,20 @@ function createTrackStatsExtractor() {
504
684
  put("keyframe_requests", keyframeRequests);
505
685
  }
506
686
  }
687
+ const frameStages = [];
688
+ if (track.kind === "video") {
689
+ const stages = inbound ? [
690
+ stage(stat.id, "jitter_buffer", stat.jitterBufferDelay, stat.jitterBufferEmittedCount),
691
+ stage(stat.id, "decode", stat.totalDecodeTime, stat.framesDecoded)
692
+ ] : [stage(stat.id, "encode", stat.totalEncodeTime, stat.framesEncoded)];
693
+ for (const s of stages) {
694
+ if (s !== void 0) {
695
+ frameStages.push(s);
696
+ }
697
+ }
698
+ }
699
+ const codecImplementation = inbound ? stat.decoderImplementation : stat.encoderImplementation;
700
+ const hardwareCodec = inbound ? stat.powerEfficientDecoder : stat.powerEfficientEncoder;
507
701
  stats.push({
508
702
  timestamp: Date.now(),
509
703
  trackName: track.name,
@@ -511,11 +705,30 @@ function createTrackStatsExtractor() {
511
705
  direction,
512
706
  codec,
513
707
  paused: pausedTracks.includes(track.name),
514
- metrics
708
+ metrics,
709
+ ...frameStages.length > 0 ? { frameStages } : {},
710
+ ...track.kind === "video" && codecImplementation !== void 0 ? { codecImplementation } : {},
711
+ ...track.kind === "video" && hardwareCodec !== void 0 ? { hardwareCodec } : {}
515
712
  });
516
713
  });
517
714
  return stats;
518
715
  };
716
+ return Object.assign(extract, {
717
+ commit() {
718
+ for (const [id, sample] of pendingBytes) {
719
+ lastBytes.set(id, sample);
720
+ }
721
+ for (const [key, total] of pendingStages) {
722
+ if (total === void 0) {
723
+ lastStages.delete(key);
724
+ } else {
725
+ lastStages.set(key, total);
726
+ }
727
+ }
728
+ pendingBytes = /* @__PURE__ */ new Map();
729
+ pendingStages = /* @__PURE__ */ new Map();
730
+ }
731
+ });
519
732
  }
520
733
  function toClientConnectionStat(report, timeToConnectMs) {
521
734
  const pair = activeCandidatePair(report);
@@ -654,7 +867,7 @@ function describeCause(cause) {
654
867
  // package.json
655
868
  var package_default = {
656
869
  name: "@reactor-team/js-sdk",
657
- version: "3.2.0",
870
+ version: "3.3.0",
658
871
  description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
659
872
  license: "Apache-2.0",
660
873
  repository: {
@@ -1168,10 +1381,18 @@ var Reactor = class {
1168
1381
  * with it, so a batch sent before the timings exist doesn't use it up.
1169
1382
  * Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
1170
1383
  hasSentConnectTime = false;
1384
+ /** Whether this connection has sent a client-stats batch yet. The first
1385
+ * one goes out even with no reading, for the environment the bridge adds
1386
+ * to it. Reset with `hasSentConnectTime`. */
1387
+ hasSentClientStats = false;
1171
1388
  /** Set on the "connecting" status transition, cleared once `connectionTimings`
1172
1389
  * is finalized on "ready" — see `handleStatusChanged()`. */
1173
1390
  connectStartTime;
1174
1391
  waitingStartTime;
1392
+ /** The tail of every `pushFrame()` so far, settled either way — pushes
1393
+ * reach the binding in call order however long each one's pixels take to
1394
+ * read, and `step()` waits for the ones called before it. */
1395
+ pendingPushes = Promise.resolve();
1175
1396
  emitter = new Emitter();
1176
1397
  /** Serializes connect()/reconnect()/disconnect() (and the free() inside
1177
1398
  * disconnect()/[Symbol.dispose]) against each other. Calling into the
@@ -1189,6 +1410,15 @@ var Reactor = class {
1189
1410
  * out every read active when it started, before it releases the ones
1190
1411
  * that arrived while it ran. */
1191
1412
  writerActive = false;
1413
+ /** Counts `disconnect()`s and disposal. A push captures it when called, and
1414
+ * one whose frame was still being read when it moved is dropped, rather
1415
+ * than recreating a client to carry the frame into the next session. */
1416
+ connectionEpoch = 0;
1417
+ /** Input tracks published on this connection, each with the publication
1418
+ * that claimed it: a push needs one, and a `PublishedTrack` only feeds the
1419
+ * publication it came from. Cleared with the connection. */
1420
+ publishedInputs = /* @__PURE__ */ new Map();
1421
+ publications = 0;
1192
1422
  /** Readers parked in `withReaderAccess()`'s wait loop, woken once
1193
1423
  * `withWriterAccess()` finishes; each re-checks `writerActive` rather
1194
1424
  * than assuming its turn, since another writer already queued behind the
@@ -1461,14 +1691,49 @@ var Reactor = class {
1461
1691
  * disconnect() freeing the wasm client mid-await would otherwise resume
1462
1692
  * into a freed client — the same class of use-after-free `queue` exists to
1463
1693
  * prevent for connect/disconnect/reconnect themselves.
1694
+ *
1695
+ * Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
1696
+ * track in a stepped session.
1697
+ *
1698
+ * In a `generation: "stepped"` session, input frames never travel over
1699
+ * RTP: `track` is optional, and the frames are pushed with
1700
+ * `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
1701
+ * passed there is ignored, with a console warning — nothing is sent from
1702
+ * it. In a streamed session `track` is required, and omitting it rejects
1703
+ * with an `InvalidStateError`.
1704
+ *
1705
+ * ```ts
1706
+ * const camera = await reactor.publishTrack('camera');
1707
+ * await camera.pushFrame(imageData);
1708
+ * const result = await reactor.step();
1709
+ * ```
1464
1710
  */
1465
1711
  async publishTrack(name, track) {
1466
1712
  this.assertNotDisposed();
1467
1713
  try {
1714
+ let media = track;
1715
+ if (this.isStepped() && media !== void 0) {
1716
+ console.warn(
1717
+ `[Reactor] publishTrack('${name}'): ignoring the MediaStreamTrack \u2014 a stepped session sends no RTP and takes frames through track.pushFrame(...) on the track publishTrack() returns.`
1718
+ );
1719
+ media = void 0;
1720
+ }
1721
+ if (!this.isStepped() && media === void 0) {
1722
+ throw new InvalidStateError(
1723
+ `publishTrack('${name}') needs a MediaStreamTrack in a streamed session.`,
1724
+ { operation: "publishTrack" }
1725
+ );
1726
+ }
1727
+ const publication = this.publications += 1;
1468
1728
  await this.queue.push(async () => {
1469
1729
  const client = await this.getOrCreateClient();
1470
- await client.publishTrack(name, track);
1730
+ await client.publishTrack(name, media);
1731
+ this.publishedInputs.set(name, publication);
1471
1732
  }, "publishTrack");
1733
+ return {
1734
+ name,
1735
+ pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
1736
+ };
1472
1737
  } catch (cause) {
1473
1738
  throw this.captureError(cause);
1474
1739
  }
@@ -1485,6 +1750,7 @@ var Reactor = class {
1485
1750
  await this.queue.push(async () => {
1486
1751
  const client = await this.getOrCreateClient();
1487
1752
  await client.unpublishTrack(name);
1753
+ this.publishedInputs.delete(name);
1488
1754
  }, "unpublishTrack");
1489
1755
  } catch (cause) {
1490
1756
  this.emitError(cause);
@@ -1557,6 +1823,175 @@ var Reactor = class {
1557
1823
  throw this.captureError(cause);
1558
1824
  }
1559
1825
  }
1826
+ // ── Stepped generation ──────────────────────────────────────────────────
1827
+ /**
1828
+ * Runs one step of a `generation: "stepped"` session and resolves with
1829
+ * what it produced, per output track.
1830
+ *
1831
+ * The request carries `state`, `commands` and every frame pushed since the
1832
+ * previous step. The runtime applies them in that order — each `state` key
1833
+ * as the model's `set_<key>` command, then `commands`, then the frames —
1834
+ * and then generates. They are separate calls, not a transaction: one that
1835
+ * fails does not undo the ones before it. All of it is optional:
1836
+ * `step()` with nothing set and nothing pushed runs on the model's current
1837
+ * state, which commands sent outside a step (`sendCommand()`) also change.
1838
+ * A file goes in the way it does for a command: upload it with
1839
+ * `uploadFile()`, then put the `FileRef` in a command's `data` or a state
1840
+ * value.
1841
+ *
1842
+ * A step that ran but that the model reported as failed still resolves,
1843
+ * with `result.error` set. `result.files` lists the files the model kept
1844
+ * with the step (name, content type, size); their bytes are served over
1845
+ * HTTP from the session's step results, not carried here.
1846
+ *
1847
+ * Rejects with a `StepRefusedError` when the runtime or the model turns
1848
+ * the step down (the runtime checks a whole request first, so nothing it
1849
+ * carried was applied), `StepNotReadyError` when the session cannot take a
1850
+ * step yet (neither ran the step), `StepCommandFailedError` when a state key
1851
+ * or command failed (the calls before it stay applied; the step did not
1852
+ * run); both name the part at fault in `entry`, and `StepCommandFailedError`
1853
+ * the handler's code in `command_code`,
1854
+ * `StepSessionEndedError` when the session ended before the result came
1855
+ * back, `StepResultTooLargeError` when the result does not fit in one
1856
+ * message, `MessageTooLargeError` when the request itself is too large
1857
+ * (nothing was sent), `InvalidStateError` outside a ready stepped
1858
+ * session, or whatever else failed the step (`DisconnectedError`,
1859
+ * `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
1860
+ * when its request cannot be sent, they are dropped with it rather than
1861
+ * kept for the next step.
1862
+ *
1863
+ * ```ts
1864
+ * const camera = await reactor.publishTrack('camera');
1865
+ * await camera.pushFrame(imageData);
1866
+ * const result = await reactor.step({ state: { prompt: 'a cat' } });
1867
+ * if (result.error) console.warn('step failed:', result.error);
1868
+ * const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
1869
+ * if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
1870
+ * ```
1871
+ */
1872
+ async step(options = {}) {
1873
+ this.assertNotDisposed();
1874
+ try {
1875
+ const state = rewriteNestedFileRefs(options.state ?? null);
1876
+ const commands = (options.commands ?? []).map(
1877
+ (command) => command.data ? { ...command, data: rewriteNestedFileRefs(command.data) } : command
1878
+ );
1879
+ await this.pendingPushes;
1880
+ return await this.withReaderAccess(async () => {
1881
+ const client = await this.getOrCreateClient();
1882
+ return client.step(state, commands);
1883
+ });
1884
+ } catch (cause) {
1885
+ throw this.captureError(cause);
1886
+ }
1887
+ }
1888
+ /**
1889
+ * Buffers one input frame for the next `step()` on the sendonly track
1890
+ * `trackName`. Nothing is sent until that step, which carries every frame
1891
+ * pushed since the one before it, per track in push order. The
1892
+ * `PublishedTrack` that `publishTrack()` resolves with does the same
1893
+ * through `track.pushFrame(frame)`.
1894
+ *
1895
+ * Video: a `VideoFrame` (cloned when called, so the caller may close it as
1896
+ * soon as this returns; it stays the caller's), an
1897
+ * `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
1898
+ * by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
1899
+ * `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
1900
+ * applied, so a 90° frame arrives with its width and height swapped.
1901
+ * Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
1902
+ * PCM at the rate the model's track declares.
1903
+ *
1904
+ * Pushes keep their call order, and a `step()` waits for every push called
1905
+ * before it. A refused push rejects: await it, or `.catch()` it in a frame
1906
+ * loop, or every bad frame is an unhandled rejection. What waits for
1907
+ * the next step is at most one message's worth of frames: pushing faster
1908
+ * than steps run drops the oldest frames, across tracks. Rejects with
1909
+ * an `InvalidStateError` outside a stepped session, for a track not
1910
+ * published on this connection (`publishTrack(trackName)` first), for a
1911
+ * frame whose size does not match its dimensions, or for a name the model
1912
+ * does not declare as a sendonly track of that kind.
1913
+ */
1914
+ async pushFrame(trackName, frame, options = {}) {
1915
+ return this.pushTo(trackName, void 0, frame, options);
1916
+ }
1917
+ /** `pushFrame()`, for the track's current publication or, from a
1918
+ * `PublishedTrack`, only for the `publication` it came from. */
1919
+ async pushTo(trackName, publication, frame, options = {}) {
1920
+ this.assertNotDisposed();
1921
+ const epoch = this.connectionEpoch;
1922
+ let held;
1923
+ try {
1924
+ held = holdFrame(frame);
1925
+ } catch (cause) {
1926
+ throw this.captureError(cause);
1927
+ }
1928
+ const push = this.pendingPushes.then(async () => {
1929
+ let pixels;
1930
+ try {
1931
+ if (!this.isStepped()) {
1932
+ throw new InvalidStateError(
1933
+ `pushFrame('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
1934
+ { operation: "pushFrame" }
1935
+ );
1936
+ }
1937
+ pixels = isAudioFrame(held.frame) ? void 0 : await toPixels(held.frame);
1938
+ } finally {
1939
+ held.release();
1940
+ }
1941
+ await this.withReaderAccess(async () => {
1942
+ if (epoch !== this.connectionEpoch) {
1943
+ throw new InvalidStateError(
1944
+ `pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
1945
+ { operation: "pushFrame" }
1946
+ );
1947
+ }
1948
+ const current = this.publishedInputs.get(trackName);
1949
+ if (current === void 0 || publication !== void 0 && publication !== current) {
1950
+ throw new InvalidStateError(
1951
+ publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
1952
+ { operation: "pushFrame" }
1953
+ );
1954
+ }
1955
+ const client = await this.getOrCreateClient();
1956
+ if (pixels) {
1957
+ client.pushVideoFrame(
1958
+ trackName,
1959
+ pixels.data,
1960
+ pixels.width,
1961
+ pixels.height,
1962
+ pixels.format,
1963
+ options.metadata ?? null,
1964
+ options.captureTimeUs ?? null
1965
+ );
1966
+ } else if (isAudioFrame(frame)) {
1967
+ client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
1968
+ }
1969
+ });
1970
+ });
1971
+ this.pendingPushes = push.catch(() => void 0);
1972
+ try {
1973
+ await push;
1974
+ } catch (cause) {
1975
+ throw this.captureError(cause);
1976
+ }
1977
+ }
1978
+ /** Drops every frame buffered for the next step, pushes called before it
1979
+ * included: it waits its turn behind them, as a `step()` does. */
1980
+ async clearStepInputs() {
1981
+ this.assertNotDisposed();
1982
+ const clear = this.pendingPushes.then(
1983
+ () => this.withReaderAccess(() => {
1984
+ this.client?.clearStepInputs();
1985
+ return Promise.resolve();
1986
+ })
1987
+ );
1988
+ this.pendingPushes = clear.catch(() => void 0);
1989
+ try {
1990
+ await clear;
1991
+ } catch (cause) {
1992
+ throw this.captureError(cause);
1993
+ }
1994
+ }
1560
1995
  // ── Recording ───────────────────────────────────────────────────────────
1561
1996
  /**
1562
1997
  * Requests a clip covering the last `durationSeconds` of the session.
@@ -1855,6 +2290,7 @@ var Reactor = class {
1855
2290
  const extractConnectionStats = createConnectionStatsExtractor();
1856
2291
  const extractTrackStats = createTrackStatsExtractor();
1857
2292
  this.hasSentConnectTime = false;
2293
+ this.hasSentClientStats = false;
1858
2294
  this.stopStatsPollers = [
1859
2295
  pollPeerStats(client, LOCAL_STATS_INTERVAL_MS, (report) => {
1860
2296
  this.stats = { ...extractConnectionStats(report), connectionTimings: this.connectionTimings };
@@ -1872,8 +2308,9 @@ var Reactor = class {
1872
2308
  * track has one yet (none negotiated onto a named track, or no codec
1873
2309
  * reported yet), so the runtime gets the bandwidth estimate, the
1874
2310
  * round-trip time and the time to connect without waiting on the tracks.
1875
- * Skipped only when there is nothing at all to say; one is due again in
1876
- * `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
2311
+ * Skipped only when there is nothing at all to say and a batch has already
2312
+ * gone out for this connection, so the first one still carries the
2313
+ * environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
1877
2314
  * actually sent for this connection — see `hasSentConnectTime`.
1878
2315
  * `sendClientStats` throws synchronously on a closing connection; caught
1879
2316
  * here for the same reason `getStats()`'s rejection is above. */
@@ -1881,11 +2318,13 @@ var Reactor = class {
1881
2318
  const trackStats = extractTrackStats(report, this.trackMapping(), this.pausedTracks());
1882
2319
  const timeToConnectMs = this.hasSentConnectTime ? void 0 : this.connectionTimings?.totalMs;
1883
2320
  const connectionStat = toClientConnectionStat(report, timeToConnectMs);
1884
- if (trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
2321
+ if (this.hasSentClientStats && trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
1885
2322
  return;
1886
2323
  }
1887
2324
  try {
1888
2325
  client.sendClientStats(trackStats, connectionStat);
2326
+ extractTrackStats.commit();
2327
+ this.hasSentClientStats = true;
1889
2328
  if (timeToConnectMs !== void 0) {
1890
2329
  this.hasSentConnectTime = true;
1891
2330
  }
@@ -1907,6 +2346,8 @@ var Reactor = class {
1907
2346
  * leave `getCapabilities()` returning the previous session's stale
1908
2347
  * tracks/commands until a new `capabilitiesReceived` lands. */
1909
2348
  resetConnectionState() {
2349
+ this.connectionEpoch += 1;
2350
+ this.publishedInputs.clear();
1910
2351
  this.stopStatsPolling();
1911
2352
  this.connectionTimings = void 0;
1912
2353
  this.connectStartTime = void 0;
@@ -1927,6 +2368,9 @@ var Reactor = class {
1927
2368
  emitError(cause) {
1928
2369
  this.emitter.emit("error", this.captureError(cause));
1929
2370
  }
2371
+ isStepped() {
2372
+ return this.clientOptions.generation === "stepped";
2373
+ }
1930
2374
  assertNotDisposed() {
1931
2375
  if (this.disposed) {
1932
2376
  throw new Error("This Reactor was disposed and can't be used again \u2014 construct a new one.");
@@ -1934,6 +2378,25 @@ var Reactor = class {
1934
2378
  }
1935
2379
  };
1936
2380
 
2381
+ // src/step-frames.ts
2382
+ function stepFrameToImageData(frame) {
2383
+ const [height, width, channels] = frame.shape;
2384
+ if (frame.dtype !== "uint8" || frame.shape.length !== 3 || channels !== 3 || height === void 0 || width === void 0 || frame.data.length !== width * height * 3) {
2385
+ throw new Error(
2386
+ `stepFrameToImageData: expected a (height, width, 3) uint8 RGB frame, got ${frame.dtype} [${frame.shape.join(", ")}]`
2387
+ );
2388
+ }
2389
+ const rgb = frame.data;
2390
+ const rgba = new Uint8ClampedArray(width * height * 4);
2391
+ for (let src = 0, dst = 0; src < rgb.length; src += 3, dst += 4) {
2392
+ rgba[dst] = rgb[src];
2393
+ rgba[dst + 1] = rgb[src + 1];
2394
+ rgba[dst + 2] = rgb[src + 2];
2395
+ rgba[dst + 3] = 255;
2396
+ }
2397
+ return new ImageData(rgba, width, height);
2398
+ }
2399
+
1937
2400
  // src/jwt.ts
1938
2401
  function normalizeJwtSource(source) {
1939
2402
  if (typeof source === "function") {
@@ -2884,9 +3347,11 @@ function defaultLabel(state) {
2884
3347
  ClipPlayer,
2885
3348
  ConflictError,
2886
3349
  DEFAULT_PLAYLIST_POLL_SLACK_MS,
3350
+ DataChannelChunkingRequiredError,
2887
3351
  DecodeError,
2888
3352
  DisconnectedError,
2889
3353
  FileRef,
3354
+ GenerationModeMismatchError,
2890
3355
  InvalidStateError,
2891
3356
  MessageTooLargeError,
2892
3357
  NetworkError,
@@ -2901,8 +3366,14 @@ function defaultLabel(state) {
2901
3366
  RequestTimeoutError,
2902
3367
  ServerError,
2903
3368
  SessionTerminalError,
3369
+ StepCommandFailedError,
3370
+ StepNotReadyError,
3371
+ StepRefusedError,
3372
+ StepResultTooLargeError,
3373
+ StepSessionEndedError,
2904
3374
  TransportError,
2905
3375
  UnauthorizedError,
3376
+ UnsupportedFrameEncodingError,
2906
3377
  VersionMismatchError,
2907
3378
  WebcamStream,
2908
3379
  createPlayableManifestUrl,
@@ -2911,6 +3382,7 @@ function defaultLabel(state) {
2911
3382
  isFileRef,
2912
3383
  normalizeJwtSource,
2913
3384
  parsePlaylist,
3385
+ stepFrameToImageData,
2914
3386
  useClipDownload,
2915
3387
  useReactor,
2916
3388
  useReactorInternalMessage,