@reactor-team/js-sdk 3.1.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -44,6 +44,14 @@ var ReactorError = class extends Error {
44
44
  operation;
45
45
  /** Backoff hint the platform sent. */
46
46
  retry_after_ms;
47
+ /** For a refused step or a step whose call failed, the part of the step
48
+ * request at fault: `"state.<key>"`, `"commands[<i>]"` or
49
+ * `"media_input[<i>]"`. */
50
+ entry;
51
+ /** For a step whose call failed, the stable code its handler raised (its
52
+ * `CommandError` code, or `internal_error`), to branch on instead of the
53
+ * message. */
54
+ command_code;
47
55
  /** When `reactor-core` reported this failure. For a purely local one (no
48
56
  * payload at all — see {@link ReactorErrorOptions}), this is synthesized
49
57
  * as the construction time instead, since there's nothing to report —
@@ -56,7 +64,7 @@ var ReactorError = class extends Error {
56
64
  * compatibility — prefer `timestamp_ms`. */
57
65
  timestamp;
58
66
  constructor(message, options = {}) {
59
- super(message);
67
+ super(message, options.cause === void 0 ? void 0 : { cause: options.cause });
60
68
  this.name = "ReactorError";
61
69
  this.code = options.code ?? new.target.code;
62
70
  this.recoverable = options.recoverable ?? false;
@@ -64,6 +72,8 @@ var ReactorError = class extends Error {
64
72
  this.operation = options.operation;
65
73
  this.retry_after_ms = options.retry_after_ms;
66
74
  this.retryAfter = options.retry_after_ms;
75
+ this.entry = options.entry;
76
+ this.command_code = options.command_code;
67
77
  this.timestamp_ms = options.timestamp_ms ?? Date.now();
68
78
  this.timestamp = this.timestamp_ms;
69
79
  }
@@ -119,6 +129,30 @@ var AbortedError = class extends ReactorError {
119
129
  var RecorderDisabledError = class extends ReactorError {
120
130
  static code = "RECORDER_DISABLED";
121
131
  };
132
+ var GenerationModeMismatchError = class extends ReactorError {
133
+ static code = "GENERATION_MODE_MISMATCH";
134
+ };
135
+ var DataChannelChunkingRequiredError = class extends ReactorError {
136
+ static code = "DATA_CHANNEL_CHUNKING_REQUIRED";
137
+ };
138
+ var StepRefusedError = class extends ReactorError {
139
+ static code = "step_refused";
140
+ };
141
+ var StepNotReadyError = class extends ReactorError {
142
+ static code = "step_not_ready";
143
+ };
144
+ var StepResultTooLargeError = class extends ReactorError {
145
+ static code = "step_result_too_large";
146
+ };
147
+ var StepCommandFailedError = class extends ReactorError {
148
+ static code = "step_command_failed";
149
+ };
150
+ var StepSessionEndedError = class extends ReactorError {
151
+ static code = "step_session_ended";
152
+ };
153
+ var UnsupportedFrameEncodingError = class extends ReactorError {
154
+ static code = "UNSUPPORTED_FRAME_ENCODING";
155
+ };
122
156
  var ERROR_CLASSES = [
123
157
  NetworkError,
124
158
  UnauthorizedError,
@@ -136,7 +170,15 @@ var ERROR_CLASSES = [
136
170
  DisconnectedError,
137
171
  RequestTimeoutError,
138
172
  AbortedError,
139
- RecorderDisabledError
173
+ RecorderDisabledError,
174
+ GenerationModeMismatchError,
175
+ DataChannelChunkingRequiredError,
176
+ StepRefusedError,
177
+ StepNotReadyError,
178
+ StepResultTooLargeError,
179
+ StepCommandFailedError,
180
+ StepSessionEndedError,
181
+ UnsupportedFrameEncodingError
140
182
  ];
141
183
  var BY_CODE = new Map(
142
184
  ERROR_CLASSES.map((errorClass) => [errorClass.code, errorClass])
@@ -157,7 +199,12 @@ function toReactorError(cause) {
157
199
  status: payload.status,
158
200
  operation: payload.operation,
159
201
  retry_after_ms: payload.retry_after_ms,
160
- timestamp_ms: payload.timestamp_ms
202
+ timestamp_ms: payload.timestamp_ms,
203
+ entry: payload.entry,
204
+ command_code: payload.command_code,
205
+ // A thrown JS error (a failed wasm load, say) keeps its stack and its own
206
+ // `cause` reachable; reactor-core's plain payload objects have neither.
207
+ cause: cause instanceof Error ? cause : void 0
161
208
  });
162
209
  }
163
210
 
@@ -292,6 +339,112 @@ function isSerializedAsObject(value) {
292
339
  return typeof value === "object" && value !== null && !ArrayBuffer.isView(value);
293
340
  }
294
341
 
342
+ // src/internal/push-frames.ts
343
+ function isAudioFrame(frame) {
344
+ return "samples" in frame;
345
+ }
346
+ function isVideoFrame(frame) {
347
+ return typeof frame.copyTo === "function" && "displayWidth" in frame;
348
+ }
349
+ function holdFrame(frame) {
350
+ if (!isAudioFrame(frame) && isVideoFrame(frame)) {
351
+ const clone = frame.clone();
352
+ return { frame: clone, release: () => clone.close() };
353
+ }
354
+ return { frame, release: () => {
355
+ } };
356
+ }
357
+ async function toPixels(frame) {
358
+ if (isVideoFrame(frame)) {
359
+ return videoFramePixels(frame);
360
+ }
361
+ return rawPixels(frame);
362
+ }
363
+ function rawPixels(frame) {
364
+ const { data, width, height } = frame;
365
+ const bytes = data instanceof Uint8Array ? data : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
366
+ return { data: bytes, width, height, format: "format" in frame && frame.format || "rgba" };
367
+ }
368
+ var NATIVE_FORMATS = {
369
+ RGBA: "rgba",
370
+ RGBX: "rgba",
371
+ BGRA: "bgra",
372
+ BGRX: "bgra"
373
+ };
374
+ function orientationOf(frame) {
375
+ const { rotation, flip } = frame;
376
+ return { rotation: ((rotation ?? 0) % 360 + 360) % 360, flip: flip ?? false };
377
+ }
378
+ async function videoFramePixels(frame) {
379
+ const rect = frame.visibleRect;
380
+ const width = rect?.width ?? frame.codedWidth;
381
+ const height = rect?.height ?? frame.codedHeight;
382
+ const native = frame.format ? NATIVE_FORMATS[frame.format] : void 0;
383
+ const orientation = orientationOf(frame);
384
+ if (native) {
385
+ const data = new Uint8Array(width * height * 4);
386
+ await frame.copyTo(data, { layout: [{ offset: 0, stride: width * 4 }] });
387
+ return orient({ data, width, height, format: native }, orientation);
388
+ }
389
+ try {
390
+ const data = new Uint8Array(width * height * 4);
391
+ const options = { format: "RGBA", layout: [{ offset: 0, stride: width * 4 }] };
392
+ await frame.copyTo(data, options);
393
+ return orient({ data, width, height, format: "rgba" }, orientation);
394
+ } catch {
395
+ return canvasPixels(frame, width, height, orientation);
396
+ }
397
+ }
398
+ function orient(pixels, { rotation, flip }) {
399
+ if (rotation % 90 !== 0 || rotation === 0 && !flip) {
400
+ return pixels;
401
+ }
402
+ const { width: w, height: h } = pixels;
403
+ const turned = rotation === 90 || rotation === 270;
404
+ const outWidth = turned ? h : w;
405
+ const outHeight = turned ? w : h;
406
+ const bytes = pixels.data.byteOffset % 4 === 0 ? pixels.data : pixels.data.slice();
407
+ const source = new Uint32Array(bytes.buffer, bytes.byteOffset, w * h);
408
+ const data = new Uint8Array(outWidth * outHeight * 4);
409
+ const target = new Uint32Array(data.buffer);
410
+ for (let y = 0; y < h; y++) {
411
+ for (let x = 0; x < w; x++) {
412
+ let tx;
413
+ let ty;
414
+ if (rotation === 90) {
415
+ [tx, ty] = [h - 1 - y, x];
416
+ } else if (rotation === 180) {
417
+ [tx, ty] = [w - 1 - x, h - 1 - y];
418
+ } else if (rotation === 270) {
419
+ [tx, ty] = [y, w - 1 - x];
420
+ } else {
421
+ [tx, ty] = [x, y];
422
+ }
423
+ if (flip) {
424
+ tx = outWidth - 1 - tx;
425
+ }
426
+ target[ty * outWidth + tx] = source[y * w + x];
427
+ }
428
+ }
429
+ return { data, width: outWidth, height: outHeight, format: pixels.format };
430
+ }
431
+ function canvasPixels(frame, visibleWidth, visibleHeight, { rotation }) {
432
+ const turned = rotation === 90 || rotation === 270;
433
+ const width = turned ? visibleHeight : visibleWidth;
434
+ const height = turned ? visibleWidth : visibleHeight;
435
+ if (typeof OffscreenCanvas === "undefined") {
436
+ throw new Error(
437
+ `pushFrame: cannot read a ${frame.format ?? "formatless"} VideoFrame here \u2014 push its pixels as an ImageData or a Uint8Array instead.`
438
+ );
439
+ }
440
+ const context = new OffscreenCanvas(width, height).getContext("2d");
441
+ if (!context) {
442
+ throw new Error("pushFrame: no 2D canvas context to read the VideoFrame through.");
443
+ }
444
+ context.drawImage(frame, 0, 0, width, height);
445
+ return rawPixels(context.getImageData(0, 0, width, height));
446
+ }
447
+
295
448
  // src/internal/recording.ts
296
449
  function toPublicClip(clip) {
297
450
  return {
@@ -354,7 +507,28 @@ var WIRE_CODECS = {
354
507
  };
355
508
  function createTrackStatsExtractor() {
356
509
  const lastBytes = /* @__PURE__ */ new Map();
357
- return (report, tracks, pausedTracks) => {
510
+ const lastStages = /* @__PURE__ */ new Map();
511
+ let pendingBytes = /* @__PURE__ */ new Map();
512
+ let pendingStages = /* @__PURE__ */ new Map();
513
+ const stage = (streamId, name, seconds, frames) => {
514
+ if (seconds === void 0 || frames === void 0) {
515
+ return void 0;
516
+ }
517
+ const key = `${streamId}:${name}`;
518
+ if (!Number.isFinite(seconds)) {
519
+ pendingStages.set(key, void 0);
520
+ return void 0;
521
+ }
522
+ pendingStages.set(key, { seconds, frames });
523
+ const before = lastStages.get(key);
524
+ if (before === void 0 || frames <= before.frames || seconds < before.seconds) {
525
+ return void 0;
526
+ }
527
+ return { name, totalMs: (seconds - before.seconds) * 1e3, frames: frames - before.frames };
528
+ };
529
+ const extract = (report, tracks, pausedTracks) => {
530
+ pendingBytes = /* @__PURE__ */ new Map();
531
+ pendingStages = /* @__PURE__ */ new Map();
358
532
  const reportWithLookup = report;
359
533
  const stats = [];
360
534
  const seenStreams = /* @__PURE__ */ new Set();
@@ -393,7 +567,7 @@ function createTrackStatsExtractor() {
393
567
  if (previous !== void 0 && stat.timestamp > previous.timestamp && bytes >= previous.bytes) {
394
568
  put("bitrate_bps", Math.round((bytes - previous.bytes) * 8 * 1e3 / (stat.timestamp - previous.timestamp)));
395
569
  }
396
- lastBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
570
+ pendingBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
397
571
  }
398
572
  const keyframeRequests = stat.firCount !== void 0 || stat.pliCount !== void 0 ? (stat.firCount ?? 0) + (stat.pliCount ?? 0) : void 0;
399
573
  if (inbound) {
@@ -428,6 +602,20 @@ function createTrackStatsExtractor() {
428
602
  put("keyframe_requests", keyframeRequests);
429
603
  }
430
604
  }
605
+ const frameStages = [];
606
+ if (track.kind === "video") {
607
+ const stages = inbound ? [
608
+ stage(stat.id, "jitter_buffer", stat.jitterBufferDelay, stat.jitterBufferEmittedCount),
609
+ stage(stat.id, "decode", stat.totalDecodeTime, stat.framesDecoded)
610
+ ] : [stage(stat.id, "encode", stat.totalEncodeTime, stat.framesEncoded)];
611
+ for (const s of stages) {
612
+ if (s !== void 0) {
613
+ frameStages.push(s);
614
+ }
615
+ }
616
+ }
617
+ const codecImplementation = inbound ? stat.decoderImplementation : stat.encoderImplementation;
618
+ const hardwareCodec = inbound ? stat.powerEfficientDecoder : stat.powerEfficientEncoder;
431
619
  stats.push({
432
620
  timestamp: Date.now(),
433
621
  trackName: track.name,
@@ -435,11 +623,30 @@ function createTrackStatsExtractor() {
435
623
  direction,
436
624
  codec,
437
625
  paused: pausedTracks.includes(track.name),
438
- metrics
626
+ metrics,
627
+ ...frameStages.length > 0 ? { frameStages } : {},
628
+ ...track.kind === "video" && codecImplementation !== void 0 ? { codecImplementation } : {},
629
+ ...track.kind === "video" && hardwareCodec !== void 0 ? { hardwareCodec } : {}
439
630
  });
440
631
  });
441
632
  return stats;
442
633
  };
634
+ return Object.assign(extract, {
635
+ commit() {
636
+ for (const [id, sample] of pendingBytes) {
637
+ lastBytes.set(id, sample);
638
+ }
639
+ for (const [key, total] of pendingStages) {
640
+ if (total === void 0) {
641
+ lastStages.delete(key);
642
+ } else {
643
+ lastStages.set(key, total);
644
+ }
645
+ }
646
+ pendingBytes = /* @__PURE__ */ new Map();
647
+ pendingStages = /* @__PURE__ */ new Map();
648
+ }
649
+ });
443
650
  }
444
651
  function toClientConnectionStat(report, timeToConnectMs) {
445
652
  const pair = activeCandidatePair(report);
@@ -558,27 +765,27 @@ function loadReactorWasm() {
558
765
  if (!modulePromise) {
559
766
  modulePromise = importWasmModule().catch((cause) => {
560
767
  modulePromise = null;
561
- throw new Error(
562
- "reactor-wasm failed to load. Run `mise run build:wasm` (or, from a published install, reinstall the package) and try again.",
563
- { cause }
564
- );
768
+ throw new Error(`reactor-wasm failed to load: ${describeCause(cause)}`, { cause });
565
769
  });
566
770
  }
567
771
  return modulePromise;
568
772
  }
569
773
  async function importWasmModule() {
570
- const module = await import(
571
- /* @vite-ignore */
572
- "./wasm/reactor_wasm.js"
573
- );
774
+ const module = await import("./wasm/reactor_wasm.js");
574
775
  await module.default();
575
776
  return module;
576
777
  }
778
+ function describeCause(cause) {
779
+ if (cause instanceof Error) {
780
+ return cause.message ? `${cause.name}: ${cause.message}` : cause.name;
781
+ }
782
+ return String(cause);
783
+ }
577
784
 
578
785
  // package.json
579
786
  var package_default = {
580
787
  name: "@reactor-team/js-sdk",
581
- version: "3.1.0",
788
+ version: "3.3.0",
582
789
  description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
583
790
  license: "Apache-2.0",
584
791
  repository: {
@@ -638,6 +845,11 @@ var package_default = {
638
845
  }
639
846
  };
640
847
 
848
+ // src/internal/jwt-resolver.ts
849
+ async function resolveJwtSource(source, context) {
850
+ return typeof source === "function" ? source(context) : source;
851
+ }
852
+
641
853
  // src/recording.ts
642
854
  var RecordingError = class extends Error {
643
855
  constructor(code, reason) {
@@ -657,15 +869,16 @@ async function fetchPlaylist(playlistUrl, options = {}) {
657
869
  const startedPollingAt = Date.now();
658
870
  const deadlineMs = hasDeadline ? Math.max(options.predictedReadyAtMs ?? startedPollingAt, startedPollingAt) + options.slackMs : void 0;
659
871
  const maxRetries = typeof options.maxRetries === "number" ? options.maxRetries : void 0;
660
- const init = {};
661
- if (options.signal) {
662
- init.signal = options.signal;
663
- }
664
- if (options.jwt) {
665
- init.headers = { Authorization: `Bearer ${options.jwt}` };
666
- }
667
872
  let attempt = 0;
668
873
  while (true) {
874
+ const init = {};
875
+ if (options.signal) {
876
+ init.signal = options.signal;
877
+ }
878
+ const jwt = await resolveManifestJwt(options);
879
+ if (jwt) {
880
+ init.headers = { Authorization: `Bearer ${jwt}` };
881
+ }
669
882
  let response;
670
883
  try {
671
884
  response = await fetch(playlistUrl, init);
@@ -768,7 +981,10 @@ function absolutizeManifestUrls(manifestBody, playlistUrl) {
768
981
  return out.join(eol);
769
982
  }
770
983
  async function downloadClipAsFile(clip, filename = "reactor-clip.mp4", options = {}) {
771
- const playlistOptions = { predictedReadyAtMs: clip.predictedReadyAtMs };
984
+ const playlistOptions = {
985
+ predictedReadyAtMs: clip.predictedReadyAtMs,
986
+ sessionId: clip.sessionId
987
+ };
772
988
  if (options.signal) {
773
989
  playlistOptions.signal = options.signal;
774
990
  }
@@ -1006,6 +1222,27 @@ function parseRetryAfter(header, fallbackMs) {
1006
1222
  }
1007
1223
  return fallbackMs;
1008
1224
  }
1225
+ async function resolveManifestJwt(options) {
1226
+ if (options.jwt === void 0) {
1227
+ return void 0;
1228
+ }
1229
+ const context = options.sessionId === void 0 ? {} : { sessionId: options.sessionId };
1230
+ return abortable(resolveJwtSource(options.jwt, context), options.signal);
1231
+ }
1232
+ function abortable(promise, signal) {
1233
+ if (!signal) {
1234
+ return promise;
1235
+ }
1236
+ return new Promise((resolve, reject) => {
1237
+ const onAbort = () => reject(new DOMException("Aborted", "AbortError"));
1238
+ if (signal.aborted) {
1239
+ onAbort();
1240
+ } else {
1241
+ signal.addEventListener("abort", onAbort, { once: true });
1242
+ }
1243
+ void promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort));
1244
+ });
1245
+ }
1009
1246
  function sleep(ms, signal) {
1010
1247
  return new Promise((resolve, reject) => {
1011
1248
  if (signal?.aborted) {
@@ -1062,10 +1299,18 @@ var Reactor = class {
1062
1299
  * with it, so a batch sent before the timings exist doesn't use it up.
1063
1300
  * Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
1064
1301
  hasSentConnectTime = false;
1302
+ /** Whether this connection has sent a client-stats batch yet. The first
1303
+ * one goes out even with no reading, for the environment the bridge adds
1304
+ * to it. Reset with `hasSentConnectTime`. */
1305
+ hasSentClientStats = false;
1065
1306
  /** Set on the "connecting" status transition, cleared once `connectionTimings`
1066
1307
  * is finalized on "ready" — see `handleStatusChanged()`. */
1067
1308
  connectStartTime;
1068
1309
  waitingStartTime;
1310
+ /** The tail of every `pushFrame()` so far, settled either way — pushes
1311
+ * reach the binding in call order however long each one's pixels take to
1312
+ * read, and `step()` waits for the ones called before it. */
1313
+ pendingPushes = Promise.resolve();
1069
1314
  emitter = new Emitter();
1070
1315
  /** Serializes connect()/reconnect()/disconnect() (and the free() inside
1071
1316
  * disconnect()/[Symbol.dispose]) against each other. Calling into the
@@ -1083,6 +1328,15 @@ var Reactor = class {
1083
1328
  * out every read active when it started, before it releases the ones
1084
1329
  * that arrived while it ran. */
1085
1330
  writerActive = false;
1331
+ /** Counts `disconnect()`s and disposal. A push captures it when called, and
1332
+ * one whose frame was still being read when it moved is dropped, rather
1333
+ * than recreating a client to carry the frame into the next session. */
1334
+ connectionEpoch = 0;
1335
+ /** Input tracks published on this connection, each with the publication
1336
+ * that claimed it: a push needs one, and a `PublishedTrack` only feeds the
1337
+ * publication it came from. Cleared with the connection. */
1338
+ publishedInputs = /* @__PURE__ */ new Map();
1339
+ publications = 0;
1086
1340
  /** Readers parked in `withReaderAccess()`'s wait loop, woken once
1087
1341
  * `withWriterAccess()` finishes; each re-checks `writerActive` rather
1088
1342
  * than assuming its turn, since another writer already queued behind the
@@ -1355,14 +1609,49 @@ var Reactor = class {
1355
1609
  * disconnect() freeing the wasm client mid-await would otherwise resume
1356
1610
  * into a freed client — the same class of use-after-free `queue` exists to
1357
1611
  * prevent for connect/disconnect/reconnect themselves.
1612
+ *
1613
+ * Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
1614
+ * track in a stepped session.
1615
+ *
1616
+ * In a `generation: "stepped"` session, input frames never travel over
1617
+ * RTP: `track` is optional, and the frames are pushed with
1618
+ * `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
1619
+ * passed there is ignored, with a console warning — nothing is sent from
1620
+ * it. In a streamed session `track` is required, and omitting it rejects
1621
+ * with an `InvalidStateError`.
1622
+ *
1623
+ * ```ts
1624
+ * const camera = await reactor.publishTrack('camera');
1625
+ * await camera.pushFrame(imageData);
1626
+ * const result = await reactor.step();
1627
+ * ```
1358
1628
  */
1359
1629
  async publishTrack(name, track) {
1360
1630
  this.assertNotDisposed();
1361
1631
  try {
1632
+ let media = track;
1633
+ if (this.isStepped() && media !== void 0) {
1634
+ console.warn(
1635
+ `[Reactor] publishTrack('${name}'): ignoring the MediaStreamTrack \u2014 a stepped session sends no RTP and takes frames through track.pushFrame(...) on the track publishTrack() returns.`
1636
+ );
1637
+ media = void 0;
1638
+ }
1639
+ if (!this.isStepped() && media === void 0) {
1640
+ throw new InvalidStateError(
1641
+ `publishTrack('${name}') needs a MediaStreamTrack in a streamed session.`,
1642
+ { operation: "publishTrack" }
1643
+ );
1644
+ }
1645
+ const publication = this.publications += 1;
1362
1646
  await this.queue.push(async () => {
1363
1647
  const client = await this.getOrCreateClient();
1364
- await client.publishTrack(name, track);
1648
+ await client.publishTrack(name, media);
1649
+ this.publishedInputs.set(name, publication);
1365
1650
  }, "publishTrack");
1651
+ return {
1652
+ name,
1653
+ pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
1654
+ };
1366
1655
  } catch (cause) {
1367
1656
  throw this.captureError(cause);
1368
1657
  }
@@ -1379,6 +1668,7 @@ var Reactor = class {
1379
1668
  await this.queue.push(async () => {
1380
1669
  const client = await this.getOrCreateClient();
1381
1670
  await client.unpublishTrack(name);
1671
+ this.publishedInputs.delete(name);
1382
1672
  }, "unpublishTrack");
1383
1673
  } catch (cause) {
1384
1674
  this.emitError(cause);
@@ -1451,6 +1741,175 @@ var Reactor = class {
1451
1741
  throw this.captureError(cause);
1452
1742
  }
1453
1743
  }
1744
+ // ── Stepped generation ──────────────────────────────────────────────────
1745
+ /**
1746
+ * Runs one step of a `generation: "stepped"` session and resolves with
1747
+ * what it produced, per output track.
1748
+ *
1749
+ * The request carries `state`, `commands` and every frame pushed since the
1750
+ * previous step. The runtime applies them in that order — each `state` key
1751
+ * as the model's `set_<key>` command, then `commands`, then the frames —
1752
+ * and then generates. They are separate calls, not a transaction: one that
1753
+ * fails does not undo the ones before it. All of it is optional:
1754
+ * `step()` with nothing set and nothing pushed runs on the model's current
1755
+ * state, which commands sent outside a step (`sendCommand()`) also change.
1756
+ * A file goes in the way it does for a command: upload it with
1757
+ * `uploadFile()`, then put the `FileRef` in a command's `data` or a state
1758
+ * value.
1759
+ *
1760
+ * A step that ran but that the model reported as failed still resolves,
1761
+ * with `result.error` set. `result.files` lists the files the model kept
1762
+ * with the step (name, content type, size); their bytes are served over
1763
+ * HTTP from the session's step results, not carried here.
1764
+ *
1765
+ * Rejects with a `StepRefusedError` when the runtime or the model turns
1766
+ * the step down (the runtime checks a whole request first, so nothing it
1767
+ * carried was applied), `StepNotReadyError` when the session cannot take a
1768
+ * step yet (neither ran the step), `StepCommandFailedError` when a state key
1769
+ * or command failed (the calls before it stay applied; the step did not
1770
+ * run); both name the part at fault in `entry`, and `StepCommandFailedError`
1771
+ * the handler's code in `command_code`,
1772
+ * `StepSessionEndedError` when the session ended before the result came
1773
+ * back, `StepResultTooLargeError` when the result does not fit in one
1774
+ * message, `MessageTooLargeError` when the request itself is too large
1775
+ * (nothing was sent), `InvalidStateError` outside a ready stepped
1776
+ * session, or whatever else failed the step (`DisconnectedError`,
1777
+ * `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
1778
+ * when its request cannot be sent, they are dropped with it rather than
1779
+ * kept for the next step.
1780
+ *
1781
+ * ```ts
1782
+ * const camera = await reactor.publishTrack('camera');
1783
+ * await camera.pushFrame(imageData);
1784
+ * const result = await reactor.step({ state: { prompt: 'a cat' } });
1785
+ * if (result.error) console.warn('step failed:', result.error);
1786
+ * const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
1787
+ * if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
1788
+ * ```
1789
+ */
1790
+ async step(options = {}) {
1791
+ this.assertNotDisposed();
1792
+ try {
1793
+ const state = rewriteNestedFileRefs(options.state ?? null);
1794
+ const commands = (options.commands ?? []).map(
1795
+ (command) => command.data ? { ...command, data: rewriteNestedFileRefs(command.data) } : command
1796
+ );
1797
+ await this.pendingPushes;
1798
+ return await this.withReaderAccess(async () => {
1799
+ const client = await this.getOrCreateClient();
1800
+ return client.step(state, commands);
1801
+ });
1802
+ } catch (cause) {
1803
+ throw this.captureError(cause);
1804
+ }
1805
+ }
1806
+ /**
1807
+ * Buffers one input frame for the next `step()` on the sendonly track
1808
+ * `trackName`. Nothing is sent until that step, which carries every frame
1809
+ * pushed since the one before it, per track in push order. The
1810
+ * `PublishedTrack` that `publishTrack()` resolves with does the same
1811
+ * through `track.pushFrame(frame)`.
1812
+ *
1813
+ * Video: a `VideoFrame` (cloned when called, so the caller may close it as
1814
+ * soon as this returns; it stays the caller's), an
1815
+ * `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
1816
+ * by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
1817
+ * `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
1818
+ * applied, so a 90° frame arrives with its width and height swapped.
1819
+ * Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
1820
+ * PCM at the rate the model's track declares.
1821
+ *
1822
+ * Pushes keep their call order, and a `step()` waits for every push called
1823
+ * before it. A refused push rejects: await it, or `.catch()` it in a frame
1824
+ * loop, or every bad frame is an unhandled rejection. What waits for
1825
+ * the next step is at most one message's worth of frames: pushing faster
1826
+ * than steps run drops the oldest frames, across tracks. Rejects with
1827
+ * an `InvalidStateError` outside a stepped session, for a track not
1828
+ * published on this connection (`publishTrack(trackName)` first), for a
1829
+ * frame whose size does not match its dimensions, or for a name the model
1830
+ * does not declare as a sendonly track of that kind.
1831
+ */
1832
+ async pushFrame(trackName, frame, options = {}) {
1833
+ return this.pushTo(trackName, void 0, frame, options);
1834
+ }
1835
+ /** `pushFrame()`, for the track's current publication or, from a
1836
+ * `PublishedTrack`, only for the `publication` it came from. */
1837
+ async pushTo(trackName, publication, frame, options = {}) {
1838
+ this.assertNotDisposed();
1839
+ const epoch = this.connectionEpoch;
1840
+ let held;
1841
+ try {
1842
+ held = holdFrame(frame);
1843
+ } catch (cause) {
1844
+ throw this.captureError(cause);
1845
+ }
1846
+ const push = this.pendingPushes.then(async () => {
1847
+ let pixels;
1848
+ try {
1849
+ if (!this.isStepped()) {
1850
+ throw new InvalidStateError(
1851
+ `pushFrame('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
1852
+ { operation: "pushFrame" }
1853
+ );
1854
+ }
1855
+ pixels = isAudioFrame(held.frame) ? void 0 : await toPixels(held.frame);
1856
+ } finally {
1857
+ held.release();
1858
+ }
1859
+ await this.withReaderAccess(async () => {
1860
+ if (epoch !== this.connectionEpoch) {
1861
+ throw new InvalidStateError(
1862
+ `pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
1863
+ { operation: "pushFrame" }
1864
+ );
1865
+ }
1866
+ const current = this.publishedInputs.get(trackName);
1867
+ if (current === void 0 || publication !== void 0 && publication !== current) {
1868
+ throw new InvalidStateError(
1869
+ publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
1870
+ { operation: "pushFrame" }
1871
+ );
1872
+ }
1873
+ const client = await this.getOrCreateClient();
1874
+ if (pixels) {
1875
+ client.pushVideoFrame(
1876
+ trackName,
1877
+ pixels.data,
1878
+ pixels.width,
1879
+ pixels.height,
1880
+ pixels.format,
1881
+ options.metadata ?? null,
1882
+ options.captureTimeUs ?? null
1883
+ );
1884
+ } else if (isAudioFrame(frame)) {
1885
+ client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
1886
+ }
1887
+ });
1888
+ });
1889
+ this.pendingPushes = push.catch(() => void 0);
1890
+ try {
1891
+ await push;
1892
+ } catch (cause) {
1893
+ throw this.captureError(cause);
1894
+ }
1895
+ }
1896
+ /** Drops every frame buffered for the next step, pushes called before it
1897
+ * included: it waits its turn behind them, as a `step()` does. */
1898
+ async clearStepInputs() {
1899
+ this.assertNotDisposed();
1900
+ const clear = this.pendingPushes.then(
1901
+ () => this.withReaderAccess(() => {
1902
+ this.client?.clearStepInputs();
1903
+ return Promise.resolve();
1904
+ })
1905
+ );
1906
+ this.pendingPushes = clear.catch(() => void 0);
1907
+ try {
1908
+ await clear;
1909
+ } catch (cause) {
1910
+ throw this.captureError(cause);
1911
+ }
1912
+ }
1454
1913
  // ── Recording ───────────────────────────────────────────────────────────
1455
1914
  /**
1456
1915
  * Requests a clip covering the last `durationSeconds` of the session.
@@ -1749,6 +2208,7 @@ var Reactor = class {
1749
2208
  const extractConnectionStats = createConnectionStatsExtractor();
1750
2209
  const extractTrackStats = createTrackStatsExtractor();
1751
2210
  this.hasSentConnectTime = false;
2211
+ this.hasSentClientStats = false;
1752
2212
  this.stopStatsPollers = [
1753
2213
  pollPeerStats(client, LOCAL_STATS_INTERVAL_MS, (report) => {
1754
2214
  this.stats = { ...extractConnectionStats(report), connectionTimings: this.connectionTimings };
@@ -1766,8 +2226,9 @@ var Reactor = class {
1766
2226
  * track has one yet (none negotiated onto a named track, or no codec
1767
2227
  * reported yet), so the runtime gets the bandwidth estimate, the
1768
2228
  * round-trip time and the time to connect without waiting on the tracks.
1769
- * Skipped only when there is nothing at all to say; one is due again in
1770
- * `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
2229
+ * Skipped only when there is nothing at all to say and a batch has already
2230
+ * gone out for this connection, so the first one still carries the
2231
+ * environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
1771
2232
  * actually sent for this connection — see `hasSentConnectTime`.
1772
2233
  * `sendClientStats` throws synchronously on a closing connection; caught
1773
2234
  * here for the same reason `getStats()`'s rejection is above. */
@@ -1775,11 +2236,13 @@ var Reactor = class {
1775
2236
  const trackStats = extractTrackStats(report, this.trackMapping(), this.pausedTracks());
1776
2237
  const timeToConnectMs = this.hasSentConnectTime ? void 0 : this.connectionTimings?.totalMs;
1777
2238
  const connectionStat = toClientConnectionStat(report, timeToConnectMs);
1778
- if (trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
2239
+ if (this.hasSentClientStats && trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
1779
2240
  return;
1780
2241
  }
1781
2242
  try {
1782
2243
  client.sendClientStats(trackStats, connectionStat);
2244
+ extractTrackStats.commit();
2245
+ this.hasSentClientStats = true;
1783
2246
  if (timeToConnectMs !== void 0) {
1784
2247
  this.hasSentConnectTime = true;
1785
2248
  }
@@ -1801,6 +2264,8 @@ var Reactor = class {
1801
2264
  * leave `getCapabilities()` returning the previous session's stale
1802
2265
  * tracks/commands until a new `capabilitiesReceived` lands. */
1803
2266
  resetConnectionState() {
2267
+ this.connectionEpoch += 1;
2268
+ this.publishedInputs.clear();
1804
2269
  this.stopStatsPolling();
1805
2270
  this.connectionTimings = void 0;
1806
2271
  this.connectStartTime = void 0;
@@ -1821,6 +2286,9 @@ var Reactor = class {
1821
2286
  emitError(cause) {
1822
2287
  this.emitter.emit("error", this.captureError(cause));
1823
2288
  }
2289
+ isStepped() {
2290
+ return this.clientOptions.generation === "stepped";
2291
+ }
1824
2292
  assertNotDisposed() {
1825
2293
  if (this.disposed) {
1826
2294
  throw new Error("This Reactor was disposed and can't be used again \u2014 construct a new one.");
@@ -1828,6 +2296,25 @@ var Reactor = class {
1828
2296
  }
1829
2297
  };
1830
2298
 
2299
+ // src/step-frames.ts
2300
+ function stepFrameToImageData(frame) {
2301
+ const [height, width, channels] = frame.shape;
2302
+ if (frame.dtype !== "uint8" || frame.shape.length !== 3 || channels !== 3 || height === void 0 || width === void 0 || frame.data.length !== width * height * 3) {
2303
+ throw new Error(
2304
+ `stepFrameToImageData: expected a (height, width, 3) uint8 RGB frame, got ${frame.dtype} [${frame.shape.join(", ")}]`
2305
+ );
2306
+ }
2307
+ const rgb = frame.data;
2308
+ const rgba = new Uint8ClampedArray(width * height * 4);
2309
+ for (let src = 0, dst = 0; src < rgb.length; src += 3, dst += 4) {
2310
+ rgba[dst] = rgb[src];
2311
+ rgba[dst + 1] = rgb[src + 1];
2312
+ rgba[dst + 2] = rgb[src + 2];
2313
+ rgba[dst + 3] = 255;
2314
+ }
2315
+ return new ImageData(rgba, width, height);
2316
+ }
2317
+
1831
2318
  // src/jwt.ts
1832
2319
  function normalizeJwtSource(source) {
1833
2320
  if (typeof source === "function") {
@@ -2470,11 +2957,6 @@ function describeMediaError(error) {
2470
2957
  return error?.message ? `${message} (${error.message})` : message;
2471
2958
  }
2472
2959
 
2473
- // src/internal/jwt-resolver.ts
2474
- async function resolveJwtSource(source) {
2475
- return typeof source === "function" ? source() : source;
2476
- }
2477
-
2478
2960
  // src/react/ClipPlayer.tsx
2479
2961
  import { jsx as jsx4, jsxs as jsxs2 } from "react/jsx-runtime";
2480
2962
  function ClipPlayer({
@@ -2524,21 +3006,17 @@ function ClipPlayer({
2524
3006
  const setup = async () => {
2525
3007
  try {
2526
3008
  setPhase({ kind: "waiting" });
2527
- const explicit = getJwtRef.current;
2528
- const fallback = storeRef.current?.getState().internal.reactor.getJwtResolver();
2529
- const jwt = explicit ? await explicit() : fallback !== void 0 ? await resolveJwtSource(fallback) : void 0;
2530
- if (cancelled) {
2531
- return;
2532
- }
3009
+ const source = () => getJwtRef.current ?? storeRef.current?.getState().internal.reactor.getJwtResolver();
2533
3010
  const playlistOptions = {
2534
3011
  predictedReadyAtMs: clip.predictedReadyAtMs,
3012
+ sessionId: clip.sessionId,
2535
3013
  signal: abort.signal
2536
3014
  };
2537
3015
  if (slackMsRef.current !== void 0) {
2538
3016
  playlistOptions.slackMs = slackMsRef.current;
2539
3017
  }
2540
- if (jwt !== void 0) {
2541
- playlistOptions.jwt = jwt;
3018
+ if (source() !== void 0) {
3019
+ playlistOptions.jwt = (context) => resolveJwtSource(source() ?? "", context);
2542
3020
  }
2543
3021
  const body = await fetchPlaylist(clip.playlistUrl, playlistOptions);
2544
3022
  if (cancelled) {
@@ -2684,15 +3162,13 @@ function useClipDownload(clip, options = {}) {
2684
3162
  abortRef.current = abort;
2685
3163
  setState({ kind: "downloading", fetched: 0, total: 0 });
2686
3164
  try {
2687
- const explicit = getJwtRef.current;
2688
- const fallback = storeRef.current?.getState().internal.reactor.getJwtResolver();
2689
- const jwt = explicit ? await explicit() : fallback !== void 0 ? await resolveJwtSource(fallback) : void 0;
3165
+ const source = () => getJwtRef.current ?? storeRef.current?.getState().internal.reactor.getJwtResolver();
2690
3166
  const downloadOptions = {
2691
3167
  signal: abort.signal,
2692
3168
  onProgress: ({ fetched, total }) => setState({ kind: "downloading", fetched, total })
2693
3169
  };
2694
- if (jwt !== void 0) {
2695
- downloadOptions.jwt = jwt;
3170
+ if (source() !== void 0) {
3171
+ downloadOptions.jwt = (context) => resolveJwtSource(source() ?? "", context);
2696
3172
  }
2697
3173
  const blob = await downloadClipAsFile(clipRef.current, filenameRef.current, downloadOptions);
2698
3174
  setState({ kind: "idle" });
@@ -2788,9 +3264,11 @@ export {
2788
3264
  ClipPlayer,
2789
3265
  ConflictError,
2790
3266
  DEFAULT_PLAYLIST_POLL_SLACK_MS,
3267
+ DataChannelChunkingRequiredError,
2791
3268
  DecodeError,
2792
3269
  DisconnectedError,
2793
3270
  FileRef,
3271
+ GenerationModeMismatchError,
2794
3272
  InvalidStateError,
2795
3273
  MessageTooLargeError,
2796
3274
  NetworkError,
@@ -2805,8 +3283,14 @@ export {
2805
3283
  RequestTimeoutError,
2806
3284
  ServerError,
2807
3285
  SessionTerminalError,
3286
+ StepCommandFailedError,
3287
+ StepNotReadyError,
3288
+ StepRefusedError,
3289
+ StepResultTooLargeError,
3290
+ StepSessionEndedError,
2808
3291
  TransportError,
2809
3292
  UnauthorizedError,
3293
+ UnsupportedFrameEncodingError,
2810
3294
  VersionMismatchError,
2811
3295
  WebcamStream,
2812
3296
  createPlayableManifestUrl,
@@ -2815,6 +3299,7 @@ export {
2815
3299
  isFileRef,
2816
3300
  normalizeJwtSource,
2817
3301
  parsePlaylist,
3302
+ stepFrameToImageData,
2818
3303
  useClipDownload,
2819
3304
  useReactor,
2820
3305
  useReactorInternalMessage,