@reactor-team/js-sdk 3.1.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -37,9 +37,11 @@ __export(index_exports, {
37
37
  ClipPlayer: () => ClipPlayer,
38
38
  ConflictError: () => ConflictError,
39
39
  DEFAULT_PLAYLIST_POLL_SLACK_MS: () => DEFAULT_PLAYLIST_POLL_SLACK_MS,
40
+ DataChannelChunkingRequiredError: () => DataChannelChunkingRequiredError,
40
41
  DecodeError: () => DecodeError,
41
42
  DisconnectedError: () => DisconnectedError,
42
43
  FileRef: () => FileRef,
44
+ GenerationModeMismatchError: () => GenerationModeMismatchError,
43
45
  InvalidStateError: () => InvalidStateError,
44
46
  MessageTooLargeError: () => MessageTooLargeError,
45
47
  NetworkError: () => NetworkError,
@@ -54,8 +56,14 @@ __export(index_exports, {
54
56
  RequestTimeoutError: () => RequestTimeoutError,
55
57
  ServerError: () => ServerError,
56
58
  SessionTerminalError: () => SessionTerminalError,
59
+ StepCommandFailedError: () => StepCommandFailedError,
60
+ StepNotReadyError: () => StepNotReadyError,
61
+ StepRefusedError: () => StepRefusedError,
62
+ StepResultTooLargeError: () => StepResultTooLargeError,
63
+ StepSessionEndedError: () => StepSessionEndedError,
57
64
  TransportError: () => TransportError,
58
65
  UnauthorizedError: () => UnauthorizedError,
66
+ UnsupportedFrameEncodingError: () => UnsupportedFrameEncodingError,
59
67
  VersionMismatchError: () => VersionMismatchError,
60
68
  WebcamStream: () => WebcamStream,
61
69
  createPlayableManifestUrl: () => createPlayableManifestUrl,
@@ -64,6 +72,7 @@ __export(index_exports, {
64
72
  isFileRef: () => isFileRef,
65
73
  normalizeJwtSource: () => normalizeJwtSource,
66
74
  parsePlaylist: () => parsePlaylist,
75
+ stepFrameToImageData: () => stepFrameToImageData,
67
76
  useClipDownload: () => useClipDownload,
68
77
  useReactor: () => useReactor,
69
78
  useReactorInternalMessage: () => useReactorInternalMessage,
@@ -117,6 +126,14 @@ var ReactorError = class extends Error {
117
126
  operation;
118
127
  /** Backoff hint the platform sent. */
119
128
  retry_after_ms;
129
+ /** For a refused step or a step whose call failed, the part of the step
130
+ * request at fault: `"state.<key>"`, `"commands[<i>]"` or
131
+ * `"media_input[<i>]"`. */
132
+ entry;
133
+ /** For a step whose call failed, the stable code its handler raised (its
134
+ * `CommandError` code, or `internal_error`), to branch on instead of the
135
+ * message. */
136
+ command_code;
120
137
  /** When `reactor-core` reported this failure. For a purely local one (no
121
138
  * payload at all — see {@link ReactorErrorOptions}), this is synthesized
122
139
  * as the construction time instead, since there's nothing to report —
@@ -129,7 +146,7 @@ var ReactorError = class extends Error {
129
146
  * compatibility — prefer `timestamp_ms`. */
130
147
  timestamp;
131
148
  constructor(message, options = {}) {
132
- super(message);
149
+ super(message, options.cause === void 0 ? void 0 : { cause: options.cause });
133
150
  this.name = "ReactorError";
134
151
  this.code = options.code ?? new.target.code;
135
152
  this.recoverable = options.recoverable ?? false;
@@ -137,6 +154,8 @@ var ReactorError = class extends Error {
137
154
  this.operation = options.operation;
138
155
  this.retry_after_ms = options.retry_after_ms;
139
156
  this.retryAfter = options.retry_after_ms;
157
+ this.entry = options.entry;
158
+ this.command_code = options.command_code;
140
159
  this.timestamp_ms = options.timestamp_ms ?? Date.now();
141
160
  this.timestamp = this.timestamp_ms;
142
161
  }
@@ -192,6 +211,30 @@ var AbortedError = class extends ReactorError {
192
211
  var RecorderDisabledError = class extends ReactorError {
193
212
  static code = "RECORDER_DISABLED";
194
213
  };
214
+ var GenerationModeMismatchError = class extends ReactorError {
215
+ static code = "GENERATION_MODE_MISMATCH";
216
+ };
217
+ var DataChannelChunkingRequiredError = class extends ReactorError {
218
+ static code = "DATA_CHANNEL_CHUNKING_REQUIRED";
219
+ };
220
+ var StepRefusedError = class extends ReactorError {
221
+ static code = "step_refused";
222
+ };
223
+ var StepNotReadyError = class extends ReactorError {
224
+ static code = "step_not_ready";
225
+ };
226
+ var StepResultTooLargeError = class extends ReactorError {
227
+ static code = "step_result_too_large";
228
+ };
229
+ var StepCommandFailedError = class extends ReactorError {
230
+ static code = "step_command_failed";
231
+ };
232
+ var StepSessionEndedError = class extends ReactorError {
233
+ static code = "step_session_ended";
234
+ };
235
+ var UnsupportedFrameEncodingError = class extends ReactorError {
236
+ static code = "UNSUPPORTED_FRAME_ENCODING";
237
+ };
195
238
  var ERROR_CLASSES = [
196
239
  NetworkError,
197
240
  UnauthorizedError,
@@ -209,7 +252,15 @@ var ERROR_CLASSES = [
209
252
  DisconnectedError,
210
253
  RequestTimeoutError,
211
254
  AbortedError,
212
- RecorderDisabledError
255
+ RecorderDisabledError,
256
+ GenerationModeMismatchError,
257
+ DataChannelChunkingRequiredError,
258
+ StepRefusedError,
259
+ StepNotReadyError,
260
+ StepResultTooLargeError,
261
+ StepCommandFailedError,
262
+ StepSessionEndedError,
263
+ UnsupportedFrameEncodingError
213
264
  ];
214
265
  var BY_CODE = new Map(
215
266
  ERROR_CLASSES.map((errorClass) => [errorClass.code, errorClass])
@@ -230,7 +281,12 @@ function toReactorError(cause) {
230
281
  status: payload.status,
231
282
  operation: payload.operation,
232
283
  retry_after_ms: payload.retry_after_ms,
233
- timestamp_ms: payload.timestamp_ms
284
+ timestamp_ms: payload.timestamp_ms,
285
+ entry: payload.entry,
286
+ command_code: payload.command_code,
287
+ // A thrown JS error (a failed wasm load, say) keeps its stack and its own
288
+ // `cause` reachable; reactor-core's plain payload objects have neither.
289
+ cause: cause instanceof Error ? cause : void 0
234
290
  });
235
291
  }
236
292
 
@@ -365,6 +421,112 @@ function isSerializedAsObject(value) {
365
421
  return typeof value === "object" && value !== null && !ArrayBuffer.isView(value);
366
422
  }
367
423
 
424
+ // src/internal/push-frames.ts
425
+ function isAudioFrame(frame) {
426
+ return "samples" in frame;
427
+ }
428
+ function isVideoFrame(frame) {
429
+ return typeof frame.copyTo === "function" && "displayWidth" in frame;
430
+ }
431
+ function holdFrame(frame) {
432
+ if (!isAudioFrame(frame) && isVideoFrame(frame)) {
433
+ const clone = frame.clone();
434
+ return { frame: clone, release: () => clone.close() };
435
+ }
436
+ return { frame, release: () => {
437
+ } };
438
+ }
439
+ async function toPixels(frame) {
440
+ if (isVideoFrame(frame)) {
441
+ return videoFramePixels(frame);
442
+ }
443
+ return rawPixels(frame);
444
+ }
445
+ function rawPixels(frame) {
446
+ const { data, width, height } = frame;
447
+ const bytes = data instanceof Uint8Array ? data : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
448
+ return { data: bytes, width, height, format: "format" in frame && frame.format || "rgba" };
449
+ }
450
+ var NATIVE_FORMATS = {
451
+ RGBA: "rgba",
452
+ RGBX: "rgba",
453
+ BGRA: "bgra",
454
+ BGRX: "bgra"
455
+ };
456
+ function orientationOf(frame) {
457
+ const { rotation, flip } = frame;
458
+ return { rotation: ((rotation ?? 0) % 360 + 360) % 360, flip: flip ?? false };
459
+ }
460
+ async function videoFramePixels(frame) {
461
+ const rect = frame.visibleRect;
462
+ const width = rect?.width ?? frame.codedWidth;
463
+ const height = rect?.height ?? frame.codedHeight;
464
+ const native = frame.format ? NATIVE_FORMATS[frame.format] : void 0;
465
+ const orientation = orientationOf(frame);
466
+ if (native) {
467
+ const data = new Uint8Array(width * height * 4);
468
+ await frame.copyTo(data, { layout: [{ offset: 0, stride: width * 4 }] });
469
+ return orient({ data, width, height, format: native }, orientation);
470
+ }
471
+ try {
472
+ const data = new Uint8Array(width * height * 4);
473
+ const options = { format: "RGBA", layout: [{ offset: 0, stride: width * 4 }] };
474
+ await frame.copyTo(data, options);
475
+ return orient({ data, width, height, format: "rgba" }, orientation);
476
+ } catch {
477
+ return canvasPixels(frame, width, height, orientation);
478
+ }
479
+ }
480
+ function orient(pixels, { rotation, flip }) {
481
+ if (rotation % 90 !== 0 || rotation === 0 && !flip) {
482
+ return pixels;
483
+ }
484
+ const { width: w, height: h } = pixels;
485
+ const turned = rotation === 90 || rotation === 270;
486
+ const outWidth = turned ? h : w;
487
+ const outHeight = turned ? w : h;
488
+ const bytes = pixels.data.byteOffset % 4 === 0 ? pixels.data : pixels.data.slice();
489
+ const source = new Uint32Array(bytes.buffer, bytes.byteOffset, w * h);
490
+ const data = new Uint8Array(outWidth * outHeight * 4);
491
+ const target = new Uint32Array(data.buffer);
492
+ for (let y = 0; y < h; y++) {
493
+ for (let x = 0; x < w; x++) {
494
+ let tx;
495
+ let ty;
496
+ if (rotation === 90) {
497
+ [tx, ty] = [h - 1 - y, x];
498
+ } else if (rotation === 180) {
499
+ [tx, ty] = [w - 1 - x, h - 1 - y];
500
+ } else if (rotation === 270) {
501
+ [tx, ty] = [y, w - 1 - x];
502
+ } else {
503
+ [tx, ty] = [x, y];
504
+ }
505
+ if (flip) {
506
+ tx = outWidth - 1 - tx;
507
+ }
508
+ target[ty * outWidth + tx] = source[y * w + x];
509
+ }
510
+ }
511
+ return { data, width: outWidth, height: outHeight, format: pixels.format };
512
+ }
513
+ function canvasPixels(frame, visibleWidth, visibleHeight, { rotation }) {
514
+ const turned = rotation === 90 || rotation === 270;
515
+ const width = turned ? visibleHeight : visibleWidth;
516
+ const height = turned ? visibleWidth : visibleHeight;
517
+ if (typeof OffscreenCanvas === "undefined") {
518
+ throw new Error(
519
+ `pushFrame: cannot read a ${frame.format ?? "formatless"} VideoFrame here \u2014 push its pixels as an ImageData or a Uint8Array instead.`
520
+ );
521
+ }
522
+ const context = new OffscreenCanvas(width, height).getContext("2d");
523
+ if (!context) {
524
+ throw new Error("pushFrame: no 2D canvas context to read the VideoFrame through.");
525
+ }
526
+ context.drawImage(frame, 0, 0, width, height);
527
+ return rawPixels(context.getImageData(0, 0, width, height));
528
+ }
529
+
368
530
  // src/internal/recording.ts
369
531
  function toPublicClip(clip) {
370
532
  return {
@@ -427,7 +589,28 @@ var WIRE_CODECS = {
427
589
  };
428
590
  function createTrackStatsExtractor() {
429
591
  const lastBytes = /* @__PURE__ */ new Map();
430
- return (report, tracks, pausedTracks) => {
592
+ const lastStages = /* @__PURE__ */ new Map();
593
+ let pendingBytes = /* @__PURE__ */ new Map();
594
+ let pendingStages = /* @__PURE__ */ new Map();
595
+ const stage = (streamId, name, seconds, frames) => {
596
+ if (seconds === void 0 || frames === void 0) {
597
+ return void 0;
598
+ }
599
+ const key = `${streamId}:${name}`;
600
+ if (!Number.isFinite(seconds)) {
601
+ pendingStages.set(key, void 0);
602
+ return void 0;
603
+ }
604
+ pendingStages.set(key, { seconds, frames });
605
+ const before = lastStages.get(key);
606
+ if (before === void 0 || frames <= before.frames || seconds < before.seconds) {
607
+ return void 0;
608
+ }
609
+ return { name, totalMs: (seconds - before.seconds) * 1e3, frames: frames - before.frames };
610
+ };
611
+ const extract = (report, tracks, pausedTracks) => {
612
+ pendingBytes = /* @__PURE__ */ new Map();
613
+ pendingStages = /* @__PURE__ */ new Map();
431
614
  const reportWithLookup = report;
432
615
  const stats = [];
433
616
  const seenStreams = /* @__PURE__ */ new Set();
@@ -466,7 +649,7 @@ function createTrackStatsExtractor() {
466
649
  if (previous !== void 0 && stat.timestamp > previous.timestamp && bytes >= previous.bytes) {
467
650
  put("bitrate_bps", Math.round((bytes - previous.bytes) * 8 * 1e3 / (stat.timestamp - previous.timestamp)));
468
651
  }
469
- lastBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
652
+ pendingBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
470
653
  }
471
654
  const keyframeRequests = stat.firCount !== void 0 || stat.pliCount !== void 0 ? (stat.firCount ?? 0) + (stat.pliCount ?? 0) : void 0;
472
655
  if (inbound) {
@@ -501,6 +684,20 @@ function createTrackStatsExtractor() {
501
684
  put("keyframe_requests", keyframeRequests);
502
685
  }
503
686
  }
687
+ const frameStages = [];
688
+ if (track.kind === "video") {
689
+ const stages = inbound ? [
690
+ stage(stat.id, "jitter_buffer", stat.jitterBufferDelay, stat.jitterBufferEmittedCount),
691
+ stage(stat.id, "decode", stat.totalDecodeTime, stat.framesDecoded)
692
+ ] : [stage(stat.id, "encode", stat.totalEncodeTime, stat.framesEncoded)];
693
+ for (const s of stages) {
694
+ if (s !== void 0) {
695
+ frameStages.push(s);
696
+ }
697
+ }
698
+ }
699
+ const codecImplementation = inbound ? stat.decoderImplementation : stat.encoderImplementation;
700
+ const hardwareCodec = inbound ? stat.powerEfficientDecoder : stat.powerEfficientEncoder;
504
701
  stats.push({
505
702
  timestamp: Date.now(),
506
703
  trackName: track.name,
@@ -508,11 +705,30 @@ function createTrackStatsExtractor() {
508
705
  direction,
509
706
  codec,
510
707
  paused: pausedTracks.includes(track.name),
511
- metrics
708
+ metrics,
709
+ ...frameStages.length > 0 ? { frameStages } : {},
710
+ ...track.kind === "video" && codecImplementation !== void 0 ? { codecImplementation } : {},
711
+ ...track.kind === "video" && hardwareCodec !== void 0 ? { hardwareCodec } : {}
512
712
  });
513
713
  });
514
714
  return stats;
515
715
  };
716
+ return Object.assign(extract, {
717
+ commit() {
718
+ for (const [id, sample] of pendingBytes) {
719
+ lastBytes.set(id, sample);
720
+ }
721
+ for (const [key, total] of pendingStages) {
722
+ if (total === void 0) {
723
+ lastStages.delete(key);
724
+ } else {
725
+ lastStages.set(key, total);
726
+ }
727
+ }
728
+ pendingBytes = /* @__PURE__ */ new Map();
729
+ pendingStages = /* @__PURE__ */ new Map();
730
+ }
731
+ });
516
732
  }
517
733
  function toClientConnectionStat(report, timeToConnectMs) {
518
734
  const pair = activeCandidatePair(report);
@@ -631,27 +847,27 @@ function loadReactorWasm() {
631
847
  if (!modulePromise) {
632
848
  modulePromise = importWasmModule().catch((cause) => {
633
849
  modulePromise = null;
634
- throw new Error(
635
- "reactor-wasm failed to load. Run `mise run build:wasm` (or, from a published install, reinstall the package) and try again.",
636
- { cause }
637
- );
850
+ throw new Error(`reactor-wasm failed to load: ${describeCause(cause)}`, { cause });
638
851
  });
639
852
  }
640
853
  return modulePromise;
641
854
  }
642
855
  async function importWasmModule() {
643
- const module2 = await import(
644
- /* @vite-ignore */
645
- "./wasm/reactor_wasm.js"
646
- );
856
+ const module2 = await import("./wasm/reactor_wasm.js");
647
857
  await module2.default();
648
858
  return module2;
649
859
  }
860
+ function describeCause(cause) {
861
+ if (cause instanceof Error) {
862
+ return cause.message ? `${cause.name}: ${cause.message}` : cause.name;
863
+ }
864
+ return String(cause);
865
+ }
650
866
 
651
867
  // package.json
652
868
  var package_default = {
653
869
  name: "@reactor-team/js-sdk",
654
- version: "3.1.0",
870
+ version: "3.3.0",
655
871
  description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
656
872
  license: "Apache-2.0",
657
873
  repository: {
@@ -711,6 +927,11 @@ var package_default = {
711
927
  }
712
928
  };
713
929
 
930
+ // src/internal/jwt-resolver.ts
931
+ async function resolveJwtSource(source, context) {
932
+ return typeof source === "function" ? source(context) : source;
933
+ }
934
+
714
935
  // src/recording.ts
715
936
  var RecordingError = class extends Error {
716
937
  constructor(code, reason) {
@@ -730,15 +951,16 @@ async function fetchPlaylist(playlistUrl, options = {}) {
730
951
  const startedPollingAt = Date.now();
731
952
  const deadlineMs = hasDeadline ? Math.max(options.predictedReadyAtMs ?? startedPollingAt, startedPollingAt) + options.slackMs : void 0;
732
953
  const maxRetries = typeof options.maxRetries === "number" ? options.maxRetries : void 0;
733
- const init = {};
734
- if (options.signal) {
735
- init.signal = options.signal;
736
- }
737
- if (options.jwt) {
738
- init.headers = { Authorization: `Bearer ${options.jwt}` };
739
- }
740
954
  let attempt = 0;
741
955
  while (true) {
956
+ const init = {};
957
+ if (options.signal) {
958
+ init.signal = options.signal;
959
+ }
960
+ const jwt = await resolveManifestJwt(options);
961
+ if (jwt) {
962
+ init.headers = { Authorization: `Bearer ${jwt}` };
963
+ }
742
964
  let response;
743
965
  try {
744
966
  response = await fetch(playlistUrl, init);
@@ -841,7 +1063,10 @@ function absolutizeManifestUrls(manifestBody, playlistUrl) {
841
1063
  return out.join(eol);
842
1064
  }
843
1065
  async function downloadClipAsFile(clip, filename = "reactor-clip.mp4", options = {}) {
844
- const playlistOptions = { predictedReadyAtMs: clip.predictedReadyAtMs };
1066
+ const playlistOptions = {
1067
+ predictedReadyAtMs: clip.predictedReadyAtMs,
1068
+ sessionId: clip.sessionId
1069
+ };
845
1070
  if (options.signal) {
846
1071
  playlistOptions.signal = options.signal;
847
1072
  }
@@ -1079,6 +1304,27 @@ function parseRetryAfter(header, fallbackMs) {
1079
1304
  }
1080
1305
  return fallbackMs;
1081
1306
  }
1307
+ async function resolveManifestJwt(options) {
1308
+ if (options.jwt === void 0) {
1309
+ return void 0;
1310
+ }
1311
+ const context = options.sessionId === void 0 ? {} : { sessionId: options.sessionId };
1312
+ return abortable(resolveJwtSource(options.jwt, context), options.signal);
1313
+ }
1314
+ function abortable(promise, signal) {
1315
+ if (!signal) {
1316
+ return promise;
1317
+ }
1318
+ return new Promise((resolve, reject) => {
1319
+ const onAbort = () => reject(new DOMException("Aborted", "AbortError"));
1320
+ if (signal.aborted) {
1321
+ onAbort();
1322
+ } else {
1323
+ signal.addEventListener("abort", onAbort, { once: true });
1324
+ }
1325
+ void promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort));
1326
+ });
1327
+ }
1082
1328
  function sleep(ms, signal) {
1083
1329
  return new Promise((resolve, reject) => {
1084
1330
  if (signal?.aborted) {
@@ -1135,10 +1381,18 @@ var Reactor = class {
1135
1381
  * with it, so a batch sent before the timings exist doesn't use it up.
1136
1382
  * Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
1137
1383
  hasSentConnectTime = false;
1384
+ /** Whether this connection has sent a client-stats batch yet. The first
1385
+ * one goes out even with no reading, for the environment the bridge adds
1386
+ * to it. Reset with `hasSentConnectTime`. */
1387
+ hasSentClientStats = false;
1138
1388
  /** Set on the "connecting" status transition, cleared once `connectionTimings`
1139
1389
  * is finalized on "ready" — see `handleStatusChanged()`. */
1140
1390
  connectStartTime;
1141
1391
  waitingStartTime;
1392
+ /** The tail of every `pushFrame()` so far, settled either way — pushes
1393
+ * reach the binding in call order however long each one's pixels take to
1394
+ * read, and `step()` waits for the ones called before it. */
1395
+ pendingPushes = Promise.resolve();
1142
1396
  emitter = new Emitter();
1143
1397
  /** Serializes connect()/reconnect()/disconnect() (and the free() inside
1144
1398
  * disconnect()/[Symbol.dispose]) against each other. Calling into the
@@ -1156,6 +1410,15 @@ var Reactor = class {
1156
1410
  * out every read active when it started, before it releases the ones
1157
1411
  * that arrived while it ran. */
1158
1412
  writerActive = false;
1413
+ /** Counts `disconnect()`s and disposal. A push captures it when called, and
1414
+ * one whose frame was still being read when it moved is dropped, rather
1415
+ * than recreating a client to carry the frame into the next session. */
1416
+ connectionEpoch = 0;
1417
+ /** Input tracks published on this connection, each with the publication
1418
+ * that claimed it: a push needs one, and a `PublishedTrack` only feeds the
1419
+ * publication it came from. Cleared with the connection. */
1420
+ publishedInputs = /* @__PURE__ */ new Map();
1421
+ publications = 0;
1159
1422
  /** Readers parked in `withReaderAccess()`'s wait loop, woken once
1160
1423
  * `withWriterAccess()` finishes; each re-checks `writerActive` rather
1161
1424
  * than assuming its turn, since another writer already queued behind the
@@ -1428,14 +1691,49 @@ var Reactor = class {
1428
1691
  * disconnect() freeing the wasm client mid-await would otherwise resume
1429
1692
  * into a freed client — the same class of use-after-free `queue` exists to
1430
1693
  * prevent for connect/disconnect/reconnect themselves.
1694
+ *
1695
+ * Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
1696
+ * track in a stepped session.
1697
+ *
1698
+ * In a `generation: "stepped"` session, input frames never travel over
1699
+ * RTP: `track` is optional, and the frames are pushed with
1700
+ * `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
1701
+ * passed there is ignored, with a console warning — nothing is sent from
1702
+ * it. In a streamed session `track` is required, and omitting it rejects
1703
+ * with an `InvalidStateError`.
1704
+ *
1705
+ * ```ts
1706
+ * const camera = await reactor.publishTrack('camera');
1707
+ * await camera.pushFrame(imageData);
1708
+ * const result = await reactor.step();
1709
+ * ```
1431
1710
  */
1432
1711
  async publishTrack(name, track) {
1433
1712
  this.assertNotDisposed();
1434
1713
  try {
1714
+ let media = track;
1715
+ if (this.isStepped() && media !== void 0) {
1716
+ console.warn(
1717
+ `[Reactor] publishTrack('${name}'): ignoring the MediaStreamTrack \u2014 a stepped session sends no RTP and takes frames through track.pushFrame(...) on the track publishTrack() returns.`
1718
+ );
1719
+ media = void 0;
1720
+ }
1721
+ if (!this.isStepped() && media === void 0) {
1722
+ throw new InvalidStateError(
1723
+ `publishTrack('${name}') needs a MediaStreamTrack in a streamed session.`,
1724
+ { operation: "publishTrack" }
1725
+ );
1726
+ }
1727
+ const publication = this.publications += 1;
1435
1728
  await this.queue.push(async () => {
1436
1729
  const client = await this.getOrCreateClient();
1437
- await client.publishTrack(name, track);
1730
+ await client.publishTrack(name, media);
1731
+ this.publishedInputs.set(name, publication);
1438
1732
  }, "publishTrack");
1733
+ return {
1734
+ name,
1735
+ pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
1736
+ };
1439
1737
  } catch (cause) {
1440
1738
  throw this.captureError(cause);
1441
1739
  }
@@ -1452,6 +1750,7 @@ var Reactor = class {
1452
1750
  await this.queue.push(async () => {
1453
1751
  const client = await this.getOrCreateClient();
1454
1752
  await client.unpublishTrack(name);
1753
+ this.publishedInputs.delete(name);
1455
1754
  }, "unpublishTrack");
1456
1755
  } catch (cause) {
1457
1756
  this.emitError(cause);
@@ -1524,6 +1823,175 @@ var Reactor = class {
1524
1823
  throw this.captureError(cause);
1525
1824
  }
1526
1825
  }
1826
+ // ── Stepped generation ──────────────────────────────────────────────────
1827
+ /**
1828
+ * Runs one step of a `generation: "stepped"` session and resolves with
1829
+ * what it produced, per output track.
1830
+ *
1831
+ * The request carries `state`, `commands` and every frame pushed since the
1832
+ * previous step. The runtime applies them in that order — each `state` key
1833
+ * as the model's `set_<key>` command, then `commands`, then the frames —
1834
+ * and then generates. They are separate calls, not a transaction: one that
1835
+ * fails does not undo the ones before it. All of it is optional:
1836
+ * `step()` with nothing set and nothing pushed runs on the model's current
1837
+ * state, which commands sent outside a step (`sendCommand()`) also change.
1838
+ * A file goes in the way it does for a command: upload it with
1839
+ * `uploadFile()`, then put the `FileRef` in a command's `data` or a state
1840
+ * value.
1841
+ *
1842
+ * A step that ran but that the model reported as failed still resolves,
1843
+ * with `result.error` set. `result.files` lists the files the model kept
1844
+ * with the step (name, content type, size); their bytes are served over
1845
+ * HTTP from the session's step results, not carried here.
1846
+ *
1847
+ * Rejects with a `StepRefusedError` when the runtime or the model turns
1848
+ * the step down (the runtime checks a whole request first, so nothing it
1849
+ * carried was applied), `StepNotReadyError` when the session cannot take a
1850
+ * step yet (neither ran the step), `StepCommandFailedError` when a state key
1851
+ * or command failed (the calls before it stay applied; the step did not
1852
+ * run); both name the part at fault in `entry`, and `StepCommandFailedError`
1853
+ * the handler's code in `command_code`,
1854
+ * `StepSessionEndedError` when the session ended before the result came
1855
+ * back, `StepResultTooLargeError` when the result does not fit in one
1856
+ * message, `MessageTooLargeError` when the request itself is too large
1857
+ * (nothing was sent), `InvalidStateError` outside a ready stepped
1858
+ * session, or whatever else failed the step (`DisconnectedError`,
1859
+ * `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
1860
+ * when its request cannot be sent, they are dropped with it rather than
1861
+ * kept for the next step.
1862
+ *
1863
+ * ```ts
1864
+ * const camera = await reactor.publishTrack('camera');
1865
+ * await camera.pushFrame(imageData);
1866
+ * const result = await reactor.step({ state: { prompt: 'a cat' } });
1867
+ * if (result.error) console.warn('step failed:', result.error);
1868
+ * const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
1869
+ * if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
1870
+ * ```
1871
+ */
1872
+ async step(options = {}) {
1873
+ this.assertNotDisposed();
1874
+ try {
1875
+ const state = rewriteNestedFileRefs(options.state ?? null);
1876
+ const commands = (options.commands ?? []).map(
1877
+ (command) => command.data ? { ...command, data: rewriteNestedFileRefs(command.data) } : command
1878
+ );
1879
+ await this.pendingPushes;
1880
+ return await this.withReaderAccess(async () => {
1881
+ const client = await this.getOrCreateClient();
1882
+ return client.step(state, commands);
1883
+ });
1884
+ } catch (cause) {
1885
+ throw this.captureError(cause);
1886
+ }
1887
+ }
1888
+ /**
1889
+ * Buffers one input frame for the next `step()` on the sendonly track
1890
+ * `trackName`. Nothing is sent until that step, which carries every frame
1891
+ * pushed since the one before it, per track in push order. The
1892
+ * `PublishedTrack` that `publishTrack()` resolves with does the same
1893
+ * through `track.pushFrame(frame)`.
1894
+ *
1895
+ * Video: a `VideoFrame` (cloned when called, so the caller may close it as
1896
+ * soon as this returns; it stays the caller's), an
1897
+ * `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
1898
+ * by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
1899
+ * `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
1900
+ * applied, so a 90° frame arrives with its width and height swapped.
1901
+ * Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
1902
+ * PCM at the rate the model's track declares.
1903
+ *
1904
+ * Pushes keep their call order, and a `step()` waits for every push called
1905
+ * before it. A refused push rejects: await it, or `.catch()` it in a frame
1906
+ * loop, or every bad frame is an unhandled rejection. What waits for
1907
+ * the next step is at most one message's worth of frames: pushing faster
1908
+ * than steps run drops the oldest frames, across tracks. Rejects with
1909
+ * an `InvalidStateError` outside a stepped session, for a track not
1910
+ * published on this connection (`publishTrack(trackName)` first), for a
1911
+ * frame whose size does not match its dimensions, or for a name the model
1912
+ * does not declare as a sendonly track of that kind.
1913
+ */
1914
+ async pushFrame(trackName, frame, options = {}) {
1915
+ return this.pushTo(trackName, void 0, frame, options);
1916
+ }
1917
+ /** `pushFrame()`, for the track's current publication or, from a
1918
+ * `PublishedTrack`, only for the `publication` it came from. */
1919
+ async pushTo(trackName, publication, frame, options = {}) {
1920
+ this.assertNotDisposed();
1921
+ const epoch = this.connectionEpoch;
1922
+ let held;
1923
+ try {
1924
+ held = holdFrame(frame);
1925
+ } catch (cause) {
1926
+ throw this.captureError(cause);
1927
+ }
1928
+ const push = this.pendingPushes.then(async () => {
1929
+ let pixels;
1930
+ try {
1931
+ if (!this.isStepped()) {
1932
+ throw new InvalidStateError(
1933
+ `pushFrame('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
1934
+ { operation: "pushFrame" }
1935
+ );
1936
+ }
1937
+ pixels = isAudioFrame(held.frame) ? void 0 : await toPixels(held.frame);
1938
+ } finally {
1939
+ held.release();
1940
+ }
1941
+ await this.withReaderAccess(async () => {
1942
+ if (epoch !== this.connectionEpoch) {
1943
+ throw new InvalidStateError(
1944
+ `pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
1945
+ { operation: "pushFrame" }
1946
+ );
1947
+ }
1948
+ const current = this.publishedInputs.get(trackName);
1949
+ if (current === void 0 || publication !== void 0 && publication !== current) {
1950
+ throw new InvalidStateError(
1951
+ publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
1952
+ { operation: "pushFrame" }
1953
+ );
1954
+ }
1955
+ const client = await this.getOrCreateClient();
1956
+ if (pixels) {
1957
+ client.pushVideoFrame(
1958
+ trackName,
1959
+ pixels.data,
1960
+ pixels.width,
1961
+ pixels.height,
1962
+ pixels.format,
1963
+ options.metadata ?? null,
1964
+ options.captureTimeUs ?? null
1965
+ );
1966
+ } else if (isAudioFrame(frame)) {
1967
+ client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
1968
+ }
1969
+ });
1970
+ });
1971
+ this.pendingPushes = push.catch(() => void 0);
1972
+ try {
1973
+ await push;
1974
+ } catch (cause) {
1975
+ throw this.captureError(cause);
1976
+ }
1977
+ }
1978
+ /** Drops every frame buffered for the next step, pushes called before it
1979
+ * included: it waits its turn behind them, as a `step()` does. */
1980
+ async clearStepInputs() {
1981
+ this.assertNotDisposed();
1982
+ const clear = this.pendingPushes.then(
1983
+ () => this.withReaderAccess(() => {
1984
+ this.client?.clearStepInputs();
1985
+ return Promise.resolve();
1986
+ })
1987
+ );
1988
+ this.pendingPushes = clear.catch(() => void 0);
1989
+ try {
1990
+ await clear;
1991
+ } catch (cause) {
1992
+ throw this.captureError(cause);
1993
+ }
1994
+ }
1527
1995
  // ── Recording ───────────────────────────────────────────────────────────
1528
1996
  /**
1529
1997
  * Requests a clip covering the last `durationSeconds` of the session.
@@ -1822,6 +2290,7 @@ var Reactor = class {
1822
2290
  const extractConnectionStats = createConnectionStatsExtractor();
1823
2291
  const extractTrackStats = createTrackStatsExtractor();
1824
2292
  this.hasSentConnectTime = false;
2293
+ this.hasSentClientStats = false;
1825
2294
  this.stopStatsPollers = [
1826
2295
  pollPeerStats(client, LOCAL_STATS_INTERVAL_MS, (report) => {
1827
2296
  this.stats = { ...extractConnectionStats(report), connectionTimings: this.connectionTimings };
@@ -1839,8 +2308,9 @@ var Reactor = class {
1839
2308
  * track has one yet (none negotiated onto a named track, or no codec
1840
2309
  * reported yet), so the runtime gets the bandwidth estimate, the
1841
2310
  * round-trip time and the time to connect without waiting on the tracks.
1842
- * Skipped only when there is nothing at all to say; one is due again in
1843
- * `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
2311
+ * Skipped only when there is nothing at all to say and a batch has already
2312
+ * gone out for this connection, so the first one still carries the
2313
+ * environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
1844
2314
  * actually sent for this connection — see `hasSentConnectTime`.
1845
2315
  * `sendClientStats` throws synchronously on a closing connection; caught
1846
2316
  * here for the same reason `getStats()`'s rejection is above. */
@@ -1848,11 +2318,13 @@ var Reactor = class {
1848
2318
  const trackStats = extractTrackStats(report, this.trackMapping(), this.pausedTracks());
1849
2319
  const timeToConnectMs = this.hasSentConnectTime ? void 0 : this.connectionTimings?.totalMs;
1850
2320
  const connectionStat = toClientConnectionStat(report, timeToConnectMs);
1851
- if (trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
2321
+ if (this.hasSentClientStats && trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
1852
2322
  return;
1853
2323
  }
1854
2324
  try {
1855
2325
  client.sendClientStats(trackStats, connectionStat);
2326
+ extractTrackStats.commit();
2327
+ this.hasSentClientStats = true;
1856
2328
  if (timeToConnectMs !== void 0) {
1857
2329
  this.hasSentConnectTime = true;
1858
2330
  }
@@ -1874,6 +2346,8 @@ var Reactor = class {
1874
2346
  * leave `getCapabilities()` returning the previous session's stale
1875
2347
  * tracks/commands until a new `capabilitiesReceived` lands. */
1876
2348
  resetConnectionState() {
2349
+ this.connectionEpoch += 1;
2350
+ this.publishedInputs.clear();
1877
2351
  this.stopStatsPolling();
1878
2352
  this.connectionTimings = void 0;
1879
2353
  this.connectStartTime = void 0;
@@ -1894,6 +2368,9 @@ var Reactor = class {
1894
2368
  emitError(cause) {
1895
2369
  this.emitter.emit("error", this.captureError(cause));
1896
2370
  }
2371
+ isStepped() {
2372
+ return this.clientOptions.generation === "stepped";
2373
+ }
1897
2374
  assertNotDisposed() {
1898
2375
  if (this.disposed) {
1899
2376
  throw new Error("This Reactor was disposed and can't be used again \u2014 construct a new one.");
@@ -1901,6 +2378,25 @@ var Reactor = class {
1901
2378
  }
1902
2379
  };
1903
2380
 
2381
+ // src/step-frames.ts
2382
+ function stepFrameToImageData(frame) {
2383
+ const [height, width, channels] = frame.shape;
2384
+ if (frame.dtype !== "uint8" || frame.shape.length !== 3 || channels !== 3 || height === void 0 || width === void 0 || frame.data.length !== width * height * 3) {
2385
+ throw new Error(
2386
+ `stepFrameToImageData: expected a (height, width, 3) uint8 RGB frame, got ${frame.dtype} [${frame.shape.join(", ")}]`
2387
+ );
2388
+ }
2389
+ const rgb = frame.data;
2390
+ const rgba = new Uint8ClampedArray(width * height * 4);
2391
+ for (let src = 0, dst = 0; src < rgb.length; src += 3, dst += 4) {
2392
+ rgba[dst] = rgb[src];
2393
+ rgba[dst + 1] = rgb[src + 1];
2394
+ rgba[dst + 2] = rgb[src + 2];
2395
+ rgba[dst + 3] = 255;
2396
+ }
2397
+ return new ImageData(rgba, width, height);
2398
+ }
2399
+
1904
2400
  // src/jwt.ts
1905
2401
  function normalizeJwtSource(source) {
1906
2402
  if (typeof source === "function") {
@@ -2543,11 +3039,6 @@ function describeMediaError(error) {
2543
3039
  return error?.message ? `${message} (${error.message})` : message;
2544
3040
  }
2545
3041
 
2546
- // src/internal/jwt-resolver.ts
2547
- async function resolveJwtSource(source) {
2548
- return typeof source === "function" ? source() : source;
2549
- }
2550
-
2551
3042
  // src/react/ClipPlayer.tsx
2552
3043
  var import_jsx_runtime4 = require("react/jsx-runtime");
2553
3044
  function ClipPlayer({
@@ -2597,21 +3088,17 @@ function ClipPlayer({
2597
3088
  const setup = async () => {
2598
3089
  try {
2599
3090
  setPhase({ kind: "waiting" });
2600
- const explicit = getJwtRef.current;
2601
- const fallback = storeRef.current?.getState().internal.reactor.getJwtResolver();
2602
- const jwt = explicit ? await explicit() : fallback !== void 0 ? await resolveJwtSource(fallback) : void 0;
2603
- if (cancelled) {
2604
- return;
2605
- }
3091
+ const source = () => getJwtRef.current ?? storeRef.current?.getState().internal.reactor.getJwtResolver();
2606
3092
  const playlistOptions = {
2607
3093
  predictedReadyAtMs: clip.predictedReadyAtMs,
3094
+ sessionId: clip.sessionId,
2608
3095
  signal: abort.signal
2609
3096
  };
2610
3097
  if (slackMsRef.current !== void 0) {
2611
3098
  playlistOptions.slackMs = slackMsRef.current;
2612
3099
  }
2613
- if (jwt !== void 0) {
2614
- playlistOptions.jwt = jwt;
3100
+ if (source() !== void 0) {
3101
+ playlistOptions.jwt = (context) => resolveJwtSource(source() ?? "", context);
2615
3102
  }
2616
3103
  const body = await fetchPlaylist(clip.playlistUrl, playlistOptions);
2617
3104
  if (cancelled) {
@@ -2757,15 +3244,13 @@ function useClipDownload(clip, options = {}) {
2757
3244
  abortRef.current = abort;
2758
3245
  setState({ kind: "downloading", fetched: 0, total: 0 });
2759
3246
  try {
2760
- const explicit = getJwtRef.current;
2761
- const fallback = storeRef.current?.getState().internal.reactor.getJwtResolver();
2762
- const jwt = explicit ? await explicit() : fallback !== void 0 ? await resolveJwtSource(fallback) : void 0;
3247
+ const source = () => getJwtRef.current ?? storeRef.current?.getState().internal.reactor.getJwtResolver();
2763
3248
  const downloadOptions = {
2764
3249
  signal: abort.signal,
2765
3250
  onProgress: ({ fetched, total }) => setState({ kind: "downloading", fetched, total })
2766
3251
  };
2767
- if (jwt !== void 0) {
2768
- downloadOptions.jwt = jwt;
3252
+ if (source() !== void 0) {
3253
+ downloadOptions.jwt = (context) => resolveJwtSource(source() ?? "", context);
2769
3254
  }
2770
3255
  const blob = await downloadClipAsFile(clipRef.current, filenameRef.current, downloadOptions);
2771
3256
  setState({ kind: "idle" });
@@ -2862,9 +3347,11 @@ function defaultLabel(state) {
2862
3347
  ClipPlayer,
2863
3348
  ConflictError,
2864
3349
  DEFAULT_PLAYLIST_POLL_SLACK_MS,
3350
+ DataChannelChunkingRequiredError,
2865
3351
  DecodeError,
2866
3352
  DisconnectedError,
2867
3353
  FileRef,
3354
+ GenerationModeMismatchError,
2868
3355
  InvalidStateError,
2869
3356
  MessageTooLargeError,
2870
3357
  NetworkError,
@@ -2879,8 +3366,14 @@ function defaultLabel(state) {
2879
3366
  RequestTimeoutError,
2880
3367
  ServerError,
2881
3368
  SessionTerminalError,
3369
+ StepCommandFailedError,
3370
+ StepNotReadyError,
3371
+ StepRefusedError,
3372
+ StepResultTooLargeError,
3373
+ StepSessionEndedError,
2882
3374
  TransportError,
2883
3375
  UnauthorizedError,
3376
+ UnsupportedFrameEncodingError,
2884
3377
  VersionMismatchError,
2885
3378
  WebcamStream,
2886
3379
  createPlayableManifestUrl,
@@ -2889,6 +3382,7 @@ function defaultLabel(state) {
2889
3382
  isFileRef,
2890
3383
  normalizeJwtSource,
2891
3384
  parsePlaylist,
3385
+ stepFrameToImageData,
2892
3386
  useClipDownload,
2893
3387
  useReactor,
2894
3388
  useReactorInternalMessage,