@reactor-team/js-sdk 3.2.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -44,6 +44,14 @@ var ReactorError = class extends Error {
44
44
  operation;
45
45
  /** Backoff hint the platform sent. */
46
46
  retry_after_ms;
47
+ /** For a refused step or a step whose call failed, the part of the step
48
+ * request at fault: `"state.<key>"`, `"commands[<i>]"` or
49
+ * `"media_input[<i>]"`. */
50
+ entry;
51
+ /** For a step whose call failed, the stable code its handler raised (its
52
+ * `CommandError` code, or `internal_error`), to branch on instead of the
53
+ * message. */
54
+ command_code;
47
55
  /** When `reactor-core` reported this failure. For a purely local one (no
48
56
  * payload at all — see {@link ReactorErrorOptions}), this is synthesized
49
57
  * as the construction time instead, since there's nothing to report —
@@ -64,6 +72,8 @@ var ReactorError = class extends Error {
64
72
  this.operation = options.operation;
65
73
  this.retry_after_ms = options.retry_after_ms;
66
74
  this.retryAfter = options.retry_after_ms;
75
+ this.entry = options.entry;
76
+ this.command_code = options.command_code;
67
77
  this.timestamp_ms = options.timestamp_ms ?? Date.now();
68
78
  this.timestamp = this.timestamp_ms;
69
79
  }
@@ -119,6 +129,30 @@ var AbortedError = class extends ReactorError {
119
129
  var RecorderDisabledError = class extends ReactorError {
120
130
  static code = "RECORDER_DISABLED";
121
131
  };
132
+ var GenerationModeMismatchError = class extends ReactorError {
133
+ static code = "GENERATION_MODE_MISMATCH";
134
+ };
135
+ var DataChannelChunkingRequiredError = class extends ReactorError {
136
+ static code = "DATA_CHANNEL_CHUNKING_REQUIRED";
137
+ };
138
+ var StepRefusedError = class extends ReactorError {
139
+ static code = "step_refused";
140
+ };
141
+ var StepNotReadyError = class extends ReactorError {
142
+ static code = "step_not_ready";
143
+ };
144
+ var StepResultTooLargeError = class extends ReactorError {
145
+ static code = "step_result_too_large";
146
+ };
147
+ var StepCommandFailedError = class extends ReactorError {
148
+ static code = "step_command_failed";
149
+ };
150
+ var StepSessionEndedError = class extends ReactorError {
151
+ static code = "step_session_ended";
152
+ };
153
+ var UnsupportedFrameEncodingError = class extends ReactorError {
154
+ static code = "UNSUPPORTED_FRAME_ENCODING";
155
+ };
122
156
  var ERROR_CLASSES = [
123
157
  NetworkError,
124
158
  UnauthorizedError,
@@ -136,7 +170,15 @@ var ERROR_CLASSES = [
136
170
  DisconnectedError,
137
171
  RequestTimeoutError,
138
172
  AbortedError,
139
- RecorderDisabledError
173
+ RecorderDisabledError,
174
+ GenerationModeMismatchError,
175
+ DataChannelChunkingRequiredError,
176
+ StepRefusedError,
177
+ StepNotReadyError,
178
+ StepResultTooLargeError,
179
+ StepCommandFailedError,
180
+ StepSessionEndedError,
181
+ UnsupportedFrameEncodingError
140
182
  ];
141
183
  var BY_CODE = new Map(
142
184
  ERROR_CLASSES.map((errorClass) => [errorClass.code, errorClass])
@@ -158,6 +200,8 @@ function toReactorError(cause) {
158
200
  operation: payload.operation,
159
201
  retry_after_ms: payload.retry_after_ms,
160
202
  timestamp_ms: payload.timestamp_ms,
203
+ entry: payload.entry,
204
+ command_code: payload.command_code,
161
205
  // A thrown JS error (a failed wasm load, say) keeps its stack and its own
162
206
  // `cause` reachable; reactor-core's plain payload objects have neither.
163
207
  cause: cause instanceof Error ? cause : void 0
@@ -204,6 +248,52 @@ var Emitter = class {
204
248
  }
205
249
  };
206
250
 
251
+ // src/internal/encoded-frames.ts
252
+ var CODECS = /* @__PURE__ */ new Set(["h264"]);
253
+ function isEncodedFrame(frame) {
254
+ return typeof frame === "object" && frame !== null && "codec" in frame;
255
+ }
256
+ function isChunk(data) {
257
+ return !(data instanceof Uint8Array) && !(data instanceof ArrayBuffer) && typeof data.copyTo === "function";
258
+ }
259
+ function toEncodedBytes(frame, operation, trackName) {
260
+ const refuse = (why) => new InvalidStateError(`${operation}('${trackName}'): ${why}`, { operation });
261
+ if (typeof frame !== "object" || frame === null) {
262
+ throw refuse("expected an EncodedFrame, { codec, data, width, height }");
263
+ }
264
+ const { codec, data, width, height, metadata, captureTimeUs } = frame;
265
+ if (!CODECS.has(codec)) {
266
+ throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264"`);
267
+ }
268
+ let bytes;
269
+ if (data instanceof Uint8Array) {
270
+ bytes = data;
271
+ } else if (data instanceof ArrayBuffer) {
272
+ bytes = new Uint8Array(data);
273
+ } else if (data && isChunk(data)) {
274
+ bytes = new Uint8Array(data.byteLength);
275
+ data.copyTo(bytes);
276
+ } else {
277
+ throw refuse("data must be a Uint8Array, an ArrayBuffer or an EncodedVideoChunk");
278
+ }
279
+ if (bytes.byteLength === 0) {
280
+ throw refuse("an encoded frame needs data");
281
+ }
282
+ if (!isDimension(width) || !isDimension(height)) {
283
+ throw refuse(`width and height are the decoded frame's, got ${String(width)}x${String(height)}`);
284
+ }
285
+ if (metadata != null && !(metadata instanceof Uint8Array)) {
286
+ throw refuse("metadata must be a Uint8Array");
287
+ }
288
+ if (captureTimeUs != null && !Number.isFinite(captureTimeUs)) {
289
+ throw refuse("captureTimeUs must be a finite number");
290
+ }
291
+ return { codec, data: bytes, width, height, metadata: metadata ?? null, captureTimeUs: captureTimeUs ?? null };
292
+ }
293
+ function isDimension(value) {
294
+ return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 4294967295;
295
+ }
296
+
207
297
  // src/file-ref.ts
208
298
  var FileRef = class {
209
299
  constructor(uploadId, name, mimeType, size) {
@@ -295,6 +385,112 @@ function isSerializedAsObject(value) {
295
385
  return typeof value === "object" && value !== null && !ArrayBuffer.isView(value);
296
386
  }
297
387
 
388
+ // src/internal/push-frames.ts
389
+ function isAudioFrame(frame) {
390
+ return "samples" in frame;
391
+ }
392
+ function isVideoFrame(frame) {
393
+ return typeof frame.copyTo === "function" && "displayWidth" in frame;
394
+ }
395
+ function holdFrame(frame) {
396
+ if (!isAudioFrame(frame) && isVideoFrame(frame)) {
397
+ const clone = frame.clone();
398
+ return { frame: clone, release: () => clone.close() };
399
+ }
400
+ return { frame, release: () => {
401
+ } };
402
+ }
403
+ async function toPixels(frame) {
404
+ if (isVideoFrame(frame)) {
405
+ return videoFramePixels(frame);
406
+ }
407
+ return rawPixels(frame);
408
+ }
409
+ function rawPixels(frame) {
410
+ const { data, width, height } = frame;
411
+ const bytes = data instanceof Uint8Array ? data : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
412
+ return { data: bytes, width, height, format: "format" in frame && frame.format || "rgba" };
413
+ }
414
+ var NATIVE_FORMATS = {
415
+ RGBA: "rgba",
416
+ RGBX: "rgba",
417
+ BGRA: "bgra",
418
+ BGRX: "bgra"
419
+ };
420
+ function orientationOf(frame) {
421
+ const { rotation, flip } = frame;
422
+ return { rotation: ((rotation ?? 0) % 360 + 360) % 360, flip: flip ?? false };
423
+ }
424
+ async function videoFramePixels(frame) {
425
+ const rect = frame.visibleRect;
426
+ const width = rect?.width ?? frame.codedWidth;
427
+ const height = rect?.height ?? frame.codedHeight;
428
+ const native = frame.format ? NATIVE_FORMATS[frame.format] : void 0;
429
+ const orientation = orientationOf(frame);
430
+ if (native) {
431
+ const data = new Uint8Array(width * height * 4);
432
+ await frame.copyTo(data, { layout: [{ offset: 0, stride: width * 4 }] });
433
+ return orient({ data, width, height, format: native }, orientation);
434
+ }
435
+ try {
436
+ const data = new Uint8Array(width * height * 4);
437
+ const options = { format: "RGBA", layout: [{ offset: 0, stride: width * 4 }] };
438
+ await frame.copyTo(data, options);
439
+ return orient({ data, width, height, format: "rgba" }, orientation);
440
+ } catch {
441
+ return canvasPixels(frame, width, height, orientation);
442
+ }
443
+ }
444
+ function orient(pixels, { rotation, flip }) {
445
+ if (rotation % 90 !== 0 || rotation === 0 && !flip) {
446
+ return pixels;
447
+ }
448
+ const { width: w, height: h } = pixels;
449
+ const turned = rotation === 90 || rotation === 270;
450
+ const outWidth = turned ? h : w;
451
+ const outHeight = turned ? w : h;
452
+ const bytes = pixels.data.byteOffset % 4 === 0 ? pixels.data : pixels.data.slice();
453
+ const source = new Uint32Array(bytes.buffer, bytes.byteOffset, w * h);
454
+ const data = new Uint8Array(outWidth * outHeight * 4);
455
+ const target = new Uint32Array(data.buffer);
456
+ for (let y = 0; y < h; y++) {
457
+ for (let x = 0; x < w; x++) {
458
+ let tx;
459
+ let ty;
460
+ if (rotation === 90) {
461
+ [tx, ty] = [h - 1 - y, x];
462
+ } else if (rotation === 180) {
463
+ [tx, ty] = [w - 1 - x, h - 1 - y];
464
+ } else if (rotation === 270) {
465
+ [tx, ty] = [y, w - 1 - x];
466
+ } else {
467
+ [tx, ty] = [x, y];
468
+ }
469
+ if (flip) {
470
+ tx = outWidth - 1 - tx;
471
+ }
472
+ target[ty * outWidth + tx] = source[y * w + x];
473
+ }
474
+ }
475
+ return { data, width: outWidth, height: outHeight, format: pixels.format };
476
+ }
477
+ function canvasPixels(frame, visibleWidth, visibleHeight, { rotation }) {
478
+ const turned = rotation === 90 || rotation === 270;
479
+ const width = turned ? visibleHeight : visibleWidth;
480
+ const height = turned ? visibleWidth : visibleHeight;
481
+ if (typeof OffscreenCanvas === "undefined") {
482
+ throw new Error(
483
+ `pushFrame: cannot read a ${frame.format ?? "formatless"} VideoFrame here \u2014 push its pixels as an ImageData or a Uint8Array instead.`
484
+ );
485
+ }
486
+ const context = new OffscreenCanvas(width, height).getContext("2d");
487
+ if (!context) {
488
+ throw new Error("pushFrame: no 2D canvas context to read the VideoFrame through.");
489
+ }
490
+ context.drawImage(frame, 0, 0, width, height);
491
+ return rawPixels(context.getImageData(0, 0, width, height));
492
+ }
493
+
298
494
  // src/internal/recording.ts
299
495
  function toPublicClip(clip) {
300
496
  return {
@@ -308,6 +504,67 @@ function toPublicClip(clip) {
308
504
  };
309
505
  }
310
506
 
507
+ // src/internal/step-media.ts
508
+ function holdStepMedia(media) {
509
+ if (typeof media !== "object" || media === null || Array.isArray(media)) {
510
+ throw new InvalidStateError("step(): media must map track names to arrays of frames", {
511
+ operation: "step"
512
+ });
513
+ }
514
+ const held = [];
515
+ try {
516
+ for (const [trackName, frames] of Object.entries(media)) {
517
+ if (!Array.isArray(frames)) {
518
+ throw new InvalidStateError(`step(): media['${trackName}'] must be an array of frames`, {
519
+ operation: "step"
520
+ });
521
+ }
522
+ for (const frame of frames) {
523
+ held.push(holdStepFrame(trackName, frame));
524
+ }
525
+ }
526
+ } catch (cause) {
527
+ for (const frame of held) {
528
+ frame.release();
529
+ }
530
+ throw cause;
531
+ }
532
+ return held;
533
+ }
534
+ function holdStepFrame(track, frame) {
535
+ if (isEncodedFrame(frame)) {
536
+ const operation = "pushEncodedFrame";
537
+ return { track, kind: "encoded", operation, encoded: toEncodedBytes(frame, operation, track), release: () => {
538
+ } };
539
+ }
540
+ if (typeof frame !== "object" || frame === null) {
541
+ throw new InvalidStateError(`pushFrame('${track}'): expected a frame`, { operation: "pushFrame" });
542
+ }
543
+ const held = holdFrame(frame);
544
+ return { track, kind: "raw", operation: "pushFrame", frame: held.frame, release: held.release };
545
+ }
546
+ async function readStepFrame(frame) {
547
+ const { track } = frame;
548
+ if (frame.kind === "encoded") {
549
+ return { track, kind: "encoded", operation: frame.operation, encoded: frame.encoded };
550
+ }
551
+ if (isAudioFrame(frame.frame)) {
552
+ return { track, kind: "audio", operation: frame.operation, audio: frame.frame };
553
+ }
554
+ return { track, kind: "video", operation: frame.operation, pixels: await toPixels(frame.frame) };
555
+ }
556
+ function checkStepFrame(client, frame) {
557
+ if (frame.kind === "encoded") {
558
+ const { codec, data, width, height, captureTimeUs } = frame.encoded;
559
+ client.checkEncodedVideoFrame(frame.track, codec, data.byteLength, width, height, captureTimeUs);
560
+ } else if (frame.kind === "video") {
561
+ const { data, width, height, format } = frame.pixels;
562
+ client.checkVideoFrame(frame.track, data.byteLength, width, height, format);
563
+ } else {
564
+ client.checkAudioFrame(frame.track, frame.audio.samples, frame.audio.channels ?? 1);
565
+ }
566
+ }
567
+
311
568
  // src/internal/stats.ts
312
569
  var LOCAL_STATS_INTERVAL_MS = 2e3;
313
570
  var RUNTIME_REPORT_INTERVAL_MS = 5e3;
@@ -357,7 +614,28 @@ var WIRE_CODECS = {
357
614
  };
358
615
  function createTrackStatsExtractor() {
359
616
  const lastBytes = /* @__PURE__ */ new Map();
360
- return (report, tracks, pausedTracks) => {
617
+ const lastStages = /* @__PURE__ */ new Map();
618
+ let pendingBytes = /* @__PURE__ */ new Map();
619
+ let pendingStages = /* @__PURE__ */ new Map();
620
+ const stage = (streamId, name, seconds, frames) => {
621
+ if (seconds === void 0 || frames === void 0) {
622
+ return void 0;
623
+ }
624
+ const key = `${streamId}:${name}`;
625
+ if (!Number.isFinite(seconds)) {
626
+ pendingStages.set(key, void 0);
627
+ return void 0;
628
+ }
629
+ pendingStages.set(key, { seconds, frames });
630
+ const before = lastStages.get(key);
631
+ if (before === void 0 || frames <= before.frames || seconds < before.seconds) {
632
+ return void 0;
633
+ }
634
+ return { name, totalMs: (seconds - before.seconds) * 1e3, frames: frames - before.frames };
635
+ };
636
+ const extract = (report, tracks, pausedTracks) => {
637
+ pendingBytes = /* @__PURE__ */ new Map();
638
+ pendingStages = /* @__PURE__ */ new Map();
361
639
  const reportWithLookup = report;
362
640
  const stats = [];
363
641
  const seenStreams = /* @__PURE__ */ new Set();
@@ -396,7 +674,7 @@ function createTrackStatsExtractor() {
396
674
  if (previous !== void 0 && stat.timestamp > previous.timestamp && bytes >= previous.bytes) {
397
675
  put("bitrate_bps", Math.round((bytes - previous.bytes) * 8 * 1e3 / (stat.timestamp - previous.timestamp)));
398
676
  }
399
- lastBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
677
+ pendingBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
400
678
  }
401
679
  const keyframeRequests = stat.firCount !== void 0 || stat.pliCount !== void 0 ? (stat.firCount ?? 0) + (stat.pliCount ?? 0) : void 0;
402
680
  if (inbound) {
@@ -431,6 +709,20 @@ function createTrackStatsExtractor() {
431
709
  put("keyframe_requests", keyframeRequests);
432
710
  }
433
711
  }
712
+ const frameStages = [];
713
+ if (track.kind === "video") {
714
+ const stages = inbound ? [
715
+ stage(stat.id, "jitter_buffer", stat.jitterBufferDelay, stat.jitterBufferEmittedCount),
716
+ stage(stat.id, "decode", stat.totalDecodeTime, stat.framesDecoded)
717
+ ] : [stage(stat.id, "encode", stat.totalEncodeTime, stat.framesEncoded)];
718
+ for (const s of stages) {
719
+ if (s !== void 0) {
720
+ frameStages.push(s);
721
+ }
722
+ }
723
+ }
724
+ const codecImplementation = inbound ? stat.decoderImplementation : stat.encoderImplementation;
725
+ const hardwareCodec = inbound ? stat.powerEfficientDecoder : stat.powerEfficientEncoder;
434
726
  stats.push({
435
727
  timestamp: Date.now(),
436
728
  trackName: track.name,
@@ -438,11 +730,30 @@ function createTrackStatsExtractor() {
438
730
  direction,
439
731
  codec,
440
732
  paused: pausedTracks.includes(track.name),
441
- metrics
733
+ metrics,
734
+ ...frameStages.length > 0 ? { frameStages } : {},
735
+ ...track.kind === "video" && codecImplementation !== void 0 ? { codecImplementation } : {},
736
+ ...track.kind === "video" && hardwareCodec !== void 0 ? { hardwareCodec } : {}
442
737
  });
443
738
  });
444
739
  return stats;
445
740
  };
741
+ return Object.assign(extract, {
742
+ commit() {
743
+ for (const [id, sample] of pendingBytes) {
744
+ lastBytes.set(id, sample);
745
+ }
746
+ for (const [key, total] of pendingStages) {
747
+ if (total === void 0) {
748
+ lastStages.delete(key);
749
+ } else {
750
+ lastStages.set(key, total);
751
+ }
752
+ }
753
+ pendingBytes = /* @__PURE__ */ new Map();
754
+ pendingStages = /* @__PURE__ */ new Map();
755
+ }
756
+ });
446
757
  }
447
758
  function toClientConnectionStat(report, timeToConnectMs) {
448
759
  const pair = activeCandidatePair(report);
@@ -581,7 +892,7 @@ function describeCause(cause) {
581
892
  // package.json
582
893
  var package_default = {
583
894
  name: "@reactor-team/js-sdk",
584
- version: "3.2.0",
895
+ version: "3.4.0",
585
896
  description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
586
897
  license: "Apache-2.0",
587
898
  repository: {
@@ -1095,10 +1406,19 @@ var Reactor = class {
1095
1406
  * with it, so a batch sent before the timings exist doesn't use it up.
1096
1407
  * Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
1097
1408
  hasSentConnectTime = false;
1409
+ /** Whether this connection has sent a client-stats batch yet. The first
1410
+ * one goes out even with no reading, for the environment the bridge adds
1411
+ * to it. Reset with `hasSentConnectTime`. */
1412
+ hasSentClientStats = false;
1098
1413
  /** Set on the "connecting" status transition, cleared once `connectionTimings`
1099
1414
  * is finalized on "ready" — see `handleStatusChanged()`. */
1100
1415
  connectStartTime;
1101
1416
  waitingStartTime;
1417
+ /** The tail of every `pushFrame()`/`pushEncodedFrame()` so far, settled
1418
+ * either way — pushes reach the binding in call order however long each
1419
+ * one's pixels take to read, and `step()` waits for the ones called
1420
+ * before it. */
1421
+ pendingPushes = Promise.resolve();
1102
1422
  emitter = new Emitter();
1103
1423
  /** Serializes connect()/reconnect()/disconnect() (and the free() inside
1104
1424
  * disconnect()/[Symbol.dispose]) against each other. Calling into the
@@ -1116,6 +1436,15 @@ var Reactor = class {
1116
1436
  * out every read active when it started, before it releases the ones
1117
1437
  * that arrived while it ran. */
1118
1438
  writerActive = false;
1439
+ /** Counts `disconnect()`s and disposal. A push captures it when called, and
1440
+ * one whose frame was still being read when it moved is dropped, rather
1441
+ * than recreating a client to carry the frame into the next session. */
1442
+ connectionEpoch = 0;
1443
+ /** Input tracks published on this connection, each with the publication
1444
+ * that claimed it: a push needs one, and a `PublishedTrack` only feeds the
1445
+ * publication it came from. Cleared with the connection. */
1446
+ publishedInputs = /* @__PURE__ */ new Map();
1447
+ publications = 0;
1119
1448
  /** Readers parked in `withReaderAccess()`'s wait loop, woken once
1120
1449
  * `withWriterAccess()` finishes; each re-checks `writerActive` rather
1121
1450
  * than assuming its turn, since another writer already queued behind the
@@ -1388,14 +1717,50 @@ var Reactor = class {
1388
1717
  * disconnect() freeing the wasm client mid-await would otherwise resume
1389
1718
  * into a freed client — the same class of use-after-free `queue` exists to
1390
1719
  * prevent for connect/disconnect/reconnect themselves.
1720
+ *
1721
+ * Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
1722
+ * track in a stepped session.
1723
+ *
1724
+ * In a `generation: "stepped"` session, input frames never travel over
1725
+ * RTP: `track` is optional, and the frames are pushed with
1726
+ * `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
1727
+ * passed there is ignored, with a console warning — nothing is sent from
1728
+ * it. In a streamed session `track` is required, and omitting it rejects
1729
+ * with an `InvalidStateError`.
1730
+ *
1731
+ * ```ts
1732
+ * const camera = await reactor.publishTrack('camera');
1733
+ * await camera.pushFrame(imageData);
1734
+ * const result = await reactor.step();
1735
+ * ```
1391
1736
  */
1392
1737
  async publishTrack(name, track) {
1393
1738
  this.assertNotDisposed();
1394
1739
  try {
1740
+ let media = track;
1741
+ if (this.isStepped() && media !== void 0) {
1742
+ console.warn(
1743
+ `[Reactor] publishTrack('${name}'): ignoring the MediaStreamTrack \u2014 a stepped session sends no RTP and takes frames through track.pushFrame(...) on the track publishTrack() returns.`
1744
+ );
1745
+ media = void 0;
1746
+ }
1747
+ if (!this.isStepped() && media === void 0) {
1748
+ throw new InvalidStateError(
1749
+ `publishTrack('${name}') needs a MediaStreamTrack in a streamed session.`,
1750
+ { operation: "publishTrack" }
1751
+ );
1752
+ }
1753
+ const publication = this.publications += 1;
1395
1754
  await this.queue.push(async () => {
1396
1755
  const client = await this.getOrCreateClient();
1397
- await client.publishTrack(name, track);
1756
+ await client.publishTrack(name, media);
1757
+ this.publishedInputs.set(name, publication);
1398
1758
  }, "publishTrack");
1759
+ return {
1760
+ name,
1761
+ pushFrame: (frame, options) => this.pushTo(name, publication, frame, options),
1762
+ pushEncodedFrame: (frame) => this.pushEncodedTo(name, publication, frame)
1763
+ };
1399
1764
  } catch (cause) {
1400
1765
  throw this.captureError(cause);
1401
1766
  }
@@ -1412,6 +1777,7 @@ var Reactor = class {
1412
1777
  await this.queue.push(async () => {
1413
1778
  const client = await this.getOrCreateClient();
1414
1779
  await client.unpublishTrack(name);
1780
+ this.publishedInputs.delete(name);
1415
1781
  }, "unpublishTrack");
1416
1782
  } catch (cause) {
1417
1783
  this.emitError(cause);
@@ -1484,6 +1850,337 @@ var Reactor = class {
1484
1850
  throw this.captureError(cause);
1485
1851
  }
1486
1852
  }
1853
+ // ── Stepped generation ──────────────────────────────────────────────────
1854
+ /**
1855
+ * Runs one step of a `generation: "stepped"` session and resolves with
1856
+ * what it produced, per output track.
1857
+ *
1858
+ * The request carries `state`, `commands` and every frame pushed since the
1859
+ * previous step. The runtime applies them in that order — each `state` key
1860
+ * as the model's `set_<key>` command, then `commands`, then the frames —
1861
+ * and then generates. They are separate calls, not a transaction: one that
1862
+ * fails does not undo the ones before it. All of it is optional:
1863
+ * `step()` with nothing set and nothing pushed runs on the model's current
1864
+ * state, which commands sent outside a step (`sendCommand()`) also change.
1865
+ * A file goes in the way it does for a command: upload it with
1866
+ * `uploadFile()`, then put the `FileRef` in a command's `data` or a state
1867
+ * value.
1868
+ *
1869
+ * A step that ran but that the model reported as failed still resolves,
1870
+ * with `result.error` set. `result.files` lists the files the model kept
1871
+ * with the step (name, content type, size); their bytes are served over
1872
+ * HTTP from the session's step results, not carried here.
1873
+ *
1874
+ * Rejects with a `StepRefusedError` when the runtime or the model turns
1875
+ * the step down (the runtime checks a whole request first, so nothing it
1876
+ * carried was applied), `StepNotReadyError` when the session cannot take a
1877
+ * step yet (neither ran the step), `StepCommandFailedError` when a state key
1878
+ * or command failed (the calls before it stay applied; the step did not
1879
+ * run); both name the part at fault in `entry`, and `StepCommandFailedError`
1880
+ * the handler's code in `command_code`,
1881
+ * `StepSessionEndedError` when the session ended before the result came
1882
+ * back, `StepResultTooLargeError` when the result does not fit in one
1883
+ * message, `MessageTooLargeError` when the request itself is too large
1884
+ * (nothing was sent), `InvalidStateError` outside a ready stepped
1885
+ * session, or whatever else failed the step (`DisconnectedError`,
1886
+ * `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
1887
+ * when its request cannot be sent, they are dropped with it rather than
1888
+ * kept for the next step.
1889
+ *
1890
+ * `media` gives the step frames of its own, by input track name, pushed
1891
+ * after the ones already pushed to that track: what `pushFrame()` takes,
1892
+ * or an `EncodedFrame` — H.264 the runtime decodes before the step runs,
1893
+ * so the model reads RGB frames as for a raw push. They are pushed only
1894
+ * once `state` and `commands` are well formed and the session is ready, so
1895
+ * a step refused for those leaves the frames buffered as they were. They go
1896
+ * all or nothing: a bad track or frame anywhere in `media` rejects the step
1897
+ * before any of it is pushed or sent, leaving the buffer as it was. A
1898
+ * session that leaves 'ready' once the media is in has the step refused and
1899
+ * keeps the frames for the next one, as pushed frames are.
1900
+ * A track's encoded frames of one step go in decode order, starting at a
1901
+ * keyframe; a step whose frames do not decode rejects with a
1902
+ * `StepRefusedError`.
1903
+ *
1904
+ * ```ts
1905
+ * const camera = await reactor.publishTrack('camera');
1906
+ * await camera.pushFrame(imageData);
1907
+ * const result = await reactor.step({ state: { prompt: 'a cat' } });
1908
+ * if (result.error) console.warn('step failed:', result.error);
1909
+ * const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
1910
+ * if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
1911
+ * ```
1912
+ */
1913
+ async step(options = {}) {
1914
+ this.assertNotDisposed();
1915
+ try {
1916
+ const state = rewriteNestedFileRefs(options.state ?? null);
1917
+ const commands = rewriteNestedFileRefs(options.commands ?? []);
1918
+ const previous = this.pendingPushes;
1919
+ const before = options.media !== void 0 ? this.pushStepMedia(options.media, state, commands) : previous;
1920
+ let taken;
1921
+ const bufferTaken = new Promise((resolve) => {
1922
+ taken = resolve;
1923
+ });
1924
+ const stepping = before.then(
1925
+ () => (
1926
+ // Excluded from connect()/disconnect()/reconnect() like sendCommand():
1927
+ // a step can take minutes, and must neither be freed out from under
1928
+ // nor block unrelated reads.
1929
+ this.withReaderAccess(async () => {
1930
+ const client = await this.getOrCreateClient();
1931
+ try {
1932
+ return client.step(state, commands);
1933
+ } finally {
1934
+ taken();
1935
+ }
1936
+ })
1937
+ )
1938
+ );
1939
+ stepping.catch(() => previous.then(taken));
1940
+ this.pendingPushes = bufferTaken;
1941
+ return await stepping;
1942
+ } catch (cause) {
1943
+ throw this.captureError(cause);
1944
+ }
1945
+ }
1946
+ /**
1947
+ * Buffers one input frame for the next `step()` on the sendonly track
1948
+ * `trackName`. Nothing is sent until that step, which carries every frame
1949
+ * pushed since the one before it, per track in push order. The
1950
+ * `PublishedTrack` that `publishTrack()` resolves with does the same
1951
+ * through `track.pushFrame(frame)`.
1952
+ *
1953
+ * Video: a `VideoFrame` (cloned when called, so the caller may close it as
1954
+ * soon as this returns; it stays the caller's), an
1955
+ * `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
1956
+ * by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
1957
+ * `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
1958
+ * applied, so a 90° frame arrives with its width and height swapped.
1959
+ * Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
1960
+ * PCM at the rate the model's track declares.
1961
+ *
1962
+ * Pushes keep their call order, and a `step()` waits for every push called
1963
+ * before it. A refused push rejects: await it, or `.catch()` it in a frame
1964
+ * loop, or every bad frame is an unhandled rejection. What waits for
1965
+ * the next step is at most one message's worth of frames: pushing faster
1966
+ * than steps run drops the oldest frames, across tracks. Rejects with
1967
+ * an `InvalidStateError` outside a stepped session, for a track not
1968
+ * published on this connection (`publishTrack(trackName)` first), for a
1969
+ * frame whose size does not match its dimensions, or for a name the model
1970
+ * does not declare as a sendonly track of that kind.
1971
+ */
1972
+ async pushFrame(trackName, frame, options = {}) {
1973
+ return this.pushTo(trackName, void 0, frame, options);
1974
+ }
1975
+ /** `pushFrame()`, for the track's current publication or, from a
1976
+ * `PublishedTrack`, only for the `publication` it came from. */
1977
+ async pushTo(trackName, publication, frame, options = {}) {
1978
+ this.assertNotDisposed();
1979
+ const epoch = this.connectionEpoch;
1980
+ let held;
1981
+ try {
1982
+ held = holdFrame(frame);
1983
+ } catch (cause) {
1984
+ throw this.captureError(cause);
1985
+ }
1986
+ const push = this.pendingPushes.then(async () => {
1987
+ let pixels;
1988
+ try {
1989
+ if (!this.isStepped()) {
1990
+ throw new InvalidStateError(
1991
+ `pushFrame('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
1992
+ { operation: "pushFrame" }
1993
+ );
1994
+ }
1995
+ pixels = isAudioFrame(held.frame) ? void 0 : await toPixels(held.frame);
1996
+ } finally {
1997
+ held.release();
1998
+ }
1999
+ await this.withReaderAccess(async () => {
2000
+ this.checkPushTarget("pushFrame", trackName, publication, epoch);
2001
+ const client = await this.getOrCreateClient();
2002
+ if (pixels) {
2003
+ client.pushVideoFrame(
2004
+ trackName,
2005
+ pixels.data,
2006
+ pixels.width,
2007
+ pixels.height,
2008
+ pixels.format,
2009
+ options.metadata ?? null,
2010
+ options.captureTimeUs ?? null
2011
+ );
2012
+ } else if (isAudioFrame(frame)) {
2013
+ client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
2014
+ }
2015
+ });
2016
+ });
2017
+ this.pendingPushes = push.catch(() => void 0);
2018
+ try {
2019
+ await push;
2020
+ } catch (cause) {
2021
+ throw this.captureError(cause);
2022
+ }
2023
+ }
2024
+ /**
2025
+ * Buffers one encoded video frame for the next `step()` on the sendonly
2026
+ * video track `trackName`, its bytes as they are: one H.264 access unit
2027
+ * (Annex B), with the `width` and `height` it decodes to. The
2028
+ * `PublishedTrack` that `publishTrack()` resolves with does the same
2029
+ * through `track.pushEncodedFrame(frame)`.
2030
+ *
2031
+ * The runtime decodes a track's encoded frames of one step as one stream,
2032
+ * before the step runs, and the model reads the RGB frames as if they had
2033
+ * been pushed raw. So push them in decode order, starting at a keyframe; a
2034
+ * step whose frames do not decode rejects with a `StepRefusedError`, as
2035
+ * does a runtime that predates encoded frames. Encoded and raw pushes keep
2036
+ * their call order, and a `step()` waits for every push called before it.
2037
+ *
2038
+ * ```ts
2039
+ * const encoder = new VideoEncoder({
2040
+ * output: (chunk) => void camera.pushEncodedFrame({ codec: 'h264', data: chunk, width, height }),
2041
+ * error: console.error,
2042
+ * });
2043
+ * encoder.configure({ codec: 'avc1.42001f', width, height, avc: { format: 'annexb' } });
2044
+ * ```
2045
+ *
2046
+ * Rejects with an `InvalidStateError` outside a stepped session, for a
2047
+ * track not published on this connection, for an audio or recvonly track,
2048
+ * for empty data, a zero dimension or a codec other than `"h264"`.
2049
+ */
2050
+ async pushEncodedFrame(trackName, frame) {
2051
+ return this.pushEncodedTo(trackName, void 0, frame);
2052
+ }
2053
+ /** `pushEncodedFrame()`, for the track's current publication or, from a
2054
+ * `PublishedTrack`, only for the `publication` it came from. */
2055
+ async pushEncodedTo(trackName, publication, frame) {
2056
+ const operation = "pushEncodedFrame";
2057
+ this.assertNotDisposed();
2058
+ const epoch = this.connectionEpoch;
2059
+ let encoded;
2060
+ try {
2061
+ encoded = toEncodedBytes(frame, operation, trackName);
2062
+ } catch (cause) {
2063
+ throw this.captureError(cause);
2064
+ }
2065
+ const push = this.pendingPushes.then(
2066
+ () => this.withReaderAccess(async () => {
2067
+ if (!this.isStepped()) {
2068
+ throw new InvalidStateError(
2069
+ `${operation}('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
2070
+ { operation }
2071
+ );
2072
+ }
2073
+ this.checkPushTarget(operation, trackName, publication, epoch);
2074
+ const client = await this.getOrCreateClient();
2075
+ const declared = client.tracks().find((track) => track.name === trackName);
2076
+ if (declared && (declared.kind !== "video" || declared.direction !== "sendonly")) {
2077
+ throw new InvalidStateError(
2078
+ `${operation}('${trackName}'): '${trackName}' is a ${declared.direction} ${declared.kind} track; only a sendonly video track takes encoded frames.`,
2079
+ { operation }
2080
+ );
2081
+ }
2082
+ client.pushEncodedVideoFrame(
2083
+ trackName,
2084
+ encoded.codec,
2085
+ encoded.data,
2086
+ encoded.width,
2087
+ encoded.height,
2088
+ encoded.metadata,
2089
+ encoded.captureTimeUs
2090
+ );
2091
+ })
2092
+ );
2093
+ this.pendingPushes = push.catch(() => void 0);
2094
+ try {
2095
+ await push;
2096
+ } catch (cause) {
2097
+ throw this.captureError(cause);
2098
+ }
2099
+ }
2100
+ /** Refuses a push whose connection ended since it was called, or whose
2101
+ * track is not (or no longer) published on it. */
2102
+ checkPushTarget(operation, trackName, publication, epoch) {
2103
+ if (epoch !== this.connectionEpoch) {
2104
+ throw new InvalidStateError(
2105
+ `${operation}('${trackName}'): the connection it was pushed to ended while its frame was read`,
2106
+ { operation }
2107
+ );
2108
+ }
2109
+ const current = this.publishedInputs.get(trackName);
2110
+ if (current === void 0 || publication !== void 0 && publication !== current) {
2111
+ throw new InvalidStateError(
2112
+ publication === void 0 ? `${operation}('${trackName}') needs publishTrack('${trackName}') first.` : `${operation}('${trackName}'): this track was unpublished, or its connection ended.`,
2113
+ { operation }
2114
+ );
2115
+ }
2116
+ }
2117
+ /** Pushes `step()`'s `media`, track by track and in order, each frame as
2118
+ * `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
2119
+ * checks the step's `state` and `commands`, then every frame, before the
2120
+ * first is pushed, so a refused step, or a bad track or frame anywhere,
2121
+ * leaves the buffer as it was. */
2122
+ async pushStepMedia(media, state, commands) {
2123
+ const epoch = this.connectionEpoch;
2124
+ const held = holdStepMedia(media);
2125
+ const push = this.pendingPushes.then(async () => {
2126
+ const ready = [];
2127
+ try {
2128
+ for (const frame of held) {
2129
+ ready.push(await readStepFrame(frame));
2130
+ }
2131
+ } finally {
2132
+ for (const frame of held) {
2133
+ frame.release();
2134
+ }
2135
+ }
2136
+ await this.withReaderAccess(async () => {
2137
+ const client = await this.getOrCreateClient();
2138
+ client.checkStep(state, commands);
2139
+ for (const frame of ready) {
2140
+ this.checkPushTarget(frame.operation, frame.track, void 0, epoch);
2141
+ checkStepFrame(client, frame);
2142
+ }
2143
+ for (const frame of ready) {
2144
+ if (frame.kind === "encoded") {
2145
+ client.pushEncodedVideoFrame(
2146
+ frame.track,
2147
+ frame.encoded.codec,
2148
+ frame.encoded.data,
2149
+ frame.encoded.width,
2150
+ frame.encoded.height,
2151
+ frame.encoded.metadata,
2152
+ frame.encoded.captureTimeUs
2153
+ );
2154
+ } else if (frame.kind === "video") {
2155
+ const { data, width, height, format } = frame.pixels;
2156
+ client.pushVideoFrame(frame.track, data, width, height, format, null, null);
2157
+ } else {
2158
+ const { samples, sampleRate, channels } = frame.audio;
2159
+ client.pushAudioFrame(frame.track, samples, sampleRate, channels ?? 1);
2160
+ }
2161
+ }
2162
+ });
2163
+ });
2164
+ this.pendingPushes = push.catch(() => void 0);
2165
+ await push;
2166
+ }
2167
+ /** Drops every frame buffered for the next step, pushes called before it
2168
+ * included: it waits its turn behind them, as a `step()` does. */
2169
+ async clearStepInputs() {
2170
+ this.assertNotDisposed();
2171
+ const clear = this.pendingPushes.then(
2172
+ () => this.withReaderAccess(() => {
2173
+ this.client?.clearStepInputs();
2174
+ return Promise.resolve();
2175
+ })
2176
+ );
2177
+ this.pendingPushes = clear.catch(() => void 0);
2178
+ try {
2179
+ await clear;
2180
+ } catch (cause) {
2181
+ throw this.captureError(cause);
2182
+ }
2183
+ }
1487
2184
  // ── Recording ───────────────────────────────────────────────────────────
1488
2185
  /**
1489
2186
  * Requests a clip covering the last `durationSeconds` of the session.
@@ -1782,6 +2479,7 @@ var Reactor = class {
1782
2479
  const extractConnectionStats = createConnectionStatsExtractor();
1783
2480
  const extractTrackStats = createTrackStatsExtractor();
1784
2481
  this.hasSentConnectTime = false;
2482
+ this.hasSentClientStats = false;
1785
2483
  this.stopStatsPollers = [
1786
2484
  pollPeerStats(client, LOCAL_STATS_INTERVAL_MS, (report) => {
1787
2485
  this.stats = { ...extractConnectionStats(report), connectionTimings: this.connectionTimings };
@@ -1799,8 +2497,9 @@ var Reactor = class {
1799
2497
  * track has one yet (none negotiated onto a named track, or no codec
1800
2498
  * reported yet), so the runtime gets the bandwidth estimate, the
1801
2499
  * round-trip time and the time to connect without waiting on the tracks.
1802
- * Skipped only when there is nothing at all to say; one is due again in
1803
- * `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
2500
+ * Skipped only when there is nothing at all to say and a batch has already
2501
+ * gone out for this connection, so the first one still carries the
2502
+ * environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
1804
2503
  * actually sent for this connection — see `hasSentConnectTime`.
1805
2504
  * `sendClientStats` throws synchronously on a closing connection; caught
1806
2505
  * here for the same reason `getStats()`'s rejection is above. */
@@ -1808,11 +2507,13 @@ var Reactor = class {
1808
2507
  const trackStats = extractTrackStats(report, this.trackMapping(), this.pausedTracks());
1809
2508
  const timeToConnectMs = this.hasSentConnectTime ? void 0 : this.connectionTimings?.totalMs;
1810
2509
  const connectionStat = toClientConnectionStat(report, timeToConnectMs);
1811
- if (trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
2510
+ if (this.hasSentClientStats && trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
1812
2511
  return;
1813
2512
  }
1814
2513
  try {
1815
2514
  client.sendClientStats(trackStats, connectionStat);
2515
+ extractTrackStats.commit();
2516
+ this.hasSentClientStats = true;
1816
2517
  if (timeToConnectMs !== void 0) {
1817
2518
  this.hasSentConnectTime = true;
1818
2519
  }
@@ -1834,6 +2535,8 @@ var Reactor = class {
1834
2535
  * leave `getCapabilities()` returning the previous session's stale
1835
2536
  * tracks/commands until a new `capabilitiesReceived` lands. */
1836
2537
  resetConnectionState() {
2538
+ this.connectionEpoch += 1;
2539
+ this.publishedInputs.clear();
1837
2540
  this.stopStatsPolling();
1838
2541
  this.connectionTimings = void 0;
1839
2542
  this.connectStartTime = void 0;
@@ -1854,6 +2557,9 @@ var Reactor = class {
1854
2557
  emitError(cause) {
1855
2558
  this.emitter.emit("error", this.captureError(cause));
1856
2559
  }
2560
+ isStepped() {
2561
+ return this.clientOptions.generation === "stepped";
2562
+ }
1857
2563
  assertNotDisposed() {
1858
2564
  if (this.disposed) {
1859
2565
  throw new Error("This Reactor was disposed and can't be used again \u2014 construct a new one.");
@@ -1861,6 +2567,25 @@ var Reactor = class {
1861
2567
  }
1862
2568
  };
1863
2569
 
2570
+ // src/step-frames.ts
2571
+ function stepFrameToImageData(frame) {
2572
+ const [height, width, channels] = frame.shape;
2573
+ if (frame.dtype !== "uint8" || frame.shape.length !== 3 || channels !== 3 || height === void 0 || width === void 0 || frame.data.length !== width * height * 3) {
2574
+ throw new Error(
2575
+ `stepFrameToImageData: expected a (height, width, 3) uint8 RGB frame, got ${frame.dtype} [${frame.shape.join(", ")}]`
2576
+ );
2577
+ }
2578
+ const rgb = frame.data;
2579
+ const rgba = new Uint8ClampedArray(width * height * 4);
2580
+ for (let src = 0, dst = 0; src < rgb.length; src += 3, dst += 4) {
2581
+ rgba[dst] = rgb[src];
2582
+ rgba[dst + 1] = rgb[src + 1];
2583
+ rgba[dst + 2] = rgb[src + 2];
2584
+ rgba[dst + 3] = 255;
2585
+ }
2586
+ return new ImageData(rgba, width, height);
2587
+ }
2588
+
1864
2589
  // src/jwt.ts
1865
2590
  function normalizeJwtSource(source) {
1866
2591
  if (typeof source === "function") {
@@ -2810,9 +3535,11 @@ export {
2810
3535
  ClipPlayer,
2811
3536
  ConflictError,
2812
3537
  DEFAULT_PLAYLIST_POLL_SLACK_MS,
3538
+ DataChannelChunkingRequiredError,
2813
3539
  DecodeError,
2814
3540
  DisconnectedError,
2815
3541
  FileRef,
3542
+ GenerationModeMismatchError,
2816
3543
  InvalidStateError,
2817
3544
  MessageTooLargeError,
2818
3545
  NetworkError,
@@ -2827,8 +3554,14 @@ export {
2827
3554
  RequestTimeoutError,
2828
3555
  ServerError,
2829
3556
  SessionTerminalError,
3557
+ StepCommandFailedError,
3558
+ StepNotReadyError,
3559
+ StepRefusedError,
3560
+ StepResultTooLargeError,
3561
+ StepSessionEndedError,
2830
3562
  TransportError,
2831
3563
  UnauthorizedError,
3564
+ UnsupportedFrameEncodingError,
2832
3565
  VersionMismatchError,
2833
3566
  WebcamStream,
2834
3567
  createPlayableManifestUrl,
@@ -2837,6 +3570,7 @@ export {
2837
3570
  isFileRef,
2838
3571
  normalizeJwtSource,
2839
3572
  parsePlaylist,
3573
+ stepFrameToImageData,
2840
3574
  useClipDownload,
2841
3575
  useReactor,
2842
3576
  useReactorInternalMessage,