@reactor-team/js-sdk 3.2.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -44,6 +44,14 @@ var ReactorError = class extends Error {
44
44
  operation;
45
45
  /** Backoff hint the platform sent. */
46
46
  retry_after_ms;
47
+ /** For a refused step or a step whose call failed, the part of the step
48
+ * request at fault: `"state.<key>"`, `"commands[<i>]"` or
49
+ * `"media_input[<i>]"`. */
50
+ entry;
51
+ /** For a step whose call failed, the stable code its handler raised (its
52
+ * `CommandError` code, or `internal_error`), to branch on instead of the
53
+ * message. */
54
+ command_code;
47
55
  /** When `reactor-core` reported this failure. For a purely local one (no
48
56
  * payload at all — see {@link ReactorErrorOptions}), this is synthesized
49
57
  * as the construction time instead, since there's nothing to report —
@@ -64,6 +72,8 @@ var ReactorError = class extends Error {
64
72
  this.operation = options.operation;
65
73
  this.retry_after_ms = options.retry_after_ms;
66
74
  this.retryAfter = options.retry_after_ms;
75
+ this.entry = options.entry;
76
+ this.command_code = options.command_code;
67
77
  this.timestamp_ms = options.timestamp_ms ?? Date.now();
68
78
  this.timestamp = this.timestamp_ms;
69
79
  }
@@ -119,6 +129,30 @@ var AbortedError = class extends ReactorError {
119
129
  var RecorderDisabledError = class extends ReactorError {
120
130
  static code = "RECORDER_DISABLED";
121
131
  };
132
+ var GenerationModeMismatchError = class extends ReactorError {
133
+ static code = "GENERATION_MODE_MISMATCH";
134
+ };
135
+ var DataChannelChunkingRequiredError = class extends ReactorError {
136
+ static code = "DATA_CHANNEL_CHUNKING_REQUIRED";
137
+ };
138
+ var StepRefusedError = class extends ReactorError {
139
+ static code = "step_refused";
140
+ };
141
+ var StepNotReadyError = class extends ReactorError {
142
+ static code = "step_not_ready";
143
+ };
144
+ var StepResultTooLargeError = class extends ReactorError {
145
+ static code = "step_result_too_large";
146
+ };
147
+ var StepCommandFailedError = class extends ReactorError {
148
+ static code = "step_command_failed";
149
+ };
150
+ var StepSessionEndedError = class extends ReactorError {
151
+ static code = "step_session_ended";
152
+ };
153
+ var UnsupportedFrameEncodingError = class extends ReactorError {
154
+ static code = "UNSUPPORTED_FRAME_ENCODING";
155
+ };
122
156
  var ERROR_CLASSES = [
123
157
  NetworkError,
124
158
  UnauthorizedError,
@@ -136,7 +170,15 @@ var ERROR_CLASSES = [
136
170
  DisconnectedError,
137
171
  RequestTimeoutError,
138
172
  AbortedError,
139
- RecorderDisabledError
173
+ RecorderDisabledError,
174
+ GenerationModeMismatchError,
175
+ DataChannelChunkingRequiredError,
176
+ StepRefusedError,
177
+ StepNotReadyError,
178
+ StepResultTooLargeError,
179
+ StepCommandFailedError,
180
+ StepSessionEndedError,
181
+ UnsupportedFrameEncodingError
140
182
  ];
141
183
  var BY_CODE = new Map(
142
184
  ERROR_CLASSES.map((errorClass) => [errorClass.code, errorClass])
@@ -158,6 +200,8 @@ function toReactorError(cause) {
158
200
  operation: payload.operation,
159
201
  retry_after_ms: payload.retry_after_ms,
160
202
  timestamp_ms: payload.timestamp_ms,
203
+ entry: payload.entry,
204
+ command_code: payload.command_code,
161
205
  // A thrown JS error (a failed wasm load, say) keeps its stack and its own
162
206
  // `cause` reachable; reactor-core's plain payload objects have neither.
163
207
  cause: cause instanceof Error ? cause : void 0
@@ -295,6 +339,112 @@ function isSerializedAsObject(value) {
295
339
  return typeof value === "object" && value !== null && !ArrayBuffer.isView(value);
296
340
  }
297
341
 
342
+ // src/internal/push-frames.ts
343
+ function isAudioFrame(frame) {
344
+ return "samples" in frame;
345
+ }
346
+ function isVideoFrame(frame) {
347
+ return typeof frame.copyTo === "function" && "displayWidth" in frame;
348
+ }
349
+ function holdFrame(frame) {
350
+ if (!isAudioFrame(frame) && isVideoFrame(frame)) {
351
+ const clone = frame.clone();
352
+ return { frame: clone, release: () => clone.close() };
353
+ }
354
+ return { frame, release: () => {
355
+ } };
356
+ }
357
+ async function toPixels(frame) {
358
+ if (isVideoFrame(frame)) {
359
+ return videoFramePixels(frame);
360
+ }
361
+ return rawPixels(frame);
362
+ }
363
+ function rawPixels(frame) {
364
+ const { data, width, height } = frame;
365
+ const bytes = data instanceof Uint8Array ? data : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
366
+ return { data: bytes, width, height, format: "format" in frame && frame.format || "rgba" };
367
+ }
368
+ var NATIVE_FORMATS = {
369
+ RGBA: "rgba",
370
+ RGBX: "rgba",
371
+ BGRA: "bgra",
372
+ BGRX: "bgra"
373
+ };
374
+ function orientationOf(frame) {
375
+ const { rotation, flip } = frame;
376
+ return { rotation: ((rotation ?? 0) % 360 + 360) % 360, flip: flip ?? false };
377
+ }
378
+ async function videoFramePixels(frame) {
379
+ const rect = frame.visibleRect;
380
+ const width = rect?.width ?? frame.codedWidth;
381
+ const height = rect?.height ?? frame.codedHeight;
382
+ const native = frame.format ? NATIVE_FORMATS[frame.format] : void 0;
383
+ const orientation = orientationOf(frame);
384
+ if (native) {
385
+ const data = new Uint8Array(width * height * 4);
386
+ await frame.copyTo(data, { layout: [{ offset: 0, stride: width * 4 }] });
387
+ return orient({ data, width, height, format: native }, orientation);
388
+ }
389
+ try {
390
+ const data = new Uint8Array(width * height * 4);
391
+ const options = { format: "RGBA", layout: [{ offset: 0, stride: width * 4 }] };
392
+ await frame.copyTo(data, options);
393
+ return orient({ data, width, height, format: "rgba" }, orientation);
394
+ } catch {
395
+ return canvasPixels(frame, width, height, orientation);
396
+ }
397
+ }
398
+ function orient(pixels, { rotation, flip }) {
399
+ if (rotation % 90 !== 0 || rotation === 0 && !flip) {
400
+ return pixels;
401
+ }
402
+ const { width: w, height: h } = pixels;
403
+ const turned = rotation === 90 || rotation === 270;
404
+ const outWidth = turned ? h : w;
405
+ const outHeight = turned ? w : h;
406
+ const bytes = pixels.data.byteOffset % 4 === 0 ? pixels.data : pixels.data.slice();
407
+ const source = new Uint32Array(bytes.buffer, bytes.byteOffset, w * h);
408
+ const data = new Uint8Array(outWidth * outHeight * 4);
409
+ const target = new Uint32Array(data.buffer);
410
+ for (let y = 0; y < h; y++) {
411
+ for (let x = 0; x < w; x++) {
412
+ let tx;
413
+ let ty;
414
+ if (rotation === 90) {
415
+ [tx, ty] = [h - 1 - y, x];
416
+ } else if (rotation === 180) {
417
+ [tx, ty] = [w - 1 - x, h - 1 - y];
418
+ } else if (rotation === 270) {
419
+ [tx, ty] = [y, w - 1 - x];
420
+ } else {
421
+ [tx, ty] = [x, y];
422
+ }
423
+ if (flip) {
424
+ tx = outWidth - 1 - tx;
425
+ }
426
+ target[ty * outWidth + tx] = source[y * w + x];
427
+ }
428
+ }
429
+ return { data, width: outWidth, height: outHeight, format: pixels.format };
430
+ }
431
+ function canvasPixels(frame, visibleWidth, visibleHeight, { rotation }) {
432
+ const turned = rotation === 90 || rotation === 270;
433
+ const width = turned ? visibleHeight : visibleWidth;
434
+ const height = turned ? visibleWidth : visibleHeight;
435
+ if (typeof OffscreenCanvas === "undefined") {
436
+ throw new Error(
437
+ `pushFrame: cannot read a ${frame.format ?? "formatless"} VideoFrame here \u2014 push its pixels as an ImageData or a Uint8Array instead.`
438
+ );
439
+ }
440
+ const context = new OffscreenCanvas(width, height).getContext("2d");
441
+ if (!context) {
442
+ throw new Error("pushFrame: no 2D canvas context to read the VideoFrame through.");
443
+ }
444
+ context.drawImage(frame, 0, 0, width, height);
445
+ return rawPixels(context.getImageData(0, 0, width, height));
446
+ }
447
+
298
448
  // src/internal/recording.ts
299
449
  function toPublicClip(clip) {
300
450
  return {
@@ -357,7 +507,28 @@ var WIRE_CODECS = {
357
507
  };
358
508
  function createTrackStatsExtractor() {
359
509
  const lastBytes = /* @__PURE__ */ new Map();
360
- return (report, tracks, pausedTracks) => {
510
+ const lastStages = /* @__PURE__ */ new Map();
511
+ let pendingBytes = /* @__PURE__ */ new Map();
512
+ let pendingStages = /* @__PURE__ */ new Map();
513
+ const stage = (streamId, name, seconds, frames) => {
514
+ if (seconds === void 0 || frames === void 0) {
515
+ return void 0;
516
+ }
517
+ const key = `${streamId}:${name}`;
518
+ if (!Number.isFinite(seconds)) {
519
+ pendingStages.set(key, void 0);
520
+ return void 0;
521
+ }
522
+ pendingStages.set(key, { seconds, frames });
523
+ const before = lastStages.get(key);
524
+ if (before === void 0 || frames <= before.frames || seconds < before.seconds) {
525
+ return void 0;
526
+ }
527
+ return { name, totalMs: (seconds - before.seconds) * 1e3, frames: frames - before.frames };
528
+ };
529
+ const extract = (report, tracks, pausedTracks) => {
530
+ pendingBytes = /* @__PURE__ */ new Map();
531
+ pendingStages = /* @__PURE__ */ new Map();
361
532
  const reportWithLookup = report;
362
533
  const stats = [];
363
534
  const seenStreams = /* @__PURE__ */ new Set();
@@ -396,7 +567,7 @@ function createTrackStatsExtractor() {
396
567
  if (previous !== void 0 && stat.timestamp > previous.timestamp && bytes >= previous.bytes) {
397
568
  put("bitrate_bps", Math.round((bytes - previous.bytes) * 8 * 1e3 / (stat.timestamp - previous.timestamp)));
398
569
  }
399
- lastBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
570
+ pendingBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
400
571
  }
401
572
  const keyframeRequests = stat.firCount !== void 0 || stat.pliCount !== void 0 ? (stat.firCount ?? 0) + (stat.pliCount ?? 0) : void 0;
402
573
  if (inbound) {
@@ -431,6 +602,20 @@ function createTrackStatsExtractor() {
431
602
  put("keyframe_requests", keyframeRequests);
432
603
  }
433
604
  }
605
+ const frameStages = [];
606
+ if (track.kind === "video") {
607
+ const stages = inbound ? [
608
+ stage(stat.id, "jitter_buffer", stat.jitterBufferDelay, stat.jitterBufferEmittedCount),
609
+ stage(stat.id, "decode", stat.totalDecodeTime, stat.framesDecoded)
610
+ ] : [stage(stat.id, "encode", stat.totalEncodeTime, stat.framesEncoded)];
611
+ for (const s of stages) {
612
+ if (s !== void 0) {
613
+ frameStages.push(s);
614
+ }
615
+ }
616
+ }
617
+ const codecImplementation = inbound ? stat.decoderImplementation : stat.encoderImplementation;
618
+ const hardwareCodec = inbound ? stat.powerEfficientDecoder : stat.powerEfficientEncoder;
434
619
  stats.push({
435
620
  timestamp: Date.now(),
436
621
  trackName: track.name,
@@ -438,11 +623,30 @@ function createTrackStatsExtractor() {
438
623
  direction,
439
624
  codec,
440
625
  paused: pausedTracks.includes(track.name),
441
- metrics
626
+ metrics,
627
+ ...frameStages.length > 0 ? { frameStages } : {},
628
+ ...track.kind === "video" && codecImplementation !== void 0 ? { codecImplementation } : {},
629
+ ...track.kind === "video" && hardwareCodec !== void 0 ? { hardwareCodec } : {}
442
630
  });
443
631
  });
444
632
  return stats;
445
633
  };
634
+ return Object.assign(extract, {
635
+ commit() {
636
+ for (const [id, sample] of pendingBytes) {
637
+ lastBytes.set(id, sample);
638
+ }
639
+ for (const [key, total] of pendingStages) {
640
+ if (total === void 0) {
641
+ lastStages.delete(key);
642
+ } else {
643
+ lastStages.set(key, total);
644
+ }
645
+ }
646
+ pendingBytes = /* @__PURE__ */ new Map();
647
+ pendingStages = /* @__PURE__ */ new Map();
648
+ }
649
+ });
446
650
  }
447
651
  function toClientConnectionStat(report, timeToConnectMs) {
448
652
  const pair = activeCandidatePair(report);
@@ -581,7 +785,7 @@ function describeCause(cause) {
581
785
  // package.json
582
786
  var package_default = {
583
787
  name: "@reactor-team/js-sdk",
584
- version: "3.2.0",
788
+ version: "3.3.0",
585
789
  description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
586
790
  license: "Apache-2.0",
587
791
  repository: {
@@ -1095,10 +1299,18 @@ var Reactor = class {
1095
1299
  * with it, so a batch sent before the timings exist doesn't use it up.
1096
1300
  * Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
1097
1301
  hasSentConnectTime = false;
1302
+ /** Whether this connection has sent a client-stats batch yet. The first
1303
+ * one goes out even with no reading, for the environment the bridge adds
1304
+ * to it. Reset with `hasSentConnectTime`. */
1305
+ hasSentClientStats = false;
1098
1306
  /** Set on the "connecting" status transition, cleared once `connectionTimings`
1099
1307
  * is finalized on "ready" — see `handleStatusChanged()`. */
1100
1308
  connectStartTime;
1101
1309
  waitingStartTime;
1310
+ /** The tail of every `pushFrame()` so far, settled either way — pushes
1311
+ * reach the binding in call order however long each one's pixels take to
1312
+ * read, and `step()` waits for the ones called before it. */
1313
+ pendingPushes = Promise.resolve();
1102
1314
  emitter = new Emitter();
1103
1315
  /** Serializes connect()/reconnect()/disconnect() (and the free() inside
1104
1316
  * disconnect()/[Symbol.dispose]) against each other. Calling into the
@@ -1116,6 +1328,15 @@ var Reactor = class {
1116
1328
  * out every read active when it started, before it releases the ones
1117
1329
  * that arrived while it ran. */
1118
1330
  writerActive = false;
1331
+ /** Counts `disconnect()`s and disposal. A push captures it when called, and
1332
+ * one whose frame was still being read when it moved is dropped, rather
1333
+ * than recreating a client to carry the frame into the next session. */
1334
+ connectionEpoch = 0;
1335
+ /** Input tracks published on this connection, each with the publication
1336
+ * that claimed it: a push needs one, and a `PublishedTrack` only feeds the
1337
+ * publication it came from. Cleared with the connection. */
1338
+ publishedInputs = /* @__PURE__ */ new Map();
1339
+ publications = 0;
1119
1340
  /** Readers parked in `withReaderAccess()`'s wait loop, woken once
1120
1341
  * `withWriterAccess()` finishes; each re-checks `writerActive` rather
1121
1342
  * than assuming its turn, since another writer already queued behind the
@@ -1388,14 +1609,49 @@ var Reactor = class {
1388
1609
  * disconnect() freeing the wasm client mid-await would otherwise resume
1389
1610
  * into a freed client — the same class of use-after-free `queue` exists to
1390
1611
  * prevent for connect/disconnect/reconnect themselves.
1612
+ *
1613
+ * Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
1614
+ * track in a stepped session.
1615
+ *
1616
+ * In a `generation: "stepped"` session, input frames never travel over
1617
+ * RTP: `track` is optional, and the frames are pushed with
1618
+ * `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
1619
+ * passed there is ignored, with a console warning — nothing is sent from
1620
+ * it. In a streamed session `track` is required, and omitting it rejects
1621
+ * with an `InvalidStateError`.
1622
+ *
1623
+ * ```ts
1624
+ * const camera = await reactor.publishTrack('camera');
1625
+ * await camera.pushFrame(imageData);
1626
+ * const result = await reactor.step();
1627
+ * ```
1391
1628
  */
1392
1629
  async publishTrack(name, track) {
1393
1630
  this.assertNotDisposed();
1394
1631
  try {
1632
+ let media = track;
1633
+ if (this.isStepped() && media !== void 0) {
1634
+ console.warn(
1635
+ `[Reactor] publishTrack('${name}'): ignoring the MediaStreamTrack \u2014 a stepped session sends no RTP and takes frames through track.pushFrame(...) on the track publishTrack() returns.`
1636
+ );
1637
+ media = void 0;
1638
+ }
1639
+ if (!this.isStepped() && media === void 0) {
1640
+ throw new InvalidStateError(
1641
+ `publishTrack('${name}') needs a MediaStreamTrack in a streamed session.`,
1642
+ { operation: "publishTrack" }
1643
+ );
1644
+ }
1645
+ const publication = this.publications += 1;
1395
1646
  await this.queue.push(async () => {
1396
1647
  const client = await this.getOrCreateClient();
1397
- await client.publishTrack(name, track);
1648
+ await client.publishTrack(name, media);
1649
+ this.publishedInputs.set(name, publication);
1398
1650
  }, "publishTrack");
1651
+ return {
1652
+ name,
1653
+ pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
1654
+ };
1399
1655
  } catch (cause) {
1400
1656
  throw this.captureError(cause);
1401
1657
  }
@@ -1412,6 +1668,7 @@ var Reactor = class {
1412
1668
  await this.queue.push(async () => {
1413
1669
  const client = await this.getOrCreateClient();
1414
1670
  await client.unpublishTrack(name);
1671
+ this.publishedInputs.delete(name);
1415
1672
  }, "unpublishTrack");
1416
1673
  } catch (cause) {
1417
1674
  this.emitError(cause);
@@ -1484,6 +1741,175 @@ var Reactor = class {
1484
1741
  throw this.captureError(cause);
1485
1742
  }
1486
1743
  }
1744
+ // ── Stepped generation ──────────────────────────────────────────────────
1745
+ /**
1746
+ * Runs one step of a `generation: "stepped"` session and resolves with
1747
+ * what it produced, per output track.
1748
+ *
1749
+ * The request carries `state`, `commands` and every frame pushed since the
1750
+ * previous step. The runtime applies them in that order — each `state` key
1751
+ * as the model's `set_<key>` command, then `commands`, then the frames —
1752
+ * and then generates. They are separate calls, not a transaction: one that
1753
+ * fails does not undo the ones before it. All of it is optional:
1754
+ * `step()` with nothing set and nothing pushed runs on the model's current
1755
+ * state, which commands sent outside a step (`sendCommand()`) also change.
1756
+ * A file goes in the way it does for a command: upload it with
1757
+ * `uploadFile()`, then put the `FileRef` in a command's `data` or a state
1758
+ * value.
1759
+ *
1760
+ * A step that ran but that the model reported as failed still resolves,
1761
+ * with `result.error` set. `result.files` lists the files the model kept
1762
+ * with the step (name, content type, size); their bytes are served over
1763
+ * HTTP from the session's step results, not carried here.
1764
+ *
1765
+ * Rejects with a `StepRefusedError` when the runtime or the model turns
1766
+ * the step down (the runtime checks a whole request first, so nothing it
1767
+ * carried was applied), `StepNotReadyError` when the session cannot take a
1768
+ * step yet (neither ran the step), `StepCommandFailedError` when a state key
1769
+ * or command failed (the calls before it stay applied; the step did not
1770
+ * run); both name the part at fault in `entry`, and `StepCommandFailedError`
1771
+ * the handler's code in `command_code`,
1772
+ * `StepSessionEndedError` when the session ended before the result came
1773
+ * back, `StepResultTooLargeError` when the result does not fit in one
1774
+ * message, `MessageTooLargeError` when the request itself is too large
1775
+ * (nothing was sent), `InvalidStateError` outside a ready stepped
1776
+ * session, or whatever else failed the step (`DisconnectedError`,
1777
+ * `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
1778
+ * when its request cannot be sent, they are dropped with it rather than
1779
+ * kept for the next step.
1780
+ *
1781
+ * ```ts
1782
+ * const camera = await reactor.publishTrack('camera');
1783
+ * await camera.pushFrame(imageData);
1784
+ * const result = await reactor.step({ state: { prompt: 'a cat' } });
1785
+ * if (result.error) console.warn('step failed:', result.error);
1786
+ * const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
1787
+ * if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
1788
+ * ```
1789
+ */
1790
+ async step(options = {}) {
1791
+ this.assertNotDisposed();
1792
+ try {
1793
+ const state = rewriteNestedFileRefs(options.state ?? null);
1794
+ const commands = (options.commands ?? []).map(
1795
+ (command) => command.data ? { ...command, data: rewriteNestedFileRefs(command.data) } : command
1796
+ );
1797
+ await this.pendingPushes;
1798
+ return await this.withReaderAccess(async () => {
1799
+ const client = await this.getOrCreateClient();
1800
+ return client.step(state, commands);
1801
+ });
1802
+ } catch (cause) {
1803
+ throw this.captureError(cause);
1804
+ }
1805
+ }
1806
+ /**
1807
+ * Buffers one input frame for the next `step()` on the sendonly track
1808
+ * `trackName`. Nothing is sent until that step, which carries every frame
1809
+ * pushed since the one before it, per track in push order. The
1810
+ * `PublishedTrack` that `publishTrack()` resolves with does the same
1811
+ * through `track.pushFrame(frame)`.
1812
+ *
1813
+ * Video: a `VideoFrame` (cloned when called, so the caller may close it as
1814
+ * soon as this returns; it stays the caller's), an
1815
+ * `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
1816
+ * by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
1817
+ * `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
1818
+ * applied, so a 90° frame arrives with its width and height swapped.
1819
+ * Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
1820
+ * PCM at the rate the model's track declares.
1821
+ *
1822
+ * Pushes keep their call order, and a `step()` waits for every push called
1823
+ * before it. A refused push rejects: await it, or `.catch()` it in a frame
1824
+ * loop, or every bad frame is an unhandled rejection. What waits for
1825
+ * the next step is at most one message's worth of frames: pushing faster
1826
+ * than steps run drops the oldest frames, across tracks. Rejects with
1827
+ * an `InvalidStateError` outside a stepped session, for a track not
1828
+ * published on this connection (`publishTrack(trackName)` first), for a
1829
+ * frame whose size does not match its dimensions, or for a name the model
1830
+ * does not declare as a sendonly track of that kind.
1831
+ */
1832
+ async pushFrame(trackName, frame, options = {}) {
1833
+ return this.pushTo(trackName, void 0, frame, options);
1834
+ }
1835
+ /** `pushFrame()`, for the track's current publication or, from a
1836
+ * `PublishedTrack`, only for the `publication` it came from. */
1837
+ async pushTo(trackName, publication, frame, options = {}) {
1838
+ this.assertNotDisposed();
1839
+ const epoch = this.connectionEpoch;
1840
+ let held;
1841
+ try {
1842
+ held = holdFrame(frame);
1843
+ } catch (cause) {
1844
+ throw this.captureError(cause);
1845
+ }
1846
+ const push = this.pendingPushes.then(async () => {
1847
+ let pixels;
1848
+ try {
1849
+ if (!this.isStepped()) {
1850
+ throw new InvalidStateError(
1851
+ `pushFrame('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
1852
+ { operation: "pushFrame" }
1853
+ );
1854
+ }
1855
+ pixels = isAudioFrame(held.frame) ? void 0 : await toPixels(held.frame);
1856
+ } finally {
1857
+ held.release();
1858
+ }
1859
+ await this.withReaderAccess(async () => {
1860
+ if (epoch !== this.connectionEpoch) {
1861
+ throw new InvalidStateError(
1862
+ `pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
1863
+ { operation: "pushFrame" }
1864
+ );
1865
+ }
1866
+ const current = this.publishedInputs.get(trackName);
1867
+ if (current === void 0 || publication !== void 0 && publication !== current) {
1868
+ throw new InvalidStateError(
1869
+ publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
1870
+ { operation: "pushFrame" }
1871
+ );
1872
+ }
1873
+ const client = await this.getOrCreateClient();
1874
+ if (pixels) {
1875
+ client.pushVideoFrame(
1876
+ trackName,
1877
+ pixels.data,
1878
+ pixels.width,
1879
+ pixels.height,
1880
+ pixels.format,
1881
+ options.metadata ?? null,
1882
+ options.captureTimeUs ?? null
1883
+ );
1884
+ } else if (isAudioFrame(frame)) {
1885
+ client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
1886
+ }
1887
+ });
1888
+ });
1889
+ this.pendingPushes = push.catch(() => void 0);
1890
+ try {
1891
+ await push;
1892
+ } catch (cause) {
1893
+ throw this.captureError(cause);
1894
+ }
1895
+ }
1896
+ /** Drops every frame buffered for the next step, pushes called before it
1897
+ * included: it waits its turn behind them, as a `step()` does. */
1898
+ async clearStepInputs() {
1899
+ this.assertNotDisposed();
1900
+ const clear = this.pendingPushes.then(
1901
+ () => this.withReaderAccess(() => {
1902
+ this.client?.clearStepInputs();
1903
+ return Promise.resolve();
1904
+ })
1905
+ );
1906
+ this.pendingPushes = clear.catch(() => void 0);
1907
+ try {
1908
+ await clear;
1909
+ } catch (cause) {
1910
+ throw this.captureError(cause);
1911
+ }
1912
+ }
1487
1913
  // ── Recording ───────────────────────────────────────────────────────────
1488
1914
  /**
1489
1915
  * Requests a clip covering the last `durationSeconds` of the session.
@@ -1782,6 +2208,7 @@ var Reactor = class {
1782
2208
  const extractConnectionStats = createConnectionStatsExtractor();
1783
2209
  const extractTrackStats = createTrackStatsExtractor();
1784
2210
  this.hasSentConnectTime = false;
2211
+ this.hasSentClientStats = false;
1785
2212
  this.stopStatsPollers = [
1786
2213
  pollPeerStats(client, LOCAL_STATS_INTERVAL_MS, (report) => {
1787
2214
  this.stats = { ...extractConnectionStats(report), connectionTimings: this.connectionTimings };
@@ -1799,8 +2226,9 @@ var Reactor = class {
1799
2226
  * track has one yet (none negotiated onto a named track, or no codec
1800
2227
  * reported yet), so the runtime gets the bandwidth estimate, the
1801
2228
  * round-trip time and the time to connect without waiting on the tracks.
1802
- * Skipped only when there is nothing at all to say; one is due again in
1803
- * `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
2229
+ * Skipped only when there is nothing at all to say and a batch has already
2230
+ * gone out for this connection, so the first one still carries the
2231
+ * environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
1804
2232
  * actually sent for this connection — see `hasSentConnectTime`.
1805
2233
  * `sendClientStats` throws synchronously on a closing connection; caught
1806
2234
  * here for the same reason `getStats()`'s rejection is above. */
@@ -1808,11 +2236,13 @@ var Reactor = class {
1808
2236
  const trackStats = extractTrackStats(report, this.trackMapping(), this.pausedTracks());
1809
2237
  const timeToConnectMs = this.hasSentConnectTime ? void 0 : this.connectionTimings?.totalMs;
1810
2238
  const connectionStat = toClientConnectionStat(report, timeToConnectMs);
1811
- if (trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
2239
+ if (this.hasSentClientStats && trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
1812
2240
  return;
1813
2241
  }
1814
2242
  try {
1815
2243
  client.sendClientStats(trackStats, connectionStat);
2244
+ extractTrackStats.commit();
2245
+ this.hasSentClientStats = true;
1816
2246
  if (timeToConnectMs !== void 0) {
1817
2247
  this.hasSentConnectTime = true;
1818
2248
  }
@@ -1834,6 +2264,8 @@ var Reactor = class {
1834
2264
  * leave `getCapabilities()` returning the previous session's stale
1835
2265
  * tracks/commands until a new `capabilitiesReceived` lands. */
1836
2266
  resetConnectionState() {
2267
+ this.connectionEpoch += 1;
2268
+ this.publishedInputs.clear();
1837
2269
  this.stopStatsPolling();
1838
2270
  this.connectionTimings = void 0;
1839
2271
  this.connectStartTime = void 0;
@@ -1854,6 +2286,9 @@ var Reactor = class {
1854
2286
  emitError(cause) {
1855
2287
  this.emitter.emit("error", this.captureError(cause));
1856
2288
  }
2289
+ isStepped() {
2290
+ return this.clientOptions.generation === "stepped";
2291
+ }
1857
2292
  assertNotDisposed() {
1858
2293
  if (this.disposed) {
1859
2294
  throw new Error("This Reactor was disposed and can't be used again \u2014 construct a new one.");
@@ -1861,6 +2296,25 @@ var Reactor = class {
1861
2296
  }
1862
2297
  };
1863
2298
 
2299
+ // src/step-frames.ts
2300
+ function stepFrameToImageData(frame) {
2301
+ const [height, width, channels] = frame.shape;
2302
+ if (frame.dtype !== "uint8" || frame.shape.length !== 3 || channels !== 3 || height === void 0 || width === void 0 || frame.data.length !== width * height * 3) {
2303
+ throw new Error(
2304
+ `stepFrameToImageData: expected a (height, width, 3) uint8 RGB frame, got ${frame.dtype} [${frame.shape.join(", ")}]`
2305
+ );
2306
+ }
2307
+ const rgb = frame.data;
2308
+ const rgba = new Uint8ClampedArray(width * height * 4);
2309
+ for (let src = 0, dst = 0; src < rgb.length; src += 3, dst += 4) {
2310
+ rgba[dst] = rgb[src];
2311
+ rgba[dst + 1] = rgb[src + 1];
2312
+ rgba[dst + 2] = rgb[src + 2];
2313
+ rgba[dst + 3] = 255;
2314
+ }
2315
+ return new ImageData(rgba, width, height);
2316
+ }
2317
+
1864
2318
  // src/jwt.ts
1865
2319
  function normalizeJwtSource(source) {
1866
2320
  if (typeof source === "function") {
@@ -2810,9 +3264,11 @@ export {
2810
3264
  ClipPlayer,
2811
3265
  ConflictError,
2812
3266
  DEFAULT_PLAYLIST_POLL_SLACK_MS,
3267
+ DataChannelChunkingRequiredError,
2813
3268
  DecodeError,
2814
3269
  DisconnectedError,
2815
3270
  FileRef,
3271
+ GenerationModeMismatchError,
2816
3272
  InvalidStateError,
2817
3273
  MessageTooLargeError,
2818
3274
  NetworkError,
@@ -2827,8 +3283,14 @@ export {
2827
3283
  RequestTimeoutError,
2828
3284
  ServerError,
2829
3285
  SessionTerminalError,
3286
+ StepCommandFailedError,
3287
+ StepNotReadyError,
3288
+ StepRefusedError,
3289
+ StepResultTooLargeError,
3290
+ StepSessionEndedError,
2830
3291
  TransportError,
2831
3292
  UnauthorizedError,
3293
+ UnsupportedFrameEncodingError,
2832
3294
  VersionMismatchError,
2833
3295
  WebcamStream,
2834
3296
  createPlayableManifestUrl,
@@ -2837,6 +3299,7 @@ export {
2837
3299
  isFileRef,
2838
3300
  normalizeJwtSource,
2839
3301
  parsePlaylist,
3302
+ stepFrameToImageData,
2840
3303
  useClipDownload,
2841
3304
  useReactor,
2842
3305
  useReactorInternalMessage,