@reactor-team/js-sdk 3.2.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -47,7 +47,8 @@ rather than `any`:
47
47
  | Lifecycle | `connect(options?)`, `disconnect()`, `reconnect()` |
48
48
  | Configuration | `setJwt(jwt)` |
49
49
  | Messaging | `sendCommand(command, data?, uploads?)`, `requestSchema()` |
50
- | Tracks | `publishTrack(name, track)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
50
+ | Tracks | `publishTrack(name, track?)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
51
+ | Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushEncodedVideoFrame(name, codec, data, width, height, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
51
52
  | Recording | `requestClip(seconds)`, `requestRecording()` |
52
53
  | Uploads | `uploadFile(file, name?)` |
53
54
  | Introspection | `status()`, `sessionId()`, `sessionInfo()`, `capabilities()`, `lastError()` |
@@ -56,7 +57,42 @@ rather than `any`:
56
57
 
57
58
  Payloads that come out of the core keep their wire names (`protocol_version`,
58
59
  `retry_after_ms`, `playlist_url`); the options this crate defines itself are
59
- camelCase, because nothing on the wire constrains them.
60
+ camelCase, because nothing on the wire constrains them. So is a step's result:
61
+ it is built by hand rather than serialized, because its frame data are typed
62
+ arrays (`Uint8Array` video, `Int16Array` audio) and serde has no word for one.
63
+
64
+ A client created with `generation: "stepped"` sends no input media over RTP.
65
+ `publishTrack(name)` only claims an input track (a `MediaStreamTrack` passed
66
+ there is ignored, with a console warning), `pushVideoFrame` / `pushAudioFrame`
67
+ buffer frames in the core, and the next `step()` carries them inside its one
68
+ `RequestStep`. Both of `step()`'s arguments are optional: an empty step runs on
69
+ the model's current state. Video arrives as RGBA, RGB or BGRA, and the core
70
+ converts it to the RGB a step sends in one pass. `pushEncodedVideoFrame` buffers
71
+ an H.264 access unit (Annex B) as it is, with the size it decodes to; the runtime
72
+ decodes a track's encoded frames of one step, in push order and starting at a
73
+ keyframe, before the step runs.
74
+
75
+ The buffer holds at most one data-channel message's worth of frames: pushing
76
+ faster than steps run drops the oldest frames, across tracks, and the next step
77
+ logs how many. Pushed frames belong to the step that takes them: when that
78
+ step's request cannot be sent — `MESSAGE_TOO_LARGE` among the reasons — the
79
+ step rejects and its frames are dropped with it, so the next step starts from
80
+ what is pushed after. `clearStepInputs()` drops whatever is buffered now.
81
+
82
+ `step()` resolves with `{ step, tracks, elapsed, files, error? }`. Each track
83
+ is `{ track, kind, rate, frames }`, `rate` being the rate its frames play at as
84
+ the runtime reported it (frames per second for video, samples per second for
85
+ audio; 0 when it did not say). `files`
86
+ describes the files the model kept with the step (`name`, `contentType`,
87
+ `size`); their bytes are served over HTTP from the session's step results.
88
+ `error` is set when the step ran and the model reported a failure — such a
89
+ step still resolves. A step that did not run rejects with `step_refused`
90
+ (nothing it carried was applied), `step_not_ready` (the session cannot take a
91
+ step yet) or `step_command_failed` (a state key or command failed; the calls
92
+ before it stay applied). Both of those carry `entry`, the part of the request
93
+ at fault, and `step_command_failed` also `command_code`, its handler's code. One whose result was too large for a message
94
+ rejects with `step_result_too_large`, and one whose session ended before the
95
+ result came back with `step_session_ended`.
60
96
 
61
97
  ```js
62
98
  import init, { ReactorClient } from "./pkg/reactor_wasm.js";
@@ -4,6 +4,11 @@
4
4
  export type ReactorStatus = "disconnected" | "connecting" | "waiting" | "ready";
5
5
  export type TrackKind = "audio" | "video";
6
6
  export type TrackDirection = "recvonly" | "sendonly";
7
+ export type GenerationMode = "streamed" | "stepped";
8
+ /** Byte layout of a frame given to `pushVideoFrame`. */
9
+ export type PixelFormat = "rgba" | "rgb" | "bgra";
10
+ /** Codec of an encoded frame given to `pushEncodedVideoFrame`. */
11
+ export type EncodedVideoCodec = "h264";
7
12
 
8
13
  /** Construction options. Only `modelName` is required. */
9
14
  export interface ClientOptions {
@@ -39,6 +44,22 @@ export interface ClientOptions {
39
44
  /** Preset tracks, when known ahead of time — builds the SDP offer
40
45
  * concurrently with the session-ready poll rather than after. */
41
46
  modelTracks?: TrackCapability[];
47
+ /** How the session generates: "streamed" (the default), continuously over
48
+ * media tracks, or "stepped", one `step()` at a time. `connect()` fails
49
+ * with GENERATION_MODE_MISMATCH when the runtime did not start the
50
+ * session in this mode. */
51
+ generation?: GenerationMode;
52
+ /** How long `step()` waits for a step's result, generation included.
53
+ * Default 300000. */
54
+ stepTimeoutMs?: number;
55
+ /** The largest data-channel message, in bytes, the client declares and
56
+ * accepts: from 262144 (256 KiB) to 1073741824 (1 GiB). Default 67108864
57
+ * (64 MiB). A connection carries the smaller of this and the runtime's own
58
+ * limit, which is 64 MiB unless the model raises it, so a `step()` whose
59
+ * result is larger than either fails with `StepResultTooLargeError`. A
60
+ * message is held whole in memory while it arrives, so raise this only for
61
+ * a model whose steps need it. */
62
+ maxMessageSize?: number;
42
63
  /** Console log level. Default "warn". */
43
64
  logLevel?: "off" | "error" | "warn" | "info" | "debug" | "trace";
44
65
  }
@@ -149,6 +170,11 @@ export interface ReactorError {
149
170
  operation?: string;
150
171
  /** Backoff hint the platform sent. */
151
172
  retry_after_ms?: number;
173
+ /** For `step_refused` and `step_command_failed`, the part of the step
174
+ * request at fault, e.g. `"commands[2]"`. */
175
+ entry?: string;
176
+ /** For `step_command_failed`, the code the failing handler raised. */
177
+ command_code?: string;
152
178
  timestamp_ms: number;
153
179
  }
154
180
 
@@ -170,6 +196,31 @@ export interface ClientTrackStat {
170
196
  * frame_width, frame_height, nack_count, keyframe_requests. A reading the
171
197
  * client couldn't take is left out, never sent as 0. */
172
198
  metrics: Record<string, number>;
199
+ /** Where this track's frames spent their time since the previous batch,
200
+ * one entry per stage (e.g. "encode", "jitter_buffer", "decode"). Optional;
201
+ * an entry with no frames or a time that is not a finite, non-negative
202
+ * number is left out. */
203
+ frameStages?: FrameStage[];
204
+ /** Video only: the encoder (a sent track) or decoder (a received one) the
205
+ * browser runs, from getStats()'s encoderImplementation /
206
+ * decoderImplementation, e.g. "libvpx" or "ExternalDecoder". */
207
+ codecImplementation?: string;
208
+ /** Video only: whether that implementation is the hardware one, from
209
+ * powerEfficientEncoder / powerEfficientDecoder. */
210
+ hardwareCodec?: boolean;
211
+ }
212
+
213
+ /** The time a track's frames spent in one stage of their trip, over one
214
+ * batch's window. The average per frame is `totalMs / frames`.
215
+ * encode_wait, packetize and pacer are different: they come from the
216
+ * engine's timing frames, a sample of about one slow frame per second, so
217
+ * `frames` counts sampled frames and `totalMs / frames` leans toward the
218
+ * slow ones. Treat their average as an upper bound, not a mean. */
219
+ export interface FrameStage {
220
+ name: string;
221
+ /** The time all `frames` frames spent in the stage together. */
222
+ totalMs: number;
223
+ frames: number;
173
224
  }
174
225
 
175
226
  /** A connection-wide reading, not tied to any one track — for
@@ -182,6 +233,77 @@ export interface ClientConnectionStat {
182
233
  metrics: Record<string, number>;
183
234
  }
184
235
 
236
+ /** One command of a step, run after its state, in order. */
237
+ export interface StepCommand {
238
+ command: string;
239
+ /** The command's arguments. Omitted or null for none. */
240
+ data?: Record<string, unknown> | null;
241
+ }
242
+
243
+ /** One frame a step produced. Hand-built by the binding rather than
244
+ * serialized, so camelCase, unlike the wire payloads above. */
245
+ export interface StepFrame {
246
+ /** How the frame travelled; "raw" is the only one decoded today. */
247
+ encoding: "raw" | "unspecified";
248
+ /** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
249
+ dtype: string;
250
+ /** (height, width, 3) RGB for video, (channels, samples) planar PCM for
251
+ * audio. */
252
+ shape: number[];
253
+ /** The array, row-major, as the typed array `dtype` names — a Uint8Array
254
+ * for video, an Int16Array for audio, raw bytes for a dtype with no
255
+ * typed-array counterpart. */
256
+ data:
257
+ | Uint8Array
258
+ | Int8Array
259
+ | Uint16Array
260
+ | Int16Array
261
+ | Uint32Array
262
+ | Int32Array
263
+ | Float32Array
264
+ | Float64Array;
265
+ /** What the model attached to the frame. */
266
+ metadata?: Uint8Array;
267
+ /** When the model says the frame was captured, in microseconds. */
268
+ captureTimeUs?: number;
269
+ }
270
+
271
+ /** The frames one output track produced in a step, in production order. */
272
+ export interface StepTrack {
273
+ track: string;
274
+ kind: TrackKind;
275
+ /** The rate the track's frames play at, as the runtime reported it:
276
+ * frames per second for video, samples per second for audio. 0 when the
277
+ * runtime did not say. */
278
+ rate: number;
279
+ frames: StepFrame[];
280
+ }
281
+
282
+ /** A file the model kept with a step. */
283
+ export interface StepFile {
284
+ name: string;
285
+ /** Its MIME type, e.g. "image/png". */
286
+ contentType: string;
287
+ /** Its size in bytes. */
288
+ size: number;
289
+ }
290
+
291
+ /** What one step produced. */
292
+ export interface StepResult {
293
+ /** The step's number in the session, counted from 1. */
294
+ index: number;
295
+ tracks: StepTrack[];
296
+ /** Seconds the step's `generate()` took, or the time the model reported for the step. */
297
+ elapsed: number;
298
+ /** Files the model kept with the step. Only their description: the bytes
299
+ * are served over HTTP from the session's step results. Empty when the
300
+ * model kept none or the result was not saved. */
301
+ files: StepFile[];
302
+ /** Set when the step ran and the model reported a failure. Such a step
303
+ * still resolves; only a refused step rejects. */
304
+ error?: string;
305
+ }
306
+
185
307
  export type StatusListener = (status: ReactorStatus) => void;
186
308
  export type SessionIdListener = (sessionId: string | undefined) => void;
187
309
  export type MessageListener = (message: ReactorMessage) => void;
@@ -214,6 +336,34 @@ export class ReactorClient {
214
336
  * The runtime's capabilities, or `undefined` before they arrive.
215
337
  */
216
338
  capabilities(): Capabilities | undefined;
339
+ /**
340
+ * Whether `pushAudioFrame` would refuse `samples`: throws what it would
341
+ * throw, and buffers nothing.
342
+ */
343
+ checkAudioFrame(name: string, samples: Int16Array, channels: number): void;
344
+ /**
345
+ * Whether `pushEncodedVideoFrame` would refuse a frame of `byteLength`
346
+ * bytes: throws what it would throw, and buffers nothing. A length, as
347
+ * for `checkVideoFrame`.
348
+ */
349
+ checkEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, byte_length: number, width: number, height: number, capture_time_us?: number | null): void;
350
+ /**
351
+ * Whether `step(state, commands)` would be refused now, for the session or
352
+ * for its state and commands: throws what `step()` would throw, and takes
353
+ * no frames. `step()`'s own checks, run before a step's media is pushed,
354
+ * so a refused step leaves the buffer as the caller left it.
355
+ */
356
+ checkStep(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): void;
357
+ /**
358
+ * Whether `pushVideoFrame` would refuse a frame of `byteLength` bytes:
359
+ * throws what it would throw, and buffers nothing. The frame's length
360
+ * rather than its bytes, so a frame is not copied into wasm memory twice.
361
+ */
362
+ checkVideoFrame(name: string, byte_length: number, width: number, height: number, format?: PixelFormat | null): void;
363
+ /**
364
+ * Drop every frame buffered for the next step.
365
+ */
366
+ clearStepInputs(): void;
217
367
  /**
218
368
  * Create (or adopt) a session and bring up the transport.
219
369
  */
@@ -305,8 +455,49 @@ export class ReactorClient {
305
455
  pausedTracks(): string[];
306
456
  /**
307
457
  * Claim a sendonly track and start sending `track` on it.
458
+ *
459
+ * In a stepped session the track has no RTP sender, so there is nothing
460
+ * to attach a `MediaStreamTrack` to: `publishTrack(name)` only claims the
461
+ * track, and its frames come from `pushVideoFrame` / `pushAudioFrame`.
462
+ * A `MediaStreamTrack` passed there is ignored, with a warning.
308
463
  */
309
- publishTrack(name: string, track: MediaStreamTrack): Promise<void>;
464
+ publishTrack(name: string, track?: MediaStreamTrack | null): Promise<void>;
465
+ /**
466
+ * Buffer a block of audio for the next `step()` on the sendonly track
467
+ * `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
468
+ * already at the rate the model's track declares (a step's frame carries
469
+ * no sample rate). Stepped sessions only, as `pushVideoFrame`.
470
+ */
471
+ pushAudioFrame(name: string, samples: Int16Array, sample_rate: number, channels: number): void;
472
+ /**
473
+ * Buffer an encoded video frame for the next `step()` on the sendonly
474
+ * video track `name`, its bytes as they are.
475
+ *
476
+ * * `codec` — `"h264"` (the default): `data` is one H.264 access unit,
477
+ * Annex B.
478
+ * * `width`, `height` — the frame it decodes to.
479
+ * * `metadata`, `captureTimeUs` — as for `pushVideoFrame`, carried with
480
+ * the decoded frame.
481
+ *
482
+ * The runtime decodes a track's encoded frames of one step as one
483
+ * stream before the step runs, so push them in decode order, starting at
484
+ * a keyframe; a step whose frames do not decode is refused
485
+ * (`step_refused`). Encoded and raw pushes to one track keep their push
486
+ * order. Stepped sessions only; throws on empty data or a zero dimension.
487
+ */
488
+ pushEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, data: Uint8Array, width: number, height: number, metadata?: Uint8Array | null, capture_time_us?: number | null): void;
489
+ /**
490
+ * Buffer a video frame for the next `step()` on the sendonly track `name`.
491
+ *
492
+ * * `data` — `width * height` pixels in `format`: `"rgba"` (the default,
493
+ * what `ImageData` holds), `"rgb"` or `"bgra"`. Sent as RGB.
494
+ * * `metadata` — opaque bytes the frame carries to the model.
495
+ * * `captureTimeUs` — when the frame was captured, in microseconds.
496
+ *
497
+ * Stepped sessions only: a streamed session sends media from a
498
+ * `MediaStreamTrack` given to `publishTrack`.
499
+ */
500
+ pushVideoFrame(name: string, data: Uint8Array, width: number, height: number, format?: PixelFormat | null, metadata?: Uint8Array | null, capture_time_us?: number | null): void;
310
501
  /**
311
502
  * Rebuild the transport on the same session, without ending it.
312
503
  *
@@ -387,6 +578,33 @@ export class ReactorClient {
387
578
  * `"disconnected"` | `"connecting"` | `"waiting"` | `"ready"`.
388
579
  */
389
580
  status(): ReactorStatus;
581
+ /**
582
+ * Run one step of a stepped session and resolve with what it produced.
583
+ *
584
+ * * `state` — an object of the model's state fields, each applied as its
585
+ * `set_<key>` command before `commands`; `null`/`undefined` for none.
586
+ * * `commands` — `[{ command, data? }]`, run in order after the state.
587
+ *
588
+ * Every frame pushed since the previous step rides this one, and belongs
589
+ * to it: when the request cannot be sent the step fails and its frames
590
+ * are dropped, not kept for the next step. Rejects with the core's code:
591
+ * `step_refused` (the runtime checks a whole request first, so a refused
592
+ * step changed nothing; `entry` names the part at fault),
593
+ * `step_not_ready` (the session cannot take a step yet; the step did not
594
+ * run), `step_command_failed` (a state key or command failed; the calls
595
+ * before it stay applied and the step did not run; `entry` names the call
596
+ * and `command_code` its handler's code), `step_session_ended`
597
+ * (the session ended before the result came back),
598
+ * `step_result_too_large`, `MESSAGE_TOO_LARGE` (the request is over the
599
+ * data channel's message limit; nothing was sent),
600
+ * `UNSUPPORTED_FRAME_ENCODING`, or the code of whatever else failed the
601
+ * step (`DISCONNECTED`, `REQUEST_TIMEOUT`, …).
602
+ *
603
+ * The step is checked and its frames taken before this returns, so a
604
+ * frame pushed once it has returned goes to the next step. A step refused
605
+ * by those checks throws rather than rejecting.
606
+ */
607
+ step(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): Promise<StepResult>;
390
608
  /**
391
609
  * The negotiated `name` → `mid` mapping: `[{ name, kind, direction, mid }]`.
392
610
  */
@@ -415,6 +633,11 @@ export interface InitOutput {
415
633
  readonly memory: WebAssembly.Memory;
416
634
  readonly __wbg_reactorclient_free: (a: number, b: number) => void;
417
635
  readonly reactorclient_capabilities: (a: number) => [number, number, number];
636
+ readonly reactorclient_checkAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number) => [number, number];
637
+ readonly reactorclient_checkEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => [number, number];
638
+ readonly reactorclient_checkStep: (a: number, b: number, c: number) => [number, number];
639
+ readonly reactorclient_checkVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
640
+ readonly reactorclient_clearStepInputs: (a: number) => void;
418
641
  readonly reactorclient_connect: (a: number, b: number) => any;
419
642
  readonly reactorclient_disconnect: (a: number) => any;
420
643
  readonly reactorclient_getPeerConnection: (a: number) => any;
@@ -433,7 +656,10 @@ export interface InitOutput {
433
656
  readonly reactorclient_onTrackReceived: (a: number, b: any) => void;
434
657
  readonly reactorclient_pauseTrack: (a: number, b: number, c: number) => any;
435
658
  readonly reactorclient_pausedTracks: (a: number) => [number, number, number];
436
- readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: any) => any;
659
+ readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: number) => any;
660
+ readonly reactorclient_pushAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
661
+ readonly reactorclient_pushEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
662
+ readonly reactorclient_pushVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
437
663
  readonly reactorclient_reconnect: (a: number, b: number) => any;
438
664
  readonly reactorclient_requestClip: (a: number, b: number) => any;
439
665
  readonly reactorclient_requestRecording: (a: number) => any;
@@ -446,6 +672,7 @@ export interface InitOutput {
446
672
  readonly reactorclient_setJwt: (a: number, b: number) => [number, number];
447
673
  readonly reactorclient_setTrackBitrate: (a: number, b: number, c: number, d: number, e: number) => any;
448
674
  readonly reactorclient_status: (a: number) => any;
675
+ readonly reactorclient_step: (a: number, b: number, c: number) => [number, number, number];
449
676
  readonly reactorclient_trackMapping: (a: number) => [number, number, number];
450
677
  readonly reactorclient_tracks: (a: number) => [number, number, number];
451
678
  readonly reactorclient_unpublishTrack: (a: number, b: number, c: number) => any;