@reactor-team/js-sdk 3.2.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -47,7 +47,8 @@ rather than `any`:
47
47
  | Lifecycle | `connect(options?)`, `disconnect()`, `reconnect()` |
48
48
  | Configuration | `setJwt(jwt)` |
49
49
  | Messaging | `sendCommand(command, data?, uploads?)`, `requestSchema()` |
50
- | Tracks | `publishTrack(name, track)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
50
+ | Tracks | `publishTrack(name, track?)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
51
+ | Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
51
52
  | Recording | `requestClip(seconds)`, `requestRecording()` |
52
53
  | Uploads | `uploadFile(file, name?)` |
53
54
  | Introspection | `status()`, `sessionId()`, `sessionInfo()`, `capabilities()`, `lastError()` |
@@ -56,7 +57,39 @@ rather than `any`:
56
57
 
57
58
  Payloads that come out of the core keep their wire names (`protocol_version`,
58
59
  `retry_after_ms`, `playlist_url`); the options this crate defines itself are
59
- camelCase, because nothing on the wire constrains them.
60
+ camelCase, because nothing on the wire constrains them. So is a step's result:
61
+ it is built by hand rather than serialized, because its frame data are typed
62
+ arrays (`Uint8Array` video, `Int16Array` audio) and serde has no word for one.
63
+
64
+ A client created with `generation: "stepped"` sends no input media over RTP.
65
+ `publishTrack(name)` only claims an input track (a `MediaStreamTrack` passed
66
+ there is ignored, with a console warning), `pushVideoFrame` / `pushAudioFrame`
67
+ buffer frames in the core, and the next `step()` carries them inside its one
68
+ `RequestStep`. Both of `step()`'s arguments are optional: an empty step runs on
69
+ the model's current state. Video arrives as RGBA, RGB or BGRA, and the core
70
+ converts it to the RGB a step sends in one pass.
71
+
72
+ The buffer holds at most one data-channel message's worth of frames: pushing
73
+ faster than steps run drops the oldest frames, across tracks, and the next step
74
+ logs how many. Pushed frames belong to the step that takes them: when that
75
+ step's request cannot be sent — `MESSAGE_TOO_LARGE` among the reasons — the
76
+ step rejects and its frames are dropped with it, so the next step starts from
77
+ what is pushed after. `clearStepInputs()` drops whatever is buffered now.
78
+
79
+ `step()` resolves with `{ step, tracks, elapsed, files, error? }`. Each track
80
+ is `{ track, kind, rate, frames }`, `rate` being the rate its frames play at as
81
+ the runtime reported it (frames per second for video, samples per second for
82
+ audio; 0 when it did not say). `files`
83
+ describes the files the model kept with the step (`name`, `contentType`,
84
+ `size`); their bytes are served over HTTP from the session's step results.
85
+ `error` is set when the step ran and the model reported a failure — such a
86
+ step still resolves. A step that did not run rejects with `step_refused`
87
+ (nothing it carried was applied), `step_not_ready` (the session cannot take a
88
+ step yet) or `step_command_failed` (a state key or command failed; the calls
89
+ before it stay applied). Both of those carry `entry`, the part of the request
90
+ at fault, and `step_command_failed` also `command_code`, its handler's code. One whose result was too large for a message
91
+ rejects with `step_result_too_large`, and one whose session ended before the
92
+ result came back with `step_session_ended`.
60
93
 
61
94
  ```js
62
95
  import init, { ReactorClient } from "./pkg/reactor_wasm.js";
@@ -4,6 +4,9 @@
4
4
  export type ReactorStatus = "disconnected" | "connecting" | "waiting" | "ready";
5
5
  export type TrackKind = "audio" | "video";
6
6
  export type TrackDirection = "recvonly" | "sendonly";
7
+ export type GenerationMode = "streamed" | "stepped";
8
+ /** Byte layout of a frame given to `pushVideoFrame`. */
9
+ export type PixelFormat = "rgba" | "rgb" | "bgra";
7
10
 
8
11
  /** Construction options. Only `modelName` is required. */
9
12
  export interface ClientOptions {
@@ -39,6 +42,14 @@ export interface ClientOptions {
39
42
  /** Preset tracks, when known ahead of time — builds the SDP offer
40
43
  * concurrently with the session-ready poll rather than after. */
41
44
  modelTracks?: TrackCapability[];
45
+ /** How the session generates: "streamed" (the default), continuously over
46
+ * media tracks, or "stepped", one `step()` at a time. `connect()` fails
47
+ * with GENERATION_MODE_MISMATCH when the runtime did not start the
48
+ * session in this mode. */
49
+ generation?: GenerationMode;
50
+ /** How long `step()` waits for a step's result, generation included.
51
+ * Default 300000. */
52
+ stepTimeoutMs?: number;
42
53
  /** Console log level. Default "warn". */
43
54
  logLevel?: "off" | "error" | "warn" | "info" | "debug" | "trace";
44
55
  }
@@ -149,6 +160,11 @@ export interface ReactorError {
149
160
  operation?: string;
150
161
  /** Backoff hint the platform sent. */
151
162
  retry_after_ms?: number;
163
+ /** For `step_refused` and `step_command_failed`, the part of the step
164
+ * request at fault, e.g. `"commands[2]"`. */
165
+ entry?: string;
166
+ /** For `step_command_failed`, the code the failing handler raised. */
167
+ command_code?: string;
152
168
  timestamp_ms: number;
153
169
  }
154
170
 
@@ -170,6 +186,31 @@ export interface ClientTrackStat {
170
186
  * frame_width, frame_height, nack_count, keyframe_requests. A reading the
171
187
  * client couldn't take is left out, never sent as 0. */
172
188
  metrics: Record<string, number>;
189
+ /** Where this track's frames spent their time since the previous batch,
190
+ * one entry per stage (e.g. "encode", "jitter_buffer", "decode"). Optional;
191
+ * an entry with no frames or a time that is not a finite, non-negative
192
+ * number is left out. */
193
+ frameStages?: FrameStage[];
194
+ /** Video only: the encoder (a sent track) or decoder (a received one) the
195
+ * browser runs, from getStats()'s encoderImplementation /
196
+ * decoderImplementation, e.g. "libvpx" or "ExternalDecoder". */
197
+ codecImplementation?: string;
198
+ /** Video only: whether that implementation is the hardware one, from
199
+ * powerEfficientEncoder / powerEfficientDecoder. */
200
+ hardwareCodec?: boolean;
201
+ }
202
+
203
+ /** The time a track's frames spent in one stage of their trip, over one
204
+ * batch's window. The average per frame is `totalMs / frames`.
205
+ * encode_wait, packetize and pacer are different: they come from the
206
+ * engine's timing frames, a sample of about one slow frame per second, so
207
+ * `frames` counts sampled frames and `totalMs / frames` leans toward the
208
+ * slow ones. Treat their average as an upper bound, not a mean. */
209
+ export interface FrameStage {
210
+ name: string;
211
+ /** The time all `frames` frames spent in the stage together. */
212
+ totalMs: number;
213
+ frames: number;
173
214
  }
174
215
 
175
216
  /** A connection-wide reading, not tied to any one track — for
@@ -182,6 +223,77 @@ export interface ClientConnectionStat {
182
223
  metrics: Record<string, number>;
183
224
  }
184
225
 
226
+ /** One command of a step, run after its state, in order. */
227
+ export interface StepCommand {
228
+ command: string;
229
+ /** The command's arguments. Omitted or null for none. */
230
+ data?: Record<string, unknown> | null;
231
+ }
232
+
233
+ /** One frame a step produced. Hand-built by the binding rather than
234
+ * serialized, so camelCase, unlike the wire payloads above. */
235
+ export interface StepFrame {
236
+ /** How the frame travelled; "raw" is the only one decoded today. */
237
+ encoding: "raw" | "unspecified";
238
+ /** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
239
+ dtype: string;
240
+ /** (height, width, 3) RGB for video, (channels, samples) planar PCM for
241
+ * audio. */
242
+ shape: number[];
243
+ /** The array, row-major, as the typed array `dtype` names — a Uint8Array
244
+ * for video, an Int16Array for audio, raw bytes for a dtype with no
245
+ * typed-array counterpart. */
246
+ data:
247
+ | Uint8Array
248
+ | Int8Array
249
+ | Uint16Array
250
+ | Int16Array
251
+ | Uint32Array
252
+ | Int32Array
253
+ | Float32Array
254
+ | Float64Array;
255
+ /** What the model attached to the frame. */
256
+ metadata?: Uint8Array;
257
+ /** When the model says the frame was captured, in microseconds. */
258
+ captureTimeUs?: number;
259
+ }
260
+
261
+ /** The frames one output track produced in a step, in production order. */
262
+ export interface StepTrack {
263
+ track: string;
264
+ kind: TrackKind;
265
+ /** The rate the track's frames play at, as the runtime reported it:
266
+ * frames per second for video, samples per second for audio. 0 when the
267
+ * runtime did not say. */
268
+ rate: number;
269
+ frames: StepFrame[];
270
+ }
271
+
272
+ /** A file the model kept with a step. */
273
+ export interface StepFile {
274
+ name: string;
275
+ /** Its MIME type, e.g. "image/png". */
276
+ contentType: string;
277
+ /** Its size in bytes. */
278
+ size: number;
279
+ }
280
+
281
+ /** What one step produced. */
282
+ export interface StepResult {
283
+ /** The step's number in the session, counted from 1. */
284
+ index: number;
285
+ tracks: StepTrack[];
286
+ /** Seconds the step's `generate()` took, or the time the model reported for the step. */
287
+ elapsed: number;
288
+ /** Files the model kept with the step. Only their description: the bytes
289
+ * are served over HTTP from the session's step results. Empty when the
290
+ * model kept none or the result was not saved. */
291
+ files: StepFile[];
292
+ /** Set when the step ran and the model reported a failure. Such a step
293
+ * still resolves; only a refused step rejects. */
294
+ error?: string;
295
+ }
296
+
185
297
  export type StatusListener = (status: ReactorStatus) => void;
186
298
  export type SessionIdListener = (sessionId: string | undefined) => void;
187
299
  export type MessageListener = (message: ReactorMessage) => void;
@@ -214,6 +326,10 @@ export class ReactorClient {
214
326
  * The runtime's capabilities, or `undefined` before they arrive.
215
327
  */
216
328
  capabilities(): Capabilities | undefined;
329
+ /**
330
+ * Drop every frame buffered for the next step.
331
+ */
332
+ clearStepInputs(): void;
217
333
  /**
218
334
  * Create (or adopt) a session and bring up the transport.
219
335
  */
@@ -305,8 +421,32 @@ export class ReactorClient {
305
421
  pausedTracks(): string[];
306
422
  /**
307
423
  * Claim a sendonly track and start sending `track` on it.
424
+ *
425
+ * In a stepped session the track has no RTP sender, so there is nothing
426
+ * to attach a `MediaStreamTrack` to: `publishTrack(name)` only claims the
427
+ * track, and its frames come from `pushVideoFrame` / `pushAudioFrame`.
428
+ * A `MediaStreamTrack` passed there is ignored, with a warning.
429
+ */
430
+ publishTrack(name: string, track?: MediaStreamTrack | null): Promise<void>;
431
+ /**
432
+ * Buffer a block of audio for the next `step()` on the sendonly track
433
+ * `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
434
+ * already at the rate the model's track declares (a step's frame carries
435
+ * no sample rate). Stepped sessions only, as `pushVideoFrame`.
308
436
  */
309
- publishTrack(name: string, track: MediaStreamTrack): Promise<void>;
437
+ pushAudioFrame(name: string, samples: Int16Array, sample_rate: number, channels: number): void;
438
+ /**
439
+ * Buffer a video frame for the next `step()` on the sendonly track `name`.
440
+ *
441
+ * * `data` — `width * height` pixels in `format`: `"rgba"` (the default,
442
+ * what `ImageData` holds), `"rgb"` or `"bgra"`. Sent as RGB.
443
+ * * `metadata` — opaque bytes the frame carries to the model.
444
+ * * `captureTimeUs` — when the frame was captured, in microseconds.
445
+ *
446
+ * Stepped sessions only: a streamed session sends media from a
447
+ * `MediaStreamTrack` given to `publishTrack`.
448
+ */
449
+ pushVideoFrame(name: string, data: Uint8Array, width: number, height: number, format?: PixelFormat | null, metadata?: Uint8Array | null, capture_time_us?: number | null): void;
310
450
  /**
311
451
  * Rebuild the transport on the same session, without ending it.
312
452
  *
@@ -387,6 +527,29 @@ export class ReactorClient {
387
527
  * `"disconnected"` | `"connecting"` | `"waiting"` | `"ready"`.
388
528
  */
389
529
  status(): ReactorStatus;
530
+ /**
531
+ * Run one step of a stepped session and resolve with what it produced.
532
+ *
533
+ * * `state` — an object of the model's state fields, each applied as its
534
+ * `set_<key>` command before `commands`; `null`/`undefined` for none.
535
+ * * `commands` — `[{ command, data? }]`, run in order after the state.
536
+ *
537
+ * Every frame pushed since the previous step rides this one, and belongs
538
+ * to it: when the request cannot be sent the step fails and its frames
539
+ * are dropped, not kept for the next step. Rejects with the core's code:
540
+ * `step_refused` (the runtime checks a whole request first, so a refused
541
+ * step changed nothing; `entry` names the part at fault),
542
+ * `step_not_ready` (the session cannot take a step yet; the step did not
543
+ * run), `step_command_failed` (a state key or command failed; the calls
544
+ * before it stay applied and the step did not run; `entry` names the call
545
+ * and `command_code` its handler's code), `step_session_ended`
546
+ * (the session ended before the result came back),
547
+ * `step_result_too_large`, `MESSAGE_TOO_LARGE` (the request is over the
548
+ * data channel's message limit; nothing was sent),
549
+ * `UNSUPPORTED_FRAME_ENCODING`, or the code of whatever else failed the
550
+ * step (`DISCONNECTED`, `REQUEST_TIMEOUT`, …).
551
+ */
552
+ step(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): Promise<StepResult>;
390
553
  /**
391
554
  * The negotiated `name` → `mid` mapping: `[{ name, kind, direction, mid }]`.
392
555
  */
@@ -415,6 +578,7 @@ export interface InitOutput {
415
578
  readonly memory: WebAssembly.Memory;
416
579
  readonly __wbg_reactorclient_free: (a: number, b: number) => void;
417
580
  readonly reactorclient_capabilities: (a: number) => [number, number, number];
581
+ readonly reactorclient_clearStepInputs: (a: number) => void;
418
582
  readonly reactorclient_connect: (a: number, b: number) => any;
419
583
  readonly reactorclient_disconnect: (a: number) => any;
420
584
  readonly reactorclient_getPeerConnection: (a: number) => any;
@@ -433,7 +597,9 @@ export interface InitOutput {
433
597
  readonly reactorclient_onTrackReceived: (a: number, b: any) => void;
434
598
  readonly reactorclient_pauseTrack: (a: number, b: number, c: number) => any;
435
599
  readonly reactorclient_pausedTracks: (a: number) => [number, number, number];
436
- readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: any) => any;
600
+ readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: number) => any;
601
+ readonly reactorclient_pushAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
602
+ readonly reactorclient_pushVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
437
603
  readonly reactorclient_reconnect: (a: number, b: number) => any;
438
604
  readonly reactorclient_requestClip: (a: number, b: number) => any;
439
605
  readonly reactorclient_requestRecording: (a: number) => any;
@@ -446,6 +612,7 @@ export interface InitOutput {
446
612
  readonly reactorclient_setJwt: (a: number, b: number) => [number, number];
447
613
  readonly reactorclient_setTrackBitrate: (a: number, b: number, c: number, d: number, e: number) => any;
448
614
  readonly reactorclient_status: (a: number) => any;
615
+ readonly reactorclient_step: (a: number, b: number, c: number) => any;
449
616
  readonly reactorclient_trackMapping: (a: number) => [number, number, number];
450
617
  readonly reactorclient_tracks: (a: number) => [number, number, number];
451
618
  readonly reactorclient_unpublishTrack: (a: number, b: number, c: number) => any;