@reactor-team/js-sdk 3.3.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -48,7 +48,8 @@ rather than `any`:
48
48
  | Configuration | `setJwt(jwt)` |
49
49
  | Messaging | `sendCommand(command, data?, uploads?)`, `requestSchema()` |
50
50
  | Tracks | `publishTrack(name, track?)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
51
- | Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
51
+ | Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushEncodedVideoFrame(name, codec, data, width, height, metadata?, captureTimeUs?, keyframe?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()`, `takeInputKeyframeRequest(name)` |
52
+ | Step frame codecs | `setCodecPreference(codecs?, track?)`, `setLocalStepCodecs(encode, decode)`, `stepFrameEncodings()`, `restartTrackStream(name)` |
52
53
  | Recording | `requestClip(seconds)`, `requestRecording()` |
53
54
  | Uploads | `uploadFile(file, name?)` |
54
55
  | Introspection | `status()`, `sessionId()`, `sessionInfo()`, `capabilities()`, `lastError()` |
@@ -63,20 +64,46 @@ arrays (`Uint8Array` video, `Int16Array` audio) and serde has no word for one.
63
64
 
64
65
  A client created with `generation: "stepped"` sends no input media over RTP.
65
66
  `publishTrack(name)` only claims an input track (a `MediaStreamTrack` passed
66
- there is ignored, with a console warning), `pushVideoFrame` / `pushAudioFrame`
67
- buffer frames in the core, and the next `step()` carries them inside its one
68
- `RequestStep`. Both of `step()`'s arguments are optional: an empty step runs on
69
- the model's current state. Video arrives as RGBA, RGB or BGRA, and the core
70
- converts it to the RGB a step sends in one pass.
67
+ there is ignored, with a console warning), and `pushVideoFrame` /
68
+ `pushAudioFrame` hand frames to the core. Once the session is ready, against a
69
+ runtime that takes pushed frames, each push is sent at once in a `PushFrames`
70
+ message on the data channel, ahead of any step after it; one the channel cannot
71
+ take is dropped and raised as an error event. Frames pushed before the session
72
+ is ready, or to an older runtime, are buffered in the core and sent ahead of the
73
+ next push once it can go, or inside the next `step()`'s `RequestStep`. Both of
74
+ `step()`'s arguments are optional: an empty step runs on the model's current
75
+ state. Video arrives as RGBA, RGB or BGRA, and the core converts it to the RGB a
76
+ step sends in one pass. `pushEncodedVideoFrame` pushes an encoded frame as it is (`"h264"`: one access
77
+ unit, Annex B; `"vp8"`, `"vp9"`: one frame; `"av1"`: one temporal unit), with
78
+ the size it decodes to and its keyframe mark; the runtime decodes a track's
79
+ encoded frames as one stream, in push order and starting at a keyframe, as they
80
+ arrive. A pushed frame that does not decode raises an `input_decode_failed`
81
+ error event naming the track, whose next frame must then be a keyframe.
82
+
83
+ The browser has no codecs this crate could use, so the SDK brings WebCodecs:
84
+ `setLocalStepCodecs(encode, decode)` names what it can encode and decode, and
85
+ `setCodecPreference` orders them, both read on the next offer, which
86
+ negotiates each track's codec in the SDP (`stepFrameEncodings()` says what it
87
+ settled). The SDK encodes an input track's frames in its codec and pushes them
88
+ with `pushEncodedVideoFrame`, starting at a keyframe again when
89
+ `takeInputKeyframeRequest` says the runtime could not decode them. An output
90
+ track's encoded frames come up in the result still encoded — `data` the
91
+ bitstream, `shape` the size it decodes to, `keyframe` the runtime's mark — for
92
+ the SDK to decode, and `restartTrackStream(name)` has the runtime start a track
93
+ again at a keyframe when the SDK's decoder for it lost its stream.
71
94
 
72
95
  The buffer holds at most one data-channel message's worth of frames: pushing
73
96
  faster than steps run drops the oldest frames, across tracks, and the next step
74
- logs how many. Pushed frames belong to the step that takes them: when that
97
+ logs how many. Buffered frames belong to the step that takes them: when that
75
98
  step's request cannot be sent — `MESSAGE_TOO_LARGE` among the reasons — the
76
99
  step rejects and its frames are dropped with it, so the next step starts from
77
- what is pushed after. `clearStepInputs()` drops whatever is buffered now.
100
+ what is pushed after. `clearStepInputs()` drops whatever is buffered now;
101
+ frames already sent to the runtime are not recalled.
78
102
 
79
- `step()` resolves with `{ step, tracks, elapsed, files, error? }`. Each track
103
+ `step()` resolves with `{ step, tracks, elapsed, files, error?, data? }`.
104
+ `data` is a `Uint8Array` of what the model returned for the step besides its
105
+ media, opaque to the transport (a dict the model returned arrives encoded as
106
+ JSON); a model whose output is data alone needs no output track. Each track
80
107
  is `{ track, kind, rate, frames }`, `rate` being the rate its frames play at as
81
108
  the runtime reported it (frames per second for video, samples per second for
82
109
  audio; 0 when it did not say). `files`
@@ -7,6 +7,13 @@ export type TrackDirection = "recvonly" | "sendonly";
7
7
  export type GenerationMode = "streamed" | "stepped";
8
8
  /** Byte layout of a frame given to `pushVideoFrame`. */
9
9
  export type PixelFormat = "rgba" | "rgb" | "bgra";
10
+ /** Codec of an encoded video step frame: given to `pushEncodedVideoFrame`,
11
+ * named by `setCodecPreference` and `setLocalStepCodecs`. "h264" is one
12
+ * access unit, Annex B; "vp8" and "vp9" one frame; "av1" one temporal unit
13
+ * of OBUs. */
14
+ export type EncodedVideoCodec = "h264" | "vp8" | "vp9" | "av1";
15
+ /** How a track's step frames travel on a connection: raw, or in a codec. */
16
+ export type StepFrameEncoding = "raw" | EncodedVideoCodec;
10
17
 
11
18
  /** Construction options. Only `modelName` is required. */
12
19
  export interface ClientOptions {
@@ -50,6 +57,14 @@ export interface ClientOptions {
50
57
  /** How long `step()` waits for a step's result, generation included.
51
58
  * Default 300000. */
52
59
  stepTimeoutMs?: number;
60
+ /** The largest data-channel message, in bytes, the client declares and
61
+ * accepts: from 262144 (256 KiB) to 1073741824 (1 GiB). Default 67108864
62
+ * (64 MiB). A connection carries the smaller of this and the runtime's own
63
+ * limit, which is 64 MiB unless the model raises it, so a `step()` whose
64
+ * result is larger than either fails with `StepResultTooLargeError`. A
65
+ * message is held whole in memory while it arrives, so raise this only for
66
+ * a model whose steps need it. */
67
+ maxMessageSize?: number;
53
68
  /** Console log level. Default "warn". */
54
69
  logLevel?: "off" | "error" | "warn" | "info" | "debug" | "trace";
55
70
  }
@@ -233,8 +248,10 @@ export interface StepCommand {
233
248
  /** One frame a step produced. Hand-built by the binding rather than
234
249
  * serialized, so camelCase, unlike the wire payloads above. */
235
250
  export interface StepFrame {
236
- /** How the frame travelled; "raw" is the only one decoded today. */
237
- encoding: "raw" | "unspecified";
251
+ /** How the frame travelled. A frame in a codec comes up still encoded,
252
+ * `data` holding its bitstream and `shape` the (height, width, 3) it
253
+ * decodes to, for the SDK to decode with its own codecs. */
254
+ encoding: "raw" | "unspecified" | EncodedVideoCodec;
238
255
  /** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
239
256
  dtype: string;
240
257
  /** (height, width, 3) RGB for video, (channels, samples) planar PCM for
@@ -256,6 +273,9 @@ export interface StepFrame {
256
273
  metadata?: Uint8Array;
257
274
  /** When the model says the frame was captured, in microseconds. */
258
275
  captureTimeUs?: number;
276
+ /** Set on a frame still encoded: whether the runtime marked it as a
277
+ * keyframe, one a decoder can start from. */
278
+ keyframe?: boolean;
259
279
  }
260
280
 
261
281
  /** The frames one output track produced in a step, in production order. */
@@ -292,6 +312,11 @@ export interface StepResult {
292
312
  /** Set when the step ran and the model reported a failure. Such a step
293
313
  * still resolves; only a refused step rejects. */
294
314
  error?: string;
315
+ /** What the model returned for the step besides its media, opaque to the
316
+ * transport: the bytes it returned as they are, or a dict it returned
317
+ * encoded as JSON. Arrives with the step's frames, so a model whose output
318
+ * is data alone needs no output track. Absent when it returned none. */
319
+ data?: Uint8Array;
295
320
  }
296
321
 
297
322
  export type StatusListener = (status: ReactorStatus) => void;
@@ -327,7 +352,32 @@ export class ReactorClient {
327
352
  */
328
353
  capabilities(): Capabilities | undefined;
329
354
  /**
330
- * Drop every frame buffered for the next step.
355
+ * Whether `pushAudioFrame` would refuse `samples`: throws what it would
356
+ * throw, and buffers nothing.
357
+ */
358
+ checkAudioFrame(name: string, samples: Int16Array, channels: number): void;
359
+ /**
360
+ * Whether `pushEncodedVideoFrame` would refuse a frame of `byteLength`
361
+ * bytes: throws what it would throw, and buffers nothing. A length, as
362
+ * for `checkVideoFrame`.
363
+ */
364
+ checkEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, byte_length: number, width: number, height: number, capture_time_us?: number | null): void;
365
+ /**
366
+ * Whether `step(state, commands)` would be refused now, for the session or
367
+ * for its state and commands: throws what `step()` would throw, and takes
368
+ * no frames. `step()`'s own checks, run before a step's media is pushed,
369
+ * so a refused step leaves the buffer as the caller left it.
370
+ */
371
+ checkStep(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): void;
372
+ /**
373
+ * Whether `pushVideoFrame` would refuse a frame of `byteLength` bytes:
374
+ * throws what it would throw, and buffers nothing. The frame's length
375
+ * rather than its bytes, so a frame is not copied into wasm memory twice.
376
+ */
377
+ checkVideoFrame(name: string, byte_length: number, width: number, height: number, format?: PixelFormat | null): void;
378
+ /**
379
+ * Drop every frame still buffered for the next step. Frames already sent
380
+ * to the runtime are not recalled.
331
381
  */
332
382
  clearStepInputs(): void;
333
383
  /**
@@ -429,14 +479,37 @@ export class ReactorClient {
429
479
  */
430
480
  publishTrack(name: string, track?: MediaStreamTrack | null): Promise<void>;
431
481
  /**
432
- * Buffer a block of audio for the next `step()` on the sendonly track
433
- * `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
482
+ * Push a block of audio, as `pushVideoFrame` pushes a frame, on the
483
+ * sendonly track `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
434
484
  * already at the rate the model's track declares (a step's frame carries
435
485
  * no sample rate). Stepped sessions only, as `pushVideoFrame`.
436
486
  */
437
487
  pushAudioFrame(name: string, samples: Int16Array, sample_rate: number, channels: number): void;
438
488
  /**
439
- * Buffer a video frame for the next `step()` on the sendonly track `name`.
489
+ * Push an encoded video frame on the sendonly video track `name`, its
490
+ * bytes as they are, sent or buffered as `pushVideoFrame` does.
491
+ *
492
+ * * `codec` — `"h264"` (the default; one access unit, Annex B), `"vp8"`,
493
+ * `"vp9"` (one frame) or `"av1"` (one temporal unit of OBUs).
494
+ * * `width`, `height` — the frame it decodes to.
495
+ * * `metadata`, `captureTimeUs` — as for `pushVideoFrame`, carried with
496
+ * the decoded frame.
497
+ * * `keyframe` — whether the frame is one a decoder can start from.
498
+ *
499
+ * The runtime decodes a track's encoded frames as one stream, as they
500
+ * arrive, so push them in decode order, starting at a keyframe. A pushed
501
+ * frame that does not decode raises an `input_decode_failed` error event
502
+ * whose `track` names the track, and the runtime drops the track's frames
503
+ * up to the next keyframe: `takeInputKeyframeRequest(name)` is then
504
+ * `true` once. A step whose buffered frames do not decode is refused
505
+ * (`step_refused`). Encoded and raw pushes to one track keep their push
506
+ * order. Stepped sessions only; throws on empty data or a zero dimension.
507
+ */
508
+ pushEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, data: Uint8Array, width: number, height: number, metadata?: Uint8Array | null, capture_time_us?: number | null, keyframe?: boolean | null): void;
509
+ /**
510
+ * Push a video frame on the sendonly track `name`: sent to the runtime at
511
+ * once in a ready session whose runtime takes pushed frames, buffered for
512
+ * the next `step()` otherwise.
440
513
  *
441
514
  * * `data` — `width * height` pixels in `format`: `"rgba"` (the default,
442
515
  * what `ImageData` holds), `"rgb"` or `"bgra"`. Sent as RGB.
@@ -468,6 +541,13 @@ export class ReactorClient {
468
541
  * Request the model's command schema (an OpenAPI document).
469
542
  */
470
543
  requestSchema(): Promise<unknown>;
544
+ /**
545
+ * Restart output track `name`'s encoded step frames: the runtime starts
546
+ * its next frames at a keyframe. For the SDK, when the decoder it feeds
547
+ * for the track lost its stream. The track stays paused or resumed as
548
+ * the caller left it.
549
+ */
550
+ restartTrackStream(name: string): void;
471
551
  /**
472
552
  * Resume a paused track.
473
553
  */
@@ -502,11 +582,32 @@ export class ReactorClient {
502
582
  * disconnected: model, cluster, server info, selected transport.
503
583
  */
504
584
  sessionInfo(): SessionInfo | undefined;
585
+ /**
586
+ * Set the codecs a stepped session prefers for its video step frames,
587
+ * most preferred first: for every track with `track` absent, or for one
588
+ * track in place of the session's order. `null`/`undefined` goes back to
589
+ * the default (H.264, VP9, AV1, VP8 for the session; the session's order
590
+ * for a track). An empty list for the session turns encoding off, and
591
+ * every frame travels raw; for one track it is taken as `null`.
592
+ *
593
+ * Callable before `connect()` or during the session: it takes effect on
594
+ * the next connection or reconnect. Each list is filtered by
595
+ * `setLocalStepCodecs`. Throws on a name that is not a codec.
596
+ */
597
+ setCodecPreference(codecs?: EncodedVideoCodec[] | null, track?: string | null): void;
505
598
  /**
506
599
  * Replace the token source. Takes effect on the next request, so this is
507
600
  * how a client built before sign-in gets its token.
508
601
  */
509
602
  setJwt(jwt?: JwtSource | null): void;
603
+ /**
604
+ * Set the codecs this browser can carry step frames in: `encode`, for
605
+ * the frames pushed on input tracks, and `decode`, for the frames step
606
+ * results bring on output tracks. Until set, every track is raw. Read on
607
+ * the next offer, like `setCodecPreference`. Throws on a name that is
608
+ * not a codec.
609
+ */
610
+ setLocalStepCodecs(encode: EncodedVideoCodec[], decode: EncodedVideoCodec[]): void;
510
611
  /**
511
612
  * Bound what one sender may spend, in bits per second. `undefined` or a
512
613
  * negative value leaves a bound at the browser default.
@@ -534,9 +635,9 @@ export class ReactorClient {
534
635
  * `set_<key>` command before `commands`; `null`/`undefined` for none.
535
636
  * * `commands` — `[{ command, data? }]`, run in order after the state.
536
637
  *
537
- * Every frame pushed since the previous step rides this one, and belongs
538
- * to it: when the request cannot be sent the step fails and its frames
539
- * are dropped, not kept for the next step. Rejects with the core's code:
638
+ * Every frame still buffered rides this one, and belongs to it: when the
639
+ * request cannot be sent the step fails and its frames are dropped, not
640
+ * kept for the next step. Rejects with the core's code:
540
641
  * `step_refused` (the runtime checks a whole request first, so a refused
541
642
  * step changed nothing; `entry` names the part at fault),
542
643
  * `step_not_ready` (the session cannot take a step yet; the step did not
@@ -548,8 +649,30 @@ export class ReactorClient {
548
649
  * data channel's message limit; nothing was sent),
549
650
  * `UNSUPPORTED_FRAME_ENCODING`, or the code of whatever else failed the
550
651
  * step (`DISCONNECTED`, `REQUEST_TIMEOUT`, …).
652
+ *
653
+ * The step is checked and its frames taken before this returns, so a
654
+ * frame pushed once it has returned goes to the next step. A step refused
655
+ * by those checks throws rather than rejecting.
656
+ *
657
+ * An encoded output frame comes up still encoded (`keyframe` marks one),
658
+ * for the SDK to decode; `restartTrackStream()` restarts a track whose
659
+ * decoder lost its stream.
551
660
  */
552
661
  step(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): Promise<StepResult>;
662
+ /**
663
+ * How each mapped track's step frames travel on this connection, as its
664
+ * SDP answer negotiated them: `{ [track]: "raw" | codec }`. Empty outside
665
+ * a connected stepped session; a track left out is raw.
666
+ */
667
+ stepFrameEncodings(): Record<string, StepFrameEncoding>;
668
+ /**
669
+ * Whether the sendonly video track `name`'s next encoded frame must be a
670
+ * keyframe, because the runtime could not decode its frames
671
+ * (`input_decode_failed`) or frames encoded for it were not sent. Asking
672
+ * clears the mark: the caller about to encode the track's next frame
673
+ * makes it a keyframe.
674
+ */
675
+ takeInputKeyframeRequest(name: string): boolean;
553
676
  /**
554
677
  * The negotiated `name` → `mid` mapping: `[{ name, kind, direction, mid }]`.
555
678
  */
@@ -578,6 +701,10 @@ export interface InitOutput {
578
701
  readonly memory: WebAssembly.Memory;
579
702
  readonly __wbg_reactorclient_free: (a: number, b: number) => void;
580
703
  readonly reactorclient_capabilities: (a: number) => [number, number, number];
704
+ readonly reactorclient_checkAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number) => [number, number];
705
+ readonly reactorclient_checkEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => [number, number];
706
+ readonly reactorclient_checkStep: (a: number, b: number, c: number) => [number, number];
707
+ readonly reactorclient_checkVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
581
708
  readonly reactorclient_clearStepInputs: (a: number) => void;
582
709
  readonly reactorclient_connect: (a: number, b: number) => any;
583
710
  readonly reactorclient_disconnect: (a: number) => any;
@@ -599,20 +726,26 @@ export interface InitOutput {
599
726
  readonly reactorclient_pausedTracks: (a: number) => [number, number, number];
600
727
  readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: number) => any;
601
728
  readonly reactorclient_pushAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
729
+ readonly reactorclient_pushEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number) => [number, number];
602
730
  readonly reactorclient_pushVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
603
731
  readonly reactorclient_reconnect: (a: number, b: number) => any;
604
732
  readonly reactorclient_requestClip: (a: number, b: number) => any;
605
733
  readonly reactorclient_requestRecording: (a: number) => any;
606
734
  readonly reactorclient_requestSchema: (a: number) => any;
735
+ readonly reactorclient_restartTrackStream: (a: number, b: number, c: number) => [number, number];
607
736
  readonly reactorclient_resumeTrack: (a: number, b: number, c: number) => any;
608
737
  readonly reactorclient_sendClientStats: (a: number, b: any, c: any) => [number, number];
609
738
  readonly reactorclient_sendCommand: (a: number, b: number, c: number, d: number, e: number) => any;
610
739
  readonly reactorclient_sessionId: (a: number) => [number, number];
611
740
  readonly reactorclient_sessionInfo: (a: number) => [number, number, number];
741
+ readonly reactorclient_setCodecPreference: (a: number, b: number, c: number, d: number) => [number, number];
612
742
  readonly reactorclient_setJwt: (a: number, b: number) => [number, number];
743
+ readonly reactorclient_setLocalStepCodecs: (a: number, b: any, c: any) => [number, number];
613
744
  readonly reactorclient_setTrackBitrate: (a: number, b: number, c: number, d: number, e: number) => any;
614
745
  readonly reactorclient_status: (a: number) => any;
615
- readonly reactorclient_step: (a: number, b: number, c: number) => any;
746
+ readonly reactorclient_step: (a: number, b: number, c: number) => [number, number, number];
747
+ readonly reactorclient_stepFrameEncodings: (a: number) => [number, number, number];
748
+ readonly reactorclient_takeInputKeyframeRequest: (a: number, b: number, c: number) => number;
616
749
  readonly reactorclient_trackMapping: (a: number) => [number, number, number];
617
750
  readonly reactorclient_tracks: (a: number) => [number, number, number];
618
751
  readonly reactorclient_unpublishTrack: (a: number, b: number, c: number) => any;