@reactor-team/js-sdk 3.4.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -48,7 +48,8 @@ rather than `any`:
48
48
  | Configuration | `setJwt(jwt)` |
49
49
  | Messaging | `sendCommand(command, data?, uploads?)`, `requestSchema()` |
50
50
  | Tracks | `publishTrack(name, track?)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
51
- | Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushEncodedVideoFrame(name, codec, data, width, height, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
51
+ | Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushEncodedVideoFrame(name, codec, data, width, height, metadata?, captureTimeUs?, keyframe?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()`, `takeInputKeyframeRequest(name)` |
52
+ | Step frame codecs | `setCodecPreference(codecs?, track?)`, `setLocalStepCodecs(encode, decode)`, `stepFrameEncodings()`, `restartTrackStream(name)` |
52
53
  | Recording | `requestClip(seconds)`, `requestRecording()` |
53
54
  | Uploads | `uploadFile(file, name?)` |
54
55
  | Introspection | `status()`, `sessionId()`, `sessionInfo()`, `capabilities()`, `lastError()` |
@@ -63,23 +64,46 @@ arrays (`Uint8Array` video, `Int16Array` audio) and serde has no word for one.
63
64
 
64
65
  A client created with `generation: "stepped"` sends no input media over RTP.
65
66
  `publishTrack(name)` only claims an input track (a `MediaStreamTrack` passed
66
- there is ignored, with a console warning), `pushVideoFrame` / `pushAudioFrame`
67
- buffer frames in the core, and the next `step()` carries them inside its one
68
- `RequestStep`. Both of `step()`'s arguments are optional: an empty step runs on
69
- the model's current state. Video arrives as RGBA, RGB or BGRA, and the core
70
- converts it to the RGB a step sends in one pass. `pushEncodedVideoFrame` buffers
71
- an H.264 access unit (Annex B) as it is, with the size it decodes to; the runtime
72
- decodes a track's encoded frames of one step, in push order and starting at a
73
- keyframe, before the step runs.
67
+ there is ignored, with a console warning), and `pushVideoFrame` /
68
+ `pushAudioFrame` hand frames to the core. Once the session is ready, against a
69
+ runtime that takes pushed frames, each push is sent at once in a `PushFrames`
70
+ message on the data channel, ahead of any step after it; one the channel cannot
71
+ take is dropped and raised as an error event. Frames pushed before the session
72
+ is ready, or to an older runtime, are buffered in the core and sent ahead of the
73
+ next push once it can go, or inside the next `step()`'s `RequestStep`. Both of
74
+ `step()`'s arguments are optional: an empty step runs on the model's current
75
+ state. Video arrives as RGBA, RGB or BGRA, and the core converts it to the RGB a
76
+ step sends in one pass. `pushEncodedVideoFrame` pushes an encoded frame as it is (`"h264"`: one access
77
+ unit, Annex B; `"vp8"`, `"vp9"`: one frame; `"av1"`: one temporal unit), with
78
+ the size it decodes to and its keyframe mark; the runtime decodes a track's
79
+ encoded frames as one stream, in push order and starting at a keyframe, as they
80
+ arrive. A pushed frame that does not decode raises an `input_decode_failed`
81
+ error event naming the track, whose next frame must then be a keyframe.
82
+
83
+ The browser has no codecs this crate could use, so the SDK brings WebCodecs:
84
+ `setLocalStepCodecs(encode, decode)` names what it can encode and decode, and
85
+ `setCodecPreference` orders them, both read on the next offer, which
86
+ negotiates each track's codec in the SDP (`stepFrameEncodings()` says what it
87
+ settled). The SDK encodes an input track's frames in its codec and pushes them
88
+ with `pushEncodedVideoFrame`, starting at a keyframe again when
89
+ `takeInputKeyframeRequest` says the runtime could not decode them. An output
90
+ track's encoded frames come up in the result still encoded — `data` the
91
+ bitstream, `shape` the size it decodes to, `keyframe` the runtime's mark — for
92
+ the SDK to decode, and `restartTrackStream(name)` has the runtime start a track
93
+ again at a keyframe when the SDK's decoder for it lost its stream.
74
94
 
75
95
  The buffer holds at most one data-channel message's worth of frames: pushing
76
96
  faster than steps run drops the oldest frames, across tracks, and the next step
77
- logs how many. Pushed frames belong to the step that takes them: when that
97
+ logs how many. Buffered frames belong to the step that takes them: when that
78
98
  step's request cannot be sent — `MESSAGE_TOO_LARGE` among the reasons — the
79
99
  step rejects and its frames are dropped with it, so the next step starts from
80
- what is pushed after. `clearStepInputs()` drops whatever is buffered now.
100
+ what is pushed after. `clearStepInputs()` drops whatever is buffered now;
101
+ frames already sent to the runtime are not recalled.
81
102
 
82
- `step()` resolves with `{ step, tracks, elapsed, files, error? }`. Each track
103
+ `step()` resolves with `{ step, tracks, elapsed, files, error?, data? }`.
104
+ `data` is a `Uint8Array` of what the model returned for the step besides its
105
+ media, opaque to the transport (a dict the model returned arrives encoded as
106
+ JSON); a model whose output is data alone needs no output track. Each track
83
107
  is `{ track, kind, rate, frames }`, `rate` being the rate its frames play at as
84
108
  the runtime reported it (frames per second for video, samples per second for
85
109
  audio; 0 when it did not say). `files`
@@ -7,8 +7,13 @@ export type TrackDirection = "recvonly" | "sendonly";
7
7
  export type GenerationMode = "streamed" | "stepped";
8
8
  /** Byte layout of a frame given to `pushVideoFrame`. */
9
9
  export type PixelFormat = "rgba" | "rgb" | "bgra";
10
- /** Codec of an encoded frame given to `pushEncodedVideoFrame`. */
11
- export type EncodedVideoCodec = "h264";
10
+ /** Codec of an encoded video step frame: given to `pushEncodedVideoFrame`,
11
+ * named by `setCodecPreference` and `setLocalStepCodecs`. "h264" is one
12
+ * access unit, Annex B; "vp8" and "vp9" one frame; "av1" one temporal unit
13
+ * of OBUs. */
14
+ export type EncodedVideoCodec = "h264" | "vp8" | "vp9" | "av1";
15
+ /** How a track's step frames travel on a connection: raw, or in a codec. */
16
+ export type StepFrameEncoding = "raw" | EncodedVideoCodec;
12
17
 
13
18
  /** Construction options. Only `modelName` is required. */
14
19
  export interface ClientOptions {
@@ -243,8 +248,10 @@ export interface StepCommand {
243
248
  /** One frame a step produced. Hand-built by the binding rather than
244
249
  * serialized, so camelCase, unlike the wire payloads above. */
245
250
  export interface StepFrame {
246
- /** How the frame travelled; "raw" is the only one decoded today. */
247
- encoding: "raw" | "unspecified";
251
+ /** How the frame travelled. A frame in a codec comes up still encoded,
252
+ * `data` holding its bitstream and `shape` the (height, width, 3) it
253
+ * decodes to, for the SDK to decode with its own codecs. */
254
+ encoding: "raw" | "unspecified" | EncodedVideoCodec;
248
255
  /** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
249
256
  dtype: string;
250
257
  /** (height, width, 3) RGB for video, (channels, samples) planar PCM for
@@ -266,6 +273,9 @@ export interface StepFrame {
266
273
  metadata?: Uint8Array;
267
274
  /** When the model says the frame was captured, in microseconds. */
268
275
  captureTimeUs?: number;
276
+ /** Set on a frame still encoded: whether the runtime marked it as a
277
+ * keyframe, one a decoder can start from. */
278
+ keyframe?: boolean;
269
279
  }
270
280
 
271
281
  /** The frames one output track produced in a step, in production order. */
@@ -302,6 +312,11 @@ export interface StepResult {
302
312
  /** Set when the step ran and the model reported a failure. Such a step
303
313
  * still resolves; only a refused step rejects. */
304
314
  error?: string;
315
+ /** What the model returned for the step besides its media, opaque to the
316
+ * transport: the bytes it returned as they are, or a dict it returned
317
+ * encoded as JSON. Arrives with the step's frames, so a model whose output
318
+ * is data alone needs no output track. Absent when it returned none. */
319
+ data?: Uint8Array;
305
320
  }
306
321
 
307
322
  export type StatusListener = (status: ReactorStatus) => void;
@@ -361,7 +376,8 @@ export class ReactorClient {
361
376
  */
362
377
  checkVideoFrame(name: string, byte_length: number, width: number, height: number, format?: PixelFormat | null): void;
363
378
  /**
364
- * Drop every frame buffered for the next step.
379
+ * Drop every frame still buffered for the next step. Frames already sent
380
+ * to the runtime are not recalled.
365
381
  */
366
382
  clearStepInputs(): void;
367
383
  /**
@@ -463,31 +479,37 @@ export class ReactorClient {
463
479
  */
464
480
  publishTrack(name: string, track?: MediaStreamTrack | null): Promise<void>;
465
481
  /**
466
- * Buffer a block of audio for the next `step()` on the sendonly track
467
- * `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
482
+ * Push a block of audio, as `pushVideoFrame` pushes a frame, on the
483
+ * sendonly track `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
468
484
  * already at the rate the model's track declares (a step's frame carries
469
485
  * no sample rate). Stepped sessions only, as `pushVideoFrame`.
470
486
  */
471
487
  pushAudioFrame(name: string, samples: Int16Array, sample_rate: number, channels: number): void;
472
488
  /**
473
- * Buffer an encoded video frame for the next `step()` on the sendonly
474
- * video track `name`, its bytes as they are.
489
+ * Push an encoded video frame on the sendonly video track `name`, its
490
+ * bytes as they are, sent or buffered as `pushVideoFrame` does.
475
491
  *
476
- * * `codec` — `"h264"` (the default): `data` is one H.264 access unit,
477
- * Annex B.
492
+ * * `codec` — `"h264"` (the default; one access unit, Annex B), `"vp8"`,
493
+ * `"vp9"` (one frame) or `"av1"` (one temporal unit of OBUs).
478
494
  * * `width`, `height` — the frame it decodes to.
479
495
  * * `metadata`, `captureTimeUs` — as for `pushVideoFrame`, carried with
480
496
  * the decoded frame.
497
+ * * `keyframe` — whether the frame is one a decoder can start from.
481
498
  *
482
- * The runtime decodes a track's encoded frames of one step as one
483
- * stream before the step runs, so push them in decode order, starting at
484
- * a keyframe; a step whose frames do not decode is refused
499
+ * The runtime decodes a track's encoded frames as one stream, as they
500
+ * arrive, so push them in decode order, starting at a keyframe. A pushed
501
+ * frame that does not decode raises an `input_decode_failed` error event
502
+ * whose `track` names the track, and the runtime drops the track's frames
503
+ * up to the next keyframe: `takeInputKeyframeRequest(name)` is then
504
+ * `true` once. A step whose buffered frames do not decode is refused
485
505
  * (`step_refused`). Encoded and raw pushes to one track keep their push
486
506
  * order. Stepped sessions only; throws on empty data or a zero dimension.
487
507
  */
488
- pushEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, data: Uint8Array, width: number, height: number, metadata?: Uint8Array | null, capture_time_us?: number | null): void;
508
+ pushEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, data: Uint8Array, width: number, height: number, metadata?: Uint8Array | null, capture_time_us?: number | null, keyframe?: boolean | null): void;
489
509
  /**
490
- * Buffer a video frame for the next `step()` on the sendonly track `name`.
510
+ * Push a video frame on the sendonly track `name`: sent to the runtime at
511
+ * once in a ready session whose runtime takes pushed frames, buffered for
512
+ * the next `step()` otherwise.
491
513
  *
492
514
  * * `data` — `width * height` pixels in `format`: `"rgba"` (the default,
493
515
  * what `ImageData` holds), `"rgb"` or `"bgra"`. Sent as RGB.
@@ -519,6 +541,13 @@ export class ReactorClient {
519
541
  * Request the model's command schema (an OpenAPI document).
520
542
  */
521
543
  requestSchema(): Promise<unknown>;
544
+ /**
545
+ * Restart output track `name`'s encoded step frames: the runtime starts
546
+ * its next frames at a keyframe. For the SDK, when the decoder it feeds
547
+ * for the track lost its stream. The track stays paused or resumed as
548
+ * the caller left it.
549
+ */
550
+ restartTrackStream(name: string): void;
522
551
  /**
523
552
  * Resume a paused track.
524
553
  */
@@ -553,11 +582,32 @@ export class ReactorClient {
553
582
  * disconnected: model, cluster, server info, selected transport.
554
583
  */
555
584
  sessionInfo(): SessionInfo | undefined;
585
+ /**
586
+ * Set the codecs a stepped session prefers for its video step frames,
587
+ * most preferred first: for every track with `track` absent, or for one
588
+ * track in place of the session's order. `null`/`undefined` goes back to
589
+ * the default (H.264, VP9, AV1, VP8 for the session; the session's order
590
+ * for a track). An empty list for the session turns encoding off, and
591
+ * every frame travels raw; for one track it is taken as `null`.
592
+ *
593
+ * Callable before `connect()` or during the session: it takes effect on
594
+ * the next connection or reconnect. Each list is filtered by
595
+ * `setLocalStepCodecs`. Throws on a name that is not a codec.
596
+ */
597
+ setCodecPreference(codecs?: EncodedVideoCodec[] | null, track?: string | null): void;
556
598
  /**
557
599
  * Replace the token source. Takes effect on the next request, so this is
558
600
  * how a client built before sign-in gets its token.
559
601
  */
560
602
  setJwt(jwt?: JwtSource | null): void;
603
+ /**
604
+ * Set the codecs this browser can carry step frames in: `encode`, for
605
+ * the frames pushed on input tracks, and `decode`, for the frames step
606
+ * results bring on output tracks. Until set, every track is raw. Read on
607
+ * the next offer, like `setCodecPreference`. Throws on a name that is
608
+ * not a codec.
609
+ */
610
+ setLocalStepCodecs(encode: EncodedVideoCodec[], decode: EncodedVideoCodec[]): void;
561
611
  /**
562
612
  * Bound what one sender may spend, in bits per second. `undefined` or a
563
613
  * negative value leaves a bound at the browser default.
@@ -585,9 +635,9 @@ export class ReactorClient {
585
635
  * `set_<key>` command before `commands`; `null`/`undefined` for none.
586
636
  * * `commands` — `[{ command, data? }]`, run in order after the state.
587
637
  *
588
- * Every frame pushed since the previous step rides this one, and belongs
589
- * to it: when the request cannot be sent the step fails and its frames
590
- * are dropped, not kept for the next step. Rejects with the core's code:
638
+ * Every frame still buffered rides this one, and belongs to it: when the
639
+ * request cannot be sent the step fails and its frames are dropped, not
640
+ * kept for the next step. Rejects with the core's code:
591
641
  * `step_refused` (the runtime checks a whole request first, so a refused
592
642
  * step changed nothing; `entry` names the part at fault),
593
643
  * `step_not_ready` (the session cannot take a step yet; the step did not
@@ -603,8 +653,26 @@ export class ReactorClient {
603
653
  * The step is checked and its frames taken before this returns, so a
604
654
  * frame pushed once it has returned goes to the next step. A step refused
605
655
  * by those checks throws rather than rejecting.
656
+ *
657
+ * An encoded output frame comes up still encoded (`keyframe` marks one),
658
+ * for the SDK to decode; `restartTrackStream()` restarts a track whose
659
+ * decoder lost its stream.
606
660
  */
607
661
  step(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): Promise<StepResult>;
662
+ /**
663
+ * How each mapped track's step frames travel on this connection, as its
664
+ * SDP answer negotiated them: `{ [track]: "raw" | codec }`. Empty outside
665
+ * a connected stepped session; a track left out is raw.
666
+ */
667
+ stepFrameEncodings(): Record<string, StepFrameEncoding>;
668
+ /**
669
+ * Whether the sendonly video track `name`'s next encoded frame must be a
670
+ * keyframe, because the runtime could not decode its frames
671
+ * (`input_decode_failed`) or frames encoded for it were not sent. Asking
672
+ * clears the mark: the caller about to encode the track's next frame
673
+ * makes it a keyframe.
674
+ */
675
+ takeInputKeyframeRequest(name: string): boolean;
608
676
  /**
609
677
  * The negotiated `name` → `mid` mapping: `[{ name, kind, direction, mid }]`.
610
678
  */
@@ -658,21 +726,26 @@ export interface InitOutput {
658
726
  readonly reactorclient_pausedTracks: (a: number) => [number, number, number];
659
727
  readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: number) => any;
660
728
  readonly reactorclient_pushAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
661
- readonly reactorclient_pushEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
729
+ readonly reactorclient_pushEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number) => [number, number];
662
730
  readonly reactorclient_pushVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
663
731
  readonly reactorclient_reconnect: (a: number, b: number) => any;
664
732
  readonly reactorclient_requestClip: (a: number, b: number) => any;
665
733
  readonly reactorclient_requestRecording: (a: number) => any;
666
734
  readonly reactorclient_requestSchema: (a: number) => any;
735
+ readonly reactorclient_restartTrackStream: (a: number, b: number, c: number) => [number, number];
667
736
  readonly reactorclient_resumeTrack: (a: number, b: number, c: number) => any;
668
737
  readonly reactorclient_sendClientStats: (a: number, b: any, c: any) => [number, number];
669
738
  readonly reactorclient_sendCommand: (a: number, b: number, c: number, d: number, e: number) => any;
670
739
  readonly reactorclient_sessionId: (a: number) => [number, number];
671
740
  readonly reactorclient_sessionInfo: (a: number) => [number, number, number];
741
+ readonly reactorclient_setCodecPreference: (a: number, b: number, c: number, d: number) => [number, number];
672
742
  readonly reactorclient_setJwt: (a: number, b: number) => [number, number];
743
+ readonly reactorclient_setLocalStepCodecs: (a: number, b: any, c: any) => [number, number];
673
744
  readonly reactorclient_setTrackBitrate: (a: number, b: number, c: number, d: number, e: number) => any;
674
745
  readonly reactorclient_status: (a: number) => any;
675
746
  readonly reactorclient_step: (a: number, b: number, c: number) => [number, number, number];
747
+ readonly reactorclient_stepFrameEncodings: (a: number) => [number, number, number];
748
+ readonly reactorclient_takeInputKeyframeRequest: (a: number, b: number, c: number) => number;
676
749
  readonly reactorclient_trackMapping: (a: number) => [number, number, number];
677
750
  readonly reactorclient_tracks: (a: number) => [number, number, number];
678
751
  readonly reactorclient_unpublishTrack: (a: number, b: number, c: number) => any;