@reactor-team/js-sdk 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -21
- package/dist/index.cjs +742 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +182 -66
- package/dist/index.d.ts +182 -66
- package/dist/index.js +742 -45
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +36 -12
- package/dist/wasm/reactor_wasm.d.ts +93 -20
- package/dist/wasm/reactor_wasm.js +114 -22
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +6 -1
- package/package.json +1 -1
package/dist/wasm/README.md
CHANGED
|
@@ -48,7 +48,8 @@ rather than `any`:
|
|
|
48
48
|
| Configuration | `setJwt(jwt)` |
|
|
49
49
|
| Messaging | `sendCommand(command, data?, uploads?)`, `requestSchema()` |
|
|
50
50
|
| Tracks | `publishTrack(name, track?)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
|
|
51
|
-
| Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushEncodedVideoFrame(name, codec, data, width, height, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
|
|
51
|
+
| Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushEncodedVideoFrame(name, codec, data, width, height, metadata?, captureTimeUs?, keyframe?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()`, `takeInputKeyframeRequest(name)` |
|
|
52
|
+
| Step frame codecs | `setCodecPreference(codecs?, track?)`, `setLocalStepCodecs(encode, decode)`, `stepFrameEncodings()`, `restartTrackStream(name)` |
|
|
52
53
|
| Recording | `requestClip(seconds)`, `requestRecording()` |
|
|
53
54
|
| Uploads | `uploadFile(file, name?)` |
|
|
54
55
|
| Introspection | `status()`, `sessionId()`, `sessionInfo()`, `capabilities()`, `lastError()` |
|
|
@@ -63,23 +64,46 @@ arrays (`Uint8Array` video, `Int16Array` audio) and serde has no word for one.
|
|
|
63
64
|
|
|
64
65
|
A client created with `generation: "stepped"` sends no input media over RTP.
|
|
65
66
|
`publishTrack(name)` only claims an input track (a `MediaStreamTrack` passed
|
|
66
|
-
there is ignored, with a console warning), `pushVideoFrame` /
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
67
|
+
there is ignored, with a console warning), and `pushVideoFrame` /
|
|
68
|
+
`pushAudioFrame` hand frames to the core. Once the session is ready, against a
|
|
69
|
+
runtime that takes pushed frames, each push is sent at once in a `PushFrames`
|
|
70
|
+
message on the data channel, ahead of any step after it; one the channel cannot
|
|
71
|
+
take is dropped and raised as an error event. Frames pushed before the session
|
|
72
|
+
is ready, or to an older runtime, are buffered in the core and sent ahead of the
|
|
73
|
+
next push once it can go, or inside the next `step()`'s `RequestStep`. Both of
|
|
74
|
+
`step()`'s arguments are optional: an empty step runs on the model's current
|
|
75
|
+
state. Video arrives as RGBA, RGB or BGRA, and the core converts it to the RGB a
|
|
76
|
+
step sends in one pass. `pushEncodedVideoFrame` pushes an encoded frame as it is (`"h264"`: one access
|
|
77
|
+
unit, Annex B; `"vp8"`, `"vp9"`: one frame; `"av1"`: one temporal unit), with
|
|
78
|
+
the size it decodes to and its keyframe mark; the runtime decodes a track's
|
|
79
|
+
encoded frames as one stream, in push order and starting at a keyframe, as they
|
|
80
|
+
arrive. A pushed frame that does not decode raises an `input_decode_failed`
|
|
81
|
+
error event naming the track, whose next frame must then be a keyframe.
|
|
82
|
+
|
|
83
|
+
The browser has no codecs this crate could use, so the SDK brings WebCodecs:
|
|
84
|
+
`setLocalStepCodecs(encode, decode)` names what it can encode and decode, and
|
|
85
|
+
`setCodecPreference` orders them, both read on the next offer, which
|
|
86
|
+
negotiates each track's codec in the SDP (`stepFrameEncodings()` says what it
|
|
87
|
+
settled). The SDK encodes an input track's frames in its codec and pushes them
|
|
88
|
+
with `pushEncodedVideoFrame`, starting at a keyframe again when
|
|
89
|
+
`takeInputKeyframeRequest` says the runtime could not decode them. An output
|
|
90
|
+
track's encoded frames come up in the result still encoded — `data` the
|
|
91
|
+
bitstream, `shape` the size it decodes to, `keyframe` the runtime's mark — for
|
|
92
|
+
the SDK to decode, and `restartTrackStream(name)` has the runtime start a track
|
|
93
|
+
again at a keyframe when the SDK's decoder for it lost its stream.
|
|
74
94
|
|
|
75
95
|
The buffer holds at most one data-channel message's worth of frames: pushing
|
|
76
96
|
faster than steps run drops the oldest frames, across tracks, and the next step
|
|
77
|
-
logs how many.
|
|
97
|
+
logs how many. Buffered frames belong to the step that takes them: when that
|
|
78
98
|
step's request cannot be sent — `MESSAGE_TOO_LARGE` among the reasons — the
|
|
79
99
|
step rejects and its frames are dropped with it, so the next step starts from
|
|
80
|
-
what is pushed after. `clearStepInputs()` drops whatever is buffered now
|
|
100
|
+
what is pushed after. `clearStepInputs()` drops whatever is buffered now;
|
|
101
|
+
frames already sent to the runtime are not recalled.
|
|
81
102
|
|
|
82
|
-
`step()` resolves with `{ step, tracks, elapsed, files, error? }`.
|
|
103
|
+
`step()` resolves with `{ step, tracks, elapsed, files, error?, data? }`.
|
|
104
|
+
`data` is a `Uint8Array` of what the model returned for the step besides its
|
|
105
|
+
media, opaque to the transport (a dict the model returned arrives encoded as
|
|
106
|
+
JSON); a model whose output is data alone needs no output track. Each track
|
|
83
107
|
is `{ track, kind, rate, frames }`, `rate` being the rate its frames play at as
|
|
84
108
|
the runtime reported it (frames per second for video, samples per second for
|
|
85
109
|
audio; 0 when it did not say). `files`
|
|
@@ -7,8 +7,13 @@ export type TrackDirection = "recvonly" | "sendonly";
|
|
|
7
7
|
export type GenerationMode = "streamed" | "stepped";
|
|
8
8
|
/** Byte layout of a frame given to `pushVideoFrame`. */
|
|
9
9
|
export type PixelFormat = "rgba" | "rgb" | "bgra";
|
|
10
|
-
/** Codec of an encoded frame given to `pushEncodedVideoFrame
|
|
11
|
-
|
|
10
|
+
/** Codec of an encoded video step frame: given to `pushEncodedVideoFrame`,
|
|
11
|
+
* named by `setCodecPreference` and `setLocalStepCodecs`. "h264" is one
|
|
12
|
+
* access unit, Annex B; "vp8" and "vp9" one frame; "av1" one temporal unit
|
|
13
|
+
* of OBUs. */
|
|
14
|
+
export type EncodedVideoCodec = "h264" | "vp8" | "vp9" | "av1";
|
|
15
|
+
/** How a track's step frames travel on a connection: raw, or in a codec. */
|
|
16
|
+
export type StepFrameEncoding = "raw" | EncodedVideoCodec;
|
|
12
17
|
|
|
13
18
|
/** Construction options. Only `modelName` is required. */
|
|
14
19
|
export interface ClientOptions {
|
|
@@ -243,8 +248,10 @@ export interface StepCommand {
|
|
|
243
248
|
/** One frame a step produced. Hand-built by the binding rather than
|
|
244
249
|
* serialized, so camelCase, unlike the wire payloads above. */
|
|
245
250
|
export interface StepFrame {
|
|
246
|
-
/** How the frame travelled
|
|
247
|
-
|
|
251
|
+
/** How the frame travelled. A frame in a codec comes up still encoded,
|
|
252
|
+
* `data` holding its bitstream and `shape` the (height, width, 3) it
|
|
253
|
+
* decodes to, for the SDK to decode with its own codecs. */
|
|
254
|
+
encoding: "raw" | "unspecified" | EncodedVideoCodec;
|
|
248
255
|
/** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
|
|
249
256
|
dtype: string;
|
|
250
257
|
/** (height, width, 3) RGB for video, (channels, samples) planar PCM for
|
|
@@ -266,6 +273,9 @@ export interface StepFrame {
|
|
|
266
273
|
metadata?: Uint8Array;
|
|
267
274
|
/** When the model says the frame was captured, in microseconds. */
|
|
268
275
|
captureTimeUs?: number;
|
|
276
|
+
/** Set on a frame still encoded: whether the runtime marked it as a
|
|
277
|
+
* keyframe, one a decoder can start from. */
|
|
278
|
+
keyframe?: boolean;
|
|
269
279
|
}
|
|
270
280
|
|
|
271
281
|
/** The frames one output track produced in a step, in production order. */
|
|
@@ -302,6 +312,11 @@ export interface StepResult {
|
|
|
302
312
|
/** Set when the step ran and the model reported a failure. Such a step
|
|
303
313
|
* still resolves; only a refused step rejects. */
|
|
304
314
|
error?: string;
|
|
315
|
+
/** What the model returned for the step besides its media, opaque to the
|
|
316
|
+
* transport: the bytes it returned as they are, or a dict it returned
|
|
317
|
+
* encoded as JSON. Arrives with the step's frames, so a model whose output
|
|
318
|
+
* is data alone needs no output track. Absent when it returned none. */
|
|
319
|
+
data?: Uint8Array;
|
|
305
320
|
}
|
|
306
321
|
|
|
307
322
|
export type StatusListener = (status: ReactorStatus) => void;
|
|
@@ -361,7 +376,8 @@ export class ReactorClient {
|
|
|
361
376
|
*/
|
|
362
377
|
checkVideoFrame(name: string, byte_length: number, width: number, height: number, format?: PixelFormat | null): void;
|
|
363
378
|
/**
|
|
364
|
-
* Drop every frame buffered for the next step.
|
|
379
|
+
* Drop every frame still buffered for the next step. Frames already sent
|
|
380
|
+
* to the runtime are not recalled.
|
|
365
381
|
*/
|
|
366
382
|
clearStepInputs(): void;
|
|
367
383
|
/**
|
|
@@ -463,31 +479,37 @@ export class ReactorClient {
|
|
|
463
479
|
*/
|
|
464
480
|
publishTrack(name: string, track?: MediaStreamTrack | null): Promise<void>;
|
|
465
481
|
/**
|
|
466
|
-
*
|
|
467
|
-
* `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
|
|
482
|
+
* Push a block of audio, as `pushVideoFrame` pushes a frame, on the
|
|
483
|
+
* sendonly track `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
|
|
468
484
|
* already at the rate the model's track declares (a step's frame carries
|
|
469
485
|
* no sample rate). Stepped sessions only, as `pushVideoFrame`.
|
|
470
486
|
*/
|
|
471
487
|
pushAudioFrame(name: string, samples: Int16Array, sample_rate: number, channels: number): void;
|
|
472
488
|
/**
|
|
473
|
-
*
|
|
474
|
-
*
|
|
489
|
+
* Push an encoded video frame on the sendonly video track `name`, its
|
|
490
|
+
* bytes as they are, sent or buffered as `pushVideoFrame` does.
|
|
475
491
|
*
|
|
476
|
-
* * `codec` — `"h264"` (the default
|
|
477
|
-
*
|
|
492
|
+
* * `codec` — `"h264"` (the default; one access unit, Annex B), `"vp8"`,
|
|
493
|
+
* `"vp9"` (one frame) or `"av1"` (one temporal unit of OBUs).
|
|
478
494
|
* * `width`, `height` — the frame it decodes to.
|
|
479
495
|
* * `metadata`, `captureTimeUs` — as for `pushVideoFrame`, carried with
|
|
480
496
|
* the decoded frame.
|
|
497
|
+
* * `keyframe` — whether the frame is one a decoder can start from.
|
|
481
498
|
*
|
|
482
|
-
* The runtime decodes a track's encoded frames
|
|
483
|
-
*
|
|
484
|
-
*
|
|
499
|
+
* The runtime decodes a track's encoded frames as one stream, as they
|
|
500
|
+
* arrive, so push them in decode order, starting at a keyframe. A pushed
|
|
501
|
+
* frame that does not decode raises an `input_decode_failed` error event
|
|
502
|
+
* whose `track` names the track, and the runtime drops the track's frames
|
|
503
|
+
* up to the next keyframe: `takeInputKeyframeRequest(name)` is then
|
|
504
|
+
* `true` once. A step whose buffered frames do not decode is refused
|
|
485
505
|
* (`step_refused`). Encoded and raw pushes to one track keep their push
|
|
486
506
|
* order. Stepped sessions only; throws on empty data or a zero dimension.
|
|
487
507
|
*/
|
|
488
|
-
pushEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, data: Uint8Array, width: number, height: number, metadata?: Uint8Array | null, capture_time_us?: number | null): void;
|
|
508
|
+
pushEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, data: Uint8Array, width: number, height: number, metadata?: Uint8Array | null, capture_time_us?: number | null, keyframe?: boolean | null): void;
|
|
489
509
|
/**
|
|
490
|
-
*
|
|
510
|
+
* Push a video frame on the sendonly track `name`: sent to the runtime at
|
|
511
|
+
* once in a ready session whose runtime takes pushed frames, buffered for
|
|
512
|
+
* the next `step()` otherwise.
|
|
491
513
|
*
|
|
492
514
|
* * `data` — `width * height` pixels in `format`: `"rgba"` (the default,
|
|
493
515
|
* what `ImageData` holds), `"rgb"` or `"bgra"`. Sent as RGB.
|
|
@@ -519,6 +541,13 @@ export class ReactorClient {
|
|
|
519
541
|
* Request the model's command schema (an OpenAPI document).
|
|
520
542
|
*/
|
|
521
543
|
requestSchema(): Promise<unknown>;
|
|
544
|
+
/**
|
|
545
|
+
* Restart output track `name`'s encoded step frames: the runtime starts
|
|
546
|
+
* its next frames at a keyframe. For the SDK, when the decoder it feeds
|
|
547
|
+
* for the track lost its stream. The track stays paused or resumed as
|
|
548
|
+
* the caller left it.
|
|
549
|
+
*/
|
|
550
|
+
restartTrackStream(name: string): void;
|
|
522
551
|
/**
|
|
523
552
|
* Resume a paused track.
|
|
524
553
|
*/
|
|
@@ -553,11 +582,32 @@ export class ReactorClient {
|
|
|
553
582
|
* disconnected: model, cluster, server info, selected transport.
|
|
554
583
|
*/
|
|
555
584
|
sessionInfo(): SessionInfo | undefined;
|
|
585
|
+
/**
|
|
586
|
+
* Set the codecs a stepped session prefers for its video step frames,
|
|
587
|
+
* most preferred first: for every track with `track` absent, or for one
|
|
588
|
+
* track in place of the session's order. `null`/`undefined` goes back to
|
|
589
|
+
* the default (H.264, VP9, AV1, VP8 for the session; the session's order
|
|
590
|
+
* for a track). An empty list for the session turns encoding off, and
|
|
591
|
+
* every frame travels raw; for one track it is taken as `null`.
|
|
592
|
+
*
|
|
593
|
+
* Callable before `connect()` or during the session: it takes effect on
|
|
594
|
+
* the next connection or reconnect. Each list is filtered by
|
|
595
|
+
* `setLocalStepCodecs`. Throws on a name that is not a codec.
|
|
596
|
+
*/
|
|
597
|
+
setCodecPreference(codecs?: EncodedVideoCodec[] | null, track?: string | null): void;
|
|
556
598
|
/**
|
|
557
599
|
* Replace the token source. Takes effect on the next request, so this is
|
|
558
600
|
* how a client built before sign-in gets its token.
|
|
559
601
|
*/
|
|
560
602
|
setJwt(jwt?: JwtSource | null): void;
|
|
603
|
+
/**
|
|
604
|
+
* Set the codecs this browser can carry step frames in: `encode`, for
|
|
605
|
+
* the frames pushed on input tracks, and `decode`, for the frames step
|
|
606
|
+
* results bring on output tracks. Until set, every track is raw. Read on
|
|
607
|
+
* the next offer, like `setCodecPreference`. Throws on a name that is
|
|
608
|
+
* not a codec.
|
|
609
|
+
*/
|
|
610
|
+
setLocalStepCodecs(encode: EncodedVideoCodec[], decode: EncodedVideoCodec[]): void;
|
|
561
611
|
/**
|
|
562
612
|
* Bound what one sender may spend, in bits per second. `undefined` or a
|
|
563
613
|
* negative value leaves a bound at the browser default.
|
|
@@ -585,9 +635,9 @@ export class ReactorClient {
|
|
|
585
635
|
* `set_<key>` command before `commands`; `null`/`undefined` for none.
|
|
586
636
|
* * `commands` — `[{ command, data? }]`, run in order after the state.
|
|
587
637
|
*
|
|
588
|
-
* Every frame
|
|
589
|
-
*
|
|
590
|
-
*
|
|
638
|
+
* Every frame still buffered rides this one, and belongs to it: when the
|
|
639
|
+
* request cannot be sent the step fails and its frames are dropped, not
|
|
640
|
+
* kept for the next step. Rejects with the core's code:
|
|
591
641
|
* `step_refused` (the runtime checks a whole request first, so a refused
|
|
592
642
|
* step changed nothing; `entry` names the part at fault),
|
|
593
643
|
* `step_not_ready` (the session cannot take a step yet; the step did not
|
|
@@ -603,8 +653,26 @@ export class ReactorClient {
|
|
|
603
653
|
* The step is checked and its frames taken before this returns, so a
|
|
604
654
|
* frame pushed once it has returned goes to the next step. A step refused
|
|
605
655
|
* by those checks throws rather than rejecting.
|
|
656
|
+
*
|
|
657
|
+
* An encoded output frame comes up still encoded (`keyframe` marks one),
|
|
658
|
+
* for the SDK to decode; `restartTrackStream()` restarts a track whose
|
|
659
|
+
* decoder lost its stream.
|
|
606
660
|
*/
|
|
607
661
|
step(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): Promise<StepResult>;
|
|
662
|
+
/**
|
|
663
|
+
* How each mapped track's step frames travel on this connection, as its
|
|
664
|
+
* SDP answer negotiated them: `{ [track]: "raw" | codec }`. Empty outside
|
|
665
|
+
* a connected stepped session; a track left out is raw.
|
|
666
|
+
*/
|
|
667
|
+
stepFrameEncodings(): Record<string, StepFrameEncoding>;
|
|
668
|
+
/**
|
|
669
|
+
* Whether the sendonly video track `name`'s next encoded frame must be a
|
|
670
|
+
* keyframe, because the runtime could not decode its frames
|
|
671
|
+
* (`input_decode_failed`) or frames encoded for it were not sent. Asking
|
|
672
|
+
* clears the mark: the caller about to encode the track's next frame
|
|
673
|
+
* makes it a keyframe.
|
|
674
|
+
*/
|
|
675
|
+
takeInputKeyframeRequest(name: string): boolean;
|
|
608
676
|
/**
|
|
609
677
|
* The negotiated `name` → `mid` mapping: `[{ name, kind, direction, mid }]`.
|
|
610
678
|
*/
|
|
@@ -658,21 +726,26 @@ export interface InitOutput {
|
|
|
658
726
|
readonly reactorclient_pausedTracks: (a: number) => [number, number, number];
|
|
659
727
|
readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: number) => any;
|
|
660
728
|
readonly reactorclient_pushAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
|
|
661
|
-
readonly reactorclient_pushEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
|
|
729
|
+
readonly reactorclient_pushEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number) => [number, number];
|
|
662
730
|
readonly reactorclient_pushVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
|
|
663
731
|
readonly reactorclient_reconnect: (a: number, b: number) => any;
|
|
664
732
|
readonly reactorclient_requestClip: (a: number, b: number) => any;
|
|
665
733
|
readonly reactorclient_requestRecording: (a: number) => any;
|
|
666
734
|
readonly reactorclient_requestSchema: (a: number) => any;
|
|
735
|
+
readonly reactorclient_restartTrackStream: (a: number, b: number, c: number) => [number, number];
|
|
667
736
|
readonly reactorclient_resumeTrack: (a: number, b: number, c: number) => any;
|
|
668
737
|
readonly reactorclient_sendClientStats: (a: number, b: any, c: any) => [number, number];
|
|
669
738
|
readonly reactorclient_sendCommand: (a: number, b: number, c: number, d: number, e: number) => any;
|
|
670
739
|
readonly reactorclient_sessionId: (a: number) => [number, number];
|
|
671
740
|
readonly reactorclient_sessionInfo: (a: number) => [number, number, number];
|
|
741
|
+
readonly reactorclient_setCodecPreference: (a: number, b: number, c: number, d: number) => [number, number];
|
|
672
742
|
readonly reactorclient_setJwt: (a: number, b: number) => [number, number];
|
|
743
|
+
readonly reactorclient_setLocalStepCodecs: (a: number, b: any, c: any) => [number, number];
|
|
673
744
|
readonly reactorclient_setTrackBitrate: (a: number, b: number, c: number, d: number, e: number) => any;
|
|
674
745
|
readonly reactorclient_status: (a: number) => any;
|
|
675
746
|
readonly reactorclient_step: (a: number, b: number, c: number) => [number, number, number];
|
|
747
|
+
readonly reactorclient_stepFrameEncodings: (a: number) => [number, number, number];
|
|
748
|
+
readonly reactorclient_takeInputKeyframeRequest: (a: number, b: number, c: number) => number;
|
|
676
749
|
readonly reactorclient_trackMapping: (a: number) => [number, number, number];
|
|
677
750
|
readonly reactorclient_tracks: (a: number) => [number, number, number];
|
|
678
751
|
readonly reactorclient_unpublishTrack: (a: number, b: number, c: number) => any;
|