@reactor-team/js-sdk 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -0
- package/dist/index.cjs +481 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +305 -5
- package/dist/index.d.ts +305 -5
- package/dist/index.js +472 -9
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +35 -2
- package/dist/wasm/reactor_wasm.d.ts +169 -2
- package/dist/wasm/reactor_wasm.js +238 -8
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +5 -1
- package/package.json +1 -1
package/dist/wasm/README.md
CHANGED
|
@@ -47,7 +47,8 @@ rather than `any`:
|
|
|
47
47
|
| Lifecycle | `connect(options?)`, `disconnect()`, `reconnect()` |
|
|
48
48
|
| Configuration | `setJwt(jwt)` |
|
|
49
49
|
| Messaging | `sendCommand(command, data?, uploads?)`, `requestSchema()` |
|
|
50
|
-
| Tracks | `publishTrack(name, track)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
|
|
50
|
+
| Tracks | `publishTrack(name, track?)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
|
|
51
|
+
| Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
|
|
51
52
|
| Recording | `requestClip(seconds)`, `requestRecording()` |
|
|
52
53
|
| Uploads | `uploadFile(file, name?)` |
|
|
53
54
|
| Introspection | `status()`, `sessionId()`, `sessionInfo()`, `capabilities()`, `lastError()` |
|
|
@@ -56,7 +57,39 @@ rather than `any`:
|
|
|
56
57
|
|
|
57
58
|
Payloads that come out of the core keep their wire names (`protocol_version`,
|
|
58
59
|
`retry_after_ms`, `playlist_url`); the options this crate defines itself are
|
|
59
|
-
camelCase, because nothing on the wire constrains them.
|
|
60
|
+
camelCase, because nothing on the wire constrains them. So is a step's result:
|
|
61
|
+
it is built by hand rather than serialized, because its frame data are typed
|
|
62
|
+
arrays (`Uint8Array` video, `Int16Array` audio) and serde has no word for one.
|
|
63
|
+
|
|
64
|
+
A client created with `generation: "stepped"` sends no input media over RTP.
|
|
65
|
+
`publishTrack(name)` only claims an input track (a `MediaStreamTrack` passed
|
|
66
|
+
there is ignored, with a console warning), `pushVideoFrame` / `pushAudioFrame`
|
|
67
|
+
buffer frames in the core, and the next `step()` carries them inside its one
|
|
68
|
+
`RequestStep`. Both of `step()`'s arguments are optional: an empty step runs on
|
|
69
|
+
the model's current state. Video arrives as RGBA, RGB or BGRA, and the core
|
|
70
|
+
converts it to the RGB a step sends in one pass.
|
|
71
|
+
|
|
72
|
+
The buffer holds at most one data-channel message's worth of frames: pushing
|
|
73
|
+
faster than steps run drops the oldest frames, across tracks, and the next step
|
|
74
|
+
logs how many. Pushed frames belong to the step that takes them: when that
|
|
75
|
+
step's request cannot be sent — `MESSAGE_TOO_LARGE` among the reasons — the
|
|
76
|
+
step rejects and its frames are dropped with it, so the next step starts from
|
|
77
|
+
what is pushed after. `clearStepInputs()` drops whatever is buffered now.
|
|
78
|
+
|
|
79
|
+
`step()` resolves with `{ step, tracks, elapsed, files, error? }`. Each track
|
|
80
|
+
is `{ track, kind, rate, frames }`, `rate` being the rate its frames play at as
|
|
81
|
+
the runtime reported it (frames per second for video, samples per second for
|
|
82
|
+
audio; 0 when it did not say). `files`
|
|
83
|
+
describes the files the model kept with the step (`name`, `contentType`,
|
|
84
|
+
`size`); their bytes are served over HTTP from the session's step results.
|
|
85
|
+
`error` is set when the step ran and the model reported a failure — such a
|
|
86
|
+
step still resolves. A step that did not run rejects with `step_refused`
|
|
87
|
+
(nothing it carried was applied), `step_not_ready` (the session cannot take a
|
|
88
|
+
step yet) or `step_command_failed` (a state key or command failed; the calls
|
|
89
|
+
before it stay applied). Both of those carry `entry`, the part of the request
|
|
90
|
+
at fault, and `step_command_failed` also `command_code`, its handler's code. One whose result was too large for a message
|
|
91
|
+
rejects with `step_result_too_large`, and one whose session ended before the
|
|
92
|
+
result came back with `step_session_ended`.
|
|
60
93
|
|
|
61
94
|
```js
|
|
62
95
|
import init, { ReactorClient } from "./pkg/reactor_wasm.js";
|
|
@@ -4,6 +4,9 @@
|
|
|
4
4
|
export type ReactorStatus = "disconnected" | "connecting" | "waiting" | "ready";
|
|
5
5
|
export type TrackKind = "audio" | "video";
|
|
6
6
|
export type TrackDirection = "recvonly" | "sendonly";
|
|
7
|
+
export type GenerationMode = "streamed" | "stepped";
|
|
8
|
+
/** Byte layout of a frame given to `pushVideoFrame`. */
|
|
9
|
+
export type PixelFormat = "rgba" | "rgb" | "bgra";
|
|
7
10
|
|
|
8
11
|
/** Construction options. Only `modelName` is required. */
|
|
9
12
|
export interface ClientOptions {
|
|
@@ -39,6 +42,14 @@ export interface ClientOptions {
|
|
|
39
42
|
/** Preset tracks, when known ahead of time — builds the SDP offer
|
|
40
43
|
* concurrently with the session-ready poll rather than after. */
|
|
41
44
|
modelTracks?: TrackCapability[];
|
|
45
|
+
/** How the session generates: "streamed" (the default), continuously over
|
|
46
|
+
* media tracks, or "stepped", one `step()` at a time. `connect()` fails
|
|
47
|
+
* with GENERATION_MODE_MISMATCH when the runtime did not start the
|
|
48
|
+
* session in this mode. */
|
|
49
|
+
generation?: GenerationMode;
|
|
50
|
+
/** How long `step()` waits for a step's result, generation included.
|
|
51
|
+
* Default 300000. */
|
|
52
|
+
stepTimeoutMs?: number;
|
|
42
53
|
/** Console log level. Default "warn". */
|
|
43
54
|
logLevel?: "off" | "error" | "warn" | "info" | "debug" | "trace";
|
|
44
55
|
}
|
|
@@ -149,6 +160,11 @@ export interface ReactorError {
|
|
|
149
160
|
operation?: string;
|
|
150
161
|
/** Backoff hint the platform sent. */
|
|
151
162
|
retry_after_ms?: number;
|
|
163
|
+
/** For `step_refused` and `step_command_failed`, the part of the step
|
|
164
|
+
* request at fault, e.g. `"commands[2]"`. */
|
|
165
|
+
entry?: string;
|
|
166
|
+
/** For `step_command_failed`, the code the failing handler raised. */
|
|
167
|
+
command_code?: string;
|
|
152
168
|
timestamp_ms: number;
|
|
153
169
|
}
|
|
154
170
|
|
|
@@ -170,6 +186,31 @@ export interface ClientTrackStat {
|
|
|
170
186
|
* frame_width, frame_height, nack_count, keyframe_requests. A reading the
|
|
171
187
|
* client couldn't take is left out, never sent as 0. */
|
|
172
188
|
metrics: Record<string, number>;
|
|
189
|
+
/** Where this track's frames spent their time since the previous batch,
|
|
190
|
+
* one entry per stage (e.g. "encode", "jitter_buffer", "decode"). Optional;
|
|
191
|
+
* an entry with no frames or a time that is not a finite, non-negative
|
|
192
|
+
* number is left out. */
|
|
193
|
+
frameStages?: FrameStage[];
|
|
194
|
+
/** Video only: the encoder (a sent track) or decoder (a received one) the
|
|
195
|
+
* browser runs, from getStats()'s encoderImplementation /
|
|
196
|
+
* decoderImplementation, e.g. "libvpx" or "ExternalDecoder". */
|
|
197
|
+
codecImplementation?: string;
|
|
198
|
+
/** Video only: whether that implementation is the hardware one, from
|
|
199
|
+
* powerEfficientEncoder / powerEfficientDecoder. */
|
|
200
|
+
hardwareCodec?: boolean;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** The time a track's frames spent in one stage of their trip, over one
|
|
204
|
+
* batch's window. The average per frame is `totalMs / frames`.
|
|
205
|
+
* encode_wait, packetize and pacer are different: they come from the
|
|
206
|
+
* engine's timing frames, a sample of about one slow frame per second, so
|
|
207
|
+
* `frames` counts sampled frames and `totalMs / frames` leans toward the
|
|
208
|
+
* slow ones. Treat their average as an upper bound, not a mean. */
|
|
209
|
+
export interface FrameStage {
|
|
210
|
+
name: string;
|
|
211
|
+
/** The time all `frames` frames spent in the stage together. */
|
|
212
|
+
totalMs: number;
|
|
213
|
+
frames: number;
|
|
173
214
|
}
|
|
174
215
|
|
|
175
216
|
/** A connection-wide reading, not tied to any one track — for
|
|
@@ -182,6 +223,77 @@ export interface ClientConnectionStat {
|
|
|
182
223
|
metrics: Record<string, number>;
|
|
183
224
|
}
|
|
184
225
|
|
|
226
|
+
/** One command of a step, run after its state, in order. */
|
|
227
|
+
export interface StepCommand {
|
|
228
|
+
command: string;
|
|
229
|
+
/** The command's arguments. Omitted or null for none. */
|
|
230
|
+
data?: Record<string, unknown> | null;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** One frame a step produced. Hand-built by the binding rather than
|
|
234
|
+
* serialized, so camelCase, unlike the wire payloads above. */
|
|
235
|
+
export interface StepFrame {
|
|
236
|
+
/** How the frame travelled; "raw" is the only one decoded today. */
|
|
237
|
+
encoding: "raw" | "unspecified";
|
|
238
|
+
/** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
|
|
239
|
+
dtype: string;
|
|
240
|
+
/** (height, width, 3) RGB for video, (channels, samples) planar PCM for
|
|
241
|
+
* audio. */
|
|
242
|
+
shape: number[];
|
|
243
|
+
/** The array, row-major, as the typed array `dtype` names — a Uint8Array
|
|
244
|
+
* for video, an Int16Array for audio, raw bytes for a dtype with no
|
|
245
|
+
* typed-array counterpart. */
|
|
246
|
+
data:
|
|
247
|
+
| Uint8Array
|
|
248
|
+
| Int8Array
|
|
249
|
+
| Uint16Array
|
|
250
|
+
| Int16Array
|
|
251
|
+
| Uint32Array
|
|
252
|
+
| Int32Array
|
|
253
|
+
| Float32Array
|
|
254
|
+
| Float64Array;
|
|
255
|
+
/** What the model attached to the frame. */
|
|
256
|
+
metadata?: Uint8Array;
|
|
257
|
+
/** When the model says the frame was captured, in microseconds. */
|
|
258
|
+
captureTimeUs?: number;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** The frames one output track produced in a step, in production order. */
|
|
262
|
+
export interface StepTrack {
|
|
263
|
+
track: string;
|
|
264
|
+
kind: TrackKind;
|
|
265
|
+
/** The rate the track's frames play at, as the runtime reported it:
|
|
266
|
+
* frames per second for video, samples per second for audio. 0 when the
|
|
267
|
+
* runtime did not say. */
|
|
268
|
+
rate: number;
|
|
269
|
+
frames: StepFrame[];
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** A file the model kept with a step. */
|
|
273
|
+
export interface StepFile {
|
|
274
|
+
name: string;
|
|
275
|
+
/** Its MIME type, e.g. "image/png". */
|
|
276
|
+
contentType: string;
|
|
277
|
+
/** Its size in bytes. */
|
|
278
|
+
size: number;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** What one step produced. */
|
|
282
|
+
export interface StepResult {
|
|
283
|
+
/** The step's number in the session, counted from 1. */
|
|
284
|
+
index: number;
|
|
285
|
+
tracks: StepTrack[];
|
|
286
|
+
/** Seconds the step's `generate()` took, or the time the model reported for the step. */
|
|
287
|
+
elapsed: number;
|
|
288
|
+
/** Files the model kept with the step. Only their description: the bytes
|
|
289
|
+
* are served over HTTP from the session's step results. Empty when the
|
|
290
|
+
* model kept none or the result was not saved. */
|
|
291
|
+
files: StepFile[];
|
|
292
|
+
/** Set when the step ran and the model reported a failure. Such a step
|
|
293
|
+
* still resolves; only a refused step rejects. */
|
|
294
|
+
error?: string;
|
|
295
|
+
}
|
|
296
|
+
|
|
185
297
|
export type StatusListener = (status: ReactorStatus) => void;
|
|
186
298
|
export type SessionIdListener = (sessionId: string | undefined) => void;
|
|
187
299
|
export type MessageListener = (message: ReactorMessage) => void;
|
|
@@ -214,6 +326,10 @@ export class ReactorClient {
|
|
|
214
326
|
* The runtime's capabilities, or `undefined` before they arrive.
|
|
215
327
|
*/
|
|
216
328
|
capabilities(): Capabilities | undefined;
|
|
329
|
+
/**
|
|
330
|
+
* Drop every frame buffered for the next step.
|
|
331
|
+
*/
|
|
332
|
+
clearStepInputs(): void;
|
|
217
333
|
/**
|
|
218
334
|
* Create (or adopt) a session and bring up the transport.
|
|
219
335
|
*/
|
|
@@ -305,8 +421,32 @@ export class ReactorClient {
|
|
|
305
421
|
pausedTracks(): string[];
|
|
306
422
|
/**
|
|
307
423
|
* Claim a sendonly track and start sending `track` on it.
|
|
424
|
+
*
|
|
425
|
+
* In a stepped session the track has no RTP sender, so there is nothing
|
|
426
|
+
* to attach a `MediaStreamTrack` to: `publishTrack(name)` only claims the
|
|
427
|
+
* track, and its frames come from `pushVideoFrame` / `pushAudioFrame`.
|
|
428
|
+
* A `MediaStreamTrack` passed there is ignored, with a warning.
|
|
429
|
+
*/
|
|
430
|
+
publishTrack(name: string, track?: MediaStreamTrack | null): Promise<void>;
|
|
431
|
+
/**
|
|
432
|
+
* Buffer a block of audio for the next `step()` on the sendonly track
|
|
433
|
+
* `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
|
|
434
|
+
* already at the rate the model's track declares (a step's frame carries
|
|
435
|
+
* no sample rate). Stepped sessions only, as `pushVideoFrame`.
|
|
308
436
|
*/
|
|
309
|
-
|
|
437
|
+
pushAudioFrame(name: string, samples: Int16Array, sample_rate: number, channels: number): void;
|
|
438
|
+
/**
|
|
439
|
+
* Buffer a video frame for the next `step()` on the sendonly track `name`.
|
|
440
|
+
*
|
|
441
|
+
* * `data` — `width * height` pixels in `format`: `"rgba"` (the default,
|
|
442
|
+
* what `ImageData` holds), `"rgb"` or `"bgra"`. Sent as RGB.
|
|
443
|
+
* * `metadata` — opaque bytes the frame carries to the model.
|
|
444
|
+
* * `captureTimeUs` — when the frame was captured, in microseconds.
|
|
445
|
+
*
|
|
446
|
+
* Stepped sessions only: a streamed session sends media from a
|
|
447
|
+
* `MediaStreamTrack` given to `publishTrack`.
|
|
448
|
+
*/
|
|
449
|
+
pushVideoFrame(name: string, data: Uint8Array, width: number, height: number, format?: PixelFormat | null, metadata?: Uint8Array | null, capture_time_us?: number | null): void;
|
|
310
450
|
/**
|
|
311
451
|
* Rebuild the transport on the same session, without ending it.
|
|
312
452
|
*
|
|
@@ -387,6 +527,29 @@ export class ReactorClient {
|
|
|
387
527
|
* `"disconnected"` | `"connecting"` | `"waiting"` | `"ready"`.
|
|
388
528
|
*/
|
|
389
529
|
status(): ReactorStatus;
|
|
530
|
+
/**
|
|
531
|
+
* Run one step of a stepped session and resolve with what it produced.
|
|
532
|
+
*
|
|
533
|
+
* * `state` — an object of the model's state fields, each applied as its
|
|
534
|
+
* `set_<key>` command before `commands`; `null`/`undefined` for none.
|
|
535
|
+
* * `commands` — `[{ command, data? }]`, run in order after the state.
|
|
536
|
+
*
|
|
537
|
+
* Every frame pushed since the previous step rides this one, and belongs
|
|
538
|
+
* to it: when the request cannot be sent the step fails and its frames
|
|
539
|
+
* are dropped, not kept for the next step. Rejects with the core's code:
|
|
540
|
+
* `step_refused` (the runtime checks a whole request first, so a refused
|
|
541
|
+
* step changed nothing; `entry` names the part at fault),
|
|
542
|
+
* `step_not_ready` (the session cannot take a step yet; the step did not
|
|
543
|
+
* run), `step_command_failed` (a state key or command failed; the calls
|
|
544
|
+
* before it stay applied and the step did not run; `entry` names the call
|
|
545
|
+
* and `command_code` its handler's code), `step_session_ended`
|
|
546
|
+
* (the session ended before the result came back),
|
|
547
|
+
* `step_result_too_large`, `MESSAGE_TOO_LARGE` (the request is over the
|
|
548
|
+
* data channel's message limit; nothing was sent),
|
|
549
|
+
* `UNSUPPORTED_FRAME_ENCODING`, or the code of whatever else failed the
|
|
550
|
+
* step (`DISCONNECTED`, `REQUEST_TIMEOUT`, …).
|
|
551
|
+
*/
|
|
552
|
+
step(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): Promise<StepResult>;
|
|
390
553
|
/**
|
|
391
554
|
* The negotiated `name` → `mid` mapping: `[{ name, kind, direction, mid }]`.
|
|
392
555
|
*/
|
|
@@ -415,6 +578,7 @@ export interface InitOutput {
|
|
|
415
578
|
readonly memory: WebAssembly.Memory;
|
|
416
579
|
readonly __wbg_reactorclient_free: (a: number, b: number) => void;
|
|
417
580
|
readonly reactorclient_capabilities: (a: number) => [number, number, number];
|
|
581
|
+
readonly reactorclient_clearStepInputs: (a: number) => void;
|
|
418
582
|
readonly reactorclient_connect: (a: number, b: number) => any;
|
|
419
583
|
readonly reactorclient_disconnect: (a: number) => any;
|
|
420
584
|
readonly reactorclient_getPeerConnection: (a: number) => any;
|
|
@@ -433,7 +597,9 @@ export interface InitOutput {
|
|
|
433
597
|
readonly reactorclient_onTrackReceived: (a: number, b: any) => void;
|
|
434
598
|
readonly reactorclient_pauseTrack: (a: number, b: number, c: number) => any;
|
|
435
599
|
readonly reactorclient_pausedTracks: (a: number) => [number, number, number];
|
|
436
|
-
readonly reactorclient_publishTrack: (a: number, b: number, c: number, d:
|
|
600
|
+
readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: number) => any;
|
|
601
|
+
readonly reactorclient_pushAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
|
|
602
|
+
readonly reactorclient_pushVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
|
|
437
603
|
readonly reactorclient_reconnect: (a: number, b: number) => any;
|
|
438
604
|
readonly reactorclient_requestClip: (a: number, b: number) => any;
|
|
439
605
|
readonly reactorclient_requestRecording: (a: number) => any;
|
|
@@ -446,6 +612,7 @@ export interface InitOutput {
|
|
|
446
612
|
readonly reactorclient_setJwt: (a: number, b: number) => [number, number];
|
|
447
613
|
readonly reactorclient_setTrackBitrate: (a: number, b: number, c: number, d: number, e: number) => any;
|
|
448
614
|
readonly reactorclient_status: (a: number) => any;
|
|
615
|
+
readonly reactorclient_step: (a: number, b: number, c: number) => any;
|
|
449
616
|
readonly reactorclient_trackMapping: (a: number) => [number, number, number];
|
|
450
617
|
readonly reactorclient_tracks: (a: number) => [number, number, number];
|
|
451
618
|
readonly reactorclient_unpublishTrack: (a: number, b: number, c: number) => any;
|