@reactor-team/js-sdk 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -11
- package/dist/index.cjs +539 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +340 -22
- package/dist/index.d.ts +340 -22
- package/dist/index.js +530 -45
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +35 -2
- package/dist/wasm/reactor_wasm.d.ts +184 -8
- package/dist/wasm/reactor_wasm.js +254 -11
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +5 -1
- package/package.json +1 -1
- package/dist/wasm/reactor_wasm_bg.wasm +0 -0
package/dist/wasm/README.md
CHANGED
|
@@ -47,7 +47,8 @@ rather than `any`:
|
|
|
47
47
|
| Lifecycle | `connect(options?)`, `disconnect()`, `reconnect()` |
|
|
48
48
|
| Configuration | `setJwt(jwt)` |
|
|
49
49
|
| Messaging | `sendCommand(command, data?, uploads?)`, `requestSchema()` |
|
|
50
|
-
| Tracks | `publishTrack(name, track)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
|
|
50
|
+
| Tracks | `publishTrack(name, track?)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
|
|
51
|
+
| Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
|
|
51
52
|
| Recording | `requestClip(seconds)`, `requestRecording()` |
|
|
52
53
|
| Uploads | `uploadFile(file, name?)` |
|
|
53
54
|
| Introspection | `status()`, `sessionId()`, `sessionInfo()`, `capabilities()`, `lastError()` |
|
|
@@ -56,7 +57,39 @@ rather than `any`:
|
|
|
56
57
|
|
|
57
58
|
Payloads that come out of the core keep their wire names (`protocol_version`,
|
|
58
59
|
`retry_after_ms`, `playlist_url`); the options this crate defines itself are
|
|
59
|
-
camelCase, because nothing on the wire constrains them.
|
|
60
|
+
camelCase, because nothing on the wire constrains them. So is a step's result:
|
|
61
|
+
it is built by hand rather than serialized, because its frame data are typed
|
|
62
|
+
arrays (`Uint8Array` video, `Int16Array` audio) and serde has no word for one.
|
|
63
|
+
|
|
64
|
+
A client created with `generation: "stepped"` sends no input media over RTP.
|
|
65
|
+
`publishTrack(name)` only claims an input track (a `MediaStreamTrack` passed
|
|
66
|
+
there is ignored, with a console warning), `pushVideoFrame` / `pushAudioFrame`
|
|
67
|
+
buffer frames in the core, and the next `step()` carries them inside its one
|
|
68
|
+
`RequestStep`. Both of `step()`'s arguments are optional: an empty step runs on
|
|
69
|
+
the model's current state. Video arrives as RGBA, RGB or BGRA, and the core
|
|
70
|
+
converts it to the RGB a step sends in one pass.
|
|
71
|
+
|
|
72
|
+
The buffer holds at most one data-channel message's worth of frames: pushing
|
|
73
|
+
faster than steps run drops the oldest frames, across tracks, and the next step
|
|
74
|
+
logs how many. Pushed frames belong to the step that takes them: when that
|
|
75
|
+
step's request cannot be sent — `MESSAGE_TOO_LARGE` among the reasons — the
|
|
76
|
+
step rejects and its frames are dropped with it, so the next step starts from
|
|
77
|
+
what is pushed after. `clearStepInputs()` drops whatever is buffered now.
|
|
78
|
+
|
|
79
|
+
`step()` resolves with `{ step, tracks, elapsed, files, error? }`. Each track
|
|
80
|
+
is `{ track, kind, rate, frames }`, `rate` being the rate its frames play at as
|
|
81
|
+
the runtime reported it (frames per second for video, samples per second for
|
|
82
|
+
audio; 0 when it did not say). `files`
|
|
83
|
+
describes the files the model kept with the step (`name`, `contentType`,
|
|
84
|
+
`size`); their bytes are served over HTTP from the session's step results.
|
|
85
|
+
`error` is set when the step ran and the model reported a failure — such a
|
|
86
|
+
step still resolves. A step that did not run rejects with `step_refused`
|
|
87
|
+
(nothing it carried was applied), `step_not_ready` (the session cannot take a
|
|
88
|
+
step yet) or `step_command_failed` (a state key or command failed; the calls
|
|
89
|
+
before it stay applied). Both of those carry `entry`, the part of the request
|
|
90
|
+
at fault, and `step_command_failed` also `command_code`, its handler's code. One whose result was too large for a message
|
|
91
|
+
rejects with `step_result_too_large`, and one whose session ended before the
|
|
92
|
+
result came back with `step_session_ended`.
|
|
60
93
|
|
|
61
94
|
```js
|
|
62
95
|
import init, { ReactorClient } from "./pkg/reactor_wasm.js";
|
|
@@ -4,6 +4,9 @@
|
|
|
4
4
|
export type ReactorStatus = "disconnected" | "connecting" | "waiting" | "ready";
|
|
5
5
|
export type TrackKind = "audio" | "video";
|
|
6
6
|
export type TrackDirection = "recvonly" | "sendonly";
|
|
7
|
+
export type GenerationMode = "streamed" | "stepped";
|
|
8
|
+
/** Byte layout of a frame given to `pushVideoFrame`. */
|
|
9
|
+
export type PixelFormat = "rgba" | "rgb" | "bgra";
|
|
7
10
|
|
|
8
11
|
/** Construction options. Only `modelName` is required. */
|
|
9
12
|
export interface ClientOptions {
|
|
@@ -39,6 +42,14 @@ export interface ClientOptions {
|
|
|
39
42
|
/** Preset tracks, when known ahead of time — builds the SDP offer
|
|
40
43
|
* concurrently with the session-ready poll rather than after. */
|
|
41
44
|
modelTracks?: TrackCapability[];
|
|
45
|
+
/** How the session generates: "streamed" (the default), continuously over
|
|
46
|
+
* media tracks, or "stepped", one `step()` at a time. `connect()` fails
|
|
47
|
+
* with GENERATION_MODE_MISMATCH when the runtime did not start the
|
|
48
|
+
* session in this mode. */
|
|
49
|
+
generation?: GenerationMode;
|
|
50
|
+
/** How long `step()` waits for a step's result, generation included.
|
|
51
|
+
* Default 300000. */
|
|
52
|
+
stepTimeoutMs?: number;
|
|
42
53
|
/** Console log level. Default "warn". */
|
|
43
54
|
logLevel?: "off" | "error" | "warn" | "info" | "debug" | "trace";
|
|
44
55
|
}
|
|
@@ -59,9 +70,17 @@ export interface ConnectOptions {
|
|
|
59
70
|
maxAttempts?: number;
|
|
60
71
|
}
|
|
61
72
|
|
|
62
|
-
/**
|
|
63
|
-
*
|
|
64
|
-
|
|
73
|
+
/** What a token is being resolved for. `sessionId` is set for every request
|
|
74
|
+
* that targets an existing session and unset for session creation. A
|
|
75
|
+
* resolver that mints a replacement token must bind it to that session
|
|
76
|
+
* (`resources.sessions.bind`), or the token cannot reach the session. */
|
|
77
|
+
export interface JwtRequestContext {
|
|
78
|
+
sessionId?: string;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** A token, or a resolver called before every authenticated request with a
|
|
82
|
+
* `JwtRequestContext`. Returning "" sends no Authorization header. */
|
|
83
|
+
export type JwtSource = string | ((context?: JwtRequestContext) => string | Promise<string>);
|
|
65
84
|
|
|
66
85
|
export interface TrackCapability {
|
|
67
86
|
name: string;
|
|
@@ -141,6 +160,11 @@ export interface ReactorError {
|
|
|
141
160
|
operation?: string;
|
|
142
161
|
/** Backoff hint the platform sent. */
|
|
143
162
|
retry_after_ms?: number;
|
|
163
|
+
/** For `step_refused` and `step_command_failed`, the part of the step
|
|
164
|
+
* request at fault, e.g. `"commands[2]"`. */
|
|
165
|
+
entry?: string;
|
|
166
|
+
/** For `step_command_failed`, the code the failing handler raised. */
|
|
167
|
+
command_code?: string;
|
|
144
168
|
timestamp_ms: number;
|
|
145
169
|
}
|
|
146
170
|
|
|
@@ -162,6 +186,31 @@ export interface ClientTrackStat {
|
|
|
162
186
|
* frame_width, frame_height, nack_count, keyframe_requests. A reading the
|
|
163
187
|
* client couldn't take is left out, never sent as 0. */
|
|
164
188
|
metrics: Record<string, number>;
|
|
189
|
+
/** Where this track's frames spent their time since the previous batch,
|
|
190
|
+
* one entry per stage (e.g. "encode", "jitter_buffer", "decode"). Optional;
|
|
191
|
+
* an entry with no frames or a time that is not a finite, non-negative
|
|
192
|
+
* number is left out. */
|
|
193
|
+
frameStages?: FrameStage[];
|
|
194
|
+
/** Video only: the encoder (a sent track) or decoder (a received one) the
|
|
195
|
+
* browser runs, from getStats()'s encoderImplementation /
|
|
196
|
+
* decoderImplementation, e.g. "libvpx" or "ExternalDecoder". */
|
|
197
|
+
codecImplementation?: string;
|
|
198
|
+
/** Video only: whether that implementation is the hardware one, from
|
|
199
|
+
* powerEfficientEncoder / powerEfficientDecoder. */
|
|
200
|
+
hardwareCodec?: boolean;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** The time a track's frames spent in one stage of their trip, over one
|
|
204
|
+
* batch's window. The average per frame is `totalMs / frames`.
|
|
205
|
+
* encode_wait, packetize and pacer are different: they come from the
|
|
206
|
+
* engine's timing frames, a sample of about one slow frame per second, so
|
|
207
|
+
* `frames` counts sampled frames and `totalMs / frames` leans toward the
|
|
208
|
+
* slow ones. Treat their average as an upper bound, not a mean. */
|
|
209
|
+
export interface FrameStage {
|
|
210
|
+
name: string;
|
|
211
|
+
/** The time all `frames` frames spent in the stage together. */
|
|
212
|
+
totalMs: number;
|
|
213
|
+
frames: number;
|
|
165
214
|
}
|
|
166
215
|
|
|
167
216
|
/** A connection-wide reading, not tied to any one track — for
|
|
@@ -174,6 +223,77 @@ export interface ClientConnectionStat {
|
|
|
174
223
|
metrics: Record<string, number>;
|
|
175
224
|
}
|
|
176
225
|
|
|
226
|
+
/** One command of a step, run after its state, in order. */
|
|
227
|
+
export interface StepCommand {
|
|
228
|
+
command: string;
|
|
229
|
+
/** The command's arguments. Omitted or null for none. */
|
|
230
|
+
data?: Record<string, unknown> | null;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** One frame a step produced. Hand-built by the binding rather than
|
|
234
|
+
* serialized, so camelCase, unlike the wire payloads above. */
|
|
235
|
+
export interface StepFrame {
|
|
236
|
+
/** How the frame travelled; "raw" is the only one decoded today. */
|
|
237
|
+
encoding: "raw" | "unspecified";
|
|
238
|
+
/** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
|
|
239
|
+
dtype: string;
|
|
240
|
+
/** (height, width, 3) RGB for video, (channels, samples) planar PCM for
|
|
241
|
+
* audio. */
|
|
242
|
+
shape: number[];
|
|
243
|
+
/** The array, row-major, as the typed array `dtype` names — a Uint8Array
|
|
244
|
+
* for video, an Int16Array for audio, raw bytes for a dtype with no
|
|
245
|
+
* typed-array counterpart. */
|
|
246
|
+
data:
|
|
247
|
+
| Uint8Array
|
|
248
|
+
| Int8Array
|
|
249
|
+
| Uint16Array
|
|
250
|
+
| Int16Array
|
|
251
|
+
| Uint32Array
|
|
252
|
+
| Int32Array
|
|
253
|
+
| Float32Array
|
|
254
|
+
| Float64Array;
|
|
255
|
+
/** What the model attached to the frame. */
|
|
256
|
+
metadata?: Uint8Array;
|
|
257
|
+
/** When the model says the frame was captured, in microseconds. */
|
|
258
|
+
captureTimeUs?: number;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** The frames one output track produced in a step, in production order. */
|
|
262
|
+
export interface StepTrack {
|
|
263
|
+
track: string;
|
|
264
|
+
kind: TrackKind;
|
|
265
|
+
/** The rate the track's frames play at, as the runtime reported it:
|
|
266
|
+
* frames per second for video, samples per second for audio. 0 when the
|
|
267
|
+
* runtime did not say. */
|
|
268
|
+
rate: number;
|
|
269
|
+
frames: StepFrame[];
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** A file the model kept with a step. */
|
|
273
|
+
export interface StepFile {
|
|
274
|
+
name: string;
|
|
275
|
+
/** Its MIME type, e.g. "image/png". */
|
|
276
|
+
contentType: string;
|
|
277
|
+
/** Its size in bytes. */
|
|
278
|
+
size: number;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** What one step produced. */
|
|
282
|
+
export interface StepResult {
|
|
283
|
+
/** The step's number in the session, counted from 1. */
|
|
284
|
+
index: number;
|
|
285
|
+
tracks: StepTrack[];
|
|
286
|
+
/** Seconds the step's `generate()` took, or the time the model reported for the step. */
|
|
287
|
+
elapsed: number;
|
|
288
|
+
/** Files the model kept with the step. Only their description: the bytes
|
|
289
|
+
* are served over HTTP from the session's step results. Empty when the
|
|
290
|
+
* model kept none or the result was not saved. */
|
|
291
|
+
files: StepFile[];
|
|
292
|
+
/** Set when the step ran and the model reported a failure. Such a step
|
|
293
|
+
* still resolves; only a refused step rejects. */
|
|
294
|
+
error?: string;
|
|
295
|
+
}
|
|
296
|
+
|
|
177
297
|
export type StatusListener = (status: ReactorStatus) => void;
|
|
178
298
|
export type SessionIdListener = (sessionId: string | undefined) => void;
|
|
179
299
|
export type MessageListener = (message: ReactorMessage) => void;
|
|
@@ -206,6 +326,10 @@ export class ReactorClient {
|
|
|
206
326
|
* The runtime's capabilities, or `undefined` before they arrive.
|
|
207
327
|
*/
|
|
208
328
|
capabilities(): Capabilities | undefined;
|
|
329
|
+
/**
|
|
330
|
+
* Drop every frame buffered for the next step.
|
|
331
|
+
*/
|
|
332
|
+
clearStepInputs(): void;
|
|
209
333
|
/**
|
|
210
334
|
* Create (or adopt) a session and bring up the transport.
|
|
211
335
|
*/
|
|
@@ -249,9 +373,10 @@ export class ReactorClient {
|
|
|
249
373
|
* Create a client.
|
|
250
374
|
*
|
|
251
375
|
* * `options` — see the `ClientOptions` fields; `modelName` is required.
|
|
252
|
-
* * `jwt` — a token string, a `() => string | Promise<string>`
|
|
253
|
-
* called before every authenticated request
|
|
254
|
-
* unauthenticated local runtime.
|
|
376
|
+
* * `jwt` — a token string, a `(context) => string | Promise<string>`
|
|
377
|
+
* resolver called before every authenticated request with the session
|
|
378
|
+
* it is for, or `null` for an unauthenticated local runtime.
|
|
379
|
+
* Replaceable later with `setJwt`.
|
|
255
380
|
*/
|
|
256
381
|
constructor(options: ClientOptions, jwt?: JwtSource | null);
|
|
257
382
|
/**
|
|
@@ -296,8 +421,32 @@ export class ReactorClient {
|
|
|
296
421
|
pausedTracks(): string[];
|
|
297
422
|
/**
|
|
298
423
|
* Claim a sendonly track and start sending `track` on it.
|
|
424
|
+
*
|
|
425
|
+
* In a stepped session the track has no RTP sender, so there is nothing
|
|
426
|
+
* to attach a `MediaStreamTrack` to: `publishTrack(name)` only claims the
|
|
427
|
+
* track, and its frames come from `pushVideoFrame` / `pushAudioFrame`.
|
|
428
|
+
* A `MediaStreamTrack` passed there is ignored, with a warning.
|
|
429
|
+
*/
|
|
430
|
+
publishTrack(name: string, track?: MediaStreamTrack | null): Promise<void>;
|
|
431
|
+
/**
|
|
432
|
+
* Buffer a block of audio for the next `step()` on the sendonly track
|
|
433
|
+
* `name`: interleaved signed 16-bit PCM, `channels` samples per frame,
|
|
434
|
+
* already at the rate the model's track declares (a step's frame carries
|
|
435
|
+
* no sample rate). Stepped sessions only, as `pushVideoFrame`.
|
|
299
436
|
*/
|
|
300
|
-
|
|
437
|
+
pushAudioFrame(name: string, samples: Int16Array, sample_rate: number, channels: number): void;
|
|
438
|
+
/**
|
|
439
|
+
* Buffer a video frame for the next `step()` on the sendonly track `name`.
|
|
440
|
+
*
|
|
441
|
+
* * `data` — `width * height` pixels in `format`: `"rgba"` (the default,
|
|
442
|
+
* what `ImageData` holds), `"rgb"` or `"bgra"`. Sent as RGB.
|
|
443
|
+
* * `metadata` — opaque bytes the frame carries to the model.
|
|
444
|
+
* * `captureTimeUs` — when the frame was captured, in microseconds.
|
|
445
|
+
*
|
|
446
|
+
* Stepped sessions only: a streamed session sends media from a
|
|
447
|
+
* `MediaStreamTrack` given to `publishTrack`.
|
|
448
|
+
*/
|
|
449
|
+
pushVideoFrame(name: string, data: Uint8Array, width: number, height: number, format?: PixelFormat | null, metadata?: Uint8Array | null, capture_time_us?: number | null): void;
|
|
301
450
|
/**
|
|
302
451
|
* Rebuild the transport on the same session, without ending it.
|
|
303
452
|
*
|
|
@@ -378,6 +527,29 @@ export class ReactorClient {
|
|
|
378
527
|
* `"disconnected"` | `"connecting"` | `"waiting"` | `"ready"`.
|
|
379
528
|
*/
|
|
380
529
|
status(): ReactorStatus;
|
|
530
|
+
/**
|
|
531
|
+
* Run one step of a stepped session and resolve with what it produced.
|
|
532
|
+
*
|
|
533
|
+
* * `state` — an object of the model's state fields, each applied as its
|
|
534
|
+
* `set_<key>` command before `commands`; `null`/`undefined` for none.
|
|
535
|
+
* * `commands` — `[{ command, data? }]`, run in order after the state.
|
|
536
|
+
*
|
|
537
|
+
* Every frame pushed since the previous step rides this one, and belongs
|
|
538
|
+
* to it: when the request cannot be sent the step fails and its frames
|
|
539
|
+
* are dropped, not kept for the next step. Rejects with the core's code:
|
|
540
|
+
* `step_refused` (the runtime checks a whole request first, so a refused
|
|
541
|
+
* step changed nothing; `entry` names the part at fault),
|
|
542
|
+
* `step_not_ready` (the session cannot take a step yet; the step did not
|
|
543
|
+
* run), `step_command_failed` (a state key or command failed; the calls
|
|
544
|
+
* before it stay applied and the step did not run; `entry` names the call
|
|
545
|
+
* and `command_code` its handler's code), `step_session_ended`
|
|
546
|
+
* (the session ended before the result came back),
|
|
547
|
+
* `step_result_too_large`, `MESSAGE_TOO_LARGE` (the request is over the
|
|
548
|
+
* data channel's message limit; nothing was sent),
|
|
549
|
+
* `UNSUPPORTED_FRAME_ENCODING`, or the code of whatever else failed the
|
|
550
|
+
* step (`DISCONNECTED`, `REQUEST_TIMEOUT`, …).
|
|
551
|
+
*/
|
|
552
|
+
step(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): Promise<StepResult>;
|
|
381
553
|
/**
|
|
382
554
|
* The negotiated `name` → `mid` mapping: `[{ name, kind, direction, mid }]`.
|
|
383
555
|
*/
|
|
@@ -406,6 +578,7 @@ export interface InitOutput {
|
|
|
406
578
|
readonly memory: WebAssembly.Memory;
|
|
407
579
|
readonly __wbg_reactorclient_free: (a: number, b: number) => void;
|
|
408
580
|
readonly reactorclient_capabilities: (a: number) => [number, number, number];
|
|
581
|
+
readonly reactorclient_clearStepInputs: (a: number) => void;
|
|
409
582
|
readonly reactorclient_connect: (a: number, b: number) => any;
|
|
410
583
|
readonly reactorclient_disconnect: (a: number) => any;
|
|
411
584
|
readonly reactorclient_getPeerConnection: (a: number) => any;
|
|
@@ -424,7 +597,9 @@ export interface InitOutput {
|
|
|
424
597
|
readonly reactorclient_onTrackReceived: (a: number, b: any) => void;
|
|
425
598
|
readonly reactorclient_pauseTrack: (a: number, b: number, c: number) => any;
|
|
426
599
|
readonly reactorclient_pausedTracks: (a: number) => [number, number, number];
|
|
427
|
-
readonly reactorclient_publishTrack: (a: number, b: number, c: number, d:
|
|
600
|
+
readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: number) => any;
|
|
601
|
+
readonly reactorclient_pushAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
|
|
602
|
+
readonly reactorclient_pushVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
|
|
428
603
|
readonly reactorclient_reconnect: (a: number, b: number) => any;
|
|
429
604
|
readonly reactorclient_requestClip: (a: number, b: number) => any;
|
|
430
605
|
readonly reactorclient_requestRecording: (a: number) => any;
|
|
@@ -437,6 +612,7 @@ export interface InitOutput {
|
|
|
437
612
|
readonly reactorclient_setJwt: (a: number, b: number) => [number, number];
|
|
438
613
|
readonly reactorclient_setTrackBitrate: (a: number, b: number, c: number, d: number, e: number) => any;
|
|
439
614
|
readonly reactorclient_status: (a: number) => any;
|
|
615
|
+
readonly reactorclient_step: (a: number, b: number, c: number) => any;
|
|
440
616
|
readonly reactorclient_trackMapping: (a: number) => [number, number, number];
|
|
441
617
|
readonly reactorclient_tracks: (a: number) => [number, number, number];
|
|
442
618
|
readonly reactorclient_unpublishTrack: (a: number, b: number, c: number) => any;
|