@reactor-team/js-sdk 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -0
- package/dist/index.cjs +296 -25
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +98 -4
- package/dist/index.d.ts +98 -4
- package/dist/index.js +296 -25
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +5 -2
- package/dist/wasm/reactor_wasm.d.ts +61 -1
- package/dist/wasm/reactor_wasm.js +117 -7
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +6 -1
- package/package.json +1 -1
package/dist/wasm/README.md
CHANGED
|
@@ -48,7 +48,7 @@ rather than `any`:
|
|
|
48
48
|
| Configuration | `setJwt(jwt)` |
|
|
49
49
|
| Messaging | `sendCommand(command, data?, uploads?)`, `requestSchema()` |
|
|
50
50
|
| Tracks | `publishTrack(name, track?)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
|
|
51
|
-
| Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
|
|
51
|
+
| Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushEncodedVideoFrame(name, codec, data, width, height, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
|
|
52
52
|
| Recording | `requestClip(seconds)`, `requestRecording()` |
|
|
53
53
|
| Uploads | `uploadFile(file, name?)` |
|
|
54
54
|
| Introspection | `status()`, `sessionId()`, `sessionInfo()`, `capabilities()`, `lastError()` |
|
|
@@ -67,7 +67,10 @@ there is ignored, with a console warning), `pushVideoFrame` / `pushAudioFrame`
|
|
|
67
67
|
buffer frames in the core, and the next `step()` carries them inside its one
|
|
68
68
|
`RequestStep`. Both of `step()`'s arguments are optional: an empty step runs on
|
|
69
69
|
the model's current state. Video arrives as RGBA, RGB or BGRA, and the core
|
|
70
|
-
converts it to the RGB a step sends in one pass.
|
|
70
|
+
converts it to the RGB a step sends in one pass. `pushEncodedVideoFrame` buffers
|
|
71
|
+
an H.264 access unit (Annex B) as it is, with the size it decodes to; the runtime
|
|
72
|
+
decodes a track's encoded frames of one step, in push order and starting at a
|
|
73
|
+
keyframe, before the step runs.
|
|
71
74
|
|
|
72
75
|
The buffer holds at most one data-channel message's worth of frames: pushing
|
|
73
76
|
faster than steps run drops the oldest frames, across tracks, and the next step
|
|
@@ -7,6 +7,8 @@ export type TrackDirection = "recvonly" | "sendonly";
|
|
|
7
7
|
export type GenerationMode = "streamed" | "stepped";
|
|
8
8
|
/** Byte layout of a frame given to `pushVideoFrame`. */
|
|
9
9
|
export type PixelFormat = "rgba" | "rgb" | "bgra";
|
|
10
|
+
/** Codec of an encoded frame given to `pushEncodedVideoFrame`. */
|
|
11
|
+
export type EncodedVideoCodec = "h264";
|
|
10
12
|
|
|
11
13
|
/** Construction options. Only `modelName` is required. */
|
|
12
14
|
export interface ClientOptions {
|
|
@@ -50,6 +52,14 @@ export interface ClientOptions {
|
|
|
50
52
|
/** How long `step()` waits for a step's result, generation included.
|
|
51
53
|
* Default 300000. */
|
|
52
54
|
stepTimeoutMs?: number;
|
|
55
|
+
/** The largest data-channel message, in bytes, the client declares and
|
|
56
|
+
* accepts: from 262144 (256 KiB) to 1073741824 (1 GiB). Default 67108864
|
|
57
|
+
* (64 MiB). A connection carries the smaller of this and the runtime's own
|
|
58
|
+
* limit, which is 64 MiB unless the model raises it, so a `step()` whose
|
|
59
|
+
* result is larger than either fails with `StepResultTooLargeError`. A
|
|
60
|
+
* message is held whole in memory while it arrives, so raise this only for
|
|
61
|
+
* a model whose steps need it. */
|
|
62
|
+
maxMessageSize?: number;
|
|
53
63
|
/** Console log level. Default "warn". */
|
|
54
64
|
logLevel?: "off" | "error" | "warn" | "info" | "debug" | "trace";
|
|
55
65
|
}
|
|
@@ -326,6 +336,30 @@ export class ReactorClient {
|
|
|
326
336
|
* The runtime's capabilities, or `undefined` before they arrive.
|
|
327
337
|
*/
|
|
328
338
|
capabilities(): Capabilities | undefined;
|
|
339
|
+
/**
|
|
340
|
+
* Whether `pushAudioFrame` would refuse `samples`: throws what it would
|
|
341
|
+
* throw, and buffers nothing.
|
|
342
|
+
*/
|
|
343
|
+
checkAudioFrame(name: string, samples: Int16Array, channels: number): void;
|
|
344
|
+
/**
|
|
345
|
+
* Whether `pushEncodedVideoFrame` would refuse a frame of `byteLength`
|
|
346
|
+
* bytes: throws what it would throw, and buffers nothing. A length, as
|
|
347
|
+
* for `checkVideoFrame`.
|
|
348
|
+
*/
|
|
349
|
+
checkEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, byte_length: number, width: number, height: number, capture_time_us?: number | null): void;
|
|
350
|
+
/**
|
|
351
|
+
* Whether `step(state, commands)` would be refused now, for the session or
|
|
352
|
+
* for its state and commands: throws what `step()` would throw, and takes
|
|
353
|
+
* no frames. `step()`'s own checks, run before a step's media is pushed,
|
|
354
|
+
* so a refused step leaves the buffer as the caller left it.
|
|
355
|
+
*/
|
|
356
|
+
checkStep(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): void;
|
|
357
|
+
/**
|
|
358
|
+
* Whether `pushVideoFrame` would refuse a frame of `byteLength` bytes:
|
|
359
|
+
* throws what it would throw, and buffers nothing. The frame's length
|
|
360
|
+
* rather than its bytes, so a frame is not copied into wasm memory twice.
|
|
361
|
+
*/
|
|
362
|
+
checkVideoFrame(name: string, byte_length: number, width: number, height: number, format?: PixelFormat | null): void;
|
|
329
363
|
/**
|
|
330
364
|
* Drop every frame buffered for the next step.
|
|
331
365
|
*/
|
|
@@ -435,6 +469,23 @@ export class ReactorClient {
|
|
|
435
469
|
* no sample rate). Stepped sessions only, as `pushVideoFrame`.
|
|
436
470
|
*/
|
|
437
471
|
pushAudioFrame(name: string, samples: Int16Array, sample_rate: number, channels: number): void;
|
|
472
|
+
/**
|
|
473
|
+
* Buffer an encoded video frame for the next `step()` on the sendonly
|
|
474
|
+
* video track `name`, its bytes as they are.
|
|
475
|
+
*
|
|
476
|
+
* * `codec` — `"h264"` (the default): `data` is one H.264 access unit,
|
|
477
|
+
* Annex B.
|
|
478
|
+
* * `width`, `height` — the frame it decodes to.
|
|
479
|
+
* * `metadata`, `captureTimeUs` — as for `pushVideoFrame`, carried with
|
|
480
|
+
* the decoded frame.
|
|
481
|
+
*
|
|
482
|
+
* The runtime decodes a track's encoded frames of one step as one
|
|
483
|
+
* stream before the step runs, so push them in decode order, starting at
|
|
484
|
+
* a keyframe; a step whose frames do not decode is refused
|
|
485
|
+
* (`step_refused`). Encoded and raw pushes to one track keep their push
|
|
486
|
+
* order. Stepped sessions only; throws on empty data or a zero dimension.
|
|
487
|
+
*/
|
|
488
|
+
pushEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, data: Uint8Array, width: number, height: number, metadata?: Uint8Array | null, capture_time_us?: number | null): void;
|
|
438
489
|
/**
|
|
439
490
|
* Buffer a video frame for the next `step()` on the sendonly track `name`.
|
|
440
491
|
*
|
|
@@ -548,6 +599,10 @@ export class ReactorClient {
|
|
|
548
599
|
* data channel's message limit; nothing was sent),
|
|
549
600
|
* `UNSUPPORTED_FRAME_ENCODING`, or the code of whatever else failed the
|
|
550
601
|
* step (`DISCONNECTED`, `REQUEST_TIMEOUT`, …).
|
|
602
|
+
*
|
|
603
|
+
* The step is checked and its frames taken before this returns, so a
|
|
604
|
+
* frame pushed once it has returned goes to the next step. A step refused
|
|
605
|
+
* by those checks throws rather than rejecting.
|
|
551
606
|
*/
|
|
552
607
|
step(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): Promise<StepResult>;
|
|
553
608
|
/**
|
|
@@ -578,6 +633,10 @@ export interface InitOutput {
|
|
|
578
633
|
readonly memory: WebAssembly.Memory;
|
|
579
634
|
readonly __wbg_reactorclient_free: (a: number, b: number) => void;
|
|
580
635
|
readonly reactorclient_capabilities: (a: number) => [number, number, number];
|
|
636
|
+
readonly reactorclient_checkAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number) => [number, number];
|
|
637
|
+
readonly reactorclient_checkEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => [number, number];
|
|
638
|
+
readonly reactorclient_checkStep: (a: number, b: number, c: number) => [number, number];
|
|
639
|
+
readonly reactorclient_checkVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
|
|
581
640
|
readonly reactorclient_clearStepInputs: (a: number) => void;
|
|
582
641
|
readonly reactorclient_connect: (a: number, b: number) => any;
|
|
583
642
|
readonly reactorclient_disconnect: (a: number) => any;
|
|
@@ -599,6 +658,7 @@ export interface InitOutput {
|
|
|
599
658
|
readonly reactorclient_pausedTracks: (a: number) => [number, number, number];
|
|
600
659
|
readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: number) => any;
|
|
601
660
|
readonly reactorclient_pushAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
|
|
661
|
+
readonly reactorclient_pushEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
|
|
602
662
|
readonly reactorclient_pushVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
|
|
603
663
|
readonly reactorclient_reconnect: (a: number, b: number) => any;
|
|
604
664
|
readonly reactorclient_requestClip: (a: number, b: number) => any;
|
|
@@ -612,7 +672,7 @@ export interface InitOutput {
|
|
|
612
672
|
readonly reactorclient_setJwt: (a: number, b: number) => [number, number];
|
|
613
673
|
readonly reactorclient_setTrackBitrate: (a: number, b: number, c: number, d: number, e: number) => any;
|
|
614
674
|
readonly reactorclient_status: (a: number) => any;
|
|
615
|
-
readonly reactorclient_step: (a: number, b: number, c: number) =>
|
|
675
|
+
readonly reactorclient_step: (a: number, b: number, c: number) => [number, number, number];
|
|
616
676
|
readonly reactorclient_trackMapping: (a: number) => [number, number, number];
|
|
617
677
|
readonly reactorclient_tracks: (a: number) => [number, number, number];
|
|
618
678
|
readonly reactorclient_unpublishTrack: (a: number, b: number, c: number) => any;
|