@reactor-team/js-sdk 3.3.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -48,7 +48,7 @@ rather than `any`:
48
48
  | Configuration | `setJwt(jwt)` |
49
49
  | Messaging | `sendCommand(command, data?, uploads?)`, `requestSchema()` |
50
50
  | Tracks | `publishTrack(name, track?)`, `unpublishTrack(name)`, `pauseTrack(name)`, `resumeTrack(name)`, `tracks()`, `trackMapping()`, `pausedTracks()` |
51
- | Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
51
+ | Stepped generation | `step(state?, commands?)`, `pushVideoFrame(name, data, width, height, format?, metadata?, captureTimeUs?)`, `pushEncodedVideoFrame(name, codec, data, width, height, metadata?, captureTimeUs?)`, `pushAudioFrame(name, samples, sampleRate, channels)`, `clearStepInputs()` |
52
52
  | Recording | `requestClip(seconds)`, `requestRecording()` |
53
53
  | Uploads | `uploadFile(file, name?)` |
54
54
  | Introspection | `status()`, `sessionId()`, `sessionInfo()`, `capabilities()`, `lastError()` |
@@ -67,7 +67,10 @@ there is ignored, with a console warning), `pushVideoFrame` / `pushAudioFrame`
67
67
  buffer frames in the core, and the next `step()` carries them inside its one
68
68
  `RequestStep`. Both of `step()`'s arguments are optional: an empty step runs on
69
69
  the model's current state. Video arrives as RGBA, RGB or BGRA, and the core
70
- converts it to the RGB a step sends in one pass.
70
+ converts it to the RGB a step sends in one pass. `pushEncodedVideoFrame` buffers
71
+ an H.264 access unit (Annex B) as it is, with the size it decodes to; the runtime
72
+ decodes a track's encoded frames of one step, in push order and starting at a
73
+ keyframe, before the step runs.
71
74
 
72
75
  The buffer holds at most one data-channel message's worth of frames: pushing
73
76
  faster than steps run drops the oldest frames, across tracks, and the next step
@@ -7,6 +7,8 @@ export type TrackDirection = "recvonly" | "sendonly";
7
7
  export type GenerationMode = "streamed" | "stepped";
8
8
  /** Byte layout of a frame given to `pushVideoFrame`. */
9
9
  export type PixelFormat = "rgba" | "rgb" | "bgra";
10
+ /** Codec of an encoded frame given to `pushEncodedVideoFrame`. */
11
+ export type EncodedVideoCodec = "h264";
10
12
 
11
13
  /** Construction options. Only `modelName` is required. */
12
14
  export interface ClientOptions {
@@ -50,6 +52,14 @@ export interface ClientOptions {
50
52
  /** How long `step()` waits for a step's result, generation included.
51
53
  * Default 300000. */
52
54
  stepTimeoutMs?: number;
55
+ /** The largest data-channel message, in bytes, the client declares and
56
+ * accepts: from 262144 (256 KiB) to 1073741824 (1 GiB). Default 67108864
57
+ * (64 MiB). A connection carries the smaller of this and the runtime's own
58
+ * limit, which is 64 MiB unless the model raises it, so a `step()` whose
59
+ * result is larger than either fails with `StepResultTooLargeError`. A
60
+ * message is held whole in memory while it arrives, so raise this only for
61
+ * a model whose steps need it. */
62
+ maxMessageSize?: number;
53
63
  /** Console log level. Default "warn". */
54
64
  logLevel?: "off" | "error" | "warn" | "info" | "debug" | "trace";
55
65
  }
@@ -326,6 +336,30 @@ export class ReactorClient {
326
336
  * The runtime's capabilities, or `undefined` before they arrive.
327
337
  */
328
338
  capabilities(): Capabilities | undefined;
339
+ /**
340
+ * Whether `pushAudioFrame` would refuse `samples`: throws what it would
341
+ * throw, and buffers nothing.
342
+ */
343
+ checkAudioFrame(name: string, samples: Int16Array, channels: number): void;
344
+ /**
345
+ * Whether `pushEncodedVideoFrame` would refuse a frame of `byteLength`
346
+ * bytes: throws what it would throw, and buffers nothing. A length, as
347
+ * for `checkVideoFrame`.
348
+ */
349
+ checkEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, byte_length: number, width: number, height: number, capture_time_us?: number | null): void;
350
+ /**
351
+ * Whether `step(state, commands)` would be refused now, for the session or
352
+ * for its state and commands: throws what `step()` would throw, and takes
353
+ * no frames. `step()`'s own checks, run before a step's media is pushed,
354
+ * so a refused step leaves the buffer as the caller left it.
355
+ */
356
+ checkStep(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): void;
357
+ /**
358
+ * Whether `pushVideoFrame` would refuse a frame of `byteLength` bytes:
359
+ * throws what it would throw, and buffers nothing. The frame's length
360
+ * rather than its bytes, so a frame is not copied into wasm memory twice.
361
+ */
362
+ checkVideoFrame(name: string, byte_length: number, width: number, height: number, format?: PixelFormat | null): void;
329
363
  /**
330
364
  * Drop every frame buffered for the next step.
331
365
  */
@@ -435,6 +469,23 @@ export class ReactorClient {
435
469
  * no sample rate). Stepped sessions only, as `pushVideoFrame`.
436
470
  */
437
471
  pushAudioFrame(name: string, samples: Int16Array, sample_rate: number, channels: number): void;
472
+ /**
473
+ * Buffer an encoded video frame for the next `step()` on the sendonly
474
+ * video track `name`, its bytes as they are.
475
+ *
476
+ * * `codec` — `"h264"` (the default): `data` is one H.264 access unit,
477
+ * Annex B.
478
+ * * `width`, `height` — the frame it decodes to.
479
+ * * `metadata`, `captureTimeUs` — as for `pushVideoFrame`, carried with
480
+ * the decoded frame.
481
+ *
482
+ * The runtime decodes a track's encoded frames of one step as one
483
+ * stream before the step runs, so push them in decode order, starting at
484
+ * a keyframe; a step whose frames do not decode is refused
485
+ * (`step_refused`). Encoded and raw pushes to one track keep their push
486
+ * order. Stepped sessions only; throws on empty data or a zero dimension.
487
+ */
488
+ pushEncodedVideoFrame(name: string, codec: EncodedVideoCodec | null | undefined, data: Uint8Array, width: number, height: number, metadata?: Uint8Array | null, capture_time_us?: number | null): void;
438
489
  /**
439
490
  * Buffer a video frame for the next `step()` on the sendonly track `name`.
440
491
  *
@@ -548,6 +599,10 @@ export class ReactorClient {
548
599
  * data channel's message limit; nothing was sent),
549
600
  * `UNSUPPORTED_FRAME_ENCODING`, or the code of whatever else failed the
550
601
  * step (`DISCONNECTED`, `REQUEST_TIMEOUT`, …).
602
+ *
603
+ * The step is checked and its frames taken before this returns, so a
604
+ * frame pushed once it has returned goes to the next step. A step refused
605
+ * by those checks throws rather than rejecting.
551
606
  */
552
607
  step(state?: Record<string, unknown> | null | null, commands?: StepCommand[] | null): Promise<StepResult>;
553
608
  /**
@@ -578,6 +633,10 @@ export interface InitOutput {
578
633
  readonly memory: WebAssembly.Memory;
579
634
  readonly __wbg_reactorclient_free: (a: number, b: number) => void;
580
635
  readonly reactorclient_capabilities: (a: number) => [number, number, number];
636
+ readonly reactorclient_checkAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number) => [number, number];
637
+ readonly reactorclient_checkEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => [number, number];
638
+ readonly reactorclient_checkStep: (a: number, b: number, c: number) => [number, number];
639
+ readonly reactorclient_checkVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
581
640
  readonly reactorclient_clearStepInputs: (a: number) => void;
582
641
  readonly reactorclient_connect: (a: number, b: number) => any;
583
642
  readonly reactorclient_disconnect: (a: number) => any;
@@ -599,6 +658,7 @@ export interface InitOutput {
599
658
  readonly reactorclient_pausedTracks: (a: number) => [number, number, number];
600
659
  readonly reactorclient_publishTrack: (a: number, b: number, c: number, d: number) => any;
601
660
  readonly reactorclient_pushAudioFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
661
+ readonly reactorclient_pushEncodedVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
602
662
  readonly reactorclient_pushVideoFrame: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => [number, number];
603
663
  readonly reactorclient_reconnect: (a: number, b: number) => any;
604
664
  readonly reactorclient_requestClip: (a: number, b: number) => any;
@@ -612,7 +672,7 @@ export interface InitOutput {
612
672
  readonly reactorclient_setJwt: (a: number, b: number) => [number, number];
613
673
  readonly reactorclient_setTrackBitrate: (a: number, b: number, c: number, d: number, e: number) => any;
614
674
  readonly reactorclient_status: (a: number) => any;
615
- readonly reactorclient_step: (a: number, b: number, c: number) => any;
675
+ readonly reactorclient_step: (a: number, b: number, c: number) => [number, number, number];
616
676
  readonly reactorclient_trackMapping: (a: number) => [number, number, number];
617
677
  readonly reactorclient_tracks: (a: number) => [number, number, number];
618
678
  readonly reactorclient_unpublishTrack: (a: number, b: number, c: number) => any;