@reactor-team/js-sdk 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -0
- package/dist/index.cjs +752 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +399 -5
- package/dist/index.d.ts +399 -5
- package/dist/index.js +743 -9
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +38 -2
- package/dist/wasm/reactor_wasm.d.ts +229 -2
- package/dist/wasm/reactor_wasm.js +348 -8
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +10 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -29,6 +29,8 @@ interface ReactorErrorOptions {
|
|
|
29
29
|
operation?: string | undefined;
|
|
30
30
|
retry_after_ms?: number | undefined;
|
|
31
31
|
timestamp_ms?: number | undefined;
|
|
32
|
+
entry?: string | undefined;
|
|
33
|
+
command_code?: string | undefined;
|
|
32
34
|
/** The underlying error, kept as the standard `Error.cause`. */
|
|
33
35
|
cause?: unknown;
|
|
34
36
|
}
|
|
@@ -49,6 +51,14 @@ declare class ReactorError extends Error {
|
|
|
49
51
|
readonly operation: string | undefined;
|
|
50
52
|
/** Backoff hint the platform sent. */
|
|
51
53
|
readonly retry_after_ms: number | undefined;
|
|
54
|
+
/** For a refused step or a step whose call failed, the part of the step
|
|
55
|
+
* request at fault: `"state.<key>"`, `"commands[<i>]"` or
|
|
56
|
+
* `"media_input[<i>]"`. */
|
|
57
|
+
readonly entry: string | undefined;
|
|
58
|
+
/** For a step whose call failed, the stable code its handler raised (its
|
|
59
|
+
* `CommandError` code, or `internal_error`), to branch on instead of the
|
|
60
|
+
* message. */
|
|
61
|
+
readonly command_code: string | undefined;
|
|
52
62
|
/** When `reactor-core` reported this failure. For a purely local one (no
|
|
53
63
|
* payload at all — see {@link ReactorErrorOptions}), this is synthesized
|
|
54
64
|
* as the construction time instead, since there's nothing to report —
|
|
@@ -155,6 +165,52 @@ declare class AbortedError extends ReactorError {
|
|
|
155
165
|
declare class RecorderDisabledError extends ReactorError {
|
|
156
166
|
static readonly code = "RECORDER_DISABLED";
|
|
157
167
|
}
|
|
168
|
+
/** The runtime did not start the session in the requested `generation` —
|
|
169
|
+
* most often a runtime without stepped generation, which ignores the
|
|
170
|
+
* request rather than refusing it. Not recoverable: the same `connect()`
|
|
171
|
+
* against the same runtime fails the same way. */
|
|
172
|
+
declare class GenerationModeMismatchError extends ReactorError {
|
|
173
|
+
static readonly code = "GENERATION_MODE_MISMATCH";
|
|
174
|
+
}
|
|
175
|
+
/** `connect()` asked for stepped generation over a data channel that did not
|
|
176
|
+
* negotiate chunking: this client or the runtime does not support it. A
|
|
177
|
+
* step's inputs and result each ride one data-channel message, and without
|
|
178
|
+
* chunking a single camera frame outgrows it. Not recoverable. */
|
|
179
|
+
declare class DataChannelChunkingRequiredError extends ReactorError {
|
|
180
|
+
static readonly code = "DATA_CHANNEL_CHUNKING_REQUIRED";
|
|
181
|
+
}
|
|
182
|
+
/** The runtime refused a `step()` — the request or the model turned it down.
|
|
183
|
+
* The runtime checks the whole request before it applies any of it, so a
|
|
184
|
+
* refused step changed nothing. The code is the runtime's own, passed
|
|
185
|
+
* through, hence lowercase. */
|
|
186
|
+
declare class StepRefusedError extends ReactorError {
|
|
187
|
+
static readonly code = "step_refused";
|
|
188
|
+
}
|
|
189
|
+
/** A `step()` did not run because the session cannot take a step yet — it
|
|
190
|
+
* is not in a running state that can. Nothing the step carried was applied.
|
|
191
|
+
* Lowercase, as the runtime names the outcome. */
|
|
192
|
+
declare class StepNotReadyError extends ReactorError {
|
|
193
|
+
static readonly code = "step_not_ready";
|
|
194
|
+
}
|
|
195
|
+
/** A `step()` ran, but its result did not fit in one data-channel message. */
|
|
196
|
+
declare class StepResultTooLargeError extends ReactorError {
|
|
197
|
+
static readonly code = "step_result_too_large";
|
|
198
|
+
}
|
|
199
|
+
/** A `step()` did not run because the handler of one of its state keys or
|
|
200
|
+
* commands failed. The calls before it stay applied and the ones after it
|
|
201
|
+
* were skipped. Lowercase, as the runtime names the outcome. */
|
|
202
|
+
declare class StepCommandFailedError extends ReactorError {
|
|
203
|
+
static readonly code = "step_command_failed";
|
|
204
|
+
}
|
|
205
|
+
/** The session ended before a `step()`'s result could be sent, so whether the
|
|
206
|
+
* step ran is unknown. Lowercase, as the runtime names the outcome. */
|
|
207
|
+
declare class StepSessionEndedError extends ReactorError {
|
|
208
|
+
static readonly code = "step_session_ended";
|
|
209
|
+
}
|
|
210
|
+
/** A `step()` result carried a frame in an encoding this SDK cannot decode. */
|
|
211
|
+
declare class UnsupportedFrameEncodingError extends ReactorError {
|
|
212
|
+
static readonly code = "UNSUPPORTED_FRAME_ENCODING";
|
|
213
|
+
}
|
|
158
214
|
|
|
159
215
|
/**
|
|
160
216
|
* Reference to a file uploaded via `Reactor.uploadFile()`. Pass it back in
|
|
@@ -199,6 +255,11 @@ declare function isFileRef(value: unknown): value is FileRef;
|
|
|
199
255
|
type ReactorStatus = 'disconnected' | 'connecting' | 'waiting' | 'ready';
|
|
200
256
|
type TrackKind = 'audio' | 'video';
|
|
201
257
|
type TrackDirection = 'recvonly' | 'sendonly';
|
|
258
|
+
type GenerationMode = 'streamed' | 'stepped';
|
|
259
|
+
/** Byte layout of a frame given to `pushVideoFrame`. */
|
|
260
|
+
type PixelFormat = 'rgba' | 'rgb' | 'bgra';
|
|
261
|
+
/** Codec of an encoded frame given to `pushEncodedVideoFrame`. */
|
|
262
|
+
type EncodedVideoCodec = 'h264';
|
|
202
263
|
interface ClientOptions {
|
|
203
264
|
modelName: string;
|
|
204
265
|
apiUrl?: string;
|
|
@@ -217,6 +278,14 @@ interface ClientOptions {
|
|
|
217
278
|
sdpBackoffMaxMs?: number;
|
|
218
279
|
sdpBackoffMultiplier?: number;
|
|
219
280
|
modelTracks?: TrackCapability[];
|
|
281
|
+
/** "streamed" (the default) or "stepped". */
|
|
282
|
+
generation?: GenerationMode;
|
|
283
|
+
/** How long `step()` waits for a step's result, generation included.
|
|
284
|
+
* Default 300000. */
|
|
285
|
+
stepTimeoutMs?: number;
|
|
286
|
+
/** The largest data-channel message, in bytes, the client declares and
|
|
287
|
+
* accepts: 262144 (256 KiB) to 1073741824 (1 GiB). Default 64 MiB. */
|
|
288
|
+
maxMessageSize?: number;
|
|
220
289
|
logLevel?: 'off' | 'error' | 'warn' | 'info' | 'debug' | 'trace';
|
|
221
290
|
}
|
|
222
291
|
interface ConnectOptions {
|
|
@@ -276,7 +345,151 @@ interface ReactorMessage {
|
|
|
276
345
|
type: string;
|
|
277
346
|
data: unknown;
|
|
278
347
|
}
|
|
348
|
+
/** One command of a step, run after its state, in order. */
|
|
349
|
+
interface StepCommand {
|
|
350
|
+
command: string;
|
|
351
|
+
/** The command's arguments. Omitted or null for none. */
|
|
352
|
+
data?: Record<string, unknown> | null;
|
|
353
|
+
}
|
|
354
|
+
/** One frame a step produced. Hand-built by the binding rather than
|
|
355
|
+
* serialized, so camelCase, unlike the wire payloads above. */
|
|
356
|
+
interface StepFrame {
|
|
357
|
+
/** How the frame travelled; "raw" is the only one decoded today. */
|
|
358
|
+
encoding: 'raw' | 'unspecified';
|
|
359
|
+
/** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
|
|
360
|
+
dtype: string;
|
|
361
|
+
/** (height, width, 3) RGB for video, (channels, samples) planar PCM for
|
|
362
|
+
* audio. */
|
|
363
|
+
shape: number[];
|
|
364
|
+
/** The array, row-major, as the typed array `dtype` names — a Uint8Array
|
|
365
|
+
* for video, an Int16Array for audio, raw bytes for a dtype with no
|
|
366
|
+
* typed-array counterpart. */
|
|
367
|
+
data: Uint8Array | Int8Array | Uint16Array | Int16Array | Uint32Array | Int32Array | Float32Array | Float64Array;
|
|
368
|
+
/** What the model attached to the frame. */
|
|
369
|
+
metadata?: Uint8Array;
|
|
370
|
+
/** When the model says the frame was captured, in microseconds. */
|
|
371
|
+
captureTimeUs?: number;
|
|
372
|
+
}
|
|
373
|
+
/** The frames one output track produced in a step, in production order. */
|
|
374
|
+
interface StepTrack {
|
|
375
|
+
track: string;
|
|
376
|
+
kind: TrackKind;
|
|
377
|
+
/** The rate the track's frames play at, as the runtime reported it:
|
|
378
|
+
* frames per second for video, samples per second for audio. 0 when the
|
|
379
|
+
* runtime did not say. */
|
|
380
|
+
rate: number;
|
|
381
|
+
frames: StepFrame[];
|
|
382
|
+
}
|
|
383
|
+
/** A file the model kept with a step. */
|
|
384
|
+
interface StepFile {
|
|
385
|
+
name: string;
|
|
386
|
+
/** Its MIME type, e.g. "image/png". */
|
|
387
|
+
contentType: string;
|
|
388
|
+
/** Its size in bytes. */
|
|
389
|
+
size: number;
|
|
390
|
+
}
|
|
391
|
+
/** What one step produced. */
|
|
392
|
+
interface StepResult {
|
|
393
|
+
/** The step's number in the session, counted from 1: the `step` of the
|
|
394
|
+
* session's step results (`GET /sessions/{id}/steps/{step}`). */
|
|
395
|
+
step: number;
|
|
396
|
+
tracks: StepTrack[];
|
|
397
|
+
/** Seconds the step's `generate()` took, or the time the model reported for the step. */
|
|
398
|
+
elapsed: number;
|
|
399
|
+
/** Files the model kept with the step. Only their description: the bytes
|
|
400
|
+
* are served over HTTP from the session's step results. Empty when the
|
|
401
|
+
* model kept none or the result was not saved. */
|
|
402
|
+
files: StepFile[];
|
|
403
|
+
/** Set when the step ran and the model reported a failure. Such a step
|
|
404
|
+
* still resolves; only a refused step rejects. */
|
|
405
|
+
error?: string;
|
|
406
|
+
}
|
|
279
407
|
|
|
408
|
+
/** What `Reactor.step()` sends: `state`, then `commands`, then every frame
|
|
409
|
+
* pushed since the previous step, applied in that order before the model
|
|
410
|
+
* generates. Every part is optional. A file goes in as a `FileRef` from
|
|
411
|
+
* `uploadFile()`, in a command's `data` or a state value. */
|
|
412
|
+
interface StepOptions {
|
|
413
|
+
/** Values for the model's state fields, keyed by field name. Each key runs
|
|
414
|
+
* as the model's `set_<key>` command, before `commands`. */
|
|
415
|
+
state?: Record<string, unknown>;
|
|
416
|
+
/** Commands to run after the state, in order. */
|
|
417
|
+
commands?: StepCommand[];
|
|
418
|
+
/** This step's own frames, by input track name: what `pushFrame()` takes,
|
|
419
|
+
* or an `EncodedFrame`. They go after the frames pushed to that track
|
|
420
|
+
* since the previous step, exactly as if pushed just before the step, so
|
|
421
|
+
* each track must be published first. A bad track or frame anywhere fails
|
|
422
|
+
* the step before any of them is pushed, leaving the buffer as it was. */
|
|
423
|
+
media?: Record<string, ReadonlyArray<PushFrame | EncodedFrame>>;
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* An encoded video frame for a step: one H.264 access unit (Annex B), its
|
|
427
|
+
* bytes sent as they are. `width` and `height` are the frame it decodes to.
|
|
428
|
+
*
|
|
429
|
+
* The runtime decodes a track's encoded frames of one step as one stream,
|
|
430
|
+
* before the step runs, and the model reads the RGB frames as if they had
|
|
431
|
+
* been pushed raw. So give them in decode order, starting at a keyframe.
|
|
432
|
+
*
|
|
433
|
+
* `data` may be a WebCodecs `EncodedVideoChunk` as a `VideoEncoder` hands it
|
|
434
|
+
* over; configure the encoder with `avc: { format: 'annexb' }`, as its
|
|
435
|
+
* default (`"avc"`) is length-prefixed, not Annex B.
|
|
436
|
+
*/
|
|
437
|
+
interface EncodedFrame {
|
|
438
|
+
/** Tells an encoded frame from a raw one in `step()`'s `media`. */
|
|
439
|
+
codec: EncodedVideoCodec;
|
|
440
|
+
/** One access unit. A chunk's bytes are copied out when it is pushed. */
|
|
441
|
+
data: Uint8Array | ArrayBuffer | EncodedVideoChunk;
|
|
442
|
+
/** Width of the frame it decodes to. */
|
|
443
|
+
width: number;
|
|
444
|
+
/** Height of the frame it decodes to. */
|
|
445
|
+
height: number;
|
|
446
|
+
/** Opaque bytes the decoded frame carries to the model. */
|
|
447
|
+
metadata?: Uint8Array;
|
|
448
|
+
/** When the frame was captured, in microseconds. */
|
|
449
|
+
captureTimeUs?: number;
|
|
450
|
+
}
|
|
451
|
+
/** Raw pixels for `Reactor.pushFrame()`. `ImageData` is one of these as is. */
|
|
452
|
+
interface RawVideoFrame {
|
|
453
|
+
data: Uint8Array | Uint8ClampedArray;
|
|
454
|
+
width: number;
|
|
455
|
+
height: number;
|
|
456
|
+
/** Byte layout of `data`. Default `"rgba"` — what `ImageData` holds. */
|
|
457
|
+
format?: PixelFormat;
|
|
458
|
+
}
|
|
459
|
+
/** A video frame for `Reactor.pushFrame()`: a WebCodecs `VideoFrame` (the
|
|
460
|
+
* caller keeps ownership and closes it), an `ImageData`, or raw pixels. */
|
|
461
|
+
type PushVideoFrame = VideoFrame | ImageData | RawVideoFrame;
|
|
462
|
+
/** A block of audio for `Reactor.pushFrame()`: interleaved signed 16-bit
|
|
463
|
+
* PCM, already at the sample rate the model's track declares — a step's
|
|
464
|
+
* frame carries no sample rate for the runtime to resample from. */
|
|
465
|
+
interface PushAudioFrame {
|
|
466
|
+
samples: Int16Array;
|
|
467
|
+
sampleRate: number;
|
|
468
|
+
/** Interleaved channels in `samples`. Default 1. */
|
|
469
|
+
channels?: number;
|
|
470
|
+
}
|
|
471
|
+
type PushFrame = PushVideoFrame | PushAudioFrame;
|
|
472
|
+
/** What `Reactor.publishTrack()` resolves with: the published input track. */
|
|
473
|
+
interface PublishedTrack {
|
|
474
|
+
readonly name: string;
|
|
475
|
+
/** Buffers one input frame on this track for the next `step()` of a
|
|
476
|
+
* stepped session — `reactor.pushFrame(name, frame, options)`. Rejects
|
|
477
|
+
* with an `InvalidStateError` in a streamed session, whose media comes
|
|
478
|
+
* from the `MediaStreamTrack` given to `publishTrack()`, and once the
|
|
479
|
+
* track is unpublished or its connection ends: publish again for a new
|
|
480
|
+
* handle. */
|
|
481
|
+
pushFrame(frame: PushFrame, options?: PushFrameOptions): Promise<void>;
|
|
482
|
+
/** Buffers one encoded video frame on this track for the next `step()` —
|
|
483
|
+
* `reactor.pushEncodedFrame(name, frame)`. Rejects as `pushFrame()` does. */
|
|
484
|
+
pushEncodedFrame(frame: EncodedFrame): Promise<void>;
|
|
485
|
+
}
|
|
486
|
+
/** Per-frame extras for a pushed video frame. */
|
|
487
|
+
interface PushFrameOptions {
|
|
488
|
+
/** Opaque bytes the frame carries to the model. */
|
|
489
|
+
metadata?: Uint8Array;
|
|
490
|
+
/** When the frame was captured, in microseconds. */
|
|
491
|
+
captureTimeUs?: number;
|
|
492
|
+
}
|
|
280
493
|
/** The session resource, as the coordinator reports it — see
|
|
281
494
|
* `Reactor.getSessionInfo()`. */
|
|
282
495
|
type SessionResponse = SessionInfo;
|
|
@@ -645,10 +858,19 @@ declare class Reactor implements Disposable {
|
|
|
645
858
|
* with it, so a batch sent before the timings exist doesn't use it up.
|
|
646
859
|
* Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
|
|
647
860
|
private hasSentConnectTime;
|
|
861
|
+
/** Whether this connection has sent a client-stats batch yet. The first
|
|
862
|
+
* one goes out even with no reading, for the environment the bridge adds
|
|
863
|
+
* to it. Reset with `hasSentConnectTime`. */
|
|
864
|
+
private hasSentClientStats;
|
|
648
865
|
/** Set on the "connecting" status transition, cleared once `connectionTimings`
|
|
649
866
|
* is finalized on "ready" — see `handleStatusChanged()`. */
|
|
650
867
|
private connectStartTime;
|
|
651
868
|
private waitingStartTime;
|
|
869
|
+
/** The tail of every `pushFrame()`/`pushEncodedFrame()` so far, settled
|
|
870
|
+
* either way — pushes reach the binding in call order however long each
|
|
871
|
+
* one's pixels take to read, and `step()` waits for the ones called
|
|
872
|
+
* before it. */
|
|
873
|
+
private pendingPushes;
|
|
652
874
|
private readonly emitter;
|
|
653
875
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
654
876
|
* disconnect()/[Symbol.dispose]) against each other. Calling into the
|
|
@@ -666,6 +888,15 @@ declare class Reactor implements Disposable {
|
|
|
666
888
|
* out every read active when it started, before it releases the ones
|
|
667
889
|
* that arrived while it ran. */
|
|
668
890
|
private writerActive;
|
|
891
|
+
/** Counts `disconnect()`s and disposal. A push captures it when called, and
|
|
892
|
+
* one whose frame was still being read when it moved is dropped, rather
|
|
893
|
+
* than recreating a client to carry the frame into the next session. */
|
|
894
|
+
private connectionEpoch;
|
|
895
|
+
/** Input tracks published on this connection, each with the publication
|
|
896
|
+
* that claimed it: a push needs one, and a `PublishedTrack` only feeds the
|
|
897
|
+
* publication it came from. Cleared with the connection. */
|
|
898
|
+
private readonly publishedInputs;
|
|
899
|
+
private publications;
|
|
669
900
|
/** Readers parked in `withReaderAccess()`'s wait loop, woken once
|
|
670
901
|
* `withWriterAccess()` finishes; each re-checks `writerActive` rather
|
|
671
902
|
* than assuming its turn, since another writer already queued behind the
|
|
@@ -801,8 +1032,24 @@ declare class Reactor implements Disposable {
|
|
|
801
1032
|
* disconnect() freeing the wasm client mid-await would otherwise resume
|
|
802
1033
|
* into a freed client — the same class of use-after-free `queue` exists to
|
|
803
1034
|
* prevent for connect/disconnect/reconnect themselves.
|
|
1035
|
+
*
|
|
1036
|
+
* Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
|
|
1037
|
+
* track in a stepped session.
|
|
1038
|
+
*
|
|
1039
|
+
* In a `generation: "stepped"` session, input frames never travel over
|
|
1040
|
+
* RTP: `track` is optional, and the frames are pushed with
|
|
1041
|
+
* `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
|
|
1042
|
+
* passed there is ignored, with a console warning — nothing is sent from
|
|
1043
|
+
* it. In a streamed session `track` is required, and omitting it rejects
|
|
1044
|
+
* with an `InvalidStateError`.
|
|
1045
|
+
*
|
|
1046
|
+
* ```ts
|
|
1047
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1048
|
+
* await camera.pushFrame(imageData);
|
|
1049
|
+
* const result = await reactor.step();
|
|
1050
|
+
* ```
|
|
804
1051
|
*/
|
|
805
|
-
publishTrack(name: string, track
|
|
1052
|
+
publishTrack(name: string, track?: MediaStreamTrack): Promise<PublishedTrack>;
|
|
806
1053
|
/**
|
|
807
1054
|
* Unlike every other track method, this doesn't reject — a failure is
|
|
808
1055
|
* reported through the `error` event instead, since this is commonly the
|
|
@@ -850,6 +1097,138 @@ declare class Reactor implements Disposable {
|
|
|
850
1097
|
}): Promise<void>;
|
|
851
1098
|
/** Resumes a track previously stopped with `pauseTrack()`. */
|
|
852
1099
|
resumeTrack(name: string): Promise<void>;
|
|
1100
|
+
/**
|
|
1101
|
+
* Runs one step of a `generation: "stepped"` session and resolves with
|
|
1102
|
+
* what it produced, per output track.
|
|
1103
|
+
*
|
|
1104
|
+
* The request carries `state`, `commands` and every frame pushed since the
|
|
1105
|
+
* previous step. The runtime applies them in that order — each `state` key
|
|
1106
|
+
* as the model's `set_<key>` command, then `commands`, then the frames —
|
|
1107
|
+
* and then generates. They are separate calls, not a transaction: one that
|
|
1108
|
+
* fails does not undo the ones before it. All of it is optional:
|
|
1109
|
+
* `step()` with nothing set and nothing pushed runs on the model's current
|
|
1110
|
+
* state, which commands sent outside a step (`sendCommand()`) also change.
|
|
1111
|
+
* A file goes in the way it does for a command: upload it with
|
|
1112
|
+
* `uploadFile()`, then put the `FileRef` in a command's `data` or a state
|
|
1113
|
+
* value.
|
|
1114
|
+
*
|
|
1115
|
+
* A step that ran but that the model reported as failed still resolves,
|
|
1116
|
+
* with `result.error` set. `result.files` lists the files the model kept
|
|
1117
|
+
* with the step (name, content type, size); their bytes are served over
|
|
1118
|
+
* HTTP from the session's step results, not carried here.
|
|
1119
|
+
*
|
|
1120
|
+
* Rejects with a `StepRefusedError` when the runtime or the model turns
|
|
1121
|
+
* the step down (the runtime checks a whole request first, so nothing it
|
|
1122
|
+
* carried was applied), `StepNotReadyError` when the session cannot take a
|
|
1123
|
+
* step yet (neither ran the step), `StepCommandFailedError` when a state key
|
|
1124
|
+
* or command failed (the calls before it stay applied; the step did not
|
|
1125
|
+
* run); both name the part at fault in `entry`, and `StepCommandFailedError`
|
|
1126
|
+
* the handler's code in `command_code`,
|
|
1127
|
+
* `StepSessionEndedError` when the session ended before the result came
|
|
1128
|
+
* back, `StepResultTooLargeError` when the result does not fit in one
|
|
1129
|
+
* message, `MessageTooLargeError` when the request itself is too large
|
|
1130
|
+
* (nothing was sent), `InvalidStateError` outside a ready stepped
|
|
1131
|
+
* session, or whatever else failed the step (`DisconnectedError`,
|
|
1132
|
+
* `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
|
|
1133
|
+
* when its request cannot be sent, they are dropped with it rather than
|
|
1134
|
+
* kept for the next step.
|
|
1135
|
+
*
|
|
1136
|
+
* `media` gives the step frames of its own, by input track name, pushed
|
|
1137
|
+
* after the ones already pushed to that track: what `pushFrame()` takes,
|
|
1138
|
+
* or an `EncodedFrame` — H.264 the runtime decodes before the step runs,
|
|
1139
|
+
* so the model reads RGB frames as for a raw push. They are pushed only
|
|
1140
|
+
* once `state` and `commands` are well formed and the session is ready, so
|
|
1141
|
+
* a step refused for those leaves the frames buffered as they were. They go
|
|
1142
|
+
* all or nothing: a bad track or frame anywhere in `media` rejects the step
|
|
1143
|
+
* before any of it is pushed or sent, leaving the buffer as it was. A
|
|
1144
|
+
* session that leaves 'ready' once the media is in has the step refused and
|
|
1145
|
+
* keeps the frames for the next one, as pushed frames are.
|
|
1146
|
+
* A track's encoded frames of one step go in decode order, starting at a
|
|
1147
|
+
* keyframe; a step whose frames do not decode rejects with a
|
|
1148
|
+
* `StepRefusedError`.
|
|
1149
|
+
*
|
|
1150
|
+
* ```ts
|
|
1151
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1152
|
+
* await camera.pushFrame(imageData);
|
|
1153
|
+
* const result = await reactor.step({ state: { prompt: 'a cat' } });
|
|
1154
|
+
* if (result.error) console.warn('step failed:', result.error);
|
|
1155
|
+
* const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
|
|
1156
|
+
* if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
|
|
1157
|
+
* ```
|
|
1158
|
+
*/
|
|
1159
|
+
step(options?: StepOptions): Promise<StepResult>;
|
|
1160
|
+
/**
|
|
1161
|
+
* Buffers one input frame for the next `step()` on the sendonly track
|
|
1162
|
+
* `trackName`. Nothing is sent until that step, which carries every frame
|
|
1163
|
+
* pushed since the one before it, per track in push order. The
|
|
1164
|
+
* `PublishedTrack` that `publishTrack()` resolves with does the same
|
|
1165
|
+
* through `track.pushFrame(frame)`.
|
|
1166
|
+
*
|
|
1167
|
+
* Video: a `VideoFrame` (cloned when called, so the caller may close it as
|
|
1168
|
+
* soon as this returns; it stays the caller's), an
|
|
1169
|
+
* `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
|
|
1170
|
+
* by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
|
|
1171
|
+
* `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
|
|
1172
|
+
* applied, so a 90° frame arrives with its width and height swapped.
|
|
1173
|
+
* Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
|
|
1174
|
+
* PCM at the rate the model's track declares.
|
|
1175
|
+
*
|
|
1176
|
+
* Pushes keep their call order, and a `step()` waits for every push called
|
|
1177
|
+
* before it. A refused push rejects: await it, or `.catch()` it in a frame
|
|
1178
|
+
* loop, or every bad frame is an unhandled rejection. What waits for
|
|
1179
|
+
* the next step is at most one message's worth of frames: pushing faster
|
|
1180
|
+
* than steps run drops the oldest frames, across tracks. Rejects with
|
|
1181
|
+
* an `InvalidStateError` outside a stepped session, for a track not
|
|
1182
|
+
* published on this connection (`publishTrack(trackName)` first), for a
|
|
1183
|
+
* frame whose size does not match its dimensions, or for a name the model
|
|
1184
|
+
* does not declare as a sendonly track of that kind.
|
|
1185
|
+
*/
|
|
1186
|
+
pushFrame(trackName: string, frame: PushFrame, options?: PushFrameOptions): Promise<void>;
|
|
1187
|
+
/** `pushFrame()`, for the track's current publication or, from a
|
|
1188
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
1189
|
+
private pushTo;
|
|
1190
|
+
/**
|
|
1191
|
+
* Buffers one encoded video frame for the next `step()` on the sendonly
|
|
1192
|
+
* video track `trackName`, its bytes as they are: one H.264 access unit
|
|
1193
|
+
* (Annex B), with the `width` and `height` it decodes to. The
|
|
1194
|
+
* `PublishedTrack` that `publishTrack()` resolves with does the same
|
|
1195
|
+
* through `track.pushEncodedFrame(frame)`.
|
|
1196
|
+
*
|
|
1197
|
+
* The runtime decodes a track's encoded frames of one step as one stream,
|
|
1198
|
+
* before the step runs, and the model reads the RGB frames as if they had
|
|
1199
|
+
* been pushed raw. So push them in decode order, starting at a keyframe; a
|
|
1200
|
+
* step whose frames do not decode rejects with a `StepRefusedError`, as
|
|
1201
|
+
* does a runtime that predates encoded frames. Encoded and raw pushes keep
|
|
1202
|
+
* their call order, and a `step()` waits for every push called before it.
|
|
1203
|
+
*
|
|
1204
|
+
* ```ts
|
|
1205
|
+
* const encoder = new VideoEncoder({
|
|
1206
|
+
* output: (chunk) => void camera.pushEncodedFrame({ codec: 'h264', data: chunk, width, height }),
|
|
1207
|
+
* error: console.error,
|
|
1208
|
+
* });
|
|
1209
|
+
* encoder.configure({ codec: 'avc1.42001f', width, height, avc: { format: 'annexb' } });
|
|
1210
|
+
* ```
|
|
1211
|
+
*
|
|
1212
|
+
* Rejects with an `InvalidStateError` outside a stepped session, for a
|
|
1213
|
+
* track not published on this connection, for an audio or recvonly track,
|
|
1214
|
+
* for empty data, a zero dimension or a codec other than `"h264"`.
|
|
1215
|
+
*/
|
|
1216
|
+
pushEncodedFrame(trackName: string, frame: EncodedFrame): Promise<void>;
|
|
1217
|
+
/** `pushEncodedFrame()`, for the track's current publication or, from a
|
|
1218
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
1219
|
+
private pushEncodedTo;
|
|
1220
|
+
/** Refuses a push whose connection ended since it was called, or whose
|
|
1221
|
+
* track is not (or no longer) published on it. */
|
|
1222
|
+
private checkPushTarget;
|
|
1223
|
+
/** Pushes `step()`'s `media`, track by track and in order, each frame as
|
|
1224
|
+
* `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
|
|
1225
|
+
* checks the step's `state` and `commands`, then every frame, before the
|
|
1226
|
+
* first is pushed, so a refused step, or a bad track or frame anywhere,
|
|
1227
|
+
* leaves the buffer as it was. */
|
|
1228
|
+
private pushStepMedia;
|
|
1229
|
+
/** Drops every frame buffered for the next step, pushes called before it
|
|
1230
|
+
* included: it waits its turn behind them, as a `step()` does. */
|
|
1231
|
+
clearStepInputs(): Promise<void>;
|
|
853
1232
|
/**
|
|
854
1233
|
* Requests a clip covering the last `durationSeconds` of the session.
|
|
855
1234
|
* `reactor-core` correlates the reply itself (and enforces `"ready"`), so
|
|
@@ -969,8 +1348,9 @@ declare class Reactor implements Disposable {
|
|
|
969
1348
|
* track has one yet (none negotiated onto a named track, or no codec
|
|
970
1349
|
* reported yet), so the runtime gets the bandwidth estimate, the
|
|
971
1350
|
* round-trip time and the time to connect without waiting on the tracks.
|
|
972
|
-
* Skipped only when there is nothing at all to say
|
|
973
|
-
*
|
|
1351
|
+
* Skipped only when there is nothing at all to say and a batch has already
|
|
1352
|
+
* gone out for this connection, so the first one still carries the
|
|
1353
|
+
* environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
|
|
974
1354
|
* actually sent for this connection — see `hasSentConnectTime`.
|
|
975
1355
|
* `sendClientStats` throws synchronously on a closing connection; caught
|
|
976
1356
|
* here for the same reason `getStats()`'s rejection is above. */
|
|
@@ -992,9 +1372,20 @@ declare class Reactor implements Disposable {
|
|
|
992
1372
|
* `error` event — the one place recording and emitting happen together,
|
|
993
1373
|
* so `getLastError()` never drifts from what listeners were told. */
|
|
994
1374
|
private emitError;
|
|
1375
|
+
private isStepped;
|
|
995
1376
|
private assertNotDisposed;
|
|
996
1377
|
}
|
|
997
1378
|
|
|
1379
|
+
/**
|
|
1380
|
+
* A step's video frame as an `ImageData`, ready for a canvas's
|
|
1381
|
+
* `putImageData()` (or `createImageBitmap()`).
|
|
1382
|
+
*
|
|
1383
|
+
* A step's video frames are `(height, width, 3)` `uint8` RGB; a canvas wants
|
|
1384
|
+
* RGBA, so this adds an opaque alpha. Throws on any other frame — an audio
|
|
1385
|
+
* frame, or a dtype or shape that is not RGB video.
|
|
1386
|
+
*/
|
|
1387
|
+
declare function stepFrameToImageData(frame: StepFrame): ImageData;
|
|
1388
|
+
|
|
998
1389
|
/** Lazy resolver for a Coordinator/runtime bearer token, called immediately
|
|
999
1390
|
* before each authenticated request with the session that request is for —
|
|
1000
1391
|
* see `JwtSource` and `JwtRequestContext`. A resolver that ignores the
|
|
@@ -1040,7 +1431,10 @@ interface ReactorActions {
|
|
|
1040
1431
|
disconnect: (recoverable?: boolean) => Promise<void>;
|
|
1041
1432
|
reconnect: (options?: ConnectOptions) => Promise<void>;
|
|
1042
1433
|
sendCommand: <T extends object = Record<string, unknown>>(command: string, data?: NotFunction<T>, scope?: MessageScope) => Promise<ReactorMessage | undefined>;
|
|
1043
|
-
|
|
1434
|
+
/** `Reactor.publishTrack()`. `track` is required in a streamed session
|
|
1435
|
+
* and ignored (with a warning) in a stepped one, whose frames go through
|
|
1436
|
+
* the resolved handle's `pushFrame()`. */
|
|
1437
|
+
publish: (name: string, track?: MediaStreamTrack) => Promise<PublishedTrack>;
|
|
1044
1438
|
unpublish: (name: string) => Promise<void>;
|
|
1045
1439
|
pauseTrack: (name: string) => Promise<void>;
|
|
1046
1440
|
resumeTrack: (name: string) => Promise<void>;
|
|
@@ -1342,4 +1736,4 @@ interface ClipDownloadButtonProps {
|
|
|
1342
1736
|
}
|
|
1343
1737
|
declare function ClipDownloadButton({ clip, getJwt, filename, children, className, style, disabled, onSuccess, onError, }: ClipDownloadButtonProps): react.JSX.Element;
|
|
1344
1738
|
|
|
1345
|
-
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DecodeError, DisconnectedError, type DownloadClipOptions, type FetchPlaylistOptions, FileRef, InvalidStateError, type JwtRequestContext, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|
|
1739
|
+
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DataChannelChunkingRequiredError, DecodeError, DisconnectedError, type DownloadClipOptions, type EncodedFrame, type EncodedVideoCodec, type FetchPlaylistOptions, FileRef, type GenerationMode, GenerationModeMismatchError, InvalidStateError, type JwtRequestContext, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, type PublishedTrack, type PushAudioFrame, type PushFrame, type PushFrameOptions, type PushVideoFrame, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type StepCommand, StepCommandFailedError, type StepFile, type StepFrame, StepNotReadyError, type StepOptions, StepRefusedError, type StepResult, StepResultTooLargeError, StepSessionEndedError, type StepTrack, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, UnsupportedFrameEncodingError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, stepFrameToImageData, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|