@reactor-team/js-sdk 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -11
- package/dist/index.cjs +539 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +340 -22
- package/dist/index.d.ts +340 -22
- package/dist/index.js +530 -45
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +35 -2
- package/dist/wasm/reactor_wasm.d.ts +184 -8
- package/dist/wasm/reactor_wasm.js +254 -11
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +5 -1
- package/package.json +1 -1
- package/dist/wasm/reactor_wasm_bg.wasm +0 -0
package/dist/index.d.cts
CHANGED
|
@@ -29,6 +29,10 @@ interface ReactorErrorOptions {
|
|
|
29
29
|
operation?: string | undefined;
|
|
30
30
|
retry_after_ms?: number | undefined;
|
|
31
31
|
timestamp_ms?: number | undefined;
|
|
32
|
+
entry?: string | undefined;
|
|
33
|
+
command_code?: string | undefined;
|
|
34
|
+
/** The underlying error, kept as the standard `Error.cause`. */
|
|
35
|
+
cause?: unknown;
|
|
32
36
|
}
|
|
33
37
|
declare class ReactorError extends Error {
|
|
34
38
|
/** The code this class stands for. `ReactorError` itself is the fallback,
|
|
@@ -47,6 +51,14 @@ declare class ReactorError extends Error {
|
|
|
47
51
|
readonly operation: string | undefined;
|
|
48
52
|
/** Backoff hint the platform sent. */
|
|
49
53
|
readonly retry_after_ms: number | undefined;
|
|
54
|
+
/** For a refused step or a step whose call failed, the part of the step
|
|
55
|
+
* request at fault: `"state.<key>"`, `"commands[<i>]"` or
|
|
56
|
+
* `"media_input[<i>]"`. */
|
|
57
|
+
readonly entry: string | undefined;
|
|
58
|
+
/** For a step whose call failed, the stable code its handler raised (its
|
|
59
|
+
* `CommandError` code, or `internal_error`), to branch on instead of the
|
|
60
|
+
* message. */
|
|
61
|
+
readonly command_code: string | undefined;
|
|
50
62
|
/** When `reactor-core` reported this failure. For a purely local one (no
|
|
51
63
|
* payload at all — see {@link ReactorErrorOptions}), this is synthesized
|
|
52
64
|
* as the construction time instead, since there's nothing to report —
|
|
@@ -153,6 +165,52 @@ declare class AbortedError extends ReactorError {
|
|
|
153
165
|
declare class RecorderDisabledError extends ReactorError {
|
|
154
166
|
static readonly code = "RECORDER_DISABLED";
|
|
155
167
|
}
|
|
168
|
+
/** The runtime did not start the session in the requested `generation` —
|
|
169
|
+
* most often a runtime without stepped generation, which ignores the
|
|
170
|
+
* request rather than refusing it. Not recoverable: the same `connect()`
|
|
171
|
+
* against the same runtime fails the same way. */
|
|
172
|
+
declare class GenerationModeMismatchError extends ReactorError {
|
|
173
|
+
static readonly code = "GENERATION_MODE_MISMATCH";
|
|
174
|
+
}
|
|
175
|
+
/** `connect()` asked for stepped generation over a data channel that did not
|
|
176
|
+
* negotiate chunking: this client or the runtime does not support it. A
|
|
177
|
+
* step's inputs and result each ride one data-channel message, and without
|
|
178
|
+
* chunking a single camera frame outgrows it. Not recoverable. */
|
|
179
|
+
declare class DataChannelChunkingRequiredError extends ReactorError {
|
|
180
|
+
static readonly code = "DATA_CHANNEL_CHUNKING_REQUIRED";
|
|
181
|
+
}
|
|
182
|
+
/** The runtime refused a `step()` — the request or the model turned it down.
|
|
183
|
+
* The runtime checks the whole request before it applies any of it, so a
|
|
184
|
+
* refused step changed nothing. The code is the runtime's own, passed
|
|
185
|
+
* through, hence lowercase. */
|
|
186
|
+
declare class StepRefusedError extends ReactorError {
|
|
187
|
+
static readonly code = "step_refused";
|
|
188
|
+
}
|
|
189
|
+
/** A `step()` did not run because the session cannot take a step yet — it
|
|
190
|
+
* is not in a running state that can. Nothing the step carried was applied.
|
|
191
|
+
* Lowercase, as the runtime names the outcome. */
|
|
192
|
+
declare class StepNotReadyError extends ReactorError {
|
|
193
|
+
static readonly code = "step_not_ready";
|
|
194
|
+
}
|
|
195
|
+
/** A `step()` ran, but its result did not fit in one data-channel message. */
|
|
196
|
+
declare class StepResultTooLargeError extends ReactorError {
|
|
197
|
+
static readonly code = "step_result_too_large";
|
|
198
|
+
}
|
|
199
|
+
/** A `step()` did not run because the handler of one of its state keys or
|
|
200
|
+
* commands failed. The calls before it stay applied and the ones after it
|
|
201
|
+
* were skipped. Lowercase, as the runtime names the outcome. */
|
|
202
|
+
declare class StepCommandFailedError extends ReactorError {
|
|
203
|
+
static readonly code = "step_command_failed";
|
|
204
|
+
}
|
|
205
|
+
/** The session ended before a `step()`'s result could be sent, so whether the
|
|
206
|
+
* step ran is unknown. Lowercase, as the runtime names the outcome. */
|
|
207
|
+
declare class StepSessionEndedError extends ReactorError {
|
|
208
|
+
static readonly code = "step_session_ended";
|
|
209
|
+
}
|
|
210
|
+
/** A `step()` result carried a frame in an encoding this SDK cannot decode. */
|
|
211
|
+
declare class UnsupportedFrameEncodingError extends ReactorError {
|
|
212
|
+
static readonly code = "UNSUPPORTED_FRAME_ENCODING";
|
|
213
|
+
}
|
|
156
214
|
|
|
157
215
|
/**
|
|
158
216
|
* Reference to a file uploaded via `Reactor.uploadFile()`. Pass it back in
|
|
@@ -197,6 +255,9 @@ declare function isFileRef(value: unknown): value is FileRef;
|
|
|
197
255
|
type ReactorStatus = 'disconnected' | 'connecting' | 'waiting' | 'ready';
|
|
198
256
|
type TrackKind = 'audio' | 'video';
|
|
199
257
|
type TrackDirection = 'recvonly' | 'sendonly';
|
|
258
|
+
type GenerationMode = 'streamed' | 'stepped';
|
|
259
|
+
/** Byte layout of a frame given to `pushVideoFrame`. */
|
|
260
|
+
type PixelFormat = 'rgba' | 'rgb' | 'bgra';
|
|
200
261
|
interface ClientOptions {
|
|
201
262
|
modelName: string;
|
|
202
263
|
apiUrl?: string;
|
|
@@ -215,6 +276,11 @@ interface ClientOptions {
|
|
|
215
276
|
sdpBackoffMaxMs?: number;
|
|
216
277
|
sdpBackoffMultiplier?: number;
|
|
217
278
|
modelTracks?: TrackCapability[];
|
|
279
|
+
/** "streamed" (the default) or "stepped". */
|
|
280
|
+
generation?: GenerationMode;
|
|
281
|
+
/** How long `step()` waits for a step's result, generation included.
|
|
282
|
+
* Default 300000. */
|
|
283
|
+
stepTimeoutMs?: number;
|
|
218
284
|
logLevel?: 'off' | 'error' | 'warn' | 'info' | 'debug' | 'trace';
|
|
219
285
|
}
|
|
220
286
|
interface ConnectOptions {
|
|
@@ -223,9 +289,16 @@ interface ConnectOptions {
|
|
|
223
289
|
autoResumeTracks?: boolean;
|
|
224
290
|
maxAttempts?: number;
|
|
225
291
|
}
|
|
226
|
-
/**
|
|
227
|
-
*
|
|
228
|
-
|
|
292
|
+
/** What a token is being resolved for. `sessionId` is set for every request
|
|
293
|
+
* that targets an existing session and unset for session creation. A
|
|
294
|
+
* resolver that mints a replacement token must bind it to that session
|
|
295
|
+
* (`resources.sessions.bind`), or the token cannot reach the session. */
|
|
296
|
+
interface JwtRequestContext {
|
|
297
|
+
sessionId?: string;
|
|
298
|
+
}
|
|
299
|
+
/** A token, or a resolver called before every authenticated request with a
|
|
300
|
+
* `JwtRequestContext`. Returning "" sends no Authorization header. */
|
|
301
|
+
type JwtSource = string | ((context?: JwtRequestContext) => string | Promise<string>);
|
|
229
302
|
interface TrackCapability {
|
|
230
303
|
name: string;
|
|
231
304
|
kind: TrackKind;
|
|
@@ -267,7 +340,116 @@ interface ReactorMessage {
|
|
|
267
340
|
type: string;
|
|
268
341
|
data: unknown;
|
|
269
342
|
}
|
|
343
|
+
/** One command of a step, run after its state, in order. */
|
|
344
|
+
interface StepCommand {
|
|
345
|
+
command: string;
|
|
346
|
+
/** The command's arguments. Omitted or null for none. */
|
|
347
|
+
data?: Record<string, unknown> | null;
|
|
348
|
+
}
|
|
349
|
+
/** One frame a step produced. Hand-built by the binding rather than
|
|
350
|
+
* serialized, so camelCase, unlike the wire payloads above. */
|
|
351
|
+
interface StepFrame {
|
|
352
|
+
/** How the frame travelled; "raw" is the only one decoded today. */
|
|
353
|
+
encoding: 'raw' | 'unspecified';
|
|
354
|
+
/** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
|
|
355
|
+
dtype: string;
|
|
356
|
+
/** (height, width, 3) RGB for video, (channels, samples) planar PCM for
|
|
357
|
+
* audio. */
|
|
358
|
+
shape: number[];
|
|
359
|
+
/** The array, row-major, as the typed array `dtype` names — a Uint8Array
|
|
360
|
+
* for video, an Int16Array for audio, raw bytes for a dtype with no
|
|
361
|
+
* typed-array counterpart. */
|
|
362
|
+
data: Uint8Array | Int8Array | Uint16Array | Int16Array | Uint32Array | Int32Array | Float32Array | Float64Array;
|
|
363
|
+
/** What the model attached to the frame. */
|
|
364
|
+
metadata?: Uint8Array;
|
|
365
|
+
/** When the model says the frame was captured, in microseconds. */
|
|
366
|
+
captureTimeUs?: number;
|
|
367
|
+
}
|
|
368
|
+
/** The frames one output track produced in a step, in production order. */
|
|
369
|
+
interface StepTrack {
|
|
370
|
+
track: string;
|
|
371
|
+
kind: TrackKind;
|
|
372
|
+
/** The rate the track's frames play at, as the runtime reported it:
|
|
373
|
+
* frames per second for video, samples per second for audio. 0 when the
|
|
374
|
+
* runtime did not say. */
|
|
375
|
+
rate: number;
|
|
376
|
+
frames: StepFrame[];
|
|
377
|
+
}
|
|
378
|
+
/** A file the model kept with a step. */
|
|
379
|
+
interface StepFile {
|
|
380
|
+
name: string;
|
|
381
|
+
/** Its MIME type, e.g. "image/png". */
|
|
382
|
+
contentType: string;
|
|
383
|
+
/** Its size in bytes. */
|
|
384
|
+
size: number;
|
|
385
|
+
}
|
|
386
|
+
/** What one step produced. */
|
|
387
|
+
interface StepResult {
|
|
388
|
+
/** The step's number in the session, counted from 1: the `step` of the
|
|
389
|
+
* session's step results (`GET /sessions/{id}/steps/{step}`). */
|
|
390
|
+
step: number;
|
|
391
|
+
tracks: StepTrack[];
|
|
392
|
+
/** Seconds the step's `generate()` took, or the time the model reported for the step. */
|
|
393
|
+
elapsed: number;
|
|
394
|
+
/** Files the model kept with the step. Only their description: the bytes
|
|
395
|
+
* are served over HTTP from the session's step results. Empty when the
|
|
396
|
+
* model kept none or the result was not saved. */
|
|
397
|
+
files: StepFile[];
|
|
398
|
+
/** Set when the step ran and the model reported a failure. Such a step
|
|
399
|
+
* still resolves; only a refused step rejects. */
|
|
400
|
+
error?: string;
|
|
401
|
+
}
|
|
270
402
|
|
|
403
|
+
/** What `Reactor.step()` sends: `state`, then `commands`, then every frame
|
|
404
|
+
* pushed since the previous step, applied in that order before the model
|
|
405
|
+
* generates. Every part is optional. A file goes in as a `FileRef` from
|
|
406
|
+
* `uploadFile()`, in a command's `data` or a state value. */
|
|
407
|
+
interface StepOptions {
|
|
408
|
+
/** Values for the model's state fields, keyed by field name. Each key runs
|
|
409
|
+
* as the model's `set_<key>` command, before `commands`. */
|
|
410
|
+
state?: Record<string, unknown>;
|
|
411
|
+
/** Commands to run after the state, in order. */
|
|
412
|
+
commands?: StepCommand[];
|
|
413
|
+
}
|
|
414
|
+
/** Raw pixels for `Reactor.pushFrame()`. `ImageData` is one of these as is. */
|
|
415
|
+
interface RawVideoFrame {
|
|
416
|
+
data: Uint8Array | Uint8ClampedArray;
|
|
417
|
+
width: number;
|
|
418
|
+
height: number;
|
|
419
|
+
/** Byte layout of `data`. Default `"rgba"` — what `ImageData` holds. */
|
|
420
|
+
format?: PixelFormat;
|
|
421
|
+
}
|
|
422
|
+
/** A video frame for `Reactor.pushFrame()`: a WebCodecs `VideoFrame` (the
|
|
423
|
+
* caller keeps ownership and closes it), an `ImageData`, or raw pixels. */
|
|
424
|
+
type PushVideoFrame = VideoFrame | ImageData | RawVideoFrame;
|
|
425
|
+
/** A block of audio for `Reactor.pushFrame()`: interleaved signed 16-bit
|
|
426
|
+
* PCM, already at the sample rate the model's track declares — a step's
|
|
427
|
+
* frame carries no sample rate for the runtime to resample from. */
|
|
428
|
+
interface PushAudioFrame {
|
|
429
|
+
samples: Int16Array;
|
|
430
|
+
sampleRate: number;
|
|
431
|
+
/** Interleaved channels in `samples`. Default 1. */
|
|
432
|
+
channels?: number;
|
|
433
|
+
}
|
|
434
|
+
type PushFrame = PushVideoFrame | PushAudioFrame;
|
|
435
|
+
/** What `Reactor.publishTrack()` resolves with: the published input track. */
|
|
436
|
+
interface PublishedTrack {
|
|
437
|
+
readonly name: string;
|
|
438
|
+
/** Buffers one input frame on this track for the next `step()` of a
|
|
439
|
+
* stepped session — `reactor.pushFrame(name, frame, options)`. Rejects
|
|
440
|
+
* with an `InvalidStateError` in a streamed session, whose media comes
|
|
441
|
+
* from the `MediaStreamTrack` given to `publishTrack()`, and once the
|
|
442
|
+
* track is unpublished or its connection ends: publish again for a new
|
|
443
|
+
* handle. */
|
|
444
|
+
pushFrame(frame: PushFrame, options?: PushFrameOptions): Promise<void>;
|
|
445
|
+
}
|
|
446
|
+
/** Per-frame extras for a pushed video frame. */
|
|
447
|
+
interface PushFrameOptions {
|
|
448
|
+
/** Opaque bytes the frame carries to the model. */
|
|
449
|
+
metadata?: Uint8Array;
|
|
450
|
+
/** When the frame was captured, in microseconds. */
|
|
451
|
+
captureTimeUs?: number;
|
|
452
|
+
}
|
|
271
453
|
/** The session resource, as the coordinator reports it — see
|
|
272
454
|
* `Reactor.getSessionInfo()`. */
|
|
273
455
|
type SessionResponse = SessionInfo;
|
|
@@ -549,10 +731,14 @@ interface FetchPlaylistOptions {
|
|
|
549
731
|
*/
|
|
550
732
|
signal?: AbortSignal;
|
|
551
733
|
/**
|
|
552
|
-
* Coordinator JWT
|
|
553
|
-
*
|
|
734
|
+
* Coordinator JWT for the manifest GET: a string, or a resolver called
|
|
735
|
+
* before every poll with `{ sessionId }`, so a long 202 wait can cross a
|
|
736
|
+
* token refresh. Omit in local mode (HttpRuntime).
|
|
554
737
|
*/
|
|
555
|
-
jwt?:
|
|
738
|
+
jwt?: JwtSource;
|
|
739
|
+
/** The clip's session, handed to a resolver `jwt`. `downloadClipAsFile()`
|
|
740
|
+
* fills it from `clip.sessionId`. */
|
|
741
|
+
sessionId?: string;
|
|
556
742
|
}
|
|
557
743
|
/**
|
|
558
744
|
* Fetch `playlistUrl`, polling on `202 Accepted` — returned while the
|
|
@@ -584,9 +770,10 @@ declare function parsePlaylist(manifestBody: string, playlistUrl: string): Parse
|
|
|
584
770
|
*/
|
|
585
771
|
declare function createPlayableManifestUrl(manifestBody: string, playlistUrl: string): string;
|
|
586
772
|
interface DownloadClipOptions {
|
|
587
|
-
/** Coordinator JWT for the manifest GET
|
|
588
|
-
*
|
|
589
|
-
|
|
773
|
+
/** Coordinator JWT for the manifest GET: a string, or a resolver called
|
|
774
|
+
* per poll with `{ sessionId: clip.sessionId }`. The chunks the manifest
|
|
775
|
+
* references are S3 presigned URLs, fetched unauthenticated. */
|
|
776
|
+
jwt?: JwtSource;
|
|
590
777
|
/** Cancels both the playlist poll and any in-flight chunk fetches. */
|
|
591
778
|
signal?: AbortSignal;
|
|
592
779
|
/** Called after each chunk completes — useful for progress UI. */
|
|
@@ -631,10 +818,18 @@ declare class Reactor implements Disposable {
|
|
|
631
818
|
* with it, so a batch sent before the timings exist doesn't use it up.
|
|
632
819
|
* Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
|
|
633
820
|
private hasSentConnectTime;
|
|
821
|
+
/** Whether this connection has sent a client-stats batch yet. The first
|
|
822
|
+
* one goes out even with no reading, for the environment the bridge adds
|
|
823
|
+
* to it. Reset with `hasSentConnectTime`. */
|
|
824
|
+
private hasSentClientStats;
|
|
634
825
|
/** Set on the "connecting" status transition, cleared once `connectionTimings`
|
|
635
826
|
* is finalized on "ready" — see `handleStatusChanged()`. */
|
|
636
827
|
private connectStartTime;
|
|
637
828
|
private waitingStartTime;
|
|
829
|
+
/** The tail of every `pushFrame()` so far, settled either way — pushes
|
|
830
|
+
* reach the binding in call order however long each one's pixels take to
|
|
831
|
+
* read, and `step()` waits for the ones called before it. */
|
|
832
|
+
private pendingPushes;
|
|
638
833
|
private readonly emitter;
|
|
639
834
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
640
835
|
* disconnect()/[Symbol.dispose]) against each other. Calling into the
|
|
@@ -652,6 +847,15 @@ declare class Reactor implements Disposable {
|
|
|
652
847
|
* out every read active when it started, before it releases the ones
|
|
653
848
|
* that arrived while it ran. */
|
|
654
849
|
private writerActive;
|
|
850
|
+
/** Counts `disconnect()`s and disposal. A push captures it when called, and
|
|
851
|
+
* one whose frame was still being read when it moved is dropped, rather
|
|
852
|
+
* than recreating a client to carry the frame into the next session. */
|
|
853
|
+
private connectionEpoch;
|
|
854
|
+
/** Input tracks published on this connection, each with the publication
|
|
855
|
+
* that claimed it: a push needs one, and a `PublishedTrack` only feeds the
|
|
856
|
+
* publication it came from. Cleared with the connection. */
|
|
857
|
+
private readonly publishedInputs;
|
|
858
|
+
private publications;
|
|
655
859
|
/** Readers parked in `withReaderAccess()`'s wait loop, woken once
|
|
656
860
|
* `withWriterAccess()` finishes; each re-checks `writerActive` rather
|
|
657
861
|
* than assuming its turn, since another writer already queued behind the
|
|
@@ -787,8 +991,24 @@ declare class Reactor implements Disposable {
|
|
|
787
991
|
* disconnect() freeing the wasm client mid-await would otherwise resume
|
|
788
992
|
* into a freed client — the same class of use-after-free `queue` exists to
|
|
789
993
|
* prevent for connect/disconnect/reconnect themselves.
|
|
994
|
+
*
|
|
995
|
+
* Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
|
|
996
|
+
* track in a stepped session.
|
|
997
|
+
*
|
|
998
|
+
* In a `generation: "stepped"` session, input frames never travel over
|
|
999
|
+
* RTP: `track` is optional, and the frames are pushed with
|
|
1000
|
+
* `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
|
|
1001
|
+
* passed there is ignored, with a console warning — nothing is sent from
|
|
1002
|
+
* it. In a streamed session `track` is required, and omitting it rejects
|
|
1003
|
+
* with an `InvalidStateError`.
|
|
1004
|
+
*
|
|
1005
|
+
* ```ts
|
|
1006
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1007
|
+
* await camera.pushFrame(imageData);
|
|
1008
|
+
* const result = await reactor.step();
|
|
1009
|
+
* ```
|
|
790
1010
|
*/
|
|
791
|
-
publishTrack(name: string, track
|
|
1011
|
+
publishTrack(name: string, track?: MediaStreamTrack): Promise<PublishedTrack>;
|
|
792
1012
|
/**
|
|
793
1013
|
* Unlike every other track method, this doesn't reject — a failure is
|
|
794
1014
|
* reported through the `error` event instead, since this is commonly the
|
|
@@ -836,6 +1056,85 @@ declare class Reactor implements Disposable {
|
|
|
836
1056
|
}): Promise<void>;
|
|
837
1057
|
/** Resumes a track previously stopped with `pauseTrack()`. */
|
|
838
1058
|
resumeTrack(name: string): Promise<void>;
|
|
1059
|
+
/**
|
|
1060
|
+
* Runs one step of a `generation: "stepped"` session and resolves with
|
|
1061
|
+
* what it produced, per output track.
|
|
1062
|
+
*
|
|
1063
|
+
* The request carries `state`, `commands` and every frame pushed since the
|
|
1064
|
+
* previous step. The runtime applies them in that order — each `state` key
|
|
1065
|
+
* as the model's `set_<key>` command, then `commands`, then the frames —
|
|
1066
|
+
* and then generates. They are separate calls, not a transaction: one that
|
|
1067
|
+
* fails does not undo the ones before it. All of it is optional:
|
|
1068
|
+
* `step()` with nothing set and nothing pushed runs on the model's current
|
|
1069
|
+
* state, which commands sent outside a step (`sendCommand()`) also change.
|
|
1070
|
+
* A file goes in the way it does for a command: upload it with
|
|
1071
|
+
* `uploadFile()`, then put the `FileRef` in a command's `data` or a state
|
|
1072
|
+
* value.
|
|
1073
|
+
*
|
|
1074
|
+
* A step that ran but that the model reported as failed still resolves,
|
|
1075
|
+
* with `result.error` set. `result.files` lists the files the model kept
|
|
1076
|
+
* with the step (name, content type, size); their bytes are served over
|
|
1077
|
+
* HTTP from the session's step results, not carried here.
|
|
1078
|
+
*
|
|
1079
|
+
* Rejects with a `StepRefusedError` when the runtime or the model turns
|
|
1080
|
+
* the step down (the runtime checks a whole request first, so nothing it
|
|
1081
|
+
* carried was applied), `StepNotReadyError` when the session cannot take a
|
|
1082
|
+
* step yet (neither ran the step), `StepCommandFailedError` when a state key
|
|
1083
|
+
* or command failed (the calls before it stay applied; the step did not
|
|
1084
|
+
* run); both name the part at fault in `entry`, and `StepCommandFailedError`
|
|
1085
|
+
* the handler's code in `command_code`,
|
|
1086
|
+
* `StepSessionEndedError` when the session ended before the result came
|
|
1087
|
+
* back, `StepResultTooLargeError` when the result does not fit in one
|
|
1088
|
+
* message, `MessageTooLargeError` when the request itself is too large
|
|
1089
|
+
* (nothing was sent), `InvalidStateError` outside a ready stepped
|
|
1090
|
+
* session, or whatever else failed the step (`DisconnectedError`,
|
|
1091
|
+
* `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
|
|
1092
|
+
* when its request cannot be sent, they are dropped with it rather than
|
|
1093
|
+
* kept for the next step.
|
|
1094
|
+
*
|
|
1095
|
+
* ```ts
|
|
1096
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1097
|
+
* await camera.pushFrame(imageData);
|
|
1098
|
+
* const result = await reactor.step({ state: { prompt: 'a cat' } });
|
|
1099
|
+
* if (result.error) console.warn('step failed:', result.error);
|
|
1100
|
+
* const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
|
|
1101
|
+
* if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
|
|
1102
|
+
* ```
|
|
1103
|
+
*/
|
|
1104
|
+
step(options?: StepOptions): Promise<StepResult>;
|
|
1105
|
+
/**
|
|
1106
|
+
* Buffers one input frame for the next `step()` on the sendonly track
|
|
1107
|
+
* `trackName`. Nothing is sent until that step, which carries every frame
|
|
1108
|
+
* pushed since the one before it, per track in push order. The
|
|
1109
|
+
* `PublishedTrack` that `publishTrack()` resolves with does the same
|
|
1110
|
+
* through `track.pushFrame(frame)`.
|
|
1111
|
+
*
|
|
1112
|
+
* Video: a `VideoFrame` (cloned when called, so the caller may close it as
|
|
1113
|
+
* soon as this returns; it stays the caller's), an
|
|
1114
|
+
* `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
|
|
1115
|
+
* by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
|
|
1116
|
+
* `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
|
|
1117
|
+
* applied, so a 90° frame arrives with its width and height swapped.
|
|
1118
|
+
* Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
|
|
1119
|
+
* PCM at the rate the model's track declares.
|
|
1120
|
+
*
|
|
1121
|
+
* Pushes keep their call order, and a `step()` waits for every push called
|
|
1122
|
+
* before it. A refused push rejects: await it, or `.catch()` it in a frame
|
|
1123
|
+
* loop, or every bad frame is an unhandled rejection. What waits for
|
|
1124
|
+
* the next step is at most one message's worth of frames: pushing faster
|
|
1125
|
+
* than steps run drops the oldest frames, across tracks. Rejects with
|
|
1126
|
+
* an `InvalidStateError` outside a stepped session, for a track not
|
|
1127
|
+
* published on this connection (`publishTrack(trackName)` first), for a
|
|
1128
|
+
* frame whose size does not match its dimensions, or for a name the model
|
|
1129
|
+
* does not declare as a sendonly track of that kind.
|
|
1130
|
+
*/
|
|
1131
|
+
pushFrame(trackName: string, frame: PushFrame, options?: PushFrameOptions): Promise<void>;
|
|
1132
|
+
/** `pushFrame()`, for the track's current publication or, from a
|
|
1133
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
1134
|
+
private pushTo;
|
|
1135
|
+
/** Drops every frame buffered for the next step, pushes called before it
|
|
1136
|
+
* included: it waits its turn behind them, as a `step()` does. */
|
|
1137
|
+
clearStepInputs(): Promise<void>;
|
|
839
1138
|
/**
|
|
840
1139
|
* Requests a clip covering the last `durationSeconds` of the session.
|
|
841
1140
|
* `reactor-core` correlates the reply itself (and enforces `"ready"`), so
|
|
@@ -955,8 +1254,9 @@ declare class Reactor implements Disposable {
|
|
|
955
1254
|
* track has one yet (none negotiated onto a named track, or no codec
|
|
956
1255
|
* reported yet), so the runtime gets the bandwidth estimate, the
|
|
957
1256
|
* round-trip time and the time to connect without waiting on the tracks.
|
|
958
|
-
* Skipped only when there is nothing at all to say
|
|
959
|
-
*
|
|
1257
|
+
* Skipped only when there is nothing at all to say and a batch has already
|
|
1258
|
+
* gone out for this connection, so the first one still carries the
|
|
1259
|
+
* environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
|
|
960
1260
|
* actually sent for this connection — see `hasSentConnectTime`.
|
|
961
1261
|
* `sendClientStats` throws synchronously on a closing connection; caught
|
|
962
1262
|
* here for the same reason `getStats()`'s rejection is above. */
|
|
@@ -978,12 +1278,25 @@ declare class Reactor implements Disposable {
|
|
|
978
1278
|
* `error` event — the one place recording and emitting happen together,
|
|
979
1279
|
* so `getLastError()` never drifts from what listeners were told. */
|
|
980
1280
|
private emitError;
|
|
1281
|
+
private isStepped;
|
|
981
1282
|
private assertNotDisposed;
|
|
982
1283
|
}
|
|
983
1284
|
|
|
1285
|
+
/**
|
|
1286
|
+
* A step's video frame as an `ImageData`, ready for a canvas's
|
|
1287
|
+
* `putImageData()` (or `createImageBitmap()`).
|
|
1288
|
+
*
|
|
1289
|
+
* A step's video frames are `(height, width, 3)` `uint8` RGB; a canvas wants
|
|
1290
|
+
* RGBA, so this adds an opaque alpha. Throws on any other frame — an audio
|
|
1291
|
+
* frame, or a dtype or shape that is not RGB video.
|
|
1292
|
+
*/
|
|
1293
|
+
declare function stepFrameToImageData(frame: StepFrame): ImageData;
|
|
1294
|
+
|
|
984
1295
|
/** Lazy resolver for a Coordinator/runtime bearer token, called immediately
|
|
985
|
-
* before each authenticated request
|
|
986
|
-
|
|
1296
|
+
* before each authenticated request with the session that request is for —
|
|
1297
|
+
* see `JwtSource` and `JwtRequestContext`. A resolver that ignores the
|
|
1298
|
+
* context keeps working. */
|
|
1299
|
+
type JwtResolver = (context?: JwtRequestContext) => string | Promise<string>;
|
|
987
1300
|
/**
|
|
988
1301
|
* Wraps a `JwtSource` into a `JwtResolver`: a static string becomes a
|
|
989
1302
|
* resolver that always returns it, a resolver passes through unchanged.
|
|
@@ -1024,7 +1337,10 @@ interface ReactorActions {
|
|
|
1024
1337
|
disconnect: (recoverable?: boolean) => Promise<void>;
|
|
1025
1338
|
reconnect: (options?: ConnectOptions) => Promise<void>;
|
|
1026
1339
|
sendCommand: <T extends object = Record<string, unknown>>(command: string, data?: NotFunction<T>, scope?: MessageScope) => Promise<ReactorMessage | undefined>;
|
|
1027
|
-
|
|
1340
|
+
/** `Reactor.publishTrack()`. `track` is required in a streamed session
|
|
1341
|
+
* and ignored (with a warning) in a stepped one, whose frames go through
|
|
1342
|
+
* the resolved handle's `pushFrame()`. */
|
|
1343
|
+
publish: (name: string, track?: MediaStreamTrack) => Promise<PublishedTrack>;
|
|
1028
1344
|
unpublish: (name: string) => Promise<void>;
|
|
1029
1345
|
pauseTrack: (name: string) => Promise<void>;
|
|
1030
1346
|
resumeTrack: (name: string) => Promise<void>;
|
|
@@ -1184,13 +1500,14 @@ interface ClipPlayerProps {
|
|
|
1184
1500
|
clip: Clip;
|
|
1185
1501
|
/**
|
|
1186
1502
|
* Lazy resolver for the Coordinator JWT used on the manifest GET, called
|
|
1187
|
-
*
|
|
1503
|
+
* before every poll with `{ sessionId: clip.sessionId }` so token refreshes
|
|
1504
|
+
* are picked up automatically.
|
|
1188
1505
|
*
|
|
1189
1506
|
* Required outside a `ReactorProvider` in production; optional inside one
|
|
1190
1507
|
* (inherits the provider's resolver). Omit in local-dev mode (HttpRuntime)
|
|
1191
1508
|
* — the manifest endpoint there is auth-free.
|
|
1192
1509
|
*/
|
|
1193
|
-
getJwt?:
|
|
1510
|
+
getJwt?: JwtResolver;
|
|
1194
1511
|
/**
|
|
1195
1512
|
* Opt into a bounded wait: give up polling the manifest with
|
|
1196
1513
|
* `CLIP_NOT_READY` once `max(clip.predictedReadyAtMs, pollStart) + slackMs`
|
|
@@ -1243,13 +1560,14 @@ interface UseClipDownloadOptions {
|
|
|
1243
1560
|
filename?: string | null;
|
|
1244
1561
|
/**
|
|
1245
1562
|
* Lazy resolver for the Coordinator JWT used on the manifest GET. Called
|
|
1246
|
-
*
|
|
1247
|
-
* refreshes are picked up
|
|
1563
|
+
* before every poll of a {@link UseClipDownloadResult.download} with
|
|
1564
|
+
* `{ sessionId: clip.sessionId }`, so token refreshes are picked up
|
|
1565
|
+
* automatically.
|
|
1248
1566
|
*
|
|
1249
1567
|
* Optional inside a `ReactorProvider` (inherits the provider's resolver)
|
|
1250
1568
|
* and in local-dev mode.
|
|
1251
1569
|
*/
|
|
1252
|
-
getJwt?:
|
|
1570
|
+
getJwt?: JwtResolver;
|
|
1253
1571
|
}
|
|
1254
1572
|
interface UseClipDownloadResult {
|
|
1255
1573
|
/** Current state of the most recent download attempt. */
|
|
@@ -1294,7 +1612,7 @@ interface ClipDownloadButtonProps {
|
|
|
1294
1612
|
clip: Clip;
|
|
1295
1613
|
/** Lazy JWT resolver. Optional inside a `ReactorProvider` (inherits the
|
|
1296
1614
|
* provider's resolver) and in local-dev mode. See `ClipPlayerProps.getJwt`. */
|
|
1297
|
-
getJwt?:
|
|
1615
|
+
getJwt?: JwtResolver;
|
|
1298
1616
|
/** Filename for the saved MP4. Default `"reactor-clip.mp4"`. */
|
|
1299
1617
|
filename?: string;
|
|
1300
1618
|
/**
|
|
@@ -1324,4 +1642,4 @@ interface ClipDownloadButtonProps {
|
|
|
1324
1642
|
}
|
|
1325
1643
|
declare function ClipDownloadButton({ clip, getJwt, filename, children, className, style, disabled, onSuccess, onError, }: ClipDownloadButtonProps): react.JSX.Element;
|
|
1326
1644
|
|
|
1327
|
-
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DecodeError, DisconnectedError, type DownloadClipOptions, type FetchPlaylistOptions, FileRef, InvalidStateError, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|
|
1645
|
+
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DataChannelChunkingRequiredError, DecodeError, DisconnectedError, type DownloadClipOptions, type FetchPlaylistOptions, FileRef, type GenerationMode, GenerationModeMismatchError, InvalidStateError, type JwtRequestContext, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, type PublishedTrack, type PushAudioFrame, type PushFrame, type PushFrameOptions, type PushVideoFrame, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type StepCommand, StepCommandFailedError, type StepFile, type StepFrame, StepNotReadyError, type StepOptions, StepRefusedError, type StepResult, StepResultTooLargeError, StepSessionEndedError, type StepTrack, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, UnsupportedFrameEncodingError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, stepFrameToImageData, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|