@reactor-team/js-sdk 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -0
- package/dist/index.cjs +481 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +305 -5
- package/dist/index.d.ts +305 -5
- package/dist/index.js +472 -9
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +35 -2
- package/dist/wasm/reactor_wasm.d.ts +169 -2
- package/dist/wasm/reactor_wasm.js +238 -8
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +5 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -29,6 +29,8 @@ interface ReactorErrorOptions {
|
|
|
29
29
|
operation?: string | undefined;
|
|
30
30
|
retry_after_ms?: number | undefined;
|
|
31
31
|
timestamp_ms?: number | undefined;
|
|
32
|
+
entry?: string | undefined;
|
|
33
|
+
command_code?: string | undefined;
|
|
32
34
|
/** The underlying error, kept as the standard `Error.cause`. */
|
|
33
35
|
cause?: unknown;
|
|
34
36
|
}
|
|
@@ -49,6 +51,14 @@ declare class ReactorError extends Error {
|
|
|
49
51
|
readonly operation: string | undefined;
|
|
50
52
|
/** Backoff hint the platform sent. */
|
|
51
53
|
readonly retry_after_ms: number | undefined;
|
|
54
|
+
/** For a refused step or a step whose call failed, the part of the step
|
|
55
|
+
* request at fault: `"state.<key>"`, `"commands[<i>]"` or
|
|
56
|
+
* `"media_input[<i>]"`. */
|
|
57
|
+
readonly entry: string | undefined;
|
|
58
|
+
/** For a step whose call failed, the stable code its handler raised (its
|
|
59
|
+
* `CommandError` code, or `internal_error`), to branch on instead of the
|
|
60
|
+
* message. */
|
|
61
|
+
readonly command_code: string | undefined;
|
|
52
62
|
/** When `reactor-core` reported this failure. For a purely local one (no
|
|
53
63
|
* payload at all — see {@link ReactorErrorOptions}), this is synthesized
|
|
54
64
|
* as the construction time instead, since there's nothing to report —
|
|
@@ -155,6 +165,52 @@ declare class AbortedError extends ReactorError {
|
|
|
155
165
|
declare class RecorderDisabledError extends ReactorError {
|
|
156
166
|
static readonly code = "RECORDER_DISABLED";
|
|
157
167
|
}
|
|
168
|
+
/** The runtime did not start the session in the requested `generation` —
|
|
169
|
+
* most often a runtime without stepped generation, which ignores the
|
|
170
|
+
* request rather than refusing it. Not recoverable: the same `connect()`
|
|
171
|
+
* against the same runtime fails the same way. */
|
|
172
|
+
declare class GenerationModeMismatchError extends ReactorError {
|
|
173
|
+
static readonly code = "GENERATION_MODE_MISMATCH";
|
|
174
|
+
}
|
|
175
|
+
/** `connect()` asked for stepped generation over a data channel that did not
|
|
176
|
+
* negotiate chunking: this client or the runtime does not support it. A
|
|
177
|
+
* step's inputs and result each ride one data-channel message, and without
|
|
178
|
+
* chunking a single camera frame outgrows it. Not recoverable. */
|
|
179
|
+
declare class DataChannelChunkingRequiredError extends ReactorError {
|
|
180
|
+
static readonly code = "DATA_CHANNEL_CHUNKING_REQUIRED";
|
|
181
|
+
}
|
|
182
|
+
/** The runtime refused a `step()` — the request or the model turned it down.
|
|
183
|
+
* The runtime checks the whole request before it applies any of it, so a
|
|
184
|
+
* refused step changed nothing. The code is the runtime's own, passed
|
|
185
|
+
* through, hence lowercase. */
|
|
186
|
+
declare class StepRefusedError extends ReactorError {
|
|
187
|
+
static readonly code = "step_refused";
|
|
188
|
+
}
|
|
189
|
+
/** A `step()` did not run because the session cannot take a step yet — it
|
|
190
|
+
* is not in a running state that can. Nothing the step carried was applied.
|
|
191
|
+
* Lowercase, as the runtime names the outcome. */
|
|
192
|
+
declare class StepNotReadyError extends ReactorError {
|
|
193
|
+
static readonly code = "step_not_ready";
|
|
194
|
+
}
|
|
195
|
+
/** A `step()` ran, but its result did not fit in one data-channel message. */
|
|
196
|
+
declare class StepResultTooLargeError extends ReactorError {
|
|
197
|
+
static readonly code = "step_result_too_large";
|
|
198
|
+
}
|
|
199
|
+
/** A `step()` did not run because the handler of one of its state keys or
|
|
200
|
+
* commands failed. The calls before it stay applied and the ones after it
|
|
201
|
+
* were skipped. Lowercase, as the runtime names the outcome. */
|
|
202
|
+
declare class StepCommandFailedError extends ReactorError {
|
|
203
|
+
static readonly code = "step_command_failed";
|
|
204
|
+
}
|
|
205
|
+
/** The session ended before a `step()`'s result could be sent, so whether the
|
|
206
|
+
* step ran is unknown. Lowercase, as the runtime names the outcome. */
|
|
207
|
+
declare class StepSessionEndedError extends ReactorError {
|
|
208
|
+
static readonly code = "step_session_ended";
|
|
209
|
+
}
|
|
210
|
+
/** A `step()` result carried a frame in an encoding this SDK cannot decode. */
|
|
211
|
+
declare class UnsupportedFrameEncodingError extends ReactorError {
|
|
212
|
+
static readonly code = "UNSUPPORTED_FRAME_ENCODING";
|
|
213
|
+
}
|
|
158
214
|
|
|
159
215
|
/**
|
|
160
216
|
* Reference to a file uploaded via `Reactor.uploadFile()`. Pass it back in
|
|
@@ -199,6 +255,9 @@ declare function isFileRef(value: unknown): value is FileRef;
|
|
|
199
255
|
type ReactorStatus = 'disconnected' | 'connecting' | 'waiting' | 'ready';
|
|
200
256
|
type TrackKind = 'audio' | 'video';
|
|
201
257
|
type TrackDirection = 'recvonly' | 'sendonly';
|
|
258
|
+
type GenerationMode = 'streamed' | 'stepped';
|
|
259
|
+
/** Byte layout of a frame given to `pushVideoFrame`. */
|
|
260
|
+
type PixelFormat = 'rgba' | 'rgb' | 'bgra';
|
|
202
261
|
interface ClientOptions {
|
|
203
262
|
modelName: string;
|
|
204
263
|
apiUrl?: string;
|
|
@@ -217,6 +276,11 @@ interface ClientOptions {
|
|
|
217
276
|
sdpBackoffMaxMs?: number;
|
|
218
277
|
sdpBackoffMultiplier?: number;
|
|
219
278
|
modelTracks?: TrackCapability[];
|
|
279
|
+
/** "streamed" (the default) or "stepped". */
|
|
280
|
+
generation?: GenerationMode;
|
|
281
|
+
/** How long `step()` waits for a step's result, generation included.
|
|
282
|
+
* Default 300000. */
|
|
283
|
+
stepTimeoutMs?: number;
|
|
220
284
|
logLevel?: 'off' | 'error' | 'warn' | 'info' | 'debug' | 'trace';
|
|
221
285
|
}
|
|
222
286
|
interface ConnectOptions {
|
|
@@ -276,7 +340,116 @@ interface ReactorMessage {
|
|
|
276
340
|
type: string;
|
|
277
341
|
data: unknown;
|
|
278
342
|
}
|
|
343
|
+
/** One command of a step, run after its state, in order. */
|
|
344
|
+
interface StepCommand {
|
|
345
|
+
command: string;
|
|
346
|
+
/** The command's arguments. Omitted or null for none. */
|
|
347
|
+
data?: Record<string, unknown> | null;
|
|
348
|
+
}
|
|
349
|
+
/** One frame a step produced. Hand-built by the binding rather than
|
|
350
|
+
* serialized, so camelCase, unlike the wire payloads above. */
|
|
351
|
+
interface StepFrame {
|
|
352
|
+
/** How the frame travelled; "raw" is the only one decoded today. */
|
|
353
|
+
encoding: 'raw' | 'unspecified';
|
|
354
|
+
/** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
|
|
355
|
+
dtype: string;
|
|
356
|
+
/** (height, width, 3) RGB for video, (channels, samples) planar PCM for
|
|
357
|
+
* audio. */
|
|
358
|
+
shape: number[];
|
|
359
|
+
/** The array, row-major, as the typed array `dtype` names — a Uint8Array
|
|
360
|
+
* for video, an Int16Array for audio, raw bytes for a dtype with no
|
|
361
|
+
* typed-array counterpart. */
|
|
362
|
+
data: Uint8Array | Int8Array | Uint16Array | Int16Array | Uint32Array | Int32Array | Float32Array | Float64Array;
|
|
363
|
+
/** What the model attached to the frame. */
|
|
364
|
+
metadata?: Uint8Array;
|
|
365
|
+
/** When the model says the frame was captured, in microseconds. */
|
|
366
|
+
captureTimeUs?: number;
|
|
367
|
+
}
|
|
368
|
+
/** The frames one output track produced in a step, in production order. */
|
|
369
|
+
interface StepTrack {
|
|
370
|
+
track: string;
|
|
371
|
+
kind: TrackKind;
|
|
372
|
+
/** The rate the track's frames play at, as the runtime reported it:
|
|
373
|
+
* frames per second for video, samples per second for audio. 0 when the
|
|
374
|
+
* runtime did not say. */
|
|
375
|
+
rate: number;
|
|
376
|
+
frames: StepFrame[];
|
|
377
|
+
}
|
|
378
|
+
/** A file the model kept with a step. */
|
|
379
|
+
interface StepFile {
|
|
380
|
+
name: string;
|
|
381
|
+
/** Its MIME type, e.g. "image/png". */
|
|
382
|
+
contentType: string;
|
|
383
|
+
/** Its size in bytes. */
|
|
384
|
+
size: number;
|
|
385
|
+
}
|
|
386
|
+
/** What one step produced. */
|
|
387
|
+
interface StepResult {
|
|
388
|
+
/** The step's number in the session, counted from 1: the `step` of the
|
|
389
|
+
* session's step results (`GET /sessions/{id}/steps/{step}`). */
|
|
390
|
+
step: number;
|
|
391
|
+
tracks: StepTrack[];
|
|
392
|
+
/** Seconds the step's `generate()` took, or the time the model reported for the step. */
|
|
393
|
+
elapsed: number;
|
|
394
|
+
/** Files the model kept with the step. Only their description: the bytes
|
|
395
|
+
* are served over HTTP from the session's step results. Empty when the
|
|
396
|
+
* model kept none or the result was not saved. */
|
|
397
|
+
files: StepFile[];
|
|
398
|
+
/** Set when the step ran and the model reported a failure. Such a step
|
|
399
|
+
* still resolves; only a refused step rejects. */
|
|
400
|
+
error?: string;
|
|
401
|
+
}
|
|
279
402
|
|
|
403
|
+
/** What `Reactor.step()` sends: `state`, then `commands`, then every frame
|
|
404
|
+
* pushed since the previous step, applied in that order before the model
|
|
405
|
+
* generates. Every part is optional. A file goes in as a `FileRef` from
|
|
406
|
+
* `uploadFile()`, in a command's `data` or a state value. */
|
|
407
|
+
interface StepOptions {
|
|
408
|
+
/** Values for the model's state fields, keyed by field name. Each key runs
|
|
409
|
+
* as the model's `set_<key>` command, before `commands`. */
|
|
410
|
+
state?: Record<string, unknown>;
|
|
411
|
+
/** Commands to run after the state, in order. */
|
|
412
|
+
commands?: StepCommand[];
|
|
413
|
+
}
|
|
414
|
+
/** Raw pixels for `Reactor.pushFrame()`. `ImageData` is one of these as is. */
|
|
415
|
+
interface RawVideoFrame {
|
|
416
|
+
data: Uint8Array | Uint8ClampedArray;
|
|
417
|
+
width: number;
|
|
418
|
+
height: number;
|
|
419
|
+
/** Byte layout of `data`. Default `"rgba"` — what `ImageData` holds. */
|
|
420
|
+
format?: PixelFormat;
|
|
421
|
+
}
|
|
422
|
+
/** A video frame for `Reactor.pushFrame()`: a WebCodecs `VideoFrame` (the
|
|
423
|
+
* caller keeps ownership and closes it), an `ImageData`, or raw pixels. */
|
|
424
|
+
type PushVideoFrame = VideoFrame | ImageData | RawVideoFrame;
|
|
425
|
+
/** A block of audio for `Reactor.pushFrame()`: interleaved signed 16-bit
|
|
426
|
+
* PCM, already at the sample rate the model's track declares — a step's
|
|
427
|
+
* frame carries no sample rate for the runtime to resample from. */
|
|
428
|
+
interface PushAudioFrame {
|
|
429
|
+
samples: Int16Array;
|
|
430
|
+
sampleRate: number;
|
|
431
|
+
/** Interleaved channels in `samples`. Default 1. */
|
|
432
|
+
channels?: number;
|
|
433
|
+
}
|
|
434
|
+
type PushFrame = PushVideoFrame | PushAudioFrame;
|
|
435
|
+
/** What `Reactor.publishTrack()` resolves with: the published input track. */
|
|
436
|
+
interface PublishedTrack {
|
|
437
|
+
readonly name: string;
|
|
438
|
+
/** Buffers one input frame on this track for the next `step()` of a
|
|
439
|
+
* stepped session — `reactor.pushFrame(name, frame, options)`. Rejects
|
|
440
|
+
* with an `InvalidStateError` in a streamed session, whose media comes
|
|
441
|
+
* from the `MediaStreamTrack` given to `publishTrack()`, and once the
|
|
442
|
+
* track is unpublished or its connection ends: publish again for a new
|
|
443
|
+
* handle. */
|
|
444
|
+
pushFrame(frame: PushFrame, options?: PushFrameOptions): Promise<void>;
|
|
445
|
+
}
|
|
446
|
+
/** Per-frame extras for a pushed video frame. */
|
|
447
|
+
interface PushFrameOptions {
|
|
448
|
+
/** Opaque bytes the frame carries to the model. */
|
|
449
|
+
metadata?: Uint8Array;
|
|
450
|
+
/** When the frame was captured, in microseconds. */
|
|
451
|
+
captureTimeUs?: number;
|
|
452
|
+
}
|
|
280
453
|
/** The session resource, as the coordinator reports it — see
|
|
281
454
|
* `Reactor.getSessionInfo()`. */
|
|
282
455
|
type SessionResponse = SessionInfo;
|
|
@@ -645,10 +818,18 @@ declare class Reactor implements Disposable {
|
|
|
645
818
|
* with it, so a batch sent before the timings exist doesn't use it up.
|
|
646
819
|
* Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
|
|
647
820
|
private hasSentConnectTime;
|
|
821
|
+
/** Whether this connection has sent a client-stats batch yet. The first
|
|
822
|
+
* one goes out even with no reading, for the environment the bridge adds
|
|
823
|
+
* to it. Reset with `hasSentConnectTime`. */
|
|
824
|
+
private hasSentClientStats;
|
|
648
825
|
/** Set on the "connecting" status transition, cleared once `connectionTimings`
|
|
649
826
|
* is finalized on "ready" — see `handleStatusChanged()`. */
|
|
650
827
|
private connectStartTime;
|
|
651
828
|
private waitingStartTime;
|
|
829
|
+
/** The tail of every `pushFrame()` so far, settled either way — pushes
|
|
830
|
+
* reach the binding in call order however long each one's pixels take to
|
|
831
|
+
* read, and `step()` waits for the ones called before it. */
|
|
832
|
+
private pendingPushes;
|
|
652
833
|
private readonly emitter;
|
|
653
834
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
654
835
|
* disconnect()/[Symbol.dispose]) against each other. Calling into the
|
|
@@ -666,6 +847,15 @@ declare class Reactor implements Disposable {
|
|
|
666
847
|
* out every read active when it started, before it releases the ones
|
|
667
848
|
* that arrived while it ran. */
|
|
668
849
|
private writerActive;
|
|
850
|
+
/** Counts `disconnect()`s and disposal. A push captures it when called, and
|
|
851
|
+
* one whose frame was still being read when it moved is dropped, rather
|
|
852
|
+
* than recreating a client to carry the frame into the next session. */
|
|
853
|
+
private connectionEpoch;
|
|
854
|
+
/** Input tracks published on this connection, each with the publication
|
|
855
|
+
* that claimed it: a push needs one, and a `PublishedTrack` only feeds the
|
|
856
|
+
* publication it came from. Cleared with the connection. */
|
|
857
|
+
private readonly publishedInputs;
|
|
858
|
+
private publications;
|
|
669
859
|
/** Readers parked in `withReaderAccess()`'s wait loop, woken once
|
|
670
860
|
* `withWriterAccess()` finishes; each re-checks `writerActive` rather
|
|
671
861
|
* than assuming its turn, since another writer already queued behind the
|
|
@@ -801,8 +991,24 @@ declare class Reactor implements Disposable {
|
|
|
801
991
|
* disconnect() freeing the wasm client mid-await would otherwise resume
|
|
802
992
|
* into a freed client — the same class of use-after-free `queue` exists to
|
|
803
993
|
* prevent for connect/disconnect/reconnect themselves.
|
|
994
|
+
*
|
|
995
|
+
* Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
|
|
996
|
+
* track in a stepped session.
|
|
997
|
+
*
|
|
998
|
+
* In a `generation: "stepped"` session, input frames never travel over
|
|
999
|
+
* RTP: `track` is optional, and the frames are pushed with
|
|
1000
|
+
* `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
|
|
1001
|
+
* passed there is ignored, with a console warning — nothing is sent from
|
|
1002
|
+
* it. In a streamed session `track` is required, and omitting it rejects
|
|
1003
|
+
* with an `InvalidStateError`.
|
|
1004
|
+
*
|
|
1005
|
+
* ```ts
|
|
1006
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1007
|
+
* await camera.pushFrame(imageData);
|
|
1008
|
+
* const result = await reactor.step();
|
|
1009
|
+
* ```
|
|
804
1010
|
*/
|
|
805
|
-
publishTrack(name: string, track
|
|
1011
|
+
publishTrack(name: string, track?: MediaStreamTrack): Promise<PublishedTrack>;
|
|
806
1012
|
/**
|
|
807
1013
|
* Unlike every other track method, this doesn't reject — a failure is
|
|
808
1014
|
* reported through the `error` event instead, since this is commonly the
|
|
@@ -850,6 +1056,85 @@ declare class Reactor implements Disposable {
|
|
|
850
1056
|
}): Promise<void>;
|
|
851
1057
|
/** Resumes a track previously stopped with `pauseTrack()`. */
|
|
852
1058
|
resumeTrack(name: string): Promise<void>;
|
|
1059
|
+
/**
|
|
1060
|
+
* Runs one step of a `generation: "stepped"` session and resolves with
|
|
1061
|
+
* what it produced, per output track.
|
|
1062
|
+
*
|
|
1063
|
+
* The request carries `state`, `commands` and every frame pushed since the
|
|
1064
|
+
* previous step. The runtime applies them in that order — each `state` key
|
|
1065
|
+
* as the model's `set_<key>` command, then `commands`, then the frames —
|
|
1066
|
+
* and then generates. They are separate calls, not a transaction: one that
|
|
1067
|
+
* fails does not undo the ones before it. All of it is optional:
|
|
1068
|
+
* `step()` with nothing set and nothing pushed runs on the model's current
|
|
1069
|
+
* state, which commands sent outside a step (`sendCommand()`) also change.
|
|
1070
|
+
* A file goes in the way it does for a command: upload it with
|
|
1071
|
+
* `uploadFile()`, then put the `FileRef` in a command's `data` or a state
|
|
1072
|
+
* value.
|
|
1073
|
+
*
|
|
1074
|
+
* A step that ran but that the model reported as failed still resolves,
|
|
1075
|
+
* with `result.error` set. `result.files` lists the files the model kept
|
|
1076
|
+
* with the step (name, content type, size); their bytes are served over
|
|
1077
|
+
* HTTP from the session's step results, not carried here.
|
|
1078
|
+
*
|
|
1079
|
+
* Rejects with a `StepRefusedError` when the runtime or the model turns
|
|
1080
|
+
* the step down (the runtime checks a whole request first, so nothing it
|
|
1081
|
+
* carried was applied), `StepNotReadyError` when the session cannot take a
|
|
1082
|
+
* step yet (neither ran the step), `StepCommandFailedError` when a state key
|
|
1083
|
+
* or command failed (the calls before it stay applied; the step did not
|
|
1084
|
+
* run); both name the part at fault in `entry`, and `StepCommandFailedError`
|
|
1085
|
+
* the handler's code in `command_code`,
|
|
1086
|
+
* `StepSessionEndedError` when the session ended before the result came
|
|
1087
|
+
* back, `StepResultTooLargeError` when the result does not fit in one
|
|
1088
|
+
* message, `MessageTooLargeError` when the request itself is too large
|
|
1089
|
+
* (nothing was sent), `InvalidStateError` outside a ready stepped
|
|
1090
|
+
* session, or whatever else failed the step (`DisconnectedError`,
|
|
1091
|
+
* `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
|
|
1092
|
+
* when its request cannot be sent, they are dropped with it rather than
|
|
1093
|
+
* kept for the next step.
|
|
1094
|
+
*
|
|
1095
|
+
* ```ts
|
|
1096
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1097
|
+
* await camera.pushFrame(imageData);
|
|
1098
|
+
* const result = await reactor.step({ state: { prompt: 'a cat' } });
|
|
1099
|
+
* if (result.error) console.warn('step failed:', result.error);
|
|
1100
|
+
* const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
|
|
1101
|
+
* if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
|
|
1102
|
+
* ```
|
|
1103
|
+
*/
|
|
1104
|
+
step(options?: StepOptions): Promise<StepResult>;
|
|
1105
|
+
/**
|
|
1106
|
+
* Buffers one input frame for the next `step()` on the sendonly track
|
|
1107
|
+
* `trackName`. Nothing is sent until that step, which carries every frame
|
|
1108
|
+
* pushed since the one before it, per track in push order. The
|
|
1109
|
+
* `PublishedTrack` that `publishTrack()` resolves with does the same
|
|
1110
|
+
* through `track.pushFrame(frame)`.
|
|
1111
|
+
*
|
|
1112
|
+
* Video: a `VideoFrame` (cloned when called, so the caller may close it as
|
|
1113
|
+
* soon as this returns; it stays the caller's), an
|
|
1114
|
+
* `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
|
|
1115
|
+
* by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
|
|
1116
|
+
* `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
|
|
1117
|
+
* applied, so a 90° frame arrives with its width and height swapped.
|
|
1118
|
+
* Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
|
|
1119
|
+
* PCM at the rate the model's track declares.
|
|
1120
|
+
*
|
|
1121
|
+
* Pushes keep their call order, and a `step()` waits for every push called
|
|
1122
|
+
* before it. A refused push rejects: await it, or `.catch()` it in a frame
|
|
1123
|
+
* loop, or every bad frame is an unhandled rejection. What waits for
|
|
1124
|
+
* the next step is at most one message's worth of frames: pushing faster
|
|
1125
|
+
* than steps run drops the oldest frames, across tracks. Rejects with
|
|
1126
|
+
* an `InvalidStateError` outside a stepped session, for a track not
|
|
1127
|
+
* published on this connection (`publishTrack(trackName)` first), for a
|
|
1128
|
+
* frame whose size does not match its dimensions, or for a name the model
|
|
1129
|
+
* does not declare as a sendonly track of that kind.
|
|
1130
|
+
*/
|
|
1131
|
+
pushFrame(trackName: string, frame: PushFrame, options?: PushFrameOptions): Promise<void>;
|
|
1132
|
+
/** `pushFrame()`, for the track's current publication or, from a
|
|
1133
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
1134
|
+
private pushTo;
|
|
1135
|
+
/** Drops every frame buffered for the next step, pushes called before it
|
|
1136
|
+
* included: it waits its turn behind them, as a `step()` does. */
|
|
1137
|
+
clearStepInputs(): Promise<void>;
|
|
853
1138
|
/**
|
|
854
1139
|
* Requests a clip covering the last `durationSeconds` of the session.
|
|
855
1140
|
* `reactor-core` correlates the reply itself (and enforces `"ready"`), so
|
|
@@ -969,8 +1254,9 @@ declare class Reactor implements Disposable {
|
|
|
969
1254
|
* track has one yet (none negotiated onto a named track, or no codec
|
|
970
1255
|
* reported yet), so the runtime gets the bandwidth estimate, the
|
|
971
1256
|
* round-trip time and the time to connect without waiting on the tracks.
|
|
972
|
-
* Skipped only when there is nothing at all to say
|
|
973
|
-
*
|
|
1257
|
+
* Skipped only when there is nothing at all to say and a batch has already
|
|
1258
|
+
* gone out for this connection, so the first one still carries the
|
|
1259
|
+
* environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
|
|
974
1260
|
* actually sent for this connection — see `hasSentConnectTime`.
|
|
975
1261
|
* `sendClientStats` throws synchronously on a closing connection; caught
|
|
976
1262
|
* here for the same reason `getStats()`'s rejection is above. */
|
|
@@ -992,9 +1278,20 @@ declare class Reactor implements Disposable {
|
|
|
992
1278
|
* `error` event — the one place recording and emitting happen together,
|
|
993
1279
|
* so `getLastError()` never drifts from what listeners were told. */
|
|
994
1280
|
private emitError;
|
|
1281
|
+
private isStepped;
|
|
995
1282
|
private assertNotDisposed;
|
|
996
1283
|
}
|
|
997
1284
|
|
|
1285
|
+
/**
|
|
1286
|
+
* A step's video frame as an `ImageData`, ready for a canvas's
|
|
1287
|
+
* `putImageData()` (or `createImageBitmap()`).
|
|
1288
|
+
*
|
|
1289
|
+
* A step's video frames are `(height, width, 3)` `uint8` RGB; a canvas wants
|
|
1290
|
+
* RGBA, so this adds an opaque alpha. Throws on any other frame — an audio
|
|
1291
|
+
* frame, or a dtype or shape that is not RGB video.
|
|
1292
|
+
*/
|
|
1293
|
+
declare function stepFrameToImageData(frame: StepFrame): ImageData;
|
|
1294
|
+
|
|
998
1295
|
/** Lazy resolver for a Coordinator/runtime bearer token, called immediately
|
|
999
1296
|
* before each authenticated request with the session that request is for —
|
|
1000
1297
|
* see `JwtSource` and `JwtRequestContext`. A resolver that ignores the
|
|
@@ -1040,7 +1337,10 @@ interface ReactorActions {
|
|
|
1040
1337
|
disconnect: (recoverable?: boolean) => Promise<void>;
|
|
1041
1338
|
reconnect: (options?: ConnectOptions) => Promise<void>;
|
|
1042
1339
|
sendCommand: <T extends object = Record<string, unknown>>(command: string, data?: NotFunction<T>, scope?: MessageScope) => Promise<ReactorMessage | undefined>;
|
|
1043
|
-
|
|
1340
|
+
/** `Reactor.publishTrack()`. `track` is required in a streamed session
|
|
1341
|
+
* and ignored (with a warning) in a stepped one, whose frames go through
|
|
1342
|
+
* the resolved handle's `pushFrame()`. */
|
|
1343
|
+
publish: (name: string, track?: MediaStreamTrack) => Promise<PublishedTrack>;
|
|
1044
1344
|
unpublish: (name: string) => Promise<void>;
|
|
1045
1345
|
pauseTrack: (name: string) => Promise<void>;
|
|
1046
1346
|
resumeTrack: (name: string) => Promise<void>;
|
|
@@ -1342,4 +1642,4 @@ interface ClipDownloadButtonProps {
|
|
|
1342
1642
|
}
|
|
1343
1643
|
declare function ClipDownloadButton({ clip, getJwt, filename, children, className, style, disabled, onSuccess, onError, }: ClipDownloadButtonProps): react.JSX.Element;
|
|
1344
1644
|
|
|
1345
|
-
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DecodeError, DisconnectedError, type DownloadClipOptions, type FetchPlaylistOptions, FileRef, InvalidStateError, type JwtRequestContext, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|
|
1645
|
+
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DataChannelChunkingRequiredError, DecodeError, DisconnectedError, type DownloadClipOptions, type FetchPlaylistOptions, FileRef, type GenerationMode, GenerationModeMismatchError, InvalidStateError, type JwtRequestContext, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, type PublishedTrack, type PushAudioFrame, type PushFrame, type PushFrameOptions, type PushVideoFrame, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type StepCommand, StepCommandFailedError, type StepFile, type StepFrame, StepNotReadyError, type StepOptions, StepRefusedError, type StepResult, StepResultTooLargeError, StepSessionEndedError, type StepTrack, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, UnsupportedFrameEncodingError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, stepFrameToImageData, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|