@reactor-team/js-sdk 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -21
- package/dist/index.cjs +742 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +182 -66
- package/dist/index.d.ts +182 -66
- package/dist/index.js +742 -45
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +36 -12
- package/dist/wasm/reactor_wasm.d.ts +93 -20
- package/dist/wasm/reactor_wasm.js +114 -22
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +6 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -212,28 +212,6 @@ declare class UnsupportedFrameEncodingError extends ReactorError {
|
|
|
212
212
|
static readonly code = "UNSUPPORTED_FRAME_ENCODING";
|
|
213
213
|
}
|
|
214
214
|
|
|
215
|
-
/**
|
|
216
|
-
* Reference to a file uploaded via `Reactor.uploadFile()`. Pass it back in
|
|
217
|
-
* `sendCommand()`'s `data` wherever the model's schema declares a file: as a
|
|
218
|
-
* top-level value it's extracted and sent as a separate upload reference,
|
|
219
|
-
* and inside an array or object it's serialized as one — see
|
|
220
|
-
* `extractFileRefs()`.
|
|
221
|
-
*/
|
|
222
|
-
declare class FileRef {
|
|
223
|
-
readonly uploadId: string;
|
|
224
|
-
readonly name: string;
|
|
225
|
-
readonly mimeType: string;
|
|
226
|
-
readonly size: number;
|
|
227
|
-
constructor(uploadId: string, name: string, mimeType: string, size: number);
|
|
228
|
-
}
|
|
229
|
-
/**
|
|
230
|
-
* Structural check for anything shaped like a `FileRef`, for callers who'd
|
|
231
|
-
* rather duck-type than rely on `instanceof` (e.g. across two copies of this
|
|
232
|
-
* package bundled into the same page). `extractFileRefs()` uses this too,
|
|
233
|
-
* for the same reason.
|
|
234
|
-
*/
|
|
235
|
-
declare function isFileRef(value: unknown): value is FileRef;
|
|
236
|
-
|
|
237
215
|
/**
|
|
238
216
|
* The `reactor-wasm` contract, mirrored by hand.
|
|
239
217
|
*
|
|
@@ -259,7 +237,9 @@ type GenerationMode = 'streamed' | 'stepped';
|
|
|
259
237
|
/** Byte layout of a frame given to `pushVideoFrame`. */
|
|
260
238
|
type PixelFormat = 'rgba' | 'rgb' | 'bgra';
|
|
261
239
|
/** Codec of an encoded frame given to `pushEncodedVideoFrame`. */
|
|
262
|
-
type EncodedVideoCodec = 'h264';
|
|
240
|
+
type EncodedVideoCodec = 'h264' | 'vp8' | 'vp9' | 'av1';
|
|
241
|
+
/** How a track's step frames travel on a connection: raw, or in a codec. */
|
|
242
|
+
type StepFrameEncoding = 'raw' | EncodedVideoCodec;
|
|
263
243
|
interface ClientOptions {
|
|
264
244
|
modelName: string;
|
|
265
245
|
apiUrl?: string;
|
|
@@ -354,8 +334,11 @@ interface StepCommand {
|
|
|
354
334
|
/** One frame a step produced. Hand-built by the binding rather than
|
|
355
335
|
* serialized, so camelCase, unlike the wire payloads above. */
|
|
356
336
|
interface StepFrame {
|
|
357
|
-
/** How the frame travelled
|
|
358
|
-
|
|
337
|
+
/** How the frame travelled. From the binding, a frame in a codec comes
|
|
338
|
+
* up still encoded (`data` its bitstream, `shape` the (height, width, 3)
|
|
339
|
+
* it decodes to), and the SDK decodes it to RGB before `step()` resolves;
|
|
340
|
+
* `encoding` still names the codec it travelled in. */
|
|
341
|
+
encoding: 'raw' | 'unspecified' | EncodedVideoCodec;
|
|
359
342
|
/** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
|
|
360
343
|
dtype: string;
|
|
361
344
|
/** (height, width, 3) RGB for video, (channels, samples) planar PCM for
|
|
@@ -369,6 +352,9 @@ interface StepFrame {
|
|
|
369
352
|
metadata?: Uint8Array;
|
|
370
353
|
/** When the model says the frame was captured, in microseconds. */
|
|
371
354
|
captureTimeUs?: number;
|
|
355
|
+
/** Set on a frame that travelled in a codec: whether the runtime marked
|
|
356
|
+
* it as a keyframe. */
|
|
357
|
+
keyframe?: boolean;
|
|
372
358
|
}
|
|
373
359
|
/** The frames one output track produced in a step, in production order. */
|
|
374
360
|
interface StepTrack {
|
|
@@ -403,11 +389,37 @@ interface StepResult {
|
|
|
403
389
|
/** Set when the step ran and the model reported a failure. Such a step
|
|
404
390
|
* still resolves; only a refused step rejects. */
|
|
405
391
|
error?: string;
|
|
392
|
+
/** What the model returned for the step besides its media, opaque to the
|
|
393
|
+
* transport: the bytes it returned as they are, or a dict it returned
|
|
394
|
+
* encoded as JSON. Arrives with the step's frames, so a model whose output
|
|
395
|
+
* is data alone needs no output track. Absent when it returned none. */
|
|
396
|
+
data?: Uint8Array;
|
|
406
397
|
}
|
|
407
398
|
|
|
408
|
-
/**
|
|
409
|
-
*
|
|
410
|
-
*
|
|
399
|
+
/**
|
|
400
|
+
* Reference to a file uploaded via `Reactor.uploadFile()`. Pass it back in
|
|
401
|
+
* `sendCommand()`'s `data` wherever the model's schema declares a file: as a
|
|
402
|
+
* top-level value it's extracted and sent as a separate upload reference,
|
|
403
|
+
* and inside an array or object it's serialized as one — see
|
|
404
|
+
* `extractFileRefs()`.
|
|
405
|
+
*/
|
|
406
|
+
declare class FileRef {
|
|
407
|
+
readonly uploadId: string;
|
|
408
|
+
readonly name: string;
|
|
409
|
+
readonly mimeType: string;
|
|
410
|
+
readonly size: number;
|
|
411
|
+
constructor(uploadId: string, name: string, mimeType: string, size: number);
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Structural check for anything shaped like a `FileRef`, for callers who'd
|
|
415
|
+
* rather duck-type than rely on `instanceof` (e.g. across two copies of this
|
|
416
|
+
* package bundled into the same page). `extractFileRefs()` uses this too,
|
|
417
|
+
* for the same reason.
|
|
418
|
+
*/
|
|
419
|
+
declare function isFileRef(value: unknown): value is FileRef;
|
|
420
|
+
|
|
421
|
+
/** What `Reactor.step()` sends: `state`, then `commands`, then any frames
|
|
422
|
+
* still buffered, applied in that order before the model generates. Every part is optional. A file goes in as a `FileRef` from
|
|
411
423
|
* `uploadFile()`, in a command's `data` or a state value. */
|
|
412
424
|
interface StepOptions {
|
|
413
425
|
/** Values for the model's state fields, keyed by field name. Each key runs
|
|
@@ -422,13 +434,19 @@ interface StepOptions {
|
|
|
422
434
|
* the step before any of them is pushed, leaving the buffer as it was. */
|
|
423
435
|
media?: Record<string, ReadonlyArray<PushFrame | EncodedFrame>>;
|
|
424
436
|
}
|
|
437
|
+
/** Options of `Reactor.setCodecPreference()`. */
|
|
438
|
+
interface CodecPreferenceOptions {
|
|
439
|
+
/** The one track the list is for, in place of the session's list. */
|
|
440
|
+
track?: string;
|
|
441
|
+
}
|
|
425
442
|
/**
|
|
426
|
-
* An encoded video frame for a step: one H.264
|
|
427
|
-
*
|
|
443
|
+
* An encoded video frame for a step, its bytes sent as they are: one H.264
|
|
444
|
+
* access unit (Annex B), one VP8 or VP9 frame, or one AV1 temporal unit of
|
|
445
|
+
* OBUs. `width` and `height` are the frame it decodes to.
|
|
428
446
|
*
|
|
429
|
-
* The runtime decodes a track's encoded frames
|
|
430
|
-
*
|
|
431
|
-
*
|
|
447
|
+
* The runtime decodes a track's encoded frames as one stream, as they
|
|
448
|
+
* arrive, and the model reads the RGB frames as if they had been pushed raw.
|
|
449
|
+
* So give them in decode order, starting at a keyframe.
|
|
432
450
|
*
|
|
433
451
|
* `data` may be a WebCodecs `EncodedVideoChunk` as a `VideoEncoder` hands it
|
|
434
452
|
* over; configure the encoder with `avc: { format: 'annexb' }`, as its
|
|
@@ -447,6 +465,9 @@ interface EncodedFrame {
|
|
|
447
465
|
metadata?: Uint8Array;
|
|
448
466
|
/** When the frame was captured, in microseconds. */
|
|
449
467
|
captureTimeUs?: number;
|
|
468
|
+
/** Whether the frame is one a decoder can start from. Read from an
|
|
469
|
+
* `EncodedVideoChunk`'s `type` when not given. */
|
|
470
|
+
keyframe?: boolean;
|
|
450
471
|
}
|
|
451
472
|
/** Raw pixels for `Reactor.pushFrame()`. `ImageData` is one of these as is. */
|
|
452
473
|
interface RawVideoFrame {
|
|
@@ -472,14 +493,15 @@ type PushFrame = PushVideoFrame | PushAudioFrame;
|
|
|
472
493
|
/** What `Reactor.publishTrack()` resolves with: the published input track. */
|
|
473
494
|
interface PublishedTrack {
|
|
474
495
|
readonly name: string;
|
|
475
|
-
/**
|
|
476
|
-
*
|
|
496
|
+
/** Pushes one input frame on this track to a stepped session, sent at once
|
|
497
|
+
* or buffered for the next `step()` — `reactor.pushFrame(name, frame,
|
|
498
|
+
* options)`. Rejects
|
|
477
499
|
* with an `InvalidStateError` in a streamed session, whose media comes
|
|
478
500
|
* from the `MediaStreamTrack` given to `publishTrack()`, and once the
|
|
479
501
|
* track is unpublished or its connection ends: publish again for a new
|
|
480
502
|
* handle. */
|
|
481
503
|
pushFrame(frame: PushFrame, options?: PushFrameOptions): Promise<void>;
|
|
482
|
-
/**
|
|
504
|
+
/** Pushes one encoded video frame on this track, as `pushFrame()` does —
|
|
483
505
|
* `reactor.pushEncodedFrame(name, frame)`. Rejects as `pushFrame()` does. */
|
|
484
506
|
pushEncodedFrame(frame: EncodedFrame): Promise<void>;
|
|
485
507
|
}
|
|
@@ -871,6 +893,17 @@ declare class Reactor implements Disposable {
|
|
|
871
893
|
* one's pixels take to read, and `step()` waits for the ones called
|
|
872
894
|
* before it. */
|
|
873
895
|
private pendingPushes;
|
|
896
|
+
/** Stepped: every `setCodecPreference()` so far, keyed by track (`null`
|
|
897
|
+
* for the session's list), replayed into a client created later. */
|
|
898
|
+
private readonly codecPreferences;
|
|
899
|
+
/** Stepped: what this browser's WebCodecs can encode and decode, probed
|
|
900
|
+
* once, on the first stepped connect. */
|
|
901
|
+
private localStepCodecs;
|
|
902
|
+
/** Stepped: the input tracks' encoders and the output tracks' decoders of
|
|
903
|
+
* the current connection, dropped as it ends so the next one's streams
|
|
904
|
+
* start at keyframes. */
|
|
905
|
+
private readonly inputEncoders;
|
|
906
|
+
private readonly outputDecoders;
|
|
874
907
|
private readonly emitter;
|
|
875
908
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
876
909
|
* disconnect()/[Symbol.dispose]) against each other. Calling into the
|
|
@@ -1101,10 +1134,10 @@ declare class Reactor implements Disposable {
|
|
|
1101
1134
|
* Runs one step of a `generation: "stepped"` session and resolves with
|
|
1102
1135
|
* what it produced, per output track.
|
|
1103
1136
|
*
|
|
1104
|
-
* The request carries `state
|
|
1105
|
-
*
|
|
1106
|
-
* as the model's `set_<key>` command,
|
|
1107
|
-
* and then generates. They are separate calls, not a transaction: one that
|
|
1137
|
+
* The request carries `state` and `commands`; the frames pushed before it
|
|
1138
|
+
* have already reached the runtime (see `pushFrame()`). The runtime applies
|
|
1139
|
+
* them in that order — each `state` key as the model's `set_<key>` command,
|
|
1140
|
+
* then `commands` — and then generates. They are separate calls, not a transaction: one that
|
|
1108
1141
|
* fails does not undo the ones before it. All of it is optional:
|
|
1109
1142
|
* `step()` with nothing set and nothing pushed runs on the model's current
|
|
1110
1143
|
* state, which commands sent outside a step (`sendCommand()`) also change.
|
|
@@ -1115,7 +1148,9 @@ declare class Reactor implements Disposable {
|
|
|
1115
1148
|
* A step that ran but that the model reported as failed still resolves,
|
|
1116
1149
|
* with `result.error` set. `result.files` lists the files the model kept
|
|
1117
1150
|
* with the step (name, content type, size); their bytes are served over
|
|
1118
|
-
* HTTP from the session's step results, not carried here.
|
|
1151
|
+
* HTTP from the session's step results, not carried here. `result.data`
|
|
1152
|
+
* holds what the model returned besides its media, as opaque bytes (a dict
|
|
1153
|
+
* arrives encoded as JSON); it is absent when the model returned none.
|
|
1119
1154
|
*
|
|
1120
1155
|
* Rejects with a `StepRefusedError` when the runtime or the model turns
|
|
1121
1156
|
* the step down (the runtime checks a whole request first, so nothing it
|
|
@@ -1129,7 +1164,7 @@ declare class Reactor implements Disposable {
|
|
|
1129
1164
|
* message, `MessageTooLargeError` when the request itself is too large
|
|
1130
1165
|
* (nothing was sent), `InvalidStateError` outside a ready stepped
|
|
1131
1166
|
* session, or whatever else failed the step (`DisconnectedError`,
|
|
1132
|
-
* `RequestTimeoutError`, ...). The frames
|
|
1167
|
+
* `RequestTimeoutError`, ...). The frames buffered for a step belong to it:
|
|
1133
1168
|
* when its request cannot be sent, they are dropped with it rather than
|
|
1134
1169
|
* kept for the next step.
|
|
1135
1170
|
*
|
|
@@ -1143,9 +1178,19 @@ declare class Reactor implements Disposable {
|
|
|
1143
1178
|
* before any of it is pushed or sent, leaving the buffer as it was. A
|
|
1144
1179
|
* session that leaves 'ready' once the media is in has the step refused and
|
|
1145
1180
|
* keeps the frames for the next one, as pushed frames are.
|
|
1146
|
-
* A track's encoded frames
|
|
1147
|
-
*
|
|
1148
|
-
* `StepRefusedError
|
|
1181
|
+
* A track's encoded frames go in decode order, starting at a keyframe; a
|
|
1182
|
+
* step whose buffered frames do not decode rejects with a
|
|
1183
|
+
* `StepRefusedError` (see `pushEncodedFrame()` for frames already sent).
|
|
1184
|
+
*
|
|
1185
|
+
* Video step frames travel encoded both ways when this browser and the
|
|
1186
|
+
* runtime share a codec (H.264 first by default; see
|
|
1187
|
+
* `setCodecPreference()`), on WebCodecs: the frames pushed on an input
|
|
1188
|
+
* track are encoded, and an output track's frames are decoded, so a result
|
|
1189
|
+
* still holds RGB frames whose `encoding` names the codec they travelled
|
|
1190
|
+
* in. Each track is one stream for the connection. When one of its
|
|
1191
|
+
* decoders lost its stream, the step rejects with a `DecodeError`, the SDK
|
|
1192
|
+
* pauses and resumes that track so the runtime starts it again at a
|
|
1193
|
+
* keyframe, and the track's frames fail until that keyframe arrives.
|
|
1149
1194
|
*
|
|
1150
1195
|
* ```ts
|
|
1151
1196
|
* const camera = await reactor.publishTrack('camera');
|
|
@@ -1158,11 +1203,15 @@ declare class Reactor implements Disposable {
|
|
|
1158
1203
|
*/
|
|
1159
1204
|
step(options?: StepOptions): Promise<StepResult>;
|
|
1160
1205
|
/**
|
|
1161
|
-
*
|
|
1162
|
-
*
|
|
1163
|
-
*
|
|
1164
|
-
*
|
|
1165
|
-
*
|
|
1206
|
+
* Pushes one input frame on the sendonly track `trackName` of a stepped
|
|
1207
|
+
* session. Once the session is ready, against a runtime that takes pushed
|
|
1208
|
+
* frames, the frame is sent to the runtime at once over the data channel,
|
|
1209
|
+
* ahead of any `step()` after it; one the channel cannot take is dropped
|
|
1210
|
+
* and raised as an error event. A frame pushed before the session is ready,
|
|
1211
|
+
* or to an older runtime, is buffered and sent with the next push or
|
|
1212
|
+
* `step()`, per track in push order. The `PublishedTrack` that
|
|
1213
|
+
* `publishTrack()` resolves with does the same through
|
|
1214
|
+
* `track.pushFrame(frame)`.
|
|
1166
1215
|
*
|
|
1167
1216
|
* Video: a `VideoFrame` (cloned when called, so the caller may close it as
|
|
1168
1217
|
* soon as this returns; it stays the caller's), an
|
|
@@ -1173,9 +1222,15 @@ declare class Reactor implements Disposable {
|
|
|
1173
1222
|
* Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
|
|
1174
1223
|
* PCM at the rate the model's track declares.
|
|
1175
1224
|
*
|
|
1225
|
+
* A video frame travels encoded when the connection negotiated a codec for
|
|
1226
|
+
* the track (see `setCodecPreference()`): it is encoded here, with
|
|
1227
|
+
* WebCodecs, as the next frame of the track's stream, and the model still
|
|
1228
|
+
* reads RGB. A track's first frame on a connection is a keyframe, and so
|
|
1229
|
+
* is the next one after the runtime could not decode its frames.
|
|
1230
|
+
*
|
|
1176
1231
|
* Pushes keep their call order, and a `step()` waits for every push called
|
|
1177
1232
|
* before it. A refused push rejects: await it, or `.catch()` it in a frame
|
|
1178
|
-
* loop, or every bad frame is an unhandled rejection. What
|
|
1233
|
+
* loop, or every bad frame is an unhandled rejection. What is buffered for
|
|
1179
1234
|
* the next step is at most one message's worth of frames: pushing faster
|
|
1180
1235
|
* than steps run drops the oldest frames, across tracks. Rejects with
|
|
1181
1236
|
* an `InvalidStateError` outside a stepped session, for a track not
|
|
@@ -1188,18 +1243,22 @@ declare class Reactor implements Disposable {
|
|
|
1188
1243
|
* `PublishedTrack`, only for the `publication` it came from. */
|
|
1189
1244
|
private pushTo;
|
|
1190
1245
|
/**
|
|
1191
|
-
*
|
|
1192
|
-
* video track `trackName`, its bytes as they are: one
|
|
1193
|
-
* (Annex B),
|
|
1194
|
-
* `
|
|
1195
|
-
*
|
|
1246
|
+
* Pushes one encoded video frame, sent or buffered as `pushFrame()` does,
|
|
1247
|
+
* on the sendonly video track `trackName`, its bytes as they are: one
|
|
1248
|
+
* H.264 access unit (Annex B), one VP8 or VP9 frame, or one AV1 temporal
|
|
1249
|
+
* unit, with the `width` and `height` it decodes to. The `PublishedTrack`
|
|
1250
|
+
* that `publishTrack()` resolves with does the same through
|
|
1251
|
+
* `track.pushEncodedFrame(frame)`.
|
|
1196
1252
|
*
|
|
1197
|
-
* The runtime decodes a track's encoded frames
|
|
1198
|
-
*
|
|
1199
|
-
*
|
|
1200
|
-
*
|
|
1201
|
-
*
|
|
1202
|
-
*
|
|
1253
|
+
* The runtime decodes a track's encoded frames as one stream, as they
|
|
1254
|
+
* arrive, and the model reads the RGB frames as if they had been pushed
|
|
1255
|
+
* raw. So push them in decode order, starting at a keyframe (`keyframe`, or
|
|
1256
|
+
* a chunk's `type`). A pushed frame that does not decode raises an
|
|
1257
|
+
* `input_decode_failed` error event naming the track, whose next frame
|
|
1258
|
+
* must then be a keyframe; a step whose buffered frames do not decode
|
|
1259
|
+
* rejects with a `StepRefusedError`, as does a runtime that predates
|
|
1260
|
+
* encoded frames. Encoded and raw pushes keep their call order, and a
|
|
1261
|
+
* `step()` waits for every push called before it.
|
|
1203
1262
|
*
|
|
1204
1263
|
* ```ts
|
|
1205
1264
|
* const encoder = new VideoEncoder({
|
|
@@ -1211,7 +1270,7 @@ declare class Reactor implements Disposable {
|
|
|
1211
1270
|
*
|
|
1212
1271
|
* Rejects with an `InvalidStateError` outside a stepped session, for a
|
|
1213
1272
|
* track not published on this connection, for an audio or recvonly track,
|
|
1214
|
-
* for empty data, a zero dimension or
|
|
1273
|
+
* for empty data, a zero dimension or an unknown codec.
|
|
1215
1274
|
*/
|
|
1216
1275
|
pushEncodedFrame(trackName: string, frame: EncodedFrame): Promise<void>;
|
|
1217
1276
|
/** `pushEncodedFrame()`, for the track's current publication or, from a
|
|
@@ -1220,14 +1279,68 @@ declare class Reactor implements Disposable {
|
|
|
1220
1279
|
/** Refuses a push whose connection ended since it was called, or whose
|
|
1221
1280
|
* track is not (or no longer) published on it. */
|
|
1222
1281
|
private checkPushTarget;
|
|
1282
|
+
/** Hands `pixels` to the binding for `track`: encoded first, on the
|
|
1283
|
+
* track's encoder, when the connection negotiated a codec for the track's
|
|
1284
|
+
* step frames and this browser can encode; raw otherwise, and when the
|
|
1285
|
+
* encoder fails (its next frame is then a keyframe). Checked by the
|
|
1286
|
+
* binding before anything is encoded, so a bad frame is refused either
|
|
1287
|
+
* way. `stillTarget` throws when the push's connection or publication is
|
|
1288
|
+
* gone: asked again once the encode is awaited, since a reconnect can
|
|
1289
|
+
* start meanwhile, and neither the encoded frame nor its raw fallback may
|
|
1290
|
+
* then reach the next session. */
|
|
1291
|
+
private pushPixels;
|
|
1292
|
+
/** Gives the binding the codecs this browser can carry step frames in,
|
|
1293
|
+
* probed once, so the next offer can negotiate them. A streamed session
|
|
1294
|
+
* carries no step frames. */
|
|
1295
|
+
/** Restart each output track whose decoder lost its stream: a pause and
|
|
1296
|
+
* a resume of that track, so the runtime starts it again at a keyframe. A
|
|
1297
|
+
* restart that cannot be sent is left to the next connection, whose
|
|
1298
|
+
* streams start at keyframes anyway. */
|
|
1299
|
+
private restartLostTracks;
|
|
1300
|
+
private offerStepCodecs;
|
|
1223
1301
|
/** Pushes `step()`'s `media`, track by track and in order, each frame as
|
|
1224
1302
|
* `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
|
|
1225
1303
|
* checks the step's `state` and `commands`, then every frame, before the
|
|
1226
1304
|
* first is pushed, so a refused step, or a bad track or frame anywhere,
|
|
1227
1305
|
* leaves the buffer as it was. */
|
|
1228
1306
|
private pushStepMedia;
|
|
1229
|
-
/**
|
|
1230
|
-
*
|
|
1307
|
+
/**
|
|
1308
|
+
* Chooses the codecs a stepped session carries its video step frames in,
|
|
1309
|
+
* most preferred first, from `"h264"`, `"vp9"`, `"av1"` and `"vp8"`.
|
|
1310
|
+
* `null` goes back to the default, H.264 first. An empty list for the
|
|
1311
|
+
* session turns encoding off, so every frame travels raw:
|
|
1312
|
+
*
|
|
1313
|
+
* ```ts
|
|
1314
|
+
* reactor.setCodecPreference(['vp9', 'h264']); // VP9 when both ends have it
|
|
1315
|
+
* reactor.setCodecPreference([]); // raw frames
|
|
1316
|
+
* reactor.setCodecPreference(['av1'], { track: 'main_video' });
|
|
1317
|
+
* ```
|
|
1318
|
+
*
|
|
1319
|
+
* With `track`, the list applies to that track only, in place of the
|
|
1320
|
+
* session's; an empty one there goes back to the session's, since one
|
|
1321
|
+
* track cannot turn encoding off alone. The list is an order, not a
|
|
1322
|
+
* limit: the codecs it leaves out follow it, so a runtime without the
|
|
1323
|
+
* preferred ones takes another. Of those, the ones this browser's
|
|
1324
|
+
* WebCodecs can encode (input tracks) or decode (output tracks) are
|
|
1325
|
+
* offered, and the runtime takes per track the first it shares; a track
|
|
1326
|
+
* they share no codec for stays raw.
|
|
1327
|
+
*
|
|
1328
|
+
* Whatever the codec, `step()` results and the frames `pushFrame()` takes
|
|
1329
|
+
* stay RGB: the encoding is between this client and the runtime. Callable
|
|
1330
|
+
* before `connect()` or during a session; it takes effect on the next
|
|
1331
|
+
* connection or reconnect, and the live connection keeps what it
|
|
1332
|
+
* negotiated (see `getStepFrameEncodings()`). A streamed session ignores
|
|
1333
|
+
* it. Throws an `InvalidStateError` on a name that is not a codec.
|
|
1334
|
+
*/
|
|
1335
|
+
setCodecPreference(codecs: EncodedVideoCodec[] | null, options?: CodecPreferenceOptions): void;
|
|
1336
|
+
/** How each track's step frames travel on this connection, as its offer
|
|
1337
|
+
* and answer negotiated them: `"raw"`, or the codec of an encoded video
|
|
1338
|
+
* track. Empty outside a connected stepped session; a track left out is
|
|
1339
|
+
* raw. */
|
|
1340
|
+
getStepFrameEncodings(): Record<string, StepFrameEncoding>;
|
|
1341
|
+
/** Drops every frame still buffered for the next step, pushes called before
|
|
1342
|
+
* it included: it waits its turn behind them, as a `step()` does. Frames
|
|
1343
|
+
* already sent to the runtime are not recalled. */
|
|
1231
1344
|
clearStepInputs(): Promise<void>;
|
|
1232
1345
|
/**
|
|
1233
1346
|
* Requests a clip covering the last `durationSeconds` of the session.
|
|
@@ -1364,6 +1477,9 @@ declare class Reactor implements Disposable {
|
|
|
1364
1477
|
* leave `getCapabilities()` returning the previous session's stale
|
|
1365
1478
|
* tracks/commands until a new `capabilitiesReceived` lands. */
|
|
1366
1479
|
private resetConnectionState;
|
|
1480
|
+
/** Drops every step frame encoder and decoder: each track's stream lived
|
|
1481
|
+
* with the connection that ended, and the next one's start at keyframes. */
|
|
1482
|
+
private resetStepStreams;
|
|
1367
1483
|
/** Wraps `cause` and records it as the most recent failure, for a call
|
|
1368
1484
|
* that's about to throw rather than emit — see `getLastError()`'s doc
|
|
1369
1485
|
* comment. */
|
|
@@ -1736,4 +1852,4 @@ interface ClipDownloadButtonProps {
|
|
|
1736
1852
|
}
|
|
1737
1853
|
declare function ClipDownloadButton({ clip, getJwt, filename, children, className, style, disabled, onSuccess, onError, }: ClipDownloadButtonProps): react.JSX.Element;
|
|
1738
1854
|
|
|
1739
|
-
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DataChannelChunkingRequiredError, DecodeError, DisconnectedError, type DownloadClipOptions, type EncodedFrame, type EncodedVideoCodec, type FetchPlaylistOptions, FileRef, type GenerationMode, GenerationModeMismatchError, InvalidStateError, type JwtRequestContext, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, type PublishedTrack, type PushAudioFrame, type PushFrame, type PushFrameOptions, type PushVideoFrame, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type StepCommand, StepCommandFailedError, type StepFile, type StepFrame, StepNotReadyError, type StepOptions, StepRefusedError, type StepResult, StepResultTooLargeError, StepSessionEndedError, type StepTrack, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, UnsupportedFrameEncodingError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, stepFrameToImageData, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|
|
1855
|
+
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CodecPreferenceOptions, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DataChannelChunkingRequiredError, DecodeError, DisconnectedError, type DownloadClipOptions, type EncodedFrame, type EncodedVideoCodec, type FetchPlaylistOptions, FileRef, type GenerationMode, GenerationModeMismatchError, InvalidStateError, type JwtRequestContext, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, type PublishedTrack, type PushAudioFrame, type PushFrame, type PushFrameOptions, type PushVideoFrame, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type StepCommand, StepCommandFailedError, type StepFile, type StepFrame, type StepFrameEncoding, StepNotReadyError, type StepOptions, StepRefusedError, type StepResult, StepResultTooLargeError, StepSessionEndedError, type StepTrack, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, UnsupportedFrameEncodingError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, stepFrameToImageData, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|