@reactor-team/js-sdk 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -9
- package/dist/index.cjs +1013 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +257 -47
- package/dist/index.d.ts +257 -47
- package/dist/index.js +1013 -45
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +36 -9
- package/dist/wasm/reactor_wasm.d.ts +143 -10
- package/dist/wasm/reactor_wasm.js +216 -14
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +11 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -212,28 +212,6 @@ declare class UnsupportedFrameEncodingError extends ReactorError {
|
|
|
212
212
|
static readonly code = "UNSUPPORTED_FRAME_ENCODING";
|
|
213
213
|
}
|
|
214
214
|
|
|
215
|
-
/**
|
|
216
|
-
* Reference to a file uploaded via `Reactor.uploadFile()`. Pass it back in
|
|
217
|
-
* `sendCommand()`'s `data` wherever the model's schema declares a file: as a
|
|
218
|
-
* top-level value it's extracted and sent as a separate upload reference,
|
|
219
|
-
* and inside an array or object it's serialized as one — see
|
|
220
|
-
* `extractFileRefs()`.
|
|
221
|
-
*/
|
|
222
|
-
declare class FileRef {
|
|
223
|
-
readonly uploadId: string;
|
|
224
|
-
readonly name: string;
|
|
225
|
-
readonly mimeType: string;
|
|
226
|
-
readonly size: number;
|
|
227
|
-
constructor(uploadId: string, name: string, mimeType: string, size: number);
|
|
228
|
-
}
|
|
229
|
-
/**
|
|
230
|
-
* Structural check for anything shaped like a `FileRef`, for callers who'd
|
|
231
|
-
* rather duck-type than rely on `instanceof` (e.g. across two copies of this
|
|
232
|
-
* package bundled into the same page). `extractFileRefs()` uses this too,
|
|
233
|
-
* for the same reason.
|
|
234
|
-
*/
|
|
235
|
-
declare function isFileRef(value: unknown): value is FileRef;
|
|
236
|
-
|
|
237
215
|
/**
|
|
238
216
|
* The `reactor-wasm` contract, mirrored by hand.
|
|
239
217
|
*
|
|
@@ -258,6 +236,10 @@ type TrackDirection = 'recvonly' | 'sendonly';
|
|
|
258
236
|
type GenerationMode = 'streamed' | 'stepped';
|
|
259
237
|
/** Byte layout of a frame given to `pushVideoFrame`. */
|
|
260
238
|
type PixelFormat = 'rgba' | 'rgb' | 'bgra';
|
|
239
|
+
/** Codec of an encoded frame given to `pushEncodedVideoFrame`. */
|
|
240
|
+
type EncodedVideoCodec = 'h264' | 'vp8' | 'vp9' | 'av1';
|
|
241
|
+
/** How a track's step frames travel on a connection: raw, or in a codec. */
|
|
242
|
+
type StepFrameEncoding = 'raw' | EncodedVideoCodec;
|
|
261
243
|
interface ClientOptions {
|
|
262
244
|
modelName: string;
|
|
263
245
|
apiUrl?: string;
|
|
@@ -281,6 +263,9 @@ interface ClientOptions {
|
|
|
281
263
|
/** How long `step()` waits for a step's result, generation included.
|
|
282
264
|
* Default 300000. */
|
|
283
265
|
stepTimeoutMs?: number;
|
|
266
|
+
/** The largest data-channel message, in bytes, the client declares and
|
|
267
|
+
* accepts: 262144 (256 KiB) to 1073741824 (1 GiB). Default 64 MiB. */
|
|
268
|
+
maxMessageSize?: number;
|
|
284
269
|
logLevel?: 'off' | 'error' | 'warn' | 'info' | 'debug' | 'trace';
|
|
285
270
|
}
|
|
286
271
|
interface ConnectOptions {
|
|
@@ -349,8 +334,11 @@ interface StepCommand {
|
|
|
349
334
|
/** One frame a step produced. Hand-built by the binding rather than
|
|
350
335
|
* serialized, so camelCase, unlike the wire payloads above. */
|
|
351
336
|
interface StepFrame {
|
|
352
|
-
/** How the frame travelled
|
|
353
|
-
|
|
337
|
+
/** How the frame travelled. From the binding, a frame in a codec comes
|
|
338
|
+
* up still encoded (`data` its bitstream, `shape` the (height, width, 3)
|
|
339
|
+
* it decodes to), and the SDK decodes it to RGB before `step()` resolves;
|
|
340
|
+
* `encoding` still names the codec it travelled in. */
|
|
341
|
+
encoding: 'raw' | 'unspecified' | EncodedVideoCodec;
|
|
354
342
|
/** Element type, in NumPy notation: "uint8" (video), "int16" (audio). */
|
|
355
343
|
dtype: string;
|
|
356
344
|
/** (height, width, 3) RGB for video, (channels, samples) planar PCM for
|
|
@@ -364,6 +352,9 @@ interface StepFrame {
|
|
|
364
352
|
metadata?: Uint8Array;
|
|
365
353
|
/** When the model says the frame was captured, in microseconds. */
|
|
366
354
|
captureTimeUs?: number;
|
|
355
|
+
/** Set on a frame that travelled in a codec: whether the runtime marked
|
|
356
|
+
* it as a keyframe. */
|
|
357
|
+
keyframe?: boolean;
|
|
367
358
|
}
|
|
368
359
|
/** The frames one output track produced in a step, in production order. */
|
|
369
360
|
interface StepTrack {
|
|
@@ -398,11 +389,37 @@ interface StepResult {
|
|
|
398
389
|
/** Set when the step ran and the model reported a failure. Such a step
|
|
399
390
|
* still resolves; only a refused step rejects. */
|
|
400
391
|
error?: string;
|
|
392
|
+
/** What the model returned for the step besides its media, opaque to the
|
|
393
|
+
* transport: the bytes it returned as they are, or a dict it returned
|
|
394
|
+
* encoded as JSON. Arrives with the step's frames, so a model whose output
|
|
395
|
+
* is data alone needs no output track. Absent when it returned none. */
|
|
396
|
+
data?: Uint8Array;
|
|
401
397
|
}
|
|
402
398
|
|
|
403
|
-
/**
|
|
404
|
-
*
|
|
405
|
-
*
|
|
399
|
+
/**
|
|
400
|
+
* Reference to a file uploaded via `Reactor.uploadFile()`. Pass it back in
|
|
401
|
+
* `sendCommand()`'s `data` wherever the model's schema declares a file: as a
|
|
402
|
+
* top-level value it's extracted and sent as a separate upload reference,
|
|
403
|
+
* and inside an array or object it's serialized as one — see
|
|
404
|
+
* `extractFileRefs()`.
|
|
405
|
+
*/
|
|
406
|
+
declare class FileRef {
|
|
407
|
+
readonly uploadId: string;
|
|
408
|
+
readonly name: string;
|
|
409
|
+
readonly mimeType: string;
|
|
410
|
+
readonly size: number;
|
|
411
|
+
constructor(uploadId: string, name: string, mimeType: string, size: number);
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Structural check for anything shaped like a `FileRef`, for callers who'd
|
|
415
|
+
* rather duck-type than rely on `instanceof` (e.g. across two copies of this
|
|
416
|
+
* package bundled into the same page). `extractFileRefs()` uses this too,
|
|
417
|
+
* for the same reason.
|
|
418
|
+
*/
|
|
419
|
+
declare function isFileRef(value: unknown): value is FileRef;
|
|
420
|
+
|
|
421
|
+
/** What `Reactor.step()` sends: `state`, then `commands`, then any frames
|
|
422
|
+
* still buffered, applied in that order before the model generates. Every part is optional. A file goes in as a `FileRef` from
|
|
406
423
|
* `uploadFile()`, in a command's `data` or a state value. */
|
|
407
424
|
interface StepOptions {
|
|
408
425
|
/** Values for the model's state fields, keyed by field name. Each key runs
|
|
@@ -410,6 +427,47 @@ interface StepOptions {
|
|
|
410
427
|
state?: Record<string, unknown>;
|
|
411
428
|
/** Commands to run after the state, in order. */
|
|
412
429
|
commands?: StepCommand[];
|
|
430
|
+
/** This step's own frames, by input track name: what `pushFrame()` takes,
|
|
431
|
+
* or an `EncodedFrame`. They go after the frames pushed to that track
|
|
432
|
+
* since the previous step, exactly as if pushed just before the step, so
|
|
433
|
+
* each track must be published first. A bad track or frame anywhere fails
|
|
434
|
+
* the step before any of them is pushed, leaving the buffer as it was. */
|
|
435
|
+
media?: Record<string, ReadonlyArray<PushFrame | EncodedFrame>>;
|
|
436
|
+
}
|
|
437
|
+
/** Options of `Reactor.setCodecPreference()`. */
|
|
438
|
+
interface CodecPreferenceOptions {
|
|
439
|
+
/** The one track the list is for, in place of the session's list. */
|
|
440
|
+
track?: string;
|
|
441
|
+
}
|
|
442
|
+
/**
|
|
443
|
+
* An encoded video frame for a step, its bytes sent as they are: one H.264
|
|
444
|
+
* access unit (Annex B), one VP8 or VP9 frame, or one AV1 temporal unit of
|
|
445
|
+
* OBUs. `width` and `height` are the frame it decodes to.
|
|
446
|
+
*
|
|
447
|
+
* The runtime decodes a track's encoded frames as one stream, as they
|
|
448
|
+
* arrive, and the model reads the RGB frames as if they had been pushed raw.
|
|
449
|
+
* So give them in decode order, starting at a keyframe.
|
|
450
|
+
*
|
|
451
|
+
* `data` may be a WebCodecs `EncodedVideoChunk` as a `VideoEncoder` hands it
|
|
452
|
+
* over; configure the encoder with `avc: { format: 'annexb' }`, as its
|
|
453
|
+
* default (`"avc"`) is length-prefixed, not Annex B.
|
|
454
|
+
*/
|
|
455
|
+
interface EncodedFrame {
|
|
456
|
+
/** Tells an encoded frame from a raw one in `step()`'s `media`. */
|
|
457
|
+
codec: EncodedVideoCodec;
|
|
458
|
+
/** One access unit. A chunk's bytes are copied out when it is pushed. */
|
|
459
|
+
data: Uint8Array | ArrayBuffer | EncodedVideoChunk;
|
|
460
|
+
/** Width of the frame it decodes to. */
|
|
461
|
+
width: number;
|
|
462
|
+
/** Height of the frame it decodes to. */
|
|
463
|
+
height: number;
|
|
464
|
+
/** Opaque bytes the decoded frame carries to the model. */
|
|
465
|
+
metadata?: Uint8Array;
|
|
466
|
+
/** When the frame was captured, in microseconds. */
|
|
467
|
+
captureTimeUs?: number;
|
|
468
|
+
/** Whether the frame is one a decoder can start from. Read from an
|
|
469
|
+
* `EncodedVideoChunk`'s `type` when not given. */
|
|
470
|
+
keyframe?: boolean;
|
|
413
471
|
}
|
|
414
472
|
/** Raw pixels for `Reactor.pushFrame()`. `ImageData` is one of these as is. */
|
|
415
473
|
interface RawVideoFrame {
|
|
@@ -435,13 +493,17 @@ type PushFrame = PushVideoFrame | PushAudioFrame;
|
|
|
435
493
|
/** What `Reactor.publishTrack()` resolves with: the published input track. */
|
|
436
494
|
interface PublishedTrack {
|
|
437
495
|
readonly name: string;
|
|
438
|
-
/**
|
|
439
|
-
*
|
|
496
|
+
/** Pushes one input frame on this track to a stepped session, sent at once
|
|
497
|
+
* or buffered for the next `step()` — `reactor.pushFrame(name, frame,
|
|
498
|
+
* options)`. Rejects
|
|
440
499
|
* with an `InvalidStateError` in a streamed session, whose media comes
|
|
441
500
|
* from the `MediaStreamTrack` given to `publishTrack()`, and once the
|
|
442
501
|
* track is unpublished or its connection ends: publish again for a new
|
|
443
502
|
* handle. */
|
|
444
503
|
pushFrame(frame: PushFrame, options?: PushFrameOptions): Promise<void>;
|
|
504
|
+
/** Pushes one encoded video frame on this track, as `pushFrame()` does —
|
|
505
|
+
* `reactor.pushEncodedFrame(name, frame)`. Rejects as `pushFrame()` does. */
|
|
506
|
+
pushEncodedFrame(frame: EncodedFrame): Promise<void>;
|
|
445
507
|
}
|
|
446
508
|
/** Per-frame extras for a pushed video frame. */
|
|
447
509
|
interface PushFrameOptions {
|
|
@@ -826,10 +888,22 @@ declare class Reactor implements Disposable {
|
|
|
826
888
|
* is finalized on "ready" — see `handleStatusChanged()`. */
|
|
827
889
|
private connectStartTime;
|
|
828
890
|
private waitingStartTime;
|
|
829
|
-
/** The tail of every `pushFrame()` so far, settled
|
|
830
|
-
* reach the binding in call order however long each
|
|
831
|
-
* read, and `step()` waits for the ones called
|
|
891
|
+
/** The tail of every `pushFrame()`/`pushEncodedFrame()` so far, settled
|
|
892
|
+
* either way — pushes reach the binding in call order however long each
|
|
893
|
+
* one's pixels take to read, and `step()` waits for the ones called
|
|
894
|
+
* before it. */
|
|
832
895
|
private pendingPushes;
|
|
896
|
+
/** Stepped: every `setCodecPreference()` so far, keyed by track (`null`
|
|
897
|
+
* for the session's list), replayed into a client created later. */
|
|
898
|
+
private readonly codecPreferences;
|
|
899
|
+
/** Stepped: what this browser's WebCodecs can encode and decode, probed
|
|
900
|
+
* once, on the first stepped connect. */
|
|
901
|
+
private localStepCodecs;
|
|
902
|
+
/** Stepped: the input tracks' encoders and the output tracks' decoders of
|
|
903
|
+
* the current connection, dropped as it ends so the next one's streams
|
|
904
|
+
* start at keyframes. */
|
|
905
|
+
private readonly inputEncoders;
|
|
906
|
+
private readonly outputDecoders;
|
|
833
907
|
private readonly emitter;
|
|
834
908
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
835
909
|
* disconnect()/[Symbol.dispose]) against each other. Calling into the
|
|
@@ -1060,10 +1134,10 @@ declare class Reactor implements Disposable {
|
|
|
1060
1134
|
* Runs one step of a `generation: "stepped"` session and resolves with
|
|
1061
1135
|
* what it produced, per output track.
|
|
1062
1136
|
*
|
|
1063
|
-
* The request carries `state
|
|
1064
|
-
*
|
|
1065
|
-
* as the model's `set_<key>` command,
|
|
1066
|
-
* and then generates. They are separate calls, not a transaction: one that
|
|
1137
|
+
* The request carries `state` and `commands`; the frames pushed before it
|
|
1138
|
+
* have already reached the runtime (see `pushFrame()`). The runtime applies
|
|
1139
|
+
* them in that order — each `state` key as the model's `set_<key>` command,
|
|
1140
|
+
* then `commands` — and then generates. They are separate calls, not a transaction: one that
|
|
1067
1141
|
* fails does not undo the ones before it. All of it is optional:
|
|
1068
1142
|
* `step()` with nothing set and nothing pushed runs on the model's current
|
|
1069
1143
|
* state, which commands sent outside a step (`sendCommand()`) also change.
|
|
@@ -1074,7 +1148,9 @@ declare class Reactor implements Disposable {
|
|
|
1074
1148
|
* A step that ran but that the model reported as failed still resolves,
|
|
1075
1149
|
* with `result.error` set. `result.files` lists the files the model kept
|
|
1076
1150
|
* with the step (name, content type, size); their bytes are served over
|
|
1077
|
-
* HTTP from the session's step results, not carried here.
|
|
1151
|
+
* HTTP from the session's step results, not carried here. `result.data`
|
|
1152
|
+
* holds what the model returned besides its media, as opaque bytes (a dict
|
|
1153
|
+
* arrives encoded as JSON); it is absent when the model returned none.
|
|
1078
1154
|
*
|
|
1079
1155
|
* Rejects with a `StepRefusedError` when the runtime or the model turns
|
|
1080
1156
|
* the step down (the runtime checks a whole request first, so nothing it
|
|
@@ -1088,10 +1164,34 @@ declare class Reactor implements Disposable {
|
|
|
1088
1164
|
* message, `MessageTooLargeError` when the request itself is too large
|
|
1089
1165
|
* (nothing was sent), `InvalidStateError` outside a ready stepped
|
|
1090
1166
|
* session, or whatever else failed the step (`DisconnectedError`,
|
|
1091
|
-
* `RequestTimeoutError`, ...). The frames
|
|
1167
|
+
* `RequestTimeoutError`, ...). The frames buffered for a step belong to it:
|
|
1092
1168
|
* when its request cannot be sent, they are dropped with it rather than
|
|
1093
1169
|
* kept for the next step.
|
|
1094
1170
|
*
|
|
1171
|
+
* `media` gives the step frames of its own, by input track name, pushed
|
|
1172
|
+
* after the ones already pushed to that track: what `pushFrame()` takes,
|
|
1173
|
+
* or an `EncodedFrame` — H.264 the runtime decodes before the step runs,
|
|
1174
|
+
* so the model reads RGB frames as for a raw push. They are pushed only
|
|
1175
|
+
* once `state` and `commands` are well formed and the session is ready, so
|
|
1176
|
+
* a step refused for those leaves the frames buffered as they were. They go
|
|
1177
|
+
* all or nothing: a bad track or frame anywhere in `media` rejects the step
|
|
1178
|
+
* before any of it is pushed or sent, leaving the buffer as it was. A
|
|
1179
|
+
* session that leaves 'ready' once the media is in has the step refused and
|
|
1180
|
+
* keeps the frames for the next one, as pushed frames are.
|
|
1181
|
+
* A track's encoded frames go in decode order, starting at a keyframe; a
|
|
1182
|
+
* step whose buffered frames do not decode rejects with a
|
|
1183
|
+
* `StepRefusedError` (see `pushEncodedFrame()` for frames already sent).
|
|
1184
|
+
*
|
|
1185
|
+
* Video step frames travel encoded both ways when this browser and the
|
|
1186
|
+
* runtime share a codec (H.264 first by default; see
|
|
1187
|
+
* `setCodecPreference()`), on WebCodecs: the frames pushed on an input
|
|
1188
|
+
* track are encoded, and an output track's frames are decoded, so a result
|
|
1189
|
+
* still holds RGB frames whose `encoding` names the codec they travelled
|
|
1190
|
+
* in. Each track is one stream for the connection. When one of its
|
|
1191
|
+
* decoders lost its stream, the step rejects with a `DecodeError`, the SDK
|
|
1192
|
+
* pauses and resumes that track so the runtime starts it again at a
|
|
1193
|
+
* keyframe, and the track's frames fail until that keyframe arrives.
|
|
1194
|
+
*
|
|
1095
1195
|
* ```ts
|
|
1096
1196
|
* const camera = await reactor.publishTrack('camera');
|
|
1097
1197
|
* await camera.pushFrame(imageData);
|
|
@@ -1103,11 +1203,15 @@ declare class Reactor implements Disposable {
|
|
|
1103
1203
|
*/
|
|
1104
1204
|
step(options?: StepOptions): Promise<StepResult>;
|
|
1105
1205
|
/**
|
|
1106
|
-
*
|
|
1107
|
-
*
|
|
1108
|
-
*
|
|
1109
|
-
*
|
|
1110
|
-
*
|
|
1206
|
+
* Pushes one input frame on the sendonly track `trackName` of a stepped
|
|
1207
|
+
* session. Once the session is ready, against a runtime that takes pushed
|
|
1208
|
+
* frames, the frame is sent to the runtime at once over the data channel,
|
|
1209
|
+
* ahead of any `step()` after it; one the channel cannot take is dropped
|
|
1210
|
+
* and raised as an error event. A frame pushed before the session is ready,
|
|
1211
|
+
* or to an older runtime, is buffered and sent with the next push or
|
|
1212
|
+
* `step()`, per track in push order. The `PublishedTrack` that
|
|
1213
|
+
* `publishTrack()` resolves with does the same through
|
|
1214
|
+
* `track.pushFrame(frame)`.
|
|
1111
1215
|
*
|
|
1112
1216
|
* Video: a `VideoFrame` (cloned when called, so the caller may close it as
|
|
1113
1217
|
* soon as this returns; it stays the caller's), an
|
|
@@ -1118,9 +1222,15 @@ declare class Reactor implements Disposable {
|
|
|
1118
1222
|
* Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
|
|
1119
1223
|
* PCM at the rate the model's track declares.
|
|
1120
1224
|
*
|
|
1225
|
+
* A video frame travels encoded when the connection negotiated a codec for
|
|
1226
|
+
* the track (see `setCodecPreference()`): it is encoded here, with
|
|
1227
|
+
* WebCodecs, as the next frame of the track's stream, and the model still
|
|
1228
|
+
* reads RGB. A track's first frame on a connection is a keyframe, and so
|
|
1229
|
+
* is the next one after the runtime could not decode its frames.
|
|
1230
|
+
*
|
|
1121
1231
|
* Pushes keep their call order, and a `step()` waits for every push called
|
|
1122
1232
|
* before it. A refused push rejects: await it, or `.catch()` it in a frame
|
|
1123
|
-
* loop, or every bad frame is an unhandled rejection. What
|
|
1233
|
+
* loop, or every bad frame is an unhandled rejection. What is buffered for
|
|
1124
1234
|
* the next step is at most one message's worth of frames: pushing faster
|
|
1125
1235
|
* than steps run drops the oldest frames, across tracks. Rejects with
|
|
1126
1236
|
* an `InvalidStateError` outside a stepped session, for a track not
|
|
@@ -1132,8 +1242,105 @@ declare class Reactor implements Disposable {
|
|
|
1132
1242
|
/** `pushFrame()`, for the track's current publication or, from a
|
|
1133
1243
|
* `PublishedTrack`, only for the `publication` it came from. */
|
|
1134
1244
|
private pushTo;
|
|
1135
|
-
/**
|
|
1136
|
-
*
|
|
1245
|
+
/**
|
|
1246
|
+
* Pushes one encoded video frame, sent or buffered as `pushFrame()` does,
|
|
1247
|
+
* on the sendonly video track `trackName`, its bytes as they are: one
|
|
1248
|
+
* H.264 access unit (Annex B), one VP8 or VP9 frame, or one AV1 temporal
|
|
1249
|
+
* unit, with the `width` and `height` it decodes to. The `PublishedTrack`
|
|
1250
|
+
* that `publishTrack()` resolves with does the same through
|
|
1251
|
+
* `track.pushEncodedFrame(frame)`.
|
|
1252
|
+
*
|
|
1253
|
+
* The runtime decodes a track's encoded frames as one stream, as they
|
|
1254
|
+
* arrive, and the model reads the RGB frames as if they had been pushed
|
|
1255
|
+
* raw. So push them in decode order, starting at a keyframe (`keyframe`, or
|
|
1256
|
+
* a chunk's `type`). A pushed frame that does not decode raises an
|
|
1257
|
+
* `input_decode_failed` error event naming the track, whose next frame
|
|
1258
|
+
* must then be a keyframe; a step whose buffered frames do not decode
|
|
1259
|
+
* rejects with a `StepRefusedError`, as does a runtime that predates
|
|
1260
|
+
* encoded frames. Encoded and raw pushes keep their call order, and a
|
|
1261
|
+
* `step()` waits for every push called before it.
|
|
1262
|
+
*
|
|
1263
|
+
* ```ts
|
|
1264
|
+
* const encoder = new VideoEncoder({
|
|
1265
|
+
* output: (chunk) => void camera.pushEncodedFrame({ codec: 'h264', data: chunk, width, height }),
|
|
1266
|
+
* error: console.error,
|
|
1267
|
+
* });
|
|
1268
|
+
* encoder.configure({ codec: 'avc1.42001f', width, height, avc: { format: 'annexb' } });
|
|
1269
|
+
* ```
|
|
1270
|
+
*
|
|
1271
|
+
* Rejects with an `InvalidStateError` outside a stepped session, for a
|
|
1272
|
+
* track not published on this connection, for an audio or recvonly track,
|
|
1273
|
+
* for empty data, a zero dimension or an unknown codec.
|
|
1274
|
+
*/
|
|
1275
|
+
pushEncodedFrame(trackName: string, frame: EncodedFrame): Promise<void>;
|
|
1276
|
+
/** `pushEncodedFrame()`, for the track's current publication or, from a
|
|
1277
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
1278
|
+
private pushEncodedTo;
|
|
1279
|
+
/** Refuses a push whose connection ended since it was called, or whose
|
|
1280
|
+
* track is not (or no longer) published on it. */
|
|
1281
|
+
private checkPushTarget;
|
|
1282
|
+
/** Hands `pixels` to the binding for `track`: encoded first, on the
|
|
1283
|
+
* track's encoder, when the connection negotiated a codec for the track's
|
|
1284
|
+
* step frames and this browser can encode; raw otherwise, and when the
|
|
1285
|
+
* encoder fails (its next frame is then a keyframe). Checked by the
|
|
1286
|
+
* binding before anything is encoded, so a bad frame is refused either
|
|
1287
|
+
* way. `stillTarget` throws when the push's connection or publication is
|
|
1288
|
+
* gone: asked again once the encode is awaited, since a reconnect can
|
|
1289
|
+
* start meanwhile, and neither the encoded frame nor its raw fallback may
|
|
1290
|
+
* then reach the next session. */
|
|
1291
|
+
private pushPixels;
|
|
1292
|
+
/** Gives the binding the codecs this browser can carry step frames in,
|
|
1293
|
+
* probed once, so the next offer can negotiate them. A streamed session
|
|
1294
|
+
* carries no step frames. */
|
|
1295
|
+
/** Restart each output track whose decoder lost its stream: a pause and
|
|
1296
|
+
* a resume of that track, so the runtime starts it again at a keyframe. A
|
|
1297
|
+
* restart that cannot be sent is left to the next connection, whose
|
|
1298
|
+
* streams start at keyframes anyway. */
|
|
1299
|
+
private restartLostTracks;
|
|
1300
|
+
private offerStepCodecs;
|
|
1301
|
+
/** Pushes `step()`'s `media`, track by track and in order, each frame as
|
|
1302
|
+
* `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
|
|
1303
|
+
* checks the step's `state` and `commands`, then every frame, before the
|
|
1304
|
+
* first is pushed, so a refused step, or a bad track or frame anywhere,
|
|
1305
|
+
* leaves the buffer as it was. */
|
|
1306
|
+
private pushStepMedia;
|
|
1307
|
+
/**
|
|
1308
|
+
* Chooses the codecs a stepped session carries its video step frames in,
|
|
1309
|
+
* most preferred first, from `"h264"`, `"vp9"`, `"av1"` and `"vp8"`.
|
|
1310
|
+
* `null` goes back to the default, H.264 first. An empty list for the
|
|
1311
|
+
* session turns encoding off, so every frame travels raw:
|
|
1312
|
+
*
|
|
1313
|
+
* ```ts
|
|
1314
|
+
* reactor.setCodecPreference(['vp9', 'h264']); // VP9 when both ends have it
|
|
1315
|
+
* reactor.setCodecPreference([]); // raw frames
|
|
1316
|
+
* reactor.setCodecPreference(['av1'], { track: 'main_video' });
|
|
1317
|
+
* ```
|
|
1318
|
+
*
|
|
1319
|
+
* With `track`, the list applies to that track only, in place of the
|
|
1320
|
+
* session's; an empty one there goes back to the session's, since one
|
|
1321
|
+
* track cannot turn encoding off alone. The list is an order, not a
|
|
1322
|
+
* limit: the codecs it leaves out follow it, so a runtime without the
|
|
1323
|
+
* preferred ones takes another. Of those, the ones this browser's
|
|
1324
|
+
* WebCodecs can encode (input tracks) or decode (output tracks) are
|
|
1325
|
+
* offered, and the runtime takes per track the first it shares; a track
|
|
1326
|
+
* they share no codec for stays raw.
|
|
1327
|
+
*
|
|
1328
|
+
* Whatever the codec, `step()` results and the frames `pushFrame()` takes
|
|
1329
|
+
* stay RGB: the encoding is between this client and the runtime. Callable
|
|
1330
|
+
* before `connect()` or during a session; it takes effect on the next
|
|
1331
|
+
* connection or reconnect, and the live connection keeps what it
|
|
1332
|
+
* negotiated (see `getStepFrameEncodings()`). A streamed session ignores
|
|
1333
|
+
* it. Throws an `InvalidStateError` on a name that is not a codec.
|
|
1334
|
+
*/
|
|
1335
|
+
setCodecPreference(codecs: EncodedVideoCodec[] | null, options?: CodecPreferenceOptions): void;
|
|
1336
|
+
/** How each track's step frames travel on this connection, as its offer
|
|
1337
|
+
* and answer negotiated them: `"raw"`, or the codec of an encoded video
|
|
1338
|
+
* track. Empty outside a connected stepped session; a track left out is
|
|
1339
|
+
* raw. */
|
|
1340
|
+
getStepFrameEncodings(): Record<string, StepFrameEncoding>;
|
|
1341
|
+
/** Drops every frame still buffered for the next step, pushes called before
|
|
1342
|
+
* it included: it waits its turn behind them, as a `step()` does. Frames
|
|
1343
|
+
* already sent to the runtime are not recalled. */
|
|
1137
1344
|
clearStepInputs(): Promise<void>;
|
|
1138
1345
|
/**
|
|
1139
1346
|
* Requests a clip covering the last `durationSeconds` of the session.
|
|
@@ -1270,6 +1477,9 @@ declare class Reactor implements Disposable {
|
|
|
1270
1477
|
* leave `getCapabilities()` returning the previous session's stale
|
|
1271
1478
|
* tracks/commands until a new `capabilitiesReceived` lands. */
|
|
1272
1479
|
private resetConnectionState;
|
|
1480
|
+
/** Drops every step frame encoder and decoder: each track's stream lived
|
|
1481
|
+
* with the connection that ended, and the next one's start at keyframes. */
|
|
1482
|
+
private resetStepStreams;
|
|
1273
1483
|
/** Wraps `cause` and records it as the most recent failure, for a call
|
|
1274
1484
|
* that's about to throw rather than emit — see `getLastError()`'s doc
|
|
1275
1485
|
* comment. */
|
|
@@ -1642,4 +1852,4 @@ interface ClipDownloadButtonProps {
|
|
|
1642
1852
|
}
|
|
1643
1853
|
declare function ClipDownloadButton({ clip, getJwt, filename, children, className, style, disabled, onSuccess, onError, }: ClipDownloadButtonProps): react.JSX.Element;
|
|
1644
1854
|
|
|
1645
|
-
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DataChannelChunkingRequiredError, DecodeError, DisconnectedError, type DownloadClipOptions, type FetchPlaylistOptions, FileRef, type GenerationMode, GenerationModeMismatchError, InvalidStateError, type JwtRequestContext, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, type PublishedTrack, type PushAudioFrame, type PushFrame, type PushFrameOptions, type PushVideoFrame, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type StepCommand, StepCommandFailedError, type StepFile, type StepFrame, StepNotReadyError, type StepOptions, StepRefusedError, type StepResult, StepResultTooLargeError, StepSessionEndedError, type StepTrack, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, UnsupportedFrameEncodingError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, stepFrameToImageData, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|
|
1855
|
+
export { AbortedError, BadRequestError, type Capabilities, type Clip, ClipDownloadButton, type ClipDownloadButtonProps, type ClipDownloadState, type ClipKind, ClipPlayer, type ClipPlayerProps, type CodecPreferenceOptions, type CommandCapability, ConflictError, type ConnectOptions, type ConnectionStats, type ConnectionTimings, DEFAULT_PLAYLIST_POLL_SLACK_MS, DataChannelChunkingRequiredError, DecodeError, DisconnectedError, type DownloadClipOptions, type EncodedFrame, type EncodedVideoCodec, type FetchPlaylistOptions, FileRef, type GenerationMode, GenerationModeMismatchError, InvalidStateError, type JwtRequestContext, type JwtResolver, type JwtSource, type MessageScope, MessageTooLargeError, type ModelSchema, type ModelSchemaOperation, type ModelSchemaPathItem, type ModerationAction, type ModerationEvent, NetworkError, NotFoundError, type PublishedTrack, type PushAudioFrame, type PushFrame, type PushFrameOptions, type PushVideoFrame, RateLimitedError, Reactor, type ReactorActions, type ReactorConnectOptions, ReactorError, type ReactorEventMap, type ReactorEventName, type ReactorMessage, type ReactorOptions, ReactorProvider, type ReactorProviderProps, type ReactorState, type ReactorStatus, type ReactorStore, ReactorView, type ReactorViewProps, RecorderDisabledError, RecordingError, RequestTimeoutError, ServerError, type SessionResponse, SessionTerminalError, type StepCommand, StepCommandFailedError, type StepFile, type StepFrame, type StepFrameEncoding, StepNotReadyError, type StepOptions, StepRefusedError, type StepResult, StepResultTooLargeError, StepSessionEndedError, type StepTrack, type TrackCapability, type TrackDirection, type TrackKind, type TrackMappingEntry, TransportError, UnauthorizedError, UnsupportedFrameEncodingError, type UseClipDownloadOptions, type UseClipDownloadResult, VersionMismatchError, WebcamStream, type WebcamStreamProps, createPlayableManifestUrl, downloadClipAsFile, fetchPlaylist, isFileRef, normalizeJwtSource, parsePlaylist, stepFrameToImageData, useClipDownload, useReactor, useReactorInternalMessage, useReactorMessage, useStats };
|