@reactor-team/js-sdk 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -0
- package/dist/index.cjs +752 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +399 -5
- package/dist/index.d.ts +399 -5
- package/dist/index.js +743 -9
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +38 -2
- package/dist/wasm/reactor_wasm.d.ts +229 -2
- package/dist/wasm/reactor_wasm.js +348 -8
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +10 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -37,9 +37,11 @@ __export(index_exports, {
|
|
|
37
37
|
ClipPlayer: () => ClipPlayer,
|
|
38
38
|
ConflictError: () => ConflictError,
|
|
39
39
|
DEFAULT_PLAYLIST_POLL_SLACK_MS: () => DEFAULT_PLAYLIST_POLL_SLACK_MS,
|
|
40
|
+
DataChannelChunkingRequiredError: () => DataChannelChunkingRequiredError,
|
|
40
41
|
DecodeError: () => DecodeError,
|
|
41
42
|
DisconnectedError: () => DisconnectedError,
|
|
42
43
|
FileRef: () => FileRef,
|
|
44
|
+
GenerationModeMismatchError: () => GenerationModeMismatchError,
|
|
43
45
|
InvalidStateError: () => InvalidStateError,
|
|
44
46
|
MessageTooLargeError: () => MessageTooLargeError,
|
|
45
47
|
NetworkError: () => NetworkError,
|
|
@@ -54,8 +56,14 @@ __export(index_exports, {
|
|
|
54
56
|
RequestTimeoutError: () => RequestTimeoutError,
|
|
55
57
|
ServerError: () => ServerError,
|
|
56
58
|
SessionTerminalError: () => SessionTerminalError,
|
|
59
|
+
StepCommandFailedError: () => StepCommandFailedError,
|
|
60
|
+
StepNotReadyError: () => StepNotReadyError,
|
|
61
|
+
StepRefusedError: () => StepRefusedError,
|
|
62
|
+
StepResultTooLargeError: () => StepResultTooLargeError,
|
|
63
|
+
StepSessionEndedError: () => StepSessionEndedError,
|
|
57
64
|
TransportError: () => TransportError,
|
|
58
65
|
UnauthorizedError: () => UnauthorizedError,
|
|
66
|
+
UnsupportedFrameEncodingError: () => UnsupportedFrameEncodingError,
|
|
59
67
|
VersionMismatchError: () => VersionMismatchError,
|
|
60
68
|
WebcamStream: () => WebcamStream,
|
|
61
69
|
createPlayableManifestUrl: () => createPlayableManifestUrl,
|
|
@@ -64,6 +72,7 @@ __export(index_exports, {
|
|
|
64
72
|
isFileRef: () => isFileRef,
|
|
65
73
|
normalizeJwtSource: () => normalizeJwtSource,
|
|
66
74
|
parsePlaylist: () => parsePlaylist,
|
|
75
|
+
stepFrameToImageData: () => stepFrameToImageData,
|
|
67
76
|
useClipDownload: () => useClipDownload,
|
|
68
77
|
useReactor: () => useReactor,
|
|
69
78
|
useReactorInternalMessage: () => useReactorInternalMessage,
|
|
@@ -117,6 +126,14 @@ var ReactorError = class extends Error {
|
|
|
117
126
|
operation;
|
|
118
127
|
/** Backoff hint the platform sent. */
|
|
119
128
|
retry_after_ms;
|
|
129
|
+
/** For a refused step or a step whose call failed, the part of the step
|
|
130
|
+
* request at fault: `"state.<key>"`, `"commands[<i>]"` or
|
|
131
|
+
* `"media_input[<i>]"`. */
|
|
132
|
+
entry;
|
|
133
|
+
/** For a step whose call failed, the stable code its handler raised (its
|
|
134
|
+
* `CommandError` code, or `internal_error`), to branch on instead of the
|
|
135
|
+
* message. */
|
|
136
|
+
command_code;
|
|
120
137
|
/** When `reactor-core` reported this failure. For a purely local one (no
|
|
121
138
|
* payload at all — see {@link ReactorErrorOptions}), this is synthesized
|
|
122
139
|
* as the construction time instead, since there's nothing to report —
|
|
@@ -137,6 +154,8 @@ var ReactorError = class extends Error {
|
|
|
137
154
|
this.operation = options.operation;
|
|
138
155
|
this.retry_after_ms = options.retry_after_ms;
|
|
139
156
|
this.retryAfter = options.retry_after_ms;
|
|
157
|
+
this.entry = options.entry;
|
|
158
|
+
this.command_code = options.command_code;
|
|
140
159
|
this.timestamp_ms = options.timestamp_ms ?? Date.now();
|
|
141
160
|
this.timestamp = this.timestamp_ms;
|
|
142
161
|
}
|
|
@@ -192,6 +211,30 @@ var AbortedError = class extends ReactorError {
|
|
|
192
211
|
var RecorderDisabledError = class extends ReactorError {
|
|
193
212
|
static code = "RECORDER_DISABLED";
|
|
194
213
|
};
|
|
214
|
+
var GenerationModeMismatchError = class extends ReactorError {
|
|
215
|
+
static code = "GENERATION_MODE_MISMATCH";
|
|
216
|
+
};
|
|
217
|
+
var DataChannelChunkingRequiredError = class extends ReactorError {
|
|
218
|
+
static code = "DATA_CHANNEL_CHUNKING_REQUIRED";
|
|
219
|
+
};
|
|
220
|
+
var StepRefusedError = class extends ReactorError {
|
|
221
|
+
static code = "step_refused";
|
|
222
|
+
};
|
|
223
|
+
var StepNotReadyError = class extends ReactorError {
|
|
224
|
+
static code = "step_not_ready";
|
|
225
|
+
};
|
|
226
|
+
var StepResultTooLargeError = class extends ReactorError {
|
|
227
|
+
static code = "step_result_too_large";
|
|
228
|
+
};
|
|
229
|
+
var StepCommandFailedError = class extends ReactorError {
|
|
230
|
+
static code = "step_command_failed";
|
|
231
|
+
};
|
|
232
|
+
var StepSessionEndedError = class extends ReactorError {
|
|
233
|
+
static code = "step_session_ended";
|
|
234
|
+
};
|
|
235
|
+
var UnsupportedFrameEncodingError = class extends ReactorError {
|
|
236
|
+
static code = "UNSUPPORTED_FRAME_ENCODING";
|
|
237
|
+
};
|
|
195
238
|
var ERROR_CLASSES = [
|
|
196
239
|
NetworkError,
|
|
197
240
|
UnauthorizedError,
|
|
@@ -209,7 +252,15 @@ var ERROR_CLASSES = [
|
|
|
209
252
|
DisconnectedError,
|
|
210
253
|
RequestTimeoutError,
|
|
211
254
|
AbortedError,
|
|
212
|
-
RecorderDisabledError
|
|
255
|
+
RecorderDisabledError,
|
|
256
|
+
GenerationModeMismatchError,
|
|
257
|
+
DataChannelChunkingRequiredError,
|
|
258
|
+
StepRefusedError,
|
|
259
|
+
StepNotReadyError,
|
|
260
|
+
StepResultTooLargeError,
|
|
261
|
+
StepCommandFailedError,
|
|
262
|
+
StepSessionEndedError,
|
|
263
|
+
UnsupportedFrameEncodingError
|
|
213
264
|
];
|
|
214
265
|
var BY_CODE = new Map(
|
|
215
266
|
ERROR_CLASSES.map((errorClass) => [errorClass.code, errorClass])
|
|
@@ -231,6 +282,8 @@ function toReactorError(cause) {
|
|
|
231
282
|
operation: payload.operation,
|
|
232
283
|
retry_after_ms: payload.retry_after_ms,
|
|
233
284
|
timestamp_ms: payload.timestamp_ms,
|
|
285
|
+
entry: payload.entry,
|
|
286
|
+
command_code: payload.command_code,
|
|
234
287
|
// A thrown JS error (a failed wasm load, say) keeps its stack and its own
|
|
235
288
|
// `cause` reachable; reactor-core's plain payload objects have neither.
|
|
236
289
|
cause: cause instanceof Error ? cause : void 0
|
|
@@ -277,6 +330,52 @@ var Emitter = class {
|
|
|
277
330
|
}
|
|
278
331
|
};
|
|
279
332
|
|
|
333
|
+
// src/internal/encoded-frames.ts
|
|
334
|
+
var CODECS = /* @__PURE__ */ new Set(["h264"]);
|
|
335
|
+
function isEncodedFrame(frame) {
|
|
336
|
+
return typeof frame === "object" && frame !== null && "codec" in frame;
|
|
337
|
+
}
|
|
338
|
+
function isChunk(data) {
|
|
339
|
+
return !(data instanceof Uint8Array) && !(data instanceof ArrayBuffer) && typeof data.copyTo === "function";
|
|
340
|
+
}
|
|
341
|
+
function toEncodedBytes(frame, operation, trackName) {
|
|
342
|
+
const refuse = (why) => new InvalidStateError(`${operation}('${trackName}'): ${why}`, { operation });
|
|
343
|
+
if (typeof frame !== "object" || frame === null) {
|
|
344
|
+
throw refuse("expected an EncodedFrame, { codec, data, width, height }");
|
|
345
|
+
}
|
|
346
|
+
const { codec, data, width, height, metadata, captureTimeUs } = frame;
|
|
347
|
+
if (!CODECS.has(codec)) {
|
|
348
|
+
throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264"`);
|
|
349
|
+
}
|
|
350
|
+
let bytes;
|
|
351
|
+
if (data instanceof Uint8Array) {
|
|
352
|
+
bytes = data;
|
|
353
|
+
} else if (data instanceof ArrayBuffer) {
|
|
354
|
+
bytes = new Uint8Array(data);
|
|
355
|
+
} else if (data && isChunk(data)) {
|
|
356
|
+
bytes = new Uint8Array(data.byteLength);
|
|
357
|
+
data.copyTo(bytes);
|
|
358
|
+
} else {
|
|
359
|
+
throw refuse("data must be a Uint8Array, an ArrayBuffer or an EncodedVideoChunk");
|
|
360
|
+
}
|
|
361
|
+
if (bytes.byteLength === 0) {
|
|
362
|
+
throw refuse("an encoded frame needs data");
|
|
363
|
+
}
|
|
364
|
+
if (!isDimension(width) || !isDimension(height)) {
|
|
365
|
+
throw refuse(`width and height are the decoded frame's, got ${String(width)}x${String(height)}`);
|
|
366
|
+
}
|
|
367
|
+
if (metadata != null && !(metadata instanceof Uint8Array)) {
|
|
368
|
+
throw refuse("metadata must be a Uint8Array");
|
|
369
|
+
}
|
|
370
|
+
if (captureTimeUs != null && !Number.isFinite(captureTimeUs)) {
|
|
371
|
+
throw refuse("captureTimeUs must be a finite number");
|
|
372
|
+
}
|
|
373
|
+
return { codec, data: bytes, width, height, metadata: metadata ?? null, captureTimeUs: captureTimeUs ?? null };
|
|
374
|
+
}
|
|
375
|
+
function isDimension(value) {
|
|
376
|
+
return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 4294967295;
|
|
377
|
+
}
|
|
378
|
+
|
|
280
379
|
// src/file-ref.ts
|
|
281
380
|
var FileRef = class {
|
|
282
381
|
constructor(uploadId, name, mimeType, size) {
|
|
@@ -368,6 +467,112 @@ function isSerializedAsObject(value) {
|
|
|
368
467
|
return typeof value === "object" && value !== null && !ArrayBuffer.isView(value);
|
|
369
468
|
}
|
|
370
469
|
|
|
470
|
+
// src/internal/push-frames.ts
|
|
471
|
+
function isAudioFrame(frame) {
|
|
472
|
+
return "samples" in frame;
|
|
473
|
+
}
|
|
474
|
+
function isVideoFrame(frame) {
|
|
475
|
+
return typeof frame.copyTo === "function" && "displayWidth" in frame;
|
|
476
|
+
}
|
|
477
|
+
function holdFrame(frame) {
|
|
478
|
+
if (!isAudioFrame(frame) && isVideoFrame(frame)) {
|
|
479
|
+
const clone = frame.clone();
|
|
480
|
+
return { frame: clone, release: () => clone.close() };
|
|
481
|
+
}
|
|
482
|
+
return { frame, release: () => {
|
|
483
|
+
} };
|
|
484
|
+
}
|
|
485
|
+
async function toPixels(frame) {
|
|
486
|
+
if (isVideoFrame(frame)) {
|
|
487
|
+
return videoFramePixels(frame);
|
|
488
|
+
}
|
|
489
|
+
return rawPixels(frame);
|
|
490
|
+
}
|
|
491
|
+
function rawPixels(frame) {
|
|
492
|
+
const { data, width, height } = frame;
|
|
493
|
+
const bytes = data instanceof Uint8Array ? data : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
494
|
+
return { data: bytes, width, height, format: "format" in frame && frame.format || "rgba" };
|
|
495
|
+
}
|
|
496
|
+
var NATIVE_FORMATS = {
|
|
497
|
+
RGBA: "rgba",
|
|
498
|
+
RGBX: "rgba",
|
|
499
|
+
BGRA: "bgra",
|
|
500
|
+
BGRX: "bgra"
|
|
501
|
+
};
|
|
502
|
+
function orientationOf(frame) {
|
|
503
|
+
const { rotation, flip } = frame;
|
|
504
|
+
return { rotation: ((rotation ?? 0) % 360 + 360) % 360, flip: flip ?? false };
|
|
505
|
+
}
|
|
506
|
+
async function videoFramePixels(frame) {
|
|
507
|
+
const rect = frame.visibleRect;
|
|
508
|
+
const width = rect?.width ?? frame.codedWidth;
|
|
509
|
+
const height = rect?.height ?? frame.codedHeight;
|
|
510
|
+
const native = frame.format ? NATIVE_FORMATS[frame.format] : void 0;
|
|
511
|
+
const orientation = orientationOf(frame);
|
|
512
|
+
if (native) {
|
|
513
|
+
const data = new Uint8Array(width * height * 4);
|
|
514
|
+
await frame.copyTo(data, { layout: [{ offset: 0, stride: width * 4 }] });
|
|
515
|
+
return orient({ data, width, height, format: native }, orientation);
|
|
516
|
+
}
|
|
517
|
+
try {
|
|
518
|
+
const data = new Uint8Array(width * height * 4);
|
|
519
|
+
const options = { format: "RGBA", layout: [{ offset: 0, stride: width * 4 }] };
|
|
520
|
+
await frame.copyTo(data, options);
|
|
521
|
+
return orient({ data, width, height, format: "rgba" }, orientation);
|
|
522
|
+
} catch {
|
|
523
|
+
return canvasPixels(frame, width, height, orientation);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
function orient(pixels, { rotation, flip }) {
|
|
527
|
+
if (rotation % 90 !== 0 || rotation === 0 && !flip) {
|
|
528
|
+
return pixels;
|
|
529
|
+
}
|
|
530
|
+
const { width: w, height: h } = pixels;
|
|
531
|
+
const turned = rotation === 90 || rotation === 270;
|
|
532
|
+
const outWidth = turned ? h : w;
|
|
533
|
+
const outHeight = turned ? w : h;
|
|
534
|
+
const bytes = pixels.data.byteOffset % 4 === 0 ? pixels.data : pixels.data.slice();
|
|
535
|
+
const source = new Uint32Array(bytes.buffer, bytes.byteOffset, w * h);
|
|
536
|
+
const data = new Uint8Array(outWidth * outHeight * 4);
|
|
537
|
+
const target = new Uint32Array(data.buffer);
|
|
538
|
+
for (let y = 0; y < h; y++) {
|
|
539
|
+
for (let x = 0; x < w; x++) {
|
|
540
|
+
let tx;
|
|
541
|
+
let ty;
|
|
542
|
+
if (rotation === 90) {
|
|
543
|
+
[tx, ty] = [h - 1 - y, x];
|
|
544
|
+
} else if (rotation === 180) {
|
|
545
|
+
[tx, ty] = [w - 1 - x, h - 1 - y];
|
|
546
|
+
} else if (rotation === 270) {
|
|
547
|
+
[tx, ty] = [y, w - 1 - x];
|
|
548
|
+
} else {
|
|
549
|
+
[tx, ty] = [x, y];
|
|
550
|
+
}
|
|
551
|
+
if (flip) {
|
|
552
|
+
tx = outWidth - 1 - tx;
|
|
553
|
+
}
|
|
554
|
+
target[ty * outWidth + tx] = source[y * w + x];
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
return { data, width: outWidth, height: outHeight, format: pixels.format };
|
|
558
|
+
}
|
|
559
|
+
function canvasPixels(frame, visibleWidth, visibleHeight, { rotation }) {
|
|
560
|
+
const turned = rotation === 90 || rotation === 270;
|
|
561
|
+
const width = turned ? visibleHeight : visibleWidth;
|
|
562
|
+
const height = turned ? visibleWidth : visibleHeight;
|
|
563
|
+
if (typeof OffscreenCanvas === "undefined") {
|
|
564
|
+
throw new Error(
|
|
565
|
+
`pushFrame: cannot read a ${frame.format ?? "formatless"} VideoFrame here \u2014 push its pixels as an ImageData or a Uint8Array instead.`
|
|
566
|
+
);
|
|
567
|
+
}
|
|
568
|
+
const context = new OffscreenCanvas(width, height).getContext("2d");
|
|
569
|
+
if (!context) {
|
|
570
|
+
throw new Error("pushFrame: no 2D canvas context to read the VideoFrame through.");
|
|
571
|
+
}
|
|
572
|
+
context.drawImage(frame, 0, 0, width, height);
|
|
573
|
+
return rawPixels(context.getImageData(0, 0, width, height));
|
|
574
|
+
}
|
|
575
|
+
|
|
371
576
|
// src/internal/recording.ts
|
|
372
577
|
function toPublicClip(clip) {
|
|
373
578
|
return {
|
|
@@ -381,6 +586,67 @@ function toPublicClip(clip) {
|
|
|
381
586
|
};
|
|
382
587
|
}
|
|
383
588
|
|
|
589
|
+
// src/internal/step-media.ts
|
|
590
|
+
function holdStepMedia(media) {
|
|
591
|
+
if (typeof media !== "object" || media === null || Array.isArray(media)) {
|
|
592
|
+
throw new InvalidStateError("step(): media must map track names to arrays of frames", {
|
|
593
|
+
operation: "step"
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
const held = [];
|
|
597
|
+
try {
|
|
598
|
+
for (const [trackName, frames] of Object.entries(media)) {
|
|
599
|
+
if (!Array.isArray(frames)) {
|
|
600
|
+
throw new InvalidStateError(`step(): media['${trackName}'] must be an array of frames`, {
|
|
601
|
+
operation: "step"
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
for (const frame of frames) {
|
|
605
|
+
held.push(holdStepFrame(trackName, frame));
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
} catch (cause) {
|
|
609
|
+
for (const frame of held) {
|
|
610
|
+
frame.release();
|
|
611
|
+
}
|
|
612
|
+
throw cause;
|
|
613
|
+
}
|
|
614
|
+
return held;
|
|
615
|
+
}
|
|
616
|
+
function holdStepFrame(track, frame) {
|
|
617
|
+
if (isEncodedFrame(frame)) {
|
|
618
|
+
const operation = "pushEncodedFrame";
|
|
619
|
+
return { track, kind: "encoded", operation, encoded: toEncodedBytes(frame, operation, track), release: () => {
|
|
620
|
+
} };
|
|
621
|
+
}
|
|
622
|
+
if (typeof frame !== "object" || frame === null) {
|
|
623
|
+
throw new InvalidStateError(`pushFrame('${track}'): expected a frame`, { operation: "pushFrame" });
|
|
624
|
+
}
|
|
625
|
+
const held = holdFrame(frame);
|
|
626
|
+
return { track, kind: "raw", operation: "pushFrame", frame: held.frame, release: held.release };
|
|
627
|
+
}
|
|
628
|
+
async function readStepFrame(frame) {
|
|
629
|
+
const { track } = frame;
|
|
630
|
+
if (frame.kind === "encoded") {
|
|
631
|
+
return { track, kind: "encoded", operation: frame.operation, encoded: frame.encoded };
|
|
632
|
+
}
|
|
633
|
+
if (isAudioFrame(frame.frame)) {
|
|
634
|
+
return { track, kind: "audio", operation: frame.operation, audio: frame.frame };
|
|
635
|
+
}
|
|
636
|
+
return { track, kind: "video", operation: frame.operation, pixels: await toPixels(frame.frame) };
|
|
637
|
+
}
|
|
638
|
+
function checkStepFrame(client, frame) {
|
|
639
|
+
if (frame.kind === "encoded") {
|
|
640
|
+
const { codec, data, width, height, captureTimeUs } = frame.encoded;
|
|
641
|
+
client.checkEncodedVideoFrame(frame.track, codec, data.byteLength, width, height, captureTimeUs);
|
|
642
|
+
} else if (frame.kind === "video") {
|
|
643
|
+
const { data, width, height, format } = frame.pixels;
|
|
644
|
+
client.checkVideoFrame(frame.track, data.byteLength, width, height, format);
|
|
645
|
+
} else {
|
|
646
|
+
client.checkAudioFrame(frame.track, frame.audio.samples, frame.audio.channels ?? 1);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
|
|
384
650
|
// src/internal/stats.ts
|
|
385
651
|
var LOCAL_STATS_INTERVAL_MS = 2e3;
|
|
386
652
|
var RUNTIME_REPORT_INTERVAL_MS = 5e3;
|
|
@@ -430,7 +696,28 @@ var WIRE_CODECS = {
|
|
|
430
696
|
};
|
|
431
697
|
function createTrackStatsExtractor() {
|
|
432
698
|
const lastBytes = /* @__PURE__ */ new Map();
|
|
433
|
-
|
|
699
|
+
const lastStages = /* @__PURE__ */ new Map();
|
|
700
|
+
let pendingBytes = /* @__PURE__ */ new Map();
|
|
701
|
+
let pendingStages = /* @__PURE__ */ new Map();
|
|
702
|
+
const stage = (streamId, name, seconds, frames) => {
|
|
703
|
+
if (seconds === void 0 || frames === void 0) {
|
|
704
|
+
return void 0;
|
|
705
|
+
}
|
|
706
|
+
const key = `${streamId}:${name}`;
|
|
707
|
+
if (!Number.isFinite(seconds)) {
|
|
708
|
+
pendingStages.set(key, void 0);
|
|
709
|
+
return void 0;
|
|
710
|
+
}
|
|
711
|
+
pendingStages.set(key, { seconds, frames });
|
|
712
|
+
const before = lastStages.get(key);
|
|
713
|
+
if (before === void 0 || frames <= before.frames || seconds < before.seconds) {
|
|
714
|
+
return void 0;
|
|
715
|
+
}
|
|
716
|
+
return { name, totalMs: (seconds - before.seconds) * 1e3, frames: frames - before.frames };
|
|
717
|
+
};
|
|
718
|
+
const extract = (report, tracks, pausedTracks) => {
|
|
719
|
+
pendingBytes = /* @__PURE__ */ new Map();
|
|
720
|
+
pendingStages = /* @__PURE__ */ new Map();
|
|
434
721
|
const reportWithLookup = report;
|
|
435
722
|
const stats = [];
|
|
436
723
|
const seenStreams = /* @__PURE__ */ new Set();
|
|
@@ -469,7 +756,7 @@ function createTrackStatsExtractor() {
|
|
|
469
756
|
if (previous !== void 0 && stat.timestamp > previous.timestamp && bytes >= previous.bytes) {
|
|
470
757
|
put("bitrate_bps", Math.round((bytes - previous.bytes) * 8 * 1e3 / (stat.timestamp - previous.timestamp)));
|
|
471
758
|
}
|
|
472
|
-
|
|
759
|
+
pendingBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
|
|
473
760
|
}
|
|
474
761
|
const keyframeRequests = stat.firCount !== void 0 || stat.pliCount !== void 0 ? (stat.firCount ?? 0) + (stat.pliCount ?? 0) : void 0;
|
|
475
762
|
if (inbound) {
|
|
@@ -504,6 +791,20 @@ function createTrackStatsExtractor() {
|
|
|
504
791
|
put("keyframe_requests", keyframeRequests);
|
|
505
792
|
}
|
|
506
793
|
}
|
|
794
|
+
const frameStages = [];
|
|
795
|
+
if (track.kind === "video") {
|
|
796
|
+
const stages = inbound ? [
|
|
797
|
+
stage(stat.id, "jitter_buffer", stat.jitterBufferDelay, stat.jitterBufferEmittedCount),
|
|
798
|
+
stage(stat.id, "decode", stat.totalDecodeTime, stat.framesDecoded)
|
|
799
|
+
] : [stage(stat.id, "encode", stat.totalEncodeTime, stat.framesEncoded)];
|
|
800
|
+
for (const s of stages) {
|
|
801
|
+
if (s !== void 0) {
|
|
802
|
+
frameStages.push(s);
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
const codecImplementation = inbound ? stat.decoderImplementation : stat.encoderImplementation;
|
|
807
|
+
const hardwareCodec = inbound ? stat.powerEfficientDecoder : stat.powerEfficientEncoder;
|
|
507
808
|
stats.push({
|
|
508
809
|
timestamp: Date.now(),
|
|
509
810
|
trackName: track.name,
|
|
@@ -511,11 +812,30 @@ function createTrackStatsExtractor() {
|
|
|
511
812
|
direction,
|
|
512
813
|
codec,
|
|
513
814
|
paused: pausedTracks.includes(track.name),
|
|
514
|
-
metrics
|
|
815
|
+
metrics,
|
|
816
|
+
...frameStages.length > 0 ? { frameStages } : {},
|
|
817
|
+
...track.kind === "video" && codecImplementation !== void 0 ? { codecImplementation } : {},
|
|
818
|
+
...track.kind === "video" && hardwareCodec !== void 0 ? { hardwareCodec } : {}
|
|
515
819
|
});
|
|
516
820
|
});
|
|
517
821
|
return stats;
|
|
518
822
|
};
|
|
823
|
+
return Object.assign(extract, {
|
|
824
|
+
commit() {
|
|
825
|
+
for (const [id, sample] of pendingBytes) {
|
|
826
|
+
lastBytes.set(id, sample);
|
|
827
|
+
}
|
|
828
|
+
for (const [key, total] of pendingStages) {
|
|
829
|
+
if (total === void 0) {
|
|
830
|
+
lastStages.delete(key);
|
|
831
|
+
} else {
|
|
832
|
+
lastStages.set(key, total);
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
pendingBytes = /* @__PURE__ */ new Map();
|
|
836
|
+
pendingStages = /* @__PURE__ */ new Map();
|
|
837
|
+
}
|
|
838
|
+
});
|
|
519
839
|
}
|
|
520
840
|
function toClientConnectionStat(report, timeToConnectMs) {
|
|
521
841
|
const pair = activeCandidatePair(report);
|
|
@@ -654,7 +974,7 @@ function describeCause(cause) {
|
|
|
654
974
|
// package.json
|
|
655
975
|
var package_default = {
|
|
656
976
|
name: "@reactor-team/js-sdk",
|
|
657
|
-
version: "3.
|
|
977
|
+
version: "3.4.0",
|
|
658
978
|
description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
|
|
659
979
|
license: "Apache-2.0",
|
|
660
980
|
repository: {
|
|
@@ -1168,10 +1488,19 @@ var Reactor = class {
|
|
|
1168
1488
|
* with it, so a batch sent before the timings exist doesn't use it up.
|
|
1169
1489
|
* Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
|
|
1170
1490
|
hasSentConnectTime = false;
|
|
1491
|
+
/** Whether this connection has sent a client-stats batch yet. The first
|
|
1492
|
+
* one goes out even with no reading, for the environment the bridge adds
|
|
1493
|
+
* to it. Reset with `hasSentConnectTime`. */
|
|
1494
|
+
hasSentClientStats = false;
|
|
1171
1495
|
/** Set on the "connecting" status transition, cleared once `connectionTimings`
|
|
1172
1496
|
* is finalized on "ready" — see `handleStatusChanged()`. */
|
|
1173
1497
|
connectStartTime;
|
|
1174
1498
|
waitingStartTime;
|
|
1499
|
+
/** The tail of every `pushFrame()`/`pushEncodedFrame()` so far, settled
|
|
1500
|
+
* either way — pushes reach the binding in call order however long each
|
|
1501
|
+
* one's pixels take to read, and `step()` waits for the ones called
|
|
1502
|
+
* before it. */
|
|
1503
|
+
pendingPushes = Promise.resolve();
|
|
1175
1504
|
emitter = new Emitter();
|
|
1176
1505
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
1177
1506
|
* disconnect()/[Symbol.dispose]) against each other. Calling into the
|
|
@@ -1189,6 +1518,15 @@ var Reactor = class {
|
|
|
1189
1518
|
* out every read active when it started, before it releases the ones
|
|
1190
1519
|
* that arrived while it ran. */
|
|
1191
1520
|
writerActive = false;
|
|
1521
|
+
/** Counts `disconnect()`s and disposal. A push captures it when called, and
|
|
1522
|
+
* one whose frame was still being read when it moved is dropped, rather
|
|
1523
|
+
* than recreating a client to carry the frame into the next session. */
|
|
1524
|
+
connectionEpoch = 0;
|
|
1525
|
+
/** Input tracks published on this connection, each with the publication
|
|
1526
|
+
* that claimed it: a push needs one, and a `PublishedTrack` only feeds the
|
|
1527
|
+
* publication it came from. Cleared with the connection. */
|
|
1528
|
+
publishedInputs = /* @__PURE__ */ new Map();
|
|
1529
|
+
publications = 0;
|
|
1192
1530
|
/** Readers parked in `withReaderAccess()`'s wait loop, woken once
|
|
1193
1531
|
* `withWriterAccess()` finishes; each re-checks `writerActive` rather
|
|
1194
1532
|
* than assuming its turn, since another writer already queued behind the
|
|
@@ -1461,14 +1799,50 @@ var Reactor = class {
|
|
|
1461
1799
|
* disconnect() freeing the wasm client mid-await would otherwise resume
|
|
1462
1800
|
* into a freed client — the same class of use-after-free `queue` exists to
|
|
1463
1801
|
* prevent for connect/disconnect/reconnect themselves.
|
|
1802
|
+
*
|
|
1803
|
+
* Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
|
|
1804
|
+
* track in a stepped session.
|
|
1805
|
+
*
|
|
1806
|
+
* In a `generation: "stepped"` session, input frames never travel over
|
|
1807
|
+
* RTP: `track` is optional, and the frames are pushed with
|
|
1808
|
+
* `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
|
|
1809
|
+
* passed there is ignored, with a console warning — nothing is sent from
|
|
1810
|
+
* it. In a streamed session `track` is required, and omitting it rejects
|
|
1811
|
+
* with an `InvalidStateError`.
|
|
1812
|
+
*
|
|
1813
|
+
* ```ts
|
|
1814
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1815
|
+
* await camera.pushFrame(imageData);
|
|
1816
|
+
* const result = await reactor.step();
|
|
1817
|
+
* ```
|
|
1464
1818
|
*/
|
|
1465
1819
|
async publishTrack(name, track) {
|
|
1466
1820
|
this.assertNotDisposed();
|
|
1467
1821
|
try {
|
|
1822
|
+
let media = track;
|
|
1823
|
+
if (this.isStepped() && media !== void 0) {
|
|
1824
|
+
console.warn(
|
|
1825
|
+
`[Reactor] publishTrack('${name}'): ignoring the MediaStreamTrack \u2014 a stepped session sends no RTP and takes frames through track.pushFrame(...) on the track publishTrack() returns.`
|
|
1826
|
+
);
|
|
1827
|
+
media = void 0;
|
|
1828
|
+
}
|
|
1829
|
+
if (!this.isStepped() && media === void 0) {
|
|
1830
|
+
throw new InvalidStateError(
|
|
1831
|
+
`publishTrack('${name}') needs a MediaStreamTrack in a streamed session.`,
|
|
1832
|
+
{ operation: "publishTrack" }
|
|
1833
|
+
);
|
|
1834
|
+
}
|
|
1835
|
+
const publication = this.publications += 1;
|
|
1468
1836
|
await this.queue.push(async () => {
|
|
1469
1837
|
const client = await this.getOrCreateClient();
|
|
1470
|
-
await client.publishTrack(name,
|
|
1838
|
+
await client.publishTrack(name, media);
|
|
1839
|
+
this.publishedInputs.set(name, publication);
|
|
1471
1840
|
}, "publishTrack");
|
|
1841
|
+
return {
|
|
1842
|
+
name,
|
|
1843
|
+
pushFrame: (frame, options) => this.pushTo(name, publication, frame, options),
|
|
1844
|
+
pushEncodedFrame: (frame) => this.pushEncodedTo(name, publication, frame)
|
|
1845
|
+
};
|
|
1472
1846
|
} catch (cause) {
|
|
1473
1847
|
throw this.captureError(cause);
|
|
1474
1848
|
}
|
|
@@ -1485,6 +1859,7 @@ var Reactor = class {
|
|
|
1485
1859
|
await this.queue.push(async () => {
|
|
1486
1860
|
const client = await this.getOrCreateClient();
|
|
1487
1861
|
await client.unpublishTrack(name);
|
|
1862
|
+
this.publishedInputs.delete(name);
|
|
1488
1863
|
}, "unpublishTrack");
|
|
1489
1864
|
} catch (cause) {
|
|
1490
1865
|
this.emitError(cause);
|
|
@@ -1557,6 +1932,337 @@ var Reactor = class {
|
|
|
1557
1932
|
throw this.captureError(cause);
|
|
1558
1933
|
}
|
|
1559
1934
|
}
|
|
1935
|
+
// ── Stepped generation ──────────────────────────────────────────────────
|
|
1936
|
+
/**
|
|
1937
|
+
* Runs one step of a `generation: "stepped"` session and resolves with
|
|
1938
|
+
* what it produced, per output track.
|
|
1939
|
+
*
|
|
1940
|
+
* The request carries `state`, `commands` and every frame pushed since the
|
|
1941
|
+
* previous step. The runtime applies them in that order — each `state` key
|
|
1942
|
+
* as the model's `set_<key>` command, then `commands`, then the frames —
|
|
1943
|
+
* and then generates. They are separate calls, not a transaction: one that
|
|
1944
|
+
* fails does not undo the ones before it. All of it is optional:
|
|
1945
|
+
* `step()` with nothing set and nothing pushed runs on the model's current
|
|
1946
|
+
* state, which commands sent outside a step (`sendCommand()`) also change.
|
|
1947
|
+
* A file goes in the way it does for a command: upload it with
|
|
1948
|
+
* `uploadFile()`, then put the `FileRef` in a command's `data` or a state
|
|
1949
|
+
* value.
|
|
1950
|
+
*
|
|
1951
|
+
* A step that ran but that the model reported as failed still resolves,
|
|
1952
|
+
* with `result.error` set. `result.files` lists the files the model kept
|
|
1953
|
+
* with the step (name, content type, size); their bytes are served over
|
|
1954
|
+
* HTTP from the session's step results, not carried here.
|
|
1955
|
+
*
|
|
1956
|
+
* Rejects with a `StepRefusedError` when the runtime or the model turns
|
|
1957
|
+
* the step down (the runtime checks a whole request first, so nothing it
|
|
1958
|
+
* carried was applied), `StepNotReadyError` when the session cannot take a
|
|
1959
|
+
* step yet (neither ran the step), `StepCommandFailedError` when a state key
|
|
1960
|
+
* or command failed (the calls before it stay applied; the step did not
|
|
1961
|
+
* run); both name the part at fault in `entry`, and `StepCommandFailedError`
|
|
1962
|
+
* the handler's code in `command_code`,
|
|
1963
|
+
* `StepSessionEndedError` when the session ended before the result came
|
|
1964
|
+
* back, `StepResultTooLargeError` when the result does not fit in one
|
|
1965
|
+
* message, `MessageTooLargeError` when the request itself is too large
|
|
1966
|
+
* (nothing was sent), `InvalidStateError` outside a ready stepped
|
|
1967
|
+
* session, or whatever else failed the step (`DisconnectedError`,
|
|
1968
|
+
* `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
|
|
1969
|
+
* when its request cannot be sent, they are dropped with it rather than
|
|
1970
|
+
* kept for the next step.
|
|
1971
|
+
*
|
|
1972
|
+
* `media` gives the step frames of its own, by input track name, pushed
|
|
1973
|
+
* after the ones already pushed to that track: what `pushFrame()` takes,
|
|
1974
|
+
* or an `EncodedFrame` — H.264 the runtime decodes before the step runs,
|
|
1975
|
+
* so the model reads RGB frames as for a raw push. They are pushed only
|
|
1976
|
+
* once `state` and `commands` are well formed and the session is ready, so
|
|
1977
|
+
* a step refused for those leaves the frames buffered as they were. They go
|
|
1978
|
+
* all or nothing: a bad track or frame anywhere in `media` rejects the step
|
|
1979
|
+
* before any of it is pushed or sent, leaving the buffer as it was. A
|
|
1980
|
+
* session that leaves 'ready' once the media is in has the step refused and
|
|
1981
|
+
* keeps the frames for the next one, as pushed frames are.
|
|
1982
|
+
* A track's encoded frames of one step go in decode order, starting at a
|
|
1983
|
+
* keyframe; a step whose frames do not decode rejects with a
|
|
1984
|
+
* `StepRefusedError`.
|
|
1985
|
+
*
|
|
1986
|
+
* ```ts
|
|
1987
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1988
|
+
* await camera.pushFrame(imageData);
|
|
1989
|
+
* const result = await reactor.step({ state: { prompt: 'a cat' } });
|
|
1990
|
+
* if (result.error) console.warn('step failed:', result.error);
|
|
1991
|
+
* const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
|
|
1992
|
+
* if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
|
|
1993
|
+
* ```
|
|
1994
|
+
*/
|
|
1995
|
+
async step(options = {}) {
|
|
1996
|
+
this.assertNotDisposed();
|
|
1997
|
+
try {
|
|
1998
|
+
const state = rewriteNestedFileRefs(options.state ?? null);
|
|
1999
|
+
const commands = rewriteNestedFileRefs(options.commands ?? []);
|
|
2000
|
+
const previous = this.pendingPushes;
|
|
2001
|
+
const before = options.media !== void 0 ? this.pushStepMedia(options.media, state, commands) : previous;
|
|
2002
|
+
let taken;
|
|
2003
|
+
const bufferTaken = new Promise((resolve) => {
|
|
2004
|
+
taken = resolve;
|
|
2005
|
+
});
|
|
2006
|
+
const stepping = before.then(
|
|
2007
|
+
() => (
|
|
2008
|
+
// Excluded from connect()/disconnect()/reconnect() like sendCommand():
|
|
2009
|
+
// a step can take minutes, and must neither be freed out from under
|
|
2010
|
+
// nor block unrelated reads.
|
|
2011
|
+
this.withReaderAccess(async () => {
|
|
2012
|
+
const client = await this.getOrCreateClient();
|
|
2013
|
+
try {
|
|
2014
|
+
return client.step(state, commands);
|
|
2015
|
+
} finally {
|
|
2016
|
+
taken();
|
|
2017
|
+
}
|
|
2018
|
+
})
|
|
2019
|
+
)
|
|
2020
|
+
);
|
|
2021
|
+
stepping.catch(() => previous.then(taken));
|
|
2022
|
+
this.pendingPushes = bufferTaken;
|
|
2023
|
+
return await stepping;
|
|
2024
|
+
} catch (cause) {
|
|
2025
|
+
throw this.captureError(cause);
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
2028
|
+
/**
|
|
2029
|
+
* Buffers one input frame for the next `step()` on the sendonly track
|
|
2030
|
+
* `trackName`. Nothing is sent until that step, which carries every frame
|
|
2031
|
+
* pushed since the one before it, per track in push order. The
|
|
2032
|
+
* `PublishedTrack` that `publishTrack()` resolves with does the same
|
|
2033
|
+
* through `track.pushFrame(frame)`.
|
|
2034
|
+
*
|
|
2035
|
+
* Video: a `VideoFrame` (cloned when called, so the caller may close it as
|
|
2036
|
+
* soon as this returns; it stays the caller's), an
|
|
2037
|
+
* `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
|
|
2038
|
+
* by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
|
|
2039
|
+
* `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
|
|
2040
|
+
* applied, so a 90° frame arrives with its width and height swapped.
|
|
2041
|
+
* Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
|
|
2042
|
+
* PCM at the rate the model's track declares.
|
|
2043
|
+
*
|
|
2044
|
+
* Pushes keep their call order, and a `step()` waits for every push called
|
|
2045
|
+
* before it. A refused push rejects: await it, or `.catch()` it in a frame
|
|
2046
|
+
* loop, or every bad frame is an unhandled rejection. What waits for
|
|
2047
|
+
* the next step is at most one message's worth of frames: pushing faster
|
|
2048
|
+
* than steps run drops the oldest frames, across tracks. Rejects with
|
|
2049
|
+
* an `InvalidStateError` outside a stepped session, for a track not
|
|
2050
|
+
* published on this connection (`publishTrack(trackName)` first), for a
|
|
2051
|
+
* frame whose size does not match its dimensions, or for a name the model
|
|
2052
|
+
* does not declare as a sendonly track of that kind.
|
|
2053
|
+
*/
|
|
2054
|
+
async pushFrame(trackName, frame, options = {}) {
|
|
2055
|
+
return this.pushTo(trackName, void 0, frame, options);
|
|
2056
|
+
}
|
|
2057
|
+
/** `pushFrame()`, for the track's current publication or, from a
|
|
2058
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
2059
|
+
async pushTo(trackName, publication, frame, options = {}) {
|
|
2060
|
+
this.assertNotDisposed();
|
|
2061
|
+
const epoch = this.connectionEpoch;
|
|
2062
|
+
let held;
|
|
2063
|
+
try {
|
|
2064
|
+
held = holdFrame(frame);
|
|
2065
|
+
} catch (cause) {
|
|
2066
|
+
throw this.captureError(cause);
|
|
2067
|
+
}
|
|
2068
|
+
const push = this.pendingPushes.then(async () => {
|
|
2069
|
+
let pixels;
|
|
2070
|
+
try {
|
|
2071
|
+
if (!this.isStepped()) {
|
|
2072
|
+
throw new InvalidStateError(
|
|
2073
|
+
`pushFrame('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
|
|
2074
|
+
{ operation: "pushFrame" }
|
|
2075
|
+
);
|
|
2076
|
+
}
|
|
2077
|
+
pixels = isAudioFrame(held.frame) ? void 0 : await toPixels(held.frame);
|
|
2078
|
+
} finally {
|
|
2079
|
+
held.release();
|
|
2080
|
+
}
|
|
2081
|
+
await this.withReaderAccess(async () => {
|
|
2082
|
+
this.checkPushTarget("pushFrame", trackName, publication, epoch);
|
|
2083
|
+
const client = await this.getOrCreateClient();
|
|
2084
|
+
if (pixels) {
|
|
2085
|
+
client.pushVideoFrame(
|
|
2086
|
+
trackName,
|
|
2087
|
+
pixels.data,
|
|
2088
|
+
pixels.width,
|
|
2089
|
+
pixels.height,
|
|
2090
|
+
pixels.format,
|
|
2091
|
+
options.metadata ?? null,
|
|
2092
|
+
options.captureTimeUs ?? null
|
|
2093
|
+
);
|
|
2094
|
+
} else if (isAudioFrame(frame)) {
|
|
2095
|
+
client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
|
|
2096
|
+
}
|
|
2097
|
+
});
|
|
2098
|
+
});
|
|
2099
|
+
this.pendingPushes = push.catch(() => void 0);
|
|
2100
|
+
try {
|
|
2101
|
+
await push;
|
|
2102
|
+
} catch (cause) {
|
|
2103
|
+
throw this.captureError(cause);
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
/**
|
|
2107
|
+
* Buffers one encoded video frame for the next `step()` on the sendonly
|
|
2108
|
+
* video track `trackName`, its bytes as they are: one H.264 access unit
|
|
2109
|
+
* (Annex B), with the `width` and `height` it decodes to. The
|
|
2110
|
+
* `PublishedTrack` that `publishTrack()` resolves with does the same
|
|
2111
|
+
* through `track.pushEncodedFrame(frame)`.
|
|
2112
|
+
*
|
|
2113
|
+
* The runtime decodes a track's encoded frames of one step as one stream,
|
|
2114
|
+
* before the step runs, and the model reads the RGB frames as if they had
|
|
2115
|
+
* been pushed raw. So push them in decode order, starting at a keyframe; a
|
|
2116
|
+
* step whose frames do not decode rejects with a `StepRefusedError`, as
|
|
2117
|
+
* does a runtime that predates encoded frames. Encoded and raw pushes keep
|
|
2118
|
+
* their call order, and a `step()` waits for every push called before it.
|
|
2119
|
+
*
|
|
2120
|
+
* ```ts
|
|
2121
|
+
* const encoder = new VideoEncoder({
|
|
2122
|
+
* output: (chunk) => void camera.pushEncodedFrame({ codec: 'h264', data: chunk, width, height }),
|
|
2123
|
+
* error: console.error,
|
|
2124
|
+
* });
|
|
2125
|
+
* encoder.configure({ codec: 'avc1.42001f', width, height, avc: { format: 'annexb' } });
|
|
2126
|
+
* ```
|
|
2127
|
+
*
|
|
2128
|
+
* Rejects with an `InvalidStateError` outside a stepped session, for a
|
|
2129
|
+
* track not published on this connection, for an audio or recvonly track,
|
|
2130
|
+
* for empty data, a zero dimension or a codec other than `"h264"`.
|
|
2131
|
+
*/
|
|
2132
|
+
async pushEncodedFrame(trackName, frame) {
|
|
2133
|
+
return this.pushEncodedTo(trackName, void 0, frame);
|
|
2134
|
+
}
|
|
2135
|
+
/** `pushEncodedFrame()`, for the track's current publication or, from a
|
|
2136
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
2137
|
+
async pushEncodedTo(trackName, publication, frame) {
|
|
2138
|
+
const operation = "pushEncodedFrame";
|
|
2139
|
+
this.assertNotDisposed();
|
|
2140
|
+
const epoch = this.connectionEpoch;
|
|
2141
|
+
let encoded;
|
|
2142
|
+
try {
|
|
2143
|
+
encoded = toEncodedBytes(frame, operation, trackName);
|
|
2144
|
+
} catch (cause) {
|
|
2145
|
+
throw this.captureError(cause);
|
|
2146
|
+
}
|
|
2147
|
+
const push = this.pendingPushes.then(
|
|
2148
|
+
() => this.withReaderAccess(async () => {
|
|
2149
|
+
if (!this.isStepped()) {
|
|
2150
|
+
throw new InvalidStateError(
|
|
2151
|
+
`${operation}('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
|
|
2152
|
+
{ operation }
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
this.checkPushTarget(operation, trackName, publication, epoch);
|
|
2156
|
+
const client = await this.getOrCreateClient();
|
|
2157
|
+
const declared = client.tracks().find((track) => track.name === trackName);
|
|
2158
|
+
if (declared && (declared.kind !== "video" || declared.direction !== "sendonly")) {
|
|
2159
|
+
throw new InvalidStateError(
|
|
2160
|
+
`${operation}('${trackName}'): '${trackName}' is a ${declared.direction} ${declared.kind} track; only a sendonly video track takes encoded frames.`,
|
|
2161
|
+
{ operation }
|
|
2162
|
+
);
|
|
2163
|
+
}
|
|
2164
|
+
client.pushEncodedVideoFrame(
|
|
2165
|
+
trackName,
|
|
2166
|
+
encoded.codec,
|
|
2167
|
+
encoded.data,
|
|
2168
|
+
encoded.width,
|
|
2169
|
+
encoded.height,
|
|
2170
|
+
encoded.metadata,
|
|
2171
|
+
encoded.captureTimeUs
|
|
2172
|
+
);
|
|
2173
|
+
})
|
|
2174
|
+
);
|
|
2175
|
+
this.pendingPushes = push.catch(() => void 0);
|
|
2176
|
+
try {
|
|
2177
|
+
await push;
|
|
2178
|
+
} catch (cause) {
|
|
2179
|
+
throw this.captureError(cause);
|
|
2180
|
+
}
|
|
2181
|
+
}
|
|
2182
|
+
/** Refuses a push whose connection ended since it was called, or whose
|
|
2183
|
+
* track is not (or no longer) published on it. */
|
|
2184
|
+
checkPushTarget(operation, trackName, publication, epoch) {
|
|
2185
|
+
if (epoch !== this.connectionEpoch) {
|
|
2186
|
+
throw new InvalidStateError(
|
|
2187
|
+
`${operation}('${trackName}'): the connection it was pushed to ended while its frame was read`,
|
|
2188
|
+
{ operation }
|
|
2189
|
+
);
|
|
2190
|
+
}
|
|
2191
|
+
const current = this.publishedInputs.get(trackName);
|
|
2192
|
+
if (current === void 0 || publication !== void 0 && publication !== current) {
|
|
2193
|
+
throw new InvalidStateError(
|
|
2194
|
+
publication === void 0 ? `${operation}('${trackName}') needs publishTrack('${trackName}') first.` : `${operation}('${trackName}'): this track was unpublished, or its connection ended.`,
|
|
2195
|
+
{ operation }
|
|
2196
|
+
);
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
/** Pushes `step()`'s `media`, track by track and in order, each frame as
|
|
2200
|
+
* `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
|
|
2201
|
+
* checks the step's `state` and `commands`, then every frame, before the
|
|
2202
|
+
* first is pushed, so a refused step, or a bad track or frame anywhere,
|
|
2203
|
+
* leaves the buffer as it was. */
|
|
2204
|
+
async pushStepMedia(media, state, commands) {
|
|
2205
|
+
const epoch = this.connectionEpoch;
|
|
2206
|
+
const held = holdStepMedia(media);
|
|
2207
|
+
const push = this.pendingPushes.then(async () => {
|
|
2208
|
+
const ready = [];
|
|
2209
|
+
try {
|
|
2210
|
+
for (const frame of held) {
|
|
2211
|
+
ready.push(await readStepFrame(frame));
|
|
2212
|
+
}
|
|
2213
|
+
} finally {
|
|
2214
|
+
for (const frame of held) {
|
|
2215
|
+
frame.release();
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2218
|
+
await this.withReaderAccess(async () => {
|
|
2219
|
+
const client = await this.getOrCreateClient();
|
|
2220
|
+
client.checkStep(state, commands);
|
|
2221
|
+
for (const frame of ready) {
|
|
2222
|
+
this.checkPushTarget(frame.operation, frame.track, void 0, epoch);
|
|
2223
|
+
checkStepFrame(client, frame);
|
|
2224
|
+
}
|
|
2225
|
+
for (const frame of ready) {
|
|
2226
|
+
if (frame.kind === "encoded") {
|
|
2227
|
+
client.pushEncodedVideoFrame(
|
|
2228
|
+
frame.track,
|
|
2229
|
+
frame.encoded.codec,
|
|
2230
|
+
frame.encoded.data,
|
|
2231
|
+
frame.encoded.width,
|
|
2232
|
+
frame.encoded.height,
|
|
2233
|
+
frame.encoded.metadata,
|
|
2234
|
+
frame.encoded.captureTimeUs
|
|
2235
|
+
);
|
|
2236
|
+
} else if (frame.kind === "video") {
|
|
2237
|
+
const { data, width, height, format } = frame.pixels;
|
|
2238
|
+
client.pushVideoFrame(frame.track, data, width, height, format, null, null);
|
|
2239
|
+
} else {
|
|
2240
|
+
const { samples, sampleRate, channels } = frame.audio;
|
|
2241
|
+
client.pushAudioFrame(frame.track, samples, sampleRate, channels ?? 1);
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
});
|
|
2245
|
+
});
|
|
2246
|
+
this.pendingPushes = push.catch(() => void 0);
|
|
2247
|
+
await push;
|
|
2248
|
+
}
|
|
2249
|
+
/** Drops every frame buffered for the next step, pushes called before it
|
|
2250
|
+
* included: it waits its turn behind them, as a `step()` does. */
|
|
2251
|
+
async clearStepInputs() {
|
|
2252
|
+
this.assertNotDisposed();
|
|
2253
|
+
const clear = this.pendingPushes.then(
|
|
2254
|
+
() => this.withReaderAccess(() => {
|
|
2255
|
+
this.client?.clearStepInputs();
|
|
2256
|
+
return Promise.resolve();
|
|
2257
|
+
})
|
|
2258
|
+
);
|
|
2259
|
+
this.pendingPushes = clear.catch(() => void 0);
|
|
2260
|
+
try {
|
|
2261
|
+
await clear;
|
|
2262
|
+
} catch (cause) {
|
|
2263
|
+
throw this.captureError(cause);
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
1560
2266
|
// ── Recording ───────────────────────────────────────────────────────────
|
|
1561
2267
|
/**
|
|
1562
2268
|
* Requests a clip covering the last `durationSeconds` of the session.
|
|
@@ -1855,6 +2561,7 @@ var Reactor = class {
|
|
|
1855
2561
|
const extractConnectionStats = createConnectionStatsExtractor();
|
|
1856
2562
|
const extractTrackStats = createTrackStatsExtractor();
|
|
1857
2563
|
this.hasSentConnectTime = false;
|
|
2564
|
+
this.hasSentClientStats = false;
|
|
1858
2565
|
this.stopStatsPollers = [
|
|
1859
2566
|
pollPeerStats(client, LOCAL_STATS_INTERVAL_MS, (report) => {
|
|
1860
2567
|
this.stats = { ...extractConnectionStats(report), connectionTimings: this.connectionTimings };
|
|
@@ -1872,8 +2579,9 @@ var Reactor = class {
|
|
|
1872
2579
|
* track has one yet (none negotiated onto a named track, or no codec
|
|
1873
2580
|
* reported yet), so the runtime gets the bandwidth estimate, the
|
|
1874
2581
|
* round-trip time and the time to connect without waiting on the tracks.
|
|
1875
|
-
* Skipped only when there is nothing at all to say
|
|
1876
|
-
*
|
|
2582
|
+
* Skipped only when there is nothing at all to say and a batch has already
|
|
2583
|
+
* gone out for this connection, so the first one still carries the
|
|
2584
|
+
* environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
|
|
1877
2585
|
* actually sent for this connection — see `hasSentConnectTime`.
|
|
1878
2586
|
* `sendClientStats` throws synchronously on a closing connection; caught
|
|
1879
2587
|
* here for the same reason `getStats()`'s rejection is above. */
|
|
@@ -1881,11 +2589,13 @@ var Reactor = class {
|
|
|
1881
2589
|
const trackStats = extractTrackStats(report, this.trackMapping(), this.pausedTracks());
|
|
1882
2590
|
const timeToConnectMs = this.hasSentConnectTime ? void 0 : this.connectionTimings?.totalMs;
|
|
1883
2591
|
const connectionStat = toClientConnectionStat(report, timeToConnectMs);
|
|
1884
|
-
if (trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
|
|
2592
|
+
if (this.hasSentClientStats && trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
|
|
1885
2593
|
return;
|
|
1886
2594
|
}
|
|
1887
2595
|
try {
|
|
1888
2596
|
client.sendClientStats(trackStats, connectionStat);
|
|
2597
|
+
extractTrackStats.commit();
|
|
2598
|
+
this.hasSentClientStats = true;
|
|
1889
2599
|
if (timeToConnectMs !== void 0) {
|
|
1890
2600
|
this.hasSentConnectTime = true;
|
|
1891
2601
|
}
|
|
@@ -1907,6 +2617,8 @@ var Reactor = class {
|
|
|
1907
2617
|
* leave `getCapabilities()` returning the previous session's stale
|
|
1908
2618
|
* tracks/commands until a new `capabilitiesReceived` lands. */
|
|
1909
2619
|
resetConnectionState() {
|
|
2620
|
+
this.connectionEpoch += 1;
|
|
2621
|
+
this.publishedInputs.clear();
|
|
1910
2622
|
this.stopStatsPolling();
|
|
1911
2623
|
this.connectionTimings = void 0;
|
|
1912
2624
|
this.connectStartTime = void 0;
|
|
@@ -1927,6 +2639,9 @@ var Reactor = class {
|
|
|
1927
2639
|
emitError(cause) {
|
|
1928
2640
|
this.emitter.emit("error", this.captureError(cause));
|
|
1929
2641
|
}
|
|
2642
|
+
isStepped() {
|
|
2643
|
+
return this.clientOptions.generation === "stepped";
|
|
2644
|
+
}
|
|
1930
2645
|
assertNotDisposed() {
|
|
1931
2646
|
if (this.disposed) {
|
|
1932
2647
|
throw new Error("This Reactor was disposed and can't be used again \u2014 construct a new one.");
|
|
@@ -1934,6 +2649,25 @@ var Reactor = class {
|
|
|
1934
2649
|
}
|
|
1935
2650
|
};
|
|
1936
2651
|
|
|
2652
|
+
// src/step-frames.ts
|
|
2653
|
+
function stepFrameToImageData(frame) {
|
|
2654
|
+
const [height, width, channels] = frame.shape;
|
|
2655
|
+
if (frame.dtype !== "uint8" || frame.shape.length !== 3 || channels !== 3 || height === void 0 || width === void 0 || frame.data.length !== width * height * 3) {
|
|
2656
|
+
throw new Error(
|
|
2657
|
+
`stepFrameToImageData: expected a (height, width, 3) uint8 RGB frame, got ${frame.dtype} [${frame.shape.join(", ")}]`
|
|
2658
|
+
);
|
|
2659
|
+
}
|
|
2660
|
+
const rgb = frame.data;
|
|
2661
|
+
const rgba = new Uint8ClampedArray(width * height * 4);
|
|
2662
|
+
for (let src = 0, dst = 0; src < rgb.length; src += 3, dst += 4) {
|
|
2663
|
+
rgba[dst] = rgb[src];
|
|
2664
|
+
rgba[dst + 1] = rgb[src + 1];
|
|
2665
|
+
rgba[dst + 2] = rgb[src + 2];
|
|
2666
|
+
rgba[dst + 3] = 255;
|
|
2667
|
+
}
|
|
2668
|
+
return new ImageData(rgba, width, height);
|
|
2669
|
+
}
|
|
2670
|
+
|
|
1937
2671
|
// src/jwt.ts
|
|
1938
2672
|
function normalizeJwtSource(source) {
|
|
1939
2673
|
if (typeof source === "function") {
|
|
@@ -2884,9 +3618,11 @@ function defaultLabel(state) {
|
|
|
2884
3618
|
ClipPlayer,
|
|
2885
3619
|
ConflictError,
|
|
2886
3620
|
DEFAULT_PLAYLIST_POLL_SLACK_MS,
|
|
3621
|
+
DataChannelChunkingRequiredError,
|
|
2887
3622
|
DecodeError,
|
|
2888
3623
|
DisconnectedError,
|
|
2889
3624
|
FileRef,
|
|
3625
|
+
GenerationModeMismatchError,
|
|
2890
3626
|
InvalidStateError,
|
|
2891
3627
|
MessageTooLargeError,
|
|
2892
3628
|
NetworkError,
|
|
@@ -2901,8 +3637,14 @@ function defaultLabel(state) {
|
|
|
2901
3637
|
RequestTimeoutError,
|
|
2902
3638
|
ServerError,
|
|
2903
3639
|
SessionTerminalError,
|
|
3640
|
+
StepCommandFailedError,
|
|
3641
|
+
StepNotReadyError,
|
|
3642
|
+
StepRefusedError,
|
|
3643
|
+
StepResultTooLargeError,
|
|
3644
|
+
StepSessionEndedError,
|
|
2904
3645
|
TransportError,
|
|
2905
3646
|
UnauthorizedError,
|
|
3647
|
+
UnsupportedFrameEncodingError,
|
|
2906
3648
|
VersionMismatchError,
|
|
2907
3649
|
WebcamStream,
|
|
2908
3650
|
createPlayableManifestUrl,
|
|
@@ -2911,6 +3653,7 @@ function defaultLabel(state) {
|
|
|
2911
3653
|
isFileRef,
|
|
2912
3654
|
normalizeJwtSource,
|
|
2913
3655
|
parsePlaylist,
|
|
3656
|
+
stepFrameToImageData,
|
|
2914
3657
|
useClipDownload,
|
|
2915
3658
|
useReactor,
|
|
2916
3659
|
useReactorInternalMessage,
|