@reactor-team/js-sdk 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -0
- package/dist/index.cjs +481 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +305 -5
- package/dist/index.d.ts +305 -5
- package/dist/index.js +472 -9
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +35 -2
- package/dist/wasm/reactor_wasm.d.ts +169 -2
- package/dist/wasm/reactor_wasm.js +238 -8
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +5 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -44,6 +44,14 @@ var ReactorError = class extends Error {
|
|
|
44
44
|
operation;
|
|
45
45
|
/** Backoff hint the platform sent. */
|
|
46
46
|
retry_after_ms;
|
|
47
|
+
/** For a refused step or a step whose call failed, the part of the step
|
|
48
|
+
* request at fault: `"state.<key>"`, `"commands[<i>]"` or
|
|
49
|
+
* `"media_input[<i>]"`. */
|
|
50
|
+
entry;
|
|
51
|
+
/** For a step whose call failed, the stable code its handler raised (its
|
|
52
|
+
* `CommandError` code, or `internal_error`), to branch on instead of the
|
|
53
|
+
* message. */
|
|
54
|
+
command_code;
|
|
47
55
|
/** When `reactor-core` reported this failure. For a purely local one (no
|
|
48
56
|
* payload at all — see {@link ReactorErrorOptions}), this is synthesized
|
|
49
57
|
* as the construction time instead, since there's nothing to report —
|
|
@@ -64,6 +72,8 @@ var ReactorError = class extends Error {
|
|
|
64
72
|
this.operation = options.operation;
|
|
65
73
|
this.retry_after_ms = options.retry_after_ms;
|
|
66
74
|
this.retryAfter = options.retry_after_ms;
|
|
75
|
+
this.entry = options.entry;
|
|
76
|
+
this.command_code = options.command_code;
|
|
67
77
|
this.timestamp_ms = options.timestamp_ms ?? Date.now();
|
|
68
78
|
this.timestamp = this.timestamp_ms;
|
|
69
79
|
}
|
|
@@ -119,6 +129,30 @@ var AbortedError = class extends ReactorError {
|
|
|
119
129
|
var RecorderDisabledError = class extends ReactorError {
|
|
120
130
|
static code = "RECORDER_DISABLED";
|
|
121
131
|
};
|
|
132
|
+
var GenerationModeMismatchError = class extends ReactorError {
|
|
133
|
+
static code = "GENERATION_MODE_MISMATCH";
|
|
134
|
+
};
|
|
135
|
+
var DataChannelChunkingRequiredError = class extends ReactorError {
|
|
136
|
+
static code = "DATA_CHANNEL_CHUNKING_REQUIRED";
|
|
137
|
+
};
|
|
138
|
+
var StepRefusedError = class extends ReactorError {
|
|
139
|
+
static code = "step_refused";
|
|
140
|
+
};
|
|
141
|
+
var StepNotReadyError = class extends ReactorError {
|
|
142
|
+
static code = "step_not_ready";
|
|
143
|
+
};
|
|
144
|
+
var StepResultTooLargeError = class extends ReactorError {
|
|
145
|
+
static code = "step_result_too_large";
|
|
146
|
+
};
|
|
147
|
+
var StepCommandFailedError = class extends ReactorError {
|
|
148
|
+
static code = "step_command_failed";
|
|
149
|
+
};
|
|
150
|
+
var StepSessionEndedError = class extends ReactorError {
|
|
151
|
+
static code = "step_session_ended";
|
|
152
|
+
};
|
|
153
|
+
var UnsupportedFrameEncodingError = class extends ReactorError {
|
|
154
|
+
static code = "UNSUPPORTED_FRAME_ENCODING";
|
|
155
|
+
};
|
|
122
156
|
var ERROR_CLASSES = [
|
|
123
157
|
NetworkError,
|
|
124
158
|
UnauthorizedError,
|
|
@@ -136,7 +170,15 @@ var ERROR_CLASSES = [
|
|
|
136
170
|
DisconnectedError,
|
|
137
171
|
RequestTimeoutError,
|
|
138
172
|
AbortedError,
|
|
139
|
-
RecorderDisabledError
|
|
173
|
+
RecorderDisabledError,
|
|
174
|
+
GenerationModeMismatchError,
|
|
175
|
+
DataChannelChunkingRequiredError,
|
|
176
|
+
StepRefusedError,
|
|
177
|
+
StepNotReadyError,
|
|
178
|
+
StepResultTooLargeError,
|
|
179
|
+
StepCommandFailedError,
|
|
180
|
+
StepSessionEndedError,
|
|
181
|
+
UnsupportedFrameEncodingError
|
|
140
182
|
];
|
|
141
183
|
var BY_CODE = new Map(
|
|
142
184
|
ERROR_CLASSES.map((errorClass) => [errorClass.code, errorClass])
|
|
@@ -158,6 +200,8 @@ function toReactorError(cause) {
|
|
|
158
200
|
operation: payload.operation,
|
|
159
201
|
retry_after_ms: payload.retry_after_ms,
|
|
160
202
|
timestamp_ms: payload.timestamp_ms,
|
|
203
|
+
entry: payload.entry,
|
|
204
|
+
command_code: payload.command_code,
|
|
161
205
|
// A thrown JS error (a failed wasm load, say) keeps its stack and its own
|
|
162
206
|
// `cause` reachable; reactor-core's plain payload objects have neither.
|
|
163
207
|
cause: cause instanceof Error ? cause : void 0
|
|
@@ -295,6 +339,112 @@ function isSerializedAsObject(value) {
|
|
|
295
339
|
return typeof value === "object" && value !== null && !ArrayBuffer.isView(value);
|
|
296
340
|
}
|
|
297
341
|
|
|
342
|
+
// src/internal/push-frames.ts
|
|
343
|
+
function isAudioFrame(frame) {
|
|
344
|
+
return "samples" in frame;
|
|
345
|
+
}
|
|
346
|
+
function isVideoFrame(frame) {
|
|
347
|
+
return typeof frame.copyTo === "function" && "displayWidth" in frame;
|
|
348
|
+
}
|
|
349
|
+
function holdFrame(frame) {
|
|
350
|
+
if (!isAudioFrame(frame) && isVideoFrame(frame)) {
|
|
351
|
+
const clone = frame.clone();
|
|
352
|
+
return { frame: clone, release: () => clone.close() };
|
|
353
|
+
}
|
|
354
|
+
return { frame, release: () => {
|
|
355
|
+
} };
|
|
356
|
+
}
|
|
357
|
+
async function toPixels(frame) {
|
|
358
|
+
if (isVideoFrame(frame)) {
|
|
359
|
+
return videoFramePixels(frame);
|
|
360
|
+
}
|
|
361
|
+
return rawPixels(frame);
|
|
362
|
+
}
|
|
363
|
+
function rawPixels(frame) {
|
|
364
|
+
const { data, width, height } = frame;
|
|
365
|
+
const bytes = data instanceof Uint8Array ? data : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
366
|
+
return { data: bytes, width, height, format: "format" in frame && frame.format || "rgba" };
|
|
367
|
+
}
|
|
368
|
+
var NATIVE_FORMATS = {
|
|
369
|
+
RGBA: "rgba",
|
|
370
|
+
RGBX: "rgba",
|
|
371
|
+
BGRA: "bgra",
|
|
372
|
+
BGRX: "bgra"
|
|
373
|
+
};
|
|
374
|
+
function orientationOf(frame) {
|
|
375
|
+
const { rotation, flip } = frame;
|
|
376
|
+
return { rotation: ((rotation ?? 0) % 360 + 360) % 360, flip: flip ?? false };
|
|
377
|
+
}
|
|
378
|
+
async function videoFramePixels(frame) {
|
|
379
|
+
const rect = frame.visibleRect;
|
|
380
|
+
const width = rect?.width ?? frame.codedWidth;
|
|
381
|
+
const height = rect?.height ?? frame.codedHeight;
|
|
382
|
+
const native = frame.format ? NATIVE_FORMATS[frame.format] : void 0;
|
|
383
|
+
const orientation = orientationOf(frame);
|
|
384
|
+
if (native) {
|
|
385
|
+
const data = new Uint8Array(width * height * 4);
|
|
386
|
+
await frame.copyTo(data, { layout: [{ offset: 0, stride: width * 4 }] });
|
|
387
|
+
return orient({ data, width, height, format: native }, orientation);
|
|
388
|
+
}
|
|
389
|
+
try {
|
|
390
|
+
const data = new Uint8Array(width * height * 4);
|
|
391
|
+
const options = { format: "RGBA", layout: [{ offset: 0, stride: width * 4 }] };
|
|
392
|
+
await frame.copyTo(data, options);
|
|
393
|
+
return orient({ data, width, height, format: "rgba" }, orientation);
|
|
394
|
+
} catch {
|
|
395
|
+
return canvasPixels(frame, width, height, orientation);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
function orient(pixels, { rotation, flip }) {
|
|
399
|
+
if (rotation % 90 !== 0 || rotation === 0 && !flip) {
|
|
400
|
+
return pixels;
|
|
401
|
+
}
|
|
402
|
+
const { width: w, height: h } = pixels;
|
|
403
|
+
const turned = rotation === 90 || rotation === 270;
|
|
404
|
+
const outWidth = turned ? h : w;
|
|
405
|
+
const outHeight = turned ? w : h;
|
|
406
|
+
const bytes = pixels.data.byteOffset % 4 === 0 ? pixels.data : pixels.data.slice();
|
|
407
|
+
const source = new Uint32Array(bytes.buffer, bytes.byteOffset, w * h);
|
|
408
|
+
const data = new Uint8Array(outWidth * outHeight * 4);
|
|
409
|
+
const target = new Uint32Array(data.buffer);
|
|
410
|
+
for (let y = 0; y < h; y++) {
|
|
411
|
+
for (let x = 0; x < w; x++) {
|
|
412
|
+
let tx;
|
|
413
|
+
let ty;
|
|
414
|
+
if (rotation === 90) {
|
|
415
|
+
[tx, ty] = [h - 1 - y, x];
|
|
416
|
+
} else if (rotation === 180) {
|
|
417
|
+
[tx, ty] = [w - 1 - x, h - 1 - y];
|
|
418
|
+
} else if (rotation === 270) {
|
|
419
|
+
[tx, ty] = [y, w - 1 - x];
|
|
420
|
+
} else {
|
|
421
|
+
[tx, ty] = [x, y];
|
|
422
|
+
}
|
|
423
|
+
if (flip) {
|
|
424
|
+
tx = outWidth - 1 - tx;
|
|
425
|
+
}
|
|
426
|
+
target[ty * outWidth + tx] = source[y * w + x];
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
return { data, width: outWidth, height: outHeight, format: pixels.format };
|
|
430
|
+
}
|
|
431
|
+
function canvasPixels(frame, visibleWidth, visibleHeight, { rotation }) {
|
|
432
|
+
const turned = rotation === 90 || rotation === 270;
|
|
433
|
+
const width = turned ? visibleHeight : visibleWidth;
|
|
434
|
+
const height = turned ? visibleWidth : visibleHeight;
|
|
435
|
+
if (typeof OffscreenCanvas === "undefined") {
|
|
436
|
+
throw new Error(
|
|
437
|
+
`pushFrame: cannot read a ${frame.format ?? "formatless"} VideoFrame here \u2014 push its pixels as an ImageData or a Uint8Array instead.`
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
const context = new OffscreenCanvas(width, height).getContext("2d");
|
|
441
|
+
if (!context) {
|
|
442
|
+
throw new Error("pushFrame: no 2D canvas context to read the VideoFrame through.");
|
|
443
|
+
}
|
|
444
|
+
context.drawImage(frame, 0, 0, width, height);
|
|
445
|
+
return rawPixels(context.getImageData(0, 0, width, height));
|
|
446
|
+
}
|
|
447
|
+
|
|
298
448
|
// src/internal/recording.ts
|
|
299
449
|
function toPublicClip(clip) {
|
|
300
450
|
return {
|
|
@@ -357,7 +507,28 @@ var WIRE_CODECS = {
|
|
|
357
507
|
};
|
|
358
508
|
function createTrackStatsExtractor() {
|
|
359
509
|
const lastBytes = /* @__PURE__ */ new Map();
|
|
360
|
-
|
|
510
|
+
const lastStages = /* @__PURE__ */ new Map();
|
|
511
|
+
let pendingBytes = /* @__PURE__ */ new Map();
|
|
512
|
+
let pendingStages = /* @__PURE__ */ new Map();
|
|
513
|
+
const stage = (streamId, name, seconds, frames) => {
|
|
514
|
+
if (seconds === void 0 || frames === void 0) {
|
|
515
|
+
return void 0;
|
|
516
|
+
}
|
|
517
|
+
const key = `${streamId}:${name}`;
|
|
518
|
+
if (!Number.isFinite(seconds)) {
|
|
519
|
+
pendingStages.set(key, void 0);
|
|
520
|
+
return void 0;
|
|
521
|
+
}
|
|
522
|
+
pendingStages.set(key, { seconds, frames });
|
|
523
|
+
const before = lastStages.get(key);
|
|
524
|
+
if (before === void 0 || frames <= before.frames || seconds < before.seconds) {
|
|
525
|
+
return void 0;
|
|
526
|
+
}
|
|
527
|
+
return { name, totalMs: (seconds - before.seconds) * 1e3, frames: frames - before.frames };
|
|
528
|
+
};
|
|
529
|
+
const extract = (report, tracks, pausedTracks) => {
|
|
530
|
+
pendingBytes = /* @__PURE__ */ new Map();
|
|
531
|
+
pendingStages = /* @__PURE__ */ new Map();
|
|
361
532
|
const reportWithLookup = report;
|
|
362
533
|
const stats = [];
|
|
363
534
|
const seenStreams = /* @__PURE__ */ new Set();
|
|
@@ -396,7 +567,7 @@ function createTrackStatsExtractor() {
|
|
|
396
567
|
if (previous !== void 0 && stat.timestamp > previous.timestamp && bytes >= previous.bytes) {
|
|
397
568
|
put("bitrate_bps", Math.round((bytes - previous.bytes) * 8 * 1e3 / (stat.timestamp - previous.timestamp)));
|
|
398
569
|
}
|
|
399
|
-
|
|
570
|
+
pendingBytes.set(stat.id, { bytes, timestamp: stat.timestamp });
|
|
400
571
|
}
|
|
401
572
|
const keyframeRequests = stat.firCount !== void 0 || stat.pliCount !== void 0 ? (stat.firCount ?? 0) + (stat.pliCount ?? 0) : void 0;
|
|
402
573
|
if (inbound) {
|
|
@@ -431,6 +602,20 @@ function createTrackStatsExtractor() {
|
|
|
431
602
|
put("keyframe_requests", keyframeRequests);
|
|
432
603
|
}
|
|
433
604
|
}
|
|
605
|
+
const frameStages = [];
|
|
606
|
+
if (track.kind === "video") {
|
|
607
|
+
const stages = inbound ? [
|
|
608
|
+
stage(stat.id, "jitter_buffer", stat.jitterBufferDelay, stat.jitterBufferEmittedCount),
|
|
609
|
+
stage(stat.id, "decode", stat.totalDecodeTime, stat.framesDecoded)
|
|
610
|
+
] : [stage(stat.id, "encode", stat.totalEncodeTime, stat.framesEncoded)];
|
|
611
|
+
for (const s of stages) {
|
|
612
|
+
if (s !== void 0) {
|
|
613
|
+
frameStages.push(s);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
const codecImplementation = inbound ? stat.decoderImplementation : stat.encoderImplementation;
|
|
618
|
+
const hardwareCodec = inbound ? stat.powerEfficientDecoder : stat.powerEfficientEncoder;
|
|
434
619
|
stats.push({
|
|
435
620
|
timestamp: Date.now(),
|
|
436
621
|
trackName: track.name,
|
|
@@ -438,11 +623,30 @@ function createTrackStatsExtractor() {
|
|
|
438
623
|
direction,
|
|
439
624
|
codec,
|
|
440
625
|
paused: pausedTracks.includes(track.name),
|
|
441
|
-
metrics
|
|
626
|
+
metrics,
|
|
627
|
+
...frameStages.length > 0 ? { frameStages } : {},
|
|
628
|
+
...track.kind === "video" && codecImplementation !== void 0 ? { codecImplementation } : {},
|
|
629
|
+
...track.kind === "video" && hardwareCodec !== void 0 ? { hardwareCodec } : {}
|
|
442
630
|
});
|
|
443
631
|
});
|
|
444
632
|
return stats;
|
|
445
633
|
};
|
|
634
|
+
return Object.assign(extract, {
|
|
635
|
+
commit() {
|
|
636
|
+
for (const [id, sample] of pendingBytes) {
|
|
637
|
+
lastBytes.set(id, sample);
|
|
638
|
+
}
|
|
639
|
+
for (const [key, total] of pendingStages) {
|
|
640
|
+
if (total === void 0) {
|
|
641
|
+
lastStages.delete(key);
|
|
642
|
+
} else {
|
|
643
|
+
lastStages.set(key, total);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
pendingBytes = /* @__PURE__ */ new Map();
|
|
647
|
+
pendingStages = /* @__PURE__ */ new Map();
|
|
648
|
+
}
|
|
649
|
+
});
|
|
446
650
|
}
|
|
447
651
|
function toClientConnectionStat(report, timeToConnectMs) {
|
|
448
652
|
const pair = activeCandidatePair(report);
|
|
@@ -581,7 +785,7 @@ function describeCause(cause) {
|
|
|
581
785
|
// package.json
|
|
582
786
|
var package_default = {
|
|
583
787
|
name: "@reactor-team/js-sdk",
|
|
584
|
-
version: "3.
|
|
788
|
+
version: "3.3.0",
|
|
585
789
|
description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
|
|
586
790
|
license: "Apache-2.0",
|
|
587
791
|
repository: {
|
|
@@ -1095,10 +1299,18 @@ var Reactor = class {
|
|
|
1095
1299
|
* with it, so a batch sent before the timings exist doesn't use it up.
|
|
1096
1300
|
* Reset per `startStatsPolling()` call, i.e. per connect()/reconnect(). */
|
|
1097
1301
|
hasSentConnectTime = false;
|
|
1302
|
+
/** Whether this connection has sent a client-stats batch yet. The first
|
|
1303
|
+
* one goes out even with no reading, for the environment the bridge adds
|
|
1304
|
+
* to it. Reset with `hasSentConnectTime`. */
|
|
1305
|
+
hasSentClientStats = false;
|
|
1098
1306
|
/** Set on the "connecting" status transition, cleared once `connectionTimings`
|
|
1099
1307
|
* is finalized on "ready" — see `handleStatusChanged()`. */
|
|
1100
1308
|
connectStartTime;
|
|
1101
1309
|
waitingStartTime;
|
|
1310
|
+
/** The tail of every `pushFrame()` so far, settled either way — pushes
|
|
1311
|
+
* reach the binding in call order however long each one's pixels take to
|
|
1312
|
+
* read, and `step()` waits for the ones called before it. */
|
|
1313
|
+
pendingPushes = Promise.resolve();
|
|
1102
1314
|
emitter = new Emitter();
|
|
1103
1315
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
1104
1316
|
* disconnect()/[Symbol.dispose]) against each other. Calling into the
|
|
@@ -1116,6 +1328,15 @@ var Reactor = class {
|
|
|
1116
1328
|
* out every read active when it started, before it releases the ones
|
|
1117
1329
|
* that arrived while it ran. */
|
|
1118
1330
|
writerActive = false;
|
|
1331
|
+
/** Counts `disconnect()`s and disposal. A push captures it when called, and
|
|
1332
|
+
* one whose frame was still being read when it moved is dropped, rather
|
|
1333
|
+
* than recreating a client to carry the frame into the next session. */
|
|
1334
|
+
connectionEpoch = 0;
|
|
1335
|
+
/** Input tracks published on this connection, each with the publication
|
|
1336
|
+
* that claimed it: a push needs one, and a `PublishedTrack` only feeds the
|
|
1337
|
+
* publication it came from. Cleared with the connection. */
|
|
1338
|
+
publishedInputs = /* @__PURE__ */ new Map();
|
|
1339
|
+
publications = 0;
|
|
1119
1340
|
/** Readers parked in `withReaderAccess()`'s wait loop, woken once
|
|
1120
1341
|
* `withWriterAccess()` finishes; each re-checks `writerActive` rather
|
|
1121
1342
|
* than assuming its turn, since another writer already queued behind the
|
|
@@ -1388,14 +1609,49 @@ var Reactor = class {
|
|
|
1388
1609
|
* disconnect() freeing the wasm client mid-await would otherwise resume
|
|
1389
1610
|
* into a freed client — the same class of use-after-free `queue` exists to
|
|
1390
1611
|
* prevent for connect/disconnect/reconnect themselves.
|
|
1612
|
+
*
|
|
1613
|
+
* Resolves with a `PublishedTrack` handle whose `pushFrame()` feeds the
|
|
1614
|
+
* track in a stepped session.
|
|
1615
|
+
*
|
|
1616
|
+
* In a `generation: "stepped"` session, input frames never travel over
|
|
1617
|
+
* RTP: `track` is optional, and the frames are pushed with
|
|
1618
|
+
* `published.pushFrame(...)` for the next `step()`. A `MediaStreamTrack`
|
|
1619
|
+
* passed there is ignored, with a console warning — nothing is sent from
|
|
1620
|
+
* it. In a streamed session `track` is required, and omitting it rejects
|
|
1621
|
+
* with an `InvalidStateError`.
|
|
1622
|
+
*
|
|
1623
|
+
* ```ts
|
|
1624
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1625
|
+
* await camera.pushFrame(imageData);
|
|
1626
|
+
* const result = await reactor.step();
|
|
1627
|
+
* ```
|
|
1391
1628
|
*/
|
|
1392
1629
|
async publishTrack(name, track) {
|
|
1393
1630
|
this.assertNotDisposed();
|
|
1394
1631
|
try {
|
|
1632
|
+
let media = track;
|
|
1633
|
+
if (this.isStepped() && media !== void 0) {
|
|
1634
|
+
console.warn(
|
|
1635
|
+
`[Reactor] publishTrack('${name}'): ignoring the MediaStreamTrack \u2014 a stepped session sends no RTP and takes frames through track.pushFrame(...) on the track publishTrack() returns.`
|
|
1636
|
+
);
|
|
1637
|
+
media = void 0;
|
|
1638
|
+
}
|
|
1639
|
+
if (!this.isStepped() && media === void 0) {
|
|
1640
|
+
throw new InvalidStateError(
|
|
1641
|
+
`publishTrack('${name}') needs a MediaStreamTrack in a streamed session.`,
|
|
1642
|
+
{ operation: "publishTrack" }
|
|
1643
|
+
);
|
|
1644
|
+
}
|
|
1645
|
+
const publication = this.publications += 1;
|
|
1395
1646
|
await this.queue.push(async () => {
|
|
1396
1647
|
const client = await this.getOrCreateClient();
|
|
1397
|
-
await client.publishTrack(name,
|
|
1648
|
+
await client.publishTrack(name, media);
|
|
1649
|
+
this.publishedInputs.set(name, publication);
|
|
1398
1650
|
}, "publishTrack");
|
|
1651
|
+
return {
|
|
1652
|
+
name,
|
|
1653
|
+
pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
|
|
1654
|
+
};
|
|
1399
1655
|
} catch (cause) {
|
|
1400
1656
|
throw this.captureError(cause);
|
|
1401
1657
|
}
|
|
@@ -1412,6 +1668,7 @@ var Reactor = class {
|
|
|
1412
1668
|
await this.queue.push(async () => {
|
|
1413
1669
|
const client = await this.getOrCreateClient();
|
|
1414
1670
|
await client.unpublishTrack(name);
|
|
1671
|
+
this.publishedInputs.delete(name);
|
|
1415
1672
|
}, "unpublishTrack");
|
|
1416
1673
|
} catch (cause) {
|
|
1417
1674
|
this.emitError(cause);
|
|
@@ -1484,6 +1741,175 @@ var Reactor = class {
|
|
|
1484
1741
|
throw this.captureError(cause);
|
|
1485
1742
|
}
|
|
1486
1743
|
}
|
|
1744
|
+
// ── Stepped generation ──────────────────────────────────────────────────
|
|
1745
|
+
/**
|
|
1746
|
+
* Runs one step of a `generation: "stepped"` session and resolves with
|
|
1747
|
+
* what it produced, per output track.
|
|
1748
|
+
*
|
|
1749
|
+
* The request carries `state`, `commands` and every frame pushed since the
|
|
1750
|
+
* previous step. The runtime applies them in that order — each `state` key
|
|
1751
|
+
* as the model's `set_<key>` command, then `commands`, then the frames —
|
|
1752
|
+
* and then generates. They are separate calls, not a transaction: one that
|
|
1753
|
+
* fails does not undo the ones before it. All of it is optional:
|
|
1754
|
+
* `step()` with nothing set and nothing pushed runs on the model's current
|
|
1755
|
+
* state, which commands sent outside a step (`sendCommand()`) also change.
|
|
1756
|
+
* A file goes in the way it does for a command: upload it with
|
|
1757
|
+
* `uploadFile()`, then put the `FileRef` in a command's `data` or a state
|
|
1758
|
+
* value.
|
|
1759
|
+
*
|
|
1760
|
+
* A step that ran but that the model reported as failed still resolves,
|
|
1761
|
+
* with `result.error` set. `result.files` lists the files the model kept
|
|
1762
|
+
* with the step (name, content type, size); their bytes are served over
|
|
1763
|
+
* HTTP from the session's step results, not carried here.
|
|
1764
|
+
*
|
|
1765
|
+
* Rejects with a `StepRefusedError` when the runtime or the model turns
|
|
1766
|
+
* the step down (the runtime checks a whole request first, so nothing it
|
|
1767
|
+
* carried was applied), `StepNotReadyError` when the session cannot take a
|
|
1768
|
+
* step yet (neither ran the step), `StepCommandFailedError` when a state key
|
|
1769
|
+
* or command failed (the calls before it stay applied; the step did not
|
|
1770
|
+
* run); both name the part at fault in `entry`, and `StepCommandFailedError`
|
|
1771
|
+
* the handler's code in `command_code`,
|
|
1772
|
+
* `StepSessionEndedError` when the session ended before the result came
|
|
1773
|
+
* back, `StepResultTooLargeError` when the result does not fit in one
|
|
1774
|
+
* message, `MessageTooLargeError` when the request itself is too large
|
|
1775
|
+
* (nothing was sent), `InvalidStateError` outside a ready stepped
|
|
1776
|
+
* session, or whatever else failed the step (`DisconnectedError`,
|
|
1777
|
+
* `RequestTimeoutError`, ...). The frames pushed for a step belong to it:
|
|
1778
|
+
* when its request cannot be sent, they are dropped with it rather than
|
|
1779
|
+
* kept for the next step.
|
|
1780
|
+
*
|
|
1781
|
+
* ```ts
|
|
1782
|
+
* const camera = await reactor.publishTrack('camera');
|
|
1783
|
+
* await camera.pushFrame(imageData);
|
|
1784
|
+
* const result = await reactor.step({ state: { prompt: 'a cat' } });
|
|
1785
|
+
* if (result.error) console.warn('step failed:', result.error);
|
|
1786
|
+
* const frame = result.tracks.find((t) => t.track === 'main_video')?.frames[0];
|
|
1787
|
+
* if (frame) ctx.putImageData(stepFrameToImageData(frame), 0, 0);
|
|
1788
|
+
* ```
|
|
1789
|
+
*/
|
|
1790
|
+
async step(options = {}) {
|
|
1791
|
+
this.assertNotDisposed();
|
|
1792
|
+
try {
|
|
1793
|
+
const state = rewriteNestedFileRefs(options.state ?? null);
|
|
1794
|
+
const commands = (options.commands ?? []).map(
|
|
1795
|
+
(command) => command.data ? { ...command, data: rewriteNestedFileRefs(command.data) } : command
|
|
1796
|
+
);
|
|
1797
|
+
await this.pendingPushes;
|
|
1798
|
+
return await this.withReaderAccess(async () => {
|
|
1799
|
+
const client = await this.getOrCreateClient();
|
|
1800
|
+
return client.step(state, commands);
|
|
1801
|
+
});
|
|
1802
|
+
} catch (cause) {
|
|
1803
|
+
throw this.captureError(cause);
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
/**
|
|
1807
|
+
* Buffers one input frame for the next `step()` on the sendonly track
|
|
1808
|
+
* `trackName`. Nothing is sent until that step, which carries every frame
|
|
1809
|
+
* pushed since the one before it, per track in push order. The
|
|
1810
|
+
* `PublishedTrack` that `publishTrack()` resolves with does the same
|
|
1811
|
+
* through `track.pushFrame(frame)`.
|
|
1812
|
+
*
|
|
1813
|
+
* Video: a `VideoFrame` (cloned when called, so the caller may close it as
|
|
1814
|
+
* soon as this returns; it stays the caller's), an
|
|
1815
|
+
* `ImageData`, or `{ data, width, height, format? }` raw pixels (`"rgba"`
|
|
1816
|
+
* by default, or `"rgb"`/`"bgra"`). Sent to the model as RGB. A
|
|
1817
|
+
* `VideoFrame` is sent as it is displayed: its `rotation` and `flip` are
|
|
1818
|
+
* applied, so a 90° frame arrives with its width and height swapped.
|
|
1819
|
+
* Audio: `{ samples, sampleRate, channels? }`, interleaved `Int16Array`
|
|
1820
|
+
* PCM at the rate the model's track declares.
|
|
1821
|
+
*
|
|
1822
|
+
* Pushes keep their call order, and a `step()` waits for every push called
|
|
1823
|
+
* before it. A refused push rejects: await it, or `.catch()` it in a frame
|
|
1824
|
+
* loop, or every bad frame is an unhandled rejection. What waits for
|
|
1825
|
+
* the next step is at most one message's worth of frames: pushing faster
|
|
1826
|
+
* than steps run drops the oldest frames, across tracks. Rejects with
|
|
1827
|
+
* an `InvalidStateError` outside a stepped session, for a track not
|
|
1828
|
+
* published on this connection (`publishTrack(trackName)` first), for a
|
|
1829
|
+
* frame whose size does not match its dimensions, or for a name the model
|
|
1830
|
+
* does not declare as a sendonly track of that kind.
|
|
1831
|
+
*/
|
|
1832
|
+
async pushFrame(trackName, frame, options = {}) {
|
|
1833
|
+
return this.pushTo(trackName, void 0, frame, options);
|
|
1834
|
+
}
|
|
1835
|
+
/** `pushFrame()`, for the track's current publication or, from a
|
|
1836
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
1837
|
+
async pushTo(trackName, publication, frame, options = {}) {
|
|
1838
|
+
this.assertNotDisposed();
|
|
1839
|
+
const epoch = this.connectionEpoch;
|
|
1840
|
+
let held;
|
|
1841
|
+
try {
|
|
1842
|
+
held = holdFrame(frame);
|
|
1843
|
+
} catch (cause) {
|
|
1844
|
+
throw this.captureError(cause);
|
|
1845
|
+
}
|
|
1846
|
+
const push = this.pendingPushes.then(async () => {
|
|
1847
|
+
let pixels;
|
|
1848
|
+
try {
|
|
1849
|
+
if (!this.isStepped()) {
|
|
1850
|
+
throw new InvalidStateError(
|
|
1851
|
+
`pushFrame('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
|
|
1852
|
+
{ operation: "pushFrame" }
|
|
1853
|
+
);
|
|
1854
|
+
}
|
|
1855
|
+
pixels = isAudioFrame(held.frame) ? void 0 : await toPixels(held.frame);
|
|
1856
|
+
} finally {
|
|
1857
|
+
held.release();
|
|
1858
|
+
}
|
|
1859
|
+
await this.withReaderAccess(async () => {
|
|
1860
|
+
if (epoch !== this.connectionEpoch) {
|
|
1861
|
+
throw new InvalidStateError(
|
|
1862
|
+
`pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
|
|
1863
|
+
{ operation: "pushFrame" }
|
|
1864
|
+
);
|
|
1865
|
+
}
|
|
1866
|
+
const current = this.publishedInputs.get(trackName);
|
|
1867
|
+
if (current === void 0 || publication !== void 0 && publication !== current) {
|
|
1868
|
+
throw new InvalidStateError(
|
|
1869
|
+
publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
|
|
1870
|
+
{ operation: "pushFrame" }
|
|
1871
|
+
);
|
|
1872
|
+
}
|
|
1873
|
+
const client = await this.getOrCreateClient();
|
|
1874
|
+
if (pixels) {
|
|
1875
|
+
client.pushVideoFrame(
|
|
1876
|
+
trackName,
|
|
1877
|
+
pixels.data,
|
|
1878
|
+
pixels.width,
|
|
1879
|
+
pixels.height,
|
|
1880
|
+
pixels.format,
|
|
1881
|
+
options.metadata ?? null,
|
|
1882
|
+
options.captureTimeUs ?? null
|
|
1883
|
+
);
|
|
1884
|
+
} else if (isAudioFrame(frame)) {
|
|
1885
|
+
client.pushAudioFrame(trackName, frame.samples, frame.sampleRate, frame.channels ?? 1);
|
|
1886
|
+
}
|
|
1887
|
+
});
|
|
1888
|
+
});
|
|
1889
|
+
this.pendingPushes = push.catch(() => void 0);
|
|
1890
|
+
try {
|
|
1891
|
+
await push;
|
|
1892
|
+
} catch (cause) {
|
|
1893
|
+
throw this.captureError(cause);
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
/** Drops every frame buffered for the next step, pushes called before it
|
|
1897
|
+
* included: it waits its turn behind them, as a `step()` does. */
|
|
1898
|
+
async clearStepInputs() {
|
|
1899
|
+
this.assertNotDisposed();
|
|
1900
|
+
const clear = this.pendingPushes.then(
|
|
1901
|
+
() => this.withReaderAccess(() => {
|
|
1902
|
+
this.client?.clearStepInputs();
|
|
1903
|
+
return Promise.resolve();
|
|
1904
|
+
})
|
|
1905
|
+
);
|
|
1906
|
+
this.pendingPushes = clear.catch(() => void 0);
|
|
1907
|
+
try {
|
|
1908
|
+
await clear;
|
|
1909
|
+
} catch (cause) {
|
|
1910
|
+
throw this.captureError(cause);
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1487
1913
|
// ── Recording ───────────────────────────────────────────────────────────
|
|
1488
1914
|
/**
|
|
1489
1915
|
* Requests a clip covering the last `durationSeconds` of the session.
|
|
@@ -1782,6 +2208,7 @@ var Reactor = class {
|
|
|
1782
2208
|
const extractConnectionStats = createConnectionStatsExtractor();
|
|
1783
2209
|
const extractTrackStats = createTrackStatsExtractor();
|
|
1784
2210
|
this.hasSentConnectTime = false;
|
|
2211
|
+
this.hasSentClientStats = false;
|
|
1785
2212
|
this.stopStatsPollers = [
|
|
1786
2213
|
pollPeerStats(client, LOCAL_STATS_INTERVAL_MS, (report) => {
|
|
1787
2214
|
this.stats = { ...extractConnectionStats(report), connectionTimings: this.connectionTimings };
|
|
@@ -1799,8 +2226,9 @@ var Reactor = class {
|
|
|
1799
2226
|
* track has one yet (none negotiated onto a named track, or no codec
|
|
1800
2227
|
* reported yet), so the runtime gets the bandwidth estimate, the
|
|
1801
2228
|
* round-trip time and the time to connect without waiting on the tracks.
|
|
1802
|
-
* Skipped only when there is nothing at all to say
|
|
1803
|
-
*
|
|
2229
|
+
* Skipped only when there is nothing at all to say and a batch has already
|
|
2230
|
+
* gone out for this connection, so the first one still carries the
|
|
2231
|
+
* environment; one is due again in `RUNTIME_REPORT_INTERVAL_MS`. The time to connect goes on the first batch
|
|
1804
2232
|
* actually sent for this connection — see `hasSentConnectTime`.
|
|
1805
2233
|
* `sendClientStats` throws synchronously on a closing connection; caught
|
|
1806
2234
|
* here for the same reason `getStats()`'s rejection is above. */
|
|
@@ -1808,11 +2236,13 @@ var Reactor = class {
|
|
|
1808
2236
|
const trackStats = extractTrackStats(report, this.trackMapping(), this.pausedTracks());
|
|
1809
2237
|
const timeToConnectMs = this.hasSentConnectTime ? void 0 : this.connectionTimings?.totalMs;
|
|
1810
2238
|
const connectionStat = toClientConnectionStat(report, timeToConnectMs);
|
|
1811
|
-
if (trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
|
|
2239
|
+
if (this.hasSentClientStats && trackStats.length === 0 && Object.keys(connectionStat.metrics).length === 0) {
|
|
1812
2240
|
return;
|
|
1813
2241
|
}
|
|
1814
2242
|
try {
|
|
1815
2243
|
client.sendClientStats(trackStats, connectionStat);
|
|
2244
|
+
extractTrackStats.commit();
|
|
2245
|
+
this.hasSentClientStats = true;
|
|
1816
2246
|
if (timeToConnectMs !== void 0) {
|
|
1817
2247
|
this.hasSentConnectTime = true;
|
|
1818
2248
|
}
|
|
@@ -1834,6 +2264,8 @@ var Reactor = class {
|
|
|
1834
2264
|
* leave `getCapabilities()` returning the previous session's stale
|
|
1835
2265
|
* tracks/commands until a new `capabilitiesReceived` lands. */
|
|
1836
2266
|
resetConnectionState() {
|
|
2267
|
+
this.connectionEpoch += 1;
|
|
2268
|
+
this.publishedInputs.clear();
|
|
1837
2269
|
this.stopStatsPolling();
|
|
1838
2270
|
this.connectionTimings = void 0;
|
|
1839
2271
|
this.connectStartTime = void 0;
|
|
@@ -1854,6 +2286,9 @@ var Reactor = class {
|
|
|
1854
2286
|
emitError(cause) {
|
|
1855
2287
|
this.emitter.emit("error", this.captureError(cause));
|
|
1856
2288
|
}
|
|
2289
|
+
isStepped() {
|
|
2290
|
+
return this.clientOptions.generation === "stepped";
|
|
2291
|
+
}
|
|
1857
2292
|
assertNotDisposed() {
|
|
1858
2293
|
if (this.disposed) {
|
|
1859
2294
|
throw new Error("This Reactor was disposed and can't be used again \u2014 construct a new one.");
|
|
@@ -1861,6 +2296,25 @@ var Reactor = class {
|
|
|
1861
2296
|
}
|
|
1862
2297
|
};
|
|
1863
2298
|
|
|
2299
|
+
// src/step-frames.ts
|
|
2300
|
+
function stepFrameToImageData(frame) {
|
|
2301
|
+
const [height, width, channels] = frame.shape;
|
|
2302
|
+
if (frame.dtype !== "uint8" || frame.shape.length !== 3 || channels !== 3 || height === void 0 || width === void 0 || frame.data.length !== width * height * 3) {
|
|
2303
|
+
throw new Error(
|
|
2304
|
+
`stepFrameToImageData: expected a (height, width, 3) uint8 RGB frame, got ${frame.dtype} [${frame.shape.join(", ")}]`
|
|
2305
|
+
);
|
|
2306
|
+
}
|
|
2307
|
+
const rgb = frame.data;
|
|
2308
|
+
const rgba = new Uint8ClampedArray(width * height * 4);
|
|
2309
|
+
for (let src = 0, dst = 0; src < rgb.length; src += 3, dst += 4) {
|
|
2310
|
+
rgba[dst] = rgb[src];
|
|
2311
|
+
rgba[dst + 1] = rgb[src + 1];
|
|
2312
|
+
rgba[dst + 2] = rgb[src + 2];
|
|
2313
|
+
rgba[dst + 3] = 255;
|
|
2314
|
+
}
|
|
2315
|
+
return new ImageData(rgba, width, height);
|
|
2316
|
+
}
|
|
2317
|
+
|
|
1864
2318
|
// src/jwt.ts
|
|
1865
2319
|
function normalizeJwtSource(source) {
|
|
1866
2320
|
if (typeof source === "function") {
|
|
@@ -2810,9 +3264,11 @@ export {
|
|
|
2810
3264
|
ClipPlayer,
|
|
2811
3265
|
ConflictError,
|
|
2812
3266
|
DEFAULT_PLAYLIST_POLL_SLACK_MS,
|
|
3267
|
+
DataChannelChunkingRequiredError,
|
|
2813
3268
|
DecodeError,
|
|
2814
3269
|
DisconnectedError,
|
|
2815
3270
|
FileRef,
|
|
3271
|
+
GenerationModeMismatchError,
|
|
2816
3272
|
InvalidStateError,
|
|
2817
3273
|
MessageTooLargeError,
|
|
2818
3274
|
NetworkError,
|
|
@@ -2827,8 +3283,14 @@ export {
|
|
|
2827
3283
|
RequestTimeoutError,
|
|
2828
3284
|
ServerError,
|
|
2829
3285
|
SessionTerminalError,
|
|
3286
|
+
StepCommandFailedError,
|
|
3287
|
+
StepNotReadyError,
|
|
3288
|
+
StepRefusedError,
|
|
3289
|
+
StepResultTooLargeError,
|
|
3290
|
+
StepSessionEndedError,
|
|
2830
3291
|
TransportError,
|
|
2831
3292
|
UnauthorizedError,
|
|
3293
|
+
UnsupportedFrameEncodingError,
|
|
2832
3294
|
VersionMismatchError,
|
|
2833
3295
|
WebcamStream,
|
|
2834
3296
|
createPlayableManifestUrl,
|
|
@@ -2837,6 +3299,7 @@ export {
|
|
|
2837
3299
|
isFileRef,
|
|
2838
3300
|
normalizeJwtSource,
|
|
2839
3301
|
parsePlaylist,
|
|
3302
|
+
stepFrameToImageData,
|
|
2840
3303
|
useClipDownload,
|
|
2841
3304
|
useReactor,
|
|
2842
3305
|
useReactorInternalMessage,
|