@reactor-team/js-sdk 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -0
- package/dist/index.cjs +296 -25
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +98 -4
- package/dist/index.d.ts +98 -4
- package/dist/index.js +296 -25
- package/dist/index.js.map +1 -1
- package/dist/wasm/README.md +5 -2
- package/dist/wasm/reactor_wasm.d.ts +61 -1
- package/dist/wasm/reactor_wasm.js +117 -7
- package/dist/wasm/reactor_wasm_bg.wasm.d.ts +6 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -248,6 +248,52 @@ var Emitter = class {
|
|
|
248
248
|
}
|
|
249
249
|
};
|
|
250
250
|
|
|
251
|
+
// src/internal/encoded-frames.ts
|
|
252
|
+
var CODECS = /* @__PURE__ */ new Set(["h264"]);
|
|
253
|
+
function isEncodedFrame(frame) {
|
|
254
|
+
return typeof frame === "object" && frame !== null && "codec" in frame;
|
|
255
|
+
}
|
|
256
|
+
function isChunk(data) {
|
|
257
|
+
return !(data instanceof Uint8Array) && !(data instanceof ArrayBuffer) && typeof data.copyTo === "function";
|
|
258
|
+
}
|
|
259
|
+
function toEncodedBytes(frame, operation, trackName) {
|
|
260
|
+
const refuse = (why) => new InvalidStateError(`${operation}('${trackName}'): ${why}`, { operation });
|
|
261
|
+
if (typeof frame !== "object" || frame === null) {
|
|
262
|
+
throw refuse("expected an EncodedFrame, { codec, data, width, height }");
|
|
263
|
+
}
|
|
264
|
+
const { codec, data, width, height, metadata, captureTimeUs } = frame;
|
|
265
|
+
if (!CODECS.has(codec)) {
|
|
266
|
+
throw refuse(`unknown codec ${JSON.stringify(codec)}; expected "h264"`);
|
|
267
|
+
}
|
|
268
|
+
let bytes;
|
|
269
|
+
if (data instanceof Uint8Array) {
|
|
270
|
+
bytes = data;
|
|
271
|
+
} else if (data instanceof ArrayBuffer) {
|
|
272
|
+
bytes = new Uint8Array(data);
|
|
273
|
+
} else if (data && isChunk(data)) {
|
|
274
|
+
bytes = new Uint8Array(data.byteLength);
|
|
275
|
+
data.copyTo(bytes);
|
|
276
|
+
} else {
|
|
277
|
+
throw refuse("data must be a Uint8Array, an ArrayBuffer or an EncodedVideoChunk");
|
|
278
|
+
}
|
|
279
|
+
if (bytes.byteLength === 0) {
|
|
280
|
+
throw refuse("an encoded frame needs data");
|
|
281
|
+
}
|
|
282
|
+
if (!isDimension(width) || !isDimension(height)) {
|
|
283
|
+
throw refuse(`width and height are the decoded frame's, got ${String(width)}x${String(height)}`);
|
|
284
|
+
}
|
|
285
|
+
if (metadata != null && !(metadata instanceof Uint8Array)) {
|
|
286
|
+
throw refuse("metadata must be a Uint8Array");
|
|
287
|
+
}
|
|
288
|
+
if (captureTimeUs != null && !Number.isFinite(captureTimeUs)) {
|
|
289
|
+
throw refuse("captureTimeUs must be a finite number");
|
|
290
|
+
}
|
|
291
|
+
return { codec, data: bytes, width, height, metadata: metadata ?? null, captureTimeUs: captureTimeUs ?? null };
|
|
292
|
+
}
|
|
293
|
+
function isDimension(value) {
|
|
294
|
+
return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 4294967295;
|
|
295
|
+
}
|
|
296
|
+
|
|
251
297
|
// src/file-ref.ts
|
|
252
298
|
var FileRef = class {
|
|
253
299
|
constructor(uploadId, name, mimeType, size) {
|
|
@@ -458,6 +504,67 @@ function toPublicClip(clip) {
|
|
|
458
504
|
};
|
|
459
505
|
}
|
|
460
506
|
|
|
507
|
+
// src/internal/step-media.ts
|
|
508
|
+
function holdStepMedia(media) {
|
|
509
|
+
if (typeof media !== "object" || media === null || Array.isArray(media)) {
|
|
510
|
+
throw new InvalidStateError("step(): media must map track names to arrays of frames", {
|
|
511
|
+
operation: "step"
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
const held = [];
|
|
515
|
+
try {
|
|
516
|
+
for (const [trackName, frames] of Object.entries(media)) {
|
|
517
|
+
if (!Array.isArray(frames)) {
|
|
518
|
+
throw new InvalidStateError(`step(): media['${trackName}'] must be an array of frames`, {
|
|
519
|
+
operation: "step"
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
for (const frame of frames) {
|
|
523
|
+
held.push(holdStepFrame(trackName, frame));
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
} catch (cause) {
|
|
527
|
+
for (const frame of held) {
|
|
528
|
+
frame.release();
|
|
529
|
+
}
|
|
530
|
+
throw cause;
|
|
531
|
+
}
|
|
532
|
+
return held;
|
|
533
|
+
}
|
|
534
|
+
function holdStepFrame(track, frame) {
|
|
535
|
+
if (isEncodedFrame(frame)) {
|
|
536
|
+
const operation = "pushEncodedFrame";
|
|
537
|
+
return { track, kind: "encoded", operation, encoded: toEncodedBytes(frame, operation, track), release: () => {
|
|
538
|
+
} };
|
|
539
|
+
}
|
|
540
|
+
if (typeof frame !== "object" || frame === null) {
|
|
541
|
+
throw new InvalidStateError(`pushFrame('${track}'): expected a frame`, { operation: "pushFrame" });
|
|
542
|
+
}
|
|
543
|
+
const held = holdFrame(frame);
|
|
544
|
+
return { track, kind: "raw", operation: "pushFrame", frame: held.frame, release: held.release };
|
|
545
|
+
}
|
|
546
|
+
async function readStepFrame(frame) {
|
|
547
|
+
const { track } = frame;
|
|
548
|
+
if (frame.kind === "encoded") {
|
|
549
|
+
return { track, kind: "encoded", operation: frame.operation, encoded: frame.encoded };
|
|
550
|
+
}
|
|
551
|
+
if (isAudioFrame(frame.frame)) {
|
|
552
|
+
return { track, kind: "audio", operation: frame.operation, audio: frame.frame };
|
|
553
|
+
}
|
|
554
|
+
return { track, kind: "video", operation: frame.operation, pixels: await toPixels(frame.frame) };
|
|
555
|
+
}
|
|
556
|
+
function checkStepFrame(client, frame) {
|
|
557
|
+
if (frame.kind === "encoded") {
|
|
558
|
+
const { codec, data, width, height, captureTimeUs } = frame.encoded;
|
|
559
|
+
client.checkEncodedVideoFrame(frame.track, codec, data.byteLength, width, height, captureTimeUs);
|
|
560
|
+
} else if (frame.kind === "video") {
|
|
561
|
+
const { data, width, height, format } = frame.pixels;
|
|
562
|
+
client.checkVideoFrame(frame.track, data.byteLength, width, height, format);
|
|
563
|
+
} else {
|
|
564
|
+
client.checkAudioFrame(frame.track, frame.audio.samples, frame.audio.channels ?? 1);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
|
|
461
568
|
// src/internal/stats.ts
|
|
462
569
|
var LOCAL_STATS_INTERVAL_MS = 2e3;
|
|
463
570
|
var RUNTIME_REPORT_INTERVAL_MS = 5e3;
|
|
@@ -785,7 +892,7 @@ function describeCause(cause) {
|
|
|
785
892
|
// package.json
|
|
786
893
|
var package_default = {
|
|
787
894
|
name: "@reactor-team/js-sdk",
|
|
788
|
-
version: "3.
|
|
895
|
+
version: "3.4.0",
|
|
789
896
|
description: "Connect your JavaScript/TypeScript app to a live Reactor model in the browser.",
|
|
790
897
|
license: "Apache-2.0",
|
|
791
898
|
repository: {
|
|
@@ -1307,9 +1414,10 @@ var Reactor = class {
|
|
|
1307
1414
|
* is finalized on "ready" — see `handleStatusChanged()`. */
|
|
1308
1415
|
connectStartTime;
|
|
1309
1416
|
waitingStartTime;
|
|
1310
|
-
/** The tail of every `pushFrame()` so far, settled
|
|
1311
|
-
* reach the binding in call order however long each
|
|
1312
|
-
* read, and `step()` waits for the ones called
|
|
1417
|
+
/** The tail of every `pushFrame()`/`pushEncodedFrame()` so far, settled
|
|
1418
|
+
* either way — pushes reach the binding in call order however long each
|
|
1419
|
+
* one's pixels take to read, and `step()` waits for the ones called
|
|
1420
|
+
* before it. */
|
|
1313
1421
|
pendingPushes = Promise.resolve();
|
|
1314
1422
|
emitter = new Emitter();
|
|
1315
1423
|
/** Serializes connect()/reconnect()/disconnect() (and the free() inside
|
|
@@ -1650,7 +1758,8 @@ var Reactor = class {
|
|
|
1650
1758
|
}, "publishTrack");
|
|
1651
1759
|
return {
|
|
1652
1760
|
name,
|
|
1653
|
-
pushFrame: (frame, options) => this.pushTo(name, publication, frame, options)
|
|
1761
|
+
pushFrame: (frame, options) => this.pushTo(name, publication, frame, options),
|
|
1762
|
+
pushEncodedFrame: (frame) => this.pushEncodedTo(name, publication, frame)
|
|
1654
1763
|
};
|
|
1655
1764
|
} catch (cause) {
|
|
1656
1765
|
throw this.captureError(cause);
|
|
@@ -1778,6 +1887,20 @@ var Reactor = class {
|
|
|
1778
1887
|
* when its request cannot be sent, they are dropped with it rather than
|
|
1779
1888
|
* kept for the next step.
|
|
1780
1889
|
*
|
|
1890
|
+
* `media` gives the step frames of its own, by input track name, pushed
|
|
1891
|
+
* after the ones already pushed to that track: what `pushFrame()` takes,
|
|
1892
|
+
* or an `EncodedFrame` — H.264 the runtime decodes before the step runs,
|
|
1893
|
+
* so the model reads RGB frames as for a raw push. They are pushed only
|
|
1894
|
+
* once `state` and `commands` are well formed and the session is ready, so
|
|
1895
|
+
* a step refused for those leaves the frames buffered as they were. They go
|
|
1896
|
+
* all or nothing: a bad track or frame anywhere in `media` rejects the step
|
|
1897
|
+
* before any of it is pushed or sent, leaving the buffer as it was. A
|
|
1898
|
+
* session that leaves 'ready' once the media is in has the step refused and
|
|
1899
|
+
* keeps the frames for the next one, as pushed frames are.
|
|
1900
|
+
* A track's encoded frames of one step go in decode order, starting at a
|
|
1901
|
+
* keyframe; a step whose frames do not decode rejects with a
|
|
1902
|
+
* `StepRefusedError`.
|
|
1903
|
+
*
|
|
1781
1904
|
* ```ts
|
|
1782
1905
|
* const camera = await reactor.publishTrack('camera');
|
|
1783
1906
|
* await camera.pushFrame(imageData);
|
|
@@ -1791,14 +1914,31 @@ var Reactor = class {
|
|
|
1791
1914
|
this.assertNotDisposed();
|
|
1792
1915
|
try {
|
|
1793
1916
|
const state = rewriteNestedFileRefs(options.state ?? null);
|
|
1794
|
-
const commands = (options.commands ?? [])
|
|
1795
|
-
|
|
1796
|
-
);
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
return client.step(state, commands);
|
|
1917
|
+
const commands = rewriteNestedFileRefs(options.commands ?? []);
|
|
1918
|
+
const previous = this.pendingPushes;
|
|
1919
|
+
const before = options.media !== void 0 ? this.pushStepMedia(options.media, state, commands) : previous;
|
|
1920
|
+
let taken;
|
|
1921
|
+
const bufferTaken = new Promise((resolve) => {
|
|
1922
|
+
taken = resolve;
|
|
1801
1923
|
});
|
|
1924
|
+
const stepping = before.then(
|
|
1925
|
+
() => (
|
|
1926
|
+
// Excluded from connect()/disconnect()/reconnect() like sendCommand():
|
|
1927
|
+
// a step can take minutes, and must neither be freed out from under
|
|
1928
|
+
// nor block unrelated reads.
|
|
1929
|
+
this.withReaderAccess(async () => {
|
|
1930
|
+
const client = await this.getOrCreateClient();
|
|
1931
|
+
try {
|
|
1932
|
+
return client.step(state, commands);
|
|
1933
|
+
} finally {
|
|
1934
|
+
taken();
|
|
1935
|
+
}
|
|
1936
|
+
})
|
|
1937
|
+
)
|
|
1938
|
+
);
|
|
1939
|
+
stepping.catch(() => previous.then(taken));
|
|
1940
|
+
this.pendingPushes = bufferTaken;
|
|
1941
|
+
return await stepping;
|
|
1802
1942
|
} catch (cause) {
|
|
1803
1943
|
throw this.captureError(cause);
|
|
1804
1944
|
}
|
|
@@ -1857,19 +1997,7 @@ var Reactor = class {
|
|
|
1857
1997
|
held.release();
|
|
1858
1998
|
}
|
|
1859
1999
|
await this.withReaderAccess(async () => {
|
|
1860
|
-
|
|
1861
|
-
throw new InvalidStateError(
|
|
1862
|
-
`pushFrame('${trackName}'): the connection it was pushed to ended while its frame was read`,
|
|
1863
|
-
{ operation: "pushFrame" }
|
|
1864
|
-
);
|
|
1865
|
-
}
|
|
1866
|
-
const current = this.publishedInputs.get(trackName);
|
|
1867
|
-
if (current === void 0 || publication !== void 0 && publication !== current) {
|
|
1868
|
-
throw new InvalidStateError(
|
|
1869
|
-
publication === void 0 ? `pushFrame('${trackName}') needs publishTrack('${trackName}') first.` : `pushFrame('${trackName}'): this track was unpublished, or its connection ended.`,
|
|
1870
|
-
{ operation: "pushFrame" }
|
|
1871
|
-
);
|
|
1872
|
-
}
|
|
2000
|
+
this.checkPushTarget("pushFrame", trackName, publication, epoch);
|
|
1873
2001
|
const client = await this.getOrCreateClient();
|
|
1874
2002
|
if (pixels) {
|
|
1875
2003
|
client.pushVideoFrame(
|
|
@@ -1893,6 +2021,149 @@ var Reactor = class {
|
|
|
1893
2021
|
throw this.captureError(cause);
|
|
1894
2022
|
}
|
|
1895
2023
|
}
|
|
2024
|
+
/**
|
|
2025
|
+
* Buffers one encoded video frame for the next `step()` on the sendonly
|
|
2026
|
+
* video track `trackName`, its bytes as they are: one H.264 access unit
|
|
2027
|
+
* (Annex B), with the `width` and `height` it decodes to. The
|
|
2028
|
+
* `PublishedTrack` that `publishTrack()` resolves with does the same
|
|
2029
|
+
* through `track.pushEncodedFrame(frame)`.
|
|
2030
|
+
*
|
|
2031
|
+
* The runtime decodes a track's encoded frames of one step as one stream,
|
|
2032
|
+
* before the step runs, and the model reads the RGB frames as if they had
|
|
2033
|
+
* been pushed raw. So push them in decode order, starting at a keyframe; a
|
|
2034
|
+
* step whose frames do not decode rejects with a `StepRefusedError`, as
|
|
2035
|
+
* does a runtime that predates encoded frames. Encoded and raw pushes keep
|
|
2036
|
+
* their call order, and a `step()` waits for every push called before it.
|
|
2037
|
+
*
|
|
2038
|
+
* ```ts
|
|
2039
|
+
* const encoder = new VideoEncoder({
|
|
2040
|
+
* output: (chunk) => void camera.pushEncodedFrame({ codec: 'h264', data: chunk, width, height }),
|
|
2041
|
+
* error: console.error,
|
|
2042
|
+
* });
|
|
2043
|
+
* encoder.configure({ codec: 'avc1.42001f', width, height, avc: { format: 'annexb' } });
|
|
2044
|
+
* ```
|
|
2045
|
+
*
|
|
2046
|
+
* Rejects with an `InvalidStateError` outside a stepped session, for a
|
|
2047
|
+
* track not published on this connection, for an audio or recvonly track,
|
|
2048
|
+
* for empty data, a zero dimension or a codec other than `"h264"`.
|
|
2049
|
+
*/
|
|
2050
|
+
async pushEncodedFrame(trackName, frame) {
|
|
2051
|
+
return this.pushEncodedTo(trackName, void 0, frame);
|
|
2052
|
+
}
|
|
2053
|
+
/** `pushEncodedFrame()`, for the track's current publication or, from a
|
|
2054
|
+
* `PublishedTrack`, only for the `publication` it came from. */
|
|
2055
|
+
async pushEncodedTo(trackName, publication, frame) {
|
|
2056
|
+
const operation = "pushEncodedFrame";
|
|
2057
|
+
this.assertNotDisposed();
|
|
2058
|
+
const epoch = this.connectionEpoch;
|
|
2059
|
+
let encoded;
|
|
2060
|
+
try {
|
|
2061
|
+
encoded = toEncodedBytes(frame, operation, trackName);
|
|
2062
|
+
} catch (cause) {
|
|
2063
|
+
throw this.captureError(cause);
|
|
2064
|
+
}
|
|
2065
|
+
const push = this.pendingPushes.then(
|
|
2066
|
+
() => this.withReaderAccess(async () => {
|
|
2067
|
+
if (!this.isStepped()) {
|
|
2068
|
+
throw new InvalidStateError(
|
|
2069
|
+
`${operation}('${trackName}') needs a session created with generation: "stepped"; a streamed session sends media from a MediaStreamTrack given to publishTrack().`,
|
|
2070
|
+
{ operation }
|
|
2071
|
+
);
|
|
2072
|
+
}
|
|
2073
|
+
this.checkPushTarget(operation, trackName, publication, epoch);
|
|
2074
|
+
const client = await this.getOrCreateClient();
|
|
2075
|
+
const declared = client.tracks().find((track) => track.name === trackName);
|
|
2076
|
+
if (declared && (declared.kind !== "video" || declared.direction !== "sendonly")) {
|
|
2077
|
+
throw new InvalidStateError(
|
|
2078
|
+
`${operation}('${trackName}'): '${trackName}' is a ${declared.direction} ${declared.kind} track; only a sendonly video track takes encoded frames.`,
|
|
2079
|
+
{ operation }
|
|
2080
|
+
);
|
|
2081
|
+
}
|
|
2082
|
+
client.pushEncodedVideoFrame(
|
|
2083
|
+
trackName,
|
|
2084
|
+
encoded.codec,
|
|
2085
|
+
encoded.data,
|
|
2086
|
+
encoded.width,
|
|
2087
|
+
encoded.height,
|
|
2088
|
+
encoded.metadata,
|
|
2089
|
+
encoded.captureTimeUs
|
|
2090
|
+
);
|
|
2091
|
+
})
|
|
2092
|
+
);
|
|
2093
|
+
this.pendingPushes = push.catch(() => void 0);
|
|
2094
|
+
try {
|
|
2095
|
+
await push;
|
|
2096
|
+
} catch (cause) {
|
|
2097
|
+
throw this.captureError(cause);
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
/** Refuses a push whose connection ended since it was called, or whose
|
|
2101
|
+
* track is not (or no longer) published on it. */
|
|
2102
|
+
checkPushTarget(operation, trackName, publication, epoch) {
|
|
2103
|
+
if (epoch !== this.connectionEpoch) {
|
|
2104
|
+
throw new InvalidStateError(
|
|
2105
|
+
`${operation}('${trackName}'): the connection it was pushed to ended while its frame was read`,
|
|
2106
|
+
{ operation }
|
|
2107
|
+
);
|
|
2108
|
+
}
|
|
2109
|
+
const current = this.publishedInputs.get(trackName);
|
|
2110
|
+
if (current === void 0 || publication !== void 0 && publication !== current) {
|
|
2111
|
+
throw new InvalidStateError(
|
|
2112
|
+
publication === void 0 ? `${operation}('${trackName}') needs publishTrack('${trackName}') first.` : `${operation}('${trackName}'): this track was unpublished, or its connection ended.`,
|
|
2113
|
+
{ operation }
|
|
2114
|
+
);
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
/** Pushes `step()`'s `media`, track by track and in order, each frame as
|
|
2118
|
+
* `pushFrame()` or `pushEncodedFrame()` would. All or nothing: the binding
|
|
2119
|
+
* checks the step's `state` and `commands`, then every frame, before the
|
|
2120
|
+
* first is pushed, so a refused step, or a bad track or frame anywhere,
|
|
2121
|
+
* leaves the buffer as it was. */
|
|
2122
|
+
async pushStepMedia(media, state, commands) {
|
|
2123
|
+
const epoch = this.connectionEpoch;
|
|
2124
|
+
const held = holdStepMedia(media);
|
|
2125
|
+
const push = this.pendingPushes.then(async () => {
|
|
2126
|
+
const ready = [];
|
|
2127
|
+
try {
|
|
2128
|
+
for (const frame of held) {
|
|
2129
|
+
ready.push(await readStepFrame(frame));
|
|
2130
|
+
}
|
|
2131
|
+
} finally {
|
|
2132
|
+
for (const frame of held) {
|
|
2133
|
+
frame.release();
|
|
2134
|
+
}
|
|
2135
|
+
}
|
|
2136
|
+
await this.withReaderAccess(async () => {
|
|
2137
|
+
const client = await this.getOrCreateClient();
|
|
2138
|
+
client.checkStep(state, commands);
|
|
2139
|
+
for (const frame of ready) {
|
|
2140
|
+
this.checkPushTarget(frame.operation, frame.track, void 0, epoch);
|
|
2141
|
+
checkStepFrame(client, frame);
|
|
2142
|
+
}
|
|
2143
|
+
for (const frame of ready) {
|
|
2144
|
+
if (frame.kind === "encoded") {
|
|
2145
|
+
client.pushEncodedVideoFrame(
|
|
2146
|
+
frame.track,
|
|
2147
|
+
frame.encoded.codec,
|
|
2148
|
+
frame.encoded.data,
|
|
2149
|
+
frame.encoded.width,
|
|
2150
|
+
frame.encoded.height,
|
|
2151
|
+
frame.encoded.metadata,
|
|
2152
|
+
frame.encoded.captureTimeUs
|
|
2153
|
+
);
|
|
2154
|
+
} else if (frame.kind === "video") {
|
|
2155
|
+
const { data, width, height, format } = frame.pixels;
|
|
2156
|
+
client.pushVideoFrame(frame.track, data, width, height, format, null, null);
|
|
2157
|
+
} else {
|
|
2158
|
+
const { samples, sampleRate, channels } = frame.audio;
|
|
2159
|
+
client.pushAudioFrame(frame.track, samples, sampleRate, channels ?? 1);
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2162
|
+
});
|
|
2163
|
+
});
|
|
2164
|
+
this.pendingPushes = push.catch(() => void 0);
|
|
2165
|
+
await push;
|
|
2166
|
+
}
|
|
1896
2167
|
/** Drops every frame buffered for the next step, pushes called before it
|
|
1897
2168
|
* included: it waits its turn behind them, as a `step()` does. */
|
|
1898
2169
|
async clearStepInputs() {
|