@decartai/sdk 0.1.20 → 0.1.22

package/README.md

@@ -88,6 +88,18 @@ A connected realtime session exposes an SDK `subscribeToken` once it reaches a
 connected state. Share that SDK token with viewers — `client.realtime.subscribe`
 uses it to request receive-only LiveKit credentials from Decart, then connects to
 the LiveKit room for the styled output stream. No viewer camera is required.
+Browser-produced streams can include LiveKit frame metadata for glass-to-glass
+latency, so viewers also need a browser runtime that can create the SDK's
+frame-metadata worker; unsupported runtimes reject those tokens before joining.
+
+> **Viewers must be on this SDK version or newer.** A browser publisher now
+> advertises frame timing by default, which makes the server append a packet
+> trailer to every frame in the room. Viewers on an older `@decartai/sdk`
+> ignore the token's `frame_timing` flag and join without a strip worker — they
+> connect successfully but decode nothing, so the video element stays black with
+> no error raised. React Native viewers cannot strip trailers at all and are
+> rejected with `UNSUPPORTED_PLATFORM_FEATURE`. Upgrade viewers before upgrading
+> publishers.
 
 **Producer** — capture the token from the active session:
 
@@ -175,12 +187,12 @@ realtimeClient.getConnectionQuality(); // latest report, or null before the firs
 > setInterval(() => console.log(realtimeClient.getConnectionQuality()?.metrics.g2gMs), 1000);
 > ```
 
-**Glass-to-glass latency (opt-in, diagnostic).** Network RTT hides the dominant cost in
-real-time video — model inference — so a session can read "good" while actually feeling laggy.
-Set `debugQuality: true` to measure the *real* camera→display latency. The SDK attaches a capture
-timestamp using LiveKit frame metadata; the server propagates it through inference and the SDK
-matches it to output playout. This surfaces **startup** (`ttffMs`) and **steady-state**
-(`g2gMs`) latency. When present, glass-to-glass drives the latency verdict instead of RTT.
+**Glass-to-glass latency.** Network RTT hides the dominant cost in real-time video — model
+inference — so a session can read "good" while actually feeling laggy. In browser sessions, the SDK
+automatically attaches a capture timestamp using LiveKit frame metadata when the required worker is
+available; the server propagates it through inference and the SDK matches it to output playout.
+This surfaces **startup** (`ttffMs`) and **steady-state** (`g2gMs`) latency. When present,
+glass-to-glass drives the latency verdict instead of RTT.
 
 > Frame metadata is currently experimental in LiveKit and requires encoded-transform support.
 > It does not alter visible pixels. `g2gDropRatio` remains `null` until frame IDs are propagated
@@ -189,7 +201,6 @@ matches it to output playout. This surfaces **startup** (`ttffMs`) and **steady-
 ```typescript
 const realtimeClient = await client.realtime.connect(stream, {
   model,
-  debugQuality: true,
 });
 
 // g2g updates every stats tick — read it from the `stats` event (the
@@ -338,9 +349,9 @@ Add them to `app.json`:
 Run `npx expo prebuild` and rebuild the native app. LiveKit does not run in Expo
 Go. Bare React Native apps must follow LiveKit's native setup instructions.
 
-Outgoing `mirror`, `debugQuality`, and deep connectivity preflight are browser-only
-and fail with `UNSUPPORTED_PLATFORM_FEATURE` on React Native. Mirror the local
-preview with your native video view instead.
+Outgoing `mirror` and deep connectivity preflight are browser-only and fail with
+`UNSUPPORTED_PLATFORM_FEATURE` on React Native. Mirror the local preview with your
+native video view instead.
 
 ## Development
 

package/dist/package.js

@@ -1,4 +1,4 @@
 //#region package.json
-var version = "0.1.20";
+var version = "0.1.22";
 //#endregion
 export { version };

package/dist/realtime/browser/frame-metadata-diagnostics.js

@@ -52,8 +52,8 @@ const TIME_SYNC_UPDATE = "timeSyncUpdate";
 function createFrameReader(tracker) {
 	let attachedTrack = null;
 	const onTimeSyncUpdate = ({ timestamp, rtpTimestamp }) => {
-		const frameMetadata = attachedTrack?.lookupFrameMetadata({ rtpTimestamp });
-		if (frameMetadata) tracker.recordFrame(frameMetadata.userTimestamp, performance.timeOrigin + timestamp);
+		const frameMetadata = attachedTrack?.lookupFrameMetadata?.({ rtpTimestamp });
+		if (frameMetadata) tracker.recordFrame(frameMetadata.userTimestamp, timestamp);
 	};
 	const detach = () => {
 		attachedTrack?.off(TIME_SYNC_UPDATE, onTimeSyncUpdate);
@@ -83,8 +83,27 @@ function createBrowserFrameMetadataDiagnostics() {
 		dispose: () => reader.dispose()
 	};
 }
+const WORKER_URL = () => new URL("./frame-metadata-worker.js", import.meta.url);
 function createFrameMetadataWorker() {
-	return new Worker(new URL("./frame-metadata-worker.js", import.meta.url));
+	return new Worker(WORKER_URL());
+}
+function isFrameMetadataWorkerSameOrigin(workerUrl, pageOrigin) {
+	if (workerUrl.protocol !== "http:" && workerUrl.protocol !== "https:") return true;
+	return workerUrl.origin === pageOrigin;
+}
+function isFrameMetadataRuntimeSupported() {
+	if (typeof window === "undefined") return false;
+	try {
+		if (!isFrameMetadataWorkerSameOrigin(WORKER_URL(), window.location?.origin)) return false;
+	} catch {
+		return false;
+	}
+	const maybeWindow = window;
+	const userAgent = window.navigator?.userAgent?.toLowerCase() ?? "";
+	const isChromiumBased = /(?:chrome|chromium|crmo)\//.test(userAgent) && !/crios\//.test(userAgent);
+	const scriptTransformSupported = typeof maybeWindow.RTCRtpScriptTransform !== "undefined" && !isChromiumBased;
+	const insertableStreamsSupported = typeof maybeWindow.RTCRtpSender?.prototype?.createEncodedStreams !== "undefined" && typeof maybeWindow.RTCRtpReceiver?.prototype?.createEncodedStreams !== "undefined";
+	return scriptTransformSupported || insertableStreamsSupported;
 }
 //#endregion
-export { FrameMetadataTracker, createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker };
+export { FrameMetadataTracker, createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker, isFrameMetadataRuntimeSupported, isFrameMetadataWorkerSameOrigin };

package/dist/realtime/browser/index.js

@@ -1,5 +1,6 @@
 import { createRealTimeClient } from "../client.js";
 import { createRealTimeSubscribeClient } from "../subscribe-client.js";
+import { createFrameMetadataWorker, isFrameMetadataRuntimeSupported } from "./frame-metadata-diagnostics.js";
 import { createPreflight } from "./preflight.js";
 import { prepareBrowserConnection } from "./prepare-connection.js";
 //#region src/realtime/browser/index.ts
@@ -16,7 +17,9 @@ const createBrowserRealtime = (options) => {
 		baseUrl: options.subscribeBaseUrl,
 		apiKey: options.apiKey,
 		integration: options.integration,
-		logger: options.logger
+		logger: options.logger,
+		createFrameMetadataWorker,
+		isFrameMetadataRuntimeSupported
 	});
 	const preflight = createPreflight({
 		logger: options.logger,

package/dist/realtime/browser/preflight.js

@@ -166,7 +166,6 @@ async function runActiveProbe(args) {
 		source = createSyntheticSource(model.width, model.height, resolveFpsNumber(model.fps));
 		const connectTask = connect(source.stream, {
 			model,
-			debugQuality: true,
 			onRemoteStream: () => {}
 		});
 		signal?.addEventListener("abort", () => connectTask.then((c) => c.disconnect()).catch(() => {}), { once: true });

package/dist/realtime/browser/prepare-connection.js

@@ -1,10 +1,10 @@
 import { RealtimeObservability } from "../observability/realtime-observability.js";
 import { isDesktopSafari } from "../../utils/platform.js";
 import { createLiveKitMediaChannel } from "../media-channel.js";
-import { createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker } from "./frame-metadata-diagnostics.js";
+import { createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker, isFrameMetadataRuntimeSupported } from "./frame-metadata-diagnostics.js";
 import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
 //#region src/realtime/browser/prepare-connection.ts
-const prepareBrowserConnection = ({ stream, mirror, debugQuality, preferredVideoCodec, fps, logger, observability: observabilityOptions }) => {
+const prepareBrowserConnection = ({ stream, mirror, preferredVideoCodec, fps, logger, observability: observabilityOptions }) => {
 	let inputStream = stream ?? new MediaStream();
 	let disposeMirroring = () => {};
 	if (mirror !== false) try {
@@ -18,11 +18,12 @@ const prepareBrowserConnection = ({ stream, mirror, debugQuality, preferredVideo
 		logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
 	}
 	let pendingWorker;
-	if (debugQuality) try {
+	if (isFrameMetadataRuntimeSupported()) try {
 		pendingWorker = createFrameMetadataWorker();
 	} catch (error) {
-		logger.warn("Frame-metadata worker unavailable; glass-to-glass latency measurement disabled", { error: error instanceof Error ? error.message : String(error) });
+		logger.debug("Frame-metadata worker unavailable; glass-to-glass latency measurement disabled", { error: error instanceof Error ? error.message : String(error) });
 	}
+	else logger.debug("Frame-metadata runtime unavailable; glass-to-glass latency measurement disabled");
 	const frameTiming = pendingWorker !== void 0;
 	const takeFrameMetadataWorker = () => {
 		if (pendingWorker) {

package/dist/realtime/client.js

@@ -27,11 +27,9 @@ const realTimeClientConnectOptionsSchema = z.object({
 		"vp9"
 	]).optional(),
 	/**
-	* Opt-in quality measurement using LiveKit frame metadata. The capture
-	* timestamp is propagated through inference and matched to output playout to
-	* surface true glass-to-glass `g2gMs` / `ttffMs` on the `stats` and
-	* `connectionQuality` signals. Browser-only and experimental because it
-	* relies on LiveKit's frame-metadata worker and encoded transforms.
+	* @deprecated Glass-to-glass measurement now runs automatically in browsers
+	* when LiveKit frame metadata is available. This legacy flag is accepted for
+	* compatibility and no longer gates measurement.
 	*/
 	debugQuality: z.boolean().optional()
 });
@@ -43,7 +41,6 @@ const createRealTimeClient = (opts) => {
 		if (!parsedOptions.success) throw parsedOptions.error;
 		const { onRemoteStream, onConnectionChange, onConnectionQuality, onQueuePosition, initialState, resolution, preferredVideoCodec } = parsedOptions.data;
 		const mirror = parsedOptions.data.mirror ?? false;
-		const debugQuality = parsedOptions.data.debugQuality ?? false;
 		let session;
 		let observability;
 		let preparedConnection;
@@ -59,7 +56,6 @@ const createRealTimeClient = (opts) => {
 			preparedConnection = opts.prepareConnection({
 				stream,
 				mirror,
-				debugQuality,
 				preferredVideoCodec,
 				fps: resolveFpsNumber(options.model.fps),
 				logger,

package/dist/realtime/config-realtime.js

@@ -19,7 +19,8 @@ const REALTIME_CONFIG = {
 			"invalid session",
 			"401",
 			"invalid api key",
-			"unauthorized"
+			"unauthorized",
+			"failed to create livekit frame-metadata worker"
 		]
 	},
 	methods: {

package/dist/realtime/media-channel.js

@@ -46,7 +46,9 @@ var LiveKitMediaChannel = class {
 			if (this.config.createFrameMetadataWorker) try {
 				worker = this.config.createFrameMetadataWorker();
 			} catch (error) {
-				this.logger.warn("Failed to create LiveKit frame-metadata worker; continuing without latency metrics", { error: error instanceof Error ? error.message : String(error) });
+				const detail = error instanceof Error ? error.message : String(error);
+				this.logger.warn("Failed to create LiveKit frame-metadata worker", { error: detail });
+				throw new Error(`Failed to create LiveKit frame-metadata worker: ${detail}`, { cause: error });
 			}
 			this.frameMetadataEnabled = worker !== void 0;
 			try {

package/dist/realtime/observability/connection-quality.d.ts

@@ -16,10 +16,10 @@ type ConnectionQualityMetrics = {
   rttMs: number | null;
   /**
    * Mid-stream (steady-state) glass-to-glass latency (ms) — the real per-frame
-   * camera→display latency through the model, excluding startup. Only populated
-   * when the opt-in frame-metadata measurement is on (`connect({ debugQuality: true })`)
-   * and past warm-up; null otherwise. When present it drives the latency verdict
-   * instead of `rttMs`.
+   * camera→display latency through the model, excluding startup. Populated in
+   * browser sessions when LiveKit frame metadata is available and past warm-up;
+   * null otherwise. When present it drives the latency verdict instead of
+   * `rttMs`.
    */
   g2gMs: number | null;
   /**

package/dist/realtime/observability/webrtc-stats.d.ts

@@ -133,9 +133,8 @@ type WebRTCStats = {
   };
   /**
    * True glass-to-glass latency + end-to-end drop signal, merged in by
-   * `RealtimeObservability` when the opt-in frame-metadata measurement is active.
-   * Null otherwise — the stats collector does not
-   * populate it.
+   * `RealtimeObservability` when frame-metadata measurement is active. Null
+   * otherwise — the stats collector does not populate it.
    */
   glassToGlass: G2GMetrics | null;
 };

package/dist/realtime/react-native/prepare-connection.js

@@ -10,10 +10,9 @@ function assertReactNativeReady() {
 function unsupportedReactNativeFeature(feature) {
 	throw createUnsupportedPlatformFeatureError(feature, "React Native");
 }
-const prepareReactNativeConnection = ({ stream, mirror, debugQuality, preferredVideoCodec, observability: observabilityOptions }) => {
+const prepareReactNativeConnection = ({ stream, mirror, preferredVideoCodec, observability: observabilityOptions }) => {
 	assertReactNativeReady();
 	if (mirror !== false) unsupportedReactNativeFeature("Outgoing video mirroring");
-	if (debugQuality) unsupportedReactNativeFeature("debugQuality");
 	return {
 		stream: stream ?? new MediaStream(),
 		observability: new RealtimeObservability(observabilityOptions),

package/dist/realtime/stream-session.js

@@ -4,8 +4,11 @@ import { SignalingChannel } from "./signaling-channel.js";
 import mitt from "mitt";
 import pRetry, { AbortError } from "p-retry";
 //#region src/realtime/stream-session.ts
-function encodeSubscribeToken(roomName) {
-	return btoa(JSON.stringify({ room_name: roomName }));
+function encodeSubscribeToken(roomName, options = {}) {
+	return btoa(JSON.stringify({
+		room_name: roomName,
+		...options.frameTiming ? { frame_timing: true } : {}
+	}));
 }
 function getInitialImageSizeKb(image) {
 	if (!image) return null;
@@ -137,7 +140,7 @@ var StreamSession = class {
 			this.setState("connected");
 			this.events.emit("sessionStarted", {
 				sessionId: roomInfo.sessionId,
-				subscribeToken: encodeSubscribeToken(roomInfo.roomName)
+				subscribeToken: encodeSubscribeToken(roomInfo.roomName, { frameTiming: this.config.frameTiming })
 			});
 		} catch (error) {
 			this.config.observability?.finishConnectionBreakdown({

package/dist/realtime/subscribe-client.js

@@ -1,15 +1,28 @@
-import { classifyWebrtcError } from "../utils/errors.js";
+import { ERROR_CODES, classifyWebrtcError, createSDKError } from "../utils/errors.js";
 import { createConsoleLogger } from "../utils/logger.js";
 import { createEventBuffer } from "./event-buffer.js";
 import { REALTIME_CONFIG } from "./config-realtime.js";
 import { loadLiveKitClient } from "./livekit.js";
 import { RealtimeObservability } from "./observability/realtime-observability.js";
 //#region src/realtime/subscribe-client.ts
+function isDecartSDKError(error) {
+	return typeof error === "object" && error !== null && typeof error.code === "string" && typeof error.message === "string";
+}
+function createFrameMetadataSubscribeUnsupportedError(detail) {
+	const suffix = detail ? `: ${detail}` : "";
+	return createSDKError(ERROR_CODES.UNSUPPORTED_PLATFORM_FEATURE, `This realtime stream requires LiveKit frame metadata, which this SDK environment does not support${suffix}.`, {
+		feature: "LiveKit frame metadata subscribe",
+		platform: "this SDK environment"
+	});
+}
 function decodeSubscribeToken(token) {
 	try {
 		const payload = JSON.parse(atob(token));
 		if (!payload.room_name || typeof payload.room_name !== "string") throw new Error("Invalid subscribe token format");
-		return { room_name: payload.room_name };
+		return {
+			room_name: payload.room_name,
+			...payload.frame_timing === true ? { frame_timing: true } : {}
+		};
 	} catch {
 		throw new Error("Invalid subscribe token");
 	}
@@ -50,10 +63,11 @@ const createRealTimeSubscribeClient = (opts) => {
 	const { baseUrl, apiKey, integration } = opts;
 	const logger = opts.logger ?? createConsoleLogger("info");
 	const subscribe = async (options) => {
-		const { room_name: roomName } = decodeSubscribeToken(options.token);
+		const { room_name: roomName, frame_timing: frameTiming } = decodeSubscribeToken(options.token);
 		const { emitter, emitOrBuffer, flush, stop } = createEventBuffer();
 		let observability;
 		let room;
+		let frameMetadataWorker;
 		let currentState = "connecting";
 		let remoteStream = null;
 		const setState = (state) => {
@@ -77,7 +91,25 @@ const createRealTimeSubscribeClient = (opts) => {
 				apiKey,
 				roomName
 			});
-			room = new LiveKitRoom(REALTIME_CONFIG.livekit.roomOptions);
+			if (frameTiming) {
+				if (!opts.createFrameMetadataWorker) throw createFrameMetadataSubscribeUnsupportedError("this platform has no frame-metadata worker");
+				if (!(opts.isFrameMetadataRuntimeSupported?.() ?? false)) throw createFrameMetadataSubscribeUnsupportedError("encoded transforms are unavailable or the SDK is served cross-origin");
+				try {
+					frameMetadataWorker = opts.createFrameMetadataWorker();
+				} catch (error) {
+					throw createFrameMetadataSubscribeUnsupportedError(`failed to create the required worker (${error instanceof Error ? error.message : String(error)})`);
+				}
+			}
+			try {
+				room = new LiveKitRoom({
+					...REALTIME_CONFIG.livekit.roomOptions,
+					...frameMetadataWorker ? { frameMetadata: { worker: frameMetadataWorker } } : {}
+				});
+			} catch (error) {
+				frameMetadataWorker?.terminate();
+				frameMetadataWorker = void 0;
+				throw error;
+			}
 			const activeRoom = room;
 			activeRoom.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
 				if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
@@ -114,6 +146,11 @@ const createRealTimeSubscribeClient = (opts) => {
 		} catch (error) {
 			observability?.stop();
 			if (room) room.disconnect().catch(() => {});
+			frameMetadataWorker?.terminate();
+			if (isDecartSDKError(error)) {
+				logger.error("Realtime subscribe error", { error: error.message });
+				throw error;
+			}
 			const err = error instanceof Error ? error : new Error(String(error));
 			logger.error("Realtime subscribe error", { error: err.message });
 			throw classifyWebrtcError(err);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decartai/sdk",
-  "version": "0.1.20",
+  "version": "0.1.22",
   "description": "Decart's JavaScript SDK",
   "type": "module",
   "license": "MIT",