@decartai/sdk 0.1.17 → 0.1.18

package/README.md

@@ -88,18 +88,6 @@ A connected realtime session exposes an SDK `subscribeToken` once it reaches a
 connected state. Share that SDK token with viewers — `client.realtime.subscribe`
 uses it to request receive-only LiveKit credentials from Decart, then connects to
 the LiveKit room for the styled output stream. No viewer camera is required.
-Browser-produced streams can include LiveKit frame metadata for glass-to-glass
-latency, so viewers also need a browser runtime that can create the SDK's
-frame-metadata worker; unsupported runtimes reject those tokens before joining.
-
-> **Viewers must be on this SDK version or newer.** A browser publisher now
-> advertises frame timing by default, which makes the server append a packet
-> trailer to every frame in the room. Viewers on an older `@decartai/sdk`
-> ignore the token's `frame_timing` flag and join without a strip worker — they
-> connect successfully but decode nothing, so the video element stays black with
-> no error raised. React Native viewers cannot strip trailers at all and are
-> rejected with `UNSUPPORTED_PLATFORM_FEATURE`. Upgrade viewers before upgrading
-> publishers.
 
 **Producer** — capture the token from the active session:
 
@@ -187,12 +175,12 @@ realtimeClient.getConnectionQuality(); // latest report, or null before the firs
 > setInterval(() => console.log(realtimeClient.getConnectionQuality()?.metrics.g2gMs), 1000);
 > ```
 
-**Glass-to-glass latency.** Network RTT hides the dominant cost in real-time video — model
-inference — so a session can read "good" while actually feeling laggy. In browser sessions, the SDK
-automatically attaches a capture timestamp using LiveKit frame metadata when the required worker is
-available; the server propagates it through inference and the SDK matches it to output playout.
-This surfaces **startup** (`ttffMs`) and **steady-state** (`g2gMs`) latency. When present,
-glass-to-glass drives the latency verdict instead of RTT.
+**Glass-to-glass latency (opt-in, diagnostic).** Network RTT hides the dominant cost in
+real-time video — model inference — so a session can read "good" while actually feeling laggy.
+Set `debugQuality: true` to measure the *real* camera→display latency. The SDK attaches a capture
+timestamp using LiveKit frame metadata; the server propagates it through inference and the SDK
+matches it to output playout. This surfaces **startup** (`ttffMs`) and **steady-state**
+(`g2gMs`) latency. When present, glass-to-glass drives the latency verdict instead of RTT.
 
 > Frame metadata is currently experimental in LiveKit and requires encoded-transform support.
 > It does not alter visible pixels. `g2gDropRatio` remains `null` until frame IDs are propagated
@@ -201,6 +189,7 @@ glass-to-glass drives the latency verdict instead of RTT.
 ```typescript
 const realtimeClient = await client.realtime.connect(stream, {
   model,
+  debugQuality: true,
 });
 
 // g2g updates every stats tick — read it from the `stats` event (the
@@ -349,9 +338,9 @@ Add them to `app.json`:
 Run `npx expo prebuild` and rebuild the native app. LiveKit does not run in Expo
 Go. Bare React Native apps must follow LiveKit's native setup instructions.
 
-Outgoing `mirror` and deep connectivity preflight are browser-only and fail with
-`UNSUPPORTED_PLATFORM_FEATURE` on React Native. Mirror the local preview with your
-native video view instead.
+Outgoing `mirror`, `debugQuality`, and deep connectivity preflight are browser-only
+and fail with `UNSUPPORTED_PLATFORM_FEATURE` on React Native. Mirror the local
+preview with your native video view instead.
 
 ## Development
 

package/dist/package.js

@@ -1,4 +1,4 @@
 //#region package.json
-var version = "0.1.17";
+var version = "0.1.18";
 //#endregion
 export { version };

package/dist/realtime/browser/frame-metadata-diagnostics.js

@@ -52,8 +52,8 @@ const TIME_SYNC_UPDATE = "timeSyncUpdate";
 function createFrameReader(tracker) {
 	let attachedTrack = null;
 	const onTimeSyncUpdate = ({ timestamp, rtpTimestamp }) => {
-		const frameMetadata = attachedTrack?.lookupFrameMetadata?.({ rtpTimestamp });
-		if (frameMetadata) tracker.recordFrame(frameMetadata.userTimestamp, timestamp);
+		const frameMetadata = attachedTrack?.lookupFrameMetadata({ rtpTimestamp });
+		if (frameMetadata) tracker.recordFrame(frameMetadata.userTimestamp, performance.timeOrigin + timestamp);
 	};
 	const detach = () => {
 		attachedTrack?.off(TIME_SYNC_UPDATE, onTimeSyncUpdate);
@@ -86,14 +86,5 @@ function createBrowserFrameMetadataDiagnostics() {
 function createFrameMetadataWorker() {
 	return new Worker(new URL("./frame-metadata-worker.js", import.meta.url));
 }
-function isFrameMetadataRuntimeSupported() {
-	if (typeof window === "undefined") return false;
-	const maybeWindow = window;
-	const userAgent = window.navigator?.userAgent?.toLowerCase() ?? "";
-	const isChromiumBased = /(?:chrome|chromium|crmo)\//.test(userAgent) && !/crios\//.test(userAgent);
-	const scriptTransformSupported = typeof maybeWindow.RTCRtpScriptTransform !== "undefined" && !isChromiumBased;
-	const insertableStreamsSupported = typeof maybeWindow.RTCRtpSender?.prototype?.createEncodedStreams !== "undefined" && typeof maybeWindow.RTCRtpReceiver?.prototype?.createEncodedStreams !== "undefined";
-	return scriptTransformSupported || insertableStreamsSupported;
-}
 //#endregion
-export { FrameMetadataTracker, createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker, isFrameMetadataRuntimeSupported };
+export { FrameMetadataTracker, createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker };

package/dist/realtime/browser/index.js

@@ -1,6 +1,5 @@
 import { createRealTimeClient } from "../client.js";
 import { createRealTimeSubscribeClient } from "../subscribe-client.js";
-import { createFrameMetadataWorker, isFrameMetadataRuntimeSupported } from "./frame-metadata-diagnostics.js";
 import { createPreflight } from "./preflight.js";
 import { prepareBrowserConnection } from "./prepare-connection.js";
 //#region src/realtime/browser/index.ts
@@ -17,9 +16,7 @@ const createBrowserRealtime = (options) => {
 		baseUrl: options.subscribeBaseUrl,
 		apiKey: options.apiKey,
 		integration: options.integration,
-		logger: options.logger,
-		createFrameMetadataWorker,
-		isFrameMetadataRuntimeSupported
+		logger: options.logger
 	});
 	const preflight = createPreflight({
 		logger: options.logger,

package/dist/realtime/browser/preflight.js

@@ -166,6 +166,7 @@ async function runActiveProbe(args) {
 		source = createSyntheticSource(model.width, model.height, resolveFpsNumber(model.fps));
 		const connectTask = connect(source.stream, {
 			model,
+			debugQuality: true,
 			onRemoteStream: () => {}
 		});
 		signal?.addEventListener("abort", () => connectTask.then((c) => c.disconnect()).catch(() => {}), { once: true });

package/dist/realtime/browser/prepare-connection.js

@@ -1,10 +1,10 @@
 import { RealtimeObservability } from "../observability/realtime-observability.js";
 import { isDesktopSafari } from "../../utils/platform.js";
 import { createLiveKitMediaChannel } from "../media-channel.js";
-import { createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker, isFrameMetadataRuntimeSupported } from "./frame-metadata-diagnostics.js";
+import { createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker } from "./frame-metadata-diagnostics.js";
 import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
 //#region src/realtime/browser/prepare-connection.ts
-const prepareBrowserConnection = ({ stream, mirror, preferredVideoCodec, fps, logger, observability: observabilityOptions }) => {
+const prepareBrowserConnection = ({ stream, mirror, debugQuality, preferredVideoCodec, fps, logger, observability: observabilityOptions }) => {
 	let inputStream = stream ?? new MediaStream();
 	let disposeMirroring = () => {};
 	if (mirror !== false) try {
@@ -18,12 +18,11 @@ const prepareBrowserConnection = ({ stream, mirror, preferredVideoCodec, fps, lo
 		logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
 	}
 	let pendingWorker;
-	if (isFrameMetadataRuntimeSupported()) try {
+	if (debugQuality) try {
 		pendingWorker = createFrameMetadataWorker();
 	} catch (error) {
-		logger.debug("Frame-metadata worker unavailable; glass-to-glass latency measurement disabled", { error: error instanceof Error ? error.message : String(error) });
+		logger.warn("Frame-metadata worker unavailable; glass-to-glass latency measurement disabled", { error: error instanceof Error ? error.message : String(error) });
 	}
-	else logger.debug("Frame-metadata runtime unavailable; glass-to-glass latency measurement disabled");
 	const frameTiming = pendingWorker !== void 0;
 	const takeFrameMetadataWorker = () => {
 		if (pendingWorker) {

package/dist/realtime/client.js

@@ -27,9 +27,11 @@ const realTimeClientConnectOptionsSchema = z.object({
 		"vp9"
 	]).optional(),
 	/**
-	* @deprecated Glass-to-glass measurement now runs automatically in browsers
-	* when LiveKit frame metadata is available. This legacy flag is accepted for
-	* compatibility and no longer gates measurement.
+	* Opt-in quality measurement using LiveKit frame metadata. The capture
+	* timestamp is propagated through inference and matched to output playout to
+	* surface true glass-to-glass `g2gMs` / `ttffMs` on the `stats` and
+	* `connectionQuality` signals. Browser-only and experimental because it
+	* relies on LiveKit's frame-metadata worker and encoded transforms.
 	*/
 	debugQuality: z.boolean().optional()
 });
@@ -41,6 +43,7 @@ const createRealTimeClient = (opts) => {
 		if (!parsedOptions.success) throw parsedOptions.error;
 		const { onRemoteStream, onConnectionChange, onConnectionQuality, onQueuePosition, initialState, resolution, preferredVideoCodec } = parsedOptions.data;
 		const mirror = parsedOptions.data.mirror ?? false;
+		const debugQuality = parsedOptions.data.debugQuality ?? false;
 		let session;
 		let observability;
 		let preparedConnection;
@@ -56,6 +59,7 @@ const createRealTimeClient = (opts) => {
 			preparedConnection = opts.prepareConnection({
 				stream,
 				mirror,
+				debugQuality,
 				preferredVideoCodec,
 				fps: resolveFpsNumber(options.model.fps),
 				logger,

package/dist/realtime/config-realtime.js

@@ -19,8 +19,7 @@ const REALTIME_CONFIG = {
 			"invalid session",
 			"401",
 			"invalid api key",
-			"unauthorized",
-			"failed to create livekit frame-metadata worker"
+			"unauthorized"
 		]
 	},
 	methods: {

package/dist/realtime/media-channel.js

@@ -46,9 +46,7 @@ var LiveKitMediaChannel = class {
 			if (this.config.createFrameMetadataWorker) try {
 				worker = this.config.createFrameMetadataWorker();
 			} catch (error) {
-				const detail = error instanceof Error ? error.message : String(error);
-				this.logger.warn("Failed to create LiveKit frame-metadata worker", { error: detail });
-				throw new Error(`Failed to create LiveKit frame-metadata worker: ${detail}`, { cause: error });
+				this.logger.warn("Failed to create LiveKit frame-metadata worker; continuing without latency metrics", { error: error instanceof Error ? error.message : String(error) });
 			}
 			this.frameMetadataEnabled = worker !== void 0;
 			try {

package/dist/realtime/observability/connection-quality.d.ts

@@ -16,10 +16,10 @@ type ConnectionQualityMetrics = {
   rttMs: number | null;
   /**
    * Mid-stream (steady-state) glass-to-glass latency (ms) — the real per-frame
-   * camera→display latency through the model, excluding startup. Populated in
-   * browser sessions when LiveKit frame metadata is available and past warm-up;
-   * null otherwise. When present it drives the latency verdict instead of
-   * `rttMs`.
+   * camera→display latency through the model, excluding startup. Only populated
+   * when the opt-in frame-metadata measurement is on (`connect({ debugQuality: true })`)
+   * and past warm-up; null otherwise. When present it drives the latency verdict
+   * instead of `rttMs`.
    */
   g2gMs: number | null;
   /**

package/dist/realtime/observability/webrtc-stats.d.ts

@@ -133,8 +133,9 @@ type WebRTCStats = {
   };
   /**
    * True glass-to-glass latency + end-to-end drop signal, merged in by
-   * `RealtimeObservability` when frame-metadata measurement is active. Null
-   * otherwise — the stats collector does not populate it.
+   * `RealtimeObservability` when the opt-in frame-metadata measurement is active.
+   * Null otherwise — the stats collector does not
+   * populate it.
    */
   glassToGlass: G2GMetrics | null;
 };

package/dist/realtime/react-native/prepare-connection.js

@@ -10,9 +10,10 @@ function assertReactNativeReady() {
 function unsupportedReactNativeFeature(feature) {
 	throw createUnsupportedPlatformFeatureError(feature, "React Native");
 }
-const prepareReactNativeConnection = ({ stream, mirror, preferredVideoCodec, observability: observabilityOptions }) => {
+const prepareReactNativeConnection = ({ stream, mirror, debugQuality, preferredVideoCodec, observability: observabilityOptions }) => {
 	assertReactNativeReady();
 	if (mirror !== false) unsupportedReactNativeFeature("Outgoing video mirroring");
+	if (debugQuality) unsupportedReactNativeFeature("debugQuality");
 	return {
 		stream: stream ?? new MediaStream(),
 		observability: new RealtimeObservability(observabilityOptions),

package/dist/realtime/stream-session.js

@@ -4,11 +4,8 @@ import { SignalingChannel } from "./signaling-channel.js";
 import mitt from "mitt";
 import pRetry, { AbortError } from "p-retry";
 //#region src/realtime/stream-session.ts
-function encodeSubscribeToken(roomName, options = {}) {
-	return btoa(JSON.stringify({
-		room_name: roomName,
-		...options.frameTiming ? { frame_timing: true } : {}
-	}));
+function encodeSubscribeToken(roomName) {
+	return btoa(JSON.stringify({ room_name: roomName }));
 }
 function getInitialImageSizeKb(image) {
 	if (!image) return null;
@@ -140,7 +137,7 @@ var StreamSession = class {
 			this.setState("connected");
 			this.events.emit("sessionStarted", {
 				sessionId: roomInfo.sessionId,
-				subscribeToken: encodeSubscribeToken(roomInfo.roomName, { frameTiming: this.config.frameTiming })
+				subscribeToken: encodeSubscribeToken(roomInfo.roomName)
 			});
 		} catch (error) {
 			this.config.observability?.finishConnectionBreakdown({

package/dist/realtime/subscribe-client.js

@@ -1,28 +1,15 @@
-import { ERROR_CODES, classifyWebrtcError, createSDKError } from "../utils/errors.js";
+import { classifyWebrtcError } from "../utils/errors.js";
 import { createConsoleLogger } from "../utils/logger.js";
 import { createEventBuffer } from "./event-buffer.js";
 import { REALTIME_CONFIG } from "./config-realtime.js";
 import { loadLiveKitClient } from "./livekit.js";
 import { RealtimeObservability } from "./observability/realtime-observability.js";
 //#region src/realtime/subscribe-client.ts
-function isDecartSDKError(error) {
-	return typeof error === "object" && error !== null && typeof error.code === "string" && typeof error.message === "string";
-}
-function createFrameMetadataSubscribeUnsupportedError(detail) {
-	const suffix = detail ? `: ${detail}` : "";
-	return createSDKError(ERROR_CODES.UNSUPPORTED_PLATFORM_FEATURE, `This realtime stream requires LiveKit frame metadata, which this SDK environment does not support${suffix}.`, {
-		feature: "LiveKit frame metadata subscribe",
-		platform: "this SDK environment"
-	});
-}
 function decodeSubscribeToken(token) {
 	try {
 		const payload = JSON.parse(atob(token));
 		if (!payload.room_name || typeof payload.room_name !== "string") throw new Error("Invalid subscribe token format");
-		return {
-			room_name: payload.room_name,
-			...payload.frame_timing === true ? { frame_timing: true } : {}
-		};
+		return { room_name: payload.room_name };
 	} catch {
 		throw new Error("Invalid subscribe token");
 	}
@@ -63,11 +50,10 @@ const createRealTimeSubscribeClient = (opts) => {
 	const { baseUrl, apiKey, integration } = opts;
 	const logger = opts.logger ?? createConsoleLogger("info");
 	const subscribe = async (options) => {
-		const { room_name: roomName, frame_timing: frameTiming } = decodeSubscribeToken(options.token);
+		const { room_name: roomName } = decodeSubscribeToken(options.token);
 		const { emitter, emitOrBuffer, flush, stop } = createEventBuffer();
 		let observability;
 		let room;
-		let frameMetadataWorker;
 		let currentState = "connecting";
 		let remoteStream = null;
 		const setState = (state) => {
@@ -91,25 +77,7 @@ const createRealTimeSubscribeClient = (opts) => {
 				apiKey,
 				roomName
 			});
-			if (frameTiming) {
-				if (!opts.createFrameMetadataWorker) throw createFrameMetadataSubscribeUnsupportedError("this platform has no frame-metadata worker");
-				if (!(opts.isFrameMetadataRuntimeSupported?.() ?? false)) throw createFrameMetadataSubscribeUnsupportedError("encoded transforms are unavailable");
-				try {
-					frameMetadataWorker = opts.createFrameMetadataWorker();
-				} catch (error) {
-					throw createFrameMetadataSubscribeUnsupportedError(`failed to create the required worker (${error instanceof Error ? error.message : String(error)})`);
-				}
-			}
-			try {
-				room = new LiveKitRoom({
-					...REALTIME_CONFIG.livekit.roomOptions,
-					...frameMetadataWorker ? { frameMetadata: { worker: frameMetadataWorker } } : {}
-				});
-			} catch (error) {
-				frameMetadataWorker?.terminate();
-				frameMetadataWorker = void 0;
-				throw error;
-			}
+			room = new LiveKitRoom(REALTIME_CONFIG.livekit.roomOptions);
 			const activeRoom = room;
 			activeRoom.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
 				if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
@@ -146,11 +114,6 @@ const createRealTimeSubscribeClient = (opts) => {
 		} catch (error) {
 			observability?.stop();
 			if (room) room.disconnect().catch(() => {});
-			frameMetadataWorker?.terminate();
-			if (isDecartSDKError(error)) {
-				logger.error("Realtime subscribe error", { error: error.message });
-				throw error;
-			}
 			const err = error instanceof Error ? error : new Error(String(error));
 			logger.error("Realtime subscribe error", { error: err.message });
 			throw classifyWebrtcError(err);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decartai/sdk",
-  "version": "0.1.17",
+  "version": "0.1.18",
   "description": "Decart's JavaScript SDK",
   "type": "module",
   "license": "MIT",