@decartai/sdk 0.1.13 → 0.1.15

package/dist/realtime/{preflight.js → browser/preflight.js}

@@ -1,109 +1,8 @@
-import { resolveFpsNumber } from "../shared/model.js";
-import { REALTIME_CONFIG } from "./config-realtime.js";
-import { extractSignals, scoreLowerBetter, worst } from "./observability/connection-quality.js";
-//#region src/realtime/preflight.ts
-/** Extract the candidate type ("host" | "srflx" | "prflx" | "relay") from an ICE candidate. */
-function candidateType(candidate) {
-	if (candidate.type) return candidate.type;
-	return /\btyp (\w+)/.exec(candidate.candidate)?.[1] ?? "";
-}
-async function gatherIceCandidates(iceServers, timeoutMs, signal, logger) {
-	if (signal?.aborted) return {
-		transport: "failed",
-		rttMs: null
-	};
-	const PC = globalThis?.RTCPeerConnection;
-	if (typeof PC !== "function") {
-		logger.warn("preflight: RTCPeerConnection unavailable in this environment");
-		return {
-			transport: "failed",
-			rttMs: null
-		};
-	}
-	let pc = null;
-	try {
-		pc = new PC({ iceServers });
-		pc.createDataChannel("decart-preflight");
-		let sawSrflx = false;
-		let sawOtherCandidate = false;
-		let firstSrflxAt = null;
-		const start = performance.now();
-		await new Promise((resolve) => {
-			let settled = false;
-			const finish = () => {
-				if (settled) return;
-				settled = true;
-				clearTimeout(timer);
-				resolve();
-			};
-			const timer = setTimeout(finish, timeoutMs);
-			signal?.addEventListener("abort", finish, { once: true });
-			const peer = pc;
-			peer.onicecandidate = (event) => {
-				if (!event.candidate || event.candidate.candidate === "") {
-					finish();
-					return;
-				}
-				if (candidateType(event.candidate) === "srflx") {
-					sawSrflx = true;
-					if (firstSrflxAt === null) firstSrflxAt = performance.now();
-				} else sawOtherCandidate = true;
-			};
-			peer.onicegatheringstatechange = () => {
-				if (peer.iceGatheringState === "complete") finish();
-			};
-			peer.createOffer().then((offer) => peer.setLocalDescription(offer)).catch((error) => {
-				logger.warn("preflight: failed to create offer", { error: error instanceof Error ? error.message : String(error) });
-				finish();
-			});
-		});
-		const rttMs = firstSrflxAt !== null ? Math.round(firstSrflxAt - start) : null;
-		let transport;
-		if (sawSrflx) transport = "udp";
-		else if (sawOtherCandidate) transport = "relay";
-		else transport = "failed";
-		return {
-			transport,
-			rttMs
-		};
-	} catch (error) {
-		logger.warn("preflight: connectivity probe threw", { error: error instanceof Error ? error.message : String(error) });
-		return {
-			transport: "failed",
-			rttMs: null
-		};
-	} finally {
-		try {
-			pc?.close();
-		} catch {}
-	}
-}
-/** Map probe metrics to a connectivity quality verdict. Pure. */
-function classifyConnectivity(metrics, thresholds) {
-	const reasons = [];
-	let quality;
-	if (metrics.transport === "failed") {
-		quality = "critical";
-		reasons.push("Could not establish any WebRTC connectivity (no ICE candidates gathered). Real-time streaming is unlikely to work on this network.");
-	} else if (metrics.transport === "relay") {
-		quality = "poor";
-		reasons.push("Direct UDP connectivity could not be confirmed; the session will need a TURN relay, which adds latency and can't be verified without starting a session.");
-	} else if (metrics.rttMs != null && metrics.rttMs > thresholds.marginalMs) {
-		quality = "poor";
-		reasons.push(`Network round-trip time is high (~${metrics.rttMs}ms > ${thresholds.marginalMs}ms); the real-time experience may feel laggy.`);
-	} else if (metrics.rttMs != null && metrics.rttMs > thresholds.goodMs) {
-		quality = "fair";
-		reasons.push(`Network round-trip time is elevated (~${metrics.rttMs}ms > ${thresholds.goodMs}ms).`);
-	} else quality = "good";
-	return {
-		quality,
-		metrics: {
-			transport: metrics.transport,
-			rttMs: metrics.rttMs
-		},
-		reasons
-	};
-}
+import { resolveFpsNumber } from "../../shared/model.js";
+import { REALTIME_CONFIG } from "../config-realtime.js";
+import { extractSignals, scoreLowerBetter, worst } from "../observability/connection-quality.js";
+import { classifyConnectivity, gatherIceCandidates } from "../preflight-connectivity.js";
+//#region src/realtime/browser/preflight.ts
 /**
 * Classify an active-probe result. Judges startup (TTFF) and steady-state
 * (mid-stream glass-to-glass) latency separately — both are real experienced
@@ -131,9 +30,9 @@ function classifyActiveProbe(metrics, thresholds) {
 		if (q !== "good") reasons.push(`Mid-stream glass-to-glass latency is ~${metrics.g2gMs}ms (good ≤ ${g.goodMs}ms); the real-time experience may feel laggy.`);
 	}
 	if (metrics.ttffMs == null && metrics.g2gMs == null) if (metrics.rttMs != null) {
-		reasons.push("Could not measure glass-to-glass latency during the probe (no marker round-trip); using network RTT instead.");
+		reasons.push("Could not measure glass-to-glass latency during the probe (no frame metadata); using network RTT instead.");
 		dims.push(scoreLowerBetter(metrics.rttMs, thresholds.rtt.goodMs, thresholds.rtt.fairMs, thresholds.rtt.poorMs));
-	} else reasons.push("The probe connected but could not measure latency (no marker round-trip and no RTT sample).");
+	} else reasons.push("The probe connected but could not measure latency (no frame metadata and no RTT sample).");
 	if (metrics.g2gDropRatio != null) {
 		const d = thresholds.g2gDrop;
 		const q = scoreLowerBetter(metrics.g2gDropRatio, d.good, d.fair, d.poor);
@@ -182,10 +81,9 @@ function activeMetricsFromStats(stats) {
 	};
 }
 /**
-* Animated synthetic video source — no camera permission needed; content is
-* irrelevant to the marker. Sized to the model's exact input dimensions so the
-* server doesn't resize/crop the frame, which would move the bottom-left marker
-* out of where the server reads it (breaking the round trip).
+* Animated synthetic video source — no camera permission needed. It uses the
+* model's input dimensions to exercise the same encode/inference path as a real
+* session without unnecessary server-side resizing.
 */
 function createSyntheticSource(width, height, fps) {
 	if (typeof document === "undefined") throw new Error("deep connectivity probe requires a DOM environment (document is undefined)");
@@ -232,11 +130,18 @@ const ABORTED_DEEP_PROBE = {
 /** Unblock `promise` early when `signal` aborts. */
 function raceAbort(promise, signal) {
 	if (!signal) return promise;
-	if (signal.aborted) throw signal.reason ?? new DOMException("Aborted", "AbortError");
+	if (signal.aborted) throw signal.reason ?? createAbortError();
 	return Promise.race([promise, new Promise((_, reject) => {
-		signal.addEventListener("abort", () => reject(signal.reason ?? new DOMException("Aborted", "AbortError")), { once: true });
+		signal.addEventListener("abort", () => reject(signal.reason ?? createAbortError()), { once: true });
 	})]);
 }
+function createAbortError() {
+	const DOMExceptionConstructor = globalThis.DOMException;
+	if (typeof DOMExceptionConstructor === "function") return new DOMExceptionConstructor("Aborted", "AbortError");
+	const error = /* @__PURE__ */ new Error("Aborted");
+	error.name = "AbortError";
+	return error;
+}
 /** Resolve once enough g2g samples exist or the probe window elapses (never rejects). */
 function waitForProbe(getLatest, minSamples, durationMs, signal) {
 	return new Promise((resolve) => {
@@ -308,4 +213,4 @@ const createPreflight = ({ logger, connect }) => {
 	return { checkConnectivity };
 };
 //#endregion
-export { createPreflight };
+export { classifyActiveProbe, createPreflight };

package/dist/realtime/browser/prepare-connection.js

@@ -0,0 +1,57 @@
+import { RealtimeObservability } from "../observability/realtime-observability.js";
+import { isDesktopSafari } from "../../utils/platform.js";
+import { createLiveKitMediaChannel } from "../media-channel.js";
+import { createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker } from "./frame-metadata-diagnostics.js";
+import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
+//#region src/realtime/browser/prepare-connection.ts
+const prepareBrowserConnection = ({ stream, mirror, debugQuality, preferredVideoCodec, fps, logger, observability: observabilityOptions }) => {
+	let inputStream = stream ?? new MediaStream();
+	let disposeMirroring = () => {};
+	if (mirror !== false) try {
+		const videoTrack = inputStream.getVideoTracks?.()[0];
+		if (videoTrack && (mirror === true || shouldMirrorTrack(videoTrack))) {
+			const mirrored = createMirroredStream(inputStream, { fps });
+			inputStream = mirrored.stream;
+			disposeMirroring = mirrored.dispose;
+		} else if (mirror === true && !videoTrack) logger.warn("mirror: true requested but no video track was found on the input stream");
+	} catch (error) {
+		logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
+	}
+	let pendingWorker;
+	if (debugQuality) try {
+		pendingWorker = createFrameMetadataWorker();
+	} catch (error) {
+		logger.warn("Frame-metadata worker unavailable; glass-to-glass latency measurement disabled", { error: error instanceof Error ? error.message : String(error) });
+	}
+	const frameTiming = pendingWorker !== void 0;
+	const takeFrameMetadataWorker = () => {
+		if (pendingWorker) {
+			const worker = pendingWorker;
+			pendingWorker = void 0;
+			return worker;
+		}
+		return createFrameMetadataWorker();
+	};
+	const observability = new RealtimeObservability({
+		...observabilityOptions,
+		glassToGlass: frameTiming ? createBrowserFrameMetadataDiagnostics() : void 0
+	});
+	const safariCodec = isDesktopSafari() ? "vp8" : void 0;
+	return {
+		stream: inputStream,
+		observability,
+		frameTiming,
+		videoCodec: safariCodec ?? preferredVideoCodec,
+		queryParams: { ...safariCodec ? { livekit_server_codec: safariCodec } : {} },
+		createMediaChannel: (config) => createLiveKitMediaChannel({
+			...config,
+			createFrameMetadataWorker: frameTiming ? takeFrameMetadataWorker : void 0
+		}),
+		dispose: () => {
+			disposeMirroring();
+			pendingWorker?.terminate();
+		}
+	};
+};
+//#endregion
+export { prepareBrowserConnection };

package/dist/realtime/client.d.ts

@@ -14,7 +14,7 @@ declare const realTimeClientInitialStateSchema: z.ZodObject<{
     text: z.ZodString;
     enhance: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
   }, z.core.$strip>>;
-  image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
+  image: z.ZodOptional<z.ZodUnion<readonly [z.ZodType<Blob, unknown, z.core.$ZodTypeInternals<Blob, unknown>>, z.ZodType<File, unknown, z.core.$ZodTypeInternals<File, unknown>>, z.ZodString]>>;
   passthrough: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
 type OnRemoteStreamFn = (stream: MediaStream) => void;
@@ -46,7 +46,7 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
       text: z.ZodString;
       enhance: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
     }, z.core.$strip>>;
-    image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
+    image: z.ZodOptional<z.ZodUnion<readonly [z.ZodType<Blob, unknown, z.core.$ZodTypeInternals<Blob, unknown>>, z.ZodType<File, unknown, z.core.$ZodTypeInternals<File, unknown>>, z.ZodString]>>;
     passthrough: z.ZodOptional<z.ZodBoolean>;
   }, z.core.$strip>>;
   queryParams: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
@@ -57,12 +57,19 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
   }>>;
   preferredVideoCodec: z.ZodOptional<z.ZodEnum<{
     h264: "h264";
+    vp8: "vp8";
     vp9: "vp9";
   }>>;
   debugQuality: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
-type RealTimeClientConnectOptions = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model"> & {
+type RealtimeMediaStream = {
+  getTracks(): unknown[];
+  getAudioTracks(): unknown[];
+  getVideoTracks(): unknown[];
+};
+type RealTimeClientConnectOptions<TStream extends RealtimeMediaStream = MediaStream> = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model" | "onRemoteStream"> & {
   model: ModelDefinition | CustomModelDefinition;
+  onRemoteStream: (stream: TStream) => void;
 };
 type Events = {
   connectionChange: ConnectionState;
@@ -101,4 +108,4 @@ type RealTimeClient = {
   replaceVideoTrack: (track: MediaStreamTrack) => Promise<void>;
 };
 //#endregion
-export { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState };
+export { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState, RealtimeMediaStream };

package/dist/realtime/client.js

@@ -1,14 +1,11 @@
 import { classifyWebrtcError } from "../utils/errors.js";
+import { createConsoleLogger } from "../utils/logger.js";
 import { isFileRefId } from "../files/types.js";
 import { modelDefinitionSchema, resolveFpsNumber } from "../shared/model.js";
 import { modelStateSchema } from "../shared/types.js";
-import { createConsoleLogger } from "../utils/logger.js";
 import { imageToBase64 } from "../utils/media.js";
-import { isDesktopSafari } from "../utils/platform.js";
 import { createEventBuffer } from "./event-buffer.js";
 import { realtimeMethods } from "./methods.js";
-import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
-import { RealtimeObservability } from "./observability/realtime-observability.js";
 import { StreamSession } from "./stream-session.js";
 import { z } from "zod";
 //#region src/realtime/client.ts
@@ -23,7 +20,19 @@ const realTimeClientConnectOptionsSchema = z.object({
 	queryParams: z.record(z.string(), z.string()).optional(),
 	mirror: z.union([z.literal("auto"), z.boolean()]).optional(),
 	resolution: z.enum(["720p", "1080p"]).optional(),
-	preferredVideoCodec: z.enum(["h264", "vp9"]).optional(),
+	/** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */
+	preferredVideoCodec: z.enum([
+		"h264",
+		"vp8",
+		"vp9"
+	]).optional(),
+	/**
+	* Opt-in quality measurement using LiveKit frame metadata. The capture
+	* timestamp is propagated through inference and matched to output playout to
+	* surface true glass-to-glass `g2gMs` / `ttffMs` on the `stats` and
+	* `connectionQuality` signals. Browser-only and experimental because it
+	* relies on LiveKit's frame-metadata worker and encoded transforms.
+	*/
 	debugQuality: z.boolean().optional()
 });
 const createRealTimeClient = (opts) => {
@@ -34,20 +43,10 @@ const createRealTimeClient = (opts) => {
 		if (!parsedOptions.success) throw parsedOptions.error;
 		const { onRemoteStream, onConnectionChange, onConnectionQuality, onQueuePosition, initialState, resolution, preferredVideoCodec } = parsedOptions.data;
 		const mirror = parsedOptions.data.mirror ?? false;
-		let inputStream = stream ?? new MediaStream();
-		let mirroredStream;
-		if (mirror !== false) try {
-			const firstVideoTrack = inputStream.getVideoTracks?.()[0];
-			if (firstVideoTrack && (mirror === true || shouldMirrorTrack(firstVideoTrack))) {
-				mirroredStream = createMirroredStream(inputStream, { fps: resolveFpsNumber(options.model.fps) });
-				inputStream = mirroredStream.stream;
-			} else if (mirror === true && !firstVideoTrack) logger.warn("mirror: true requested but no video track was found on the input stream");
-		} catch (error) {
-			logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
-		}
 		const debugQuality = parsedOptions.data.debugQuality ?? false;
 		let session;
 		let observability;
+		let preparedConnection;
 		try {
 			const initialImageRef = isFileRefId(initialState?.image) ? initialState.image : void 0;
 			const initialImage = initialImageRef === void 0 && initialState?.image ? await imageToBase64(initialState.image) : void 0;
@@ -57,41 +56,47 @@ const createRealTimeClient = (opts) => {
 			} : void 0;
 			const url = `${baseUrl}${options.model.urlPath}`;
 			const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer();
-			observability = new RealtimeObservability({
-				telemetryEnabled: opts.telemetryEnabled,
-				apiKey,
-				model: options.model.name,
-				integration,
+			preparedConnection = opts.prepareConnection({
+				stream,
+				mirror,
+				debugQuality,
+				preferredVideoCodec,
+				fps: resolveFpsNumber(options.model.fps),
 				logger,
-				onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
-				onStats: (stats) => emitOrBuffer("stats", stats),
-				onConnectionQuality: (report) => {
-					emitOrBuffer("connectionQuality", report);
-					onConnectionQuality?.(report);
-				},
-				debugQuality
+				observability: {
+					telemetryEnabled: opts.telemetryEnabled,
+					apiKey,
+					model: options.model.name,
+					integration,
+					logger,
+					onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
+					onStats: (stats) => emitOrBuffer("stats", stats),
+					onConnectionQuality: (report) => {
+						emitOrBuffer("connectionQuality", report);
+						onConnectionQuality?.(report);
+					}
+				}
 			});
-			if (debugQuality) inputStream = observability.attachOutgoingStream(inputStream, resolveFpsNumber(options.model.fps));
-			const safariCodec = isDesktopSafari() ? "vp8" : void 0;
-			const publishCodec = safariCodec ?? preferredVideoCodec;
+			observability = preparedConnection.observability;
 			session = new StreamSession({
 				url: `${url}?${new URLSearchParams({
-					...safariCodec ? { livekit_server_codec: safariCodec } : {},
+					...preparedConnection.queryParams ?? {},
 					...options.queryParams ?? {},
-					...debugQuality ? { pixel_latency: "1" } : {},
 					api_key: apiKey,
 					model: options.model.name,
 					...resolution ? { resolution } : {}
 				}).toString()}`,
 				integration,
 				observability,
-				localStream: inputStream,
+				frameTiming: preparedConnection.frameTiming,
+				localStream: preparedConnection.stream,
 				initialImage,
 				initialImageRef,
 				initialPrompt,
 				initialPassthrough: initialState?.passthrough,
 				logger,
-				videoCodec: publishCodec
+				videoCodec: preparedConnection.videoCodec,
+				createMediaChannel: preparedConnection.createMediaChannel
 			});
 			let sessionId = null;
 			let subscribeToken = null;
@@ -126,7 +131,7 @@ const createRealTimeClient = (opts) => {
 					observability?.stop();
 					stop();
 					activeSession.disconnect();
-					mirroredStream?.dispose();
+					preparedConnection?.dispose();
 				},
 				on: eventEmitter.on,
 				off: eventEmitter.off,
@@ -158,7 +163,7 @@ const createRealTimeClient = (opts) => {
 		} catch (error) {
 			observability?.stop();
 			session?.disconnect();
-			mirroredStream?.dispose();
+			preparedConnection?.dispose();
 			throw error;
 		}
 	};

package/dist/realtime/config-realtime.js

@@ -44,43 +44,78 @@ const REALTIME_CONFIG = {
 		telemetryReportIntervalMs: 1e4,
 		telemetryUrl: "https://platform.decart.ai/api/v1/telemetry",
 		telemetryMaxItemsPerReport: 120,
+		/**
+		* Thresholds for the derived in-session connection-quality signal
+		* (see observability/connection-quality.ts). Tuned for a camera-up
+		* real-time pipeline (~3–3.5 Mbps upstream, model fps ~25–30). All
+		* values are tunable here so behaviour can change without code edits.
+		*/
 		connectionQuality: {
+			/** Rolling-window size used to smooth raw per-sample metrics. */
 			windowSamples: 5,
+			/**
+			* Samples to wait before the bitrate dimensions count — the encoder
+			* and BWE ramp for several seconds after connect, so early low
+			* bitrate is not a slow network. RTT/loss/stall start scoring sooner.
+			* `decart-try-on` waits 30 samples watching inbound bitrate alone.
+			*/
 			warmupSamples: 8,
+			/** Consecutive worse samples required before the level downgrades. */
 			downgradeConsecutive: 5,
+			/** Consecutive better samples required before the level upgrades (recover slow). */
 			upgradeConsecutive: 5,
+			/** Round-trip time bands (ms). Bands widen by relayExtraMs on TURN-relayed paths. */
 			rtt: {
 				goodMs: 150,
 				fairMs: 300,
 				poorMs: 500,
 				relayExtraMs: 100
 			},
+			/**
+			* Mid-stream (steady-state) true glass-to-glass latency bands (ms) — the
+			* real per-frame camera→display latency through the model, used for the
+			* latency dimension *instead of* RTT when frame-metadata measurement is on.
+			* Already includes both network legs, so relayExtraMs does not apply.
+			* Excludes startup (see `ttff`). Anchored to Datadog
+			* `rt.stream.pipeline_latency_ms` (server-side median ~285ms / p95 ~510ms)
+			* plus network + jitter-buffer + decode headroom. Tune with real data.
+			*/
 			glassToGlass: {
 				goodMs: 500,
 				fairMs: 900,
 				poorMs: 1500
 			},
+			/**
+			* Time-to-first-frame bands (ms) — startup latency from connect to the
+			* first rendered model frame. An order of magnitude larger than mid-stream
+			* and judged separately (a slow first frame is a different problem from a
+			* laggy steady state). ~4–5s is an acceptable ("fair") cold start today.
+			*/
 			ttff: {
 				goodMs: 4e3,
 				fairMs: 6e3,
 				poorMs: 1e4
 			},
+			/** Fraction of outbound packets the server reports lost (0..1). */
 			loss: {
 				good: .001,
 				fair: .01,
 				poor: .05
 			},
+			/** End-to-end frame drop ratio (0..1), when a frame identity is available. */
 			g2gDrop: {
 				good: .001,
 				fair: .01,
 				poor: .05
 			},
+			/** Upstream headroom = available BWE ÷ the intended publish bitrate (requiredUpstreamKbps). */
 			upstream: {
 				goodRatio: 1,
 				fairRatio: .8,
 				poorRatio: .5,
 				requiredUpstreamKbps: 3500
 			},
+			/** Rendered (inbound) frames-per-second. */
 			stall: {
 				goodFps: 20,
 				fairFps: 12,
@@ -88,13 +123,29 @@ const REALTIME_CONFIG = {
 			}
 		}
 	},
+	/**
+	* SDK-only preflight (see preflight.ts). Validates WebRTC reachability
+	* (does UDP egress work / will the path need TURN) and latency via a
+	* throwaway RTCPeerConnection — no backend session, no media server.
+	*/
 	preflight: {
+		/** Public STUN servers used to gather server-reflexive candidates. */
 		defaultStunUrls: ["stun:stun.l.google.com:19302"],
+		/** Abort candidate gathering after this long. */
 		iceGatherTimeoutMs: 5e3,
+		/** RTT bands (ms) for the preflight verdict. */
 		rtt: {
 			goodMs: 150,
 			marginalMs: 300
 		},
+		/**
+		* Deep probe (`checkConnectivity({ deep: true, model })`): briefly opens a
+		* real session with a synthetic source + frame-metadata measurement to get a
+		* true glass-to-glass verdict, then tears it down. Costs a short GPU session.
+		* The verdict reuses the in-session `connectionQuality` bands. Duration must
+		* cover TTFF (~4–5s) + mid-stream warm-up (~2s) before steady-state samples
+		* accrue; resolves early once `minSamples` exist.
+		*/
 		active: {
 			durationMs: 12e3,
 			minSamples: 5

package/dist/realtime/livekit.js

@@ -0,0 +1,26 @@
+import { createLiveKitInitializationError } from "../utils/errors.js";
+//#region src/realtime/livekit.ts
+let liveKitModulePromise;
+function validateLiveKitModule(module) {
+	const invalid = [
+		typeof module.Room !== "function" && "Room",
+		(typeof module.RoomEvent !== "object" || module.RoomEvent === null) && "RoomEvent",
+		(module.Track === null || typeof module.Track !== "object" && typeof module.Track !== "function") && "Track",
+		module.Track?.Source?.Camera !== "camera" && "Track.Source.Camera",
+		(typeof module.ConnectionState !== "object" || module.ConnectionState === null) && "ConnectionState"
+	].filter((name) => name !== false);
+	if (invalid.length > 0) throw createLiveKitInitializationError(`livekit-client is missing or has invalid required exports: ${invalid.join(", ")}`);
+	return module;
+}
+/** Load the browser-neutral LiveKit client only when realtime is actually used. */
+function loadLiveKitClient() {
+	liveKitModulePromise ??= import("livekit-client").then(validateLiveKitModule).catch((error) => {
+		liveKitModulePromise = void 0;
+		if (typeof error === "object" && error !== null && "code" in error && error.code === "LIVEKIT_INITIALIZATION_ERROR") throw error;
+		const cause = error instanceof Error ? error : new Error(String(error));
+		throw createLiveKitInitializationError(`Failed to initialize livekit-client: ${cause.message}`, cause);
+	});
+	return liveKitModulePromise;
+}
+//#endregion
+export { loadLiveKitClient, validateLiveKitModule };

package/dist/realtime/media-channel.js

@@ -1,24 +1,28 @@
 import { createConsoleLogger } from "../utils/logger.js";
 import { REALTIME_CONFIG } from "./config-realtime.js";
+import { loadLiveKitClient } from "./livekit.js";
 import mitt from "mitt";
-import { Room, RoomEvent, Track } from "livekit-client";
 //#region src/realtime/media-channel.ts
-function getDefaultVideoPublishOptions(videoCodec) {
+function getDefaultVideoPublishOptions(source, videoCodec, frameMetadata = false) {
 	const resolvedCodec = videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec;
 	const maxBitrate = resolvedCodec === "vp9" ? REALTIME_CONFIG.livekit.vp9MaxVideoBitrateBps : REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps;
 	return {
-		source: Track.Source.Camera,
+		source,
 		videoCodec: resolvedCodec,
 		simulcast: resolvedCodec !== "vp9",
 		videoEncoding: {
 			maxBitrate,
 			maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps
-		}
+		},
+		...frameMetadata ? { frameMetadata: { timestamp: true } } : {}
 	};
 }
-var MediaChannel = class {
+var LiveKitMediaChannel = class {
+	config;
 	room = null;
+	cameraTrackSource = null;
 	remoteStream = null;
+	frameMetadataEnabled = false;
 	events = mitt();
 	logger;
 	constructor(config) {
@@ -35,14 +39,34 @@ var MediaChannel = class {
 		this.events.off(event, handler);
 	}
 	async connect(opts) {
-		this.room ??= new Room(REALTIME_CONFIG.livekit.roomOptions);
+		const { Room: LiveKitRoom, RoomEvent, Track } = await loadLiveKitClient();
+		this.cameraTrackSource = Track.Source.Camera;
+		if (!this.room) {
+			let worker;
+			if (this.config.createFrameMetadataWorker) try {
+				worker = this.config.createFrameMetadataWorker();
+			} catch (error) {
+				this.logger.warn("Failed to create LiveKit frame-metadata worker; continuing without latency metrics", { error: error instanceof Error ? error.message : String(error) });
+			}
+			this.frameMetadataEnabled = worker !== void 0;
+			try {
+				this.room = new LiveKitRoom({
+					...REALTIME_CONFIG.livekit.roomOptions,
+					...worker ? { frameMetadata: { worker } } : {}
+				});
+			} catch (error) {
+				worker?.terminate();
+				this.frameMetadataEnabled = false;
+				throw error;
+			}
+		}
 		const room = this.room;
 		room.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
 			if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
-			if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
+			if (track.kind !== "video" && track.kind !== "audio") return;
 			const mediaStreamTrack = track.mediaStreamTrack;
 			if (mediaStreamTrack) {
-				if (track.kind === Track.Kind.Video) this.config.observability?.attachRemoteVideoTrack(mediaStreamTrack);
+				if (track.kind === "video") this.config.observability?.attachRemoteVideoTrack(track);
 				const tracks = this.remoteStream?.getTracks() ?? [];
 				if (!tracks.includes(mediaStreamTrack)) tracks.push(mediaStreamTrack);
 				this.remoteStream = new MediaStream(tracks);
@@ -74,15 +98,20 @@ var MediaChannel = class {
 	disconnect() {
 		const room = this.room;
 		this.room = null;
+		this.cameraTrackSource = null;
 		this.remoteStream = null;
+		this.frameMetadataEnabled = false;
 		this.config.observability?.setLiveKitRoom(null);
 		if (room) room.disconnect().catch(() => {});
 	}
 	async publishTracks(stream) {
 		if (!this.room) return;
-		for (const track of stream.getTracks()) if (track.kind === "video") await this.room.localParticipant.publishTrack(track, getDefaultVideoPublishOptions(this.config.videoCodec));
-		else await this.room.localParticipant.publishTrack(track);
+		for (const track of stream.getTracks()) if (track.kind === "video") {
+			if (!this.cameraTrackSource) throw new Error("Cannot publish video track: media channel is not connected");
+			await this.room.localParticipant.publishTrack(track, getDefaultVideoPublishOptions(this.cameraTrackSource, this.config.videoCodec, this.frameMetadataEnabled));
+		} else await this.room.localParticipant.publishTrack(track);
 	}
 };
+const createLiveKitMediaChannel = (config) => new LiveKitMediaChannel(config);
 //#endregion
-export { MediaChannel };
+export { LiveKitMediaChannel, createLiveKitMediaChannel, getDefaultVideoPublishOptions };

package/dist/realtime/methods.d.ts

@@ -4,7 +4,7 @@ import { z } from "zod";
 declare const setInputSchema: z.ZodObject<{
   prompt: z.ZodOptional<z.ZodString>;
   enhance: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-  image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString, z.ZodNull]>>;
+  image: z.ZodOptional<z.ZodUnion<readonly [z.ZodType<Blob, unknown, z.core.$ZodTypeInternals<Blob, unknown>>, z.ZodType<File, unknown, z.core.$ZodTypeInternals<File, unknown>>, z.ZodString, z.ZodNull]>>;
 }, z.core.$strip>;
 type SetInput = z.input<typeof setInputSchema>;
 //#endregion