@decartai/sdk 0.1.12 → 0.1.14

package/dist/realtime/{observability → browser}/glass-to-glass.js

@@ -1,23 +1,6 @@
-import { createFrameTransformPump } from "../mirror-stream.js";
+import { createFrameTransformPump } from "./mirror-stream.js";
 import { MAX_MARKER_HEIGHT, MIN_MARKER_WIDTH, read, stamp } from "./pixel-marker.js";
-//#region src/realtime/observability/glass-to-glass.ts
-/**
-* True glass-to-glass latency measurement for the realtime pipeline.
-*
-* Opt-in (the marker is visible in the output and pixel work has a cost). When
-* enabled, the SDK stamps a monotonic sequence number into the bottom-left of
-* every outgoing frame ({@link createStampPump}) and reads it back off the
-* rendered remote frames ({@link createMarkerReader}); the server re-stamps the
-* seq from input to matching output (its `pixel_latency` mode). The
-* {@link SeqTracker} matches stamp time to render time to compute the real
-* camera→display latency through the model, and infers end-to-end frame drops
-* from seqs that are stamped but never rendered.
-*
-* The stamp pump is built on the shared `createFrameTransformPump` (Insertable
-* Streams where available, canvas `captureStream` fallback). The reader is a
-* passive offscreen-`<video>` tap so it never consumes or re-encodes the track
-* the consumer displays.
-*/
+//#region src/realtime/browser/glass-to-glass.ts
 /** Bound on in-flight seqs; a seq that ages out unmatched is an end-to-end drop. Cf. server `_MAX_PENDING`. */
 const MAX_PENDING = 256;
 /** Rolling window for the latency percentiles (≈10s at 30fps). */

package/dist/realtime/browser/index.js

@@ -0,0 +1,32 @@
+import { createRealTimeClient } from "../client.js";
+import { createRealTimeSubscribeClient } from "../subscribe-client.js";
+import { createPreflight } from "./preflight.js";
+import { prepareBrowserConnection } from "./prepare-connection.js";
+//#region src/realtime/browser/index.ts
+const createBrowserRealtime = (options) => {
+	const publish = createRealTimeClient({
+		baseUrl: options.publishBaseUrl,
+		apiKey: options.apiKey,
+		integration: options.integration,
+		logger: options.logger,
+		telemetryEnabled: options.telemetryEnabled,
+		prepareConnection: prepareBrowserConnection
+	});
+	const subscribe = createRealTimeSubscribeClient({
+		baseUrl: options.subscribeBaseUrl,
+		apiKey: options.apiKey,
+		integration: options.integration,
+		logger: options.logger
+	});
+	const preflight = createPreflight({
+		logger: options.logger,
+		connect: publish.connect
+	});
+	return {
+		connect: (stream, connectOptions) => publish.connect(stream, connectOptions),
+		subscribe: subscribe.subscribe,
+		checkConnectivity: preflight.checkConnectivity
+	};
+};
+//#endregion
+export { createBrowserRealtime };

package/dist/realtime/{mirror-stream.js → browser/mirror-stream.js}

@@ -1,4 +1,4 @@
-//#region src/realtime/mirror-stream.ts
+//#region src/realtime/browser/mirror-stream.ts
 function isMediaStreamTrackProcessorSupported() {
 	return typeof globalThis !== "undefined" && typeof globalThis.MediaStreamTrackProcessor === "function" && typeof globalThis.MediaStreamTrackGenerator === "function";
 }

package/dist/realtime/{observability → browser}/pixel-marker.js

@@ -1,4 +1,4 @@
-//#region src/realtime/observability/pixel-marker.ts
+//#region src/realtime/browser/pixel-marker.ts
 const SYNC = [
 	200,
 	50,

package/dist/realtime/{preflight.js → browser/preflight.js}

@@ -1,109 +1,8 @@
-import { resolveFpsNumber } from "../shared/model.js";
-import { REALTIME_CONFIG } from "./config-realtime.js";
-import { extractSignals, scoreLowerBetter, worst } from "./observability/connection-quality.js";
-//#region src/realtime/preflight.ts
-/** Extract the candidate type ("host" | "srflx" | "prflx" | "relay") from an ICE candidate. */
-function candidateType(candidate) {
-	if (candidate.type) return candidate.type;
-	return /\btyp (\w+)/.exec(candidate.candidate)?.[1] ?? "";
-}
-async function gatherIceCandidates(iceServers, timeoutMs, signal, logger) {
-	if (signal?.aborted) return {
-		transport: "failed",
-		rttMs: null
-	};
-	const PC = globalThis?.RTCPeerConnection;
-	if (typeof PC !== "function") {
-		logger.warn("preflight: RTCPeerConnection unavailable in this environment");
-		return {
-			transport: "failed",
-			rttMs: null
-		};
-	}
-	let pc = null;
-	try {
-		pc = new PC({ iceServers });
-		pc.createDataChannel("decart-preflight");
-		let sawSrflx = false;
-		let sawOtherCandidate = false;
-		let firstSrflxAt = null;
-		const start = performance.now();
-		await new Promise((resolve) => {
-			let settled = false;
-			const finish = () => {
-				if (settled) return;
-				settled = true;
-				clearTimeout(timer);
-				resolve();
-			};
-			const timer = setTimeout(finish, timeoutMs);
-			signal?.addEventListener("abort", finish, { once: true });
-			const peer = pc;
-			peer.onicecandidate = (event) => {
-				if (!event.candidate || event.candidate.candidate === "") {
-					finish();
-					return;
-				}
-				if (candidateType(event.candidate) === "srflx") {
-					sawSrflx = true;
-					if (firstSrflxAt === null) firstSrflxAt = performance.now();
-				} else sawOtherCandidate = true;
-			};
-			peer.onicegatheringstatechange = () => {
-				if (peer.iceGatheringState === "complete") finish();
-			};
-			peer.createOffer().then((offer) => peer.setLocalDescription(offer)).catch((error) => {
-				logger.warn("preflight: failed to create offer", { error: error instanceof Error ? error.message : String(error) });
-				finish();
-			});
-		});
-		const rttMs = firstSrflxAt !== null ? Math.round(firstSrflxAt - start) : null;
-		let transport;
-		if (sawSrflx) transport = "udp";
-		else if (sawOtherCandidate) transport = "relay";
-		else transport = "failed";
-		return {
-			transport,
-			rttMs
-		};
-	} catch (error) {
-		logger.warn("preflight: connectivity probe threw", { error: error instanceof Error ? error.message : String(error) });
-		return {
-			transport: "failed",
-			rttMs: null
-		};
-	} finally {
-		try {
-			pc?.close();
-		} catch {}
-	}
-}
-/** Map probe metrics to a connectivity quality verdict. Pure. */
-function classifyConnectivity(metrics, thresholds) {
-	const reasons = [];
-	let quality;
-	if (metrics.transport === "failed") {
-		quality = "critical";
-		reasons.push("Could not establish any WebRTC connectivity (no ICE candidates gathered). Real-time streaming is unlikely to work on this network.");
-	} else if (metrics.transport === "relay") {
-		quality = "poor";
-		reasons.push("Direct UDP connectivity could not be confirmed; the session will need a TURN relay, which adds latency and can't be verified without starting a session.");
-	} else if (metrics.rttMs != null && metrics.rttMs > thresholds.marginalMs) {
-		quality = "poor";
-		reasons.push(`Network round-trip time is high (~${metrics.rttMs}ms > ${thresholds.marginalMs}ms); the real-time experience may feel laggy.`);
-	} else if (metrics.rttMs != null && metrics.rttMs > thresholds.goodMs) {
-		quality = "fair";
-		reasons.push(`Network round-trip time is elevated (~${metrics.rttMs}ms > ${thresholds.goodMs}ms).`);
-	} else quality = "good";
-	return {
-		quality,
-		metrics: {
-			transport: metrics.transport,
-			rttMs: metrics.rttMs
-		},
-		reasons
-	};
-}
+import { resolveFpsNumber } from "../../shared/model.js";
+import { REALTIME_CONFIG } from "../config-realtime.js";
+import { extractSignals, scoreLowerBetter, worst } from "../observability/connection-quality.js";
+import { classifyConnectivity, gatherIceCandidates } from "../preflight-connectivity.js";
+//#region src/realtime/browser/preflight.ts
 /**
 * Classify an active-probe result. Judges startup (TTFF) and steady-state
 * (mid-stream glass-to-glass) latency separately — both are real experienced
@@ -232,11 +131,18 @@ const ABORTED_DEEP_PROBE = {
 /** Unblock `promise` early when `signal` aborts. */
 function raceAbort(promise, signal) {
 	if (!signal) return promise;
-	if (signal.aborted) throw signal.reason ?? new DOMException("Aborted", "AbortError");
+	if (signal.aborted) throw signal.reason ?? createAbortError();
 	return Promise.race([promise, new Promise((_, reject) => {
-		signal.addEventListener("abort", () => reject(signal.reason ?? new DOMException("Aborted", "AbortError")), { once: true });
+		signal.addEventListener("abort", () => reject(signal.reason ?? createAbortError()), { once: true });
 	})]);
 }
+function createAbortError() {
+	const DOMExceptionConstructor = globalThis.DOMException;
+	if (typeof DOMExceptionConstructor === "function") return new DOMExceptionConstructor("Aborted", "AbortError");
+	const error = /* @__PURE__ */ new Error("Aborted");
+	error.name = "AbortError";
+	return error;
+}
 /** Resolve once enough g2g samples exist or the probe window elapses (never rejects). */
 function waitForProbe(getLatest, minSamples, durationMs, signal) {
 	return new Promise((resolve) => {

package/dist/realtime/browser/prepare-connection.js

@@ -0,0 +1,39 @@
+import { RealtimeObservability } from "../observability/realtime-observability.js";
+import { isDesktopSafari } from "../../utils/platform.js";
+import { createLiveKitMediaChannel } from "../media-channel.js";
+import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
+import { createBrowserGlassToGlassDiagnostics } from "./glass-to-glass-diagnostics.js";
+//#region src/realtime/browser/prepare-connection.ts
+const prepareBrowserConnection = ({ stream, mirror, debugQuality, preferredVideoCodec, fps, logger, observability: observabilityOptions }) => {
+	let inputStream = stream ?? new MediaStream();
+	let disposeMirroring = () => {};
+	if (mirror !== false) try {
+		const videoTrack = inputStream.getVideoTracks?.()[0];
+		if (videoTrack && (mirror === true || shouldMirrorTrack(videoTrack))) {
+			const mirrored = createMirroredStream(inputStream, { fps });
+			inputStream = mirrored.stream;
+			disposeMirroring = mirrored.dispose;
+		} else if (mirror === true && !videoTrack) logger.warn("mirror: true requested but no video track was found on the input stream");
+	} catch (error) {
+		logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
+	}
+	const observability = new RealtimeObservability({
+		...observabilityOptions,
+		glassToGlass: debugQuality ? createBrowserGlassToGlassDiagnostics() : void 0
+	});
+	if (debugQuality) inputStream = observability.attachOutgoingStream(inputStream, fps);
+	const safariCodec = isDesktopSafari() ? "vp8" : void 0;
+	return {
+		stream: inputStream,
+		observability,
+		videoCodec: safariCodec ?? preferredVideoCodec,
+		queryParams: {
+			...safariCodec ? { livekit_server_codec: safariCodec } : {},
+			...debugQuality ? { pixel_latency: "1" } : {}
+		},
+		createMediaChannel: createLiveKitMediaChannel,
+		dispose: disposeMirroring
+	};
+};
+//#endregion
+export { prepareBrowserConnection };

package/dist/realtime/client.d.ts

@@ -14,7 +14,7 @@ declare const realTimeClientInitialStateSchema: z.ZodObject<{
     text: z.ZodString;
     enhance: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
   }, z.core.$strip>>;
-  image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
+  image: z.ZodOptional<z.ZodUnion<readonly [z.ZodType<Blob, unknown, z.core.$ZodTypeInternals<Blob, unknown>>, z.ZodType<File, unknown, z.core.$ZodTypeInternals<File, unknown>>, z.ZodString]>>;
   passthrough: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
 type OnRemoteStreamFn = (stream: MediaStream) => void;
@@ -46,7 +46,7 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
       text: z.ZodString;
       enhance: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
     }, z.core.$strip>>;
-    image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
+    image: z.ZodOptional<z.ZodUnion<readonly [z.ZodType<Blob, unknown, z.core.$ZodTypeInternals<Blob, unknown>>, z.ZodType<File, unknown, z.core.$ZodTypeInternals<File, unknown>>, z.ZodString]>>;
     passthrough: z.ZodOptional<z.ZodBoolean>;
   }, z.core.$strip>>;
   queryParams: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
@@ -57,12 +57,19 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
   }>>;
   preferredVideoCodec: z.ZodOptional<z.ZodEnum<{
     h264: "h264";
+    vp8: "vp8";
     vp9: "vp9";
   }>>;
   debugQuality: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
-type RealTimeClientConnectOptions = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model"> & {
+type RealtimeMediaStream = {
+  getTracks(): unknown[];
+  getAudioTracks(): unknown[];
+  getVideoTracks(): unknown[];
+};
+type RealTimeClientConnectOptions<TStream extends RealtimeMediaStream = MediaStream> = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model" | "onRemoteStream"> & {
   model: ModelDefinition | CustomModelDefinition;
+  onRemoteStream: (stream: TStream) => void;
 };
 type Events = {
   connectionChange: ConnectionState;
@@ -101,4 +108,4 @@ type RealTimeClient = {
   replaceVideoTrack: (track: MediaStreamTrack) => Promise<void>;
 };
 //#endregion
-export { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState };
+export { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState, RealtimeMediaStream };

package/dist/realtime/client.js

@@ -1,14 +1,11 @@
 import { classifyWebrtcError } from "../utils/errors.js";
+import { createConsoleLogger } from "../utils/logger.js";
 import { isFileRefId } from "../files/types.js";
 import { modelDefinitionSchema, resolveFpsNumber } from "../shared/model.js";
 import { modelStateSchema } from "../shared/types.js";
-import { createConsoleLogger } from "../utils/logger.js";
 import { imageToBase64 } from "../utils/media.js";
-import { isDesktopSafari } from "../utils/platform.js";
 import { createEventBuffer } from "./event-buffer.js";
 import { realtimeMethods } from "./methods.js";
-import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
-import { RealtimeObservability } from "./observability/realtime-observability.js";
 import { StreamSession } from "./stream-session.js";
 import { z } from "zod";
 //#region src/realtime/client.ts
@@ -23,7 +20,11 @@ const realTimeClientConnectOptionsSchema = z.object({
 	queryParams: z.record(z.string(), z.string()).optional(),
 	mirror: z.union([z.literal("auto"), z.boolean()]).optional(),
 	resolution: z.enum(["720p", "1080p"]).optional(),
-	preferredVideoCodec: z.enum(["h264", "vp9"]).optional(),
+	preferredVideoCodec: z.enum([
+		"h264",
+		"vp8",
+		"vp9"
+	]).optional(),
 	debugQuality: z.boolean().optional()
 });
 const createRealTimeClient = (opts) => {
@@ -34,20 +35,10 @@ const createRealTimeClient = (opts) => {
 		if (!parsedOptions.success) throw parsedOptions.error;
 		const { onRemoteStream, onConnectionChange, onConnectionQuality, onQueuePosition, initialState, resolution, preferredVideoCodec } = parsedOptions.data;
 		const mirror = parsedOptions.data.mirror ?? false;
-		let inputStream = stream ?? new MediaStream();
-		let mirroredStream;
-		if (mirror !== false) try {
-			const firstVideoTrack = inputStream.getVideoTracks?.()[0];
-			if (firstVideoTrack && (mirror === true || shouldMirrorTrack(firstVideoTrack))) {
-				mirroredStream = createMirroredStream(inputStream, { fps: resolveFpsNumber(options.model.fps) });
-				inputStream = mirroredStream.stream;
-			} else if (mirror === true && !firstVideoTrack) logger.warn("mirror: true requested but no video track was found on the input stream");
-		} catch (error) {
-			logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
-		}
 		const debugQuality = parsedOptions.data.debugQuality ?? false;
 		let session;
 		let observability;
+		let preparedConnection;
 		try {
 			const initialImageRef = isFileRefId(initialState?.image) ? initialState.image : void 0;
 			const initialImage = initialImageRef === void 0 && initialState?.image ? await imageToBase64(initialState.image) : void 0;
@@ -57,41 +48,46 @@ const createRealTimeClient = (opts) => {
 			} : void 0;
 			const url = `${baseUrl}${options.model.urlPath}`;
 			const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer();
-			observability = new RealtimeObservability({
-				telemetryEnabled: opts.telemetryEnabled,
-				apiKey,
-				model: options.model.name,
-				integration,
+			preparedConnection = opts.prepareConnection({
+				stream,
+				mirror,
+				debugQuality,
+				preferredVideoCodec,
+				fps: resolveFpsNumber(options.model.fps),
 				logger,
-				onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
-				onStats: (stats) => emitOrBuffer("stats", stats),
-				onConnectionQuality: (report) => {
-					emitOrBuffer("connectionQuality", report);
-					onConnectionQuality?.(report);
-				},
-				debugQuality
+				observability: {
+					telemetryEnabled: opts.telemetryEnabled,
+					apiKey,
+					model: options.model.name,
+					integration,
+					logger,
+					onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
+					onStats: (stats) => emitOrBuffer("stats", stats),
+					onConnectionQuality: (report) => {
+						emitOrBuffer("connectionQuality", report);
+						onConnectionQuality?.(report);
+					}
+				}
 			});
-			if (debugQuality) inputStream = observability.attachOutgoingStream(inputStream, resolveFpsNumber(options.model.fps));
-			const safariCodec = isDesktopSafari() ? "vp8" : void 0;
-			const publishCodec = safariCodec ?? preferredVideoCodec;
+			observability = preparedConnection.observability;
 			session = new StreamSession({
 				url: `${url}?${new URLSearchParams({
-					...safariCodec ? { livekit_server_codec: safariCodec } : {},
+					...preparedConnection.queryParams ?? {},
 					...options.queryParams ?? {},
-					...debugQuality ? { pixel_latency: "1" } : {},
 					api_key: apiKey,
 					model: options.model.name,
 					...resolution ? { resolution } : {}
 				}).toString()}`,
 				integration,
 				observability,
-				localStream: inputStream,
+				localStream: preparedConnection.stream,
 				initialImage,
 				initialImageRef,
 				initialPrompt,
 				initialPassthrough: initialState?.passthrough,
 				logger,
-				videoCodec: publishCodec
+				videoCodec: preparedConnection.videoCodec,
+				createMediaChannel: preparedConnection.createMediaChannel
 			});
 			let sessionId = null;
 			let subscribeToken = null;
@@ -126,7 +122,7 @@ const createRealTimeClient = (opts) => {
 					observability?.stop();
 					stop();
 					activeSession.disconnect();
-					mirroredStream?.dispose();
+					preparedConnection?.dispose();
 				},
 				on: eventEmitter.on,
 				off: eventEmitter.off,
@@ -158,7 +154,7 @@ const createRealTimeClient = (opts) => {
 		} catch (error) {
 			observability?.stop();
 			session?.disconnect();
-			mirroredStream?.dispose();
+			preparedConnection?.dispose();
 			throw error;
 		}
 	};

package/dist/realtime/livekit.js

@@ -0,0 +1,26 @@
+import { createLiveKitInitializationError } from "../utils/errors.js";
+//#region src/realtime/livekit.ts
+let liveKitModulePromise;
+function validateLiveKitModule(module) {
+	const invalid = [
+		typeof module.Room !== "function" && "Room",
+		(typeof module.RoomEvent !== "object" || module.RoomEvent === null) && "RoomEvent",
+		(module.Track === null || typeof module.Track !== "object" && typeof module.Track !== "function") && "Track",
+		module.Track?.Source?.Camera !== "camera" && "Track.Source.Camera",
+		(typeof module.ConnectionState !== "object" || module.ConnectionState === null) && "ConnectionState"
+	].filter((name) => name !== false);
+	if (invalid.length > 0) throw createLiveKitInitializationError(`livekit-client is missing or has invalid required exports: ${invalid.join(", ")}`);
+	return module;
+}
+/** Load the browser-neutral LiveKit client only when realtime is actually used. */
+function loadLiveKitClient() {
+	liveKitModulePromise ??= import("livekit-client").then(validateLiveKitModule).catch((error) => {
+		liveKitModulePromise = void 0;
+		if (typeof error === "object" && error !== null && "code" in error && error.code === "LIVEKIT_INITIALIZATION_ERROR") throw error;
+		const cause = error instanceof Error ? error : new Error(String(error));
+		throw createLiveKitInitializationError(`Failed to initialize livekit-client: ${cause.message}`, cause);
+	});
+	return liveKitModulePromise;
+}
+//#endregion
+export { loadLiveKitClient };

package/dist/realtime/media-channel.js

@@ -1,13 +1,13 @@
 import { createConsoleLogger } from "../utils/logger.js";
 import { REALTIME_CONFIG } from "./config-realtime.js";
+import { loadLiveKitClient } from "./livekit.js";
 import mitt from "mitt";
-import { Room, RoomEvent, Track } from "livekit-client";
 //#region src/realtime/media-channel.ts
-function getDefaultVideoPublishOptions(videoCodec) {
+function getDefaultVideoPublishOptions(source, videoCodec) {
 	const resolvedCodec = videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec;
 	const maxBitrate = resolvedCodec === "vp9" ? REALTIME_CONFIG.livekit.vp9MaxVideoBitrateBps : REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps;
 	return {
-		source: Track.Source.Camera,
+		source,
 		videoCodec: resolvedCodec,
 		simulcast: resolvedCodec !== "vp9",
 		videoEncoding: {
@@ -16,8 +16,9 @@ function getDefaultVideoPublishOptions(videoCodec) {
 		}
 	};
 }
-var MediaChannel = class {
+var LiveKitMediaChannel = class {
 	room = null;
+	cameraTrackSource = null;
 	remoteStream = null;
 	events = mitt();
 	logger;
@@ -35,14 +36,16 @@ var MediaChannel = class {
 		this.events.off(event, handler);
 	}
 	async connect(opts) {
-		this.room ??= new Room(REALTIME_CONFIG.livekit.roomOptions);
+		const { Room: LiveKitRoom, RoomEvent, Track } = await loadLiveKitClient();
+		this.cameraTrackSource = Track.Source.Camera;
+		this.room ??= new LiveKitRoom(REALTIME_CONFIG.livekit.roomOptions);
 		const room = this.room;
 		room.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
 			if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
-			if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
+			if (track.kind !== "video" && track.kind !== "audio") return;
 			const mediaStreamTrack = track.mediaStreamTrack;
 			if (mediaStreamTrack) {
-				if (track.kind === Track.Kind.Video) this.config.observability?.attachRemoteVideoTrack(mediaStreamTrack);
+				if (track.kind === "video") this.config.observability?.attachRemoteVideoTrack(mediaStreamTrack);
 				const tracks = this.remoteStream?.getTracks() ?? [];
 				if (!tracks.includes(mediaStreamTrack)) tracks.push(mediaStreamTrack);
 				this.remoteStream = new MediaStream(tracks);
@@ -74,15 +77,19 @@ var MediaChannel = class {
 	disconnect() {
 		const room = this.room;
 		this.room = null;
+		this.cameraTrackSource = null;
 		this.remoteStream = null;
 		this.config.observability?.setLiveKitRoom(null);
 		if (room) room.disconnect().catch(() => {});
 	}
 	async publishTracks(stream) {
 		if (!this.room) return;
-		for (const track of stream.getTracks()) if (track.kind === "video") await this.room.localParticipant.publishTrack(track, getDefaultVideoPublishOptions(this.config.videoCodec));
-		else await this.room.localParticipant.publishTrack(track);
+		for (const track of stream.getTracks()) if (track.kind === "video") {
+			if (!this.cameraTrackSource) throw new Error("Cannot publish video track: media channel is not connected");
+			await this.room.localParticipant.publishTrack(track, getDefaultVideoPublishOptions(this.cameraTrackSource, this.config.videoCodec));
+		} else await this.room.localParticipant.publishTrack(track);
 	}
 };
+const createLiveKitMediaChannel = (config) => new LiveKitMediaChannel(config);
 //#endregion
-export { MediaChannel };
+export { createLiveKitMediaChannel };

package/dist/realtime/methods.d.ts

@@ -4,7 +4,7 @@ import { z } from "zod";
 declare const setInputSchema: z.ZodObject<{
   prompt: z.ZodOptional<z.ZodString>;
   enhance: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-  image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString, z.ZodNull]>>;
+  image: z.ZodOptional<z.ZodUnion<readonly [z.ZodType<Blob, unknown, z.core.$ZodTypeInternals<Blob, unknown>>, z.ZodType<File, unknown, z.core.$ZodTypeInternals<File, unknown>>, z.ZodString, z.ZodNull]>>;
 }, z.core.$strip>;
 type SetInput = z.input<typeof setInputSchema>;
 //#endregion

package/dist/realtime/methods.js

@@ -1,3 +1,4 @@
+import { globalInstanceSchema } from "../utils/runtime.js";
 import { isFileRefId } from "../files/types.js";
 import { REALTIME_CONFIG } from "./config-realtime.js";
 import { z } from "zod";
@@ -6,8 +7,8 @@ const setInputSchema = z.object({
 	prompt: z.string().min(1).optional(),
 	enhance: z.boolean().optional().default(true),
 	image: z.union([
-		z.instanceof(Blob),
-		z.instanceof(File),
+		globalInstanceSchema("Blob"),
+		globalInstanceSchema("File"),
 		z.string(),
 		z.null()
 	]).optional()

package/dist/realtime/observability/g2g.d.ts

@@ -0,0 +1,14 @@
+//#region src/realtime/observability/g2g.d.ts
+type G2GMetrics = {
+  /** Time from connection start to the first rendered output frame. */
+  ttffMs: number | null;
+  /** Median steady-state camera-to-display latency. */
+  medianMs: number | null;
+  /** 90th percentile steady-state camera-to-display latency. */
+  p90Ms: number | null;
+  sampleCount: number;
+  /** Recent stamped frames that never returned, from 0 to 1. */
+  dropRatio: number | null;
+};
+//#endregion
+export { G2GMetrics };

package/dist/realtime/observability/realtime-observability.js

@@ -1,6 +1,5 @@
 import { REALTIME_CONFIG } from "../config-realtime.js";
 import { ConnectionQualityEvaluator } from "./connection-quality.js";
-import { SeqTracker, createMarkerReader, createStampPump } from "./glass-to-glass.js";
 import { createLiveKitStatsProvider } from "./livekit-stats-provider.js";
 import { NullTelemetryReporter, TelemetryReporter } from "./telemetry-reporter.js";
 import { WebRTCStatsCollector } from "./webrtc-stats.js";
@@ -16,14 +15,10 @@ var RealtimeObservability = class {
 	stallStartMs = 0;
 	connectionBreakdown = null;
 	connectionQuality = new ConnectionQualityEvaluator();
-	/** Glass-to-glass: shared tracker, outgoing stamp pump, and incoming reader. Null unless `debugQuality`. */
-	seqTracker;
-	markerReader;
-	stampPump = null;
+	glassToGlass;
 	constructor(options) {
 		this.options = options;
-		this.seqTracker = options.debugQuality ? new SeqTracker() : null;
-		this.markerReader = this.seqTracker ? createMarkerReader(this.seqTracker) : null;
+		this.glassToGlass = options.glassToGlass;
 	}
 	/**
 	* Wrap the outgoing stream so each frame carries a marker, feeding the shared
@@ -32,13 +27,9 @@ var RealtimeObservability = class {
 	* writer and reader of the tracker share one lifecycle and survive reconnects.
 	*/
 	attachOutgoingStream(stream, fps) {
-		if (!this.seqTracker) return stream;
+		if (!this.glassToGlass) return stream;
 		try {
-			this.stampPump = createStampPump(stream, {
-				tracker: this.seqTracker,
-				fps
-			});
-			return this.stampPump.stream;
+			return this.glassToGlass.attachOutgoingStream(stream, fps);
 		} catch (error) {
 			this.options.logger.warn("Failed to start glass-to-glass stamp pump; continuing without it", { error: error instanceof Error ? error.message : String(error) });
 			return stream;
@@ -46,7 +37,7 @@ var RealtimeObservability = class {
 	}
 	/** Feed the remote video track to the marker reader (called from the media channel). */
 	attachRemoteVideoTrack(track) {
-		this.markerReader?.attach(track);
+		this.glassToGlass?.attachRemoteVideoTrack(track);
 	}
 	/**
 	* Mark the start of a connect attempt for glass-to-glass timing (resets the
@@ -55,7 +46,7 @@ var RealtimeObservability = class {
 	* measurement is on.
 	*/
 	markGlassToGlassStart() {
-		this.seqTracker?.markStart(performance.now());
+		this.glassToGlass?.markStart();
 	}
 	diagnostic(name, data, timestamp = Date.now()) {
 		this.options.logger.debug(name, data);
@@ -163,8 +154,7 @@ var RealtimeObservability = class {
 	}
 	stop() {
 		this.stopStats();
-		this.stampPump?.dispose();
-		this.markerReader?.dispose();
+		this.glassToGlass?.dispose();
 		this.telemetryReporter.stop();
 		this.telemetryReporter = new NullTelemetryReporter();
 		this.telemetryReporterReady = false;
@@ -175,7 +165,7 @@ var RealtimeObservability = class {
 		return this.connectionQuality.current();
 	}
 	handleStats(stats) {
-		if (this.seqTracker) stats.glassToGlass = this.seqTracker.snapshot();
+		if (this.glassToGlass) stats.glassToGlass = this.glassToGlass.snapshot();
 		this.options.onStats?.(stats);
 		this.telemetryReporter.addStats(stats);
 		this.detectVideoStall(stats);

package/dist/realtime/observability/webrtc-stats.d.ts

@@ -1,4 +1,4 @@
-import { G2GMetrics } from "./glass-to-glass.js";
+import { G2GMetrics } from "./g2g.js";
 
 //#region src/realtime/observability/webrtc-stats.d.ts
 type WebRTCStats = {