@decartai/sdk 0.1.13 → 0.1.15

package/dist/realtime/methods.js

@@ -1,3 +1,4 @@
+import { globalInstanceSchema } from "../utils/runtime.js";
 import { isFileRefId } from "../files/types.js";
 import { REALTIME_CONFIG } from "./config-realtime.js";
 import { z } from "zod";
@@ -5,9 +6,13 @@ import { z } from "zod";
 const setInputSchema = z.object({
 	prompt: z.string().min(1).optional(),
 	enhance: z.boolean().optional().default(true),
+	/**
+	* - `Blob`/`File`/data:/http(s):/base64 string: bytes traverse the wire as base64.
+	* - `"file_..."` id (from `client.files.upload(...).id`): sent as a server-side reference.
+	*/
 	image: z.union([
-		z.instanceof(Blob),
-		z.instanceof(File),
+		globalInstanceSchema("Blob"),
+		globalInstanceSchema("File"),
 		z.string(),
 		z.null()
 	]).optional()

package/dist/realtime/observability/connection-quality.d.ts

@@ -17,7 +17,7 @@ type ConnectionQualityMetrics = {
   /**
    * Mid-stream (steady-state) glass-to-glass latency (ms) — the real per-frame
    * camera→display latency through the model, excluding startup. Only populated
-   * when the opt-in pixel-marker measurement is on (`connect({ debugQuality: true })`)
+   * when the opt-in frame-metadata measurement is on (`connect({ debugQuality: true })`)
    * and past warm-up; null otherwise. When present it drives the latency verdict
    * instead of `rttMs`.
    */
@@ -36,8 +36,8 @@ type ConnectionQualityMetrics = {
   /** Server's view of upstream (client→server) jitter in ms, or null. Observational. */
   upstreamJitterMs: number | null;
   /**
-   * End-to-end frame drop ratio (0–1) inferred from the pixel-marker seq stream.
-   * Only populated under the opt-in g2g measurement; null otherwise.
+   * End-to-end frame drop ratio (0–1). Null until frame identity is propagated
+   * end-to-end; latency measurement itself only requires the capture timestamp.
    */
   g2gDropRatio: number | null;
   /** Estimated available upstream bandwidth in kbps. Chromium-only — null on Safari/Firefox. */

package/dist/realtime/observability/connection-quality.js

@@ -76,6 +76,7 @@ function minOrNull(values) {
 	return values.length === 0 ? null : Math.min(...values);
 }
 var RingBuffer = class {
+	size;
 	values = [];
 	constructor(size) {
 		this.size = size;
@@ -101,6 +102,7 @@ var RingBuffer = class {
 * warm-up state changes; `current()` returns the latest at any time.
 */
 var ConnectionQualityEvaluator = class {
+	thresholds;
 	rtt;
 	glassToGlass;
 	loss;
@@ -216,4 +218,4 @@ var ConnectionQualityEvaluator = class {
 	}
 };
 //#endregion
-export { ConnectionQualityEvaluator, extractSignals, scoreLowerBetter, worst };
+export { ConnectionQualityEvaluator, RANK, extractSignals, scoreLowerBetter, scoreMetrics, worst };

package/dist/realtime/observability/g2g.d.ts

@@ -0,0 +1,14 @@
+//#region src/realtime/observability/g2g.d.ts
+type G2GMetrics = {
+  /** Time from connection start to the first rendered output frame. */
+  ttffMs: number | null;
+  /** Median steady-state camera-to-display latency. */
+  medianMs: number | null;
+  /** 90th percentile steady-state camera-to-display latency. */
+  p90Ms: number | null;
+  sampleCount: number;
+  /** Recent stamped frames that never returned, from 0 to 1. */
+  dropRatio: number | null;
+};
+//#endregion
+export { G2GMetrics };

package/dist/realtime/observability/realtime-observability.js

@@ -1,11 +1,11 @@
 import { REALTIME_CONFIG } from "../config-realtime.js";
 import { ConnectionQualityEvaluator } from "./connection-quality.js";
-import { SeqTracker, createMarkerReader, createStampPump } from "./glass-to-glass.js";
 import { createLiveKitStatsProvider } from "./livekit-stats-provider.js";
 import { NullTelemetryReporter, TelemetryReporter } from "./telemetry-reporter.js";
 import { WebRTCStatsCollector } from "./webrtc-stats.js";
 //#region src/realtime/observability/realtime-observability.ts
 var RealtimeObservability = class {
+	options;
 	telemetryReporter = new NullTelemetryReporter();
 	telemetryReporterReady = false;
 	pendingTelemetryDiagnostics = [];
@@ -16,37 +16,14 @@ var RealtimeObservability = class {
 	stallStartMs = 0;
 	connectionBreakdown = null;
 	connectionQuality = new ConnectionQualityEvaluator();
-	/** Glass-to-glass: shared tracker, outgoing stamp pump, and incoming reader. Null unless `debugQuality`. */
-	seqTracker;
-	markerReader;
-	stampPump = null;
+	glassToGlass;
 	constructor(options) {
 		this.options = options;
-		this.seqTracker = options.debugQuality ? new SeqTracker() : null;
-		this.markerReader = this.seqTracker ? createMarkerReader(this.seqTracker) : null;
+		this.glassToGlass = options.glassToGlass;
 	}
-	/**
-	* Wrap the outgoing stream so each frame carries a marker, feeding the shared
-	* tracker that the reader matches against. Returns the stream to publish
-	* (unchanged when g2g is off or stamping can't start). Owned here so the
-	* writer and reader of the tracker share one lifecycle and survive reconnects.
-	*/
-	attachOutgoingStream(stream, fps) {
-		if (!this.seqTracker) return stream;
-		try {
-			this.stampPump = createStampPump(stream, {
-				tracker: this.seqTracker,
-				fps
-			});
-			return this.stampPump.stream;
-		} catch (error) {
-			this.options.logger.warn("Failed to start glass-to-glass stamp pump; continuing without it", { error: error instanceof Error ? error.message : String(error) });
-			return stream;
-		}
-	}
-	/** Feed the remote video track to the marker reader (called from the media channel). */
+	/** Feed the LiveKit remote video track to the frame-metadata reader. */
 	attachRemoteVideoTrack(track) {
-		this.markerReader?.attach(track);
+		this.glassToGlass?.attachRemoteVideoTrack(track);
 	}
 	/**
 	* Mark the start of a connect attempt for glass-to-glass timing (resets the
@@ -55,7 +32,7 @@ var RealtimeObservability = class {
 	* measurement is on.
 	*/
 	markGlassToGlassStart() {
-		this.seqTracker?.markStart(performance.now());
+		this.glassToGlass?.markStart();
 	}
 	diagnostic(name, data, timestamp = Date.now()) {
 		this.options.logger.debug(name, data);
@@ -163,8 +140,7 @@ var RealtimeObservability = class {
 	}
 	stop() {
 		this.stopStats();
-		this.stampPump?.dispose();
-		this.markerReader?.dispose();
+		this.glassToGlass?.dispose();
 		this.telemetryReporter.stop();
 		this.telemetryReporter = new NullTelemetryReporter();
 		this.telemetryReporterReady = false;
@@ -175,7 +151,7 @@ var RealtimeObservability = class {
 		return this.connectionQuality.current();
 	}
 	handleStats(stats) {
-		if (this.seqTracker) stats.glassToGlass = this.seqTracker.snapshot();
+		if (this.glassToGlass) stats.glassToGlass = this.glassToGlass.snapshot();
 		this.options.onStats?.(stats);
 		this.telemetryReporter.addStats(stats);
 		this.detectVideoStall(stats);

package/dist/realtime/observability/webrtc-stats.d.ts

@@ -1,4 +1,4 @@
-import { G2GMetrics } from "./glass-to-glass.js";
+import { G2GMetrics } from "./g2g.js";
 
 //#region src/realtime/observability/webrtc-stats.d.ts
 type WebRTCStats = {
@@ -133,8 +133,8 @@ type WebRTCStats = {
   };
   /**
    * True glass-to-glass latency + end-to-end drop signal, merged in by
-   * `RealtimeObservability` when the opt-in pixel-marker measurement is active
-   * (see glass-to-glass.ts). Null otherwise — the stats collector does not
+   * `RealtimeObservability` when the opt-in frame-metadata measurement is active.
+   * Null otherwise — the stats collector does not
    * populate it.
    */
   glassToGlass: G2GMetrics | null;
@@ -154,4 +154,4 @@ type IceCandidatePair = {
   remote: IceCandidateInfo;
 };
 //#endregion
-export { WebRTCStats };
+export { IceCandidateInfo, IceCandidatePair, WebRTCStats };

package/dist/realtime/preflight-connectivity.js

@@ -0,0 +1,102 @@
+//#region src/realtime/preflight-connectivity.ts
+function candidateType(candidate) {
+	if (candidate.type) return candidate.type;
+	return /\btyp (\w+)/.exec(candidate.candidate)?.[1] ?? "";
+}
+function nowMs() {
+	return globalThis.performance?.now?.() ?? Date.now();
+}
+async function gatherIceCandidates(iceServers, timeoutMs, signal, logger) {
+	if (signal?.aborted) return {
+		transport: "failed",
+		rttMs: null
+	};
+	const PC = globalThis?.RTCPeerConnection;
+	if (typeof PC !== "function") {
+		logger.warn("preflight: RTCPeerConnection unavailable in this environment");
+		return {
+			transport: "failed",
+			rttMs: null
+		};
+	}
+	let pc = null;
+	try {
+		pc = new PC({ iceServers });
+		pc.createDataChannel("decart-preflight");
+		let sawSrflx = false;
+		let sawOtherCandidate = false;
+		let firstSrflxAt = null;
+		const start = nowMs();
+		await new Promise((resolve) => {
+			let settled = false;
+			let timer;
+			const finish = () => {
+				if (settled) return;
+				settled = true;
+				clearTimeout(timer);
+				signal?.removeEventListener("abort", finish);
+				resolve();
+			};
+			timer = setTimeout(finish, timeoutMs);
+			signal?.addEventListener("abort", finish, { once: true });
+			if (signal?.aborted) finish();
+			const peer = pc;
+			peer.onicecandidate = (event) => {
+				if (!event.candidate || event.candidate.candidate === "") return finish();
+				if (candidateType(event.candidate) === "srflx") {
+					sawSrflx = true;
+					if (firstSrflxAt === null) firstSrflxAt = nowMs();
+				} else sawOtherCandidate = true;
+			};
+			peer.onicegatheringstatechange = () => {
+				if (peer.iceGatheringState === "complete") finish();
+			};
+			peer.createOffer().then((offer) => peer.setLocalDescription(offer)).catch((error) => {
+				logger.warn("preflight: failed to create offer", { error: error instanceof Error ? error.message : String(error) });
+				finish();
+			});
+		});
+		const rttMs = firstSrflxAt !== null ? Math.round(firstSrflxAt - start) : null;
+		return {
+			transport: sawSrflx ? "udp" : sawOtherCandidate ? "relay" : "failed",
+			rttMs
+		};
+	} catch (error) {
+		logger.warn("preflight: connectivity probe threw", { error: error instanceof Error ? error.message : String(error) });
+		return {
+			transport: "failed",
+			rttMs: null
+		};
+	} finally {
+		try {
+			pc?.close();
+		} catch {}
+	}
+}
+function classifyConnectivity(metrics, thresholds) {
+	const reasons = [];
+	let quality;
+	if (metrics.transport === "failed") {
+		quality = "critical";
+		reasons.push("Could not establish any WebRTC connectivity (no ICE candidates gathered). Real-time streaming is unlikely to work on this network.");
+	} else if (metrics.transport === "relay") {
+		quality = "poor";
+		reasons.push("Direct UDP connectivity could not be confirmed; the session will need a TURN relay, which adds latency and can't be verified without starting a session.");
+	} else if (metrics.rttMs != null && metrics.rttMs > thresholds.marginalMs) {
+		quality = "poor";
+		reasons.push(`Network round-trip time is high (~${metrics.rttMs}ms > ${thresholds.marginalMs}ms); the real-time experience may feel laggy.`);
+	} else if (metrics.rttMs != null && metrics.rttMs > thresholds.goodMs) {
+		quality = "fair";
+		reasons.push(`Network round-trip time is elevated (~${metrics.rttMs}ms > ${thresholds.goodMs}ms).`);
+	} else quality = "good";
+	return {
+		quality,
+		metrics: {
+			transport: metrics.transport,
+			rttMs: metrics.rttMs
+		},
+		reasons
+	};
+}
+//#endregion
+export { classifyConnectivity, gatherIceCandidates };

package/dist/realtime/preflight-types.d.ts

@@ -0,0 +1,30 @@
+import { CustomModelDefinition, ModelDefinition } from "../shared/model.js";
+import { ConnectionQuality } from "./observability/connection-quality.js";
+
+//#region src/realtime/preflight-types.d.ts
+type ConnectivityTransport = "udp" | "relay" | "failed";
+type ConnectivityMetrics = {
+  transport: ConnectivityTransport;
+  rttMs: number | null;
+  g2gMs?: number | null;
+  ttffMs?: number | null;
+  g2gDropRatio?: number | null;
+  upstreamJitterMs?: number | null;
+  packetLoss?: number | null;
+  sampleCount?: number;
+};
+type ConnectivityReport = {
+  quality: ConnectionQuality;
+  metrics: ConnectivityMetrics;
+  reasons: string[];
+};
+type CheckConnectivityOptions = {
+  iceServers?: RTCIceServer[];
+  iceGatherTimeoutMs?: number;
+  signal?: AbortSignal;
+  deep?: boolean;
+  model?: ModelDefinition | CustomModelDefinition;
+  durationMs?: number;
+};
+//#endregion
+export { CheckConnectivityOptions, ConnectivityMetrics, ConnectivityReport, ConnectivityTransport };

package/dist/realtime/react-native/index.js

@@ -0,0 +1,36 @@
+import { createRealTimeClient } from "../client.js";
+import { createRealTimeSubscribeClient } from "../subscribe-client.js";
+import { createReactNativePreflight } from "./preflight.js";
+import { assertReactNativeReady, prepareReactNativeConnection, unsupportedReactNativeFeature } from "./prepare-connection.js";
+//#region src/realtime/react-native/index.ts
+const createReactNativeRealtime = (options) => {
+	const publish = createRealTimeClient({
+		baseUrl: options.publishBaseUrl,
+		apiKey: options.apiKey,
+		integration: options.integration,
+		logger: options.logger,
+		telemetryEnabled: options.telemetryEnabled,
+		prepareConnection: prepareReactNativeConnection
+	});
+	const subscribe = createRealTimeSubscribeClient({
+		baseUrl: options.subscribeBaseUrl,
+		apiKey: options.apiKey,
+		integration: options.integration,
+		logger: options.logger
+	});
+	const preflight = createReactNativePreflight({ logger: options.logger });
+	return {
+		connect: (stream, connectOptions) => publish.connect(stream, connectOptions),
+		subscribe: (subscribeOptions) => {
+			assertReactNativeReady();
+			return subscribe.subscribe(subscribeOptions);
+		},
+		checkConnectivity: async (checkOptions = {}) => {
+			assertReactNativeReady();
+			if (checkOptions.deep) unsupportedReactNativeFeature("Deep connectivity preflight");
+			return preflight.checkConnectivity(checkOptions);
+		}
+	};
+};
+//#endregion
+export { createReactNativeRealtime };

package/dist/realtime/react-native/preflight.js

@@ -0,0 +1,10 @@
+import { REALTIME_CONFIG } from "../config-realtime.js";
+import { classifyConnectivity, gatherIceCandidates } from "../preflight-connectivity.js";
+//#region src/realtime/react-native/preflight.ts
+function createReactNativePreflight({ logger }) {
+	return { checkConnectivity: async (options = {}) => {
+		return classifyConnectivity(await gatherIceCandidates(options.iceServers ?? REALTIME_CONFIG.preflight.defaultStunUrls.map((urls) => ({ urls })), options.iceGatherTimeoutMs ?? REALTIME_CONFIG.preflight.iceGatherTimeoutMs, options.signal, logger), REALTIME_CONFIG.preflight.rtt);
+	} };
+}
+//#endregion
+export { createReactNativePreflight };

package/dist/realtime/react-native/prepare-connection.js

@@ -0,0 +1,26 @@
+import { createReactNativeSetupRequiredError, createUnsupportedPlatformFeatureError } from "../../utils/errors.js";
+import { RealtimeObservability } from "../observability/realtime-observability.js";
+import { missingReactNativeRealtimeGlobals } from "../../utils/platform.js";
+import { createLiveKitMediaChannel } from "../media-channel.js";
+//#region src/realtime/react-native/prepare-connection.ts
+function assertReactNativeReady() {
+	const missing = missingReactNativeRealtimeGlobals();
+	if (missing.length > 0) throw createReactNativeSetupRequiredError(missing);
+}
+function unsupportedReactNativeFeature(feature) {
+	throw createUnsupportedPlatformFeatureError(feature, "React Native");
+}
+const prepareReactNativeConnection = ({ stream, mirror, debugQuality, preferredVideoCodec, observability: observabilityOptions }) => {
+	assertReactNativeReady();
+	if (mirror !== false) unsupportedReactNativeFeature("Outgoing video mirroring");
+	if (debugQuality) unsupportedReactNativeFeature("debugQuality");
+	return {
+		stream: stream ?? new MediaStream(),
+		observability: new RealtimeObservability(observabilityOptions),
+		videoCodec: preferredVideoCodec,
+		createMediaChannel: createLiveKitMediaChannel,
+		dispose: () => {}
+	};
+};
+//#endregion
+export { assertReactNativeReady, prepareReactNativeConnection, unsupportedReactNativeFeature };

package/dist/realtime/signaling-channel.js

@@ -36,6 +36,7 @@ function buildInitialStateRequest(initialState) {
 	return null;
 }
 var SignalingChannel = class {
+	config;
 	ws = null;
 	events = mitt();
 	pendingAcks = [];
@@ -66,7 +67,8 @@ var SignalingChannel = class {
 		const userSetInitialState = opts.initialState != null && (opts.initialState.image != null || opts.initialState.imageRef != null || opts.initialState.prompt != null);
 		const joinMessage = {
 			type: "livekit_join",
-			passthrough: opts.passthrough ?? !userSetInitialState
+			passthrough: opts.passthrough ?? !userSetInitialState,
+			...opts.frameTiming ? { client_capabilities: { frame_timing: true } } : {}
 		};
 		if (!this.writeMessage(joinMessage)) {
 			roomInfoWait.cancel();

package/dist/realtime/stream-session.js

@@ -1,6 +1,5 @@
 import { createConsoleLogger } from "../utils/logger.js";
 import { REALTIME_CONFIG } from "./config-realtime.js";
-import { MediaChannel } from "./media-channel.js";
 import { SignalingChannel } from "./signaling-channel.js";
 import mitt from "mitt";
 import pRetry, { AbortError } from "p-retry";
@@ -17,6 +16,7 @@ function getInitialImageSizeKb(image) {
 	return Math.max(0, Math.round(bytes / 1024));
 }
 var StreamSession = class {
+	config;
 	signaling;
 	media;
 	events = mitt();
@@ -110,7 +110,8 @@ var StreamSession = class {
 			const { roomInfo, initialStateAck } = await this.signaling.openAndJoin({
 				connectTimeout: REALTIME_CONFIG.session.connectionTimeoutMs,
 				initialState,
-				passthrough: this.config.initialPassthrough
+				passthrough: this.config.initialPassthrough,
+				frameTiming: this.config.frameTiming
 			});
 			this.watchInitialStateAck(initialStateAck, attempt);
 			if (this.disposed || this.currentAttempt !== attempt) {
@@ -238,7 +239,7 @@ var StreamSession = class {
 			logger: this.logger,
 			observability: this.config.observability
 		});
-		this.media = new MediaChannel({
+		this.media = this.config.createMediaChannel({
 			observability: this.config.observability,
 			localStream: this.config.localStream,
 			logger: this.logger,
@@ -267,4 +268,4 @@ var StreamSession = class {
 	}
 };
 //#endregion
-export { StreamSession };
+export { StreamSession, encodeSubscribeToken };

package/dist/realtime/subscribe-client.js

@@ -2,8 +2,8 @@ import { classifyWebrtcError } from "../utils/errors.js";
 import { createConsoleLogger } from "../utils/logger.js";
 import { createEventBuffer } from "./event-buffer.js";
 import { REALTIME_CONFIG } from "./config-realtime.js";
+import { loadLiveKitClient } from "./livekit.js";
 import { RealtimeObservability } from "./observability/realtime-observability.js";
-import { ConnectionState, Room, RoomEvent, Track } from "livekit-client";
 //#region src/realtime/subscribe-client.ts
 function decodeSubscribeToken(token) {
 	try {
@@ -16,11 +16,11 @@ function decodeSubscribeToken(token) {
 }
 function mapLiveKitState(state) {
 	switch (state) {
-		case ConnectionState.Connecting: return "connecting";
-		case ConnectionState.Connected: return "connected";
-		case ConnectionState.Reconnecting:
-		case ConnectionState.SignalReconnecting: return "reconnecting";
-		case ConnectionState.Disconnected: return "disconnected";
+		case "connecting": return "connecting";
+		case "connected": return "connected";
+		case "reconnecting":
+		case "signalReconnecting": return "reconnecting";
+		case "disconnected": return "disconnected";
 		default: return "disconnected";
 	}
 }
@@ -63,6 +63,7 @@ const createRealTimeSubscribeClient = (opts) => {
 			emitOrBuffer("connectionChange", state);
 		};
 		try {
+			const { Room: LiveKitRoom, RoomEvent } = await loadLiveKitClient();
 			observability = new RealtimeObservability({
 				telemetryEnabled: false,
 				apiKey,
@@ -76,11 +77,11 @@ const createRealTimeSubscribeClient = (opts) => {
 				apiKey,
 				roomName
 			});
-			room = new Room(REALTIME_CONFIG.livekit.roomOptions);
+			room = new LiveKitRoom(REALTIME_CONFIG.livekit.roomOptions);
 			const activeRoom = room;
 			activeRoom.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
 				if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
-				if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
+				if (track.kind !== "video" && track.kind !== "audio") return;
 				const mediaStreamTrack = track.mediaStreamTrack;
 				if (!mediaStreamTrack) return;
 				const tracks = remoteStream?.getTracks() ?? [];
@@ -98,7 +99,7 @@ const createRealTimeSubscribeClient = (opts) => {
 			observability.setLiveKitRoom(activeRoom);
 			setState("connected");
 			const client = {
-				isConnected: () => activeRoom.state === ConnectionState.Connected,
+				isConnected: () => activeRoom.state === "connected",
 				getConnectionState: () => mapLiveKitState(activeRoom.state),
 				disconnect: () => {
 					observability?.stop();
@@ -121,4 +122,4 @@ const createRealTimeSubscribeClient = (opts) => {
 	return { subscribe };
 };
 //#endregion
-export { createRealTimeSubscribeClient };
+export { createRealTimeSubscribeClient, decodeSubscribeToken };