@decartai/sdk 0.1.4 → 0.1.6

package/dist/realtime/observability/realtime-observability.js

@@ -1,4 +1,6 @@
 import { REALTIME_CONFIG } from "../config-realtime.js";
+import { ConnectionQualityEvaluator } from "./connection-quality.js";
+import { SeqTracker, createMarkerReader, createStampPump } from "./glass-to-glass.js";
 import { createLiveKitStatsProvider } from "./livekit-stats-provider.js";
 import { NullTelemetryReporter, TelemetryReporter } from "./telemetry-reporter.js";
 import { WebRTCStatsCollector } from "./webrtc-stats.js";
@@ -13,8 +15,47 @@ var RealtimeObservability = class {
 	videoStalled = false;
 	stallStartMs = 0;
 	connectionBreakdown = null;
+	connectionQuality = new ConnectionQualityEvaluator();
+	/** Glass-to-glass: shared tracker, outgoing stamp pump, and incoming reader. Null unless `debugQuality`. */
+	seqTracker;
+	markerReader;
+	stampPump = null;
 	constructor(options) {
 		this.options = options;
+		this.seqTracker = options.debugQuality ? new SeqTracker() : null;
+		this.markerReader = this.seqTracker ? createMarkerReader(this.seqTracker) : null;
+	}
+	/**
+	* Wrap the outgoing stream so each frame carries a marker, feeding the shared
+	* tracker that the reader matches against. Returns the stream to publish
+	* (unchanged when g2g is off or stamping can't start). Owned here so the
+	* writer and reader of the tracker share one lifecycle and survive reconnects.
+	*/
+	attachOutgoingStream(stream, fps) {
+		if (!this.seqTracker) return stream;
+		try {
+			this.stampPump = createStampPump(stream, {
+				tracker: this.seqTracker,
+				fps
+			});
+			return this.stampPump.stream;
+		} catch (error) {
+			this.options.logger.warn("Failed to start glass-to-glass stamp pump; continuing without it", { error: error instanceof Error ? error.message : String(error) });
+			return stream;
+		}
+	}
+	/** Feed the remote video track to the marker reader (called from the media channel). */
+	attachRemoteVideoTrack(track) {
+		this.markerReader?.attach(track);
+	}
+	/**
+	* Mark the start of a connect attempt for glass-to-glass timing (resets the
+	* tracker and starts the TTFF clock). Called at the top of each connect/
+	* reconnect so TTFF measures the full setup→first-frame wait. No-op unless g2g
+	* measurement is on.
+	*/
+	markGlassToGlassStart() {
+		this.seqTracker?.markStart(performance.now());
 	}
 	diagnostic(name, data, timestamp = Date.now()) {
 		this.options.logger.debug(name, data);
@@ -99,7 +140,7 @@ var RealtimeObservability = class {
 		this.stopStats();
 		this.resetStallDetection();
 		this.statsCollectorSource = source;
-		if (!this.options.telemetryEnabled && !this.options.onStats) return;
+		if (!this.options.telemetryEnabled && !this.options.onStats && !this.options.onConnectionQuality) return;
 		this.statsCollector = new WebRTCStatsCollector();
 		this.statsCollector.start(source, (stats) => this.handleStats(stats));
 	}
@@ -122,16 +163,24 @@ var RealtimeObservability = class {
 	}
 	stop() {
 		this.stopStats();
+		this.stampPump?.dispose();
+		this.markerReader?.dispose();
 		this.telemetryReporter.stop();
 		this.telemetryReporter = new NullTelemetryReporter();
 		this.telemetryReporterReady = false;
 		this.pendingTelemetryDiagnostics.length = 0;
 		this.connectionBreakdown = null;
 	}
+	getConnectionQuality() {
+		return this.connectionQuality.current();
+	}
 	handleStats(stats) {
+		if (this.seqTracker) stats.glassToGlass = this.seqTracker.snapshot();
 		this.options.onStats?.(stats);
 		this.telemetryReporter.addStats(stats);
 		this.detectVideoStall(stats);
+		const report = this.connectionQuality.update(stats);
+		if (report) this.options.onConnectionQuality?.(report);
 	}
 	detectVideoStall(stats) {
 		const fps = stats.video?.framesPerSecond ?? 0;
@@ -167,6 +216,7 @@ var RealtimeObservability = class {
 	resetStallDetection() {
 		this.videoStalled = false;
 		this.stallStartMs = 0;
+		this.connectionQuality.reset();
 	}
 };
 //#endregion

package/dist/realtime/observability/telemetry-reporter.js

@@ -2,6 +2,7 @@ import { VERSION } from "../../version.js";
 import { buildAuthHeaders } from "../../shared/request.js";
 import { REALTIME_CONFIG } from "../config-realtime.js";
 //#region src/realtime/observability/telemetry-reporter.ts
+const KEEPALIVE_MAX_BODY_BYTES = 60 * 1024;
 /** No-op reporter that silently discards all data. Used when telemetry is disabled. */
 var NullTelemetryReporter = class {
 	start() {}
@@ -43,14 +44,13 @@ var TelemetryReporter = class {
 	flush() {
 		this.sendReport();
 	}
-	/** Stop the reporter and discard any buffered data. */
+	/** Stop the reporter and make one final best-effort flush. */
 	stop() {
 		if (this.intervalId !== null) {
 			clearInterval(this.intervalId);
 			this.intervalId = null;
 		}
-		this.statsBuffer = [];
-		this.diagnosticsBuffer = [];
+		this.sendReport({ keepalive: true });
 	}
 	/**
 	* Build a single chunk from the front of the buffers, respecting the configured report item cap.
@@ -74,7 +74,7 @@ var TelemetryReporter = class {
 			diagnostics: this.diagnosticsBuffer.splice(0, REALTIME_CONFIG.observability.telemetryMaxItemsPerReport)
 		};
 	}
-	sendReport() {
+	sendReport(options = {}) {
 		if (this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0) return;
 		try {
 			const commonHeaders = {
@@ -84,15 +84,22 @@ var TelemetryReporter = class {
 				}),
 				"Content-Type": "application/json"
 			};
-			let chunk = this.createReportChunk();
-			while (chunk !== null) {
+			const chunks = [];
+			let nextChunk = this.createReportChunk();
+			while (nextChunk !== null) {
+				chunks.push(nextChunk);
+				nextChunk = this.createReportChunk();
+			}
+			chunks.forEach((chunk, index) => {
+				const body = JSON.stringify(chunk);
+				const useKeepalive = options.keepalive && index === chunks.length - 1 && new TextEncoder().encode(body).byteLength <= KEEPALIVE_MAX_BODY_BYTES;
 				fetch(REALTIME_CONFIG.observability.telemetryUrl, {
 					method: "POST",
 					headers: commonHeaders,
-					body: JSON.stringify(chunk)
+					body,
+					...useKeepalive ? { keepalive: true } : {}
 				}).catch(() => {});
-				chunk = this.createReportChunk();
-			}
+			});
 		} catch {}
 	}
 };

package/dist/realtime/observability/webrtc-stats.d.ts

@@ -1,3 +1,5 @@
+import { G2GMetrics } from "./glass-to-glass.js";
+
 //#region src/realtime/observability/webrtc-stats.d.ts
 type WebRTCStats = {
   timestamp: number;
@@ -129,6 +131,13 @@ type WebRTCStats = {
      */
     selectedCandidatePairs: IceCandidatePair[];
   };
+  /**
+   * True glass-to-glass latency + end-to-end drop signal, merged in by
+   * `RealtimeObservability` when the opt-in pixel-marker measurement is active
+   * (see glass-to-glass.ts). Null otherwise — the stats collector does not
+   * populate it.
+   */
+  glassToGlass: G2GMetrics | null;
 };
 /** One side of an ICE candidate pair (sender or receiver). */
 type IceCandidateInfo = {

package/dist/realtime/observability/webrtc-stats.js

@@ -268,7 +268,8 @@ var WebRTCStatsCollector = class {
 			audio,
 			outboundVideo,
 			connection,
-			remoteInbound
+			remoteInbound,
+			glassToGlass: null
 		};
 	}
 };

package/dist/realtime/preflight.d.ts

@@ -0,0 +1,59 @@
+import { CustomModelDefinition, ModelDefinition } from "../shared/model.js";
+import { ConnectionQuality } from "./observability/connection-quality.js";
+
+//#region src/realtime/preflight.d.ts
+
+/**
+ * SDK-only connectivity preflight — run before `realtime.connect()` to decide
+ * whether to show the integration. Spins up a throwaway `RTCPeerConnection`
+ * against public STUN (no session, no inference) to check whether WebRTC can
+ * leave the network over UDP and roughly how laggy the path is. It does not
+ * measure throughput — use the in-session `connectionQuality` signal for that.
+ */
+type ConnectivityTransport = "udp" | "relay" | "failed";
+type ConnectivityMetrics = {
+  /** "udp" = direct UDP works · "relay" = will need TURN (unverified SDK-only) · "failed" = no connectivity. */
+  transport: ConnectivityTransport;
+  /** Approximate network round-trip time (ms) from time-to-first STUN candidate (or real RTT in deep mode), or null. */
+  rttMs: number | null;
+  /** Active-probe only: measured mid-stream (steady-state) glass-to-glass latency (ms), or null. */
+  g2gMs?: number | null;
+  /** Active-probe only: time-to-first-frame (ms) — startup latency to the first rendered model frame, or null. */
+  ttffMs?: number | null;
+  /** Active-probe only: end-to-end frame drop ratio (0–1), or null. */
+  g2gDropRatio?: number | null;
+  /** Active-probe only: server's view of upstream jitter (ms), or null. */
+  upstreamJitterMs?: number | null;
+  /** Active-probe only: server-reported upstream packet loss (0–1), or null. */
+  packetLoss?: number | null;
+  /** Active-probe only: number of glass-to-glass samples collected. */
+  sampleCount?: number;
+};
+type ConnectivityReport = {
+  /** Pre-connect quality on the same `good → critical` scale as the in-session signal — you decide what to do. */
+  quality: ConnectionQuality;
+  metrics: ConnectivityMetrics;
+  /** Human-readable explanations for any non-"good" verdict. */
+  reasons: string[];
+};
+type CheckConnectivityOptions = {
+  /** Override the ICE servers used for the probe. Defaults to public STUN. */
+  iceServers?: RTCIceServer[];
+  /** Abort candidate gathering after this long. Defaults to config. */
+  iceGatherTimeoutMs?: number;
+  /** Abort the probe early. */
+  signal?: AbortSignal;
+  /**
+   * Opt-in "deep" probe: instead of the STUN-only network check, briefly open a
+   * real session with a synthetic source, measure true glass-to-glass latency,
+   * then tear it down. Requires `model`. Costs a short GPU session.
+   */
+  deep?: boolean;
+  /** Required when `deep`: the realtime model to probe (latency is model-specific). */
+  model?: ModelDefinition | CustomModelDefinition;
+  /** Deep-probe duration (ms). Defaults to config. */
+  durationMs?: number;
+};
+/** Realtime `connect` injected by the SDK root so the active probe can open a session. */
+//#endregion
+export { CheckConnectivityOptions, ConnectivityMetrics, ConnectivityReport, ConnectivityTransport };

package/dist/realtime/preflight.js

@@ -0,0 +1,311 @@
+import { resolveFpsNumber } from "../shared/model.js";
+import { REALTIME_CONFIG } from "./config-realtime.js";
+import { extractSignals, scoreLowerBetter, worst } from "./observability/connection-quality.js";
+//#region src/realtime/preflight.ts
+/** Extract the candidate type ("host" | "srflx" | "prflx" | "relay") from an ICE candidate. */
+function candidateType(candidate) {
+	if (candidate.type) return candidate.type;
+	return /\btyp (\w+)/.exec(candidate.candidate)?.[1] ?? "";
+}
+async function gatherIceCandidates(iceServers, timeoutMs, signal, logger) {
+	if (signal?.aborted) return {
+		transport: "failed",
+		rttMs: null
+	};
+	const PC = globalThis?.RTCPeerConnection;
+	if (typeof PC !== "function") {
+		logger.warn("preflight: RTCPeerConnection unavailable in this environment");
+		return {
+			transport: "failed",
+			rttMs: null
+		};
+	}
+	let pc = null;
+	try {
+		pc = new PC({ iceServers });
+		pc.createDataChannel("decart-preflight");
+		let sawSrflx = false;
+		let sawOtherCandidate = false;
+		let firstSrflxAt = null;
+		const start = performance.now();
+		await new Promise((resolve) => {
+			let settled = false;
+			const finish = () => {
+				if (settled) return;
+				settled = true;
+				clearTimeout(timer);
+				resolve();
+			};
+			const timer = setTimeout(finish, timeoutMs);
+			signal?.addEventListener("abort", finish, { once: true });
+			const peer = pc;
+			peer.onicecandidate = (event) => {
+				if (!event.candidate || event.candidate.candidate === "") {
+					finish();
+					return;
+				}
+				if (candidateType(event.candidate) === "srflx") {
+					sawSrflx = true;
+					if (firstSrflxAt === null) firstSrflxAt = performance.now();
+				} else sawOtherCandidate = true;
+			};
+			peer.onicegatheringstatechange = () => {
+				if (peer.iceGatheringState === "complete") finish();
+			};
+			peer.createOffer().then((offer) => peer.setLocalDescription(offer)).catch((error) => {
+				logger.warn("preflight: failed to create offer", { error: error instanceof Error ? error.message : String(error) });
+				finish();
+			});
+		});
+		const rttMs = firstSrflxAt !== null ? Math.round(firstSrflxAt - start) : null;
+		let transport;
+		if (sawSrflx) transport = "udp";
+		else if (sawOtherCandidate) transport = "relay";
+		else transport = "failed";
+		return {
+			transport,
+			rttMs
+		};
+	} catch (error) {
+		logger.warn("preflight: connectivity probe threw", { error: error instanceof Error ? error.message : String(error) });
+		return {
+			transport: "failed",
+			rttMs: null
+		};
+	} finally {
+		try {
+			pc?.close();
+		} catch {}
+	}
+}
+/** Map probe metrics to a connectivity quality verdict. Pure. */
+function classifyConnectivity(metrics, thresholds) {
+	const reasons = [];
+	let quality;
+	if (metrics.transport === "failed") {
+		quality = "critical";
+		reasons.push("Could not establish any WebRTC connectivity (no ICE candidates gathered). Real-time streaming is unlikely to work on this network.");
+	} else if (metrics.transport === "relay") {
+		quality = "poor";
+		reasons.push("Direct UDP connectivity could not be confirmed; the session will need a TURN relay, which adds latency and can't be verified without starting a session.");
+	} else if (metrics.rttMs != null && metrics.rttMs > thresholds.marginalMs) {
+		quality = "poor";
+		reasons.push(`Network round-trip time is high (~${metrics.rttMs}ms > ${thresholds.marginalMs}ms); the real-time experience may feel laggy.`);
+	} else if (metrics.rttMs != null && metrics.rttMs > thresholds.goodMs) {
+		quality = "fair";
+		reasons.push(`Network round-trip time is elevated (~${metrics.rttMs}ms > ${thresholds.goodMs}ms).`);
+	} else quality = "good";
+	return {
+		quality,
+		metrics: {
+			transport: metrics.transport,
+			rttMs: metrics.rttMs
+		},
+		reasons
+	};
+}
+/**
+* Classify an active-probe result. Judges startup (TTFF) and steady-state
+* (mid-stream glass-to-glass) latency separately — both are real experienced
+* latency on different scales — and folds in drops + upstream loss. Falls back
+* to RTT only when neither latency could be measured. Pure.
+*/
+function classifyActiveProbe(metrics, thresholds) {
+	const reasons = [];
+	if (metrics.transport === "failed") return {
+		quality: "critical",
+		metrics,
+		reasons: ["Could not establish a realtime session for the deep probe."]
+	};
+	const dims = [];
+	if (metrics.ttffMs != null) {
+		const t = thresholds.ttff;
+		const q = scoreLowerBetter(metrics.ttffMs, t.goodMs, t.fairMs, t.poorMs);
+		dims.push(q);
+		if (q !== "good") reasons.push(`Time to first frame is ~${(metrics.ttffMs / 1e3).toFixed(1)}s (good ≤ ${(t.goodMs / 1e3).toFixed(0)}s); the session is slow to start.`);
+	}
+	if (metrics.g2gMs != null) {
+		const g = thresholds.glassToGlass;
+		const q = scoreLowerBetter(metrics.g2gMs, g.goodMs, g.fairMs, g.poorMs);
+		dims.push(q);
+		if (q !== "good") reasons.push(`Mid-stream glass-to-glass latency is ~${metrics.g2gMs}ms (good ≤ ${g.goodMs}ms); the real-time experience may feel laggy.`);
+	}
+	if (metrics.ttffMs == null && metrics.g2gMs == null) if (metrics.rttMs != null) {
+		reasons.push("Could not measure glass-to-glass latency during the probe (no marker round-trip); using network RTT instead.");
+		dims.push(scoreLowerBetter(metrics.rttMs, thresholds.rtt.goodMs, thresholds.rtt.fairMs, thresholds.rtt.poorMs));
+	} else reasons.push("The probe connected but could not measure latency (no marker round-trip and no RTT sample).");
+	if (metrics.g2gDropRatio != null) {
+		const d = thresholds.g2gDrop;
+		const q = scoreLowerBetter(metrics.g2gDropRatio, d.good, d.fair, d.poor);
+		dims.push(q);
+		if (q !== "good") reasons.push(`End-to-end frame drop ratio is ${(metrics.g2gDropRatio * 100).toFixed(1)}% (good ≤ ${d.good * 100}%).`);
+	}
+	if (metrics.packetLoss != null) {
+		const l = thresholds.loss;
+		const q = scoreLowerBetter(metrics.packetLoss, l.good, l.fair, l.poor);
+		dims.push(q);
+		if (q !== "good") reasons.push(`Upstream packet loss is ${(metrics.packetLoss * 100).toFixed(1)}% (good ≤ ${l.good * 100}%).`);
+	}
+	if (dims.length === 0) return {
+		quality: "fair",
+		metrics,
+		reasons
+	};
+	return {
+		quality: worst(...dims),
+		metrics,
+		reasons
+	};
+}
+/** Derive active-probe metrics from the latest in-session stats sample. */
+function activeMetricsFromStats(stats) {
+	if (!stats) return {
+		transport: "udp",
+		rttMs: null,
+		g2gMs: null,
+		ttffMs: null,
+		g2gDropRatio: null,
+		upstreamJitterMs: null,
+		packetLoss: null,
+		sampleCount: 0
+	};
+	const s = extractSignals(stats);
+	return {
+		transport: s.isRelayed ? "relay" : "udp",
+		rttMs: s.rttMs != null ? Math.round(s.rttMs) : null,
+		g2gMs: s.g2gMs,
+		ttffMs: s.ttffMs,
+		g2gDropRatio: s.g2gDropRatio,
+		upstreamJitterMs: s.upstreamJitterMs != null ? Math.round(s.upstreamJitterMs) : null,
+		packetLoss: s.fractionLost,
+		sampleCount: stats.glassToGlass?.sampleCount ?? 0
+	};
+}
+/**
+* Animated synthetic video source — no camera permission needed; content is
+* irrelevant to the marker. Sized to the model's exact input dimensions so the
+* server doesn't resize/crop the frame, which would move the bottom-left marker
+* out of where the server reads it (breaking the round trip).
+*/
+function createSyntheticSource(width, height, fps) {
+	if (typeof document === "undefined") throw new Error("deep connectivity probe requires a DOM environment (document is undefined)");
+	const canvas = document.createElement("canvas");
+	canvas.width = width;
+	canvas.height = height;
+	const ctx = canvas.getContext("2d");
+	if (!ctx) throw new Error("deep connectivity probe: 2D canvas context unavailable");
+	if (typeof canvas.captureStream !== "function") throw new Error("deep connectivity probe: canvas.captureStream unavailable");
+	let rafHandle = null;
+	let frame = 0;
+	const draw = () => {
+		frame++;
+		ctx.fillStyle = `hsl(${frame % 360}, 60%, 50%)`;
+		ctx.fillRect(0, 0, canvas.width, canvas.height);
+		ctx.fillStyle = "white";
+		ctx.fillRect(frame * 7 % canvas.width, 48, 96, 96);
+		rafHandle = requestAnimationFrame(draw);
+	};
+	rafHandle = requestAnimationFrame(draw);
+	const stream = canvas.captureStream(fps);
+	return {
+		stream,
+		dispose: () => {
+			if (rafHandle !== null) cancelAnimationFrame(rafHandle);
+			for (const track of stream.getTracks()) track.stop();
+		}
+	};
+}
+const ABORTED_DEEP_PROBE = {
+	quality: "fair",
+	metrics: {
+		transport: "failed",
+		rttMs: null,
+		g2gMs: null,
+		ttffMs: null,
+		g2gDropRatio: null,
+		upstreamJitterMs: null,
+		packetLoss: null,
+		sampleCount: 0
+	},
+	reasons: ["Deep connectivity probe aborted."]
+};
+/** Unblock `promise` early when `signal` aborts. */
+function raceAbort(promise, signal) {
+	if (!signal) return promise;
+	if (signal.aborted) throw signal.reason ?? new DOMException("Aborted", "AbortError");
+	return Promise.race([promise, new Promise((_, reject) => {
+		signal.addEventListener("abort", () => reject(signal.reason ?? new DOMException("Aborted", "AbortError")), { once: true });
+	})]);
+}
+/** Resolve once enough g2g samples exist or the probe window elapses (never rejects). */
+function waitForProbe(getLatest, minSamples, durationMs, signal) {
+	return new Promise((resolve) => {
+		const start = performance.now();
+		const tick = () => {
+			if (signal?.aborted) return resolve();
+			if ((getLatest()?.glassToGlass?.sampleCount ?? 0) >= minSamples) return resolve();
+			if (performance.now() - start >= durationMs) return resolve();
+			setTimeout(tick, 200);
+		};
+		setTimeout(tick, 200);
+	});
+}
+async function runActiveProbe(args) {
+	const { connect, logger, model, durationMs, signal } = args;
+	const thresholds = REALTIME_CONFIG.observability.connectionQuality;
+	let source;
+	let client;
+	let latest = null;
+	if (signal?.aborted) return ABORTED_DEEP_PROBE;
+	try {
+		source = createSyntheticSource(model.width, model.height, resolveFpsNumber(model.fps));
+		const connectTask = connect(source.stream, {
+			model,
+			debugQuality: true,
+			onRemoteStream: () => {}
+		});
+		signal?.addEventListener("abort", () => connectTask.then((c) => c.disconnect()).catch(() => {}), { once: true });
+		client = await raceAbort(connectTask, signal);
+		client.on("stats", (stats) => {
+			latest = stats;
+		});
+		await waitForProbe(() => latest, REALTIME_CONFIG.preflight.active.minSamples, durationMs, signal);
+		if (signal?.aborted) return ABORTED_DEEP_PROBE;
+	} catch (error) {
+		if (signal?.aborted) return ABORTED_DEEP_PROBE;
+		logger.warn("deep connectivity probe failed", { error: error instanceof Error ? error.message : String(error) });
+		return classifyActiveProbe({
+			transport: "failed",
+			rttMs: null,
+			g2gMs: null,
+			g2gDropRatio: null,
+			upstreamJitterMs: null,
+			packetLoss: null,
+			sampleCount: 0
+		}, thresholds);
+	} finally {
+		client?.disconnect();
+		source?.dispose();
+	}
+	return classifyActiveProbe(activeMetricsFromStats(latest), thresholds);
+}
+const DEFAULT_ICE_SERVERS = REALTIME_CONFIG.preflight.defaultStunUrls.map((urls) => ({ urls }));
+const createPreflight = ({ logger, connect }) => {
+	const checkConnectivity = async (options = {}) => {
+		if (options.deep) {
+			if (!connect) throw new Error("deep connectivity probe is unavailable (realtime client not wired)");
+			if (!options.model) throw new Error("deep connectivity probe requires a `model` (latency is model-specific)");
+			return runActiveProbe({
+				connect,
+				logger,
+				model: options.model,
+				durationMs: options.durationMs ?? REALTIME_CONFIG.preflight.active.durationMs,
+				signal: options.signal
+			});
+		}
+		return classifyConnectivity(await gatherIceCandidates(options.iceServers ?? DEFAULT_ICE_SERVERS, options.iceGatherTimeoutMs ?? REALTIME_CONFIG.preflight.iceGatherTimeoutMs, options.signal, logger), REALTIME_CONFIG.preflight.rtt);
+	};
+	return { checkConnectivity };
+};
+//#endregion
+export { createPreflight };