@decartai/sdk 0.0.67 → 0.1.0

package/dist/realtime/observability/webrtc-stats.js

@@ -0,0 +1,276 @@
+import { REALTIME_CONFIG } from "../config-realtime.js";
+//#region src/realtime/observability/webrtc-stats.ts
+var WebRTCStatsCollector = class {
+	source = null;
+	intervalId = null;
+	prevBytesVideo = 0;
+	prevBytesAudio = 0;
+	prevBytesSentVideo = 0;
+	prevTimestamp = 0;
+	prevPacketsLostVideo = 0;
+	prevFramesDropped = 0;
+	prevFreezeCount = 0;
+	prevFreezeDuration = 0;
+	prevPacketsLostAudio = 0;
+	prevNackCountInbound = 0;
+	onStats = null;
+	intervalMs;
+	constructor(options = {}) {
+		this.intervalMs = Math.max(options.intervalMs ?? REALTIME_CONFIG.observability.statsDefaultIntervalMs, REALTIME_CONFIG.observability.statsMinIntervalMs);
+	}
+	/** Attach to a stats provider and start polling. */
+	start(source, onStats) {
+		this.stop();
+		this.source = source;
+		this.onStats = onStats;
+		this.prevBytesVideo = 0;
+		this.prevBytesAudio = 0;
+		this.prevBytesSentVideo = 0;
+		this.prevTimestamp = 0;
+		this.prevPacketsLostVideo = 0;
+		this.prevFramesDropped = 0;
+		this.prevFreezeCount = 0;
+		this.prevFreezeDuration = 0;
+		this.prevPacketsLostAudio = 0;
+		this.prevNackCountInbound = 0;
+		this.intervalId = setInterval(() => this.collect(), this.intervalMs);
+	}
+	/** Stop polling and release resources. */
+	stop() {
+		if (this.intervalId !== null) {
+			clearInterval(this.intervalId);
+			this.intervalId = null;
+		}
+		this.source = null;
+		this.onStats = null;
+	}
+	isRunning() {
+		return this.intervalId !== null;
+	}
+	async collect() {
+		if (!this.source || !this.onStats) return;
+		try {
+			const rawStats = await this.source.getStats();
+			const stats = this.parse(rawStats);
+			this.onStats(stats);
+		} catch {
+			this.stop();
+		}
+	}
+	parse(rawStats) {
+		const now = performance.now();
+		const elapsed = this.prevTimestamp > 0 ? (now - this.prevTimestamp) / 1e3 : 0;
+		let video = null;
+		let audio = null;
+		let outboundVideo = null;
+		let remoteInbound = null;
+		const connection = {
+			currentRoundTripTime: null,
+			availableOutgoingBitrate: null,
+			selectedCandidatePairs: []
+		};
+		const succeededPairs = [];
+		rawStats.forEach((report) => {
+			if (report.type === "inbound-rtp" && report.kind === "video") {
+				const bytesReceived = report.bytesReceived ?? 0;
+				const bitrate = elapsed > 0 ? (bytesReceived - this.prevBytesVideo) * 8 / elapsed : 0;
+				this.prevBytesVideo = bytesReceived;
+				const r = report;
+				const packetsLost = r.packetsLost ?? 0;
+				const framesDropped = r.framesDropped ?? 0;
+				const freezeCount = r.freezeCount ?? 0;
+				const freezeDuration = r.totalFreezesDuration ?? 0;
+				const framesDecoded = r.framesDecoded ?? 0;
+				const nackCount = r.nackCount ?? 0;
+				const jbEmitted = r.jitterBufferEmittedCount ?? 0;
+				const totalDecodeTime = r.totalDecodeTime ?? 0;
+				const totalProcessingDelay = r.totalProcessingDelay ?? 0;
+				const totalInterFrameDelay = r.totalInterFrameDelay ?? 0;
+				const totalSquaredInterFrameDelay = r.totalSquaredInterFrameDelay ?? 0;
+				const jitterBufferDelay = r.jitterBufferDelay ?? 0;
+				const jitterBufferTargetDelay = r.jitterBufferTargetDelay ?? 0;
+				const jitterBufferMinimumDelay = r.jitterBufferMinimumDelay ?? 0;
+				const avgDecodeTimeMs = framesDecoded > 0 ? totalDecodeTime / framesDecoded * 1e3 : null;
+				const avgProcessingDelayMs = framesDecoded > 0 ? totalProcessingDelay / framesDecoded * 1e3 : null;
+				const avgInterFrameDelayMs = framesDecoded > 0 ? totalInterFrameDelay / framesDecoded * 1e3 : null;
+				const interFrameDelayStdDevMs = framesDecoded > 0 ? Math.sqrt(Math.max(0, totalSquaredInterFrameDelay / framesDecoded - (totalInterFrameDelay / framesDecoded) ** 2)) * 1e3 : null;
+				const avgJitterBufferMs = jbEmitted > 0 ? jitterBufferDelay / jbEmitted * 1e3 : null;
+				const jitterBufferTargetDelayMs = jbEmitted > 0 ? jitterBufferTargetDelay / jbEmitted * 1e3 : null;
+				const jitterBufferMinimumDelayMs = jbEmitted > 0 ? jitterBufferMinimumDelay / jbEmitted * 1e3 : null;
+				video = {
+					framesDecoded,
+					framesDropped,
+					framesReceived: r.framesReceived ?? 0,
+					keyFramesDecoded: r.keyFramesDecoded ?? 0,
+					framesPerSecond: r.framesPerSecond ?? 0,
+					frameWidth: r.frameWidth ?? 0,
+					frameHeight: r.frameHeight ?? 0,
+					bytesReceived,
+					packetsReceived: r.packetsReceived ?? 0,
+					packetsLost,
+					jitter: r.jitter ?? 0,
+					bitrate: Math.round(bitrate),
+					freezeCount,
+					totalFreezesDuration: freezeDuration,
+					packetsLostDelta: Math.max(0, packetsLost - this.prevPacketsLostVideo),
+					framesDroppedDelta: Math.max(0, framesDropped - this.prevFramesDropped),
+					freezeCountDelta: Math.max(0, freezeCount - this.prevFreezeCount),
+					freezeDurationDelta: Math.max(0, freezeDuration - this.prevFreezeDuration),
+					nackCount,
+					nackCountDelta: Math.max(0, nackCount - this.prevNackCountInbound),
+					pliCount: r.pliCount ?? 0,
+					firCount: r.firCount ?? 0,
+					avgDecodeTimeMs,
+					avgJitterBufferMs,
+					avgProcessingDelayMs,
+					avgInterFrameDelayMs,
+					interFrameDelayStdDevMs,
+					jitterBufferTargetDelayMs,
+					jitterBufferMinimumDelayMs,
+					decoderImplementation: r.decoderImplementation ?? ""
+				};
+				this.prevPacketsLostVideo = packetsLost;
+				this.prevFramesDropped = framesDropped;
+				this.prevFreezeCount = freezeCount;
+				this.prevFreezeDuration = freezeDuration;
+				this.prevNackCountInbound = nackCount;
+			}
+			if (report.type === "outbound-rtp" && report.kind === "video") {
+				const r = report;
+				const bytesSent = r.bytesSent ?? 0;
+				const packetsSent = r.packetsSent ?? 0;
+				const frameWidth = r.frameWidth ?? 0;
+				const frameHeight = r.frameHeight ?? 0;
+				const pixels = frameWidth * frameHeight;
+				const framesEncoded = r.framesEncoded ?? 0;
+				const totalEncodeTime = r.totalEncodeTime ?? 0;
+				const totalPacketSendDelay = r.totalPacketSendDelay ?? 0;
+				const qpSum = r.qpSum ?? 0;
+				const nackCount = r.nackCount ?? 0;
+				const pliCount = r.pliCount ?? 0;
+				const firCount = r.firCount ?? 0;
+				const retransmittedBytesSent = r.retransmittedBytesSent ?? 0;
+				const retransmittedPacketsSent = r.retransmittedPacketsSent ?? 0;
+				const targetBitrate = r.targetBitrate ?? null;
+				const avgEncodeTimeMs = framesEncoded > 0 ? totalEncodeTime / framesEncoded * 1e3 : null;
+				const avgPacketSendDelayMs = packetsSent > 0 ? totalPacketSendDelay / packetsSent * 1e3 : null;
+				const avgQp = framesEncoded > 0 ? qpSum / framesEncoded : null;
+				if (outboundVideo === null) outboundVideo = {
+					qualityLimitationReason: r.qualityLimitationReason ?? "none",
+					qualityLimitationDurations: r.qualityLimitationDurations ?? {},
+					bytesSent,
+					packetsSent,
+					framesPerSecond: r.framesPerSecond ?? 0,
+					frameWidth,
+					frameHeight,
+					bitrate: 0,
+					targetBitrateKbps: targetBitrate != null ? Math.round(targetBitrate / 1e3) : null,
+					avgEncodeTimeMs,
+					avgPacketSendDelayMs,
+					avgQp,
+					nackCount,
+					pliCount,
+					firCount,
+					retransmittedBytesSent,
+					retransmittedPacketsSent,
+					encoderImplementation: r.encoderImplementation ?? ""
+				};
+				else {
+					outboundVideo.bytesSent += bytesSent;
+					outboundVideo.packetsSent += packetsSent;
+					outboundVideo.nackCount += nackCount;
+					outboundVideo.pliCount += pliCount;
+					outboundVideo.firCount += firCount;
+					outboundVideo.retransmittedBytesSent += retransmittedBytesSent;
+					outboundVideo.retransmittedPacketsSent += retransmittedPacketsSent;
+					if (pixels > outboundVideo.frameWidth * outboundVideo.frameHeight) {
+						outboundVideo.frameWidth = frameWidth;
+						outboundVideo.frameHeight = frameHeight;
+						outboundVideo.framesPerSecond = r.framesPerSecond ?? 0;
+						outboundVideo.qualityLimitationReason = r.qualityLimitationReason ?? "none";
+						outboundVideo.qualityLimitationDurations = r.qualityLimitationDurations ?? {};
+						outboundVideo.targetBitrateKbps = targetBitrate != null ? Math.round(targetBitrate / 1e3) : null;
+						outboundVideo.avgEncodeTimeMs = avgEncodeTimeMs;
+						outboundVideo.avgPacketSendDelayMs = avgPacketSendDelayMs;
+						outboundVideo.avgQp = avgQp;
+						outboundVideo.encoderImplementation = r.encoderImplementation ?? "";
+					}
+				}
+			}
+			if (report.type === "remote-inbound-rtp" && report.kind === "video") {
+				const r = report;
+				remoteInbound = {
+					fractionLost: r.fractionLost ?? null,
+					jitter: r.jitter ?? null,
+					roundTripTime: r.roundTripTime ?? null
+				};
+			}
+			if (report.type === "inbound-rtp" && report.kind === "audio") {
+				const bytesReceived = report.bytesReceived ?? 0;
+				const bitrate = elapsed > 0 ? (bytesReceived - this.prevBytesAudio) * 8 / elapsed : 0;
+				this.prevBytesAudio = bytesReceived;
+				const r = report;
+				const audioPacketsLost = r.packetsLost ?? 0;
+				audio = {
+					bytesReceived,
+					packetsReceived: r.packetsReceived ?? 0,
+					packetsLost: audioPacketsLost,
+					jitter: r.jitter ?? 0,
+					bitrate: Math.round(bitrate),
+					packetsLostDelta: Math.max(0, audioPacketsLost - this.prevPacketsLostAudio)
+				};
+				this.prevPacketsLostAudio = audioPacketsLost;
+			}
+			if (report.type === "candidate-pair") {
+				const r = report;
+				if (r.state === "succeeded") {
+					connection.currentRoundTripTime = r.currentRoundTripTime ?? null;
+					connection.availableOutgoingBitrate = r.availableOutgoingBitrate ?? null;
+					const localId = r.localCandidateId;
+					const remoteId = r.remoteCandidateId;
+					if (localId && remoteId) succeededPairs.push({
+						localId,
+						remoteId
+					});
+				}
+			}
+		});
+		if (succeededPairs.length > 0) {
+			const toInfo = (id) => {
+				const c = rawStats.get(id);
+				if (!c) return null;
+				return {
+					candidateType: c.candidateType ?? "",
+					address: c.address ?? c.ip ?? "",
+					port: c.port ?? 0,
+					protocol: c.protocol ?? ""
+				};
+			};
+			for (const { localId, remoteId } of succeededPairs) {
+				const local = toInfo(localId);
+				const remote = toInfo(remoteId);
+				if (local && remote) connection.selectedCandidatePairs.push({
+					local,
+					remote
+				});
+			}
+		}
+		const ov = outboundVideo;
+		if (ov !== null) {
+			const outBitrate = elapsed > 0 ? (ov.bytesSent - this.prevBytesSentVideo) * 8 / elapsed : 0;
+			ov.bitrate = Math.max(0, Math.round(outBitrate));
+			this.prevBytesSentVideo = ov.bytesSent;
+		}
+		this.prevTimestamp = now;
+		return {
+			timestamp: Date.now(),
+			video,
+			audio,
+			outboundVideo,
+			connection,
+			remoteInbound
+		};
+	}
+};
+//#endregion
+export { WebRTCStatsCollector };

package/dist/realtime/signaling-channel.js

@@ -0,0 +1,286 @@
+import { buildUserAgent } from "../utils/user-agent.js";
+import { createConsoleLogger } from "../utils/logger.js";
+import { REALTIME_CONFIG } from "./config-realtime.js";
+import mitt from "mitt";
+//#region src/realtime/signaling-channel.ts
+var SignalingChannel = class {
+	ws = null;
+	events = mitt();
+	pendingAcks = [];
+	pendingRoomInfo = null;
+	connected = false;
+	closing = false;
+	logger;
+	constructor(config) {
+		this.config = config;
+		this.logger = config.logger ?? createConsoleLogger("warn");
+	}
+	on(event, handler) {
+		this.events.on(event, handler);
+	}
+	off(event, handler) {
+		this.events.off(event, handler);
+	}
+	async openAndJoin(opts = {}) {
+		const connectTimeout = opts.connectTimeout ?? REALTIME_CONFIG.signaling.connectTimeoutMs;
+		const handshakeTimeout = opts.handshakeTimeout ?? REALTIME_CONFIG.signaling.handshakeTimeoutMs;
+		this.config.observability?.startPhase("websocket-open");
+		await this.openSocket(connectTimeout);
+		this.config.observability?.endPhase("websocket-open", { success: true });
+		this.config.observability?.startPhase("room-join");
+		const roomInfoWait = this.waitForRoomInfo(handshakeTimeout);
+		if (!this.writeMessage({ type: "livekit_join" })) {
+			roomInfoWait.cancel();
+			throw new Error("WebSocket is not open");
+		}
+		let roomInfo;
+		try {
+			roomInfo = await roomInfoWait.promise;
+		} catch (error) {
+			this.rejectAllPending(error instanceof Error ? error : new Error(String(error)));
+			throw error;
+		}
+		this.config.observability?.endPhase("room-join", { success: true });
+		this.connected = true;
+		const initialStateAck = this.sendInitialStateTracked(opts.initialState);
+		initialStateAck.catch(() => {});
+		return {
+			roomInfo,
+			initialStateAck
+		};
+	}
+	async sendInitialStateTracked(initialState) {
+		if (!initialState) return;
+		this.config.observability?.startPhase("initial-state-handshake");
+		await this.sendInitialState(initialState);
+		this.config.observability?.endPhase("initial-state-handshake", { success: true });
+	}
+	close() {
+		this.closing = true;
+		this.connected = false;
+		const ws = this.ws;
+		this.ws = null;
+		if (ws) try {
+			ws.close();
+		} catch {}
+		this.rejectPendingRoomInfo(/* @__PURE__ */ new Error("Control channel closed"));
+		this.rejectAllPending(/* @__PURE__ */ new Error("Control channel closed"));
+	}
+	async sendPrompt(text, opts = {}) {
+		const ack = await this.request({
+			message: {
+				type: "prompt",
+				prompt: text,
+				enhance_prompt: opts.enhance ?? true
+			},
+			matchAck: (msg) => msg.type === "prompt_ack" && msg.prompt === text,
+			timeoutMs: opts.timeout ?? REALTIME_CONFIG.signaling.requestTimeoutMs,
+			label: "Prompt send"
+		});
+		if (!ack.success) throw new Error(ack.error ?? "Failed to send prompt");
+	}
+	async setImage(image, opts = {}) {
+		const message = {
+			type: "set_image",
+			image_data: image
+		};
+		if (opts.prompt !== void 0) message.prompt = opts.prompt;
+		if (opts.enhance !== void 0) message.enhance_prompt = opts.enhance;
+		const ack = await this.request({
+			message,
+			matchAck: (msg) => msg.type === "set_image_ack",
+			timeoutMs: opts.timeout ?? REALTIME_CONFIG.signaling.requestTimeoutMs,
+			label: "Image send"
+		});
+		if (!ack.success) throw new Error(ack.error ?? "Failed to send image");
+	}
+	async openSocket(timeout) {
+		const userAgent = encodeURIComponent(buildUserAgent(this.config.integration));
+		const separator = this.config.url.includes("?") ? "&" : "?";
+		const wsUrl = `${this.config.url}${separator}user_agent=${userAgent}`;
+		this.closing = false;
+		await new Promise((resolve, reject) => {
+			const timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`WebSocket open timeout (${timeout}ms)`)), timeout);
+			const ws = new WebSocket(wsUrl);
+			this.ws = ws;
+			ws.onopen = () => {
+				clearTimeout(timer);
+				resolve();
+			};
+			ws.onclose = (e) => {
+				clearTimeout(timer);
+				const wasConnected = this.connected;
+				const pendingCount = this.pendingAcks.length;
+				this.connected = false;
+				this.ws = null;
+				this.logger.warn("signaling: websocket closed", {
+					code: e.code,
+					reason: e.reason,
+					wasConnected,
+					closing: this.closing,
+					pendingAcks: pendingCount
+				});
+				const error = /* @__PURE__ */ new Error(`WebSocket closed: ${e.code} ${e.reason}`);
+				this.rejectPendingRoomInfo(error);
+				this.rejectAllPending(error);
+				if (wasConnected || this.closing) this.events.emit("closed", {
+					code: e.code,
+					reason: e.reason
+				});
+				else reject(error);
+			};
+			ws.onerror = () => {};
+			ws.onmessage = (e) => {
+				try {
+					this.handleMessage(JSON.parse(e.data));
+				} catch {}
+			};
+		});
+	}
+	waitForRoomInfo(timeoutMs) {
+		let cleanup = () => {};
+		return {
+			promise: new Promise((resolve, reject) => {
+				let timer = setTimeout(() => {
+					cleanup();
+					this.logger.warn("signaling: livekit_room_info timeout", { timeoutMs });
+					reject(/* @__PURE__ */ new Error(`livekit_room_info timeout (${timeoutMs}ms)`));
+				}, timeoutMs);
+				const pendingRoomInfo = {
+					resolve: (info) => {
+						cleanup();
+						resolve(info);
+					},
+					reject: (err) => {
+						cleanup();
+						reject(err);
+					},
+					cancel: () => {
+						cleanup();
+					},
+					pauseTimeout: () => {
+						if (timer) {
+							clearTimeout(timer);
+							timer = null;
+						}
+					}
+				};
+				cleanup = () => {
+					if (timer) {
+						clearTimeout(timer);
+						timer = null;
+					}
+					if (this.pendingRoomInfo === pendingRoomInfo) this.pendingRoomInfo = null;
+				};
+				this.pendingRoomInfo = pendingRoomInfo;
+			}),
+			cancel: cleanup
+		};
+	}
+	async sendInitialState(initialState) {
+		if (!initialState) return;
+		if (initialState.image !== void 0) {
+			await this.setImage(initialState.image, {
+				prompt: initialState.prompt,
+				enhance: initialState.enhance
+			});
+			return;
+		}
+		if (initialState.prompt !== void 0 && initialState.prompt !== null) await this.sendPrompt(initialState.prompt, { enhance: initialState.enhance });
+	}
+	async request({ message, matchAck, timeoutMs, label }) {
+		return new Promise((resolve, reject) => {
+			const timer = setTimeout(() => {
+				cleanup();
+				this.logger.warn("signaling: ack timed out", {
+					label,
+					timeoutMs
+				});
+				reject(/* @__PURE__ */ new Error(`${label} timed out`));
+			}, timeoutMs);
+			const entry = {
+				matches: matchAck,
+				onMatch: (msg) => {
+					cleanup();
+					resolve(msg);
+				},
+				reject: (err) => {
+					cleanup();
+					reject(err);
+				}
+			};
+			const cleanup = () => {
+				clearTimeout(timer);
+				this.pendingAcks = this.pendingAcks.filter((e) => e !== entry);
+			};
+			this.pendingAcks.push(entry);
+			if (!this.writeMessage(message)) {
+				cleanup();
+				reject(/* @__PURE__ */ new Error("WebSocket is not open"));
+			}
+		});
+	}
+	writeMessage(message) {
+		if (this.ws?.readyState !== WebSocket.OPEN) return false;
+		this.ws.send(JSON.stringify(message));
+		return true;
+	}
+	handleMessage(msg) {
+		for (const ack of [...this.pendingAcks]) if (ack.matches(msg)) {
+			ack.onMatch(msg);
+			break;
+		}
+		switch (msg.type) {
+			case "livekit_room_info":
+				this.resolvePendingRoomInfo({
+					livekitUrl: msg.livekit_url,
+					token: msg.token,
+					roomName: msg.room_name,
+					sessionId: msg.session_id
+				});
+				break;
+			case "queue_position":
+				this.pendingRoomInfo?.pauseTimeout();
+				this.events.emit("queuePosition", {
+					position: msg.position,
+					queueSize: msg.queue_size
+				});
+				break;
+			case "generation_tick":
+				this.events.emit("generationTick", { seconds: msg.seconds });
+				break;
+			case "generation_ended":
+				this.events.emit("generationEnded", {
+					seconds: msg.seconds,
+					reason: msg.reason
+				});
+				break;
+			case "error": {
+				const error = new Error(msg.error);
+				error.source = "server";
+				this.logger.error("signaling: server error received", { error: msg.error });
+				this.events.emit("serverError", error);
+				this.rejectPendingRoomInfo(error);
+				this.rejectAllPending(error);
+				break;
+			}
+		}
+	}
+	resolvePendingRoomInfo(info) {
+		const pending = this.pendingRoomInfo;
+		if (!pending) return;
+		pending.resolve(info);
+	}
+	rejectPendingRoomInfo(error) {
+		const pending = this.pendingRoomInfo;
+		if (!pending) return;
+		pending.reject(error);
+	}
+	rejectAllPending(error) {
+		const pending = this.pendingAcks;
+		this.pendingAcks = [];
+		for (const entry of pending) entry.reject(error);
+	}
+};
+//#endregion
+export { SignalingChannel };