@decartai/sdk 0.1.1 → 0.1.3

package/dist/realtime/client.d.ts

@@ -50,6 +50,10 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
     "720p": "720p";
     "1080p": "1080p";
   }>>;
+  preferredVideoCodec: z.ZodOptional<z.ZodEnum<{
+    h264: "h264";
+    vp9: "vp9";
+  }>>;
 }, z.core.$strip>;
 type RealTimeClientConnectOptions = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model"> & {
   model: ModelDefinition | CustomModelDefinition;

package/dist/realtime/client.js

@@ -21,7 +21,8 @@ const realTimeClientConnectOptionsSchema = z.object({
 	initialState: realTimeClientInitialStateSchema.optional(),
 	queryParams: z.record(z.string(), z.string()).optional(),
 	mirror: z.union([z.literal("auto"), z.boolean()]).optional(),
-	resolution: z.enum(["720p", "1080p"]).optional()
+	resolution: z.enum(["720p", "1080p"]).optional(),
+	preferredVideoCodec: z.enum(["h264", "vp9"]).optional()
 });
 const createRealTimeClient = (opts) => {
 	const { baseUrl, apiKey, integration } = opts;
@@ -29,7 +30,7 @@ const createRealTimeClient = (opts) => {
 	const connect = async (stream, options) => {
 		const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
 		if (!parsedOptions.success) throw parsedOptions.error;
-		const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution } = parsedOptions.data;
+		const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution, preferredVideoCodec } = parsedOptions.data;
 		const mirror = parsedOptions.data.mirror ?? false;
 		let inputStream = stream ?? new MediaStream();
 		let mirroredStream;
@@ -63,6 +64,7 @@ const createRealTimeClient = (opts) => {
 				onStats: (stats) => emitOrBuffer("stats", stats)
 			});
 			const safariCodec = isDesktopSafari() ? "vp8" : void 0;
+			const publishCodec = safariCodec ?? preferredVideoCodec;
 			session = new StreamSession({
 				url: `${url}?${new URLSearchParams({
 					...safariCodec ? { livekit_server_codec: safariCodec } : {},
@@ -78,7 +80,7 @@ const createRealTimeClient = (opts) => {
 				initialImageRef,
 				initialPrompt,
 				logger,
-				videoCodec: safariCodec
+				videoCodec: publishCodec
 			});
 			let sessionId = null;
 			let subscribeToken = null;

package/dist/realtime/config-realtime.js

@@ -34,6 +34,7 @@ const REALTIME_CONFIG = {
 		},
 		defaultVideoCodec: "h264",
 		defaultMaxVideoBitrateBps: 35e5,
+		vp9MaxVideoBitrateBps: 3e6,
 		defaultPublishFps: 30
 	},
 	observability: {

package/dist/realtime/media-channel.js

@@ -1,18 +1,19 @@
 import { createConsoleLogger } from "../utils/logger.js";
 import { REALTIME_CONFIG } from "./config-realtime.js";
 import mitt from "mitt";
-import { Room, RoomEvent, Track, TrackEvent } from "livekit-client";
+import { Room, RoomEvent, Track } from "livekit-client";
 //#region src/realtime/media-channel.ts
 function getDefaultVideoPublishOptions(videoCodec) {
-	const videoEncoding = {
-		maxBitrate: REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps,
-		maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps
-	};
+	const resolvedCodec = videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec;
+	const maxBitrate = resolvedCodec === "vp9" ? REALTIME_CONFIG.livekit.vp9MaxVideoBitrateBps : REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps;
 	return {
 		source: Track.Source.Camera,
-		videoCodec: videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec,
-		simulcast: true,
-		videoEncoding
+		videoCodec: resolvedCodec,
+		simulcast: resolvedCodec !== "vp9",
+		videoEncoding: {
+			maxBitrate,
+			maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps
+		}
 	};
 }
 var MediaChannel = class {
@@ -39,16 +40,13 @@ var MediaChannel = class {
 		room.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
 			if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
 			if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
-			track.attach();
 			const mediaStreamTrack = track.mediaStreamTrack;
 			if (mediaStreamTrack) {
-				this.remoteStream ??= new MediaStream();
-				if (!this.remoteStream.getTracks().includes(mediaStreamTrack)) this.remoteStream.addTrack(mediaStreamTrack);
+				const tracks = this.remoteStream?.getTracks() ?? [];
+				if (!tracks.includes(mediaStreamTrack)) tracks.push(mediaStreamTrack);
+				this.remoteStream = new MediaStream(tracks);
 				this.events.emit("remoteStream", this.remoteStream);
 			}
-			track.on(TrackEvent.VideoPlaybackStarted, () => {
-				this.events.emit("firstFrame");
-			});
 		});
 		room.on(RoomEvent.Disconnected, (reason) => {
 			this.logger.warn("livekit: room disconnected", { reason });

package/dist/realtime/signaling-channel.js

@@ -262,6 +262,9 @@ var SignalingChannel = class {
 					queueSize: msg.queue_size
 				});
 				break;
+			case "generation_started":
+				this.events.emit("generationStarted");
+				break;
 			case "generation_tick":
 				this.events.emit("generationTick", { seconds: msg.seconds });
 				break;

package/dist/realtime/stream-session.js

@@ -165,7 +165,11 @@ var StreamSession = class {
 			this.queue = qp;
 			this.events.emit("queuePosition", qp);
 		});
-		this.signaling.on("generationTick", (e) => this.events.emit("generationTick", e));
+		this.signaling.on("generationStarted", () => this.markGenerating());
+		this.signaling.on("generationTick", (e) => {
+			this.markGenerating();
+			this.events.emit("generationTick", e);
+		});
 		this.signaling.on("generationEnded", (e) => this.events.emit("generationEnded", e));
 		this.signaling.on("serverError", (err) => this.events.emit("error", err));
 		this.signaling.on("closed", (info) => this.handleConnectionLoss({
@@ -173,11 +177,11 @@ var StreamSession = class {
 			...info
 		}));
 	}
+	markGenerating() {
+		if (this.state === "connected") this.setState("generating");
+	}
 	wireMediaEvents() {
 		this.media.on("remoteStream", (stream) => this.events.emit("remoteStream", stream));
-		this.media.on("firstFrame", () => {
-			if (this.state === "connected") this.setState("generating");
-		});
 		this.media.on("disconnected", (info) => this.handleConnectionLoss({
 			source: "media",
 			reason: info.reason

package/dist/realtime/subscribe-client.js

@@ -81,11 +81,11 @@ const createRealTimeSubscribeClient = (opts) => {
 			activeRoom.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
 				if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
 				if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
-				track.attach();
 				const mediaStreamTrack = track.mediaStreamTrack;
 				if (!mediaStreamTrack) return;
-				remoteStream ??= new MediaStream();
-				if (!remoteStream.getTracks().includes(mediaStreamTrack)) remoteStream.addTrack(mediaStreamTrack);
+				const tracks = remoteStream?.getTracks() ?? [];
+				if (!tracks.includes(mediaStreamTrack)) tracks.push(mediaStreamTrack);
+				remoteStream = new MediaStream(tracks);
 				options.onRemoteStream(remoteStream);
 			});
 			activeRoom.on(RoomEvent.ConnectionStateChanged, (state) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decartai/sdk",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Decart's JavaScript SDK",
   "type": "module",
   "license": "MIT",