@decartai/sdk 0.0.66 → 0.0.67

package/README.md

@@ -62,6 +62,23 @@ realtimeClient.setPrompt("Cyberpunk city");
 realtimeClient.disconnect();
 ```
 
+#### Front-camera mirroring
+
+Pre-flip the input stream:
+
+```ts
+const realtimeClient = await client.realtime.connect(stream, {
+  model,
+  mirror: "auto", // or true to always mirror
+  // ...
+});
+```
+
+Options:
+- `false` (default) — never mirror.
+- `"auto"` — mirror when the input track reports `facingMode: "user"` (mobile front cameras).
+- `true` — always mirror (e.g. desktop webcams).
+
 ### Async Processing (Queue API)
 
 For video generation jobs, use the queue API to submit jobs and poll for results:

package/dist/realtime/client.d.ts

@@ -36,6 +36,7 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
     image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
   }, z.core.$strip>>;
   customizeOffer: z.ZodOptional<z.ZodCustom<z.core.$InferInnerFunctionTypeAsync<z.core.$ZodFunctionArgs, z.core.$ZodFunctionOut>, z.core.$InferInnerFunctionTypeAsync<z.core.$ZodFunctionArgs, z.core.$ZodFunctionOut>>>;
+  mirror: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<"auto">, z.ZodBoolean]>>;
 }, z.core.$strip>;
 type RealTimeClientConnectOptions = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model"> & {
   model: ModelDefinition | CustomModelDefinition;

package/dist/realtime/client.js

@@ -3,6 +3,7 @@ import { modelDefinitionSchema } from "../shared/model.js";
 import { modelStateSchema } from "../shared/types.js";
 import { createEventBuffer } from "./event-buffer.js";
 import { realtimeMethods } from "./methods.js";
+import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
 import { decodeSubscribeToken, encodeSubscribeToken } from "./subscribe-client.js";
 import { NullTelemetryReporter, TelemetryReporter } from "./telemetry-reporter.js";
 import { WebRTCManager } from "./webrtc-manager.js";
@@ -56,7 +57,8 @@ const realTimeClientConnectOptionsSchema = z.object({
 	model: modelDefinitionSchema,
 	onRemoteStream: z.custom((val) => typeof val === "function", { message: "onRemoteStream must be a function" }),
 	initialState: realTimeClientInitialStateSchema.optional(),
-	customizeOffer: createAsyncFunctionSchema(z.function()).optional()
+	customizeOffer: createAsyncFunctionSchema(z.function()).optional(),
+	mirror: z.union([z.literal("auto"), z.boolean()]).optional()
 });
 const createRealTimeClient = (opts) => {
 	const { baseUrl, apiKey, integration, logger } = opts;
@@ -64,7 +66,18 @@ const createRealTimeClient = (opts) => {
 		const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
 		if (!parsedOptions.success) throw parsedOptions.error;
 		const { onRemoteStream, initialState } = parsedOptions.data;
-		const inputStream = stream ?? new MediaStream();
+		const mirror = parsedOptions.data.mirror ?? false;
+		let inputStream = stream ?? new MediaStream();
+		let mirroredStream;
+		if (mirror !== false) try {
+			const firstVideoTrack = inputStream.getVideoTracks?.()[0];
+			if (firstVideoTrack && (mirror === true || shouldMirrorTrack(firstVideoTrack))) {
+				mirroredStream = createMirroredStream(inputStream, { fps: options.model.fps });
+				inputStream = mirroredStream.stream;
+			} else if (mirror === true && !firstVideoTrack) logger.warn("mirror: true requested but no video track was found on the input stream");
+		} catch (error) {
+			logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
+		}
 		let webrtcManager;
 		let telemetryReporter = new NullTelemetryReporter();
 		let handleConnectionStateChange = null;
@@ -219,6 +232,7 @@ const createRealTimeClient = (opts) => {
 					telemetryReporter.stop();
 					stop();
 					manager.cleanup();
+					mirroredStream?.dispose();
 				},
 				on: eventEmitter.on,
 				off: eventEmitter.off,
@@ -239,6 +253,7 @@ const createRealTimeClient = (opts) => {
 		} catch (error) {
 			telemetryReporter.stop();
 			webrtcManager?.cleanup();
+			mirroredStream?.dispose();
 			throw error;
 		}
 	};

package/dist/realtime/mirror-stream.js

@@ -0,0 +1,116 @@
+//#region src/realtime/mirror-stream.ts
+function isMediaStreamTrackProcessorSupported() {
+	return typeof globalThis !== "undefined" && typeof globalThis.MediaStreamTrackProcessor === "function" && typeof globalThis.MediaStreamTrackGenerator === "function";
+}
+function shouldMirrorTrack(track) {
+	if (track.kind !== "video") return false;
+	let facingMode;
+	try {
+		facingMode = track.getSettings?.().facingMode;
+	} catch {
+		return false;
+	}
+	return facingMode === "user";
+}
+function createMirroredStream(input, opts) {
+	const [sourceVideo] = input.getVideoTracks();
+	const audioTracks = input.getAudioTracks();
+	if (!sourceVideo) return {
+		stream: input,
+		dispose: () => {},
+		impl: "noop"
+	};
+	if (isMediaStreamTrackProcessorSupported()) return createWithTrackProcessor(sourceVideo, audioTracks);
+	return createWithCanvas(sourceVideo, audioTracks, opts.fps);
+}
+function createWithTrackProcessor(sourceVideo, audioTracks) {
+	const Processor = globalThis.MediaStreamTrackProcessor;
+	const Generator = globalThis.MediaStreamTrackGenerator;
+	if (!new OffscreenCanvas(1, 1).getContext("2d")) throw new Error("createMirroredStream: OffscreenCanvas 2D context unavailable");
+	const processor = new Processor({ track: sourceVideo });
+	const generator = new Generator({ kind: "video" });
+	let canvas = new OffscreenCanvas(1, 1);
+	let ctx = canvas.getContext("2d");
+	const transform = new TransformStream({ transform(frame, controller) {
+		const w = frame.displayWidth;
+		const h = frame.displayHeight;
+		if (canvas.width !== w || canvas.height !== h) {
+			canvas = new OffscreenCanvas(w, h);
+			ctx = canvas.getContext("2d");
+		}
+		let flipped;
+		try {
+			ctx.save();
+			ctx.setTransform(-1, 0, 0, 1, w, 0);
+			ctx.drawImage(frame, 0, 0, w, h);
+			ctx.restore();
+			flipped = new VideoFrame(canvas, {
+				timestamp: frame.timestamp,
+				alpha: "discard"
+			});
+			controller.enqueue(flipped);
+			flipped = void 0;
+		} finally {
+			flipped?.close();
+			frame.close();
+		}
+	} });
+	processor.readable.pipeThrough(transform).pipeTo(generator.writable).catch(() => {});
+	const stream = new MediaStream([generator, ...audioTracks]);
+	let disposed = false;
+	return {
+		stream,
+		impl: "track-processor",
+		dispose: () => {
+			if (disposed) return;
+			disposed = true;
+			generator.stop();
+		}
+	};
+}
+function createWithCanvas(sourceVideo, audioTracks, fps) {
+	if (typeof document === "undefined") throw new Error("createMirroredStream requires a DOM environment (document is undefined)");
+	const canvas = document.createElement("canvas");
+	const ctx = canvas.getContext("2d");
+	if (!ctx) throw new Error("createMirroredStream: 2D canvas context unavailable");
+	if (typeof canvas.captureStream !== "function") throw new Error("createMirroredStream: canvas.captureStream unavailable");
+	const [flippedTrack] = canvas.captureStream(fps).getVideoTracks();
+	if (!flippedTrack) throw new Error("createMirroredStream: canvas.captureStream produced no video track");
+	const video = document.createElement("video");
+	video.muted = true;
+	video.playsInline = true;
+	video.autoplay = true;
+	video.srcObject = new MediaStream([sourceVideo]);
+	let disposed = false;
+	let rafHandle = null;
+	const draw = () => {
+		if (disposed) return;
+		const w = video.videoWidth;
+		const h = video.videoHeight;
+		if (w > 0 && h > 0) {
+			if (canvas.width !== w) canvas.width = w;
+			if (canvas.height !== h) canvas.height = h;
+			ctx.save();
+			ctx.setTransform(-1, 0, 0, 1, w, 0);
+			ctx.drawImage(video, 0, 0, w, h);
+			ctx.restore();
+		}
+		rafHandle = requestAnimationFrame(draw);
+	};
+	video.play().catch(() => {});
+	rafHandle = requestAnimationFrame(draw);
+	return {
+		stream: new MediaStream([flippedTrack, ...audioTracks]),
+		impl: "canvas",
+		dispose: () => {
+			if (disposed) return;
+			disposed = true;
+			if (rafHandle !== null) cancelAnimationFrame(rafHandle);
+			flippedTrack.stop();
+			video.srcObject = null;
+		}
+	};
+}
+
+//#endregion
+export { createMirroredStream, shouldMirrorTrack };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decartai/sdk",
-  "version": "0.0.66",
+  "version": "0.0.67",
   "description": "Decart's JavaScript SDK",
   "type": "module",
   "license": "MIT",