@oh-my-pi/pi-coding-agent 17.0.9 → 17.1.1

package/src/live/protocol.ts

@@ -0,0 +1,233 @@
+/** Frameless Bidi model used by Codex Desktop live calls. */
+export const LIVE_MODEL: "gpt-live-1-codex" = "gpt-live-1-codex";
+
+/** Maximum UTF-8 payload size accepted by each context append. */
+export const CONTEXT_CHUNK_BYTES = 500;
+
+/** Semantic stream selected for appended Frameless Bidi context. */
+export type LiveContextChannel = "speakable" | "commentary";
+
+/** Text content item accepted by Frameless Bidi context appends. */
+export type LiveInputTextContent = { type: "input_text"; text: string };
+
+/** Session object posted alongside the SDP when opening a live call. */
+export type LiveSessionPayload = {
+	model: typeof LIVE_MODEL;
+	instructions: string;
+	audio: { output: { voice: string } };
+	delegation: { type: "client" };
+};
+
+/** Messages sent by the client over the Frameless Bidi data channel. */
+export type LiveClientMessage =
+	| {
+			type: "delegation.context.append";
+			delegation_item_id: string;
+			channel?: LiveContextChannel;
+			content: LiveInputTextContent[];
+	  }
+	| {
+			type: "session.context.append";
+			channel?: LiveContextChannel;
+			content: LiveInputTextContent[];
+	  }
+	| { type: "session.close" };
+
+/** Parsed Frameless Bidi server events, including unsupported wire event types. */
+export type LiveServerEvent =
+	| {
+			type: "session.started" | "session.updated";
+			session: { id: string; instructions?: string };
+	  }
+	| { type: "output_audio.delta"; audio: string }
+	| { type: "input_transcript.added" | "output_transcript.added"; item: { text: string } }
+	| { type: "turn.done"; turn: { role: "user" | "assistant"; transcript: string } }
+	| {
+			type: "delegation.created";
+			item: {
+				type: "delegation";
+				target: "client";
+				id: string;
+				content: LiveInputTextContent[];
+			};
+	  }
+	| { type: "error"; message: string }
+	| { type: "unknown"; wireType: string };
+
+type UnknownRecord = Record<string, unknown>;
+
+function isRecord(value: unknown): value is UnknownRecord {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function parsePayload(payload: unknown): UnknownRecord | null {
+	let parsed = payload;
+	if (typeof payload === "string") {
+		try {
+			parsed = JSON.parse(payload);
+		} catch {
+			return null;
+		}
+	}
+	return isRecord(parsed) ? parsed : null;
+}
+
+function parseSessionEvent(
+	type: "session.started" | "session.updated",
+	payload: UnknownRecord,
+): LiveServerEvent | null {
+	const session = payload.session;
+	if (!isRecord(session) || typeof session.id !== "string") return null;
+	if (typeof session.instructions === "string") {
+		return { type, session: { id: session.id, instructions: session.instructions } };
+	}
+	return { type, session: { id: session.id } };
+}
+
+function parseTranscriptAddedEvent(
+	type: "input_transcript.added" | "output_transcript.added",
+	payload: UnknownRecord,
+): LiveServerEvent | null {
+	const item = payload.item;
+	if (!isRecord(item) || typeof item.text !== "string") return null;
+	return { type, item: { text: item.text } };
+}
+
+function parseTurnDoneEvent(payload: UnknownRecord): LiveServerEvent | null {
+	const turn = payload.turn;
+	if (!isRecord(turn) || (turn.role !== "user" && turn.role !== "assistant")) return null;
+	if (typeof turn.transcript !== "string") return null;
+	return { type: "turn.done", turn: { role: turn.role, transcript: turn.transcript } };
+}
+
+function parseDelegationCreatedEvent(payload: UnknownRecord): LiveServerEvent | null {
+	const item = payload.item;
+	if (!isRecord(item) || item.type !== "delegation" || item.target !== "client" || typeof item.id !== "string") {
+		return null;
+	}
+	if (!Array.isArray(item.content)) return null;
+	const content: LiveInputTextContent[] = [];
+	for (const candidate of item.content) {
+		if (!isRecord(candidate) || candidate.type !== "input_text" || typeof candidate.text !== "string") continue;
+		content.push({ type: "input_text", text: candidate.text });
+	}
+	return {
+		type: "delegation.created",
+		item: { type: "delegation", target: "client", id: item.id, content },
+	};
+}
+
+function stringifyErrorValue(value: unknown): string | null {
+	if (typeof value === "string") return value;
+	if (value === undefined) return null;
+	try {
+		return JSON.stringify(value) ?? null;
+	} catch {
+		return String(value);
+	}
+}
+
+function parseErrorEvent(payload: UnknownRecord): LiveServerEvent | null {
+	if (typeof payload.message === "string") return { type: "error", message: payload.message };
+	const error = payload.error;
+	if (isRecord(error) && typeof error.message === "string") {
+		return { type: "error", message: error.message };
+	}
+	const message = stringifyErrorValue(error);
+	return message === null ? null : { type: "error", message };
+}
+
+/** Parse a JSON string or decoded value from the Frameless Bidi data channel. */
+export function parseLiveServerEvent(payload: unknown): LiveServerEvent | null {
+	const parsed = parsePayload(payload);
+	if (!parsed || typeof parsed.type !== "string") return null;
+
+	switch (parsed.type) {
+		case "session.started":
+		case "session.updated":
+			return parseSessionEvent(parsed.type, parsed);
+		case "output_audio.delta":
+			return typeof parsed.audio === "string" ? { type: parsed.type, audio: parsed.audio } : null;
+		case "input_transcript.added":
+		case "output_transcript.added":
+			return parseTranscriptAddedEvent(parsed.type, parsed);
+		case "turn.done":
+			return parseTurnDoneEvent(parsed);
+		case "delegation.created":
+			return parseDelegationCreatedEvent(parsed);
+		case "error":
+			return parseErrorEvent(parsed);
+		default:
+			return { type: "unknown", wireType: parsed.type };
+	}
+}
+
+/** Build the session object posted in the multipart WebRTC call request. */
+export function buildLiveSessionPayload(instructions: string, voice: string): LiveSessionPayload {
+	return {
+		model: LIVE_MODEL,
+		instructions,
+		audio: { output: { voice } },
+		delegation: { type: "client" },
+	};
+}
+
+/** Build a context append associated with a server-created delegation. */
+export function buildDelegationContextAppend(
+	delegationItemId: string,
+	text: string,
+	channel?: LiveContextChannel,
+): LiveClientMessage {
+	return {
+		type: "delegation.context.append",
+		delegation_item_id: delegationItemId,
+		...(channel === undefined ? {} : { channel }),
+		content: [{ type: "input_text", text }],
+	};
+}
+
+/** Build context appended to the live session outside a delegation. */
+export function buildSessionContextAppend(text: string, channel?: LiveContextChannel): LiveClientMessage {
+	return {
+		type: "session.context.append",
+		...(channel === undefined ? {} : { channel }),
+		content: [{ type: "input_text", text }],
+	};
+}
+
+/** Build the message that gracefully closes a live session. */
+export function buildSessionClose(): LiveClientMessage {
+	return { type: "session.close" };
+}
+
+function utf8ByteLength(codePoint: number): number {
+	if (codePoint <= 0x7f) return 1;
+	if (codePoint <= 0x7ff) return 2;
+	if (codePoint <= 0xffff) return 3;
+	return 4;
+}
+
+/** Split context into character-safe chunks of at most 500 UTF-8 bytes. */
+export function chunkLiveContext(text: string): string[] {
+	if (text.length === 0) return [""];
+
+	const chunks: string[] = [];
+	let chunkStart = 0;
+	let chunkBytes = 0;
+	let index = 0;
+	while (index < text.length) {
+		const codePoint = text.codePointAt(index);
+		if (codePoint === undefined) break;
+		const characterLength = codePoint > 0xffff ? 2 : 1;
+		const characterBytes = utf8ByteLength(codePoint);
+		if (chunkBytes + characterBytes > CONTEXT_CHUNK_BYTES) {
+			chunks.push(text.slice(chunkStart, index));
+			chunkStart = index;
+			chunkBytes = 0;
+		}
+		chunkBytes += characterBytes;
+		index += characterLength;
+	}
+	chunks.push(text.slice(chunkStart));
+	return chunks;
+}

package/src/live/transport.ts

@@ -0,0 +1,422 @@
+import { type AuthStorage, isAuthRetryableError, type OAuthAccess, withOAuthAccess } from "@oh-my-pi/pi-ai";
+import { getProxyForProvider, wrapFetchForProxy } from "@oh-my-pi/pi-ai/utils/proxy";
+import {
+	CODEX_BASE_URL,
+	CODEX_CLIENT_VERSION,
+	getCodexAccountId,
+	OPENAI_HEADERS,
+} from "@oh-my-pi/pi-catalog/wire/codex";
+import { LiveWebRtcPeer } from "@oh-my-pi/pi-natives";
+import { generateCodexAttestation } from "./attestation";
+import {
+	buildLiveSessionPayload,
+	type LiveClientMessage,
+	type LiveServerEvent,
+	parseLiveServerEvent,
+} from "./protocol";
+
+const SIGNALING_URL = `${CODEX_BASE_URL}/codex/realtime/calls?intent=quicksilver&architecture=avas`;
+const MAX_ERROR_BODY_LENGTH = 2_048;
+const SIDEBAND_CONNECT_ATTEMPTS = 5;
+const SIDEBAND_CONNECT_TIMEOUT_MS = 15_000;
+const LIVE_PROVIDER = "openai-codex";
+const LIVE_ORIGINATOR = "Codex Desktop";
+const LIVE_CALL_ID_PATTERN = /^rtc_[\w-]+$/;
+
+type Lifecycle = "idle" | "connecting" | "connected" | "closing" | "closed";
+
+interface LiveSignalingResult {
+	answer: string;
+	callId: string;
+	access: OAuthAccess;
+	attestation: string | undefined;
+}
+
+class LiveSignalingError extends Error {
+	status: number;
+	errorMessage: string;
+
+	constructor(status: number, message: string) {
+		super(message);
+		this.name = "LiveSignalingError";
+		this.status = status;
+		this.errorMessage = message;
+	}
+}
+
+/** Callbacks emitted by the live WebRTC transport. */
+export interface LiveTransportCallbacks {
+	onEvent(event: LiveServerEvent): void;
+	onOutputLevel(level: number): void;
+}
+
+/** Configuration required to establish a Codex live call. */
+export interface LiveTransportOptions {
+	authStorage: AuthStorage;
+	sessionId: string;
+	instructions: string;
+	voice: string;
+	callbacks: LiveTransportCallbacks;
+	signal?: AbortSignal;
+}
+
+/** Extracts the server-assigned `rtc_*` call ID from a signaling Location header. */
+export function parseLiveCallId(location: string | null): string | undefined {
+	if (!location) return undefined;
+	return location
+		.split("?", 1)[0]
+		?.split("/")
+		.find(segment => LIVE_CALL_ID_PATTERN.test(segment));
+}
+
+/** Builds the Frameless Bidi sideband WebSocket URL for an accepted Codex call. */
+export function buildLiveSidebandUrl(callId: string): string {
+	const url = new URL(`https://api.openai.com/v1/live/${encodeURIComponent(callId)}`);
+	url.protocol = "wss:";
+	return url.toString();
+}
+
+function liveSessionHeaders(
+	access: OAuthAccess,
+	sessionId: string,
+	realtimeSessionId: string,
+	attestation: string | undefined,
+): Record<string, string> {
+	const headers: Record<string, string> = {
+		Authorization: `Bearer ${access.accessToken}`,
+		"OpenAI-Alpha": "quicksilver=v2",
+		"User-Agent": `Codex Desktop/${CODEX_CLIENT_VERSION}`,
+		"x-session-id": realtimeSessionId,
+		[OPENAI_HEADERS.ORIGINATOR]: LIVE_ORIGINATOR,
+		[OPENAI_HEADERS.VERSION]: CODEX_CLIENT_VERSION,
+		[OPENAI_HEADERS.SCOPED_SESSION_ID]: sessionId,
+		[OPENAI_HEADERS.THREAD_ID]: sessionId,
+	};
+	const accountId = access.accountId ?? getCodexAccountId(access.accessToken);
+	if (accountId) headers[OPENAI_HEADERS.ACCOUNT_ID] = accountId;
+	if (attestation) headers["x-oai-attestation"] = attestation;
+	return headers;
+}
+
+function boundedErrorBody(body: string, statusText: string): string {
+	const normalized = body.trim().replaceAll(/\s+/g, " ");
+	if (!normalized) return statusText || "empty response body";
+	if (normalized.length <= MAX_ERROR_BODY_LENGTH) return normalized;
+	return `${normalized.slice(0, MAX_ERROR_BODY_LENGTH)}…`;
+}
+
+function isAuthError(error: unknown): boolean {
+	return isAuthRetryableError(error);
+}
+
+function abortReason(signal: AbortSignal | undefined): Error {
+	if (signal?.reason instanceof Error) return signal.reason;
+	return new DOMException("Live connection aborted", "AbortError");
+}
+
+/** Native WebRTC transport for a Codex Frameless Bidi live session. */
+export class CodexLiveTransport {
+	readonly #options: LiveTransportOptions;
+	#peer: LiveWebRtcPeer | undefined;
+	readonly #realtimeSessionId = crypto.randomUUID();
+	#sideband: Bun.WebSocket | undefined;
+	#state: Lifecycle = "idle";
+	#connectPromise: Promise<void> | undefined;
+	#closePromise: Promise<void> | undefined;
+	#sendTail: Promise<void> = Promise.resolve();
+	#muted = false;
+	#unexpectedFailureReported = false;
+	readonly #abortListener: () => void;
+
+	constructor(options: LiveTransportOptions) {
+		this.#options = options;
+		this.#abortListener = () => {
+			void this.close();
+		};
+		if (!options.signal?.aborted) options.signal?.addEventListener("abort", this.#abortListener, { once: true });
+	}
+
+	/** Establish the native peer, perform Codex signaling, and wait for the data channel. */
+	connect(): Promise<void> {
+		if (this.#state === "connected") return Promise.resolve();
+		if (this.#connectPromise) return this.#connectPromise;
+		if (this.#state === "closing" || this.#state === "closed")
+			return Promise.reject(new Error("Live transport is closed"));
+		if (this.#options.signal?.aborted) return Promise.reject(abortReason(this.#options.signal));
+		this.#state = "connecting";
+		const operation = this.#connect().catch(async error => {
+			await this.close();
+			throw error;
+		});
+		this.#connectPromise = operation;
+		return operation;
+	}
+
+	async #connect(): Promise<void> {
+		const peer = new LiveWebRtcPeer(
+			(error, payload) => {
+				if (error) {
+					this.#handlePeerFailure(error.message);
+				} else {
+					this.#handleServerEvent(payload);
+				}
+			},
+			(error, level) => {
+				if (error) {
+					this.#handlePeerFailure(error.message);
+				} else {
+					this.#handleOutputLevel(level);
+				}
+			},
+			(error, message) => this.#handlePeerFailure(error?.message ?? message),
+		);
+		this.#peer = peer;
+		const offer = await peer.createOffer();
+		if (this.#state !== "connecting") throw abortReason(this.#options.signal);
+		const signaling = await this.#signal(offer);
+		await peer.acceptAnswer(signaling.answer);
+		peer.setMuted(this.#muted);
+		await peer.waitForOpen();
+		if (this.#state !== "connecting") throw abortReason(this.#options.signal);
+		await this.#connectSideband(signaling.callId, signaling.access, signaling.attestation);
+		if (this.#state !== "connecting") throw abortReason(this.#options.signal);
+		this.#state = "connected";
+	}
+
+	async #signal(offer: string): Promise<LiveSignalingResult> {
+		const attestation = await generateCodexAttestation();
+		return await withOAuthAccess(
+			this.#options.authStorage,
+			LIVE_PROVIDER,
+			access => this.#signalWithAccess(offer, access, attestation),
+			{
+				sessionId: this.#options.sessionId,
+				signal: this.#options.signal,
+				isAuthError,
+				missingAccessMessage: "No Codex OAuth credential is available for a live call.",
+			},
+		);
+	}
+
+	async #signalWithAccess(
+		offer: string,
+		access: OAuthAccess,
+		attestation: string | undefined,
+	): Promise<LiveSignalingResult> {
+		const headers = new Headers({
+			...liveSessionHeaders(access, this.#options.sessionId, this.#realtimeSessionId, attestation),
+			Accept: "*/*",
+			"Content-Type": "application/json",
+		});
+		const fetchImpl = wrapFetchForProxy(fetch, LIVE_PROVIDER);
+		const response = await fetchImpl(SIGNALING_URL, {
+			method: "POST",
+			headers,
+			body: JSON.stringify({
+				sdp: offer,
+				session: buildLiveSessionPayload(this.#options.instructions, this.#options.voice),
+			}),
+			signal: this.#options.signal,
+		});
+		const responseBody = await response.text();
+		if (!response.ok) {
+			const detail = boundedErrorBody(responseBody, response.statusText);
+			throw new LiveSignalingError(response.status, `Codex live signaling failed (${response.status}): ${detail}`);
+		}
+		const answer = responseBody;
+		if (!answer.trim())
+			throw new LiveSignalingError(response.status, "Codex live signaling returned an empty SDP answer");
+		const callId = parseLiveCallId(response.headers.get("location"));
+		if (!callId) {
+			throw new LiveSignalingError(response.status, "Codex live signaling returned no valid call ID");
+		}
+		return { answer, callId, access, attestation };
+	}
+
+	async #connectSideband(callId: string, access: OAuthAccess, attestation: string | undefined): Promise<void> {
+		let failure = new Error("Codex live sideband connection failed");
+		for (let attempt = 0; attempt < SIDEBAND_CONNECT_ATTEMPTS; attempt++) {
+			try {
+				await this.#openSideband(callId, access, attestation);
+				return;
+			} catch (cause) {
+				failure = cause instanceof Error ? cause : new Error(String(cause));
+				if (this.#options.signal?.aborted) throw abortReason(this.#options.signal);
+				if (attempt + 1 < SIDEBAND_CONNECT_ATTEMPTS) await Bun.sleep(200 * 2 ** attempt);
+			}
+		}
+		throw failure;
+	}
+
+	async #openSideband(callId: string, access: OAuthAccess, attestation: string | undefined): Promise<void> {
+		const url = buildLiveSidebandUrl(callId);
+		const options = {
+			headers: liveSessionHeaders(access, this.#options.sessionId, this.#realtimeSessionId, attestation),
+			proxy: getProxyForProvider(LIVE_PROVIDER),
+		} satisfies Bun.WebSocketOptions;
+		const socket: Bun.WebSocket = Reflect.construct(WebSocket, [url, options]);
+		socket.binaryType = "nodebuffer";
+		const { promise, resolve, reject } = Promise.withResolvers<void>();
+		let opened = false;
+		let settled = false;
+		let timeout: NodeJS.Timeout | undefined;
+		const cleanup = (): void => {
+			if (timeout) {
+				clearTimeout(timeout);
+				timeout = undefined;
+			}
+			this.#options.signal?.removeEventListener("abort", onAbort);
+		};
+		const rejectConnect = (error: Error): void => {
+			if (settled) return;
+			settled = true;
+			cleanup();
+			reject(error);
+		};
+		const onAbort = (): void => {
+			socket.close(1000, "aborted");
+			rejectConnect(abortReason(this.#options.signal));
+		};
+		socket.onopen = () => {
+			if (settled) {
+				socket.close(1000, "stale");
+				return;
+			}
+			opened = true;
+			settled = true;
+			cleanup();
+			this.#sideband = socket;
+			resolve();
+		};
+		socket.onmessage = event => {
+			if (typeof event.data !== "string") {
+				this.#reportFailure("Codex live sideband returned an unexpected binary frame.");
+				return;
+			}
+			this.#handleSidebandEvent(event.data);
+		};
+		socket.onerror = event => {
+			const detail = event instanceof ErrorEvent && event.message ? `: ${event.message}` : "";
+			if (!opened) {
+				rejectConnect(new Error(`Codex live sideband connection failed${detail}`));
+				socket.close(1011, "connection failed");
+				return;
+			}
+			this.#reportFailure(`Codex live sideband failed${detail}`);
+		};
+		socket.onclose = event => {
+			if (!opened) {
+				rejectConnect(new Error(`Codex live sideband closed before connecting (${event.code})`));
+				return;
+			}
+			if (this.#sideband !== socket) return;
+			this.#sideband = undefined;
+			if (this.#state === "connecting" || this.#state === "connected") {
+				const detail = event.reason ? `: ${event.reason}` : "";
+				this.#reportFailure(`Codex live sideband closed (${event.code})${detail}`);
+			}
+		};
+		if (this.#options.signal?.aborted) {
+			onAbort();
+		} else {
+			this.#options.signal?.addEventListener("abort", onAbort, { once: true });
+			timeout = setTimeout(() => {
+				socket.close(1000, "connect timeout");
+				rejectConnect(new Error("Codex live sideband connection timed out"));
+			}, SIDEBAND_CONNECT_TIMEOUT_MS);
+			timeout.unref?.();
+		}
+		await promise;
+	}
+
+	#handleSidebandEvent(payload: string): void {
+		if (this.#state === "closing" || this.#state === "closed") return;
+		const event = parseLiveServerEvent(payload);
+		if (!event) return;
+		try {
+			this.#options.callbacks.onEvent(event);
+		} catch {}
+	}
+
+	#handleServerEvent(payload: string): void {
+		if (this.#state === "closing" || this.#state === "closed") return;
+		const event = parseLiveServerEvent(payload);
+		if (!event || (this.#sideband?.readyState === WebSocket.OPEN && event.type !== "error")) return;
+		try {
+			this.#options.callbacks.onEvent(event);
+		} catch {}
+	}
+
+	#handleOutputLevel(level: number): void {
+		if (this.#state !== "connected" || !Number.isFinite(level)) return;
+		try {
+			this.#options.callbacks.onOutputLevel(Math.min(1, Math.max(0, level)));
+		} catch {}
+	}
+
+	#handlePeerFailure(message: string): void {
+		this.#reportFailure(message);
+	}
+
+	#reportFailure(message: string): void {
+		if ((this.#state !== "connecting" && this.#state !== "connected") || this.#unexpectedFailureReported) {
+			return;
+		}
+		this.#unexpectedFailureReported = true;
+		try {
+			this.#options.callbacks.onEvent({ type: "error", message });
+		} catch {}
+	}
+
+	/** Serialize one Frameless Bidi control message onto the call's sideband WebSocket. */
+	send(message: LiveClientMessage): Promise<void> {
+		const operation = this.#sendTail.then(() => {
+			if (this.#state !== "connected") throw new Error("Live transport is not connected");
+			const sideband = this.#sideband;
+			if (!sideband || sideband.readyState !== WebSocket.OPEN) {
+				throw new Error("Codex live sideband is not connected");
+			}
+			sideband.send(JSON.stringify(message));
+		});
+		this.#sendTail = operation.catch(() => {});
+		return operation;
+	}
+
+	/** Queue 16 kHz mono Float32 PCM for native Opus transmission. */
+	pushAudio(samples: Float32Array): void {
+		if (this.#state !== "connected" || this.#muted || samples.length === 0) return;
+		this.#peer?.pushAudio(samples);
+	}
+
+	/** Enable or disable the native audio source and discard partial input when muted. */
+	async setMuted(muted: boolean): Promise<void> {
+		this.#muted = muted;
+		if (this.#state === "connected") this.#peer?.setMuted(muted);
+	}
+
+	/** Stop sideband signaling and the native WebRTC media peer. Safe to call repeatedly. */
+	close(): Promise<void> {
+		if (this.#closePromise) return this.#closePromise;
+		this.#state = "closing";
+		const operation = this.#close();
+		this.#closePromise = operation;
+		return operation;
+	}
+
+	async #close(): Promise<void> {
+		this.#options.signal?.removeEventListener("abort", this.#abortListener);
+		const sideband = this.#sideband;
+		const peer = this.#peer;
+		this.#sideband = undefined;
+		this.#peer = undefined;
+		if (sideband && (sideband.readyState === WebSocket.OPEN || sideband.readyState === WebSocket.CONNECTING)) {
+			sideband.close(1000, "done");
+		}
+		if (peer) {
+			try {
+				await peer.close();
+			} catch {}
+		}
+		this.#state = "closed";
+	}
+}