@monotykamary/pi-better-openai 0.1.23

package/src/live/index.ts

@@ -0,0 +1,197 @@
+import { type ExtensionAPI, type ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import { getCodexCredentials } from "../codex-auth.ts";
+import type { ResolvedConfig } from "../config.ts";
+import { sanitizeDiagnosticError } from "../format.ts";
+import {
+  LiveSessionController,
+  type LiveSessionControllerOptions,
+  type LiveTranscript,
+} from "./controller.ts";
+import { LiveVisualizer, LIVE_VISUALIZER_TOGGLE_KEY } from "./visualizer.ts";
+
+export const LIVE_COMMAND = "live";
+export const LIVE_DELEGATION_MESSAGE_TYPE = "better-openai-live-delegation";
+
+interface LiveSessionRuntime {
+  start(): Promise<void>;
+  stop(): Promise<void>;
+  toggleMute(): void;
+  handleAgentMessage(message: unknown): void;
+  handleAgentSettled(): void;
+}
+
+interface ActiveLiveRun {
+  session: LiveSessionRuntime;
+  finishUi(result: LiveUiResult): void;
+}
+
+type LiveUiResult = { error?: Error };
+
+type LiveSessionFactory = (options: LiveSessionControllerOptions) => LiveSessionRuntime;
+
+export interface LiveRegistrationDependencies {
+  createSession?: LiveSessionFactory;
+}
+
+function messageText(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .filter(
+      (item): item is { type: "text"; text: string } =>
+        typeof item === "object" &&
+        item !== null &&
+        "type" in item &&
+        item.type === "text" &&
+        "text" in item &&
+        typeof item.text === "string",
+    )
+    .map((item) => item.text)
+    .join("\n");
+}
+
+export function registerOpenAILive(
+  pi: ExtensionAPI,
+  getConfig: (ctx: ExtensionContext) => ResolvedConfig,
+  dependencies: LiveRegistrationDependencies = {},
+): { isActive(): boolean; stop(): Promise<void> } {
+  const createSession =
+    dependencies.createSession ?? ((options) => new LiveSessionController(options));
+  let activeRun: ActiveLiveRun | undefined;
+  let settling: Promise<void> | undefined;
+
+  async function stopRun(run: ActiveLiveRun): Promise<void> {
+    run.finishUi({});
+    await run.session.stop();
+  }
+
+  async function stopActive(): Promise<void> {
+    const run = activeRun;
+    if (!run) {
+      if (settling) await settling;
+      return;
+    }
+    await stopRun(run);
+  }
+
+  async function start(ctx: ExtensionContext): Promise<void> {
+    if (ctx.mode !== "tui") {
+      ctx.ui.notify("Live voice requires interactive TUI mode.", "warning");
+      return;
+    }
+    const cfg = getConfig(ctx);
+    if (!cfg.live.enabled) {
+      ctx.ui.notify("Live voice is disabled. Enable it in /openai-settings.", "warning");
+      return;
+    }
+    if (settling) await settling;
+
+    const result = await ctx.ui.custom<LiveUiResult>((tui, theme, _keybindings, done) => {
+      let completed = false;
+      let session: LiveSessionRuntime;
+      const finishUi = (value: LiveUiResult) => {
+        if (completed) return;
+        completed = true;
+        done(value);
+      };
+      const visualizer = new LiveVisualizer({
+        theme,
+        requestRender: () => tui.requestRender(),
+        onStop: () => finishUi({}),
+        onToggleMute: () => session.toggleMute(),
+      });
+      const updateTranscript = (transcript: LiveTranscript | undefined) => {
+        visualizer.setTranscript(transcript);
+      };
+      session = createSession({
+        sessionId: ctx.sessionManager.getSessionId(),
+        voice: cfg.live.voice,
+        getCredentials: (signal) => getCodexCredentials(ctx, signal),
+        delegate: (request) => {
+          pi.sendMessage(
+            {
+              customType: LIVE_DELEGATION_MESSAGE_TYPE,
+              content: request,
+              display: true,
+              details: { source: "live" },
+            },
+            { triggerTurn: true, deliverAs: "steer" },
+          );
+        },
+        callbacks: {
+          onPhase: (phase) => visualizer.setPhase(phase),
+          onLevels: (input) => visualizer.setInputLevel(input),
+          onTranscript: updateTranscript,
+          onTerminal: (error) => finishUi(error ? { error } : {}),
+        },
+      });
+      activeRun = { session, finishUi };
+      setImmediate(() => {
+        void session.start().catch((cause) => {
+          const error = cause instanceof Error ? cause : new Error(String(cause));
+          finishUi({ error });
+        });
+      });
+      return visualizer;
+    });
+
+    const run = activeRun;
+    if (run) {
+      const cleanup = stopRun(run).catch(() => undefined);
+      settling = cleanup;
+      await cleanup;
+      if (activeRun === run) activeRun = undefined;
+      if (settling === cleanup) settling = undefined;
+    }
+    if (result.error) ctx.ui.notify(sanitizeDiagnosticError(result.error.message), "error");
+  }
+
+  async function toggle(ctx: ExtensionContext): Promise<void> {
+    if (activeRun) {
+      activeRun.finishUi({});
+      return;
+    }
+    await start(ctx);
+  }
+
+  pi.registerMessageRenderer(LIVE_DELEGATION_MESSAGE_TYPE, (message, _options, theme) => {
+    const text = messageText(message.content).trim();
+    const label = theme.fg("accent", theme.bold("Live request"));
+    return new Text(`${label}\n${theme.fg("customMessageText", text)}`, 1, 0);
+  });
+
+  pi.registerCommand(LIVE_COMMAND, {
+    description: "Start or stop Codex-backed realtime voice mode",
+    handler: async (args, ctx) => {
+      if (args.trim()) {
+        ctx.ui.notify("Usage: /live", "error");
+        return;
+      }
+      await toggle(ctx);
+    },
+  });
+
+  pi.registerShortcut(LIVE_VISUALIZER_TOGGLE_KEY, {
+    description: "Start or stop Better OpenAI live voice mode",
+    handler: toggle,
+  });
+
+  pi.on("message_end", (event) => {
+    activeRun?.session.handleAgentMessage(event.message);
+  });
+
+  pi.on("agent_settled", () => {
+    activeRun?.session.handleAgentSettled();
+  });
+
+  pi.on("session_shutdown", async () => {
+    await stopActive();
+    activeRun = undefined;
+  });
+
+  return {
+    isActive: () => activeRun !== undefined || settling !== undefined,
+    stop: stopActive,
+  };
+}

package/src/live/native.ts

@@ -0,0 +1,109 @@
+import { createRequire } from "node:module";
+
+export interface LiveAudioCapture {
+  stop(): void;
+}
+
+export interface LiveAudioCaptureConstructor {
+  new (
+    sampleRate: number,
+    onAudio: (error: Error | null, samples: Float32Array) => void,
+  ): LiveAudioCapture;
+}
+
+export interface LiveWebRtcPeerInstance {
+  createOffer(): Promise<string>;
+  acceptAnswer(sdp: string): Promise<void>;
+  waitForOpen(timeoutMs?: number): Promise<void>;
+  pushAudio(samples: Float32Array): void;
+  setMuted(muted: boolean): void;
+  close(): Promise<void>;
+}
+
+export interface LiveWebRtcPeerConstructor {
+  new (
+    onEvent: (error: Error | null, payload: string) => void,
+    onLevel: (error: Error | null, level: number) => void,
+    onFailure: (error: Error | null, message: string) => void,
+  ): LiveWebRtcPeerInstance;
+}
+
+export type DeviceCheckTokenResult = {
+  supported: boolean;
+  tokenBase64?: string;
+  error?: string;
+  latencyMs: number;
+};
+
+export interface LiveNativeBindings {
+  AudioCapture: LiveAudioCaptureConstructor;
+  LiveWebRtcPeer: LiveWebRtcPeerConstructor;
+  deviceCheckGenerateToken(): Promise<DeviceCheckTokenResult>;
+  __ompInstallTokioRuntime(): void;
+}
+
+const NATIVE_PACKAGES = {
+  "darwin-arm64": "@oh-my-pi/pi-natives-darwin-arm64",
+  "darwin-x64": "@oh-my-pi/pi-natives-darwin-x64",
+  "linux-arm64": "@oh-my-pi/pi-natives-linux-arm64",
+  "linux-x64": "@oh-my-pi/pi-natives-linux-x64",
+  "win32-x64": "@oh-my-pi/pi-natives-win32-x64",
+} as const;
+
+const runtimeRequire = createRequire(import.meta.url);
+let cachedBindings: LiveNativeBindings | undefined;
+
+type NativeTarget = keyof typeof NATIVE_PACKAGES;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+export function resolveLiveNativePackage(
+  platform: NodeJS.Platform | string,
+  arch: string,
+): string | undefined {
+  return NATIVE_PACKAGES[`${platform}-${arch}` as NativeTarget];
+}
+
+export function validateLiveNativeBindings(value: unknown): LiveNativeBindings {
+  if (
+    !isRecord(value) ||
+    typeof value.AudioCapture !== "function" ||
+    typeof value.LiveWebRtcPeer !== "function" ||
+    typeof value.deviceCheckGenerateToken !== "function" ||
+    typeof value.__ompInstallTokioRuntime !== "function"
+  ) {
+    throw new Error(
+      "The installed live native package does not expose the required audio/WebRTC API.",
+    );
+  }
+  return value as unknown as LiveNativeBindings;
+}
+
+export function loadLiveNative(
+  requirePackage: (packageName: string) => unknown = runtimeRequire,
+): LiveNativeBindings {
+  if (cachedBindings && requirePackage === runtimeRequire) return cachedBindings;
+  const packageName = resolveLiveNativePackage(process.platform, process.arch);
+  if (!packageName) {
+    throw new Error(
+      `Live voice is not available on ${process.platform}-${process.arch}. Supported targets: ${Object.keys(NATIVE_PACKAGES).join(", ")}.`,
+    );
+  }
+
+  let loaded: unknown;
+  try {
+    loaded = requirePackage(packageName);
+  } catch (cause) {
+    const detail = cause instanceof Error ? cause.message : String(cause);
+    throw new Error(`Unable to load ${packageName}: ${detail}`, { cause });
+  }
+
+  const bindings = validateLiveNativeBindings(loaded);
+  bindings.__ompInstallTokioRuntime();
+  if (requirePackage === runtimeRequire) cachedBindings = bindings;
+  return bindings;
+}
+
+export const LIVE_NATIVE_TARGETS = Object.keys(NATIVE_PACKAGES);

package/src/live/protocol.ts

@@ -0,0 +1,234 @@
+export const LIVE_MODEL = "gpt-live-1-codex" as const;
+
+export const CONTEXT_CHUNK_BYTES = 500;
+
+export type LiveContextChannel = "speakable" | "commentary";
+
+export type LiveInputTextContent = { type: "input_text"; text: string };
+
+export type LiveSessionPayload = {
+  model: typeof LIVE_MODEL;
+  instructions: string;
+  audio: { output: { voice: string } };
+  delegation: { type: "client" };
+};
+
+export type LiveClientMessage =
+  | {
+      type: "delegation.context.append";
+      delegation_item_id: string;
+      channel?: LiveContextChannel;
+      content: LiveInputTextContent[];
+    }
+  | {
+      type: "session.context.append";
+      channel?: LiveContextChannel;
+      content: LiveInputTextContent[];
+    }
+  | { type: "session.close" };
+
+export type LiveServerEvent =
+  | {
+      type: "session.started" | "session.updated";
+      session: { id: string; instructions?: string };
+    }
+  | { type: "output_audio.delta"; audio: string }
+  | { type: "input_transcript.added" | "output_transcript.added"; item: { text: string } }
+  | { type: "turn.done"; turn: { role: "user" | "assistant"; transcript: string } }
+  | {
+      type: "delegation.created";
+      item: {
+        type: "delegation";
+        target: "client";
+        id: string;
+        content: LiveInputTextContent[];
+      };
+    }
+  | { type: "error"; message: string }
+  | { type: "unknown"; wireType: string };
+
+type UnknownRecord = Record<string, unknown>;
+
+function isRecord(value: unknown): value is UnknownRecord {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function parsePayload(payload: unknown): UnknownRecord | null {
+  let parsed = payload;
+  if (typeof payload === "string") {
+    try {
+      parsed = JSON.parse(payload) as unknown;
+    } catch {
+      return null;
+    }
+  }
+  return isRecord(parsed) ? parsed : null;
+}
+
+function parseSessionEvent(
+  type: "session.started" | "session.updated",
+  payload: UnknownRecord,
+): LiveServerEvent | null {
+  const session = payload.session;
+  if (!isRecord(session) || typeof session.id !== "string") return null;
+  if (typeof session.instructions === "string") {
+    return { type, session: { id: session.id, instructions: session.instructions } };
+  }
+  return { type, session: { id: session.id } };
+}
+
+function parseTranscriptAddedEvent(
+  type: "input_transcript.added" | "output_transcript.added",
+  payload: UnknownRecord,
+): LiveServerEvent | null {
+  const item = payload.item;
+  if (!isRecord(item) || typeof item.text !== "string") return null;
+  return { type, item: { text: item.text } };
+}
+
+function parseTurnDoneEvent(payload: UnknownRecord): LiveServerEvent | null {
+  const turn = payload.turn;
+  if (!isRecord(turn) || (turn.role !== "user" && turn.role !== "assistant")) return null;
+  if (typeof turn.transcript !== "string") return null;
+  return { type: "turn.done", turn: { role: turn.role, transcript: turn.transcript } };
+}
+
+function parseDelegationCreatedEvent(payload: UnknownRecord): LiveServerEvent | null {
+  const item = payload.item;
+  if (
+    !isRecord(item) ||
+    item.type !== "delegation" ||
+    item.target !== "client" ||
+    typeof item.id !== "string"
+  ) {
+    return null;
+  }
+  if (!Array.isArray(item.content)) return null;
+  const content: LiveInputTextContent[] = [];
+  for (const candidate of item.content) {
+    if (
+      !isRecord(candidate) ||
+      candidate.type !== "input_text" ||
+      typeof candidate.text !== "string"
+    ) {
+      continue;
+    }
+    content.push({ type: "input_text", text: candidate.text });
+  }
+  return {
+    type: "delegation.created",
+    item: { type: "delegation", target: "client", id: item.id, content },
+  };
+}
+
+function stringifyErrorValue(value: unknown): string | null {
+  if (typeof value === "string") return value;
+  if (value === undefined) return null;
+  try {
+    return JSON.stringify(value) ?? null;
+  } catch {
+    return String(value);
+  }
+}
+
+function parseErrorEvent(payload: UnknownRecord): LiveServerEvent | null {
+  if (typeof payload.message === "string") return { type: "error", message: payload.message };
+  const error = payload.error;
+  if (isRecord(error) && typeof error.message === "string") {
+    return { type: "error", message: error.message };
+  }
+  const message = stringifyErrorValue(error);
+  return message === null ? null : { type: "error", message };
+}
+
+export function parseLiveServerEvent(payload: unknown): LiveServerEvent | null {
+  const parsed = parsePayload(payload);
+  if (!parsed || typeof parsed.type !== "string") return null;
+
+  switch (parsed.type) {
+    case "session.started":
+    case "session.updated":
+      return parseSessionEvent(parsed.type, parsed);
+    case "output_audio.delta":
+      return typeof parsed.audio === "string" ? { type: parsed.type, audio: parsed.audio } : null;
+    case "input_transcript.added":
+    case "output_transcript.added":
+      return parseTranscriptAddedEvent(parsed.type, parsed);
+    case "turn.done":
+      return parseTurnDoneEvent(parsed);
+    case "delegation.created":
+      return parseDelegationCreatedEvent(parsed);
+    case "error":
+      return parseErrorEvent(parsed);
+    default:
+      return { type: "unknown", wireType: parsed.type };
+  }
+}
+
+export function buildLiveSessionPayload(instructions: string, voice: string): LiveSessionPayload {
+  return {
+    model: LIVE_MODEL,
+    instructions,
+    audio: { output: { voice } },
+    delegation: { type: "client" },
+  };
+}
+
+export function buildDelegationContextAppend(
+  delegationItemId: string,
+  text: string,
+  channel?: LiveContextChannel,
+): LiveClientMessage {
+  return {
+    type: "delegation.context.append",
+    delegation_item_id: delegationItemId,
+    ...(channel === undefined ? {} : { channel }),
+    content: [{ type: "input_text", text }],
+  };
+}
+
+export function buildSessionContextAppend(
+  text: string,
+  channel?: LiveContextChannel,
+): LiveClientMessage {
+  return {
+    type: "session.context.append",
+    ...(channel === undefined ? {} : { channel }),
+    content: [{ type: "input_text", text }],
+  };
+}
+
+export function buildSessionClose(): LiveClientMessage {
+  return { type: "session.close" };
+}
+
+function utf8ByteLength(codePoint: number): number {
+  if (codePoint <= 0x7f) return 1;
+  if (codePoint <= 0x7ff) return 2;
+  if (codePoint <= 0xffff) return 3;
+  return 4;
+}
+
+export function chunkLiveContext(text: string): string[] {
+  if (text.length === 0) return [""];
+
+  const chunks: string[] = [];
+  let chunkStart = 0;
+  let chunkBytes = 0;
+  let index = 0;
+  while (index < text.length) {
+    const codePoint = text.codePointAt(index);
+    if (codePoint === undefined) break;
+    const characterLength = codePoint > 0xffff ? 2 : 1;
+    const characterBytes = utf8ByteLength(codePoint);
+    if (chunkBytes + characterBytes > CONTEXT_CHUNK_BYTES) {
+      chunks.push(text.slice(chunkStart, index));
+      chunkStart = index;
+      chunkBytes = 0;
+    }
+    chunkBytes += characterBytes;
+    index += characterLength;
+  }
+  chunks.push(text.slice(chunkStart));
+  return chunks;
+}