@iinm/plain-agent 1.7.19 → 1.7.21

package/src/voiceInputGemini.mjs

@@ -1,16 +1,10 @@
 import {
-  createCJKSpaceNormalizer,
-  detectRecorder,
-  failVoiceSessionAsync,
-  getRecorderCandidates,
-  isCommandAvailable,
   isObjectLike,
-  startRecorder,
-  VOICE_DEBUG,
+  startWebSocketVoiceSession,
 } from "./voiceInputSession.mjs";
 
 /**
- * @import { VoiceRecorderConfig, VoiceSession, VoiceSessionCallbacks } from "./voiceInputSession.mjs"
+ * @import { VoiceProviderHooks, VoiceRecorderConfig, VoiceSession, VoiceSessionCallbacks } from "./voiceInputSession.mjs"
  */
 
 /**
@@ -45,213 +39,67 @@ const GEMINI_LABEL = "Gemini Live";
  * @returns {VoiceSession}
  */
 export function startGeminiVoiceSession({ config, callbacks }) {
-  const recorder =
-    config.recorder ??
-    detectRecorder(getRecorderCandidates(GEMINI_SAMPLE_RATE));
-  if (!recorder) {
-    return failVoiceSessionAsync(
-      callbacks,
-      new Error(
-        "No voice recorder found. Install arecord, sox, or ffmpeg (or set `voiceInput.recorder`).",
-      ),
-    );
-  }
-
-  if (!isCommandAvailable(recorder.command)) {
-    return failVoiceSessionAsync(
-      callbacks,
-      new Error(
-        `Voice recorder command "${recorder.command}" not found on PATH.`,
-      ),
-    );
-  }
-
-  const model = config.model ?? GEMINI_DEFAULT_MODEL;
-  const base = config.baseURL ?? GEMINI_DEFAULT_WS;
-
-  let stopped = false;
-  let closeEmitted = false;
-  let ready = false;
-  /** @type {Buffer[]} */
-  const pendingAudio = [];
-  const normalizer = createCJKSpaceNormalizer();
-
-  const emitClose = () => {
-    if (closeEmitted) return;
-    closeEmitted = true;
-    callbacks.onClose?.();
-  };
-
-  const ws = new WebSocket(`${base}?key=${encodeURIComponent(config.apiKey)}`);
-  ws.binaryType = "arraybuffer";
-
-  const rec = startRecorder({
-    recorder,
-    onAudio(chunk) {
-      if (stopped) return;
-      if (ready && ws.readyState === WebSocket.OPEN) {
-        sendAudio(chunk);
-      } else {
-        pendingAudio.push(chunk);
+  /** @type {VoiceProviderHooks<VoiceInputGeminiConfig>} */
+  const hooks = {
+    label: GEMINI_LABEL,
+    sampleRate: GEMINI_SAMPLE_RATE,
+    buildWsUrl(config) {
+      const base = config.baseURL ?? GEMINI_DEFAULT_WS;
+      return `${base}?key=${encodeURIComponent(config.apiKey)}`;
+    },
+    buildSetupMessage(config) {
+      const model = config.model ?? GEMINI_DEFAULT_MODEL;
+      /** @type {Record<string, unknown>} */
+      const generationConfig = {
+        // https://ai.google.dev/gemini-api/docs/live-api/capabilities#response-modalities
+        // > The native audio models only support `AUDIO` response modality.
+        responseModalities: ["AUDIO"],
+        maxOutputTokens: 1,
+      };
+      if (model.includes("2.5")) {
+        generationConfig.thinkingConfig = { thinkingBudget: 0 };
+      }
+      /** @type {Record<string, unknown>} */
+      const setup = {
+        model: `models/${model}`,
+        generationConfig,
+        inputAudioTranscription: {},
+      };
+      if (config.language) {
+        setup.systemInstruction = {
+          parts: [{ text: `The user is speaking in ${config.language}.` }],
+        };
       }
+      return { setup };
     },
-    onError(err) {
-      if (!stopped) callbacks.onError(err);
-      stop();
+    isReadyMessage(message) {
+      return isObjectLike(message) && "setupComplete" in message;
     },
-    onExit() {
-      stop();
+    extractTranscript(message) {
+      if (!isObjectLike(message)) return undefined;
+      const serverContent = message.serverContent;
+      if (!isObjectLike(serverContent)) return undefined;
+      const transcription = serverContent.inputTranscription;
+      if (
+        isObjectLike(transcription) &&
+        typeof transcription.text === "string" &&
+        transcription.text.length > 0
+      ) {
+        return transcription.text;
+      }
+      return undefined;
     },
-  });
-
-  /**
-   * @param {Buffer} chunk
-   */
-  function sendAudio(chunk) {
-    const payload = {
-      realtimeInput: {
-        audio: {
-          data: chunk.toString("base64"),
-          mimeType: `audio/pcm;rate=${GEMINI_SAMPLE_RATE}`,
+    buildAudioPayload(chunk, sampleRate) {
+      return {
+        realtimeInput: {
+          audio: {
+            data: chunk.toString("base64"),
+            mimeType: `audio/pcm;rate=${sampleRate}`,
+          },
         },
-      },
-    };
-    try {
-      ws.send(JSON.stringify(payload));
-    } catch {
-      // connection may have just closed
-    }
-  }
-
-  ws.addEventListener("open", () => {
-    /** @type {Record<string, unknown>} */
-    const generationConfig = {
-      // https://ai.google.dev/gemini-api/docs/live-api/capabilities#response-modalities
-      // > The native audio models only support `AUDIO` response modality.
-      responseModalities: ["AUDIO"],
-      maxOutputTokens: 1,
-    };
-    if (model.includes("2.5")) {
-      generationConfig.thinkingConfig = { thinkingBudget: 0 };
-    }
-    /** @type {Record<string, unknown>} */
-    const setup = {
-      model: `models/${model}`,
-      generationConfig,
-      inputAudioTranscription: {},
-    };
-    if (config.language) {
-      setup.systemInstruction = {
-        parts: [{ text: `The user is speaking in ${config.language}.` }],
       };
-    }
-    try {
-      ws.send(JSON.stringify({ setup }));
-    } catch (err) {
-      callbacks.onError(
-        new Error(
-          `Failed to send setup message: ${err instanceof Error ? err.message : String(err)}`,
-        ),
-      );
-      stop();
-    }
-  });
-
-  ws.addEventListener("message", (event) => {
-    if (stopped) return;
-    let raw = "";
-    let message;
-    try {
-      raw =
-        typeof event.data === "string"
-          ? event.data
-          : Buffer.from(/** @type {ArrayBuffer} */ (event.data)).toString(
-              "utf8",
-            );
-      message = JSON.parse(raw);
-    } catch (err) {
-      callbacks.onError(
-        new Error(
-          `Failed to parse server message: ${err instanceof Error ? err.message : String(err)}`,
-        ),
-      );
-      return;
-    }
-    if (!isObjectLike(message)) return;
-    if (VOICE_DEBUG) {
-      process.stderr.write(`[voiceInput] <- ${raw.slice(0, 800)}\n`);
-    }
-
-    if (!ready && "setupComplete" in message) {
-      ready = true;
-      for (const chunk of pendingAudio.splice(0)) {
-        if (ws.readyState === WebSocket.OPEN) sendAudio(chunk);
-      }
-      return;
-    }
-
-    const serverContent = message.serverContent;
-    if (!isObjectLike(serverContent)) return;
-    const transcription = serverContent.inputTranscription;
-    if (
-      isObjectLike(transcription) &&
-      typeof transcription.text === "string" &&
-      transcription.text.length > 0
-    ) {
-      const normalized = normalizer.push(transcription.text);
-      if (normalized.length > 0) {
-        callbacks.onTranscript(normalized);
-      }
-    }
-  });
-
-  ws.addEventListener("error", (event) => {
-    if (stopped) return;
-    const message =
-      /** @type {{ message?: string }} */ (event).message ?? "WebSocket error";
-    callbacks.onError(new Error(`${GEMINI_LABEL} WebSocket error: ${message}`));
-    stop();
-  });
-
-  ws.addEventListener("close", (event) => {
-    if (!stopped && event.code !== 1000 && event.code !== 1005) {
-      const reason = event.reason ? `: ${event.reason}` : "";
-      callbacks.onError(
-        new Error(
-          `${GEMINI_LABEL} WebSocket closed (code ${event.code}${reason})`,
-        ),
-      );
-    }
-    stopped = true;
-    rec.stop();
-    emitClose();
-  });
-
-  if (VOICE_DEBUG) {
-    process.stderr.write(
-      `[voiceInput] driver=${GEMINI_LABEL} recorder=${recorder.command} ${recorder.args.join(" ")}\n`,
-    );
-  }
-
-  /**
-   * @returns {Promise<void>}
-   */
-  async function stop() {
-    if (stopped) return;
-    stopped = true;
-    rec.stop();
-    if (
-      ws.readyState === WebSocket.OPEN ||
-      ws.readyState === WebSocket.CONNECTING
-    ) {
-      try {
-        ws.close(1000, "client stop");
-      } catch {
-        // ignore
-      }
-    }
-    emitClose();
-  }
+    },
+  };
 
-  return { stop };
+  return startWebSocketVoiceSession({ hooks, config, callbacks });
 }

package/src/voiceInputOpenAI.mjs

@@ -1,16 +1,10 @@
 import {
-  createCJKSpaceNormalizer,
-  detectRecorder,
-  failVoiceSessionAsync,
-  getRecorderCandidates,
-  isCommandAvailable,
   isObjectLike,
-  startRecorder,
-  VOICE_DEBUG,
+  startWebSocketVoiceSession,
 } from "./voiceInputSession.mjs";
 
 /**
- * @import { VoiceRecorderConfig, VoiceSession, VoiceSessionCallbacks } from "./voiceInputSession.mjs"
+ * @import { VoiceProviderHooks, VoiceRecorderConfig, VoiceSession, VoiceSessionCallbacks } from "./voiceInputSession.mjs"
  */
 
 /**
@@ -40,222 +34,71 @@ const OPENAI_LABEL = "OpenAI Realtime";
  * @returns {VoiceSession}
  */
 export function startOpenAIVoiceSession({ config, callbacks }) {
-  const recorder =
-    config.recorder ??
-    detectRecorder(getRecorderCandidates(OPENAI_SAMPLE_RATE));
-  if (!recorder) {
-    return failVoiceSessionAsync(
-      callbacks,
-      new Error(
-        "No voice recorder found. Install arecord, sox, or ffmpeg (or set `voiceInput.recorder`).",
-      ),
-    );
-  }
-
-  if (!isCommandAvailable(recorder.command)) {
-    return failVoiceSessionAsync(
-      callbacks,
-      new Error(
-        `Voice recorder command "${recorder.command}" not found on PATH.`,
-      ),
-    );
-  }
-
-  const model = config.model ?? OPENAI_DEFAULT_MODEL;
-  const base = config.baseURL ?? OPENAI_DEFAULT_WS;
-
-  let stopped = false;
-  let closeEmitted = false;
-  let ready = false;
-  /** @type {Buffer[]} */
-  const pendingAudio = [];
-  const normalizer = createCJKSpaceNormalizer();
-
-  const emitClose = () => {
-    if (closeEmitted) return;
-    closeEmitted = true;
-    callbacks.onClose?.();
-  };
-
-  // Node's global WebSocket (undici) accepts a non-standard `headers`
-  // option. The built-in typings only declare the standards-compliant
-  // constructor, so cast through `WebSocket`-as-constructor.
-  const Ctor = /** @type {new (url: string, opts?: unknown) => WebSocket} */ (
-    /** @type {unknown} */ (WebSocket)
-  );
-  const ws = new Ctor(`${base}?intent=transcription`, {
-    headers: {
-      Authorization: `Bearer ${config.apiKey}`,
-      "OpenAI-Beta": "realtime=v1",
-    },
-  });
-  ws.binaryType = "arraybuffer";
-
-  const rec = startRecorder({
-    recorder,
-    onAudio(chunk) {
-      if (stopped) return;
-      if (ready && ws.readyState === WebSocket.OPEN) {
-        sendAudio(chunk);
-      } else {
-        pendingAudio.push(chunk);
-      }
+  /** @type {VoiceProviderHooks<VoiceInputOpenAIConfig>} */
+  const hooks = {
+    label: OPENAI_LABEL,
+    sampleRate: OPENAI_SAMPLE_RATE,
+    buildWsUrl(config) {
+      const base = config.baseURL ?? OPENAI_DEFAULT_WS;
+      return `${base}?intent=transcription`;
     },
-    onError(err) {
-      if (!stopped) callbacks.onError(err);
-      stop();
+    buildWsOptions(config) {
+      return {
+        headers: {
+          Authorization: `Bearer ${config.apiKey}`,
+          "OpenAI-Beta": "realtime=v1",
+        },
+      };
     },
-    onExit() {
-      stop();
+    buildSetupMessage(config) {
+      const model = config.model ?? OPENAI_DEFAULT_MODEL;
+      /** @type {{ model: string, language?: string }} */
+      const transcription = { model };
+      if (config.language) transcription.language = config.language;
+      // The `?intent=transcription` endpoint uses the flat transcription-session
+      // schema, not the nested `session.audio.input.*` realtime schema.
+      return {
+        type: "transcription_session.update",
+        session: {
+          input_audio_format: "pcm16",
+          input_audio_transcription: transcription,
+          turn_detection: { type: "server_vad" },
+        },
+      };
     },
-  });
-
-  /**
-   * @param {Buffer} chunk
-   */
-  function sendAudio(chunk) {
-    const payload = {
-      type: "input_audio_buffer.append",
-      audio: chunk.toString("base64"),
-    };
-    try {
-      ws.send(JSON.stringify(payload));
-    } catch {
-      // connection may have just closed
-    }
-  }
-
-  ws.addEventListener("open", () => {
-    /** @type {{ model: string, language?: string }} */
-    const transcription = { model };
-    if (config.language) transcription.language = config.language;
-    // The `?intent=transcription` endpoint uses the flat transcription-session
-    // schema, not the nested `session.audio.input.*` realtime schema.
-    const setup = {
-      type: "transcription_session.update",
-      session: {
-        input_audio_format: "pcm16",
-        input_audio_transcription: transcription,
-        turn_detection: { type: "server_vad" },
-      },
-    };
-    try {
-      ws.send(JSON.stringify(setup));
-    } catch (err) {
-      callbacks.onError(
-        new Error(
-          `Failed to send setup message: ${err instanceof Error ? err.message : String(err)}`,
-        ),
-      );
-      stop();
-    }
-  });
-
-  ws.addEventListener("message", (event) => {
-    if (stopped) return;
-    let raw = "";
-    let message;
-    try {
-      raw =
-        typeof event.data === "string"
-          ? event.data
-          : Buffer.from(/** @type {ArrayBuffer} */ (event.data)).toString(
-              "utf8",
-            );
-      message = JSON.parse(raw);
-    } catch (err) {
-      callbacks.onError(
-        new Error(
-          `Failed to parse server message: ${err instanceof Error ? err.message : String(err)}`,
-        ),
-      );
-      return;
-    }
-    if (!isObjectLike(message)) return;
-    if (VOICE_DEBUG) {
-      process.stderr.write(`[voiceInput] <- ${raw.slice(0, 800)}\n`);
-    }
-
-    if (message.type === "error" && isObjectLike(message.error)) {
-      const detail =
-        typeof message.error.message === "string"
-          ? message.error.message
-          : JSON.stringify(message.error);
-      callbacks.onError(new Error(`${OPENAI_LABEL} error: ${detail}`));
-      return;
-    }
-
-    if (
-      !ready &&
-      (message.type === "transcription_session.created" ||
-        message.type === "transcription_session.updated")
-    ) {
-      ready = true;
-      for (const chunk of pendingAudio.splice(0)) {
-        if (ws.readyState === WebSocket.OPEN) sendAudio(chunk);
-      }
-      return;
-    }
-
-    if (
-      message.type === "conversation.item.input_audio_transcription.delta" &&
-      typeof message.delta === "string" &&
-      message.delta.length > 0
-    ) {
-      const normalized = normalizer.push(message.delta);
-      if (normalized.length > 0) {
-        callbacks.onTranscript(normalized);
-      }
-    }
-  });
-
-  ws.addEventListener("error", (event) => {
-    if (stopped) return;
-    const message =
-      /** @type {{ message?: string }} */ (event).message ?? "WebSocket error";
-    callbacks.onError(new Error(`${OPENAI_LABEL} WebSocket error: ${message}`));
-    stop();
-  });
-
-  ws.addEventListener("close", (event) => {
-    if (!stopped && event.code !== 1000 && event.code !== 1005) {
-      const reason = event.reason ? `: ${event.reason}` : "";
-      callbacks.onError(
-        new Error(
-          `${OPENAI_LABEL} WebSocket closed (code ${event.code}${reason})`,
-        ),
+    isReadyMessage(message) {
+      return (
+        isObjectLike(message) &&
+        (message.type === "transcription_session.created" ||
+          message.type === "transcription_session.updated")
       );
-    }
-    stopped = true;
-    rec.stop();
-    emitClose();
-  });
-
-  if (VOICE_DEBUG) {
-    process.stderr.write(
-      `[voiceInput] driver=${OPENAI_LABEL} recorder=${recorder.command} ${recorder.args.join(" ")}\n`,
-    );
-  }
-
-  /**
-   * @returns {Promise<void>}
-   */
-  async function stop() {
-    if (stopped) return;
-    stopped = true;
-    rec.stop();
-    if (
-      ws.readyState === WebSocket.OPEN ||
-      ws.readyState === WebSocket.CONNECTING
-    ) {
-      try {
-        ws.close(1000, "client stop");
-      } catch {
-        // ignore
+    },
+    extractError(message) {
+      if (!isObjectLike(message) || message.type !== "error") return undefined;
+      const error = message.error;
+      if (!isObjectLike(error)) return undefined;
+      return typeof error.message === "string"
+        ? error.message
+        : JSON.stringify(error);
+    },
+    extractTranscript(message) {
+      if (
+        isObjectLike(message) &&
+        message.type === "conversation.item.input_audio_transcription.delta" &&
+        typeof message.delta === "string" &&
+        message.delta.length > 0
+      ) {
+        return message.delta;
       }
-    }
-    emitClose();
-  }
+      return undefined;
+    },
+    buildAudioPayload(chunk, _sampleRate) {
+      return {
+        type: "input_audio_buffer.append",
+        audio: chunk.toString("base64"),
+      };
+    },
+  };
 
-  return { stop };
+  return startWebSocketVoiceSession({ hooks, config, callbacks });
 }