@iinm/plain-agent 1.7.18 → 1.7.20

package/src/voiceInputGemini.mjs

@@ -0,0 +1,105 @@
+import {
+  isObjectLike,
+  startWebSocketVoiceSession,
+} from "./voiceInputSession.mjs";
+
+/**
+ * @import { VoiceProviderHooks, VoiceRecorderConfig, VoiceSession, VoiceSessionCallbacks } from "./voiceInputSession.mjs"
+ */
+
+/**
+ * @typedef {Object} VoiceInputGeminiConfig
+ * @property {"gemini"} provider
+ * @property {string} apiKey
+ * @property {string} [model] - Defaults to "gemini-3.1-flash-live-preview".
+ * @property {string} [language] - ISO-639-1 code (e.g. "ja", "en"). Passed to the model as a system instruction since Gemini Live has no native language hint for input transcription.
+ * @property {string} [baseURL]
+ * @property {VoiceRecorderConfig} [recorder]
+ * @property {string} [toggleKey]
+ */
+
+const GEMINI_DEFAULT_MODEL = "gemini-3.1-flash-live-preview";
+const GEMINI_DEFAULT_WS =
+  "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
+const GEMINI_SAMPLE_RATE = 16000;
+const GEMINI_LABEL = "Gemini Live";
+
+/**
+ * Start a voice input session backed by the Gemini Live BidiGenerateContent
+ * WebSocket. Spawns a recorder, streams PCM as base64 JSON messages, and
+ * forwards transcript deltas via `onTranscript`.
+ *
+ * Gemini Live was designed for voice agents, not pure STT, so the setup
+ * message forces `maxOutputTokens: 1` and disables thinking on 2.5 models
+ * to minimise wasted audio output.
+ *
+ * @param {object} options
+ * @param {VoiceInputGeminiConfig} options.config
+ * @param {VoiceSessionCallbacks} options.callbacks
+ * @returns {VoiceSession}
+ */
+export function startGeminiVoiceSession({ config, callbacks }) {
+  /** @type {VoiceProviderHooks<VoiceInputGeminiConfig>} */
+  const hooks = {
+    label: GEMINI_LABEL,
+    sampleRate: GEMINI_SAMPLE_RATE,
+    buildWsUrl(config) {
+      const base = config.baseURL ?? GEMINI_DEFAULT_WS;
+      return `${base}?key=${encodeURIComponent(config.apiKey)}`;
+    },
+    buildSetupMessage(config) {
+      const model = config.model ?? GEMINI_DEFAULT_MODEL;
+      /** @type {Record<string, unknown>} */
+      const generationConfig = {
+        // https://ai.google.dev/gemini-api/docs/live-api/capabilities#response-modalities
+        // > The native audio models only support `AUDIO` response modality.
+        responseModalities: ["AUDIO"],
+        maxOutputTokens: 1,
+      };
+      if (model.includes("2.5")) {
+        generationConfig.thinkingConfig = { thinkingBudget: 0 };
+      }
+      /** @type {Record<string, unknown>} */
+      const setup = {
+        model: `models/${model}`,
+        generationConfig,
+        inputAudioTranscription: {},
+      };
+      if (config.language) {
+        setup.systemInstruction = {
+          parts: [{ text: `The user is speaking in ${config.language}.` }],
+        };
+      }
+      return { setup };
+    },
+    isReadyMessage(message) {
+      return isObjectLike(message) && "setupComplete" in message;
+    },
+    extractTranscript(message) {
+      if (!isObjectLike(message)) return undefined;
+      const serverContent = message.serverContent;
+      if (!isObjectLike(serverContent)) return undefined;
+      const transcription = serverContent.inputTranscription;
+      if (
+        isObjectLike(transcription) &&
+        typeof transcription.text === "string" &&
+        transcription.text.length > 0
+      ) {
+        return transcription.text;
+      }
+      return undefined;
+    },
+    buildAudioPayload(chunk, sampleRate) {
+      return {
+        realtimeInput: {
+          audio: {
+            data: chunk.toString("base64"),
+            mimeType: `audio/pcm;rate=${sampleRate}`,
+          },
+        },
+      };
+    },
+  };
+
+  return startWebSocketVoiceSession({ hooks, config, callbacks });
+}

package/src/voiceInputOpenAI.mjs

@@ -0,0 +1,104 @@
+import {
+  isObjectLike,
+  startWebSocketVoiceSession,
+} from "./voiceInputSession.mjs";
+
+/**
+ * @import { VoiceProviderHooks, VoiceRecorderConfig, VoiceSession, VoiceSessionCallbacks } from "./voiceInputSession.mjs"
+ */
+
+/**
+ * @typedef {Object} VoiceInputOpenAIConfig
+ * @property {"openai"} provider
+ * @property {string} apiKey
+ * @property {string} [model] - Defaults to "gpt-4o-transcribe".
+ * @property {string} [language] - ISO-639-1 code (e.g. "ja", "en"). Improves accuracy and latency when set.
+ * @property {string} [baseURL]
+ * @property {VoiceRecorderConfig} [recorder]
+ * @property {string} [toggleKey] - "ctrl-<char>". Defaults to "ctrl-o".
+ */
+
+const OPENAI_DEFAULT_MODEL = "gpt-4o-transcribe";
+const OPENAI_DEFAULT_WS = "wss://api.openai.com/v1/realtime";
+const OPENAI_SAMPLE_RATE = 24000;
+const OPENAI_LABEL = "OpenAI Realtime";
+
+/**
+ * Start a voice input session backed by the OpenAI Realtime transcription
+ * WebSocket. Spawns a recorder, streams PCM as base64 JSON messages, and
+ * forwards transcript deltas via `onTranscript`.
+ *
+ * @param {object} options
+ * @param {VoiceInputOpenAIConfig} options.config
+ * @param {VoiceSessionCallbacks} options.callbacks
+ * @returns {VoiceSession}
+ */
+export function startOpenAIVoiceSession({ config, callbacks }) {
+  /** @type {VoiceProviderHooks<VoiceInputOpenAIConfig>} */
+  const hooks = {
+    label: OPENAI_LABEL,
+    sampleRate: OPENAI_SAMPLE_RATE,
+    buildWsUrl(config) {
+      const base = config.baseURL ?? OPENAI_DEFAULT_WS;
+      return `${base}?intent=transcription`;
+    },
+    buildWsOptions(config) {
+      return {
+        headers: {
+          Authorization: `Bearer ${config.apiKey}`,
+          "OpenAI-Beta": "realtime=v1",
+        },
+      };
+    },
+    buildSetupMessage(config) {
+      const model = config.model ?? OPENAI_DEFAULT_MODEL;
+      /** @type {{ model: string, language?: string }} */
+      const transcription = { model };
+      if (config.language) transcription.language = config.language;
+      // The `?intent=transcription` endpoint uses the flat transcription-session
+      // schema, not the nested `session.audio.input.*` realtime schema.
+      return {
+        type: "transcription_session.update",
+        session: {
+          input_audio_format: "pcm16",
+          input_audio_transcription: transcription,
+          turn_detection: { type: "server_vad" },
+        },
+      };
+    },
+    isReadyMessage(message) {
+      return (
+        isObjectLike(message) &&
+        (message.type === "transcription_session.created" ||
+          message.type === "transcription_session.updated")
+      );
+    },
+    extractError(message) {
+      if (!isObjectLike(message) || message.type !== "error") return undefined;
+      const error = message.error;
+      if (!isObjectLike(error)) return undefined;
+      return typeof error.message === "string"
+        ? error.message
+        : JSON.stringify(error);
+    },
+    extractTranscript(message) {
+      if (
+        isObjectLike(message) &&
+        message.type === "conversation.item.input_audio_transcription.delta" &&
+        typeof message.delta === "string" &&
+        message.delta.length > 0
+      ) {
+        return message.delta;
+      }
+      return undefined;
+    },
+    buildAudioPayload(chunk, _sampleRate) {
+      return {
+        type: "input_audio_buffer.append",
+        audio: chunk.toString("base64"),
+      };
+    },
+  };
+
+  return startWebSocketVoiceSession({ hooks, config, callbacks });
+}