@hiper2d/ai-agents 0.1.4 → 0.2.0

package/dist/index.mjs

@@ -1618,6 +1618,263 @@ function extractTokenUsageFromResponse7(response) {
   return extractMistralTokenUsage(response);
 }
 
+// src/voice/voice-catalog.ts
+var VOICE_MODEL_CONSTANTS = {
+  OPENAI_TTS: "gpt-4o-mini-tts",
+  OPENAI_STT: "whisper-1",
+  // ai.google.dev/gemini-api/docs/speech-generation
+  GOOGLE_TTS: "gemini-3.1-flash-tts-preview",
+  // ai.google.dev/gemini-api/docs/transcribe — Interactions API only (see google-stt.ts)
+  GOOGLE_STT: "gemini-3.5-transcribe"
+};
+var SUPPORTED_VOICE_PROVIDERS = ["openai", "google"];
+var VOICE_PROVIDER_API_KEY = {
+  openai: API_KEY_CONSTANTS.OPENAI,
+  google: API_KEY_CONSTANTS.GOOGLE
+};
+var VOICE_MODEL_PRICING = {
+  [VOICE_MODEL_CONSTANTS.OPENAI_TTS]: { pricePerMillionCharacters: 15 },
+  [VOICE_MODEL_CONSTANTS.OPENAI_STT]: { pricePerMinute: 6e-3 },
+  // Measured 2026-09-05: ~32 audio tokens per second of speech, so a 15-second
+  // line is ~$0.01 — about 3-5x an OpenAI line of the same length.
+  [VOICE_MODEL_CONSTANTS.GOOGLE_TTS]: { textInputPricePerM: 1, audioOutputPricePerM: 20 },
+  // ~25 audio tokens per second in, ~175 text tokens per minute out: ≈ $0.005/min.
+  [VOICE_MODEL_CONSTANTS.GOOGLE_STT]: { audioInputPricePerM: 2, textOutputPricePerM: 12 }
+};
+
+// src/voice/voice-pricing.ts
+function roundUSD(value) {
+  return parseFloat((value || 0).toFixed(6));
+}
+function calculateOpenAiTtsCost(characterCount) {
+  const rate = VOICE_MODEL_PRICING[VOICE_MODEL_CONSTANTS.OPENAI_TTS]?.pricePerMillionCharacters ?? 0;
+  if (!characterCount || characterCount <= 0 || rate <= 0) return 0;
+  return roundUSD(characterCount / 1e6 * rate);
+}
+function calculateOpenAiSttCost(durationSeconds) {
+  const rate = VOICE_MODEL_PRICING[VOICE_MODEL_CONSTANTS.OPENAI_STT]?.pricePerMinute ?? 0;
+  if (!durationSeconds || durationSeconds <= 0 || rate <= 0) return 0;
+  return roundUSD(durationSeconds / 60 * rate);
+}
+function calculateGeminiTtsCost(usage) {
+  const pricing = VOICE_MODEL_PRICING[VOICE_MODEL_CONSTANTS.GOOGLE_TTS];
+  return tokenPairCost(usage, pricing?.textInputPricePerM ?? 0, pricing?.audioOutputPricePerM ?? 0);
+}
+function calculateGeminiSttCost(usage) {
+  const pricing = VOICE_MODEL_PRICING[VOICE_MODEL_CONSTANTS.GOOGLE_STT];
+  return tokenPairCost(usage, pricing?.audioInputPricePerM ?? 0, pricing?.textOutputPricePerM ?? 0);
+}
+function tokenPairCost(usage, inputRate, outputRate) {
+  const inputTokens = Math.max(0, usage.inputTokens || 0);
+  const outputTokens = Math.max(0, usage.outputTokens || 0);
+  return inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate;
+}
+
+// src/voice/openai-tts.ts
+import { OpenAI } from "openai";
+async function generateOpenAiTtsAudio(text, apiKey, options = {}) {
+  const client = new OpenAI({ apiKey });
+  const speechOptions = {
+    model: VOICE_MODEL_CONSTANTS.OPENAI_TTS,
+    voice: options.voice || "alloy",
+    input: text,
+    speed: options.speed || 1,
+    response_format: options.format || "wav"
+  };
+  if (options.instructions) {
+    speechOptions.instructions = options.instructions;
+  }
+  const response = await client.audio.speech.create(speechOptions);
+  return await response.arrayBuffer();
+}
+
+// src/voice/openai-stt.ts
+import { OpenAI as OpenAI2 } from "openai";
+async function transcribeWithOpenAi(audioBuffer, apiKey, options = {}) {
+  const client = new OpenAI2({ apiKey });
+  const audioFile = new File([new Uint8Array(audioBuffer)], options.fileName || "audio.webm", { type: options.mimeType || "audio/webm" });
+  const transcription = await client.audio.transcriptions.create({
+    file: audioFile,
+    model: VOICE_MODEL_CONSTANTS.OPENAI_STT,
+    language: options.language || "en",
+    prompt: options.prompt,
+    temperature: options.temperature || 0,
+    response_format: "verbose_json"
+  });
+  const explicitDuration = Number(transcription?.duration) || 0;
+  const segments = Array.isArray(transcription?.segments) ? transcription.segments : [];
+  const segmentsDuration = segments.reduce((max, segment) => {
+    const end = Number(segment?.end);
+    return end > max ? end : max;
+  }, 0);
+  const text = typeof transcription?.text === "string" ? transcription.text : segments.map((segment) => segment?.text || "").join(" ");
+  return { text: text.trim(), durationSeconds: explicitDuration || segmentsDuration };
+}
+
+// src/voice/openai-voice-agent.ts
+var OpenAiVoiceAgent = class {
+  constructor(apiKey) {
+    this.apiKey = apiKey;
+  }
+  apiKey;
+  provider = "openai";
+  ttsModel = VOICE_MODEL_CONSTANTS.OPENAI_TTS;
+  sttModel = VOICE_MODEL_CONSTANTS.OPENAI_STT;
+  async speak(request) {
+    const audio = await generateOpenAiTtsAudio(request.text, this.apiKey, {
+      voice: request.voice,
+      instructions: request.voiceStyle || void 0
+    });
+    const characters = request.text.length;
+    return { audio, costUSD: calculateOpenAiTtsCost(characters), usage: { characters } };
+  }
+  async transcribe(request) {
+    const { text, durationSeconds } = await transcribeWithOpenAi(request.audio, this.apiKey, {
+      language: request.language,
+      prompt: request.prompt,
+      fileName: request.fileName,
+      mimeType: request.mimeType
+    });
+    return { text, durationSeconds, costUSD: calculateOpenAiSttCost(durationSeconds), usage: { audioSeconds: durationSeconds } };
+  }
+};
+
+// src/voice/google-tts.ts
+import { GoogleGenAI } from "@google/genai";
+var AUDIO_TOKENS_PER_SECOND = 32;
+var SAMPLE_RATE = 24e3;
+var PCM_BYTES_PER_SECOND = SAMPLE_RATE * 2;
+function buildGoogleTtsPrompt(text, voiceStyle) {
+  const style = voiceStyle?.trim().replace(/[:.!,;\s]+$/, "");
+  if (!style) return text;
+  const isShort = style.split(/\s+/).length <= 3 && !/[.!?,;]/.test(style);
+  return isShort ? `Say ${style}: ${text}` : `${style}:
+${text}`;
+}
+function pcmToWav(pcmData, sampleRate = SAMPLE_RATE) {
+  const numChannels = 1;
+  const bitsPerSample = 16;
+  const blockAlign = numChannels * (bitsPerSample / 8);
+  const byteRate = sampleRate * blockAlign;
+  const headerSize = 44;
+  const buffer = new ArrayBuffer(headerSize + pcmData.length);
+  const view = new DataView(buffer);
+  const writeString = (offset, str) => {
+    for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i));
+  };
+  writeString(0, "RIFF");
+  view.setUint32(4, 36 + pcmData.length, true);
+  writeString(8, "WAVE");
+  writeString(12, "fmt ");
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, numChannels, true);
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, byteRate, true);
+  view.setUint16(32, blockAlign, true);
+  view.setUint16(34, bitsPerSample, true);
+  writeString(36, "data");
+  view.setUint32(40, pcmData.length, true);
+  new Uint8Array(buffer, headerSize).set(pcmData);
+  return buffer;
+}
+async function generateGoogleTtsAudio(text, apiKey, options) {
+  const client = new GoogleGenAI({ apiKey });
+  const response = await client.models.generateContent({
+    model: VOICE_MODEL_CONSTANTS.GOOGLE_TTS,
+    contents: [{ parts: [{ text: buildGoogleTtsPrompt(text, options.voiceStyle) }] }],
+    config: {
+      responseModalities: ["AUDIO"],
+      speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: options.voiceName } } }
+    }
+  });
+  const parts = response.candidates?.[0]?.content?.parts ?? [];
+  const audioPart = parts.find((part) => part.inlineData?.mimeType?.startsWith("audio/"));
+  if (!audioPart?.inlineData?.data) {
+    throw new Error("No audio data in Google TTS response");
+  }
+  const pcmData = new Uint8Array(Buffer.from(audioPart.inlineData.data, "base64"));
+  const usageMetadata = response.usageMetadata ?? {};
+  const inputTokens = usageMetadata.promptTokenCount ?? 0;
+  const reportedOutput = usageMetadata.candidatesTokenCount;
+  const outputTokens = reportedOutput && reportedOutput > 0 ? reportedOutput : Math.ceil(pcmData.length / PCM_BYTES_PER_SECOND * AUDIO_TOKENS_PER_SECOND);
+  return { audio: pcmToWav(pcmData), usage: { inputTokens, outputTokens } };
+}
+
+// src/voice/google-stt.ts
+import { GoogleGenAI as GoogleGenAI2 } from "@google/genai";
+var GEMINI_AUDIO_TOKENS_PER_SECOND = 25;
+async function transcribeWithGemini(audioBuffer, apiKey, options = {}) {
+  const client = new GoogleGenAI2({ apiKey });
+  const interaction = await client.interactions.create({
+    model: VOICE_MODEL_CONSTANTS.GOOGLE_STT,
+    input: [{ type: "audio", data: Buffer.from(audioBuffer).toString("base64"), mime_type: options.mimeType || "audio/webm" }]
+  });
+  const text = typeof interaction?.output_text === "string" ? interaction.output_text.trim() : "";
+  const usage = interaction?.usage ?? {};
+  const byModality = (rows, modality) => (rows ?? []).filter((r) => r?.modality === modality).reduce((sum, r) => sum + (Number(r?.tokens) || 0), 0);
+  const inputTokens = byModality(usage.input_tokens_by_modality, "audio") || usage.total_input_tokens || 0;
+  const invocationOutput = (usage.model_invocation_token_counts ?? []).reduce((sum, inv) => sum + (inv?.candidates_tokens_details ?? []).reduce((s, d) => s + (Number(d?.tokens) || 0), 0), 0);
+  const outputTokens = usage.total_output_tokens || invocationOutput || Math.ceil(text.length / 4);
+  return { text, durationSeconds: inputTokens / GEMINI_AUDIO_TOKENS_PER_SECOND, usage: { inputTokens, outputTokens } };
+}
+
+// src/voice/google-voice-agent.ts
+var GoogleVoiceAgent = class {
+  constructor(apiKey) {
+    this.apiKey = apiKey;
+  }
+  apiKey;
+  provider = "google";
+  ttsModel = VOICE_MODEL_CONSTANTS.GOOGLE_TTS;
+  sttModel = VOICE_MODEL_CONSTANTS.GOOGLE_STT;
+  async speak(request) {
+    const { audio, usage } = await generateGoogleTtsAudio(request.text, this.apiKey, {
+      voiceName: request.voice,
+      voiceStyle: request.voiceStyle
+    });
+    return { audio, costUSD: calculateGeminiTtsCost(usage), usage };
+  }
+  async transcribe(request) {
+    const { text, durationSeconds, usage } = await transcribeWithGemini(request.audio, this.apiKey, { mimeType: request.mimeType });
+    return { text, durationSeconds, costUSD: calculateGeminiSttCost(usage), usage };
+  }
+};
+
+// src/voice/voice-agent-factory.ts
+var VoiceAgentFactory = class _VoiceAgentFactory {
+  /** Build an agent from a key already resolved by the caller. */
+  static createAgent(provider, apiKey) {
+    switch (provider) {
+      case "openai":
+        return new OpenAiVoiceAgent(apiKey);
+      case "google":
+        return new GoogleVoiceAgent(apiKey);
+      default:
+        throw new Error(`Unknown voice provider: ${provider}`);
+    }
+  }
+  /**
+   * Build an agent from a key map (the shape AgentFactory takes), reading the
+   * provider's key by its API_KEY_CONSTANTS name. Throws when the key is missing
+   * so the host can report a misconfiguration before any SDK is touched.
+   */
+  static createAgentFromKeys(provider, apiKeys) {
+    if (!SUPPORTED_VOICE_PROVIDERS.includes(provider)) {
+      throw new Error(`Unknown voice provider: ${provider}`);
+    }
+    const keyName = VOICE_PROVIDER_API_KEY[provider];
+    const apiKey = apiKeys[keyName];
+    if (!apiKey) {
+      throw new Error(`Missing API key ${keyName} for voice provider ${provider}`);
+    }
+    return _VoiceAgentFactory.createAgent(provider, apiKey);
+  }
+};
+function createVoiceAgent(provider, apiKey) {
+  return VoiceAgentFactory.createAgent(provider, apiKey);
+}
+
 // src/agents/abstract-agent.ts
 var AbstractAgent = class {
   name;
@@ -1764,8 +2021,7 @@ ${msg.content}` };
 };
 
 // src/agents/gpt-5-agent.ts
-import OpenAI from "openai";
-import { z as z2 } from "zod";
+import OpenAI3 from "openai";
 import { zodTextFormat } from "openai/helpers/zod";
 var Gpt5Agent = class extends AbstractAgent {
   client;
@@ -1781,7 +2037,7 @@ var Gpt5Agent = class extends AbstractAgent {
   };
   constructor(name, instruction, model, apiKey, temperature, enableThinking = false, agentLoggingConfig = DEFAULT_LOGGING_CONFIG.agents) {
     super(name, instruction, model, temperature, enableThinking, agentLoggingConfig);
-    this.client = new OpenAI({
+    this.client = new OpenAI3({
       apiKey
     });
   }
@@ -1799,12 +2055,7 @@ var Gpt5Agent = class extends AbstractAgent {
         `System: ${this.instruction}`,
         ...this.prepareMessages(messages).map((msg) => `${msg.role === "user" ? "User" : "Assistant"}: ${msg.content}`)
       ].join("\n\n");
-      let schemaToSend = zodSchema;
-      if (this.enableThinking && zodSchema instanceof z2.ZodObject) {
-        schemaToSend = zodSchema.extend({
-          thinking: z2.string().describe("Your internal chain-of-thought reasoning process used to arrive at the final answer.")
-        });
-      }
+      const schemaToSend = zodSchema;
       let response;
       try {
         response = await this.client.responses.parse({
@@ -2318,7 +2569,7 @@ ${schemaDescription}`;
 };
 
 // src/agents/google-agent.ts
-import { GoogleGenAI } from "@google/genai";
+import { GoogleGenAI as GoogleGenAI3 } from "@google/genai";
 var GoogleAgent = class extends AbstractAgent {
   client;
   defaultConfig = {
@@ -2337,7 +2588,7 @@ var GoogleAgent = class extends AbstractAgent {
   };
   constructor(name, instruction, model, apiKey, enableThinking = false, agentLoggingConfig = DEFAULT_LOGGING_CONFIG.agents) {
     super(name, instruction, model, 0.2, enableThinking, agentLoggingConfig);
-    this.client = new GoogleGenAI({
+    this.client = new GoogleGenAI3({
       apiKey
     });
   }
@@ -2898,7 +3149,7 @@ ${schemaDescription}`
 };
 
 // src/agents/deepseek-v2-agent.ts
-import OpenAI2 from "openai";
+import OpenAI4 from "openai";
 var DeepSeekV2Agent = class extends AbstractAgent {
   client;
   // Log message templates
@@ -2914,7 +3165,7 @@ var DeepSeekV2Agent = class extends AbstractAgent {
   };
   constructor(name, instruction, model, apiKey, temperature, enableThinking = false, agentLoggingConfig = DEFAULT_LOGGING_CONFIG.agents) {
     super(name, instruction, model, temperature, enableThinking, agentLoggingConfig);
-    this.client = new OpenAI2({
+    this.client = new OpenAI4({
       baseURL: "https://api.deepseek.com",
       apiKey
     });
@@ -3106,7 +3357,7 @@ ${schemaDescription}`
 };
 
 // src/agents/grok-agent.ts
-import { OpenAI as OpenAI3 } from "openai";
+import { OpenAI as OpenAI5 } from "openai";
 var GrokAgent = class extends AbstractAgent {
   client;
   // Log message templates
@@ -3123,7 +3374,7 @@ var GrokAgent = class extends AbstractAgent {
     super(name, instruction, model, temperature, enableThinking, agentLoggingConfig);
     const convId = stableHashHex(`${name}
 ${instruction}`);
-    this.client = new OpenAI3({
+    this.client = new OpenAI5({
       apiKey,
       baseURL: "https://api.x.ai/v1",
       timeout: 12e5,
@@ -3296,7 +3547,7 @@ ${input[0].content}`;
 };
 
 // src/agents/kimi-agent.ts
-import { OpenAI as OpenAI4 } from "openai";
+import { OpenAI as OpenAI6 } from "openai";
 var KimiAgent = class extends AbstractAgent {
   client;
   // kimi-k3 rejects any temperature other than 1, so we never send the field.
@@ -3323,7 +3574,7 @@ var KimiAgent = class extends AbstractAgent {
   };
   constructor(name, instruction, model, apiKey, temperature, enableThinking = false, agentLoggingConfig = DEFAULT_LOGGING_CONFIG.agents) {
     super(name, instruction, model, temperature, enableThinking, agentLoggingConfig);
-    this.client = new OpenAI4({
+    this.client = new OpenAI6({
       apiKey,
       baseURL: "https://api.moonshot.ai/v1"
     });
@@ -3514,7 +3765,7 @@ ${openAIMessages[0].content}`;
 };
 
 // src/agents/glm-agent.ts
-import { OpenAI as OpenAI5 } from "openai";
+import { OpenAI as OpenAI7 } from "openai";
 var GlmAgent = class extends AbstractAgent {
   client;
   // A getter, not a field: `maxOutputTokens` can be raised after construction, and a field
@@ -3540,7 +3791,7 @@ var GlmAgent = class extends AbstractAgent {
   };
   constructor(name, instruction, model, apiKey, temperature, enableThinking = false, agentLoggingConfig = DEFAULT_LOGGING_CONFIG.agents) {
     super(name, instruction, model, temperature, enableThinking, agentLoggingConfig);
-    this.client = new OpenAI5({
+    this.client = new OpenAI7({
       apiKey,
       baseURL: "https://api.z.ai/api/paas/v4/"
     });
@@ -3694,7 +3945,7 @@ ${openAIMessages[0].content}`;
 };
 
 // src/agents/fugu-agent.ts
-import { OpenAI as OpenAI6 } from "openai";
+import { OpenAI as OpenAI8 } from "openai";
 var FuguAgent = class extends AbstractAgent {
   client;
   // A getter, not a field: `maxOutputTokens` can be raised after construction, and a field
@@ -3718,7 +3969,7 @@ var FuguAgent = class extends AbstractAgent {
   };
   constructor(name, instruction, model, apiKey, enableThinking = false, agentLoggingConfig = DEFAULT_LOGGING_CONFIG.agents) {
     super(name, instruction, model, 1, enableThinking, agentLoggingConfig);
-    this.client = new OpenAI6({
+    this.client = new OpenAI8({
       apiKey,
       baseURL: "https://api.sakana.ai/v1",
       timeout: 12e5
@@ -3885,7 +4136,7 @@ ${schemaDescription}`;
 };
 
 // src/agents/qwen-agent.ts
-import { OpenAI as OpenAI7 } from "openai";
+import { OpenAI as OpenAI9 } from "openai";
 var QwenAgent = class extends AbstractAgent {
   client;
   // A getter, not a field: `maxOutputTokens` can be raised after construction, and a field
@@ -3910,7 +4161,7 @@ var QwenAgent = class extends AbstractAgent {
   };
   constructor(name, instruction, model, apiKey, temperature, enableThinking = false, agentLoggingConfig = DEFAULT_LOGGING_CONFIG.agents) {
     super(name, instruction, model, temperature, enableThinking, agentLoggingConfig);
-    this.client = new OpenAI7({
+    this.client = new OpenAI9({
       apiKey,
       baseURL: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
     });
@@ -4085,7 +4336,7 @@ ${openAIMessages[0].content}`;
 };
 
 // src/agents/minimax-agent.ts
-import { OpenAI as OpenAI8 } from "openai";
+import { OpenAI as OpenAI10 } from "openai";
 var MiniMaxAgent = class extends AbstractAgent {
   client;
   // A getter, not a field: `maxOutputTokens` can be raised after construction, and a field
@@ -4110,7 +4361,7 @@ var MiniMaxAgent = class extends AbstractAgent {
   };
   constructor(name, instruction, model, apiKey, temperature, enableThinking = false, agentLoggingConfig = DEFAULT_LOGGING_CONFIG.agents) {
     super(name, instruction, model, temperature, enableThinking, agentLoggingConfig);
-    this.client = new OpenAI8({
+    this.client = new OpenAI10({
       apiKey,
       baseURL: "https://api.minimax.io/v1"
     });
@@ -4347,10 +4598,12 @@ export {
   DeepSeekV2Agent,
   FUGU_REASONING_EFFORTS,
   FuguAgent,
+  GEMINI_AUDIO_TOKENS_PER_SECOND,
   GEMINI_REASONING_EFFORTS,
   GLM_REASONING_EFFORTS,
   GlmAgent,
   GoogleAgent,
+  GoogleVoiceAgent,
   Gpt5Agent,
   GrokAgent,
   KimiAgent,
@@ -4368,23 +4621,35 @@ export {
   ModelRefusalError,
   ModelUnavailableError,
   OPENAI_REASONING_EFFORTS,
+  OpenAiVoiceAgent,
   QwenAgent,
   REASONING_EFFORT_SCALE,
+  SUPPORTED_VOICE_PROVIDERS,
   SupportedAiKeyNames,
   SupportedAiModels,
+  VOICE_MODEL_CONSTANTS,
+  VOICE_MODEL_PRICING,
+  VOICE_PROVIDER_API_KEY,
+  VoiceAgentFactory,
   ZodSchemaConverter,
+  buildGoogleTtsPrompt,
   calculateAnthropicCost,
   calculateCost,
   calculateDeepSeekCost,
+  calculateGeminiSttCost,
+  calculateGeminiTtsCost,
   calculateGoogleCost,
   calculateGrokCost,
   calculateKimiCost,
   calculateMistralCost,
   calculateModelCost,
   calculateOpenAICost,
+  calculateOpenAiSttCost,
+  calculateOpenAiTtsCost,
   clampReasoningEffort,
   cleanResponse,
   createCatalog,
+  createVoiceAgent,
   extractAnthropicTokenUsage,
   extractTokenUsageFromResponse5 as extractAnthropicTokenUsageFromResponse,
   extractDeepSeekTokenUsage,
@@ -4402,6 +4667,8 @@ export {
   extractTokenUsageFromResponse as extractOpenAITokenUsageFromResponse,
   extractTokenUsage,
   extractUsageAndCalculateCost,
+  generateGoogleTtsAudio,
+  generateOpenAiTtsAudio,
   generateSchemaInstructions,
   getModelConfigByApiName,
   getModelDisplayName,
@@ -4418,6 +4685,7 @@ export {
   modelIsFast,
   needsPromptBasedSchema,
   parseAndValidateLlmJson,
+  pcmToWav,
   safeValidateResponse,
   setLlmLogger,
   stableHashHex,
@@ -4429,6 +4697,8 @@ export {
   toGeminiEffort,
   toGlmEffort,
   toOpenAIEffort,
+  transcribeWithGemini,
+  transcribeWithOpenAi,
   validateResponse
 };
 //# sourceMappingURL=index.mjs.map