@luanpoppe/ai 1.1.4 → 1.1.6

package/src/ai.ts

@@ -1,4 +1,13 @@
-import { AIModels, LLMModelConfig } from "./langchain/models";
+import {
+  AIModels,
+  DEFAULT_OLLAMA_BASE_URL,
+  LLMModelConfig,
+} from "./langchain/models";
+import {
+  isDeepSeekJsonObjectOnlyModel,
+  mergeSystemPromptWithJsonSchema,
+  parseJsonFromAssistantMessages,
+} from "./langchain/structured-output";
 import z from "zod";
 import { createAgent, modelRetryMiddleware } from "langchain";
 import type { AIAgent } from "./@types/agent";
@@ -27,6 +36,12 @@ type AIConstructor = {
   googleGeminiToken?: string;
   openAIApiKey?: string;
   openRouterApiKey?: string;
+  /** URL base da API nativa do Ollama (modelos `ollama/*`). Padrão: http://127.0.0.1:11434 */
+  ollamaBaseUrl?: string;
+  /** URL base OpenAI-compatible para modelos `local/*` (ex.: http://localhost:1234/v1) */
+  localBaseUrl?: string;
+  /** API key enviada ao servidor local; muitos aceitam qualquer valor */
+  localApiKey?: string;
   /** Lista padrão de modelos de fallback (usada em call/callStructuredOutput quando não passada no método) */
   aiModelsFallback?: AIModelNames[];
   /** Configuração de persistência de histórico (memory, sqlite, postgres, redis, mongodb) ou instância AIMemory */
@@ -164,24 +179,56 @@ export class AI {
 
     const { result: response, agent } = await this.invokeWithRetryAndFallback(
       params,
-      (paramsForModel) =>
-        createAgent({
-          ...this.standardAgent(paramsForModel, checkpointer),
-          responseFormat: this.normalizeSchemaForOpenAI(
-            outputSchema,
-            paramsForModel.aiModel,
-          ) as any,
-        }),
+      (paramsForModel) => {
+        const useJsonObject = isDeepSeekJsonObjectOnlyModel(
+          paramsForModel.aiModel,
+        );
+        const agentParams = useJsonObject
+          ? this.withDeepSeekJsonObjectParams(paramsForModel, outputSchema)
+          : paramsForModel;
+
+        return createAgent({
+          ...this.standardAgent(agentParams, checkpointer),
+          ...(useJsonObject
+            ? { responseFormat: undefined as any }
+            : {
+                responseFormat: this.normalizeSchemaForOpenAI(
+                  outputSchema,
+                  paramsForModel.aiModel,
+                ) as any,
+              }),
+        });
+      },
       (agent) => agent.invoke({ messages }, invokeConfig as any),
     );
 
     this._memory?.setAgent(agent);
 
-    const parsedResponse = outputSchema.parse(response?.structuredResponse);
+    const parsedResponse =
+      response?.structuredResponse != null
+        ? outputSchema.parse(response.structuredResponse)
+        : outputSchema.parse(parseJsonFromAssistantMessages(response.messages));
 
     return { response: parsedResponse };
   }
 
+  private withDeepSeekJsonObjectParams<T extends AICallParams>(
+    params: T,
+    outputSchema: z.ZodSchema,
+  ): T {
+    return {
+      ...params,
+      systemPrompt: mergeSystemPromptWithJsonSchema(
+        params.systemPrompt,
+        outputSchema,
+      ),
+      modelConfig: {
+        ...params.modelConfig,
+        openRouterForceJsonObject: true,
+      },
+    };
+  }
+
   /**
    * Normaliza schemas Zod para compatibilidade com OpenAI/OpenRouter
    * OpenAI exige que todos os campos em properties estejam no array required
@@ -278,6 +325,38 @@ export class AI {
         ...config,
         model: modelName,
         apiKey: this.config.openRouterApiKey,
+        openRouterProvider: modelConfig?.openRouterProvider,
+        openRouterAllowAllProviders: modelConfig?.openRouterAllowAllProviders,
+        openRouterForceJsonObject: modelConfig?.openRouterForceJsonObject,
+      });
+    }
+
+    if (aiModel.startsWith("ollama/")) {
+      const modelName = aiModel.replace(/^ollama\//, "");
+      return AIModels.ollama({
+        model: modelName,
+        baseUrl: modelConfig?.baseUrl ?? this.config.ollamaBaseUrl ?? DEFAULT_OLLAMA_BASE_URL,
+        maxTokens: config.maxTokens,
+        temperature: config.temperature,
+        numCtx: modelConfig?.numCtx,
+      });
+    }
+
+    if (aiModel.startsWith("local/")) {
+      const modelName = aiModel.replace(/^local\//, "");
+      const baseURL = modelConfig?.baseUrl ?? this.config.localBaseUrl;
+      if (!baseURL) {
+        throw new Error(
+          '[@luanpoppe/ai] localBaseUrl é obrigatório no construtor de AI (ou modelConfig.baseUrl na chamada) para modelos "local/*". ' +
+            "Ex.: localBaseUrl: \"http://localhost:1234/v1\" (LM Studio) ou \"http://127.0.0.1:11434/v1\" (Ollama OpenAI-compatible).",
+        );
+      }
+      return AIModels.openaiCompatible({
+        model: modelName,
+        baseURL,
+        apiKey: this.config.localApiKey,
+        maxTokens: config.maxTokens,
+        temperature: config.temperature,
       });
     }
 

package/src/audio/gemini-transcription.ts

@@ -0,0 +1,47 @@
+import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
+import { HumanMessage } from "langchain";
+import type { AudioBuffer } from "../@types/audio";
+import { AudioUtils } from "../utils/audio-utils";
+import type { GeminiTranscriptionOptions } from "./types";
+
+const DEFAULT_GEMINI_TRANSCRIBE_PROMPT =
+  "Transcreva o áudio a seguir em texto. Se possível, inclua timestamps aproximados no formato [mm:ss]. Responda apenas com a transcrição.";
+
+export async function transcribeWithGeminiPrompt(
+  audioBuffer: AudioBuffer,
+  options: GeminiTranscriptionOptions = {},
+): Promise<string> {
+  const apiKey = options.googleGeminiToken;
+  if (!apiKey) {
+    throw new Error(
+      "googleGeminiToken é obrigatório para transcribeWithGeminiPrompt.",
+    );
+  }
+
+  const model = new ChatGoogleGenerativeAI({
+    model: options.model ?? "gemini-2.5-flash",
+    apiKey,
+  });
+
+  const base64Data = AudioUtils.bufferToBase64(audioBuffer);
+  const mimeType = AudioUtils.detectAudioMimeType(audioBuffer);
+
+  const message = new HumanMessage({
+    content: [
+      {
+        type: "text",
+        text: options.prompt ?? DEFAULT_GEMINI_TRANSCRIBE_PROMPT,
+      },
+      {
+        type: "audio",
+        source_type: "base64",
+        data: base64Data,
+        mime_type: mimeType,
+      },
+    ] as any,
+  } as any);
+
+  const response = await model.invoke([message]);
+  const content = response.content;
+  return typeof content === "string" ? content : JSON.stringify(content);
+}

package/src/audio/index.ts

@@ -0,0 +1,101 @@
+import type { AudioBuffer } from "../@types/audio";
+import type {
+  GeminiTranscriptionOptions,
+  OpenAITranscriptionDetailed,
+  OpenAITranscriptionOptions,
+  OpenRouterSpeechOptions,
+  OpenRouterTranscriptionOptions,
+  OpenRouterTranscriptionResult,
+  OpenAISpeechOptions,
+  SpeechResult,
+} from "./types";
+import {
+  transcribeDetailedOpenAI,
+  transcribeDiarizedOpenAI,
+  transcribeFileDetailedOpenAI,
+  transcribeFileWithWhisperOpenAI,
+  transcribeToSrtOpenAI,
+  transcribeToVttOpenAI,
+  transcribeWithWhisperOpenAI,
+  translateOpenAI,
+} from "./transcription-openai";
+import { transcribeOpenRouter } from "./transcription-openrouter";
+import { speakOpenAI, speakOpenAIStream } from "./speech-openai";
+import { speakOpenRouter, speakOpenRouterStream } from "./speech-openrouter";
+import { transcribeWithGeminiPrompt } from "./gemini-transcription";
+
+export type {
+  WhisperModel,
+  OpenAITranscriptionOptions,
+  OpenAITranscriptionResponseFormat,
+  OpenAITranscriptionDetailed,
+  OpenRouterTranscriptionOptions,
+  OpenRouterTranscriptionResult,
+  OpenAISpeechOptions,
+  OpenAISpeechModel,
+  OpenRouterSpeechOptions,
+  SpeechResult,
+  TranscriptionWord,
+  TranscriptionSegment,
+  DiarizedSegment,
+  GeminiTranscriptionOptions,
+} from "./types";
+
+export { ANTHROPIC_NO_NATIVE_AUDIO } from "./types";
+
+export class AIAudio {
+  static transcribeDetailedOpenAI = transcribeDetailedOpenAI;
+  static transcribeWithWhisper = transcribeWithWhisperOpenAI;
+  static transcribeFileWithWhisper = transcribeFileWithWhisperOpenAI;
+  static transcribeToSrtOpenAI = transcribeToSrtOpenAI;
+  static transcribeToVttOpenAI = transcribeToVttOpenAI;
+  static transcribeDiarizedOpenAI = transcribeDiarizedOpenAI;
+  static translateOpenAI = translateOpenAI;
+  static transcribeOpenRouter = transcribeOpenRouter;
+  static speakOpenAI = speakOpenAI;
+  static speakOpenAIStream = speakOpenAIStream;
+  static speakOpenRouter = speakOpenRouter;
+  static speakOpenRouterStream = speakOpenRouterStream;
+  static transcribeWithGeminiPrompt = transcribeWithGeminiPrompt;
+
+  /** @deprecated Use AIAudio.transcribeWithWhisper */
+  static transcribeWithWhisperLegacy(
+    audioBuffer: AudioBuffer,
+    options?: OpenAITranscriptionOptions,
+    openAIApiKey?: string,
+  ): Promise<string> {
+    return transcribeWithWhisperOpenAI(audioBuffer, options, openAIApiKey);
+  }
+
+  static async transcribeFileDetailedOpenAI(
+    filePath: string,
+    options?: OpenAITranscriptionOptions,
+    openAIApiKey?: string,
+  ): Promise<OpenAITranscriptionDetailed> {
+    return transcribeFileDetailedOpenAI(filePath, options, openAIApiKey);
+  }
+
+  static async transcribeDetailedOpenRouter(
+    audioBuffer: AudioBuffer,
+    options: OpenRouterTranscriptionOptions,
+    openRouterApiKey?: string,
+  ): Promise<OpenRouterTranscriptionResult> {
+    return transcribeOpenRouter(audioBuffer, options, openRouterApiKey);
+  }
+
+  static async speak(
+    text: string,
+    options: OpenAISpeechOptions & { provider?: "openai" },
+    openAIApiKey?: string,
+  ): Promise<SpeechResult> {
+    return speakOpenAI(text, options, openAIApiKey);
+  }
+
+  static async speakViaOpenRouter(
+    text: string,
+    options: OpenRouterSpeechOptions,
+    openRouterApiKey?: string,
+  ): Promise<SpeechResult> {
+    return speakOpenRouter(text, options, openRouterApiKey);
+  }
+}

package/src/audio/openrouter-client.ts

@@ -0,0 +1,4 @@
+export {
+  OPENROUTER_API_BASE,
+  createOpenRouterOpenAIClient,
+} from "../utils/openrouter-client";

package/src/audio/speech-openai.ts

@@ -0,0 +1,79 @@
+import OpenAI from "openai";
+import type { OpenAISpeechOptions, SpeechResult } from "./types";
+
+const CONTENT_TYPE_BY_FORMAT: Record<string, string> = {
+  mp3: "audio/mpeg",
+  opus: "audio/opus",
+  aac: "audio/aac",
+  flac: "audio/flac",
+  wav: "audio/wav",
+  pcm: "audio/pcm",
+};
+
+function validateSpeechOptions(options: OpenAISpeechOptions): void {
+  const model = options.model ?? "gpt-4o-mini-tts";
+  if (
+    options.instructions &&
+    model !== "gpt-4o-mini-tts" &&
+    model !== "gpt-4o-mini-tts-2025-12-15"
+  ) {
+    throw new Error(
+      "[@luanpoppe/ai] instructions em TTS só é suportado por modelos gpt-4o-mini-tts.",
+    );
+  }
+}
+
+function createOpenAIClient(apiKey?: string): OpenAI {
+  if (!apiKey) {
+    throw new Error("OpenAI API key is not passed in the speech parameters");
+  }
+  return new OpenAI({ apiKey });
+}
+
+export async function speakOpenAI(
+  text: string,
+  options: OpenAISpeechOptions,
+  openAIApiKey?: string,
+): Promise<SpeechResult> {
+  validateSpeechOptions(options);
+  const openai = createOpenAIClient(openAIApiKey);
+  const responseFormat = options.responseFormat ?? "mp3";
+  const model = options.model ?? "gpt-4o-mini-tts";
+
+  const response = await openai.audio.speech.create({
+    model,
+    voice: options.voice as OpenAI.Audio.SpeechCreateParams["voice"],
+    input: text,
+    response_format: responseFormat,
+    ...(options.speed !== undefined ? { speed: options.speed } : {}),
+    ...(options.instructions ? { instructions: options.instructions } : {}),
+  });
+
+  const arrayBuffer = await response.arrayBuffer();
+  return {
+    audio: Buffer.from(arrayBuffer),
+    contentType: CONTENT_TYPE_BY_FORMAT[responseFormat] ?? "application/octet-stream",
+  };
+}
+
+export async function speakOpenAIStream(
+  text: string,
+  options: OpenAISpeechOptions,
+  openAIApiKey?: string,
+): Promise<ReadableStream<Uint8Array>> {
+  validateSpeechOptions(options);
+  const openai = createOpenAIClient(openAIApiKey);
+  const responseFormat = options.responseFormat ?? "mp3";
+  const model = options.model ?? "gpt-4o-mini-tts";
+
+  const stream = await openai.audio.speech.create({
+    model,
+    voice: options.voice as OpenAI.Audio.SpeechCreateParams["voice"],
+    input: text,
+    response_format: responseFormat,
+    ...(options.speed !== undefined ? { speed: options.speed } : {}),
+    ...(options.instructions ? { instructions: options.instructions } : {}),
+  });
+
+  return stream.body as ReadableStream<Uint8Array>;
+}

package/src/audio/speech-openrouter.ts

@@ -0,0 +1,121 @@
+import type { OpenRouterSpeechOptions, SpeechResult } from "./types";
+import { createOpenRouterOpenAIClient } from "./openrouter-client";
+import { resolveOpenRouterProvider } from "../langchain/models";
+
+const CONTENT_TYPE_BY_FORMAT: Record<string, string> = {
+  mp3: "audio/mpeg",
+  pcm: "audio/pcm",
+};
+
+function isGeminiTtsModel(model: string): boolean {
+  return /gemini/i.test(model) && /tts/i.test(model);
+}
+
+function resolveOpenRouterSpeechFormat(
+  model: string,
+  responseFormat?: OpenRouterSpeechOptions["responseFormat"],
+): "mp3" | "pcm" {
+  if (responseFormat) return responseFormat;
+  // Gemini TTS no OpenRouter aceita apenas pcm (ver docs OR /audio/speech)
+  if (isGeminiTtsModel(model)) return "pcm";
+  return "mp3";
+}
+
+function validateOpenRouterSpeechOptions(options: OpenRouterSpeechOptions): void {
+  if (!options.model) {
+    throw new Error("[@luanpoppe/ai] model é obrigatório para speakOpenRouter.");
+  }
+  if (!options.voice) {
+    throw new Error("[@luanpoppe/ai] voice é obrigatório para speakOpenRouter.");
+  }
+}
+
+export async function speakOpenRouter(
+  text: string,
+  options: OpenRouterSpeechOptions,
+  openRouterApiKey?: string,
+): Promise<SpeechResult> {
+  validateOpenRouterSpeechOptions(options);
+  if (!openRouterApiKey) {
+    throw new Error(
+      "OpenRouter API key is not passed in the speech parameters",
+    );
+  }
+
+  const client = createOpenRouterOpenAIClient(openRouterApiKey);
+  const responseFormat = resolveOpenRouterSpeechFormat(
+    options.model,
+    options.responseFormat,
+  );
+  const provider = resolveOpenRouterProvider(options.model, {
+    openRouterProvider: options.openRouterProvider,
+    openRouterAllowAllProviders: options.openRouterAllowAllProviders,
+  });
+
+  const body: Record<string, unknown> = {
+    model: options.model,
+    input: text,
+    voice: options.voice,
+    response_format: responseFormat,
+  };
+
+  if (options.speed !== undefined) body.speed = options.speed;
+  if (provider || options.instructions) {
+    const mergedProvider: Record<string, unknown> = { ...(provider ?? {}) };
+    if (options.instructions) {
+      mergedProvider.options = {
+        openai: { instructions: options.instructions },
+      };
+    }
+    body.provider = mergedProvider;
+  }
+
+  const response = await client.audio.speech.create(
+    body as unknown as Parameters<typeof client.audio.speech.create>[0],
+  );
+
+  const arrayBuffer = await response.arrayBuffer();
+  return {
+    audio: Buffer.from(arrayBuffer),
+    contentType: CONTENT_TYPE_BY_FORMAT[responseFormat] ?? "application/octet-stream",
+  };
+}
+
+export async function speakOpenRouterStream(
+  text: string,
+  options: OpenRouterSpeechOptions,
+  openRouterApiKey?: string,
+): Promise<ReadableStream<Uint8Array>> {
+  validateOpenRouterSpeechOptions(options);
+  if (!openRouterApiKey) {
+    throw new Error(
+      "OpenRouter API key is not passed in the speech parameters",
+    );
+  }
+
+  const client = createOpenRouterOpenAIClient(openRouterApiKey);
+  const responseFormat = resolveOpenRouterSpeechFormat(
+    options.model,
+    options.responseFormat,
+  );
+  const provider = resolveOpenRouterProvider(options.model, {
+    openRouterProvider: options.openRouterProvider,
+    openRouterAllowAllProviders: options.openRouterAllowAllProviders,
+  });
+
+  const body: Record<string, unknown> = {
+    model: options.model,
+    input: text,
+    voice: options.voice,
+    response_format: responseFormat,
+  };
+
+  if (options.speed !== undefined) body.speed = options.speed;
+  if (provider) body.provider = provider;
+
+  const stream = await client.audio.speech.create(
+    body as unknown as Parameters<typeof client.audio.speech.create>[0],
+  );
+
+  return stream.body as ReadableStream<Uint8Array>;
+}