@luanpoppe/ai 1.1.5 → 1.1.6

package/src/audio/index.ts

@@ -0,0 +1,101 @@
+import type { AudioBuffer } from "../@types/audio";
+import type {
+  GeminiTranscriptionOptions,
+  OpenAITranscriptionDetailed,
+  OpenAITranscriptionOptions,
+  OpenRouterSpeechOptions,
+  OpenRouterTranscriptionOptions,
+  OpenRouterTranscriptionResult,
+  OpenAISpeechOptions,
+  SpeechResult,
+} from "./types";
+import {
+  transcribeDetailedOpenAI,
+  transcribeDiarizedOpenAI,
+  transcribeFileDetailedOpenAI,
+  transcribeFileWithWhisperOpenAI,
+  transcribeToSrtOpenAI,
+  transcribeToVttOpenAI,
+  transcribeWithWhisperOpenAI,
+  translateOpenAI,
+} from "./transcription-openai";
+import { transcribeOpenRouter } from "./transcription-openrouter";
+import { speakOpenAI, speakOpenAIStream } from "./speech-openai";
+import { speakOpenRouter, speakOpenRouterStream } from "./speech-openrouter";
+import { transcribeWithGeminiPrompt } from "./gemini-transcription";
+
+export type {
+  WhisperModel,
+  OpenAITranscriptionOptions,
+  OpenAITranscriptionResponseFormat,
+  OpenAITranscriptionDetailed,
+  OpenRouterTranscriptionOptions,
+  OpenRouterTranscriptionResult,
+  OpenAISpeechOptions,
+  OpenAISpeechModel,
+  OpenRouterSpeechOptions,
+  SpeechResult,
+  TranscriptionWord,
+  TranscriptionSegment,
+  DiarizedSegment,
+  GeminiTranscriptionOptions,
+} from "./types";
+
+export { ANTHROPIC_NO_NATIVE_AUDIO } from "./types";
+
+export class AIAudio {
+  static transcribeDetailedOpenAI = transcribeDetailedOpenAI;
+  static transcribeWithWhisper = transcribeWithWhisperOpenAI;
+  static transcribeFileWithWhisper = transcribeFileWithWhisperOpenAI;
+  static transcribeToSrtOpenAI = transcribeToSrtOpenAI;
+  static transcribeToVttOpenAI = transcribeToVttOpenAI;
+  static transcribeDiarizedOpenAI = transcribeDiarizedOpenAI;
+  static translateOpenAI = translateOpenAI;
+  static transcribeOpenRouter = transcribeOpenRouter;
+  static speakOpenAI = speakOpenAI;
+  static speakOpenAIStream = speakOpenAIStream;
+  static speakOpenRouter = speakOpenRouter;
+  static speakOpenRouterStream = speakOpenRouterStream;
+  static transcribeWithGeminiPrompt = transcribeWithGeminiPrompt;
+
+  /** @deprecated Use AIAudio.transcribeWithWhisper */
+  static transcribeWithWhisperLegacy(
+    audioBuffer: AudioBuffer,
+    options?: OpenAITranscriptionOptions,
+    openAIApiKey?: string,
+  ): Promise<string> {
+    return transcribeWithWhisperOpenAI(audioBuffer, options, openAIApiKey);
+  }
+
+  static async transcribeFileDetailedOpenAI(
+    filePath: string,
+    options?: OpenAITranscriptionOptions,
+    openAIApiKey?: string,
+  ): Promise<OpenAITranscriptionDetailed> {
+    return transcribeFileDetailedOpenAI(filePath, options, openAIApiKey);
+  }
+
+  static async transcribeDetailedOpenRouter(
+    audioBuffer: AudioBuffer,
+    options: OpenRouterTranscriptionOptions,
+    openRouterApiKey?: string,
+  ): Promise<OpenRouterTranscriptionResult> {
+    return transcribeOpenRouter(audioBuffer, options, openRouterApiKey);
+  }
+
+  static async speak(
+    text: string,
+    options: OpenAISpeechOptions & { provider?: "openai" },
+    openAIApiKey?: string,
+  ): Promise<SpeechResult> {
+    return speakOpenAI(text, options, openAIApiKey);
+  }
+
+  static async speakViaOpenRouter(
+    text: string,
+    options: OpenRouterSpeechOptions,
+    openRouterApiKey?: string,
+  ): Promise<SpeechResult> {
+    return speakOpenRouter(text, options, openRouterApiKey);
+  }
+}

package/src/audio/openrouter-client.ts

@@ -0,0 +1,4 @@
+export {
+  OPENROUTER_API_BASE,
+  createOpenRouterOpenAIClient,
+} from "../utils/openrouter-client";

package/src/audio/speech-openai.ts

@@ -0,0 +1,79 @@
+import OpenAI from "openai";
+import type { OpenAISpeechOptions, SpeechResult } from "./types";
+
+const CONTENT_TYPE_BY_FORMAT: Record<string, string> = {
+  mp3: "audio/mpeg",
+  opus: "audio/opus",
+  aac: "audio/aac",
+  flac: "audio/flac",
+  wav: "audio/wav",
+  pcm: "audio/pcm",
+};
+
+function validateSpeechOptions(options: OpenAISpeechOptions): void {
+  const model = options.model ?? "gpt-4o-mini-tts";
+  if (
+    options.instructions &&
+    model !== "gpt-4o-mini-tts" &&
+    model !== "gpt-4o-mini-tts-2025-12-15"
+  ) {
+    throw new Error(
+      "[@luanpoppe/ai] instructions em TTS só é suportado por modelos gpt-4o-mini-tts.",
+    );
+  }
+}
+
+function createOpenAIClient(apiKey?: string): OpenAI {
+  if (!apiKey) {
+    throw new Error("OpenAI API key is not passed in the speech parameters");
+  }
+  return new OpenAI({ apiKey });
+}
+
+export async function speakOpenAI(
+  text: string,
+  options: OpenAISpeechOptions,
+  openAIApiKey?: string,
+): Promise<SpeechResult> {
+  validateSpeechOptions(options);
+  const openai = createOpenAIClient(openAIApiKey);
+  const responseFormat = options.responseFormat ?? "mp3";
+  const model = options.model ?? "gpt-4o-mini-tts";
+
+  const response = await openai.audio.speech.create({
+    model,
+    voice: options.voice as OpenAI.Audio.SpeechCreateParams["voice"],
+    input: text,
+    response_format: responseFormat,
+    ...(options.speed !== undefined ? { speed: options.speed } : {}),
+    ...(options.instructions ? { instructions: options.instructions } : {}),
+  });
+
+  const arrayBuffer = await response.arrayBuffer();
+  return {
+    audio: Buffer.from(arrayBuffer),
+    contentType: CONTENT_TYPE_BY_FORMAT[responseFormat] ?? "application/octet-stream",
+  };
+}
+
+export async function speakOpenAIStream(
+  text: string,
+  options: OpenAISpeechOptions,
+  openAIApiKey?: string,
+): Promise<ReadableStream<Uint8Array>> {
+  validateSpeechOptions(options);
+  const openai = createOpenAIClient(openAIApiKey);
+  const responseFormat = options.responseFormat ?? "mp3";
+  const model = options.model ?? "gpt-4o-mini-tts";
+
+  const stream = await openai.audio.speech.create({
+    model,
+    voice: options.voice as OpenAI.Audio.SpeechCreateParams["voice"],
+    input: text,
+    response_format: responseFormat,
+    ...(options.speed !== undefined ? { speed: options.speed } : {}),
+    ...(options.instructions ? { instructions: options.instructions } : {}),
+  });
+
+  return stream.body as ReadableStream<Uint8Array>;
+}

package/src/audio/speech-openrouter.ts

@@ -0,0 +1,121 @@
+import type { OpenRouterSpeechOptions, SpeechResult } from "./types";
+import { createOpenRouterOpenAIClient } from "./openrouter-client";
+import { resolveOpenRouterProvider } from "../langchain/models";
+
+const CONTENT_TYPE_BY_FORMAT: Record<string, string> = {
+  mp3: "audio/mpeg",
+  pcm: "audio/pcm",
+};
+
+function isGeminiTtsModel(model: string): boolean {
+  return /gemini/i.test(model) && /tts/i.test(model);
+}
+
+function resolveOpenRouterSpeechFormat(
+  model: string,
+  responseFormat?: OpenRouterSpeechOptions["responseFormat"],
+): "mp3" | "pcm" {
+  if (responseFormat) return responseFormat;
+  // Gemini TTS no OpenRouter aceita apenas pcm (ver docs OR /audio/speech)
+  if (isGeminiTtsModel(model)) return "pcm";
+  return "mp3";
+}
+
+function validateOpenRouterSpeechOptions(options: OpenRouterSpeechOptions): void {
+  if (!options.model) {
+    throw new Error("[@luanpoppe/ai] model é obrigatório para speakOpenRouter.");
+  }
+  if (!options.voice) {
+    throw new Error("[@luanpoppe/ai] voice é obrigatório para speakOpenRouter.");
+  }
+}
+
+export async function speakOpenRouter(
+  text: string,
+  options: OpenRouterSpeechOptions,
+  openRouterApiKey?: string,
+): Promise<SpeechResult> {
+  validateOpenRouterSpeechOptions(options);
+  if (!openRouterApiKey) {
+    throw new Error(
+      "OpenRouter API key is not passed in the speech parameters",
+    );
+  }
+
+  const client = createOpenRouterOpenAIClient(openRouterApiKey);
+  const responseFormat = resolveOpenRouterSpeechFormat(
+    options.model,
+    options.responseFormat,
+  );
+  const provider = resolveOpenRouterProvider(options.model, {
+    openRouterProvider: options.openRouterProvider,
+    openRouterAllowAllProviders: options.openRouterAllowAllProviders,
+  });
+
+  const body: Record<string, unknown> = {
+    model: options.model,
+    input: text,
+    voice: options.voice,
+    response_format: responseFormat,
+  };
+
+  if (options.speed !== undefined) body.speed = options.speed;
+  if (provider || options.instructions) {
+    const mergedProvider: Record<string, unknown> = { ...(provider ?? {}) };
+    if (options.instructions) {
+      mergedProvider.options = {
+        openai: { instructions: options.instructions },
+      };
+    }
+    body.provider = mergedProvider;
+  }
+
+  const response = await client.audio.speech.create(
+    body as unknown as Parameters<typeof client.audio.speech.create>[0],
+  );
+
+  const arrayBuffer = await response.arrayBuffer();
+  return {
+    audio: Buffer.from(arrayBuffer),
+    contentType: CONTENT_TYPE_BY_FORMAT[responseFormat] ?? "application/octet-stream",
+  };
+}
+
+export async function speakOpenRouterStream(
+  text: string,
+  options: OpenRouterSpeechOptions,
+  openRouterApiKey?: string,
+): Promise<ReadableStream<Uint8Array>> {
+  validateOpenRouterSpeechOptions(options);
+  if (!openRouterApiKey) {
+    throw new Error(
+      "OpenRouter API key is not passed in the speech parameters",
+    );
+  }
+
+  const client = createOpenRouterOpenAIClient(openRouterApiKey);
+  const responseFormat = resolveOpenRouterSpeechFormat(
+    options.model,
+    options.responseFormat,
+  );
+  const provider = resolveOpenRouterProvider(options.model, {
+    openRouterProvider: options.openRouterProvider,
+    openRouterAllowAllProviders: options.openRouterAllowAllProviders,
+  });
+
+  const body: Record<string, unknown> = {
+    model: options.model,
+    input: text,
+    voice: options.voice,
+    response_format: responseFormat,
+  };
+
+  if (options.speed !== undefined) body.speed = options.speed;
+  if (provider) body.provider = provider;
+
+  const stream = await client.audio.speech.create(
+    body as unknown as Parameters<typeof client.audio.speech.create>[0],
+  );
+
+  return stream.body as ReadableStream<Uint8Array>;
+}

package/src/audio/transcription-openai.ts

@@ -0,0 +1,259 @@
+import * as fs from "fs";
+import * as path from "path";
+import OpenAI from "openai";
+import { toFile } from "openai";
+import type { AudioBuffer } from "../@types/audio";
+import type {
+  OpenAITranscriptionDetailed,
+  OpenAITranscriptionOptions,
+  TranscriptionSegment,
+  TranscriptionWord,
+  DiarizedSegment,
+} from "./types";
+import { getAudioExtension, toAudioBuffer } from "./utils";
+
+function createOpenAIClient(apiKey?: string): OpenAI {
+  if (apiKey) {
+    return new OpenAI({ apiKey });
+  }
+  return new OpenAI();
+}
+
+async function createAudioFile(
+  audioBuffer: AudioBuffer,
+  format?: OpenAITranscriptionOptions["format"],
+) {
+  const buffer = toAudioBuffer(audioBuffer);
+  const extension = getAudioExtension(format);
+  const fileName = `whisper-${Date.now()}.${extension}`;
+  return toFile(buffer, fileName);
+}
+
+function validateTranscriptionOptions(options: OpenAITranscriptionOptions): void {
+  const model = options.model ?? "whisper-1";
+  const format = options.responseFormat ?? "text";
+
+  if (
+    options.timestampGranularities?.length &&
+    (model !== "whisper-1" || format !== "verbose_json")
+  ) {
+    throw new Error(
+      "[@luanpoppe/ai] timestampGranularities só é suportado com model whisper-1 e responseFormat verbose_json.",
+    );
+  }
+
+  if (model === "gpt-4o-transcribe-diarize" && format === "diarized_json") {
+    if (!options.chunkingStrategy) {
+      options.chunkingStrategy = "auto";
+    }
+  }
+}
+
+function mapVerboseResponse(
+  response: Record<string, unknown>,
+): OpenAITranscriptionDetailed {
+  const words = Array.isArray(response.words)
+    ? (response.words as TranscriptionWord[])
+    : undefined;
+  const segments = Array.isArray(response.segments)
+    ? (response.segments as TranscriptionSegment[])
+    : undefined;
+
+  const detailed: OpenAITranscriptionDetailed = {
+    text: String(response.text ?? ""),
+  };
+  if (typeof response.language === "string") {
+    detailed.language = response.language;
+  }
+  if (typeof response.duration === "number") {
+    detailed.duration = response.duration;
+  }
+  if (words) detailed.words = words;
+  if (segments) detailed.segments = segments;
+  return detailed;
+}
+
+function mapDiarizedResponse(
+  response: Record<string, unknown>,
+): OpenAITranscriptionDetailed {
+  const segments = Array.isArray(response.segments)
+    ? (response.segments as Array<Record<string, unknown>>)
+    : [];
+
+  const speakers: DiarizedSegment[] = segments.map((seg) => ({
+    speaker: String(seg.speaker ?? "unknown"),
+    start: Number(seg.start ?? 0),
+    end: Number(seg.end ?? 0),
+    text: String(seg.text ?? ""),
+  }));
+
+  return {
+    text: String(response.text ?? speakers.map((s) => s.text).join(" ")),
+    speakers,
+    segments: speakers.map((s, id) => ({
+      id,
+      start: s.start,
+      end: s.end,
+      text: s.text,
+    })),
+  };
+}
+
+function mapTranscriptionResponse(
+  response: unknown,
+  responseFormat: OpenAITranscriptionOptions["responseFormat"],
+): OpenAITranscriptionDetailed {
+  if (typeof response === "string") {
+    return { text: response };
+  }
+
+  const obj = response as Record<string, unknown>;
+  if (responseFormat === "diarized_json") {
+    return mapDiarizedResponse(obj);
+  }
+  if (responseFormat === "verbose_json") {
+    return mapVerboseResponse(obj);
+  }
+  return { text: String(obj.text ?? "") };
+}
+
+export async function transcribeDetailedOpenAI(
+  audioBuffer: AudioBuffer,
+  options: OpenAITranscriptionOptions = {},
+  openAIApiKey?: string,
+): Promise<OpenAITranscriptionDetailed> {
+  validateTranscriptionOptions(options);
+
+  const openai = createOpenAIClient(openAIApiKey);
+  const file = await createAudioFile(audioBuffer, options.format);
+  const responseFormat = options.responseFormat ?? "verbose_json";
+  const model = options.model ?? "whisper-1";
+
+  const params: OpenAI.Audio.TranscriptionCreateParams = {
+    file,
+    model,
+    response_format: responseFormat as OpenAI.Audio.TranscriptionCreateParams["response_format"],
+  };
+
+  if (options.languageIn2Digits) params.language = options.languageIn2Digits;
+  if (options.prompt) params.prompt = options.prompt;
+  if (options.temperature !== undefined) params.temperature = options.temperature;
+  if (options.timestampGranularities) {
+    params.timestamp_granularities = options.timestampGranularities;
+  }
+  if (options.chunkingStrategy) {
+    (params as OpenAI.Audio.TranscriptionCreateParams & {
+      chunking_strategy?: unknown;
+    }).chunking_strategy = options.chunkingStrategy;
+  }
+
+  const response = await openai.audio.transcriptions.create(params);
+  return mapTranscriptionResponse(response, responseFormat);
+}
+
+export async function transcribeWithWhisperOpenAI(
+  audioBuffer: AudioBuffer,
+  options: OpenAITranscriptionOptions = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const detailed = await transcribeDetailedOpenAI(
+    audioBuffer,
+    { ...options, responseFormat: options.responseFormat ?? "text" },
+    openAIApiKey,
+  );
+  return detailed.text;
+}
+
+export async function transcribeToSrtOpenAI(
+  audioBuffer: AudioBuffer,
+  options: Omit<OpenAITranscriptionOptions, "responseFormat"> = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const detailed = await transcribeDetailedOpenAI(
+    audioBuffer,
+    { ...options, model: options.model ?? "whisper-1", responseFormat: "srt" },
+    openAIApiKey,
+  );
+  return detailed.text;
+}
+
+export async function transcribeToVttOpenAI(
+  audioBuffer: AudioBuffer,
+  options: Omit<OpenAITranscriptionOptions, "responseFormat"> = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const detailed = await transcribeDetailedOpenAI(
+    audioBuffer,
+    { ...options, model: options.model ?? "whisper-1", responseFormat: "vtt" },
+    openAIApiKey,
+  );
+  return detailed.text;
+}
+
+export async function transcribeDiarizedOpenAI(
+  audioBuffer: AudioBuffer,
+  options: Omit<OpenAITranscriptionOptions, "model" | "responseFormat"> = {},
+  openAIApiKey?: string,
+): Promise<OpenAITranscriptionDetailed> {
+  return transcribeDetailedOpenAI(
+    audioBuffer,
+    {
+      ...options,
+      model: "gpt-4o-transcribe-diarize",
+      responseFormat: "diarized_json",
+      chunkingStrategy: options.chunkingStrategy ?? "auto",
+    },
+    openAIApiKey,
+  );
+}
+
+export async function translateOpenAI(
+  audioBuffer: AudioBuffer,
+  options: Omit<OpenAITranscriptionOptions, "responseFormat"> = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const openai = createOpenAIClient(openAIApiKey);
+  const file = await createAudioFile(audioBuffer, options.format);
+
+  const response = await openai.audio.translations.create({
+    file,
+    model: "whisper-1",
+    ...(options.prompt ? { prompt: options.prompt } : {}),
+    ...(options.temperature !== undefined
+      ? { temperature: options.temperature }
+      : {}),
+  });
+
+  return typeof response === "string" ? response : response.text;
+}
+
+export async function transcribeFileDetailedOpenAI(
+  filePath: string,
+  options: OpenAITranscriptionOptions = {},
+  openAIApiKey?: string,
+): Promise<OpenAITranscriptionDetailed> {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`Arquivo não encontrado: ${filePath}`);
+  }
+  const audioBuffer = fs.readFileSync(filePath);
+  const format =
+    options.format ?? (path.extname(filePath).replace(/^\./, "") || "mp3");
+  return transcribeDetailedOpenAI(
+    audioBuffer,
+    { ...options, format },
+    openAIApiKey,
+  );
+}
+
+export async function transcribeFileWithWhisperOpenAI(
+  filePath: string,
+  options: OpenAITranscriptionOptions = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const detailed = await transcribeFileDetailedOpenAI(
+    filePath,
+    options,
+    openAIApiKey,
+  );
+  return detailed.text;
+}

package/src/audio/transcription-openrouter.ts

@@ -0,0 +1,59 @@
+import type { AudioBuffer } from "../@types/audio";
+import { resolveOpenRouterProvider } from "../langchain/models";
+import type {
+  OpenRouterTranscriptionOptions,
+  OpenRouterTranscriptionResult,
+} from "./types";
+import { bufferToBase64, getAudioExtension } from "./utils";
+
+const OPENROUTER_TRANSCRIPTIONS_URL =
+  "https://openrouter.ai/api/v1/audio/transcriptions";
+
+export async function transcribeOpenRouter(
+  audioBuffer: AudioBuffer,
+  options: OpenRouterTranscriptionOptions,
+  openRouterApiKey?: string,
+): Promise<OpenRouterTranscriptionResult> {
+  if (!openRouterApiKey) {
+    throw new Error(
+      "OpenRouter API key is not passed in the transcription parameters",
+    );
+  }
+
+  const format = getAudioExtension(options.format);
+  const provider = resolveOpenRouterProvider(options.model, {
+    openRouterProvider: options.openRouterProvider,
+    openRouterAllowAllProviders: options.openRouterAllowAllProviders,
+  });
+
+  const body: Record<string, unknown> = {
+    model: options.model,
+    input_audio: {
+      data: bufferToBase64(audioBuffer),
+      format,
+    },
+  };
+
+  if (options.language) body.language = options.language;
+  if (options.temperature !== undefined) body.temperature = options.temperature;
+  if (provider) body.provider = provider;
+
+  const response = await fetch(OPENROUTER_TRANSCRIPTIONS_URL, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${openRouterApiKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(body),
+  });
+
+  if (!response.ok) {
+    const errText = await response.text();
+    throw new Error(
+      `[@luanpoppe/ai] OpenRouter STT falhou (${response.status}): ${errText}`,
+    );
+  }
+
+  const data = (await response.json()) as OpenRouterTranscriptionResult;
+  return data;
+}