@luanpoppe/ai 1.1.4 → 1.1.6

package/src/audio/transcription-openai.ts

@@ -0,0 +1,259 @@
+import * as fs from "fs";
+import * as path from "path";
+import OpenAI from "openai";
+import { toFile } from "openai";
+import type { AudioBuffer } from "../@types/audio";
+import type {
+  OpenAITranscriptionDetailed,
+  OpenAITranscriptionOptions,
+  TranscriptionSegment,
+  TranscriptionWord,
+  DiarizedSegment,
+} from "./types";
+import { getAudioExtension, toAudioBuffer } from "./utils";
+
+function createOpenAIClient(apiKey?: string): OpenAI {
+  if (apiKey) {
+    return new OpenAI({ apiKey });
+  }
+  return new OpenAI();
+}
+
+async function createAudioFile(
+  audioBuffer: AudioBuffer,
+  format?: OpenAITranscriptionOptions["format"],
+) {
+  const buffer = toAudioBuffer(audioBuffer);
+  const extension = getAudioExtension(format);
+  const fileName = `whisper-${Date.now()}.${extension}`;
+  return toFile(buffer, fileName);
+}
+
+function validateTranscriptionOptions(options: OpenAITranscriptionOptions): void {
+  const model = options.model ?? "whisper-1";
+  const format = options.responseFormat ?? "text";
+
+  if (
+    options.timestampGranularities?.length &&
+    (model !== "whisper-1" || format !== "verbose_json")
+  ) {
+    throw new Error(
+      "[@luanpoppe/ai] timestampGranularities só é suportado com model whisper-1 e responseFormat verbose_json.",
+    );
+  }
+
+  if (model === "gpt-4o-transcribe-diarize" && format === "diarized_json") {
+    if (!options.chunkingStrategy) {
+      options.chunkingStrategy = "auto";
+    }
+  }
+}
+
+function mapVerboseResponse(
+  response: Record<string, unknown>,
+): OpenAITranscriptionDetailed {
+  const words = Array.isArray(response.words)
+    ? (response.words as TranscriptionWord[])
+    : undefined;
+  const segments = Array.isArray(response.segments)
+    ? (response.segments as TranscriptionSegment[])
+    : undefined;
+
+  const detailed: OpenAITranscriptionDetailed = {
+    text: String(response.text ?? ""),
+  };
+  if (typeof response.language === "string") {
+    detailed.language = response.language;
+  }
+  if (typeof response.duration === "number") {
+    detailed.duration = response.duration;
+  }
+  if (words) detailed.words = words;
+  if (segments) detailed.segments = segments;
+  return detailed;
+}
+
+function mapDiarizedResponse(
+  response: Record<string, unknown>,
+): OpenAITranscriptionDetailed {
+  const segments = Array.isArray(response.segments)
+    ? (response.segments as Array<Record<string, unknown>>)
+    : [];
+
+  const speakers: DiarizedSegment[] = segments.map((seg) => ({
+    speaker: String(seg.speaker ?? "unknown"),
+    start: Number(seg.start ?? 0),
+    end: Number(seg.end ?? 0),
+    text: String(seg.text ?? ""),
+  }));
+
+  return {
+    text: String(response.text ?? speakers.map((s) => s.text).join(" ")),
+    speakers,
+    segments: speakers.map((s, id) => ({
+      id,
+      start: s.start,
+      end: s.end,
+      text: s.text,
+    })),
+  };
+}
+
+function mapTranscriptionResponse(
+  response: unknown,
+  responseFormat: OpenAITranscriptionOptions["responseFormat"],
+): OpenAITranscriptionDetailed {
+  if (typeof response === "string") {
+    return { text: response };
+  }
+
+  const obj = response as Record<string, unknown>;
+  if (responseFormat === "diarized_json") {
+    return mapDiarizedResponse(obj);
+  }
+  if (responseFormat === "verbose_json") {
+    return mapVerboseResponse(obj);
+  }
+  return { text: String(obj.text ?? "") };
+}
+
+export async function transcribeDetailedOpenAI(
+  audioBuffer: AudioBuffer,
+  options: OpenAITranscriptionOptions = {},
+  openAIApiKey?: string,
+): Promise<OpenAITranscriptionDetailed> {
+  validateTranscriptionOptions(options);
+
+  const openai = createOpenAIClient(openAIApiKey);
+  const file = await createAudioFile(audioBuffer, options.format);
+  const responseFormat = options.responseFormat ?? "verbose_json";
+  const model = options.model ?? "whisper-1";
+
+  const params: OpenAI.Audio.TranscriptionCreateParams = {
+    file,
+    model,
+    response_format: responseFormat as OpenAI.Audio.TranscriptionCreateParams["response_format"],
+  };
+
+  if (options.languageIn2Digits) params.language = options.languageIn2Digits;
+  if (options.prompt) params.prompt = options.prompt;
+  if (options.temperature !== undefined) params.temperature = options.temperature;
+  if (options.timestampGranularities) {
+    params.timestamp_granularities = options.timestampGranularities;
+  }
+  if (options.chunkingStrategy) {
+    (params as OpenAI.Audio.TranscriptionCreateParams & {
+      chunking_strategy?: unknown;
+    }).chunking_strategy = options.chunkingStrategy;
+  }
+
+  const response = await openai.audio.transcriptions.create(params);
+  return mapTranscriptionResponse(response, responseFormat);
+}
+
+export async function transcribeWithWhisperOpenAI(
+  audioBuffer: AudioBuffer,
+  options: OpenAITranscriptionOptions = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const detailed = await transcribeDetailedOpenAI(
+    audioBuffer,
+    { ...options, responseFormat: options.responseFormat ?? "text" },
+    openAIApiKey,
+  );
+  return detailed.text;
+}
+
+export async function transcribeToSrtOpenAI(
+  audioBuffer: AudioBuffer,
+  options: Omit<OpenAITranscriptionOptions, "responseFormat"> = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const detailed = await transcribeDetailedOpenAI(
+    audioBuffer,
+    { ...options, model: options.model ?? "whisper-1", responseFormat: "srt" },
+    openAIApiKey,
+  );
+  return detailed.text;
+}
+
+export async function transcribeToVttOpenAI(
+  audioBuffer: AudioBuffer,
+  options: Omit<OpenAITranscriptionOptions, "responseFormat"> = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const detailed = await transcribeDetailedOpenAI(
+    audioBuffer,
+    { ...options, model: options.model ?? "whisper-1", responseFormat: "vtt" },
+    openAIApiKey,
+  );
+  return detailed.text;
+}
+
+export async function transcribeDiarizedOpenAI(
+  audioBuffer: AudioBuffer,
+  options: Omit<OpenAITranscriptionOptions, "model" | "responseFormat"> = {},
+  openAIApiKey?: string,
+): Promise<OpenAITranscriptionDetailed> {
+  return transcribeDetailedOpenAI(
+    audioBuffer,
+    {
+      ...options,
+      model: "gpt-4o-transcribe-diarize",
+      responseFormat: "diarized_json",
+      chunkingStrategy: options.chunkingStrategy ?? "auto",
+    },
+    openAIApiKey,
+  );
+}
+
+export async function translateOpenAI(
+  audioBuffer: AudioBuffer,
+  options: Omit<OpenAITranscriptionOptions, "responseFormat"> = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const openai = createOpenAIClient(openAIApiKey);
+  const file = await createAudioFile(audioBuffer, options.format);
+
+  const response = await openai.audio.translations.create({
+    file,
+    model: "whisper-1",
+    ...(options.prompt ? { prompt: options.prompt } : {}),
+    ...(options.temperature !== undefined
+      ? { temperature: options.temperature }
+      : {}),
+  });
+
+  return typeof response === "string" ? response : response.text;
+}
+
+export async function transcribeFileDetailedOpenAI(
+  filePath: string,
+  options: OpenAITranscriptionOptions = {},
+  openAIApiKey?: string,
+): Promise<OpenAITranscriptionDetailed> {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`Arquivo não encontrado: ${filePath}`);
+  }
+  const audioBuffer = fs.readFileSync(filePath);
+  const format =
+    options.format ?? (path.extname(filePath).replace(/^\./, "") || "mp3");
+  return transcribeDetailedOpenAI(
+    audioBuffer,
+    { ...options, format },
+    openAIApiKey,
+  );
+}
+
+export async function transcribeFileWithWhisperOpenAI(
+  filePath: string,
+  options: OpenAITranscriptionOptions = {},
+  openAIApiKey?: string,
+): Promise<string> {
+  const detailed = await transcribeFileDetailedOpenAI(
+    filePath,
+    options,
+    openAIApiKey,
+  );
+  return detailed.text;
+}

package/src/audio/transcription-openrouter.ts

@@ -0,0 +1,59 @@
+import type { AudioBuffer } from "../@types/audio";
+import { resolveOpenRouterProvider } from "../langchain/models";
+import type {
+  OpenRouterTranscriptionOptions,
+  OpenRouterTranscriptionResult,
+} from "./types";
+import { bufferToBase64, getAudioExtension } from "./utils";
+
+const OPENROUTER_TRANSCRIPTIONS_URL =
+  "https://openrouter.ai/api/v1/audio/transcriptions";
+
+export async function transcribeOpenRouter(
+  audioBuffer: AudioBuffer,
+  options: OpenRouterTranscriptionOptions,
+  openRouterApiKey?: string,
+): Promise<OpenRouterTranscriptionResult> {
+  if (!openRouterApiKey) {
+    throw new Error(
+      "OpenRouter API key is not passed in the transcription parameters",
+    );
+  }
+
+  const format = getAudioExtension(options.format);
+  const provider = resolveOpenRouterProvider(options.model, {
+    openRouterProvider: options.openRouterProvider,
+    openRouterAllowAllProviders: options.openRouterAllowAllProviders,
+  });
+
+  const body: Record<string, unknown> = {
+    model: options.model,
+    input_audio: {
+      data: bufferToBase64(audioBuffer),
+      format,
+    },
+  };
+
+  if (options.language) body.language = options.language;
+  if (options.temperature !== undefined) body.temperature = options.temperature;
+  if (provider) body.provider = provider;
+
+  const response = await fetch(OPENROUTER_TRANSCRIPTIONS_URL, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${openRouterApiKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(body),
+  });
+
+  if (!response.ok) {
+    const errText = await response.text();
+    throw new Error(
+      `[@luanpoppe/ai] OpenRouter STT falhou (${response.status}): ${errText}`,
+    );
+  }
+
+  const data = (await response.json()) as OpenRouterTranscriptionResult;
+  return data;
+}

package/src/audio/types.ts

@@ -0,0 +1,127 @@
+import type { AudioMimeType } from "../@types/audio";
+import type { OpenRouterProviderPreferences } from "../@types/openrouter-provider";
+
+export type WhisperModel =
+  | "whisper-1"
+  | "gpt-4o-transcribe"
+  | "gpt-4o-mini-transcribe"
+  | "gpt-4o-mini-transcribe-2025-12-15"
+  | "gpt-4o-transcribe-diarize";
+
+export type OpenAITranscriptionResponseFormat =
+  | "json"
+  | "text"
+  | "srt"
+  | "verbose_json"
+  | "vtt"
+  | "diarized_json";
+
+export type OpenAITranscriptionOptions = {
+  model?: WhisperModel;
+  languageIn2Digits?: string;
+  prompt?: string;
+  responseFormat?: OpenAITranscriptionResponseFormat;
+  temperature?: number;
+  timestampGranularities?: ("word" | "segment")[];
+  format?: string | AudioMimeType;
+  /** Obrigatório para `gpt-4o-transcribe-diarize` em áudio > 30s */
+  chunkingStrategy?: "auto" | { type: "server_vad" };
+};
+
+export type TranscriptionWord = {
+  word: string;
+  start: number;
+  end: number;
+};
+
+export type TranscriptionSegment = {
+  id?: number;
+  start: number;
+  end: number;
+  text: string;
+};
+
+export type DiarizedSegment = {
+  speaker: string;
+  start: number;
+  end: number;
+  text: string;
+};
+
+export type OpenAITranscriptionDetailed = {
+  text: string;
+  language?: string;
+  duration?: number;
+  words?: TranscriptionWord[];
+  segments?: TranscriptionSegment[];
+  speakers?: DiarizedSegment[];
+};
+
+export type OpenRouterTranscriptionOptions = {
+  model: string;
+  format?: string | AudioMimeType;
+  language?: string;
+  temperature?: number;
+  openRouterProvider?: OpenRouterProviderPreferences;
+  /** Quando true, não aplica default de provider (ex.: deepseek only) */
+  openRouterAllowAllProviders?: boolean;
+};
+
+export type OpenRouterTranscriptionResult = {
+  text: string;
+  usage?: {
+    seconds?: number;
+    cost?: number;
+    input_tokens?: number;
+    output_tokens?: number;
+    total_tokens?: number;
+  };
+};
+
+export type OpenAISpeechModel =
+  | "tts-1"
+  | "tts-1-hd"
+  | "gpt-4o-mini-tts"
+  | "gpt-4o-mini-tts-2025-12-15";
+
+export type OpenAISpeechResponseFormat =
+  | "mp3"
+  | "opus"
+  | "aac"
+  | "flac"
+  | "wav"
+  | "pcm";
+
+export type OpenAISpeechOptions = {
+  model?: OpenAISpeechModel;
+  voice: string;
+  responseFormat?: OpenAISpeechResponseFormat;
+  speed?: number;
+  /** Apenas modelos gpt-4o-mini-tts* */
+  instructions?: string;
+};
+
+export type OpenRouterSpeechOptions = {
+  model: string;
+  voice: string;
+  responseFormat?: "mp3" | "pcm";
+  speed?: number;
+  instructions?: string;
+  openRouterProvider?: OpenRouterProviderPreferences;
+  openRouterAllowAllProviders?: boolean;
+};
+
+export type SpeechResult = {
+  audio: Buffer;
+  contentType: string;
+};
+
+export type GeminiTranscriptionOptions = {
+  model?: string;
+  prompt?: string;
+  googleGeminiToken?: string;
+};
+
+/** A API pública da Anthropic não oferece STT/TTS nativo (apenas texto/imagem/documento). */
+export const ANTHROPIC_NO_NATIVE_AUDIO =
+  "A API da Anthropic não suporta áudio nativo. Use STT (OpenAI/OpenRouter) e depois AI.call com modelos anthropic." as const;

package/src/audio/utils.ts

@@ -0,0 +1,20 @@
+import type { AudioBuffer, AudioMimeType } from "../@types/audio";
+import { MIME_TO_EXTENSION } from "../@types/audio";
+
+export function getAudioExtension(format?: string | AudioMimeType): string {
+  if (!format) return "mp3";
+  if (format.startsWith("audio/")) {
+    return MIME_TO_EXTENSION[format as AudioMimeType] ?? "mp3";
+  }
+  return format.replace(/^\./, "");
+}
+
+export function toAudioBuffer(audioBuffer: AudioBuffer): Buffer {
+  if (audioBuffer instanceof Buffer) return audioBuffer;
+  if (audioBuffer instanceof ArrayBuffer) return Buffer.from(audioBuffer);
+  return Buffer.from(audioBuffer as Uint8Array);
+}
+
+export function bufferToBase64(audioBuffer: AudioBuffer): string {
+  return toAudioBuffer(audioBuffer).toString("base64");
+}

package/src/embeddings/embed-gemini.ts

@@ -0,0 +1,54 @@
+import { GoogleGenerativeAIEmbeddings } from "@langchain/google-genai";
+import type { EmbeddingResult, GeminiEmbeddingOptions } from "./types";
+import { normalizeInput } from "./utils";
+
+export async function embedGemini(
+  input: string | string[],
+  options: GeminiEmbeddingOptions,
+  googleGeminiToken?: string,
+): Promise<EmbeddingResult> {
+  if (!googleGeminiToken) {
+    throw new Error(
+      "googleGeminiToken é obrigatório para embedGemini.",
+    );
+  }
+
+  const texts = normalizeInput(input);
+  const modelParams: ConstructorParameters<typeof GoogleGenerativeAIEmbeddings>[0] =
+    {
+      apiKey: googleGeminiToken,
+      model: options.model,
+    };
+  if (options.taskType) {
+    modelParams.taskType = options.taskType as NonNullable<
+      typeof modelParams.taskType
+    >;
+  }
+
+  const embeddingsModel = new GoogleGenerativeAIEmbeddings(modelParams);
+
+  const embeddings =
+    texts.length === 1
+      ? [await embeddingsModel.embedQuery(texts[0]!)]
+      : await embeddingsModel.embedDocuments(texts);
+
+  const result: EmbeddingResult = {
+    embeddings,
+    model: options.model,
+  };
+
+  if (options.dimensions !== undefined && options.model === "gemini-embedding-001") {
+    result.embeddings = result.embeddings.map((vec) =>
+      truncateAndNormalizeL2(vec, options.dimensions!),
+    );
+  }
+
+  return result;
+}
+
+function truncateAndNormalizeL2(vector: number[], dimensions: number): number[] {
+  const truncated = vector.slice(0, dimensions);
+  const norm = Math.sqrt(truncated.reduce((sum, v) => sum + v * v, 0));
+  if (norm === 0) return truncated;
+  return truncated.map((v) => v / norm);
+}

package/src/embeddings/embed-openai.ts

@@ -0,0 +1,53 @@
+import OpenAI from "openai";
+import type { OpenAIEmbeddingOptions, EmbeddingResult } from "./types";
+import { normalizeInput } from "./utils";
+
+function createOpenAIClient(apiKey?: string): OpenAI {
+  if (!apiKey) {
+    throw new Error("OpenAI API key is not passed in the embedding parameters");
+  }
+  return new OpenAI({ apiKey });
+}
+
+function mapEmbeddingResponse(
+  response: OpenAI.CreateEmbeddingResponse,
+  model: string,
+): EmbeddingResult {
+  const embeddings = response.data
+    .sort((a, b) => a.index - b.index)
+    .map((item) => item.embedding as number[]);
+
+  const result: EmbeddingResult = {
+    embeddings,
+    model: response.model ?? model,
+  };
+
+  if (response.usage) {
+    result.usage = {
+      prompt_tokens: response.usage.prompt_tokens,
+      total_tokens: response.usage.total_tokens,
+    };
+  }
+
+  return result;
+}
+
+export async function embedOpenAI(
+  input: string | string[],
+  options: OpenAIEmbeddingOptions,
+  openAIApiKey?: string,
+): Promise<EmbeddingResult> {
+  const openai = createOpenAIClient(openAIApiKey);
+  const texts = normalizeInput(input);
+
+  const response = await openai.embeddings.create({
+    model: options.model,
+    input: texts.length === 1 ? texts[0]! : texts,
+    ...(options.dimensions !== undefined ? { dimensions: options.dimensions } : {}),
+    ...(options.encodingFormat
+      ? { encoding_format: options.encodingFormat }
+      : {}),
+  });
+
+  return mapEmbeddingResponse(response, options.model);
+}

package/src/embeddings/embed-openrouter.ts

@@ -0,0 +1,64 @@
+import type OpenAI from "openai";
+import { resolveOpenRouterProvider } from "../langchain/models";
+import { createOpenRouterOpenAIClient } from "../utils/openrouter-client";
+import type { EmbeddingResult, OpenRouterEmbeddingOptions } from "./types";
+import { normalizeInput, stripOpenRouterPrefix } from "./utils";
+
+function mapEmbeddingResponse(
+  response: OpenAI.CreateEmbeddingResponse,
+  model: string,
+): EmbeddingResult {
+  const embeddings = response.data
+    .sort((a, b) => a.index - b.index)
+    .map((item) => item.embedding as number[]);
+
+  const result: EmbeddingResult = {
+    embeddings,
+    model: response.model ?? model,
+  };
+
+  if (response.usage) {
+    result.usage = {
+      prompt_tokens: response.usage.prompt_tokens,
+      total_tokens: response.usage.total_tokens,
+    };
+  }
+
+  return result;
+}
+
+export async function embedOpenRouter(
+  input: string | string[],
+  options: OpenRouterEmbeddingOptions,
+  openRouterApiKey?: string,
+): Promise<EmbeddingResult> {
+  if (!openRouterApiKey) {
+    throw new Error(
+      "OpenRouter API key is not passed in the embedding parameters",
+    );
+  }
+
+  const openRouterModel = stripOpenRouterPrefix(options.model);
+  const client = createOpenRouterOpenAIClient(openRouterApiKey);
+  const texts = normalizeInput(input);
+
+  const provider = resolveOpenRouterProvider(openRouterModel, {
+    openRouterProvider: options.openRouterProvider,
+    openRouterAllowAllProviders: options.openRouterAllowAllProviders,
+  });
+
+  const body: Record<string, unknown> = {
+    model: openRouterModel,
+    input: texts.length === 1 ? texts[0]! : texts,
+  };
+
+  if (options.dimensions !== undefined) body.dimensions = options.dimensions;
+  if (options.encodingFormat) body.encoding_format = options.encodingFormat;
+  if (provider) body.provider = provider;
+
+  const response = await client.embeddings.create(
+    body as unknown as OpenAI.EmbeddingCreateParams,
+  );
+
+  return mapEmbeddingResponse(response, openRouterModel);
+}