@tyvm/knowhow 0.0.90 → 0.0.92

package/src/clients/gemini.ts

@@ -499,29 +499,81 @@ export class GenericGeminiClient implements GenericClient {
 
     let cost = 0;
 
-    if ("promptTokenCount" in usage && usage.promptTokenCount) {
-      if (usage.promptTokenCount > 200000 && pricing.input_gt_200k) {
-        cost += (usage.promptTokenCount * pricing.input_gt_200k) / 1e6;
-      } else {
-        cost += (usage.promptTokenCount * pricing.input) / 1e6;
+    // ── Input tokens ──────────────────────────────────────────────────────────
+    // Use per-modality breakdowns when available (inputTokensDetails) so that
+    // audio tokens (which can cost 3–10× more than text) are billed correctly.
+    const inputDetails = (usage as any).inputTokensDetails as
+      | { modality?: string; tokenCount?: number }[]
+      | undefined;
+
+    if (
+      inputDetails &&
+      inputDetails.length > 0 &&
+      pricing.input_audio !== undefined
+    ) {
+      // Modality-aware billing
+      for (const detail of inputDetails) {
+        const tokens = detail.tokenCount || 0;
+        if (!tokens) continue;
+        const modality = (detail.modality || "").toUpperCase();
+        if (modality === "AUDIO") {
+          const rate = pricing.input_audio ?? pricing.input ?? 0;
+          cost += (tokens * rate) / 1e6;
+        } else {
+          // TEXT / IMAGE / VIDEO all use the base `input` rate
+          const isGt200k = tokens > 200_000 && pricing.input_gt_200k;
+          const rate = isGt200k ? pricing.input_gt_200k! : pricing.input ?? 0;
+          cost += (tokens * rate) / 1e6;
+        }
       }
+    } else if ("promptTokenCount" in usage && usage.promptTokenCount) {
+      // Fallback: no modality breakdown available — use total token count
+      const isGt200k =
+        usage.promptTokenCount > 200_000 && pricing.input_gt_200k;
+      const rate = isGt200k ? pricing.input_gt_200k! : pricing.input ?? 0;
+      cost += (usage.promptTokenCount * rate) / 1e6;
     }
 
-    if ("responseTokenCount" in usage && usage.responseTokenCount) {
-      if (usage.responseTokenCount > 200000 && pricing.output_gt_200k) {
-        cost += (usage.responseTokenCount * pricing.output_gt_200k) / 1e6;
-      } else {
-        cost += (usage.responseTokenCount * pricing.output) / 1e6;
+    // ── Output tokens ─────────────────────────────────────────────────────────
+    const outputDetails = (usage as any).outputTokensDetails as
+      | { modality?: string; tokenCount?: number }[]
+      | undefined;
+
+    if (
+      outputDetails &&
+      outputDetails.length > 0 &&
+      pricing.output_audio !== undefined
+    ) {
+      // Modality-aware billing
+      for (const detail of outputDetails) {
+        const tokens = detail.tokenCount || 0;
+        if (!tokens) continue;
+        const modality = (detail.modality || "").toUpperCase();
+        if (modality === "AUDIO") {
+          const rate = pricing.output_audio ?? pricing.output ?? 0;
+          cost += (tokens * rate) / 1e6;
+        } else {
+          const isGt200k = tokens > 200_000 && pricing.output_gt_200k;
+          const rate = isGt200k ? pricing.output_gt_200k! : pricing.output ?? 0;
+          cost += (tokens * rate) / 1e6;
+        }
       }
+    } else if ("responseTokenCount" in usage && usage.responseTokenCount) {
+      // Fallback: no modality breakdown — use total token count
+      const isGt200k =
+        usage.responseTokenCount > 200_000 && pricing.output_gt_200k;
+      const rate = isGt200k ? pricing.output_gt_200k! : pricing.output ?? 0;
+      cost += (usage.responseTokenCount * rate) / 1e6;
     }
 
+    // ── Context caching ───────────────────────────────────────────────────────
     if (
       "cachedContentTokenCount" in usage &&
       usage.cachedContentTokenCount &&
       pricing.context_caching
     ) {
       if (
-        usage.cachedContentTokenCount > 200000 &&
+        usage.cachedContentTokenCount > 200_000 &&
         pricing.context_caching_gt_200k
       ) {
         cost +=
@@ -531,6 +583,7 @@ export class GenericGeminiClient implements GenericClient {
         cost += (usage.cachedContentTokenCount * pricing.context_caching) / 1e6;
       }
     }
+
     return cost;
   }
 
@@ -675,9 +728,7 @@ export class GenericGeminiClient implements GenericClient {
         const images = generatedImages.map((img) => ({
           // imageBytes is already a base64-encoded string from the API
           // Don't re-encode it, just use it directly
-          b64_json: img.image?.imageBytes
-            ? img.image.imageBytes
-            : "",
+          b64_json: img.image?.imageBytes ? img.image.imageBytes : "",
           revised_prompt: options.prompt,
         }));
 
@@ -752,9 +803,11 @@ export class GenericGeminiClient implements GenericClient {
         },
       });
 
-      // Calculate estimated cost: $0.35 per second of video
+      // Calculate estimated cost using model-specific per-second rate
       const duration = options.duration || 5; // Default 5 seconds
-      const usdCost = (options.n || 1) * duration * 0.35;
+      const pricingEntry = GeminiTextPricing[options.model];
+      const ratePerSec = pricingEntry?.video_generation ?? 0.4; // default to $0.40/sec (Veo 3 rate)
+      const usdCost = (options.n || 1) * duration * ratePerSec;
 
       // Return the operation name as jobId so callers can use getVideoStatus / downloadVideo
       return {
@@ -769,7 +822,9 @@ export class GenericGeminiClient implements GenericClient {
     }
   }
 
-  async getVideoStatus(options: VideoStatusOptions): Promise<VideoStatusResponse> {
+  async getVideoStatus(
+    options: VideoStatusOptions
+  ): Promise<VideoStatusResponse> {
     try {
       const operation = await this.client.operations.getVideosOperation({
         operation: { name: options.jobId },
@@ -817,7 +872,9 @@ export class GenericGeminiClient implements GenericClient {
    * Download a video (or any file) via the Google GenAI Files API.
    * Pass either `fileId` (the files/* name) or `uri` (the full URI).
    */
-  async downloadVideo(options: FileDownloadOptions): Promise<FileDownloadResponse> {
+  async downloadVideo(
+    options: FileDownloadOptions
+  ): Promise<FileDownloadResponse> {
     return this.downloadFile(options);
   }
 
@@ -841,7 +898,9 @@ export class GenericGeminiClient implements GenericClient {
         uri: uploadedFile.uri,
         url: uploadedFile.downloadUri || uploadedFile.uri,
         mimeType: uploadedFile.mimeType,
-        sizeBytes: uploadedFile.sizeBytes ? Number(uploadedFile.sizeBytes) : undefined,
+        sizeBytes: uploadedFile.sizeBytes
+          ? Number(uploadedFile.sizeBytes)
+          : undefined,
       };
     } catch (error) {
       console.error("Error uploading file to Google GenAI Files API:", error);
@@ -860,7 +919,9 @@ export class GenericGeminiClient implements GenericClient {
    * For generated videos the `file` param accepts the Video object directly
    * (uri + optional mimeType), which the SDK resolves to a download URL.
    */
-  async downloadFile(options: FileDownloadOptions): Promise<FileDownloadResponse> {
+  async downloadFile(
+    options: FileDownloadOptions
+  ): Promise<FileDownloadResponse> {
     const mimeMap: Record<string, string> = {
       ".mp4": "video/mp4",
       ".webm": "video/webm",
@@ -894,7 +955,9 @@ export class GenericGeminiClient implements GenericClient {
 
       const response = await fetch(downloadUrl);
       if (!response.ok) {
-        throw new Error(`HTTP ${response.status} ${response.statusText} downloading ${downloadUrl}`);
+        throw new Error(
+          `HTTP ${response.status} ${response.statusText} downloading ${downloadUrl}`
+        );
       }
 
       const arrayBuffer = await response.arrayBuffer();
@@ -902,23 +965,31 @@ export class GenericGeminiClient implements GenericClient {
 
       // If caller supplied a filePath, write to it (creating dirs as needed)
       if (options.filePath) {
-        fsSync.mkdirSync(pathSync.dirname(options.filePath), { recursive: true });
+        fsSync.mkdirSync(pathSync.dirname(options.filePath), {
+          recursive: true,
+        });
         fsSync.writeFileSync(options.filePath, data);
       }
 
       // Infer mime type from the URI/fileId first (more reliable), then from the path
-      const sourceForExt = options.uri || options.fileId || options.filePath || "";
+      const sourceForExt =
+        options.uri || options.fileId || options.filePath || "";
       const ext = pathSync.extname(sourceForExt.split("?")[0]).toLowerCase();
       const mimeType = mimeMap[ext] || "video/mp4";
 
       return { data, mimeType };
     } catch (error) {
-      console.error("Error downloading file from Google GenAI Files API:", error);
+      console.error(
+        "Error downloading file from Google GenAI Files API:",
+        error
+      );
       throw error;
     }
   }
 
-  getContextLimit(model: string): { contextLimit: number; threshold: number } | undefined {
+  getContextLimit(
+    model: string
+  ): { contextLimit: number; threshold: number } | undefined {
     const contextLimit = ContextLimits[model];
     if (contextLimit === undefined) return undefined;
     const pricing = GeminiTextPricing[model];

package/src/clients/http.ts

@@ -1,4 +1,4 @@
-import axios from "axios";
+import http from "../utils/http";
 import {
   GenericClient,
   CompletionOptions,
@@ -91,12 +91,10 @@ export class HttpClient implements GenericClient {
         }),
       };
 
-      const response = await axios.post(
+      const response = await http.post(
         `${this.baseUrl}/v1/chat/completions`,
         body,
-        {
-          headers: this.headers,
-        }
+        { headers: this.headers as Record<string, string> }
       );
 
       const data = response.data;
@@ -123,15 +121,13 @@ export class HttpClient implements GenericClient {
 
   async createEmbedding(options: EmbeddingOptions): Promise<EmbeddingResponse> {
     return this.withRetry(async () => {
-      const response = await axios.post(
+      const response = await http.post(
         `${this.baseUrl}/v1/embeddings`,
         {
           model: options.model,
           input: options.input,
         },
-        {
-          headers: this.headers,
-        }
+        { headers: this.headers as Record<string, string> }
       );
 
       const data = response.data;
@@ -152,8 +148,8 @@ export class HttpClient implements GenericClient {
 
   async getModels() {
     return this.withRetry(async () => {
-      const response = await axios.get(`${this.baseUrl}/v1/models`, {
-        headers: this.headers,
+      const response = await http.get(`${this.baseUrl}/v1/models`, {
+        headers: this.headers as Record<string, string>,
       });
 
       const data = response.data?.data;

package/src/clients/pricing/google.ts

@@ -1,7 +1,38 @@
 import { Models, EmbeddingModels } from "../../types";
 
-export const GeminiTextPricing = {
-  // Gemini 3.x
+/**
+ * Gemini model pricing per 1M tokens (USD).
+ *
+ * Many models have modality-dependent pricing:
+ * - `input`: text / image / video input rate
+ * - `input_audio`: audio input rate (if different from text)
+ * - `output`: text output rate
+ * - `output_audio`: audio output rate (if different from text)
+ * - `context_caching`: context cache read rate (text/image/video)
+ * - `context_caching_audio`: context cache read rate for audio tokens
+ * - `context_caching_gt_200k`: rate for prompts > 200k tokens
+ * - `context_caching_storage`: storage cost per 1M tokens per hour
+ * - `image_generation`: per-image cost for image output models
+ * - `video_generation`: per-second cost for video generation models
+ */
+export interface GeminiModelPricing {
+  input?: number;
+  input_audio?: number;
+  input_gt_200k?: number;
+  output?: number;
+  output_audio?: number;
+  output_gt_200k?: number;
+  context_caching?: number;
+  context_caching_audio?: number;
+  context_caching_gt_200k?: number;
+  context_caching_storage?: number; // per 1M tokens per hour
+  image_generation?: number;
+  video_generation?: number;
+  thinking_output?: number;
+}
+
+export const GeminiPricing: Record<string, GeminiModelPricing> = {
+  // ── Gemini 3.x ────────────────────────────────────────────────────────────
   [Models.google.Gemini_31_Pro_Preview]: {
     input: 2,
     input_gt_200k: 4,
@@ -9,28 +40,49 @@ export const GeminiTextPricing = {
     output_gt_200k: 18,
     context_caching: 0.2,
     context_caching_gt_200k: 0.4,
+    context_caching_storage: 4.5,
   },
+  // gemini-3.1-flash-image-preview: text/image input, text+image output
   [Models.google.Gemini_31_Flash_Image_Preview]: {
     input: 0.5,
     output: 3,
-    image_generation: 0.045, // per 0.5K image
+    image_generation: 0.045, // per 0.5K image ($60/1M tokens × 747 tokens)
   },
+  // gemini-3.1-flash-lite-preview: audio input costs 2× text
   [Models.google.Gemini_31_Flash_Lite_Preview]: {
     input: 0.25,
+    input_audio: 0.5,
     output: 1.5,
     context_caching: 0.025,
-  },
+    context_caching_audio: 0.05,
+    context_caching_storage: 1.0,
+  },
+  // gemini-3.1-flash-live-preview: Live API, per-modality rates
+  // Input: $0.75 text, $3.00 audio/video, $1.00 image
+  // Output: $4.50 text, $12.00 audio
+  [Models.google.Gemini_31_Flash_Live_Preview]: {
+    input: 0.75,        // text
+    input_audio: 3.0,   // audio / video
+    output: 4.5,        // text
+    output_audio: 12.0, // audio
+  },
+  // gemini-3-flash-preview: audio input costs 2× text
   [Models.google.Gemini_3_Flash_Preview]: {
     input: 0.5,
+    input_audio: 1.0,
     output: 3.0,
     context_caching: 0.05,
+    context_caching_audio: 0.10,
+    context_caching_storage: 1.0,
   },
+  // gemini-3-pro-image-preview: text+image input, text+image output
   [Models.google.Gemini_3_Pro_Image_Preview]: {
     input: 2,
     output: 12,
-    image_generation: 0.134, // per 1K/2K image
+    image_generation: 0.134, // per 1K/2K image ($120/1M tokens × 1120 tokens)
   },
-  // Gemini 2.5
+
+  // ── Gemini 2.5 ────────────────────────────────────────────────────────────
   [Models.google.Gemini_25_Pro]: {
     input: 1.25,
     input_gt_200k: 2.5,
@@ -38,22 +90,34 @@ export const GeminiTextPricing = {
     output_gt_200k: 15.0,
     context_caching: 0.125,
     context_caching_gt_200k: 0.25,
+    context_caching_storage: 4.5,
   },
+  // gemini-2.5-flash: audio input costs 3.3× text; audio caching 3.3× text
   [Models.google.Gemini_25_Flash]: {
     input: 0.3,
+    input_audio: 1.0,
     output: 2.5,
     context_caching: 0.03,
+    context_caching_audio: 0.1,
+    context_caching_storage: 1.0,
   },
+  // gemini-2.5-flash-lite: audio input costs 3× text; audio caching 3× text
   [Models.google.Gemini_25_Flash_Lite]: {
     input: 0.1,
+    input_audio: 0.3,
     output: 0.4,
     context_caching: 0.01,
+    context_caching_audio: 0.03,
+    context_caching_storage: 1.0,
   },
+  // gemini-2.5-flash-preview-05-20: same modality splits as 2.5 Flash
   [Models.google.Gemini_25_Flash_Preview]: {
     input: 0.3,
+    input_audio: 1.0,
     output: 2.5,
-    thinking_output: 3.5,
-    context_caching: 0.0375,
+    context_caching: 0.03,
+    context_caching_audio: 0.1,
+    context_caching_storage: 1.0,
   },
   [Models.google.Gemini_25_Pro_Preview]: {
     input: 1.25,
@@ -62,24 +126,48 @@ export const GeminiTextPricing = {
     output_gt_200k: 15.0,
     context_caching: 0.125,
     context_caching_gt_200k: 0.25,
+    context_caching_storage: 4.5,
   },
+  // gemini-2.5-flash-image: image output model, per-image pricing
   [Models.google.Gemini_25_Flash_Image]: {
     input: 0.3,
-    output: 0.039, // per image ($30/1M tokens, 1290 tokens per image)
+    output: 0.039, // per image ($30/1M tokens × 1290 tokens)
+  },
+  // gemini-2.5-flash-live / gemini-2.5-flash-native-audio-preview-12-2025:
+  // Live API models — audio I/O costs dramatically more than text
+  // Input: $0.50 text, $3.00 audio/video
+  // Output: $2.00 text, $12.00 audio
+  [Models.google.Gemini_25_Flash_Live]: {
+    input: 0.5,
+    input_audio: 3.0,
+    output: 2.0,
+    output_audio: 12.0,
+  },
+  [Models.google.Gemini_25_Flash_Native_Audio]: {
+    input: 0.5,
+    input_audio: 3.0,
+    output: 2.0,
+    output_audio: 12.0,
   },
+  // TTS models: text-only input, audio-only output
   [Models.google.Gemini_25_Flash_TTS]: {
     input: 0.5,
-    output: 10.0,
+    output_audio: 10.0,
   },
   [Models.google.Gemini_25_Pro_TTS]: {
     input: 1.0,
-    output: 20.0,
+    output_audio: 20.0,
   },
-  // Gemini 2.0 (deprecated)
+
+  // ── Gemini 2.0 (deprecated) ───────────────────────────────────────────────
+  // gemini-2.0-flash: audio input costs 7× text; audio caching costs 7× text
   [Models.google.Gemini_20_Flash]: {
     input: 0.1,
+    input_audio: 0.7,
     output: 0.4,
     context_caching: 0.025,
+    context_caching_audio: 0.175,
+    context_caching_storage: 1.0,
   },
   [Models.google.Gemini_20_Flash_Preview_Image_Generation]: {
     input: 0.1,
@@ -90,7 +178,8 @@ export const GeminiTextPricing = {
     input: 0.075,
     output: 0.3,
   },
-  // Gemini 1.5 (legacy)
+
+  // ── Gemini 1.5 (legacy) ───────────────────────────────────────────────────
   [Models.google.Gemini_15_Flash]: {
     input: 0.075,
     output: 0.3,
@@ -106,39 +195,46 @@ export const GeminiTextPricing = {
     output: 5.0,
     context_caching: 0.3125,
   },
-  // Image generation
+
+  // ── Image generation ──────────────────────────────────────────────────────
   [Models.google.Imagen_3]: {
-    image_generation: 0.04, // Imagen 4 Standard: $0.04/image
+    image_generation: 0.04, // $0.04/image (Imagen 4 Standard)
   },
   [Models.google.Imagen_4_Fast]: {
-    image_generation: 0.02, // $0.02/image
+    image_generation: 0.02,
   },
   [Models.google.Imagen_4_Ultra]: {
-    image_generation: 0.06, // $0.06/image
+    image_generation: 0.06,
   },
-  // Video generation
+
+  // ── Video generation ──────────────────────────────────────────────────────
   [Models.google.Veo_2]: {
-    video_generation: 0.35,
+    video_generation: 0.35, // per second
   },
   [Models.google.Veo_3]: {
-    video_generation: 0.4, // $0.40/second
+    video_generation: 0.4,
   },
   [Models.google.Veo_3_Fast]: {
-    video_generation: 0.15, // $0.15/second
+    video_generation: 0.1, // 720p base rate; 1080p=$0.12, 4k=$0.30 not expressible here
   },
   [Models.google.Veo_3_1]: {
-    video_generation: 0.4, // $0.40/second (720p/1080p)
+    video_generation: 0.4, // 720p/1080p; 4k=$0.60 not expressible as single scalar
   },
   [Models.google.Veo_3_1_Fast]: {
-    video_generation: 0.15, // $0.15/second
+    video_generation: 0.1, // 720p base; 1080p=$0.12, 4k=$0.30
   },
-  // Embeddings
+
+  // ── Embeddings ────────────────────────────────────────────────────────────
   [EmbeddingModels.google.Gemini_Embedding]: {
     input: 0, // Free of charge
-    output: 0, // Free of charge
   },
   [EmbeddingModels.google.Gemini_Embedding_001]: {
     input: 0.15,
-    output: 0,
   },
 };
+
+/**
+ * @deprecated Use GeminiPricing instead.
+ * Kept as alias for backwards compatibility.
+ */
+export const GeminiTextPricing = GeminiPricing;

package/src/conversion.ts

@@ -2,7 +2,16 @@ import pdf from "pdf-parse";
 import * as fs from "fs";
 import * as path from "path";
 import { readFile, fileExists } from "./utils";
-import { services } from "./services";
+
+/**
+ * Get the MediaProcessorService from services() lazily.
+ * We import lazily to avoid circular dependency issues.
+ */
+function getMediaProcessor() {
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const { services } = require("./services") as typeof import("./services");
+  return services().MediaProcessor;
+}
 
 export async function processAudio(
   filePath: string,
@@ -17,24 +26,12 @@ export async function processAudio(
     console.log(`Transcription ${outputPath} already exists, skipping`);
     const fileContent = await readFile(outputPath, "utf8");
     return outputPath.endsWith("txt")
-      ? fileContent.split("\n")
-      : JSON.parse(fileContent);
+      ? (fileContent as string).split("\n")
+      : JSON.parse(fileContent as string);
   }
 
-  const { Downloader } = services();
-  const chunks = await Downloader.chunk(
-    filePath,
-    parsed.dir,
-    chunkTime,
-    reusePreviousTranscript
-  );
-  const transcription = await Downloader.transcribeChunks(
-    chunks,
-    outputPath,
-    reusePreviousTranscript
-  );
-
-  return transcription;
+  const mediaProcessor = getMediaProcessor();
+  return mediaProcessor.processAudio(filePath, reusePreviousTranscript, chunkTime);
 }
 
 export async function convertAudioToText(
@@ -73,48 +70,21 @@ export async function processVideo(
     chunkTime
   );
 
-  console.log("Extracting keyframes...");
-  const { Downloader } = services();
-  const videoAnalysis = await Downloader.extractKeyframes(
-    filePath,
-    outputPath,
-    reusePreviousTranscript,
-    chunkTime
-  );
-
-  return videoAnalysis.map((frame, index) => {
-    return {
-      frame,
-      transcription: transcriptions[index],
-    };
-  });
+  // Return the transcriptions as text — keyframe extraction requires the
+  // @tyvm/knowhow-module-video-downloader module
+  return transcriptions;
 }
 
 async function convertVideoToText(
   filePath: string,
   reusePreviousTranscript = true,
   chunkTime = 30
-) {
-  const processed = await processVideo(
-    filePath,
-    reusePreviousTranscript,
-    chunkTime
-  );
-
-  let fullString = "";
-
-  for (let i = 0; i < processed.length; i++) {
-    const chunk = processed[i];
-    fullString += `
-    Chunk: (${i + 1}/ ${processed.length}):
-    Start Timestamp: [${i * chunkTime}s]
-    Visual: ${chunk.frame.description}
-    Audio: ${chunk.transcription}
-    End Timestamp: [${i * chunkTime}s]
-    `;
+): Promise<string> {
+  const transcriptions = await processVideo(filePath, reusePreviousTranscript, chunkTime);
+  if (Array.isArray(transcriptions)) {
+    return transcriptions.join("\n");
   }
-
-  return fullString;
+  return String(transcriptions);
 }
 
 async function convertPdfToText(filePath: string) {
@@ -123,7 +93,7 @@ async function convertPdfToText(filePath: string) {
   return data.text;
 }
 
-export async function convertToText(filePath: string) {
+export async function convertToText(filePath: string): Promise<string> {
   const extension = filePath.split(".").pop();
 
   switch (extension) {
@@ -140,6 +110,6 @@ export async function convertToText(filePath: string) {
     case "pdf":
       return convertPdfToText(filePath);
     default:
-      return readFile(filePath, "utf8");
+      return ((await readFile(filePath, "utf8")) as string) || "";
   }
 }