@hasna/skills 0.0.2

package/skills/skill-transcript/src/providers/elevenlabs.ts

@@ -0,0 +1,133 @@
+/**
+ * ElevenLabs Scribe Provider
+ * High-accuracy speech-to-text with speaker diarization
+ */
+
+import type {
+  TranscriptionProvider,
+  TranscriptionOptions,
+  TranscriptionResult,
+  TranscriptionSegment,
+  ElevenLabsResponse
+} from '../types';
+
+// ElevenLabs supports up to 3GB files
+const MAX_FILE_SIZE = 3 * 1024 * 1024 * 1024; // 3GB
+
+export class ElevenLabsProvider implements TranscriptionProvider {
+  name = 'ElevenLabs Scribe';
+  maxFileSize = MAX_FILE_SIZE;
+  supportedFormats = ['mp3', 'mp4', 'wav', 'webm', 'm4a', 'ogg', 'flac'];
+
+  private apiKey: string;
+  private baseUrl = 'https://api.elevenlabs.io/v1';
+
+  constructor() {
+    const apiKey = process.env.ELEVENLABS_API_KEY;
+    if (!apiKey) {
+      throw new Error('ELEVENLABS_API_KEY environment variable is required');
+    }
+    this.apiKey = apiKey;
+  }
+
+  async transcribe(options: TranscriptionOptions): Promise<TranscriptionResult> {
+    const {
+      input,
+      language,
+      model = 'scribe_v1',
+      diarize = true,
+      timestamps = true
+    } = options;
+
+    console.log(`[ElevenLabs] Transcribing: ${input}`);
+    console.log(`[ElevenLabs] Model: ${model}`);
+    console.log(`[ElevenLabs] Diarization: ${diarize}`);
+
+    // Read the file
+    const file = Bun.file(input);
+    const fileBuffer = await file.arrayBuffer();
+
+    // Create form data
+    const formData = new FormData();
+    formData.append('audio', new Blob([fileBuffer]), input.split('/').pop());
+    formData.append('model_id', model);
+
+    if (language) {
+      formData.append('language_code', language);
+    }
+
+    if (diarize) {
+      formData.append('diarize', 'true');
+      formData.append('num_speakers', '32'); // Max supported speakers
+    }
+
+    if (timestamps) {
+      formData.append('timestamps_granularity', 'word');
+    }
+
+    const response = await fetch(`${this.baseUrl}/speech-to-text`, {
+      method: 'POST',
+      headers: {
+        'xi-api-key': this.apiKey
+      },
+      body: formData
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`ElevenLabs transcription failed: ${error}`);
+    }
+
+    const data = await response.json() as ElevenLabsResponse;
+    console.log(`[ElevenLabs] Transcription complete`);
+
+    return this.formatResult(data, model);
+  }
+
+  private formatResult(data: ElevenLabsResponse, model: string): TranscriptionResult {
+    const segments: TranscriptionSegment[] = [];
+
+    if (data.words) {
+      let currentSegment: TranscriptionSegment | null = null;
+      let segmentId = 0;
+
+      for (const word of data.words) {
+        if (!currentSegment || word.speaker !== currentSegment.speaker) {
+          if (currentSegment) {
+            segments.push(currentSegment);
+          }
+          currentSegment = {
+            id: segmentId++,
+            start: word.start,
+            end: word.end,
+            text: word.word,
+            speaker: word.speaker
+          };
+        } else {
+          currentSegment.end = word.end;
+          currentSegment.text += ' ' + word.word;
+        }
+      }
+
+      if (currentSegment) {
+        segments.push(currentSegment);
+      }
+    }
+
+    const speakers = data.speakers?.map(s => ({
+      id: s.speaker_id,
+      name: s.name,
+      segments: segments
+        .filter(seg => seg.speaker === s.speaker_id)
+        .map(seg => seg.id)
+    }));
+
+    return {
+      text: data.text,
+      segments: segments.length > 0 ? segments : undefined,
+      speakers,
+      model,
+      provider: 'elevenlabs'
+    };
+  }
+}

package/skills/skill-transcript/src/providers/gemini.ts

@@ -0,0 +1,192 @@
+/**
+ * Google Gemini Provider
+ * Multimodal transcription with analysis capabilities
+ */
+
+import { readFile } from 'fs/promises';
+import type {
+  TranscriptionProvider,
+  TranscriptionOptions,
+  TranscriptionResult,
+  GeminiResponse
+} from '../types';
+import { getFileSize, needsChunking, splitAudioFile, mergeTranscriptions, cleanupChunks } from '../chunker';
+import { dirname, join } from 'path';
+
+// Gemini supports up to 2GB files
+const MAX_FILE_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
+
+const MIME_TYPES: Record<string, string> = {
+  'mp3': 'audio/mpeg',
+  'wav': 'audio/wav',
+  'ogg': 'audio/ogg',
+  'flac': 'audio/flac',
+  'm4a': 'audio/mp4',
+  'aac': 'audio/aac',
+  'mp4': 'video/mp4',
+  'webm': 'video/webm'
+};
+
+export class GeminiProvider implements TranscriptionProvider {
+  name = 'Google Gemini';
+  maxFileSize = MAX_FILE_SIZE;
+  supportedFormats = ['mp3', 'wav', 'ogg', 'flac', 'm4a', 'aac', 'mp4', 'webm'];
+
+  private apiKey: string;
+  private baseUrl = 'https://generativelanguage.googleapis.com/v1beta';
+
+  constructor() {
+    const apiKey = process.env.GOOGLE_API_KEY;
+    if (!apiKey) {
+      throw new Error('GOOGLE_API_KEY environment variable is required');
+    }
+    this.apiKey = apiKey;
+  }
+
+  async transcribe(options: TranscriptionOptions): Promise<TranscriptionResult> {
+    const {
+      input,
+      language,
+      model = 'gemini-2.0-flash',
+      diarize = false,
+      timestamps = false
+    } = options;
+
+    console.log(`[Gemini] Transcribing: ${input}`);
+    console.log(`[Gemini] Model: ${model}`);
+
+    // Check file size
+    const fileSize = await getFileSize(input);
+    console.log(`[Gemini] File size: ${(fileSize / 1024 / 1024).toFixed(2)}MB`);
+
+    // For very large files, use chunking
+    if (fileSize > 100 * 1024 * 1024) { // > 100MB use chunking for efficiency
+      console.log(`[Gemini] Large file detected, using chunking for efficiency`);
+      return this.transcribeWithChunking(options);
+    }
+
+    return this.transcribeSingle(options);
+  }
+
+  private async transcribeSingle(options: TranscriptionOptions): Promise<TranscriptionResult> {
+    const {
+      input,
+      language,
+      model = 'gemini-2.0-flash',
+      diarize = false,
+      timestamps = false
+    } = options;
+
+    // Read and encode file
+    const fileBuffer = await readFile(input);
+    const base64Audio = fileBuffer.toString('base64');
+
+    // Determine MIME type
+    const ext = input.split('.').pop()?.toLowerCase() || 'mp3';
+    const mimeType = MIME_TYPES[ext] || 'audio/mpeg';
+
+    // Build prompt based on options
+    let prompt = 'Transcribe this audio accurately. ';
+
+    if (language) {
+      prompt += `The audio is in ${language}. `;
+    }
+
+    if (diarize) {
+      prompt += 'Identify and label different speakers (Speaker A, Speaker B, etc.). ';
+    }
+
+    if (timestamps) {
+      prompt += 'Include timestamps for each segment in [HH:MM:SS] format. ';
+    }
+
+    prompt += 'Provide only the transcription without any additional commentary.';
+
+    const response = await fetch(
+      `${this.baseUrl}/models/${model}:generateContent?key=${this.apiKey}`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          contents: [
+            {
+              parts: [
+                {
+                  inline_data: {
+                    mime_type: mimeType,
+                    data: base64Audio
+                  }
+                },
+                {
+                  text: prompt
+                }
+              ]
+            }
+          ],
+          generationConfig: {
+            temperature: 0,
+            maxOutputTokens: 8192
+          }
+        })
+      }
+    );
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`Gemini transcription failed: ${error}`);
+    }
+
+    const data = await response.json() as GeminiResponse;
+    const text = data.candidates?.[0]?.content?.parts?.[0]?.text || '';
+
+    console.log(`[Gemini] Transcription complete`);
+
+    return {
+      text: text.trim(),
+      model,
+      provider: 'gemini'
+    };
+  }
+
+  private async transcribeWithChunking(options: TranscriptionOptions): Promise<TranscriptionResult> {
+    const { input, model = 'gemini-2.0-flash' } = options;
+    const outputDir = dirname(input);
+
+    // Split into chunks
+    const chunks = await splitAudioFile(input, outputDir);
+    const transcriptions: string[] = [];
+
+    try {
+      // Process chunks in parallel (Gemini can handle concurrent requests)
+      const promises = chunks.map(async (chunk) => {
+        console.log(`[Gemini] Processing chunk ${chunk.index + 1}/${chunks.length}`);
+        const result = await this.transcribeSingle({
+          ...options,
+          input: chunk.path
+        });
+        return { index: chunk.index, text: result.text };
+      });
+
+      const results = await Promise.all(promises);
+
+      // Sort by index and extract texts
+      results.sort((a, b) => a.index - b.index);
+      transcriptions.push(...results.map(r => r.text));
+
+      // Merge transcriptions
+      const mergedText = mergeTranscriptions(transcriptions);
+
+      return {
+        text: mergedText,
+        model,
+        provider: 'gemini'
+      };
+    } finally {
+      // Cleanup chunks
+      const chunkDir = join(outputDir, '.chunks');
+      await cleanupChunks(chunkDir);
+    }
+  }
+}

package/skills/skill-transcript/src/providers/index.ts

@@ -0,0 +1,54 @@
+/**
+ * Provider Factory
+ * Returns the appropriate transcription provider
+ */
+
+import { ElevenLabsProvider } from './elevenlabs';
+import { OpenAIProvider } from './openai';
+import { GeminiProvider } from './gemini';
+import type { TranscriptionProvider, Provider } from '../types';
+
+export function getProvider(providerName: Provider): TranscriptionProvider {
+  switch (providerName) {
+    case 'elevenlabs':
+      return new ElevenLabsProvider();
+    case 'openai':
+      return new OpenAIProvider();
+    case 'gemini':
+      return new GeminiProvider();
+    default:
+      throw new Error(
+        `Unknown provider: ${providerName}. Available: elevenlabs, openai, gemini`
+      );
+  }
+}
+
+export function getProviderInfo(): void {
+  console.log(`
+Available Transcription Providers:
+
+ELEVENLABS (Scribe)
+  - Accuracy: 96.7% for English
+  - Max file size: 3GB / 10 hours
+  - Features: Speaker diarization (up to 32 speakers), word timestamps
+  - Cost: $0.40/hour
+  - Best for: Multi-speaker recordings, highest accuracy needs
+
+OPENAI (Whisper)
+  - Accuracy: Excellent
+  - Max file size: 25MB (auto-chunking for larger files)
+  - Features: Segment timestamps, language detection
+  - Cost: $0.006/min (GPT-4o Mini: $0.003/min)
+  - Best for: Standard transcription, good balance of cost/quality
+
+GEMINI (Flash)
+  - Accuracy: Very good
+  - Max file size: 2GB
+  - Features: Multimodal analysis, summarization
+  - Cost: ~$0.09-0.23/hour (generous free tier)
+  - Best for: Cost-sensitive, multimodal needs, free tier usage
+
+USAGE:
+  skill-transcript transcribe --provider <name> --input <file> [options]
+`);
+}

package/skills/skill-transcript/src/providers/openai.ts

@@ -0,0 +1,203 @@
+/**
+ * OpenAI Whisper Provider
+ * Industry-standard speech-to-text with automatic chunking for large files
+ */
+
+import type {
+  TranscriptionProvider,
+  TranscriptionOptions,
+  TranscriptionResult,
+  TranscriptionSegment,
+  OpenAITranscriptionResponse
+} from '../types';
+import {
+  needsChunking,
+  splitAudioFile,
+  mergeTranscriptions,
+  cleanupChunks,
+  compressAudio,
+  getFileSize
+} from '../chunker';
+import { dirname, join, basename, extname } from 'path';
+
+// OpenAI Whisper has a 25MB limit
+const MAX_FILE_SIZE = 25 * 1024 * 1024; // 25MB
+
+export class OpenAIProvider implements TranscriptionProvider {
+  name = 'OpenAI Whisper';
+  maxFileSize = MAX_FILE_SIZE;
+  supportedFormats = ['mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'wav', 'webm'];
+
+  private apiKey: string;
+  private baseUrl = 'https://api.openai.com/v1';
+
+  constructor() {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+      throw new Error('OPENAI_API_KEY environment variable is required');
+    }
+    this.apiKey = apiKey;
+  }
+
+  async transcribe(options: TranscriptionOptions): Promise<TranscriptionResult> {
+    const {
+      input,
+      language,
+      model = 'whisper-1',
+      timestamps = true
+    } = options;
+
+    console.log(`[OpenAI] Transcribing: ${input}`);
+    console.log(`[OpenAI] Model: ${model}`);
+
+    // Check if file needs chunking
+    const fileSize = await getFileSize(input);
+    console.log(`[OpenAI] File size: ${(fileSize / 1024 / 1024).toFixed(2)}MB`);
+
+    if (fileSize > this.maxFileSize) {
+      console.log(`[OpenAI] File exceeds 25MB limit, chunking required`);
+      return this.transcribeWithChunking(options);
+    }
+
+    return this.transcribeSingle(options);
+  }
+
+  private async transcribeSingle(options: TranscriptionOptions): Promise<TranscriptionResult> {
+    const {
+      input,
+      language,
+      model = 'whisper-1',
+      timestamps = true
+    } = options;
+
+    // Read the file
+    const file = Bun.file(input);
+    const fileBuffer = await file.arrayBuffer();
+
+    // Create form data
+    const formData = new FormData();
+    formData.append('file', new Blob([fileBuffer]), input.split('/').pop());
+    formData.append('model', model);
+    formData.append('response_format', timestamps ? 'verbose_json' : 'json');
+
+    if (language) {
+      formData.append('language', language);
+    }
+
+    if (timestamps) {
+      formData.append('timestamp_granularities[]', 'segment');
+      formData.append('timestamp_granularities[]', 'word');
+    }
+
+    const response = await fetch(`${this.baseUrl}/audio/transcriptions`, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${this.apiKey}`
+      },
+      body: formData
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`OpenAI transcription failed: ${error}`);
+    }
+
+    const data = await response.json() as OpenAITranscriptionResponse;
+    console.log(`[OpenAI] Transcription complete`);
+
+    return this.formatResult(data, model);
+  }
+
+  private async transcribeWithChunking(options: TranscriptionOptions): Promise<TranscriptionResult> {
+    const { input, model = 'whisper-1' } = options;
+    const outputDir = dirname(input);
+
+    // First try compression
+    const compressedPath = join(outputDir, `.compressed_${basename(input, extname(input))}.ogg`);
+
+    try {
+      console.log('[OpenAI] Attempting compression first...');
+      await compressAudio(input, compressedPath, '32k');
+      const compressedSize = await getFileSize(compressedPath);
+
+      if (compressedSize <= this.maxFileSize) {
+        console.log('[OpenAI] Compressed file is under 25MB, transcribing directly');
+        const result = await this.transcribeSingle({
+          ...options,
+          input: compressedPath
+        });
+
+        // Cleanup compressed file
+        await Bun.write(compressedPath, '').catch(() => {});
+
+        return result;
+      }
+    } catch {
+      console.log('[OpenAI] Compression failed or still too large, falling back to chunking');
+    }
+
+    // Split into chunks
+    const chunks = await splitAudioFile(input, outputDir);
+    const transcriptions: string[] = [];
+    const allSegments: TranscriptionSegment[] = [];
+
+    try {
+      // Process chunks sequentially to maintain order
+      for (const chunk of chunks) {
+        console.log(`[OpenAI] Processing chunk ${chunk.index + 1}/${chunks.length}`);
+
+        const result = await this.transcribeSingle({
+          ...options,
+          input: chunk.path
+        });
+
+        transcriptions.push(result.text);
+
+        // Adjust segment timestamps based on chunk start time
+        if (result.segments) {
+          for (const segment of result.segments) {
+            allSegments.push({
+              ...segment,
+              id: allSegments.length,
+              start: segment.start + chunk.startTime,
+              end: segment.end + chunk.startTime
+            });
+          }
+        }
+      }
+
+      // Merge transcriptions
+      const mergedText = mergeTranscriptions(transcriptions);
+
+      return {
+        text: mergedText,
+        segments: allSegments.length > 0 ? allSegments : undefined,
+        model,
+        provider: 'openai'
+      };
+    } finally {
+      // Cleanup chunks
+      const chunkDir = join(outputDir, '.chunks');
+      await cleanupChunks(chunkDir);
+    }
+  }
+
+  private formatResult(data: OpenAITranscriptionResponse, model: string): TranscriptionResult {
+    const segments: TranscriptionSegment[] = data.segments?.map((seg, i) => ({
+      id: i,
+      start: seg.start,
+      end: seg.end,
+      text: seg.text.trim(),
+      confidence: 1 - seg.no_speech_prob
+    })) || [];
+
+    return {
+      text: data.text,
+      segments: segments.length > 0 ? segments : undefined,
+      duration: data.duration,
+      language: data.language,
+      model,
+      provider: 'openai'
+    };
+  }
+}