@hed-hog/lms 0.0.365 → 0.0.370

package/src/course/course-audio-transcription.service.ts

@@ -14,6 +14,8 @@ import { promises as fs } from 'fs';
 import { tmpdir } from 'os';
 import { join } from 'path';
 import { promisify } from 'util';
+import { CourseAiUsageService } from './course-ai-usage.service';
+import { alignTextToTimings, fitsInTwoLines, TimedWord } from './subtitle.util';
 
 const execFileAsync = promisify(execFile);
 
@@ -104,6 +106,8 @@ export class CourseAudioTranscriptionService implements OnModuleInit, IJobHandle
     private readonly registry: QueueHandlerRegistry,
     @Inject(forwardRef(() => DatabaseQueueProvider))
     private readonly dbQueue: DatabaseQueueProvider,
+    @Inject(forwardRef(() => CourseAiUsageService))
+    private readonly aiUsageService: CourseAiUsageService,
   ) {}
 
   onModuleInit() {
@@ -111,6 +115,401 @@ export class CourseAudioTranscriptionService implements OnModuleInit, IJobHandle
     this.logger.log('Registered handler for "lms.audio.transcribe"');
   }
 
+  // Subtitle rules — Netflix Timed Text Style Guide (pt-BR).
+  private static readonly SUB_MAX_CHARS_PER_LINE = 42;
+  private static readonly SUB_MAX_LINES = 2;
+  private static readonly SUB_MAX_CPS = 17; // reading speed (chars per second)
+  private static readonly SUB_MAX_DURATION = 7; // seconds
+  private static readonly SUB_MIN_DURATION = 5 / 6; // ~0.833s
+  private static readonly SUB_MIN_GAP = 0.084; // ~2 frames @ 24fps
+  private static readonly SUB_PAUSE_SPLIT = 0.7; // silence that forces a new cue
+  // Soft break at commas/semicolons/colons only once the cue already holds a
+  // readable chunk (~one line), so we break at natural points without
+  // producing tiny fragments.
+  private static readonly SUB_SOFT_BREAK_MIN_CHARS = 38;
+  // Silence detection used to snap subtitle starts to where speech actually
+  // begins (Whisper word timestamps drift over intros/music and long pauses).
+  private static readonly SUB_SILENCE_NOISE_DB = -30; // silencedetect threshold
+  private static readonly SUB_SILENCE_MIN_DURATION = 0.3; // min silence length (s)
+
+  /**
+   * Parses the configured reading speed (characters per second) from the
+   * `lms-subtitle-reading-speed` setting value, e.g. "...(15 cps)". Falls back
+   * to the Netflix adult default and clamps to a sane range.
+   */
+  private resolveReadingSpeedCps(rawValue: unknown): number {
+    const match = String(rawValue ?? '').match(/(\d+)\s*cps/i);
+    const cps = match ? Number(match[1]) : NaN;
+    if (!Number.isFinite(cps) || cps < 6 || cps > 25) {
+      return CourseAudioTranscriptionService.SUB_MAX_CPS;
+    }
+    return cps;
+  }
+
+  /**
+   * Resolves the silencedetect parameters from settings, falling back to the
+   * defaults and clamping to sane ranges (noise must be negative dB; duration a
+   * small positive number of seconds).
+   */
+  private resolveSilenceParams(
+    noiseRaw: unknown,
+    durationRaw: unknown,
+  ): { noiseDb: number; minDuration: number } {
+    const Svc = CourseAudioTranscriptionService;
+    let noiseDb = Number(noiseRaw);
+    // Noise must be a strictly negative dB value (0/empty/positive are invalid).
+    if (!noiseRaw || !Number.isFinite(noiseDb) || noiseDb >= 0 || noiseDb < -90) {
+      noiseDb = Svc.SUB_SILENCE_NOISE_DB;
+    }
+    let minDuration = Number(durationRaw);
+    if (!Number.isFinite(minDuration) || minDuration <= 0 || minDuration > 10) {
+      minDuration = Svc.SUB_SILENCE_MIN_DURATION;
+    }
+    return { noiseDb, minDuration };
+  }
+
+  /**
+   * Detects silence intervals in the audio using FFmpeg's silencedetect filter.
+   * Subtitle starts are later snapped to the end of any silence they fall into,
+   * so captions appear when speech begins (not during intros/music or pauses).
+   * Returns an empty list on failure — silence alignment is best-effort.
+   */
+  private async detectSilences(
+    audioPath: string,
+    noiseDb: number = CourseAudioTranscriptionService.SUB_SILENCE_NOISE_DB,
+    minDuration: number = CourseAudioTranscriptionService.SUB_SILENCE_MIN_DURATION,
+  ): Promise<Array<{ start: number; end: number }>> {
+    try {
+      const { stderr } = await execFileAsync(
+        'ffmpeg',
+        [
+          '-i',
+          audioPath,
+          '-af',
+          `silencedetect=noise=${noiseDb}dB:d=${minDuration}`,
+          '-f',
+          'null',
+          '-',
+        ],
+        { maxBuffer: 1024 * 1024 * 20, windowsHide: true },
+      );
+
+      const silences: Array<{ start: number; end: number }> = [];
+      let curStart: number | null = null;
+      for (const line of String(stderr ?? '').split('\n')) {
+        const startMatch = line.match(/silence_start:\s*(-?[0-9.]+)/);
+        if (startMatch) {
+          curStart = Math.max(0, parseFloat(startMatch[1]));
+          continue;
+        }
+        const endMatch = line.match(/silence_end:\s*(-?[0-9.]+)/);
+        if (endMatch && curStart !== null) {
+          const end = parseFloat(endMatch[1]);
+          if (Number.isFinite(end) && end > curStart) {
+            silences.push({ start: curStart, end });
+          }
+          curStart = null;
+        }
+      }
+      return silences;
+    } catch (error) {
+      this.logger.warn(
+        `silencedetect failed, skipping start alignment: ${error instanceof Error ? error.message : String(error)}`,
+      );
+      return [];
+    }
+  }
+
+  /**
+   * Groups word-level timestamps into subtitle cues and adjusts their timing to
+   * the Netflix style guide. Text is stored on a single line; line breaking is a
+   * presentation concern handled at VTT generation (see subtitle.util.ts).
+   *
+   * `silences` (from detectSilences) is used to snap each cue's start to where
+   * speech actually begins, correcting Whisper's drift over intros/pauses.
+   */
+  private buildSubtitleSegments(
+    allWords: Array<{ word: string; start: number; end: number }>,
+    maxCps: number = CourseAudioTranscriptionService.SUB_MAX_CPS,
+    silences: Array<{ start: number; end: number }> = [],
+  ): Array<{ start_seconds: number; end_seconds: number; text: string }> {
+    const Svc = CourseAudioTranscriptionService;
+
+    // Phase 1 — group words into cues respecting char count, duration and pauses.
+    const cues: Array<Array<{ word: string; start: number; end: number }>> = [];
+    let buffer: Array<{ word: string; start: number; end: number }> = [];
+    let bufferText = '';
+
+    const flush = () => {
+      if (buffer.length === 0) return;
+      cues.push(buffer);
+      buffer = [];
+      bufferText = '';
+    };
+
+    for (const entry of allWords) {
+      const word = entry.word.trim();
+      if (!word) continue;
+
+      const prev = buffer[buffer.length - 1];
+      const gapFromPrev = prev ? entry.start - prev.end : 0;
+      const tentativeText = bufferText ? `${bufferText} ${word}` : word;
+      const tentativeDuration =
+        buffer.length > 0 ? entry.end - buffer[0].start : entry.end - entry.start;
+
+      if (
+        buffer.length > 0 &&
+        (!fitsInTwoLines(tentativeText, Svc.SUB_MAX_CHARS_PER_LINE) ||
+          tentativeDuration > Svc.SUB_MAX_DURATION ||
+          gapFromPrev >= Svc.SUB_PAUSE_SPLIT)
+      ) {
+        flush();
+      }
+
+      buffer.push(entry);
+      bufferText = bufferText ? `${bufferText} ${word}` : word;
+
+      // Break naturally at sentence-ending punctuation.
+      if (/[.!?…]$/.test(word)) {
+        flush();
+        continue;
+      }
+
+      // Soft break at clause punctuation once the cue is already substantial,
+      // so subtitles break at natural points (commas, pauses) instead of mid-clause.
+      if (/[,;:]$/.test(word) && bufferText.length >= Svc.SUB_SOFT_BREAK_MIN_CHARS) {
+        flush();
+      }
+    }
+    flush();
+
+    // Phase 1.5 — merge single-word cues that are not real pauses/emphasis.
+    const mergedCues = this.mergeLoneWordCues(cues);
+
+    // Phase 2 — build text and adjust display timing per cue.
+    const draft = mergedCues
+      .map((words) => ({
+        start: words[0].start,
+        endRaw: words[words.length - 1].end,
+        text: words.map((w) => w.word).join(' ').replace(/\s+/g, ' ').trim(),
+      }))
+      .filter((c) => c.text.length > 0);
+
+    const round = (n: number) => Math.max(0, Math.round(n * 1000) / 1000);
+
+    // Snap a start to the end of the silence it falls into (where speech begins).
+    // end_seconds is computed separately from `start`, so it will always be ≥ start.
+    const snapStart = (rawStart: number): number => {
+      for (const s of silences) {
+        if (rawStart >= s.start && rawStart < s.end) {
+          return s.end;
+        }
+      }
+      return rawStart;
+    };
+
+    const starts = draft.map((seg) => snapStart(seg.start));
+
+    return draft.map((seg, i) => {
+      const start = starts[i];
+
+      // Extend duration to satisfy minimum display time and reading speed (CPS),
+      // capped at the maximum duration.
+      let end = Math.max(
+        seg.endRaw,
+        start + Svc.SUB_MIN_DURATION,
+        start + seg.text.length / maxCps,
+      );
+      end = Math.min(end, start + Svc.SUB_MAX_DURATION);
+
+      // Never overlap the next cue (keep a minimum gap from its aligned start).
+      const nextStart = starts[i + 1];
+      if (nextStart !== undefined) {
+        const maxEnd = nextStart - Svc.SUB_MIN_GAP;
+        if (maxEnd > start && end > maxEnd) {
+          end = maxEnd;
+        }
+      }
+      if (end <= start) {
+        end = start + Svc.SUB_MIN_DURATION;
+      }
+
+      return {
+        start_seconds: round(start),
+        end_seconds: round(end),
+        text: seg.text,
+      };
+    });
+  }
+
+  /**
+   * Merges single-word cues into a neighbour when they are not separated by a
+   * real pause — avoiding isolated words on screen. A lone word kept apart by a
+   * pause (≥ SUB_PAUSE_SPLIT) is preserved, since it is a genuine pause/emphasis.
+   * Merges only happen when the combined cue still fits two lines and the max
+   * duration, so size/legibility rules are never violated.
+   */
+  private mergeLoneWordCues(
+    cues: Array<Array<{ word: string; start: number; end: number }>>,
+  ): Array<Array<{ word: string; start: number; end: number }>> {
+    const Svc = CourseAudioTranscriptionService;
+    const merged: Array<Array<{ word: string; start: number; end: number }>> = [];
+
+    for (const cue of cues) {
+      const prev = merged[merged.length - 1];
+      if (prev && (prev.length === 1 || cue.length === 1)) {
+        const gap = cue[0].start - prev[prev.length - 1].end;
+        const combined = [...prev, ...cue];
+        const combinedText = combined.map((w) => w.word).join(' ');
+        const combinedDuration = cue[cue.length - 1].end - prev[0].start;
+
+        if (
+          gap < Svc.SUB_PAUSE_SPLIT &&
+          combinedDuration <= Svc.SUB_MAX_DURATION &&
+          fitsInTwoLines(combinedText, Svc.SUB_MAX_CHARS_PER_LINE)
+        ) {
+          merged[merged.length - 1] = combined;
+          continue;
+        }
+      }
+      merged.push(cue);
+    }
+
+    return merged;
+  }
+
+  /**
+   * Transcribes a single audio chunk with the given model and returns the raw
+   * OpenAI response body. whisper-1 uses verbose_json with word/segment
+   * timestamps; the gpt-4o-transcribe family only supports json (text only).
+   */
+  private async transcribeChunk(
+    chunkBuffer: Buffer,
+    chunkName: string,
+    model: string,
+    apiKey: string,
+    language: string,
+  ): Promise<any> {
+    const formData = new FormData();
+    formData.append(
+      'file',
+      new Blob([new Uint8Array(chunkBuffer)], { type: 'audio/mpeg' }),
+      chunkName,
+    );
+    const isWhisper = model === 'whisper-1';
+    formData.append('model', model);
+    // Only whisper-1 supports verbose_json + timestamp_granularities; the
+    // gpt-4o-transcribe family returns 400 for anything other than json/text.
+    formData.append('response_format', isWhisper ? 'verbose_json' : 'json');
+    formData.append('language', language);
+    if (isWhisper) {
+      formData.append('timestamp_granularities[]', 'word');
+      formData.append('timestamp_granularities[]', 'segment');
+    }
+
+    const headers: Record<string, string> = {
+      Authorization: `Bearer ${apiKey}`,
+    };
+    const maybeHeaders = (formData as any).getHeaders?.();
+    if (maybeHeaders && typeof maybeHeaders === 'object') {
+      Object.assign(headers, maybeHeaders);
+    }
+
+    try {
+      const response = await axios.post(
+        'https://api.openai.com/v1/audio/transcriptions',
+        formData,
+        { headers },
+      );
+      return response.data;
+    } catch (axiosErr: any) {
+      const status = axiosErr?.response?.status;
+      const body = JSON.stringify(axiosErr?.response?.data ?? {});
+      this.logger.error(
+        `OpenAI transcription request failed — model=${model} language=${language} status=${status} body=${body}`,
+      );
+      throw axiosErr;
+    }
+  }
+
+  /**
+   * Converts a raw chunk transcription response into absolute-timed words,
+   * applying `offsetSeconds` (the chunk's position in the full audio):
+   *   - words[]    → real per-word timestamps (whisper-1)
+   *   - segments[] → synthesize even word timing within each segment (whisper-1
+   *                  verbose_json without word granularity)
+   *   - text only  → distribute words evenly across the chunk's speech window,
+   *                  skipping any leading silence detected by ffmpeg (gpt-4o)
+   */
+  private chunkResponseToWords(
+    data: any,
+    offsetSeconds: number,
+    silences: Array<{ start: number; end: number }>,
+  ): TimedWord[] {
+    const out: TimedWord[] = [];
+    const wordEntries: Array<{ word: string; start: number; end: number }> =
+      data?.words ?? [];
+
+    if (wordEntries.length > 0) {
+      for (const w of wordEntries) {
+        const wordText = String(w?.word ?? '').trim();
+        if (!wordText) continue;
+        out.push({
+          word: wordText,
+          start: offsetSeconds + Number(w?.start ?? 0),
+          end: offsetSeconds + Number(w?.end ?? 0),
+        });
+      }
+      return out;
+    }
+
+    const segments = data?.segments ?? [];
+    if (segments.length > 0) {
+      // whisper-1 verbose_json fallback: synthesize from segments
+      for (const segment of segments) {
+        const text = String(segment?.text ?? '').trim();
+        if (!text) continue;
+        const start = offsetSeconds + Number(segment?.start ?? 0);
+        const end = offsetSeconds + Number(segment?.end ?? 0);
+        const words = text.split(/\s+/).filter(Boolean);
+        const dur = Math.max(0, end - start);
+        const wordDur = words.length > 0 ? dur / words.length : 0;
+        for (let wi = 0; wi < words.length; wi++) {
+          out.push({
+            word: words[wi],
+            start: start + wi * wordDur,
+            end: start + (wi + 1) * wordDur,
+          });
+        }
+      }
+      return out;
+    }
+
+    // json-only models (gpt-4o-transcribe etc): distribute over the speech
+    // window of the chunk, skipping any leading silence detected by ffmpeg.
+    const fullText = String(data?.text ?? '').trim();
+    const words = fullText.split(/\s+/).filter(Boolean);
+    const chunkDur = 60;
+    const chunkSpeechOffset = silences
+      .filter(
+        (s) =>
+          s.start <= offsetSeconds + 1 &&
+          s.end > offsetSeconds &&
+          s.end <= offsetSeconds + chunkDur,
+      )
+      .reduce((max, s) => Math.max(max, s.end - offsetSeconds), 0);
+    const speechDur = Math.max(1, chunkDur - chunkSpeechOffset);
+    const wordDur = words.length > 0 ? speechDur / words.length : 0;
+    for (let wi = 0; wi < words.length; wi++) {
+      out.push({
+        word: words[wi],
+        start: offsetSeconds + chunkSpeechOffset + wi * wordDur,
+        end: offsetSeconds + chunkSpeechOffset + (wi + 1) * wordDur,
+      });
+    }
+    return out;
+  }
+
   async handle(job: {
     id: number;
     type: string;
@@ -157,8 +556,22 @@ export class CourseAudioTranscriptionService implements OnModuleInit, IJobHandle
     const settings = await this.settingService.getSettingValues([
       'ai-openai-profile-id',
       'lms-audio-transcription-enabled',
+      'lms-subtitle-reading-speed',
+      'lms-subtitle-silence-noise-db',
+      'lms-subtitle-silence-min-duration',
+      'lms-transcription-model',
     ]);
     const transcriptionEnabled = settings['lms-audio-transcription-enabled'] !== false;
+    const transcriptionModel = String(settings['lms-transcription-model'] ?? 'whisper-1');
+    // The model choice drives the mode: whisper-1 returns real timestamps, so it
+    // runs in a single pass. The gpt-4o models return no timestamps, so they run
+    // hybrid — whisper-1 supplies the timing and is merged onto the gpt-4o text.
+    const useHybrid = transcriptionModel !== 'whisper-1';
+    const readingSpeedCps = this.resolveReadingSpeedCps(settings['lms-subtitle-reading-speed']);
+    const silenceParams = this.resolveSilenceParams(
+      settings['lms-subtitle-silence-noise-db'],
+      settings['lms-subtitle-silence-min-duration'],
+    );
 
     if (!transcriptionEnabled) {
       const msg = `Transcrição desabilitada nas configurações do LMS (lms-audio-transcription-enabled=false).`;
@@ -199,7 +612,8 @@ export class CourseAudioTranscriptionService implements OnModuleInit, IJobHandle
       where: { id: courseId },
       include: { locale: true },
     });
-    const language = course?.locale?.code ?? 'pt';
+    const language = (course?.locale?.code ?? 'pt').split('-')[0];
+    const localeId: number | null = course?.locale_id ?? null;
 
     const tempDir = join(tmpdir(), `lms-transcribe-${lessonId}-${Date.now()}`);
     await fs.mkdir(tempDir, { recursive: true });
@@ -281,12 +695,15 @@ export class CourseAudioTranscriptionService implements OnModuleInit, IJobHandle
         chunks: chunks.length,
       });
 
-      const allSegments: Array<{
-        course_lesson_id: number;
-        start_seconds: number;
-        end_seconds: number;
-        text: string;
-      }> = [];
+      // Detect silence in the full normalized audio so subtitle starts can be
+      // aligned to where speech actually begins (intros, music, long pauses).
+      const silences = await this.detectSilences(
+        normalizedAudioPath,
+        silenceParams.noiseDb,
+        silenceParams.minDuration,
+      );
+
+      const allWords: Array<{ word: string; start: number; end: number }> = [];
 
       for (let i = 0; i < chunks.length; i += 1) {
         const chunkName = chunks[i];
@@ -305,44 +722,36 @@ export class CourseAudioTranscriptionService implements OnModuleInit, IJobHandle
         const offsetSeconds = i * 60;
         const chunkBuffer = await fs.readFile(chunkPath);
 
-        const formData = new FormData();
-        formData.append(
-          'file',
-          new Blob([chunkBuffer], { type: 'audio/mpeg' }),
-          chunkName,
-        );
-        formData.append('model', 'whisper-1');
-        formData.append('response_format', 'verbose_json');
-        formData.append('language', language);
-
-        const headers: Record<string, string> = {
-          Authorization: `Bearer ${apiKey}`,
-        };
-        const maybeHeaders = (formData as any).getHeaders?.();
-        if (maybeHeaders && typeof maybeHeaders === 'object') {
-          Object.assign(headers, maybeHeaders);
-        }
-
-        const response = await axios.post(
-          'https://api.openai.com/v1/audio/transcriptions',
-          formData,
-          { headers },
-        );
-
-        const segments = response.data?.segments ?? [];
-        for (const segment of segments) {
-          const text = String(segment?.text ?? '').trim();
-          if (!text) continue;
-
-          allSegments.push({
-            course_lesson_id: lessonId,
-            start_seconds: offsetSeconds + Number(segment?.start ?? 0),
-            end_seconds: offsetSeconds + Number(segment?.end ?? 0),
-            text,
-          });
+        if (useHybrid) {
+          // Hybrid: real timing from whisper-1 + premium text from the chosen
+          // model, merged by word alignment (see alignTextToTimings).
+          const [timingData, textData] = await Promise.all([
+            this.transcribeChunk(chunkBuffer, chunkName, 'whisper-1', apiKey, language),
+            this.transcribeChunk(chunkBuffer, chunkName, transcriptionModel, apiKey, language),
+          ]);
+          const timedWords = this.chunkResponseToWords(timingData, offsetSeconds, silences);
+          const cleanText = String(textData?.text ?? '').trim();
+          allWords.push(...alignTextToTimings(timedWords, cleanText));
+        } else {
+          const data = await this.transcribeChunk(
+            chunkBuffer,
+            chunkName,
+            transcriptionModel,
+            apiKey,
+            language,
+          );
+          allWords.push(...this.chunkResponseToWords(data, offsetSeconds, silences));
         }
       }
 
+      const allSegments = this.buildSubtitleSegments(allWords, readingSpeedCps, silences).map((seg) => ({
+        course_lesson_id: lessonId,
+        locale_id: localeId,
+        start_seconds: seg.start_seconds,
+        end_seconds: seg.end_seconds,
+        text: seg.text,
+      }));
+
       await emitProgress('Salvando transcrição gerada pela IA...', {
         phase: 'transcription_save',
         lessonId,
@@ -350,7 +759,7 @@ export class CourseAudioTranscriptionService implements OnModuleInit, IJobHandle
       });
       await this.prismaService.$transaction([
         (this.prismaService as any).course_lesson_transcription_segment.deleteMany({
-          where: { course_lesson_id: lessonId },
+          where: { course_lesson_id: lessonId, locale_id: localeId },
         }),
         (this.prismaService as any).course_lesson_transcription_segment.createMany({
           data: allSegments,
@@ -361,6 +770,25 @@ export class CourseAudioTranscriptionService implements OnModuleInit, IJobHandle
         `Transcription saved for lesson ${lessonId}: ${allSegments.length} segments`,
       );
 
+      // Record AI cost (transcription is billed per second of audio). Hybrid
+      // mode runs two passes (whisper-1 for timing + the chosen model for text),
+      // so both are charged.
+      const audioSeconds = allWords.length > 0 ? allWords[allWords.length - 1].end : 0;
+      await this.aiUsageService.recordTranscriptionUsage({
+        courseId,
+        lessonId,
+        model: transcriptionModel,
+        audioSeconds,
+      });
+      if (useHybrid) {
+        await this.aiUsageService.recordTranscriptionUsage({
+          courseId,
+          lessonId,
+          model: 'whisper-1',
+          audioSeconds,
+        });
+      }
+
       // Automatically trigger XP calculation after transcription completes.
       try {
         const existingMap = await (this.prismaService as any).lesson_xp_map.findUnique({