markupr 2.1.8

package/src/main/pipeline/PostProcessor.ts

@@ -0,0 +1,399 @@
+/**
+ * PostProcessor.ts - Post-Recording Pipeline Orchestrator
+ *
+ * Runs the full post-processing pipeline after a recording session stops:
+ *   1. Transcribe audio via WhisperService batch mode
+ *   2. Analyze transcript to find key moments (heuristic-based)
+ *   3. Extract video frames at those timestamps via ffmpeg
+ *   4. Return combined result for markdown report generation
+ *
+ * Designed to degrade gracefully: if frame extraction fails (e.g. ffmpeg
+ * not installed), the pipeline still returns a transcript-only result.
+ *
+ * Part of the post-processing pipeline that runs after recording stops.
+ */
+
+import { whisperService } from '../transcription/WhisperService';
+import { TranscriptAnalyzer, transcriptAnalyzer } from './TranscriptAnalyzer';
+import { FrameExtractor, frameExtractor } from './FrameExtractor';
+import type { KeyMoment } from './TranscriptAnalyzer';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface TranscriptSegment {
+  text: string;
+  startTime: number; // seconds from start of recording
+  endTime: number;
+  confidence: number;
+}
+
+export interface ExtractedFrame {
+  path: string; // path to extracted PNG
+  timestamp: number; // seconds from start of recording
+  reason: string; // why this frame was selected
+  transcriptSegment?: TranscriptSegment; // associated transcript segment
+}
+
+export interface PostProcessResult {
+  transcriptSegments: TranscriptSegment[];
+  extractedFrames: ExtractedFrame[];
+  reportPath: string;
+}
+
+export interface PostProcessProgress {
+  step: 'transcribing' | 'analyzing' | 'extracting-frames' | 'generating-report';
+  percent: number;
+  message: string;
+}
+
+export interface PostProcessOptions {
+  videoPath: string;
+  audioPath: string;
+  sessionDir: string;
+  transcriptSegments?: TranscriptSegment[];
+  aiMomentHints?: KeyMoment[];
+  onProgress?: (progress: PostProcessProgress) => void;
+}
+
+// ============================================================================
+// PostProcessor Class
+// ============================================================================
+
+export class PostProcessor {
+  private analyzer: TranscriptAnalyzer;
+  private extractor: FrameExtractor;
+
+  constructor(
+    analyzer: TranscriptAnalyzer = transcriptAnalyzer,
+    extractor: FrameExtractor = frameExtractor
+  ) {
+    this.analyzer = analyzer;
+    this.extractor = extractor;
+  }
+
+  /**
+   * Run the full post-processing pipeline.
+   *
+   * @param options - Pipeline configuration (paths, progress callback)
+   * @returns Combined result with transcript, frames, and report path
+   */
+  async process(options: PostProcessOptions): Promise<PostProcessResult> {
+    const { videoPath, audioPath, sessionDir, transcriptSegments, aiMomentHints, onProgress } = options;
+
+    const emitProgress = (progress: PostProcessProgress): void => {
+      if (onProgress) {
+        onProgress(progress);
+      }
+    };
+
+    // -----------------------------------------------------------------------
+    // Step 1: Transcribe audio (0-40%)
+    // -----------------------------------------------------------------------
+    emitProgress({
+      step: 'transcribing',
+      percent: 0,
+      message: 'Transcribing audio...',
+    });
+
+    let segments: TranscriptSegment[] = [];
+    const providedSegments = (transcriptSegments || [])
+      .map((segment) => ({
+        text: segment.text?.trim() || '',
+        startTime: Number.isFinite(segment.startTime) ? Math.max(0, segment.startTime) : 0,
+        endTime: Number.isFinite(segment.endTime)
+          ? Math.max(0, segment.endTime)
+          : (Number.isFinite(segment.startTime) ? Math.max(0, segment.startTime) + 1.5 : 1.5),
+        confidence: Number.isFinite(segment.confidence) ? segment.confidence : 0.8,
+      }))
+      .filter((segment) => segment.text.length > 0);
+
+    // Strategy: prefer Whisper file-based transcription when available because
+    // it produces higher quality output than the live-streamed segments captured
+    // during recording. Fall back to pre-provided segments only when Whisper is
+    // unavailable or fails.
+    const whisperAvailable = audioPath && whisperService.isModelAvailable();
+
+    if (whisperAvailable) {
+      this.log(
+        `Whisper model available, attempting file-based transcription` +
+        (providedSegments.length > 0
+          ? ` (${providedSegments.length} pre-provided segments available as fallback)`
+          : '')
+      );
+      try {
+        emitProgress({
+          step: 'transcribing',
+          percent: 5,
+          message: 'Loading Whisper model...',
+        });
+
+        const whisperResults = await whisperService.transcribeFile(audioPath);
+
+        segments = whisperResults.map((result) => ({
+          text: result.text,
+          startTime: result.startTime,
+          endTime: result.endTime,
+          confidence: result.confidence,
+        }));
+
+        if (segments.length > 0) {
+          emitProgress({
+            step: 'transcribing',
+            percent: 40,
+            message: `Whisper transcription complete: ${segments.length} segments`,
+          });
+          this.log(`Whisper file transcription complete: ${segments.length} segments`);
+        } else {
+          this.log('Whisper returned 0 segments, will try pre-provided segments as fallback');
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        this.log(`Whisper file transcription failed: ${message}`);
+        // Fall through to use pre-provided segments below
+      }
+    } else {
+      this.log(
+        `Whisper file transcription skipped: ${!audioPath ? 'no audio file path' : 'model not available'}`
+      );
+    }
+
+    // Fall back to pre-provided segments if Whisper did not produce results
+    if (segments.length === 0 && providedSegments.length > 0) {
+      segments = providedSegments.sort((a, b) => a.startTime - b.startTime);
+      emitProgress({
+        step: 'transcribing',
+        percent: 40,
+        message: `Using captured transcript (${segments.length} segments)`,
+      });
+      this.log(`Using pre-provided transcript segments as fallback: ${segments.length}`);
+    }
+
+    // If neither Whisper nor pre-provided segments produced anything, and we
+    // have an audio path but Whisper was not available, attempt Whisper anyway
+    // as a last resort (it will throw if the model truly cannot load).
+    if (segments.length === 0 && audioPath && !whisperAvailable) {
+      try {
+        emitProgress({
+          step: 'transcribing',
+          percent: 5,
+          message: 'Attempting transcription...',
+        });
+
+        const whisperResults = await whisperService.transcribeFile(audioPath);
+        segments = whisperResults.map((result) => ({
+          text: result.text,
+          startTime: result.startTime,
+          endTime: result.endTime,
+          confidence: result.confidence,
+        }));
+
+        if (segments.length > 0) {
+          emitProgress({
+            step: 'transcribing',
+            percent: 40,
+            message: `Transcription complete: ${segments.length} segments`,
+          });
+          this.log(`Whisper last-resort transcription complete: ${segments.length} segments`);
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        this.log(`Whisper last-resort transcription failed: ${message}`);
+
+        // No segments from any source - return empty result
+        return {
+          transcriptSegments: [],
+          extractedFrames: [],
+          reportPath: sessionDir,
+        };
+      }
+    }
+
+    if (segments.length === 0) {
+      this.log('No transcript segments found, returning empty result');
+      emitProgress({
+        step: 'generating-report',
+        percent: 100,
+        message: 'No speech detected in recording',
+      });
+
+      return {
+        transcriptSegments: [],
+        extractedFrames: [],
+        reportPath: sessionDir,
+      };
+    }
+
+    // -----------------------------------------------------------------------
+    // Step 2: Analyze transcript for key moments (40-50%)
+    // -----------------------------------------------------------------------
+    emitProgress({
+      step: 'analyzing',
+      percent: 40,
+      message: 'Analyzing transcript for key moments...',
+    });
+
+    const keyMoments = this.analyzer.analyze(segments, aiMomentHints || []);
+
+    emitProgress({
+      step: 'analyzing',
+      percent: 50,
+      message: `Found ${keyMoments.length} key moments`,
+    });
+
+    this.log(`Analysis complete: ${keyMoments.length} key moments identified`);
+
+    // -----------------------------------------------------------------------
+    // Step 3: Extract frames from video (50-90%)
+    // -----------------------------------------------------------------------
+    emitProgress({
+      step: 'extracting-frames',
+      percent: 50,
+      message: 'Extracting video frames...',
+    });
+
+    let extractedFrames: ExtractedFrame[] = [];
+
+    try {
+      const timestamps = keyMoments.map((m) => m.timestamp);
+
+      const extractionResult = await this.extractor.extract({
+        videoPath,
+        timestamps,
+        outputDir: sessionDir,
+      });
+
+      if (!extractionResult.ffmpegAvailable) {
+        this.log('ffmpeg not available, skipping frame extraction');
+        emitProgress({
+          step: 'extracting-frames',
+          percent: 90,
+          message: 'Frame extraction skipped (ffmpeg not installed)',
+        });
+      } else {
+        // Map extraction results to ExtractedFrame with associated transcript segments
+        extractedFrames = extractionResult.frames
+          .filter((f) => f.success)
+          .map((frame) => {
+            // Find the key moment that corresponds to this frame
+            const moment = keyMoments.find(
+              (m) => Math.abs(m.timestamp - frame.timestamp) < 0.5
+            );
+
+            // Find the closest transcript segment
+            const closestSegment = this.findClosestSegment(
+              frame.timestamp,
+              segments
+            );
+
+            return {
+              path: frame.path,
+              timestamp: frame.timestamp,
+              reason: moment?.reason ?? 'Extracted frame',
+              transcriptSegment: closestSegment,
+            };
+          });
+
+        emitProgress({
+          step: 'extracting-frames',
+          percent: 90,
+          message: `Extracted ${extractedFrames.length} frames`,
+        });
+
+        this.log(`Frame extraction complete: ${extractedFrames.length} frames`);
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.log(`Frame extraction failed: ${message} - continuing with transcript only`);
+
+      emitProgress({
+        step: 'extracting-frames',
+        percent: 90,
+        message: 'Frame extraction failed, continuing with transcript only',
+      });
+
+      // Continue with transcript-only result - do not throw
+    }
+
+    // -----------------------------------------------------------------------
+    // Step 4: Return result (90-100%)
+    // -----------------------------------------------------------------------
+    emitProgress({
+      step: 'generating-report',
+      percent: 90,
+      message: 'Preparing results...',
+    });
+
+    const result: PostProcessResult = {
+      transcriptSegments: segments,
+      extractedFrames,
+      reportPath: sessionDir,
+    };
+
+    emitProgress({
+      step: 'generating-report',
+      percent: 100,
+      message: 'Post-processing complete',
+    });
+
+    this.log(
+      `Pipeline complete: ${segments.length} segments, ${extractedFrames.length} frames`
+    );
+
+    return result;
+  }
+
+  // ============================================================================
+  // Private Methods
+  // ============================================================================
+
+  /**
+   * Find the transcript segment closest to a given timestamp.
+   * Prefers segments that contain the timestamp; falls back to nearest start time.
+   */
+  private findClosestSegment(
+    timestamp: number,
+    segments: TranscriptSegment[]
+  ): TranscriptSegment | undefined {
+    if (segments.length === 0) {
+      return undefined;
+    }
+
+    // First, check if timestamp falls within any segment
+    for (const segment of segments) {
+      if (timestamp >= segment.startTime && timestamp <= segment.endTime) {
+        return segment;
+      }
+    }
+
+    // Otherwise, find the closest segment by start time
+    let closest = segments[0];
+    let minDistance = Math.abs(timestamp - closest.startTime);
+
+    for (let i = 1; i < segments.length; i++) {
+      const distance = Math.abs(timestamp - segments[i].startTime);
+      if (distance < minDistance) {
+        minDistance = distance;
+        closest = segments[i];
+      }
+    }
+
+    return closest;
+  }
+
+  /**
+   * Log helper with consistent prefix.
+   */
+  private log(message: string): void {
+    const timestamp = new Date().toISOString();
+    console.log(`[PostProcessor ${timestamp}] ${message}`);
+  }
+}
+
+// ============================================================================
+// Singleton Export
+// ============================================================================
+
+export const postProcessor = new PostProcessor();
+export default PostProcessor;

package/src/main/pipeline/TranscriptAnalyzer.ts

@@ -0,0 +1,226 @@
+/**
+ * TranscriptAnalyzer.ts - Heuristic Key Moment Detection
+ *
+ * Analyzes transcript segments to identify moments where video frames
+ * should be extracted. Uses local heuristics (no AI required):
+ *
+ * - Natural pauses > 1.5s between segments
+ * - Periodic baseline captures every 15-20 seconds
+ * - Session start and end always included
+ *
+ * Part of the post-processing pipeline that runs after recording stops.
+ */
+
+import type { TranscriptSegment } from './PostProcessor';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface KeyMoment {
+  timestamp: number; // seconds from start of recording
+  reason: string; // human-readable reason for selection
+  confidence: number; // 0-1
+}
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/** Minimum gap between segments to count as a natural pause */
+const PAUSE_THRESHOLD_SECONDS = 1.5;
+
+/** Baseline periodic capture interval when no pauses are found */
+const PERIODIC_INTERVAL_SECONDS = 15;
+
+/** Maximum periodic interval to avoid sparse captures */
+const MAX_PERIODIC_INTERVAL_SECONDS = 20;
+
+/** Hard cap on returned key moments */
+const MAX_KEY_MOMENTS = 20;
+
+// ============================================================================
+// TranscriptAnalyzer Class
+// ============================================================================
+
+export class TranscriptAnalyzer {
+  /**
+   * Analyze transcript segments and return key moments where frames
+   * should be extracted from the video recording.
+   *
+   * @param segments - Array of transcript segments with timing info
+   * @param aiHints - Optional AI-informed key-moment hints to merge
+   * @returns Array of key moments sorted by timestamp, capped at 20
+   */
+  analyze(segments: TranscriptSegment[], aiHints: KeyMoment[] = []): KeyMoment[] {
+    if (segments.length === 0) {
+      return [];
+    }
+
+    const moments: KeyMoment[] = [];
+
+    // Always include session start
+    const firstSegment = segments[0];
+    moments.push({
+      timestamp: firstSegment.startTime,
+      reason: 'Session start',
+      confidence: 1.0,
+    });
+
+    // Detect natural pauses between segments
+    for (let i = 1; i < segments.length; i++) {
+      const prev = segments[i - 1];
+      const curr = segments[i];
+      const gap = curr.startTime - prev.endTime;
+
+      if (gap >= PAUSE_THRESHOLD_SECONDS) {
+        // Place the key moment at the start of the pause (end of previous segment)
+        moments.push({
+          timestamp: prev.endTime,
+          reason: 'Natural pause in narration',
+          confidence: Math.min(1.0, gap / 3.0), // Longer pauses = higher confidence
+        });
+      }
+    }
+
+    // Always include session end
+    const lastSegment = segments[segments.length - 1];
+    if (lastSegment.endTime !== firstSegment.startTime) {
+      moments.push({
+        timestamp: lastSegment.endTime,
+        reason: 'Session end',
+        confidence: 1.0,
+      });
+    }
+
+    // If we found fewer than 3 moments (just start/end), add periodic baseline captures.
+    // When AI hints exist, prefer them over periodic filler captures.
+    if (moments.length < 3 && aiHints.length === 0) {
+      const sessionStart = firstSegment.startTime;
+      const sessionEnd = lastSegment.endTime;
+      const sessionDuration = sessionEnd - sessionStart;
+
+      if (sessionDuration > PERIODIC_INTERVAL_SECONDS) {
+        // Calculate interval: target 15s but stretch up to 20s to avoid one extra capture
+        const rawCount = Math.floor(sessionDuration / PERIODIC_INTERVAL_SECONDS);
+        const interval = Math.min(
+          sessionDuration / rawCount,
+          MAX_PERIODIC_INTERVAL_SECONDS
+        );
+
+        for (
+          let t = sessionStart + interval;
+          t < sessionEnd;
+          t += interval
+        ) {
+          moments.push({
+            timestamp: t,
+            reason: 'Periodic capture',
+            confidence: 0.5,
+          });
+        }
+      }
+    }
+
+    // Merge AI hints (when available) so frame extraction can prioritize
+    // narration moments that the analysis pipeline considered important.
+    for (const hint of aiHints) {
+      if (!Number.isFinite(hint.timestamp)) {
+        continue;
+      }
+      moments.push({
+        timestamp: Math.max(0, hint.timestamp),
+        reason: hint.reason?.trim() || 'AI-highlighted context',
+        confidence: Math.max(0, Math.min(1, Number.isFinite(hint.confidence) ? hint.confidence : 0.8)),
+      });
+    }
+
+    // Deduplicate moments that are very close together (within 1 second)
+    const deduped = this.deduplicateMoments(moments);
+
+    // Sort by timestamp
+    deduped.sort((a, b) => a.timestamp - b.timestamp);
+
+    // Cap at MAX_KEY_MOMENTS, keeping highest confidence ones
+    if (deduped.length > MAX_KEY_MOMENTS) {
+      // Always keep first and last; rank the rest by confidence
+      const first = deduped[0];
+      const last = deduped[deduped.length - 1];
+      const middle = deduped
+        .slice(1, -1)
+        .sort((a, b) => {
+          const priorityDelta = this.momentPriority(b) - this.momentPriority(a);
+          if (priorityDelta !== 0) {
+            return priorityDelta;
+          }
+          return b.confidence - a.confidence;
+        })
+        .slice(0, MAX_KEY_MOMENTS - 2);
+
+      const capped = [first, ...middle, last];
+      capped.sort((a, b) => a.timestamp - b.timestamp);
+      return capped;
+    }
+
+    return deduped;
+  }
+
+  /**
+   * Remove moments that are within 1 second of each other,
+   * keeping the one with higher confidence.
+   */
+  private deduplicateMoments(moments: KeyMoment[]): KeyMoment[] {
+    if (moments.length <= 1) {
+      return moments;
+    }
+
+    // Sort by timestamp first for grouping
+    const sorted = [...moments].sort((a, b) => a.timestamp - b.timestamp);
+    const result: KeyMoment[] = [sorted[0]];
+
+    for (let i = 1; i < sorted.length; i++) {
+      const prev = result[result.length - 1];
+      const curr = sorted[i];
+
+      if (curr.timestamp - prev.timestamp < 1.0) {
+        // Prefer higher-priority moments (AI / semantic captures over periodic).
+        const currPriority = this.momentPriority(curr);
+        const prevPriority = this.momentPriority(prev);
+        if (
+          currPriority > prevPriority ||
+          (currPriority === prevPriority && curr.confidence > prev.confidence)
+        ) {
+          result[result.length - 1] = curr;
+        }
+      } else {
+        result.push(curr);
+      }
+    }
+
+    return result;
+  }
+
+  private momentPriority(moment: KeyMoment): number {
+    const reason = (moment.reason || '').toLowerCase();
+    if (reason.includes('session start') || reason.includes('session end')) {
+      return 4;
+    }
+    if (reason.includes('ai-') || reason.includes('ai ')) {
+      return 3;
+    }
+    if (reason.includes('natural pause')) {
+      return 2;
+    }
+    if (reason.includes('periodic')) {
+      return 0;
+    }
+    return 1;
+  }
+}
+
+// ============================================================================
+// Singleton Export
+// ============================================================================
+
+export const transcriptAnalyzer = new TranscriptAnalyzer();
+export default TranscriptAnalyzer;

package/src/main/pipeline/index.ts

@@ -0,0 +1,36 @@
+/**
+ * Pipeline Module - Post-Processing Pipeline
+ *
+ * After recording stops, this pipeline:
+ *   1. Transcribes audio via WhisperService (batch mode)
+ *   2. Analyzes transcript for key moments (heuristic-based)
+ *   3. Extracts video frames at those timestamps (via ffmpeg)
+ *   4. Returns structured data for markdown report generation
+ */
+
+// ============================================================================
+// Classes & Singletons
+// ============================================================================
+
+export { PostProcessor, postProcessor } from './PostProcessor';
+export { FrameExtractor, frameExtractor } from './FrameExtractor';
+export { TranscriptAnalyzer, transcriptAnalyzer } from './TranscriptAnalyzer';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export type {
+  PostProcessResult,
+  TranscriptSegment,
+  ExtractedFrame,
+  PostProcessProgress,
+  PostProcessOptions,
+} from './PostProcessor';
+
+export type {
+  FrameExtractionRequest,
+  FrameExtractionResult,
+} from './FrameExtractor';
+
+export type { KeyMoment } from './TranscriptAnalyzer';