markupr 2.1.8

package/src/main/ai/ImageOptimizer.ts

@@ -0,0 +1,176 @@
+/**
+ * ImageOptimizer - Optimize screenshots for the Claude Vision API
+ *
+ * Resizes images to Claude's recommended max dimension (1568px) and converts
+ * large PNGs to JPEG to reduce token usage. Uses Electron's built-in nativeImage
+ * so no external dependencies are needed.
+ */
+
+import { nativeImage } from 'electron';
+import type { Screenshot } from '../SessionController';
+import type { OptimizedImage, ImageOptimizeOptions } from './types';
+import { DEFAULT_IMAGE_OPTIMIZE_OPTIONS, AIPipelineError } from './types';
+
+/**
+ * Optimize a single screenshot buffer for the Claude Vision API.
+ *
+ * 1. Resize to fit within maxWidth (maintaining aspect ratio)
+ * 2. If the PNG exceeds pngToJpegThreshold, convert to JPEG at jpegQuality
+ */
+export function optimizeScreenshot(
+  buffer: Buffer,
+  width: number,
+  height: number,
+  options: ImageOptimizeOptions = DEFAULT_IMAGE_OPTIMIZE_OPTIONS,
+): OptimizedImage & { _id: string } {
+  const image = nativeImage.createFromBuffer(buffer);
+
+  if (image.isEmpty()) {
+    throw new AIPipelineError(
+      'Failed to create nativeImage from screenshot buffer',
+      'IMAGE_OPTIMIZATION_FAILED',
+    );
+  }
+
+  // Resize if wider than maxWidth (maintain aspect ratio)
+  let resized = image;
+  let finalWidth = width;
+  let finalHeight = height;
+
+  if (width > options.maxWidth) {
+    const scale = options.maxWidth / width;
+    finalWidth = Math.round(width * scale);
+    finalHeight = Math.round(height * scale);
+    resized = image.resize({ width: finalWidth, height: finalHeight, quality: 'best' });
+  }
+
+  // Start with PNG
+  let data = resized.toPNG();
+  let mediaType: 'image/png' | 'image/jpeg' = 'image/png';
+
+  // Convert to JPEG if PNG exceeds threshold
+  if (data.length > options.pngToJpegThreshold) {
+    data = resized.toJPEG(options.jpegQuality);
+    mediaType = 'image/jpeg';
+  }
+
+  return {
+    data,
+    mediaType,
+    originalScreenshotId: '', // Caller sets this
+    width: finalWidth,
+    height: finalHeight,
+    _id: '', // Caller sets this
+  };
+}
+
+/**
+ * Batch-optimize screenshots for the Claude Vision API.
+ *
+ * - Processes all screenshots through resize/compress
+ * - If more than maxScreenshots exist, selects an evenly-spaced subset
+ *   prioritizing manual/voice-command triggered screenshots
+ * - Returns optimized images ready for the API content blocks
+ */
+export function optimizeForAPI(
+  screenshots: Screenshot[],
+  options: Partial<ImageOptimizeOptions> = {},
+): OptimizedImage[] {
+  const opts: ImageOptimizeOptions = { ...DEFAULT_IMAGE_OPTIMIZE_OPTIONS, ...options };
+
+  if (screenshots.length === 0) {
+    return [];
+  }
+
+  // Select which screenshots to include
+  const selected = selectScreenshots(screenshots, opts.maxScreenshots);
+
+  // Optimize each selected screenshot
+  const optimized: OptimizedImage[] = [];
+
+  for (const screenshot of selected) {
+    try {
+      const result = optimizeScreenshot(
+        screenshot.buffer,
+        screenshot.width,
+        screenshot.height,
+        opts,
+      );
+
+      optimized.push({
+        data: result.data,
+        mediaType: result.mediaType,
+        originalScreenshotId: screenshot.id,
+        width: result.width,
+        height: result.height,
+      });
+    } catch (error) {
+      // Skip screenshots that fail to optimize (per design doc: "If still fails, skip that screenshot")
+      console.warn(
+        `[ImageOptimizer] Failed to optimize screenshot ${screenshot.id}, skipping:`,
+        error instanceof Error ? error.message : error,
+      );
+    }
+  }
+
+  return optimized;
+}
+
+/**
+ * Select which screenshots to send when the count exceeds the maximum.
+ *
+ * Strategy (from AI_PIPELINE_DESIGN.md):
+ * 1. Always include manual/voice-command triggered screenshots
+ * 2. From remaining pause-triggered ones, select evenly spaced to stay under max
+ */
+function selectScreenshots(screenshots: Screenshot[], max: number): Screenshot[] {
+  if (screenshots.length <= max) {
+    return screenshots;
+  }
+
+  // Separate priority screenshots (manual, voice-command) from pause-triggered
+  const priority: Screenshot[] = [];
+  const pauseTriggered: Screenshot[] = [];
+
+  for (const s of screenshots) {
+    if (s.trigger === 'manual' || s.trigger === 'voice-command') {
+      priority.push(s);
+    } else {
+      pauseTriggered.push(s);
+    }
+  }
+
+  // If priority screenshots alone exceed max, take evenly spaced from priority
+  if (priority.length >= max) {
+    return evenlySpaced(priority, max);
+  }
+
+  // Fill remaining slots with evenly spaced pause-triggered screenshots
+  const remainingSlots = max - priority.length;
+  const selectedPause = evenlySpaced(pauseTriggered, remainingSlots);
+
+  // Merge and sort by timestamp to maintain chronological order
+  const merged = [...priority, ...selectedPause];
+  merged.sort((a, b) => a.timestamp - b.timestamp);
+
+  return merged;
+}
+
+/**
+ * Select evenly spaced items from an array.
+ */
+function evenlySpaced<T>(items: T[], count: number): T[] {
+  if (items.length <= count) {
+    return items;
+  }
+
+  const result: T[] = [];
+  const step = (items.length - 1) / (count - 1);
+
+  for (let i = 0; i < count; i++) {
+    const index = Math.round(i * step);
+    result.push(items[index]);
+  }
+
+  return result;
+}

package/src/main/ai/StructuredMarkdownBuilder.ts

@@ -0,0 +1,379 @@
+/**
+ * StructuredMarkdownBuilder
+ *
+ * Converts Claude's structured JSON analysis (AIAnalysisResult) into polished,
+ * actionable markdown documents. The output is consumed by:
+ * 1. Claude Code via /review-feedback skill (primary consumer)
+ * 2. Users reading in GitHub or any markdown viewer
+ * 3. Other AI coding agents
+ *
+ * Maintains compatibility with the existing MarkdownGenerator's FB-XXX ID scheme
+ * and screenshot reference patterns.
+ */
+
+import type {
+  AIAnalysisResult,
+  AIFeedbackItem,
+  AIFeedbackPriority,
+  Session,
+  Screenshot,
+} from './types';
+
+// =============================================================================
+// Builder Options
+// =============================================================================
+
+export interface MarkdownBuildOptions {
+  /** Project name shown in the report header. */
+  projectName: string;
+  /** Relative path for screenshot references (e.g., "./screenshots"). */
+  screenshotDir: string;
+  /** Whether a session recording video exists. */
+  hasRecording?: boolean;
+  /** Filename of the session recording (default: "session-recording.webm"). */
+  recordingFilename?: string;
+  /** Claude model ID used for analysis (shown in footer). */
+  modelId?: string;
+}
+
+// =============================================================================
+// StructuredMarkdownBuilder
+// =============================================================================
+
+export class StructuredMarkdownBuilder {
+  /**
+   * Build a complete AI-enhanced markdown document from session data and Claude's analysis.
+   */
+  buildDocument(
+    session: Session,
+    aiResult: AIAnalysisResult,
+    options: MarkdownBuildOptions,
+  ): string {
+    const sections: string[] = [];
+
+    sections.push(this.buildHeader(session, aiResult, options));
+    sections.push(this.buildSummary(aiResult));
+    sections.push(this.buildFeedbackItems(session, aiResult, options));
+
+    if (aiResult.themes.length > 0) {
+      sections.push(this.buildThemes(aiResult.themes));
+    }
+
+    if (aiResult.positiveNotes.length > 0) {
+      sections.push(this.buildPositiveNotes(aiResult.positiveNotes));
+    }
+
+    if (options.hasRecording) {
+      sections.push(this.buildRecordingSection(options));
+    }
+
+    sections.push(this.buildSessionInfo(session, aiResult, options));
+    sections.push(this.buildFooter(options));
+
+    return sections.join('\n');
+  }
+
+  // ===========================================================================
+  // Section Builders
+  // ===========================================================================
+
+  private buildHeader(
+    session: Session,
+    aiResult: AIAnalysisResult,
+    options: MarkdownBuildOptions,
+  ): string {
+    const date = this.formatDate(session.endTime ?? Date.now());
+    const duration = session.endTime
+      ? this.formatDuration(session.endTime - session.startTime)
+      : 'In Progress';
+    const screenshotCount = session.screenshotBuffer.length;
+
+    return [
+      `# Feedback Report: ${options.projectName} - ${date}`,
+      '',
+      `> AI-analyzed by Claude | Duration: ${duration} | ${screenshotCount} screenshots | ${aiResult.items.length} items identified`,
+      '',
+    ].join('\n');
+  }
+
+  private buildSummary(aiResult: AIAnalysisResult): string {
+    return [
+      '## Summary',
+      '',
+      aiResult.summary,
+      '',
+      '---',
+      '',
+    ].join('\n');
+  }
+
+  private buildFeedbackItems(
+    session: Session,
+    aiResult: AIAnalysisResult,
+    options: MarkdownBuildOptions,
+  ): string {
+    const grouped = this.groupByPriority(aiResult.items);
+    const lines: string[] = [];
+
+    const priorityOrder: AIFeedbackPriority[] = ['Critical', 'High', 'Medium', 'Low'];
+    const sectionTitles: Record<AIFeedbackPriority, string> = {
+      Critical: 'Critical Issues',
+      High: 'High Priority',
+      Medium: 'Improvements Needed',
+      Low: 'Suggestions',
+    };
+
+    let sequentialIndex = 0;
+    for (const priority of priorityOrder) {
+      const items = grouped[priority];
+      if (!items || items.length === 0) continue;
+
+      lines.push(`## ${sectionTitles[priority]}`);
+      lines.push('');
+
+      for (const { item } of items) {
+        lines.push(
+          this.buildSingleItem(item, sequentialIndex, session, options),
+        );
+        sequentialIndex++;
+      }
+    }
+
+    return lines.join('\n');
+  }
+
+  private buildSingleItem(
+    item: AIFeedbackItem,
+    index: number,
+    session: Session,
+    options: MarkdownBuildOptions,
+  ): string {
+    const timestamp = this.estimateItemTimestamp(item, session);
+    const lines: string[] = [];
+
+    lines.push(`### ${index + 1}. ${item.title}`);
+    lines.push(`> "${item.quote}"`);
+    lines.push('');
+
+    // Screenshot references
+    for (const ssIndex of item.screenshotIndices) {
+      const screenshotFilename = this.resolveScreenshotFilename(session, ssIndex);
+      if (!screenshotFilename) {
+        continue;
+      }
+      lines.push(`![screenshot-${ssIndex + 1}](${options.screenshotDir}/${screenshotFilename})`);
+    }
+
+    if (item.screenshotIndices.length > 0) {
+      lines.push('');
+    }
+
+    lines.push(`- **Priority:** ${item.priority}`);
+    lines.push(`- **Category:** ${item.category}`);
+    lines.push(`- **Area:** ${item.area}`);
+    lines.push(`- **Timestamp:** ${timestamp}`);
+    lines.push(
+      `- **Action:** ${item.actionItem}`,
+    );
+    lines.push('');
+    lines.push('---');
+    lines.push('');
+
+    return lines.join('\n');
+  }
+
+  private resolveScreenshotFilename(session: Session, screenshotIndex: number): string | null {
+    if (!Number.isInteger(screenshotIndex) || screenshotIndex < 0) {
+      return null;
+    }
+
+    const screenshot = session.screenshotBuffer[screenshotIndex];
+    if (!screenshot) {
+      return null;
+    }
+
+    const matchedFeedbackIndex = session.feedbackItems.findIndex(
+      (item) => item.screenshot?.id === screenshot.id,
+    );
+    const index = matchedFeedbackIndex >= 0 ? matchedFeedbackIndex : screenshotIndex;
+    return `fb-${(index + 1).toString().padStart(3, '0')}.png`;
+  }
+
+  private buildThemes(themes: string[]): string {
+    const lines: string[] = [
+      '## Themes',
+      '',
+    ];
+
+    for (const theme of themes) {
+      lines.push(`- ${theme}`);
+    }
+
+    lines.push('');
+    lines.push('---');
+    lines.push('');
+    return lines.join('\n');
+  }
+
+  private buildPositiveNotes(notes: string[]): string {
+    const lines: string[] = [
+      '## Positive Notes',
+      '',
+    ];
+
+    for (const note of notes) {
+      lines.push(`- ${note}`);
+    }
+
+    lines.push('');
+    lines.push('---');
+    lines.push('');
+    return lines.join('\n');
+  }
+
+  private buildRecordingSection(options: MarkdownBuildOptions): string {
+    const filename = options.recordingFilename ?? 'session-recording.webm';
+    return [
+      '## Session Recording',
+      '',
+      `[Watch full session recording](./${filename})`,
+      '',
+      '---',
+      '',
+    ].join('\n');
+  }
+
+  private buildSessionInfo(
+    session: Session,
+    aiResult: AIAnalysisResult,
+    options: MarkdownBuildOptions,
+  ): string {
+    const duration = session.endTime
+      ? this.formatDuration(session.endTime - session.startTime)
+      : 'In Progress';
+    const screenshotCount = session.screenshotBuffer.length;
+    const modelLabel = options.modelId ?? 'Claude';
+
+    const lines: string[] = [
+      '## Session Info',
+      '',
+      `- **Session ID:** \`${session.id}\``,
+      `- **Source:** ${session.metadata.sourceName ?? 'Unknown'} (${session.metadata.windowTitle ? 'window' : 'screen'})`,
+      `- **Duration:** ${duration}`,
+      `- **Screenshots:** ${screenshotCount}`,
+    ];
+
+    if (options.hasRecording) {
+      const filename = options.recordingFilename ?? 'session-recording.webm';
+      lines.push(`- **Recording:** [${filename}](./${filename})`);
+    }
+
+    lines.push(`- **Analysis:** ${modelLabel} (AI-enhanced)`);
+    lines.push(`- **Items:** ${aiResult.metadata.totalItems} total, ${aiResult.metadata.criticalCount} critical, ${aiResult.metadata.highCount} high`);
+    lines.push('');
+    lines.push('---');
+    lines.push('');
+    return lines.join('\n');
+  }
+
+  private buildFooter(options: MarkdownBuildOptions): string {
+    const model = options.modelId ? ` with ${options.modelId}` : ' with AI analysis';
+    return `*Generated by [markupr](https://markupr.com)${model}*\n*If this report saved you time, support development: [Ko-fi](https://ko-fi.com/eddiesanjuan)*\n`;
+  }
+
+  // ===========================================================================
+  // Helpers
+  // ===========================================================================
+
+  /**
+   * Group items by priority, preserving original indices for FB-XXX IDs.
+   */
+  private groupByPriority(
+    items: AIFeedbackItem[],
+  ): Record<AIFeedbackPriority, { item: AIFeedbackItem; originalIndex: number }[]> {
+    const grouped: Record<AIFeedbackPriority, { item: AIFeedbackItem; originalIndex: number }[]> = {
+      Critical: [],
+      High: [],
+      Medium: [],
+      Low: [],
+    };
+
+    items.forEach((item, index) => {
+      grouped[item.priority].push({ item, originalIndex: index });
+    });
+
+    return grouped;
+  }
+
+  /**
+   * Format a 0-based index as FB-001, FB-002, etc.
+   */
+  formatItemId(index: number): string {
+    return `FB-${(index + 1).toString().padStart(3, '0')}`;
+  }
+
+  /**
+   * Estimate item timestamp as MM:SS relative to session start.
+   * Uses the quote text to find a matching transcript segment, or falls back to item order.
+   */
+  private estimateItemTimestamp(item: AIFeedbackItem, session: Session): string {
+    // Try to match transcript buffer by quote text
+    const quotePrefix = item.quote.slice(0, 30);
+    const matchingSegment = session.transcriptBuffer.find(
+      (seg) => seg.text.includes(quotePrefix),
+    );
+
+    if (matchingSegment) {
+      const relativeMs = matchingSegment.timestamp * 1000 - session.startTime;
+      return this.formatTimestamp(relativeMs);
+    }
+
+    // If screenshotIndices reference a real screenshot, use its timestamp
+    if (item.screenshotIndices.length > 0) {
+      const ssIndex = item.screenshotIndices[0];
+      const screenshot: Screenshot | undefined = session.screenshotBuffer[ssIndex];
+      if (screenshot) {
+        const relativeMs = screenshot.timestamp - session.startTime;
+        return this.formatTimestamp(relativeMs);
+      }
+    }
+
+    return '--:--';
+  }
+
+  /**
+   * Format milliseconds as MM:SS.
+   */
+  private formatTimestamp(ms: number): string {
+    const totalSeconds = Math.max(0, Math.floor(ms / 1000));
+    const mins = Math.floor(totalSeconds / 60);
+    const secs = totalSeconds % 60;
+    return `${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
+  }
+
+  /**
+   * Format milliseconds as M:SS for duration display.
+   */
+  private formatDuration(ms: number): string {
+    const totalSeconds = Math.floor(ms / 1000);
+    const mins = Math.floor(totalSeconds / 60);
+    const secs = totalSeconds % 60;
+    return `${mins}:${secs.toString().padStart(2, '0')}`;
+  }
+
+  /**
+   * Format a timestamp as a human-readable date string.
+   */
+  private formatDate(ms: number): string {
+    return new Date(ms).toLocaleDateString('en-US', {
+      year: 'numeric',
+      month: 'short',
+      day: 'numeric',
+    });
+  }
+}
+
+/**
+ * Singleton instance for easy import.
+ */
+export const structuredMarkdownBuilder = new StructuredMarkdownBuilder();

package/src/main/ai/index.ts

@@ -0,0 +1,16 @@
+/**
+ * AI Pipeline - Barrel Export
+ *
+ * Re-exports the pipeline manager and key types for use in the main process.
+ */
+
+export { processSession } from './AIPipelineManager';
+export type { PipelineProcessOptions } from './AIPipelineManager';
+export type {
+  AITier,
+  AIPipelineOutput,
+  AIPipelineOptions,
+  AIPipelineProgress,
+  AIPipelineStage,
+  AIAnalysisResult,
+} from './types';