markupr 2.1.8

package/src/main/ai/types.ts

@@ -0,0 +1,258 @@
+/**
+ * AI Pipeline Types
+ *
+ * Type definitions for the AI analysis pipeline that transforms raw session data
+ * (transcript + screenshots) into structured, actionable feedback documents using Claude.
+ */
+
+import type { Session } from '../SessionController';
+
+// =============================================================================
+// Tier & Configuration
+// =============================================================================
+
+/**
+ * AI analysis tier determining which pipeline is used.
+ * - 'free': Rule-based analysis (existing pipeline, no API call)
+ * - 'byok': User's own Anthropic API key stored in OS keychain
+ * - 'premium': Proxied through markupr's Cloudflare Worker
+ */
+export type AITier = 'free' | 'byok' | 'premium';
+
+/**
+ * Options for the ImageOptimizer.
+ */
+export interface ImageOptimizeOptions {
+  /** Max width in pixels. Claude recommends 1568px on longest edge. */
+  maxWidth: number;
+  /** JPEG quality (0-100) when converting from PNG. */
+  jpegQuality: number;
+  /** Size threshold in bytes above which PNG is converted to JPEG. */
+  pngToJpegThreshold: number;
+  /** Maximum number of screenshots to send to the API. */
+  maxScreenshots: number;
+}
+
+export const DEFAULT_IMAGE_OPTIMIZE_OPTIONS: ImageOptimizeOptions = {
+  maxWidth: 1568,
+  jpegQuality: 80,
+  pngToJpegThreshold: 500 * 1024, // 500KB
+  maxScreenshots: 20,
+};
+
+/**
+ * Options for ClaudeAnalyzer.
+ */
+export interface ClaudeAnalyzerOptions {
+  /** Claude model ID. */
+  model: string;
+  /** Maximum output tokens. */
+  maxTokens: number;
+  /** Sampling temperature (0-1). Lower = more deterministic. */
+  temperature: number;
+  /** API request timeout in milliseconds. */
+  timeoutMs: number;
+}
+
+export const DEFAULT_CLAUDE_ANALYZER_OPTIONS: ClaudeAnalyzerOptions = {
+  model: 'claude-sonnet-4-5-20250929',
+  maxTokens: 4096,
+  temperature: 0.3,
+  timeoutMs: 30_000,
+};
+
+/**
+ * Top-level options for the AI pipeline.
+ */
+export interface AIPipelineOptions {
+  /** Which tier to use for analysis. */
+  tier: AITier;
+  /** Anthropic API key (required for 'byok' and 'premium' tiers). */
+  apiKey?: string;
+  /** Base URL for the Anthropic API (overridden for premium proxy). */
+  baseUrl?: string;
+  /** Image optimization settings. */
+  imageOptions?: Partial<ImageOptimizeOptions>;
+  /** Claude analyzer settings. */
+  analyzerOptions?: Partial<ClaudeAnalyzerOptions>;
+}
+
+// =============================================================================
+// AI Analysis Result (Claude's structured output)
+// =============================================================================
+
+/**
+ * Category assigned by Claude to each feedback item.
+ */
+export type AIFeedbackCategory =
+  | 'Bug'
+  | 'UX Issue'
+  | 'Performance'
+  | 'Suggestion'
+  | 'Question'
+  | 'Positive Note';
+
+/**
+ * Priority level assigned by Claude.
+ */
+export type AIFeedbackPriority = 'Critical' | 'High' | 'Medium' | 'Low';
+
+/**
+ * A single feedback item from AI analysis.
+ * Maps directly to the JSON schema in the Claude system prompt.
+ */
+export interface AIFeedbackItem {
+  /** Short descriptive title (5-10 words). */
+  title: string;
+  /** Feedback category. */
+  category: AIFeedbackCategory;
+  /** Priority level. */
+  priority: AIFeedbackPriority;
+  /** User's exact words (relevant excerpt). */
+  quote: string;
+  /** Indices of related screenshots (0-based). */
+  screenshotIndices: number[];
+  /** Concrete 1-sentence action for a developer. */
+  actionItem: string;
+  /** Component or area of the app this relates to. */
+  area: string;
+}
+
+/**
+ * Metadata summary included in the AI analysis result.
+ */
+export interface AIAnalysisMetadata {
+  totalItems: number;
+  criticalCount: number;
+  highCount: number;
+}
+
+/**
+ * The complete structured output from Claude's analysis.
+ * This is the JSON response parsed from the API call.
+ */
+export interface AIAnalysisResult {
+  /** 2-3 sentence overview of key findings. */
+  summary: string;
+  /** Individual feedback items with categories, priorities, and actions. */
+  items: AIFeedbackItem[];
+  /** Cross-cutting themes identified across feedback. */
+  themes: string[];
+  /** Things the user explicitly praised. */
+  positiveNotes: string[];
+  /** Summary counts. */
+  metadata: AIAnalysisMetadata;
+}
+
+// =============================================================================
+// Optimized Image (output of ImageOptimizer)
+// =============================================================================
+
+/**
+ * Media type for images sent to the Claude API.
+ */
+export type ImageMediaType = 'image/png' | 'image/jpeg';
+
+/**
+ * An optimized image ready to be sent to the Claude Vision API.
+ */
+export interface OptimizedImage {
+  /** The image data buffer (resized and possibly re-encoded). */
+  data: Buffer;
+  /** MIME type of the image data. */
+  mediaType: ImageMediaType;
+  /** Original screenshot this was optimized from. */
+  originalScreenshotId: string;
+  /** Width after resize. */
+  width: number;
+  /** Height after resize. */
+  height: number;
+}
+
+// =============================================================================
+// Pipeline Progress & Errors
+// =============================================================================
+
+/**
+ * Progress stages emitted during AI pipeline processing.
+ */
+export type AIPipelineStage =
+  | 'optimizing-images'
+  | 'building-prompt'
+  | 'calling-api'
+  | 'parsing-response'
+  | 'building-markdown'
+  | 'complete'
+  | 'error'
+  | 'fallback';
+
+/**
+ * Progress event emitted to the renderer during processing.
+ */
+export interface AIPipelineProgress {
+  stage: AIPipelineStage;
+  message: string;
+  /** 0-100 progress percentage (approximate). */
+  percent: number;
+}
+
+/**
+ * Typed error codes for AI pipeline failures.
+ */
+export type AIPipelineErrorCode =
+  | 'API_TIMEOUT'
+  | 'API_AUTH_ERROR'
+  | 'API_RATE_LIMIT'
+  | 'API_SERVER_ERROR'
+  | 'NETWORK_OFFLINE'
+  | 'INVALID_RESPONSE'
+  | 'IMAGE_OPTIMIZATION_FAILED'
+  | 'UNKNOWN';
+
+/**
+ * Structured error from the AI pipeline.
+ */
+export class AIPipelineError extends Error {
+  constructor(
+    message: string,
+    public readonly code: AIPipelineErrorCode,
+    public readonly cause?: Error,
+  ) {
+    super(message);
+    this.name = 'AIPipelineError';
+  }
+}
+
+// =============================================================================
+// Pipeline Input/Output
+// =============================================================================
+
+/**
+ * Input to the AI pipeline — the raw session data.
+ * Re-exported for convenience; the pipeline accepts the Session from SessionController.
+ */
+export type AIPipelineInput = Session;
+
+/**
+ * Output from the AI pipeline — either AI-enhanced or fallback markdown.
+ */
+export interface AIPipelineOutput {
+  /** The final formatted markdown document. */
+  markdown: string;
+  /** Whether AI analysis was used (vs. free tier fallback). */
+  aiEnhanced: boolean;
+  /** The raw analysis result (only present when aiEnhanced is true). */
+  analysis?: AIAnalysisResult;
+  /** Processing duration in milliseconds. */
+  processingTimeMs: number;
+  /** Which tier was used. */
+  tier: AITier;
+  /** Error info if AI failed and fell back to free tier. */
+  fallbackReason?: string;
+}
+
+// =============================================================================
+// Re-exports for convenience
+// =============================================================================
+
+export type { Screenshot, Session } from '../SessionController';

package/src/main/analysis/ClarificationGenerator.ts

@@ -0,0 +1,385 @@
+/**
+ * ClarificationGenerator - Smart Question Generation for Ambiguous Feedback
+ *
+ * Detects vague or unclear feedback and generates targeted clarification questions
+ * to help users provide more actionable context.
+ *
+ * Detection patterns:
+ * - Pronouns without clear antecedent: "this", "that", "it"
+ * - Vague locations: "here", "there", "somewhere"
+ * - Unclear references: "the button", "the form" without specifics
+ * - Time references: "sometimes", "occasionally", "usually"
+ * - Vague comparisons: "too slow", "very small"
+ *
+ * Questions are skippable and answers are appended to feedback items.
+ */
+
+import type { FeedbackItem } from '../output/MarkdownGenerator';
+
+// =============================================================================
+// Types
+// =============================================================================
+
+/**
+ * Type of clarification needed
+ */
+export type ClarificationType =
+  | 'location'
+  | 'reproduction'
+  | 'expectation'
+  | 'frequency'
+  | 'reference'
+  | 'comparison';
+
+/**
+ * A single clarification question
+ */
+export interface ClarificationQuestion {
+  id: string;
+  feedbackItemId: string;
+  question: string;
+  type: ClarificationType;
+  placeholder: string;
+  /** Optional: the matched text that triggered this question */
+  matchedText?: string;
+}
+
+/**
+ * Answer to a clarification question
+ */
+export interface ClarificationAnswer {
+  questionId: string;
+  feedbackItemId: string;
+  answer: string;
+}
+
+/**
+ * Result of applying clarification answers to feedback
+ */
+export interface ClarifiedFeedback {
+  feedbackItems: FeedbackItem[];
+  clarifications: ClarificationAnswer[];
+}
+
+// =============================================================================
+// Pattern Definitions
+// =============================================================================
+
+interface AmbiguityPattern {
+  pattern: RegExp;
+  type: ClarificationType;
+  question: string;
+  placeholder: string;
+  /** Weight for prioritization (higher = more likely to ask) */
+  weight: number;
+}
+
+/**
+ * Patterns for detecting ambiguity in feedback text
+ *
+ * Note: Patterns use negative lookahead to avoid false positives where
+ * the meaning is actually clear (e.g., "it doesn't work" is clear enough)
+ */
+const AMBIGUITY_PATTERNS: AmbiguityPattern[] = [
+  // Vague pronouns (when not part of common clear phrases)
+  {
+    pattern: /\b(this|that)\b(?!\s+(is|was|should|could|would|doesn't|does not|isn't|is not)\s)/gi,
+    type: 'reference',
+    question: 'What specifically are you referring to?',
+    placeholder: 'e.g., "the save button in the header"',
+    weight: 0.8,
+  },
+  {
+    pattern: /\bit\b(?!\s+(is|was|should|could|would|doesn't|does not|works|worked))/gi,
+    type: 'reference',
+    question: 'What specifically does "it" refer to?',
+    placeholder: 'e.g., "the file upload component"',
+    weight: 0.6,
+  },
+
+  // Vague locations
+  {
+    pattern: /\b(here|there)\b(?!\s+(is|are|was|were)\s+(no|nothing|a|an|the))/gi,
+    type: 'location',
+    question: 'Where exactly in the interface?',
+    placeholder: 'e.g., "top right corner of the settings page"',
+    weight: 0.85,
+  },
+  {
+    pattern: /\bsomewhere\b/gi,
+    type: 'location',
+    question: 'Can you describe where more specifically?',
+    placeholder: 'e.g., "in the navigation menu, maybe under settings"',
+    weight: 0.9,
+  },
+
+  // Unclear element references (without labels/names)
+  {
+    pattern: /\bthe\s+(button|link)\b(?!\s+(labeled|called|named|titled|that says|with|for))/gi,
+    type: 'reference',
+    question: 'Which button/link specifically?',
+    placeholder: 'e.g., "the blue Submit button at the bottom of the form"',
+    weight: 0.75,
+  },
+  {
+    pattern: /\bthe\s+(form|field|input)\b(?!\s+(labeled|called|named|titled|for|where))/gi,
+    type: 'reference',
+    question: 'Which form/field specifically?',
+    placeholder: 'e.g., "the email input field in the signup form"',
+    weight: 0.75,
+  },
+  {
+    pattern: /\bthe\s+(menu|dropdown|select)\b(?!\s+(labeled|called|named|titled|for|with))/gi,
+    type: 'reference',
+    question: 'Which menu/dropdown specifically?',
+    placeholder: 'e.g., "the file type dropdown in the export dialog"',
+    weight: 0.75,
+  },
+  {
+    pattern: /\bthe\s+(modal|dialog|popup|panel)\b(?!\s+(labeled|called|named|titled|that))/gi,
+    type: 'reference',
+    question: 'Which modal/dialog specifically?',
+    placeholder: 'e.g., "the confirmation dialog after clicking delete"',
+    weight: 0.75,
+  },
+  {
+    pattern: /\bthe\s+(page|screen|section)\b(?!\s+(labeled|called|named|titled|where|for))/gi,
+    type: 'location',
+    question: 'Which page/screen specifically?',
+    placeholder: 'e.g., "the user profile settings page"',
+    weight: 0.7,
+  },
+
+  // Intermittent issues (frequency unclear)
+  {
+    pattern: /\b(sometimes|occasionally|usually|often|rarely|every now and then)\b/gi,
+    type: 'frequency',
+    question: 'How often does this happen?',
+    placeholder: 'e.g., "about 1 in 3 times when saving" or "only on first load"',
+    weight: 0.9,
+  },
+  {
+    pattern: /\b(intermittent(ly)?|sporadic(ally)?|random(ly)?)\b/gi,
+    type: 'frequency',
+    question: 'Can you estimate how frequently this occurs?',
+    placeholder: 'e.g., "maybe 20% of the time" or "after about 5 minutes of use"',
+    weight: 0.85,
+  },
+
+  // Vague comparisons
+  {
+    pattern: /\b(too|very|quite|rather|extremely)\s+(slow|fast|big|small|large|tiny|long|short)\b/gi,
+    type: 'comparison',
+    question: 'Can you be more specific about the expected behavior?',
+    placeholder: 'e.g., "should load in under 2 seconds" or "should be at least 200px wide"',
+    weight: 0.7,
+  },
+  {
+    pattern: /\btakes\s+(forever|too long|a while|ages)\b/gi,
+    type: 'comparison',
+    question: 'How long does it actually take, and how long should it take?',
+    placeholder: 'e.g., "takes 10 seconds but should be instant"',
+    weight: 0.8,
+  },
+];
+
+/**
+ * Questions for specific feedback categories (asked when no patterns match)
+ */
+const CATEGORY_QUESTIONS: Record<string, { question: string; placeholder: string; type: ClarificationType }> = {
+  bug: {
+    question: 'What steps led to this issue?',
+    type: 'reproduction',
+    placeholder: 'e.g., "After clicking submit, then going back..."',
+  },
+  'ux-issue': {
+    question: 'What would be a better experience?',
+    type: 'expectation',
+    placeholder: 'e.g., "I expected to see a confirmation before deleting"',
+  },
+  suggestion: {
+    question: 'How should it work ideally?',
+    type: 'expectation',
+    placeholder: 'e.g., "It should save automatically when I switch tabs"',
+  },
+};
+
+// =============================================================================
+// ClarificationGenerator Implementation
+// =============================================================================
+
+class ClarificationGeneratorImpl {
+  private readonly MAX_QUESTIONS = 5;
+
+  /**
+   * Generate clarification questions for a list of feedback items.
+   *
+   * Returns at most MAX_QUESTIONS to avoid overwhelming the user.
+   * Questions are prioritized by pattern weight and relevance.
+   */
+  generate(feedbackItems: FeedbackItem[]): ClarificationQuestion[] {
+    const allQuestions: Array<ClarificationQuestion & { weight: number }> = [];
+
+    for (const item of feedbackItems) {
+      const text = item.transcription;
+      const itemId = item.id;
+
+      // Track which question types we've already added for this item
+      const addedTypes = new Set<ClarificationType>();
+
+      // Check each ambiguity pattern
+      for (const patternDef of AMBIGUITY_PATTERNS) {
+        // Reset lastIndex for global patterns
+        patternDef.pattern.lastIndex = 0;
+
+        const match = patternDef.pattern.exec(text);
+        if (match && !addedTypes.has(patternDef.type)) {
+          addedTypes.add(patternDef.type);
+
+          allQuestions.push({
+            id: `${itemId}-${patternDef.type}`,
+            feedbackItemId: itemId,
+            question: patternDef.question,
+            type: patternDef.type,
+            placeholder: patternDef.placeholder,
+            matchedText: match[0],
+            weight: patternDef.weight,
+          });
+        }
+      }
+
+      // Check for category-specific questions
+      const category = item.category?.toLowerCase().replace(' ', '-');
+      if (category && CATEGORY_QUESTIONS[category]) {
+        const categoryQ = CATEGORY_QUESTIONS[category];
+
+        // Only add if we don't have a similar type question already
+        if (!addedTypes.has(categoryQ.type)) {
+          // Check if the text already provides the information
+          const hasReproSteps = /\b(when|after|before|then|first|next|if)\b/i.test(text);
+          const hasExpectation = /\b(should|expect|instead|ideally|rather|better)\b/i.test(text);
+
+          const shouldSkip =
+            (categoryQ.type === 'reproduction' && hasReproSteps) ||
+            (categoryQ.type === 'expectation' && hasExpectation);
+
+          if (!shouldSkip) {
+            allQuestions.push({
+              id: `${itemId}-category-${categoryQ.type}`,
+              feedbackItemId: itemId,
+              question: categoryQ.question,
+              type: categoryQ.type,
+              placeholder: categoryQ.placeholder,
+              weight: 0.5, // Lower weight for category questions
+            });
+          }
+        }
+      }
+    }
+
+    // Sort by weight (higher first) and limit
+    const sortedQuestions = allQuestions
+      .sort((a, b) => b.weight - a.weight)
+      .slice(0, this.MAX_QUESTIONS);
+
+    // Remove the weight property before returning
+    return sortedQuestions.map(({ weight: _weight, ...q }) => q);
+  }
+
+  /**
+   * Apply clarification answers to feedback items.
+   *
+   * Answers are appended to the transcription in a structured format:
+   * [Original text]
+   *
+   * **Clarification:**
+   * - Question: Answer
+   */
+  applyClarifications(
+    feedbackItems: FeedbackItem[],
+    answers: Record<string, string>
+  ): ClarifiedFeedback {
+    const clarifications: ClarificationAnswer[] = [];
+
+    const updatedItems = feedbackItems.map((item) => {
+      // Find all answers for this item
+      const itemAnswers: { question: string; answer: string }[] = [];
+
+      for (const [questionId, answer] of Object.entries(answers)) {
+        // Skip empty answers
+        if (!answer.trim()) continue;
+
+        // Check if this question belongs to this item
+        if (questionId.startsWith(item.id)) {
+          // Extract the question from the ID to find the original question text
+          const questionType = questionId.replace(`${item.id}-`, '').replace('category-', '');
+
+          // Find the matching pattern or category question
+          const patternQ = AMBIGUITY_PATTERNS.find((p) => p.type === questionType);
+          const categoryQ = Object.values(CATEGORY_QUESTIONS).find((q) => q.type === questionType);
+
+          const questionText = patternQ?.question || categoryQ?.question || 'Additional context';
+
+          itemAnswers.push({ question: questionText, answer });
+
+          clarifications.push({
+            questionId,
+            feedbackItemId: item.id,
+            answer,
+          });
+        }
+      }
+
+      // If we have answers, append them to the transcription
+      if (itemAnswers.length > 0) {
+        const clarificationText = itemAnswers
+          .map((qa) => `- ${qa.question} ${qa.answer}`)
+          .join('\n');
+
+        return {
+          ...item,
+          transcription: `${item.transcription}\n\n**Clarification:**\n${clarificationText}`,
+        };
+      }
+
+      return item;
+    });
+
+    return {
+      feedbackItems: updatedItems,
+      clarifications,
+    };
+  }
+
+  /**
+   * Check if a feedback item would benefit from clarification.
+   * Useful for showing a "needs clarification" indicator.
+   */
+  needsClarification(item: FeedbackItem): boolean {
+    const text = item.transcription;
+
+    for (const patternDef of AMBIGUITY_PATTERNS) {
+      patternDef.pattern.lastIndex = 0;
+      if (patternDef.pattern.test(text)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * Get the count of items that need clarification.
+   */
+  getClarificationCount(feedbackItems: FeedbackItem[]): number {
+    return feedbackItems.filter((item) => this.needsClarification(item)).length;
+  }
+}
+
+// =============================================================================
+// Singleton Export
+// =============================================================================
+
+export const clarificationGenerator = new ClarificationGeneratorImpl();
+export type { ClarificationGeneratorImpl as ClarificationGenerator };
+export default clarificationGenerator;