adaptive-memory-multi-model-router 2.2.5 → 2.2.7

package/src/routing/difficultyClassifier.ts

@@ -0,0 +1,420 @@
+/**
+ * A3M Router - Difficulty Classifier
+ *
+ * Classifies queries as simple/medium/complex based on features.
+ * Used to route queries to appropriate model tiers.
+ *
+ * Usage:
+ *   const classifier = new DifficultyClassifier();
+ *   const result = classifier.classify("What is Python?");
+ *   console.log(result); // { level: 'simple', confidence: 0.85, signals: [...] }
+ *
+ *   const complex = classifier.classify("Implement a red-black tree in Rust with full tests");
+ *   console.log(complex); // { level: 'complex', confidence: 0.92, signals: [...] }
+ */
+
+import { ProviderTier } from '../providers/providerConfig';
+
+// ============================================================
+// Types
+// ============================================================
+
+export type DifficultyLevel = 'simple' | 'medium' | 'complex';
+
+export interface ClassificationResult {
+  /** Difficulty level */
+  level: DifficultyLevel;
+  /** Confidence score 0-1 */
+  confidence: number;
+  /** Signals that contributed to the classification */
+  signals: string[];
+  /** Feature scores for each dimension */
+  features: {
+    length: number;      // 0-1, longer = potentially more complex
+    keywords: number;   // 0-1, presence of complex keywords
+    reasoning: number; // 0-1, reasoning requirements
+    language: number;   // 0-1, language complexity
+    code: number;        // 0-1, code-related content
+  };
+  /** Recommended model tier */
+  recommendedTier: ProviderTier;
+  /** Alternative tiers (in order of preference) */
+  alternativeTiers: ProviderTier[];
+}
+
+export interface ClassifierConfig {
+  /** Thresholds for classification */
+  thresholds?: {
+    simpleMax?: number;
+    mediumMax?: number;
+  };
+  /** Enable keyword-based classification */
+  useKeywords?: boolean;
+  /** Enable linguistic complexity analysis */
+  useLinguistic?: boolean;
+}
+
+const DEFAULT_CONFIG: Required<ClassifierConfig> = {
+  thresholds: {
+    simpleMax: 0.35,
+    mediumMax: 0.65,
+  },
+  useKeywords: true,
+  useLinguistic: true,
+};
+
+// ============================================================
+// Signal patterns
+// ============================================================
+
+const COMPLEX_KEYWORDS = [
+  // Math/Science
+  'calculate', 'compute', 'algorithm', 'mathematical', 'equation',
+  'statistical', 'probability', 'derivative', 'integral', 'matrix',
+  'vector', 'optimize', 'optimization', 'minimize', 'maximize',
+  // Code
+  'implement', 'refactor', 'architect', 'design pattern', 'factory',
+  'singleton', 'decorator', 'middleware', 'pipeline', 'async',
+  'concurrency', 'parallel', 'thread', 'process', 'memory leak',
+  'database', 'sql', 'nosql', 'index', 'shard', 'replica',
+  'api', 'rest', 'graphql', 'microservice', 'container', 'kubernetes',
+  'deploy', 'ci/cd', 'pipeline', 'test', 'mock', 'stub',
+  // Reasoning
+  'analyze', 'compare', 'contrast', 'evaluate', 'synthesis',
+  'implications', 'hypothesis', 'theory', 'principle', 'why does',
+  'explain why', 'reasoning', 'logical', 'deduce', 'infer',
+  // Academic/Complex
+  'research', 'comprehensive', 'detailed', 'thorough', 'extensive',
+  'in-depth', 'multi-step', 'hierarchical', 'nested', 'recursive',
+];
+
+const SIMPLE_KEYWORDS = [
+  'what is', 'who is', 'when did', 'where is', 'simple',
+  'basic', 'intro', 'tutorial', 'hello', 'hi ',
+  'thanks', 'please', 'help me', 'quick', 'brief',
+  'yes', 'no', 'maybe', 'ok', 'sure', 'okay',
+  'list of', 'names of', 'definition', 'meaning of',
+];
+
+const CODE_INDICATORS = [
+  'code', 'function', 'class', 'method', 'variable',
+  'const', 'let', 'var', 'return', 'import', 'export',
+  'def ', 'fn ', 'pub ', 'struct', 'enum', 'trait',
+  'python', 'javascript', 'typescript', 'java', 'rust', 'go',
+  'html', 'css', 'sql', 'bash', 'shell', 'script',
+  'bug', 'error', 'exception', 'debug', 'stack trace',
+  'api', 'endpoint', 'route', 'handler', 'controller',
+];
+
+const REASONING_INDICATORS = [
+  'why', 'how', 'because', 'therefore', 'thus',
+  'reasoning', 'logic', 'deduce', 'infer', 'conclude',
+  'implies', 'suggest', 'indicate', 'evidence',
+  'analyze', 'investigate', 'examine', 'evaluate',
+  'compare', 'differences', 'similar', 'versus',
+  'if then', 'hypothesis', 'assumption', 'premise',
+];
+
+// ============================================================
+// DifficultyClassifier
+// ============================================================
+
+export class DifficultyClassifier {
+  private config: Required<ClassifierConfig>;
+  private trainingData: Array<{ text: string; level: DifficultyLevel }> = [];
+
+  constructor(config: ClassifierConfig = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Classify a query's difficulty.
+   */
+  classify(query: string): ClassificationResult {
+    const query_lower = query.toLowerCase();
+    const words = query.split(/\s+/);
+
+    // Calculate feature scores
+    const features = {
+      length: this.calculateLengthScore(query),
+      keywords: this.calculateKeywordScore(query_lower),
+      reasoning: this.calculateReasoningScore(query_lower, words),
+      language: this.calculateLanguageScore(query_lower, words),
+      code: this.calculateCodeScore(query_lower),
+    };
+
+    // Weighted combination
+    const rawScore =
+      features.length * 0.15 +
+      features.keywords * 0.30 +
+      features.reasoning * 0.20 +
+      features.language * 0.15 +
+      features.code * 0.20;
+
+    // Clamp and determine level
+    const score = Math.max(0, Math.min(1, rawScore));
+    const level = this.determineLevel(score);
+    const confidence = this.calculateConfidence(features, score);
+
+    // Generate signals list
+    const signals = this.generateSignals(query_lower, features);
+
+    // Determine recommended tier
+    const { recommendedTier, alternativeTiers } = this.determineTier(level, features);
+
+    return {
+      level,
+      confidence: Math.round(confidence * 1000) / 1000,
+      signals,
+      features: {
+        length: Math.round(features.length * 1000) / 1000,
+        keywords: Math.round(features.keywords * 1000) / 1000,
+        reasoning: Math.round(features.reasoning * 1000) / 1000,
+        language: Math.round(features.language * 1000) / 1000,
+        code: Math.round(features.code * 1000) / 1000,
+      },
+      recommendedTier,
+      alternativeTiers,
+    };
+  }
+
+  /**
+   * Classify multiple queries.
+   */
+  classifyBatch(queries: string[]): ClassificationResult[] {
+    return queries.map(q => this.classify(q));
+  }
+
+  /**
+   * Add training example for future improvements.
+   */
+  addTrainingExample(query: string, level: DifficultyLevel): void {
+    this.trainingData.push({ text: query, level });
+  }
+
+  /**
+   * Get distribution of difficulties in a batch.
+   */
+  getDistribution(queries: string[]): Record<DifficultyLevel, number> {
+    const results = this.classifyBatch(queries);
+    const total = results.length;
+
+    const distribution = { simple: 0, medium: 0, complex: 0 };
+    for (const r of results) {
+      distribution[r.level]++;
+    }
+
+    return {
+      simple: Math.round((distribution.simple / total) * 1000) / 1000,
+      medium: Math.round((distribution.medium / total) * 1000) / 1000,
+      complex: Math.round((distribution.complex / total) * 1000) / 1000,
+    };
+  }
+
+  // ---- Feature calculations ----
+
+  private calculateLengthScore(query: string): number {
+    const words = query.split(/\s+/).length;
+    const chars = query.length;
+
+    // Normalize: 1-10 words = simple, 10-50 = medium, 50+ = complex
+    const wordScore = Math.min(words / 50, 1.0);
+    const charScore = Math.min(chars / 500, 1.0);
+
+    return (wordScore * 0.6 + charScore * 0.4);
+  }
+
+  private calculateKeywordScore(query: string): number {
+    if (!this.config.useKeywords) return 0.5;
+
+    let score = 0.5; // Base score
+
+    // Check for complex keywords
+    for (const kw of COMPLEX_KEYWORDS) {
+      if (query.includes(kw)) score += 0.1;
+    }
+
+    // Check for simple keywords
+    for (const kw of SIMPLE_KEYWORDS) {
+      if (query.includes(kw)) score -= 0.15;
+    }
+
+    return Math.max(0, Math.min(1, score));
+  }
+
+  private calculateReasoningScore(query: string, words: string[]): number {
+    let score = 0.2; // Base score
+
+    // Check reasoning indicators
+    for (const indicator of REASONING_INDICATORS) {
+      if (query.includes(indicator)) score += 0.12;
+    }
+
+    // "How" questions often require explanation
+    if (query.startsWith('how')) score += 0.1;
+
+    // Multi-step indicators (first, then, finally, etc.)
+    const multiStep = ['first', 'then', 'next', 'finally', 'after', 'before'];
+    const hasMultiStep = multiStep.filter(m => query.includes(m)).length;
+    score += hasMultiStep * 0.05;
+
+    // Question length (>3 words after question word = more complex)
+    if (words.length > 10) score += 0.1;
+
+    return Math.max(0, Math.min(1, score));
+  }
+
+  private calculateLanguageScore(query: string, words: string[]): number {
+    if (!this.config.useLinguistic) return 0.5;
+
+    // Simple heuristics for language complexity
+
+    // Average word length (longer words = more complex)
+    const avgWordLen = words.reduce((sum, w) => sum + w.length, 0) / words.length;
+    const lenScore = Math.min((avgWordLen - 3) / 4, 1.0); // 3 chars = simple, 7+ = complex
+
+    // Presence of technical/academic vocabulary
+    const technicalWords = [
+      'analysis', 'methodology', 'framework', 'paradigm', 'synthesis',
+      'theoretical', 'empirical', 'conceptual', 'phenomenon', 'correlation',
+    ];
+    let techCount = 0;
+    for (const tw of technicalWords) {
+      if (query.includes(tw)) techCount++;
+    }
+    const techScore = Math.min(techCount * 0.15, 0.4);
+
+    // Sentence complexity (commas, semicolons)
+    const punctCount = (query.match(/[,;:]/g) || []).length;
+    const punctScore = Math.min(punctCount * 0.1, 0.3);
+
+    return Math.max(0, Math.min(1, 0.3 + lenScore * 0.3 + techScore + punctScore));
+  }
+
+  private calculateCodeScore(query: string): number {
+    let score = 0.1; // Base score (low probability of code)
+
+    // Check code indicators
+    for (const indicator of CODE_INDICATORS) {
+      if (query.includes(indicator)) score += 0.1;
+    }
+
+    // Code block indicators
+    if (query.includes('```') || query.includes('`')) score += 0.15;
+
+    // Programming language mentions
+    const langs = ['python', 'javascript', 'java', 'rust', 'go', 'c++', 'typescript', 'ruby'];
+    for (const lang of langs) {
+      if (query.includes(lang)) score += 0.1;
+    }
+
+    // Code-like patterns (brackets, semicolons in unusual contexts)
+    if (/[{}\[\];]/.test(query)) score += 0.1;
+
+    return Math.max(0, Math.min(1, score));
+  }
+
+  // ---- Classification ----
+
+  private determineLevel(score: number): DifficultyLevel {
+    if (score <= this.config.thresholds.simpleMax) {
+      return 'simple';
+    }
+    if (score <= this.config.thresholds.mediumMax) {
+      return 'medium';
+    }
+    return 'complex';
+  }
+
+  private calculateConfidence(
+    features: ClassificationResult['features'],
+    score: number
+  ): number {
+    // Higher agreement between features = higher confidence
+    const featureValues = Object.values(features);
+    const mean = featureValues.reduce((a, b) => a + b, 0) / featureValues.length;
+    const variance = featureValues.reduce((sum, f) => sum + Math.pow(f - mean, 2), 0) / featureValues.length;
+    const stdDev = Math.sqrt(variance);
+
+    // Low variance = high confidence
+    const agreement = 1 - Math.min(stdDev * 2, 1);
+
+    // Distance from 0.5 also matters (extreme scores = more confident)
+    const extremity = Math.abs(score - 0.5) * 2;
+
+    return (agreement * 0.6 + extremity * 0.4);
+  }
+
+  // ---- Signal generation ----
+
+  private generateSignals(query: string, features: ClassificationResult['features']): string[] {
+    const signals: string[] = [];
+
+    if (features.length > 0.6) signals.push('long query');
+    if (features.length < 0.2) signals.push('short query');
+
+    if (features.keywords > 0.6) signals.push('complex vocabulary');
+    if (features.keywords < 0.3) signals.push('simple vocabulary');
+
+    if (features.code > 0.5) signals.push('code-related');
+    if (features.reasoning > 0.5) signals.push('reasoning required');
+    if (features.language > 0.6) signals.push('complex language');
+
+    // Specific keyword matches
+    for (const kw of COMPLEX_KEYWORDS) {
+      if (query.includes(kw)) signals.push(`keyword: ${kw.slice(0, 10)}`);
+    }
+
+    return signals;
+  }
+
+  // ---- Tier determination ----
+
+  private determineTier(
+    level: DifficultyLevel,
+    features: ClassificationResult['features']
+  ): { recommendedTier: ProviderTier; alternativeTiers: ProviderTier[] } {
+    if (level === 'simple') {
+      return {
+        recommendedTier: 'cheap',
+        alternativeTiers: ['free', 'mid'],
+      };
+    }
+
+    if (level === 'medium') {
+      // Medium but with high code = needs better model
+      if (features.code > 0.5) {
+        return {
+          recommendedTier: 'mid',
+          alternativeTiers: ['premium', 'cheap'],
+        };
+      }
+      return {
+        recommendedTier: 'mid',
+        alternativeTiers: ['cheap', 'premium'],
+      };
+    }
+
+    // Complex
+    if (features.code > 0.6 || features.reasoning > 0.6) {
+      return {
+        recommendedTier: 'premium',
+        alternativeTiers: ['mid', 'enterprise'],
+      };
+    }
+    return {
+      recommendedTier: 'premium',
+      alternativeTiers: ['mid', 'enterprise'],
+    };
+  }
+}
+
+// ============================================================
+// Factory
+// ============================================================
+
+export function createDifficultyClassifier(
+  config?: ClassifierConfig
+): DifficultyClassifier {
+  return new DifficultyClassifier(config);
+}