cognitive-core 0.0.2 → 0.1.2

package/src/learning/playbook-extractor.ts

@@ -0,0 +1,702 @@
+/**
+ * Playbook Extractor
+ * Extracts playbooks from successful trajectories
+ * Replaces the old TextPatternExtractor with more intelligent extraction
+ */
+
+import type {
+  Playbook,
+  Trajectory,
+  Refinement,
+} from '../types/index.js';
+import { createPlaybook } from '../types/index.js';
+import type { AnalysisResult } from './analyzer.js';
+import { textSimilarity } from '../utils/similarity.js';
+
+/**
+ * Result of playbook extraction
+ */
+export interface ExtractedPlaybooks {
+  /** Entirely new playbooks */
+  new: Playbook[];
+  /** Updates to existing playbooks */
+  updates: PlaybookUpdate[];
+}
+
+/**
+ * Update to an existing playbook
+ */
+export interface PlaybookUpdate {
+  id: string;
+  refinement?: Refinement;
+  newTrigger?: string;
+  newAntiPattern?: string;
+}
+
+/**
+ * Configuration for playbook extraction
+ */
+export interface PlaybookExtractorConfig {
+  /** Minimum trajectories needed to create a playbook */
+  minTrajectories: number;
+  /** Similarity threshold for merging playbooks */
+  mergeThreshold: number;
+  /** Minimum success rate for extraction */
+  minSuccessRate: number;
+}
+
+const DEFAULT_CONFIG: PlaybookExtractorConfig = {
+  minTrajectories: 2,
+  mergeThreshold: 0.85,
+  minSuccessRate: 0.6,
+};
+
+/**
+ * Playbook Extractor
+ * Intelligent extraction of playbooks from trajectories
+ */
+export class PlaybookExtractor {
+  private config: PlaybookExtractorConfig;
+
+  constructor(config?: Partial<PlaybookExtractorConfig>) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Extract playbooks from successful trajectories
+   */
+  async extract(
+    trajectories: Trajectory[],
+    analyses: AnalysisResult[],
+    existingPlaybooks: Playbook[] = []
+  ): Promise<ExtractedPlaybooks> {
+    const result: ExtractedPlaybooks = {
+      new: [],
+      updates: [],
+    };
+
+    // Filter to successful, abstractable trajectories
+    const candidates = trajectories.filter((_t, i) => {
+      const analysis = analyses[i];
+      return analysis?.success && analysis.abstractable;
+    });
+
+    if (candidates.length < this.config.minTrajectories) {
+      return result;
+    }
+
+    // Group by domain
+    const byDomain = this.groupByDomain(candidates);
+
+    for (const [domain, domainTrajectories] of byDomain) {
+      // Extract playbooks from this domain
+      const extracted = await this.extractFromDomain(
+        domain,
+        domainTrajectories,
+        analyses,
+        existingPlaybooks
+      );
+
+      result.new.push(...extracted.new);
+      result.updates.push(...extracted.updates);
+    }
+
+    // Consolidate similar playbooks
+    result.new = await this.consolidate(result.new);
+
+    return result;
+  }
+
+  /**
+   * Extract playbooks from a single domain's trajectories
+   */
+  private async extractFromDomain(
+    domain: string,
+    trajectories: Trajectory[],
+    analyses: AnalysisResult[],
+    existingPlaybooks: Playbook[]
+  ): Promise<ExtractedPlaybooks> {
+    const result: ExtractedPlaybooks = {
+      new: [],
+      updates: [],
+    };
+
+    // Find common patterns across trajectories
+    const patterns = this.findCommonPatterns(trajectories, analyses);
+
+    for (const pattern of patterns) {
+      // Check if similar playbook exists
+      const existing = this.findSimilarPlaybook(pattern, existingPlaybooks);
+
+      if (existing) {
+        // Generate update
+        const update = this.generateUpdate(existing, pattern, trajectories);
+        if (update) {
+          result.updates.push(update);
+        }
+      } else {
+        // Create new playbook
+        const playbook = this.createPlaybookFromPattern(pattern, domain);
+        result.new.push(playbook);
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Find common patterns across trajectories
+   */
+  private findCommonPatterns(
+    trajectories: Trajectory[],
+    analyses: AnalysisResult[]
+  ): ExtractedPattern[] {
+    const patterns: ExtractedPattern[] = [];
+
+    // Extract action sequences from key steps
+    const actionSequences: Array<{
+      actions: string[];
+      thought: string;
+      trajectoryId: string;
+    }> = [];
+
+    for (let i = 0; i < trajectories.length; i++) {
+      const trajectory = trajectories[i];
+      // Use index-based matching since TrainingExample doesn't have trajectoryId
+      const analysis = analyses[i];
+
+      if (!analysis) continue;
+
+      // Get key steps
+      const keySteps = analysis.keySteps
+        .map((idx) => trajectory.steps[idx])
+        .filter(Boolean);
+
+      if (keySteps.length > 0) {
+        actionSequences.push({
+          actions: keySteps.map((s) => s.action),
+          thought: keySteps[0]?.thought ?? trajectory.task.description,
+          trajectoryId: trajectory.id,
+        });
+      }
+    }
+
+    // Find repeated patterns
+    const patternCounts = new Map<string, ExtractedPattern>();
+
+    for (const seq of actionSequences) {
+      // Normalize actions
+      const normalized = seq.actions.map((a) => this.normalizeAction(a));
+      const key = normalized.join('|');
+
+      if (patternCounts.has(key)) {
+        const existing = patternCounts.get(key)!;
+        existing.frequency++;
+        existing.trajectoryIds.push(seq.trajectoryId);
+      } else {
+        patternCounts.set(key, {
+          actions: seq.actions,
+          normalizedActions: normalized,
+          thought: seq.thought,
+          frequency: 1,
+          trajectoryIds: [seq.trajectoryId],
+        });
+      }
+    }
+
+    // Filter to patterns that occur multiple times
+    for (const pattern of patternCounts.values()) {
+      if (pattern.frequency >= this.config.minTrajectories) {
+        patterns.push(pattern);
+      }
+    }
+
+    // Also extract error-recovery patterns
+    const errorPatterns = this.extractErrorPatterns(trajectories, analyses);
+    patterns.push(...errorPatterns);
+
+    return patterns;
+  }
+
+  /**
+   * Extract error-recovery patterns
+   */
+  private extractErrorPatterns(
+    trajectories: Trajectory[],
+    analyses: AnalysisResult[]
+  ): ExtractedPattern[] {
+    const patterns: ExtractedPattern[] = [];
+    const errorRecoveries = new Map<string, ExtractedPattern>();
+
+    for (let i = 0; i < trajectories.length; i++) {
+      const trajectory = trajectories[i];
+      const analysis = analyses[i];
+
+      if (!analysis?.errorPatterns) continue;
+
+      for (const errorPattern of analysis.errorPatterns) {
+        const key = errorPattern.signature;
+
+        // Find recovery actions (steps after the error)
+        const errorStepIdx = trajectory.steps.findIndex((s: { observation?: string }) =>
+          s.observation?.includes(errorPattern.signature)
+        );
+
+        if (errorStepIdx >= 0 && errorStepIdx < trajectory.steps.length - 1) {
+          const recoverySteps = trajectory.steps.slice(
+            errorStepIdx + 1,
+            errorStepIdx + 4
+          );
+
+          if (errorRecoveries.has(key)) {
+            const existing = errorRecoveries.get(key)!;
+            existing.frequency++;
+            existing.trajectoryIds.push(trajectory.id);
+          } else {
+            errorRecoveries.set(key, {
+              actions: recoverySteps.map((s: { action: string }) => s.action),
+              normalizedActions: recoverySteps.map((s: { action: string }) =>
+                this.normalizeAction(s.action)
+              ),
+              thought: `When encountering: ${errorPattern.signature}`,
+              frequency: 1,
+              trajectoryIds: [trajectory.id],
+              errorTrigger: errorPattern.signature,
+            });
+          }
+        }
+      }
+    }
+
+    for (const pattern of errorRecoveries.values()) {
+      if (pattern.frequency >= this.config.minTrajectories) {
+        patterns.push(pattern);
+      }
+    }
+
+    return patterns;
+  }
+
+  /**
+   * Normalize an action for pattern matching
+   */
+  private normalizeAction(action: string): string {
+    const lower = action.toLowerCase();
+
+    // Map to action categories
+    const categories: Record<string, string[]> = {
+      read: ['read', 'view', 'show', 'display', 'cat', 'less', 'get'],
+      write: ['write', 'create', 'add', 'insert', 'save'],
+      edit: ['edit', 'modify', 'change', 'update', 'replace'],
+      delete: ['delete', 'remove', 'drop', 'clear'],
+      search: ['search', 'find', 'grep', 'locate', 'query'],
+      run: ['run', 'execute', 'call', 'invoke', 'start'],
+      test: ['test', 'verify', 'check', 'validate', 'assert'],
+      build: ['build', 'compile', 'bundle', 'package'],
+      install: ['install', 'add', 'npm', 'yarn', 'pip'],
+      debug: ['debug', 'trace', 'log', 'inspect', 'breakpoint'],
+    };
+
+    for (const [category, keywords] of Object.entries(categories)) {
+      if (keywords.some((kw) => lower.includes(kw))) {
+        return category;
+      }
+    }
+
+    // Return first word as fallback
+    return lower.split(/\s+/)[0] ?? 'action';
+  }
+
+  /**
+   * Find similar existing playbook
+   */
+  private findSimilarPlaybook(
+    pattern: ExtractedPattern,
+    playbooks: Playbook[]
+  ): Playbook | undefined {
+    for (const playbook of playbooks) {
+      // Check situation similarity
+      const situationMatch = playbook.applicability.situations.some(
+        (s) => textSimilarity(pattern.thought, s) > this.config.mergeThreshold
+      );
+
+      if (situationMatch) return playbook;
+
+      // Check trigger match for error patterns
+      if (pattern.errorTrigger) {
+        const triggerMatch = playbook.applicability.triggers.some(
+          (t) =>
+            textSimilarity(pattern.errorTrigger!, t) > this.config.mergeThreshold
+        );
+        if (triggerMatch) return playbook;
+      }
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Generate update for existing playbook
+   */
+  private generateUpdate(
+    playbook: Playbook,
+    pattern: ExtractedPattern,
+    trajectories: Trajectory[]
+  ): PlaybookUpdate | undefined {
+    const update: PlaybookUpdate = { id: playbook.id };
+    let hasUpdate = false;
+
+    // Check for new trigger
+    if (
+      pattern.errorTrigger &&
+      !playbook.applicability.triggers.includes(pattern.errorTrigger)
+    ) {
+      update.newTrigger = pattern.errorTrigger;
+      hasUpdate = true;
+    }
+
+    // Check for context-specific refinement
+    const contexts = trajectories
+      .filter((t) => pattern.trajectoryIds.includes(t.id))
+      .map((t) => this.extractContext(t));
+
+    const uniqueContext = contexts.find(
+      (c) =>
+        !playbook.evolution.refinements.some(
+          (r) => textSimilarity(r.context, c) > 0.8
+        )
+    );
+
+    if (uniqueContext && pattern.actions.length > 0) {
+      update.refinement = {
+        context: uniqueContext,
+        addition: `Also: ${pattern.actions.slice(0, 2).join(', ')}`,
+        source: 'success',
+        addedAt: new Date(),
+      };
+      hasUpdate = true;
+    }
+
+    return hasUpdate ? update : undefined;
+  }
+
+  /**
+   * Create a playbook from an extracted pattern
+   */
+  private createPlaybookFromPattern(
+    pattern: ExtractedPattern,
+    domain: string
+  ): Playbook {
+    const name = this.generatePlaybookName(pattern);
+
+    return createPlaybook({
+      name,
+      applicability: {
+        situations: [pattern.thought],
+        triggers: pattern.errorTrigger ? [pattern.errorTrigger] : [],
+        antiPatterns: [],
+        domains: [domain],
+      },
+      guidance: {
+        strategy: pattern.thought,
+        tactics: pattern.actions.slice(0, 5),
+        steps: pattern.actions,
+      },
+      verification: {
+        successIndicators: ['Task completes successfully'],
+        failureIndicators: ['Same error persists', 'New errors introduced'],
+      },
+      evolution: {
+        version: '1.0.0',
+        createdFrom: pattern.trajectoryIds,
+        failures: [],
+        refinements: [],
+        successCount: pattern.frequency,
+        failureCount: 0,
+      },
+      provenance: {
+        origin: 'extracted',
+        recordedAt: new Date(),
+      },
+      confidence: Math.min(0.5, 0.2 + pattern.frequency * 0.1),
+      complexity:
+        pattern.actions.length <= 3
+          ? 'simple'
+          : pattern.actions.length <= 6
+            ? 'moderate'
+            : 'complex',
+      estimatedEffort: Math.ceil(pattern.actions.length * 0.5),
+    });
+  }
+
+  /**
+   * Generate a kebab-case playbook name
+   */
+  private generatePlaybookName(pattern: ExtractedPattern): string {
+    const words = pattern.thought
+      .toLowerCase()
+      .replace(/[^\w\s]/g, '')
+      .split(/\s+/)
+      .filter((w) => w.length > 2 && !this.isStopWord(w))
+      .slice(0, 4);
+
+    if (words.length === 0) {
+      return `playbook-${Date.now()}`;
+    }
+
+    return words.join('-');
+  }
+
+  /**
+   * Check if word is a stop word
+   */
+  private isStopWord(word: string): boolean {
+    const stopWords = new Set([
+      'the',
+      'a',
+      'an',
+      'and',
+      'or',
+      'but',
+      'is',
+      'are',
+      'was',
+      'were',
+      'be',
+      'been',
+      'being',
+      'have',
+      'has',
+      'had',
+      'do',
+      'does',
+      'did',
+      'will',
+      'would',
+      'could',
+      'should',
+      'may',
+      'might',
+      'must',
+      'shall',
+      'can',
+      'to',
+      'of',
+      'in',
+      'for',
+      'on',
+      'with',
+      'at',
+      'by',
+      'from',
+      'as',
+      'into',
+      'through',
+      'during',
+      'before',
+      'after',
+      'above',
+      'below',
+      'between',
+      'under',
+      'again',
+      'further',
+      'then',
+      'once',
+      'here',
+      'there',
+      'when',
+      'where',
+      'why',
+      'how',
+      'all',
+      'each',
+      'few',
+      'more',
+      'most',
+      'other',
+      'some',
+      'such',
+      'no',
+      'not',
+      'only',
+      'own',
+      'same',
+      'so',
+      'than',
+      'too',
+      'very',
+      'just',
+    ]);
+    return stopWords.has(word);
+  }
+
+  /**
+   * Extract context from a trajectory
+   */
+  private extractContext(trajectory: Trajectory): string {
+    // Try to extract meaningful context
+    const metadata = trajectory.task.metadata ?? {};
+
+    if (metadata.fileType) return `${metadata.fileType} files`;
+    if (metadata.framework) return `${metadata.framework} projects`;
+    if (metadata.language) return `${metadata.language} code`;
+
+    // Fall back to domain
+    return trajectory.task.domain;
+  }
+
+  /**
+   * Group trajectories by domain
+   */
+  private groupByDomain(
+    trajectories: Trajectory[]
+  ): Map<string, Trajectory[]> {
+    const groups = new Map<string, Trajectory[]>();
+
+    for (const trajectory of trajectories) {
+      const domain = trajectory.task.domain;
+      const existing = groups.get(domain) ?? [];
+      existing.push(trajectory);
+      groups.set(domain, existing);
+    }
+
+    return groups;
+  }
+
+  /**
+   * Consolidate similar playbooks
+   */
+  async consolidate(playbooks: Playbook[]): Promise<Playbook[]> {
+    if (playbooks.length <= 1) return playbooks;
+
+    const consolidated: Playbook[] = [];
+    const merged = new Set<string>();
+
+    for (let i = 0; i < playbooks.length; i++) {
+      if (merged.has(playbooks[i].id)) continue;
+
+      let current = playbooks[i];
+
+      for (let j = i + 1; j < playbooks.length; j++) {
+        if (merged.has(playbooks[j].id)) continue;
+
+        const other = playbooks[j];
+        const similarity = this.calculatePlaybookSimilarity(current, other);
+
+        if (similarity > this.config.mergeThreshold) {
+          current = this.mergePlaybooks(current, other);
+          merged.add(other.id);
+        }
+      }
+
+      consolidated.push(current);
+    }
+
+    return consolidated;
+  }
+
+  /**
+   * Calculate similarity between two playbooks
+   */
+  private calculatePlaybookSimilarity(a: Playbook, b: Playbook): number {
+    // Compare situations
+    let maxSituationSim = 0;
+    for (const sA of a.applicability.situations) {
+      for (const sB of b.applicability.situations) {
+        maxSituationSim = Math.max(maxSituationSim, textSimilarity(sA, sB));
+      }
+    }
+
+    // Compare guidance
+    const strategySim = textSimilarity(a.guidance.strategy, b.guidance.strategy);
+
+    // Combined score
+    return maxSituationSim * 0.6 + strategySim * 0.4;
+  }
+
+  /**
+   * Merge two similar playbooks
+   */
+  private mergePlaybooks(a: Playbook, b: Playbook): Playbook {
+    return createPlaybook({
+      id: a.id,
+      name: a.name,
+      applicability: {
+        situations: [...new Set([...a.applicability.situations, ...b.applicability.situations])],
+        triggers: [...new Set([...a.applicability.triggers, ...b.applicability.triggers])],
+        antiPatterns: [...new Set([...a.applicability.antiPatterns, ...b.applicability.antiPatterns])],
+        domains: [...new Set([...a.applicability.domains, ...b.applicability.domains])],
+      },
+      guidance: {
+        strategy: a.guidance.strategy, // Keep first
+        tactics: [...new Set([...a.guidance.tactics, ...b.guidance.tactics])],
+        steps: a.guidance.steps ?? b.guidance.steps,
+        codeExample: a.guidance.codeExample ?? b.guidance.codeExample,
+      },
+      verification: {
+        successIndicators: [...new Set([...a.verification.successIndicators, ...b.verification.successIndicators])],
+        failureIndicators: [...new Set([...a.verification.failureIndicators, ...b.verification.failureIndicators])],
+        rollbackStrategy: a.verification.rollbackStrategy ?? b.verification.rollbackStrategy,
+      },
+      evolution: {
+        version: a.evolution.version,
+        createdFrom: [...new Set([...a.evolution.createdFrom, ...b.evolution.createdFrom])],
+        failures: [...a.evolution.failures, ...b.evolution.failures],
+        refinements: [...a.evolution.refinements, ...b.evolution.refinements],
+        successCount: a.evolution.successCount + b.evolution.successCount,
+        failureCount: a.evolution.failureCount + b.evolution.failureCount,
+      },
+      provenance: a.provenance, // Preserve provenance from primary playbook
+      confidence: Math.max(a.confidence, b.confidence),
+      complexity: a.complexity,
+      estimatedEffort: Math.min(a.estimatedEffort, b.estimatedEffort),
+    });
+  }
+
+  /**
+   * Generate refinement from a failure
+   */
+  async generateRefinement(
+    _playbook: Playbook,
+    failedTrajectory: Trajectory,
+    failureContext: string
+  ): Promise<Refinement> {
+    // Analyze what was different in this failure
+    const context = this.extractContext(failedTrajectory);
+
+    // Try to identify what additional steps might help
+    const lastSteps = failedTrajectory.steps.slice(-3);
+    const suggestion = lastSteps.length > 0
+      ? `Consider checking: ${lastSteps.map((s: { action: string }) => s.action).join(', ')}`
+      : `Review the approach for ${failureContext}`;
+
+    return {
+      context,
+      addition: suggestion,
+      source: 'failure',
+      addedAt: new Date(),
+    };
+  }
+}
+
+/**
+ * Internal type for extracted patterns
+ */
+interface ExtractedPattern {
+  actions: string[];
+  normalizedActions: string[];
+  thought: string;
+  frequency: number;
+  trajectoryIds: string[];
+  errorTrigger?: string;
+}
+
+/**
+ * Create a playbook extractor
+ */
+export function createPlaybookExtractor(
+  config?: Partial<PlaybookExtractorConfig>
+): PlaybookExtractor {
+  return new PlaybookExtractor(config);
+}