cognitive-core 0.0.2 → 0.1.0

package/src/learning/llm-extractor.ts

@@ -0,0 +1,542 @@
+/**
+ * LLM-Based Playbook Extractor
+ *
+ * Uses language models to generate higher-quality playbooks
+ * from trajectories. Complements the pattern-based extractor
+ * with deeper semantic analysis.
+ */
+
+import type {
+  Playbook,
+  Trajectory,
+  Refinement,
+} from '../types/index.js';
+import { createPlaybook } from '../types/index.js';
+import type { AnalysisResult } from './analyzer.js';
+import type { LLMProvider } from '../agents/types.js';
+
+/**
+ * Configuration for LLM-based extraction
+ */
+export interface LLMExtractorConfig {
+  /** Temperature for LLM calls (default: 0.2 for determinism) */
+  temperature: number;
+  /** Maximum tokens for response */
+  maxTokens: number;
+  /** Whether to extract from single trajectories (vs only batches) */
+  extractFromSingle: boolean;
+  /** Minimum complexity for LLM extraction */
+  minComplexity: number;
+}
+
+const DEFAULT_CONFIG: LLMExtractorConfig = {
+  temperature: 0.2,
+  maxTokens: 2000,
+  extractFromSingle: true,
+  minComplexity: 3, // Minimum steps to warrant LLM extraction
+};
+
+/**
+ * Result from LLM extraction
+ */
+export interface LLMExtractionResult {
+  playbooks: Playbook[];
+  refinements: Array<{
+    playbookId: string;
+    refinement: Refinement;
+  }>;
+  /** Raw LLM response for debugging */
+  rawResponse?: string;
+}
+
+/**
+ * LLM-Based Playbook Extractor
+ */
+export class LLMPlaybookExtractor {
+  private provider: LLMProvider;
+  private config: LLMExtractorConfig;
+
+  constructor(provider: LLMProvider, config?: Partial<LLMExtractorConfig>) {
+    this.provider = provider;
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Extract playbooks from trajectories using LLM
+   */
+  async extract(
+    trajectories: Trajectory[],
+    analyses: AnalysisResult[],
+    existingPlaybooks: Playbook[] = []
+  ): Promise<LLMExtractionResult> {
+    // Filter to successful, complex enough trajectories
+    const candidates = trajectories.filter((t, i) => {
+      const analysis = analyses[i];
+      return (
+        analysis?.success &&
+        analysis.abstractable &&
+        t.steps.length >= this.config.minComplexity
+      );
+    });
+
+    if (candidates.length === 0) {
+      return { playbooks: [], refinements: [] };
+    }
+
+    // Extract playbooks from each candidate
+    const allResults: LLMExtractionResult = {
+      playbooks: [],
+      refinements: [],
+    };
+
+    // Process individually for better quality
+    if (this.config.extractFromSingle) {
+      for (let i = 0; i < candidates.length; i++) {
+        const trajectory = candidates[i];
+        const analysis = analyses.find(
+          (_, idx) => trajectories[idx] === trajectory
+        );
+
+        if (!analysis) continue;
+
+        const result = await this.extractFromSingleTrajectory(
+          trajectory,
+          analysis,
+          existingPlaybooks
+        );
+
+        allResults.playbooks.push(...result.playbooks);
+        allResults.refinements.push(...result.refinements);
+      }
+    }
+
+    // Also do batch extraction if we have multiple
+    if (candidates.length >= 2) {
+      const batchResult = await this.extractFromBatch(
+        candidates,
+        analyses.filter((_, i) => candidates.includes(trajectories[i])),
+        existingPlaybooks
+      );
+
+      // Merge with individual results (avoid duplicates)
+      for (const playbook of batchResult.playbooks) {
+        const isDuplicate = allResults.playbooks.some(
+          (p) => this.isSimilarPlaybook(p, playbook)
+        );
+        if (!isDuplicate) {
+          allResults.playbooks.push(playbook);
+        }
+      }
+
+      allResults.refinements.push(...batchResult.refinements);
+    }
+
+    return allResults;
+  }
+
+  /**
+   * Extract a playbook from a single trajectory
+   */
+  private async extractFromSingleTrajectory(
+    trajectory: Trajectory,
+    analysis: AnalysisResult,
+    existingPlaybooks: Playbook[]
+  ): Promise<LLMExtractionResult> {
+    const prompt = this.buildSingleExtractionPrompt(
+      trajectory,
+      analysis,
+      existingPlaybooks
+    );
+
+    const response = await this.provider.complete(
+      [{ role: 'user', content: prompt }],
+      {
+        temperature: this.config.temperature,
+        maxTokens: this.config.maxTokens,
+      }
+    );
+
+    return this.parseExtractionResponse(response, trajectory);
+  }
+
+  /**
+   * Extract playbooks from a batch of trajectories
+   */
+  private async extractFromBatch(
+    trajectories: Trajectory[],
+    analyses: AnalysisResult[],
+    existingPlaybooks: Playbook[]
+  ): Promise<LLMExtractionResult> {
+    const prompt = this.buildBatchExtractionPrompt(
+      trajectories,
+      analyses,
+      existingPlaybooks
+    );
+
+    const response = await this.provider.complete(
+      [{ role: 'user', content: prompt }],
+      {
+        temperature: this.config.temperature,
+        maxTokens: this.config.maxTokens * 2,
+      }
+    );
+
+    return this.parseExtractionResponse(response);
+  }
+
+  /**
+   * Build prompt for single trajectory extraction
+   */
+  private buildSingleExtractionPrompt(
+    trajectory: Trajectory,
+    analysis: AnalysisResult,
+    existingPlaybooks: Playbook[]
+  ): string {
+    const keyStepsDesc = analysis.keySteps
+      .map((idx) => {
+        const step = trajectory.steps[idx];
+        return step
+          ? `  - Step ${idx + 1}: ${step.action}${step.thought ? ` (reasoning: ${step.thought})` : ''}`
+          : '';
+      })
+      .filter(Boolean)
+      .join('\n');
+
+    const existingPlaybookNames = existingPlaybooks
+      .map((p) => `- ${p.name}: ${p.guidance.strategy.slice(0, 100)}`)
+      .slice(0, 5)
+      .join('\n');
+
+    return `Analyze this successful problem-solving trajectory and extract a reusable playbook.
+
+## Task
+${trajectory.task.description}
+Domain: ${trajectory.task.domain}
+
+## Solution Steps
+${trajectory.steps.map((s, i) => `${i + 1}. ${s.action}${s.observation ? `\n   Observation: ${s.observation.slice(0, 100)}` : ''}`).join('\n')}
+
+## Key Steps (most important for success)
+${keyStepsDesc}
+
+## Outcome
+${trajectory.outcome.success ? 'SUCCESS' : 'FAILURE'}: ${trajectory.outcome.solution ?? trajectory.outcome.errorInfo ?? 'No details'}
+
+${existingPlaybooks.length > 0 ? `## Existing Playbooks (avoid duplicates)
+${existingPlaybookNames}` : ''}
+
+## Instructions
+Extract a reusable playbook that captures the essential approach. Focus on:
+1. What situations trigger this approach
+2. The core strategy/pattern used
+3. Key tactical steps
+4. Success/failure indicators
+
+Respond in JSON format:
+{
+  "playbook": {
+    "name": "kebab-case-name",
+    "situations": ["when X happens", "for Y type problems"],
+    "triggers": ["error messages", "signals that trigger this"],
+    "antiPatterns": ["when NOT to use this"],
+    "strategy": "High-level approach description",
+    "tactics": ["specific technique 1", "specific technique 2"],
+    "steps": ["step 1", "step 2", "step 3"],
+    "successIndicators": ["how to know it worked"],
+    "failureIndicators": ["how to know it failed"],
+    "complexity": "simple|moderate|complex",
+    "estimatedEffort": 5
+  },
+  "isRefinementOf": null
+}
+
+If this is better suited as a refinement to an existing playbook, set isRefinementOf to the playbook name and include only the refinement details.`;
+  }
+
+  /**
+   * Build prompt for batch extraction
+   */
+  private buildBatchExtractionPrompt(
+    trajectories: Trajectory[],
+    analyses: AnalysisResult[],
+    existingPlaybooks: Playbook[]
+  ): string {
+    const trajectorySummaries = trajectories
+      .map((t, i) => {
+        const analysis = analyses[i];
+        const keySteps = analysis?.keySteps ?? [];
+        return `### Trajectory ${i + 1}
+Task: ${t.task.description}
+Domain: ${t.task.domain}
+Steps: ${t.steps.length}
+Key steps: ${keySteps.map((idx) => t.steps[idx]?.action ?? '').join(' → ')}
+Outcome: ${t.outcome.success ? 'SUCCESS' : 'FAILURE'}`;
+      })
+      .join('\n\n');
+
+    const existingPlaybookNames = existingPlaybooks
+      .map((p) => `- ${p.name}`)
+      .slice(0, 10)
+      .join('\n');
+
+    return `Analyze these successful trajectories and identify common patterns that can be extracted as reusable playbooks.
+
+## Trajectories
+${trajectorySummaries}
+
+${existingPlaybooks.length > 0 ? `## Existing Playbooks (avoid duplicates)
+${existingPlaybookNames}` : ''}
+
+## Instructions
+Identify patterns that appear across multiple trajectories and extract them as playbooks.
+Focus on generalizable approaches, not task-specific details.
+
+Respond in JSON format:
+{
+  "playbooks": [
+    {
+      "name": "kebab-case-name",
+      "situations": ["when X happens"],
+      "triggers": [],
+      "antiPatterns": [],
+      "strategy": "Description",
+      "tactics": ["technique 1"],
+      "steps": ["step 1", "step 2"],
+      "successIndicators": ["indicator"],
+      "failureIndicators": ["indicator"],
+      "complexity": "simple|moderate|complex",
+      "estimatedEffort": 5,
+      "sourceTrajectories": [1, 2]
+    }
+  ]
+}`;
+  }
+
+  /**
+   * Parse LLM response into extraction result
+   */
+  private parseExtractionResponse(
+    response: string,
+    sourceTrajectory?: Trajectory
+  ): LLMExtractionResult {
+    const result: LLMExtractionResult = {
+      playbooks: [],
+      refinements: [],
+      rawResponse: response,
+    };
+
+    try {
+      // Extract JSON from response
+      const jsonMatch = response.match(/\{[\s\S]*\}/);
+      if (!jsonMatch) {
+        return result;
+      }
+
+      const parsed = JSON.parse(jsonMatch[0]) as {
+        playbook?: LLMPlaybookData;
+        playbooks?: LLMPlaybookData[];
+        isRefinementOf?: string | null;
+      };
+
+      // Handle single playbook response
+      if (parsed.playbook && !parsed.isRefinementOf) {
+        const playbook = this.createPlaybookFromLLMData(
+          parsed.playbook,
+          sourceTrajectory
+        );
+        result.playbooks.push(playbook);
+      }
+
+      // Handle refinement response
+      if (parsed.playbook && parsed.isRefinementOf) {
+        result.refinements.push({
+          playbookId: parsed.isRefinementOf,
+          refinement: {
+            context: parsed.playbook.situations?.[0] ?? 'General refinement',
+            addition: parsed.playbook.strategy ?? '',
+            source: 'success',
+            addedAt: new Date(),
+          },
+        });
+      }
+
+      // Handle batch response
+      if (parsed.playbooks && Array.isArray(parsed.playbooks)) {
+        for (const pbData of parsed.playbooks) {
+          const playbook = this.createPlaybookFromLLMData(pbData);
+          result.playbooks.push(playbook);
+        }
+      }
+    } catch (error) {
+      // JSON parsing failed, return empty result
+      console.warn(
+        'Failed to parse LLM extraction response:',
+        error instanceof Error ? error.message : error
+      );
+    }
+
+    return result;
+  }
+
+  /**
+   * Create a Playbook from LLM-extracted data
+   */
+  private createPlaybookFromLLMData(
+    data: LLMPlaybookData,
+    sourceTrajectory?: Trajectory
+  ): Playbook {
+    const domain = sourceTrajectory?.task.domain ?? 'general';
+    const sourceIds = sourceTrajectory ? [sourceTrajectory.id] : [];
+
+    return createPlaybook({
+      name: data.name ?? `llm-extracted-${Date.now()}`,
+      applicability: {
+        situations: data.situations ?? [],
+        triggers: data.triggers ?? [],
+        antiPatterns: data.antiPatterns ?? [],
+        domains: [domain],
+      },
+      guidance: {
+        strategy: data.strategy ?? '',
+        tactics: data.tactics ?? [],
+        steps: data.steps ?? [],
+      },
+      verification: {
+        successIndicators: data.successIndicators ?? [],
+        failureIndicators: data.failureIndicators ?? [],
+      },
+      evolution: {
+        version: '1.0.0',
+        createdFrom: sourceIds,
+        failures: [],
+        refinements: [],
+        successCount: 1,
+        failureCount: 0,
+      },
+      confidence: 0.6, // Start with moderate confidence for LLM-extracted
+      complexity: this.parseComplexity(data.complexity),
+      estimatedEffort: data.estimatedEffort ?? 5,
+    });
+  }
+
+  /**
+   * Parse complexity string
+   */
+  private parseComplexity(
+    complexity?: string
+  ): 'simple' | 'moderate' | 'complex' {
+    const normalized = complexity?.toLowerCase();
+    if (normalized === 'simple') return 'simple';
+    if (normalized === 'complex') return 'complex';
+    return 'moderate';
+  }
+
+  /**
+   * Check if two playbooks are similar
+   */
+  private isSimilarPlaybook(a: Playbook, b: Playbook): boolean {
+    // Simple name-based check
+    if (a.name === b.name) return true;
+
+    // Check situation overlap
+    const aSituations = new Set(a.applicability.situations);
+    const bSituations = b.applicability.situations;
+    const overlap = bSituations.filter((s) => aSituations.has(s)).length;
+
+    return overlap >= Math.min(aSituations.size, bSituations.length) * 0.8;
+  }
+
+  /**
+   * Generate a refinement for an existing playbook based on a new trajectory
+   */
+  async generateRefinement(
+    playbook: Playbook,
+    trajectory: Trajectory,
+    context: string
+  ): Promise<Refinement> {
+    const prompt = `Given this existing playbook and a new trajectory, generate a refinement.
+
+## Existing Playbook: ${playbook.name}
+Strategy: ${playbook.guidance.strategy}
+Steps: ${playbook.guidance.steps?.join(' → ') ?? 'None'}
+
+## New Trajectory
+Task: ${trajectory.task.description}
+Steps taken: ${trajectory.steps.map((s) => s.action).join(' → ')}
+Outcome: ${trajectory.outcome.success ? 'SUCCESS' : 'FAILURE'}
+
+## Context
+${context}
+
+## Instructions
+Generate a refinement that captures what's new or different in this trajectory.
+
+Respond in JSON:
+{
+  "context": "when/where this refinement applies",
+  "addition": "what to add/change in the approach"
+}`;
+
+    const response = await this.provider.complete(
+      [{ role: 'user', content: prompt }],
+      {
+        temperature: this.config.temperature,
+        maxTokens: 500,
+      }
+    );
+
+    try {
+      const jsonMatch = response.match(/\{[\s\S]*\}/);
+      if (jsonMatch) {
+        const parsed = JSON.parse(jsonMatch[0]) as {
+          context?: string;
+          addition?: string;
+        };
+
+        return {
+          context: parsed.context ?? context,
+          addition: parsed.addition ?? '',
+          source: trajectory.outcome.success ? 'success' : 'failure',
+          addedAt: new Date(),
+        };
+      }
+    } catch {
+      // Fall back to simple refinement
+    }
+
+    return {
+      context,
+      addition: `Additional approach from trajectory: ${trajectory.steps[0]?.action ?? ''}`,
+      source: trajectory.outcome.success ? 'success' : 'failure',
+      addedAt: new Date(),
+    };
+  }
+}
+
+/**
+ * Internal type for LLM-extracted playbook data
+ */
+interface LLMPlaybookData {
+  name?: string;
+  situations?: string[];
+  triggers?: string[];
+  antiPatterns?: string[];
+  strategy?: string;
+  tactics?: string[];
+  steps?: string[];
+  successIndicators?: string[];
+  failureIndicators?: string[];
+  complexity?: string;
+  estimatedEffort?: number;
+  sourceTrajectories?: number[];
+}
+
+/**
+ * Create an LLM-based playbook extractor
+ */
+export function createLLMExtractor(
+  provider: LLMProvider,
+  config?: Partial<LLMExtractorConfig>
+): LLMPlaybookExtractor {
+  return new LLMPlaybookExtractor(provider, config);
+}