cognitive-core 0.1.0 → 0.1.2

package/src/learning/llm-extractor.ts

@@ -1,542 +0,0 @@
-/**
- * LLM-Based Playbook Extractor
- *
- * Uses language models to generate higher-quality playbooks
- * from trajectories. Complements the pattern-based extractor
- * with deeper semantic analysis.
- */
-
-import type {
-  Playbook,
-  Trajectory,
-  Refinement,
-} from '../types/index.js';
-import { createPlaybook } from '../types/index.js';
-import type { AnalysisResult } from './analyzer.js';
-import type { LLMProvider } from '../agents/types.js';
-
-/**
- * Configuration for LLM-based extraction
- */
-export interface LLMExtractorConfig {
-  /** Temperature for LLM calls (default: 0.2 for determinism) */
-  temperature: number;
-  /** Maximum tokens for response */
-  maxTokens: number;
-  /** Whether to extract from single trajectories (vs only batches) */
-  extractFromSingle: boolean;
-  /** Minimum complexity for LLM extraction */
-  minComplexity: number;
-}
-
-const DEFAULT_CONFIG: LLMExtractorConfig = {
-  temperature: 0.2,
-  maxTokens: 2000,
-  extractFromSingle: true,
-  minComplexity: 3, // Minimum steps to warrant LLM extraction
-};
-
-/**
- * Result from LLM extraction
- */
-export interface LLMExtractionResult {
-  playbooks: Playbook[];
-  refinements: Array<{
-    playbookId: string;
-    refinement: Refinement;
-  }>;
-  /** Raw LLM response for debugging */
-  rawResponse?: string;
-}
-
-/**
- * LLM-Based Playbook Extractor
- */
-export class LLMPlaybookExtractor {
-  private provider: LLMProvider;
-  private config: LLMExtractorConfig;
-
-  constructor(provider: LLMProvider, config?: Partial<LLMExtractorConfig>) {
-    this.provider = provider;
-    this.config = { ...DEFAULT_CONFIG, ...config };
-  }
-
-  /**
-   * Extract playbooks from trajectories using LLM
-   */
-  async extract(
-    trajectories: Trajectory[],
-    analyses: AnalysisResult[],
-    existingPlaybooks: Playbook[] = []
-  ): Promise<LLMExtractionResult> {
-    // Filter to successful, complex enough trajectories
-    const candidates = trajectories.filter((t, i) => {
-      const analysis = analyses[i];
-      return (
-        analysis?.success &&
-        analysis.abstractable &&
-        t.steps.length >= this.config.minComplexity
-      );
-    });
-
-    if (candidates.length === 0) {
-      return { playbooks: [], refinements: [] };
-    }
-
-    // Extract playbooks from each candidate
-    const allResults: LLMExtractionResult = {
-      playbooks: [],
-      refinements: [],
-    };
-
-    // Process individually for better quality
-    if (this.config.extractFromSingle) {
-      for (let i = 0; i < candidates.length; i++) {
-        const trajectory = candidates[i];
-        const analysis = analyses.find(
-          (_, idx) => trajectories[idx] === trajectory
-        );
-
-        if (!analysis) continue;
-
-        const result = await this.extractFromSingleTrajectory(
-          trajectory,
-          analysis,
-          existingPlaybooks
-        );
-
-        allResults.playbooks.push(...result.playbooks);
-        allResults.refinements.push(...result.refinements);
-      }
-    }
-
-    // Also do batch extraction if we have multiple
-    if (candidates.length >= 2) {
-      const batchResult = await this.extractFromBatch(
-        candidates,
-        analyses.filter((_, i) => candidates.includes(trajectories[i])),
-        existingPlaybooks
-      );
-
-      // Merge with individual results (avoid duplicates)
-      for (const playbook of batchResult.playbooks) {
-        const isDuplicate = allResults.playbooks.some(
-          (p) => this.isSimilarPlaybook(p, playbook)
-        );
-        if (!isDuplicate) {
-          allResults.playbooks.push(playbook);
-        }
-      }
-
-      allResults.refinements.push(...batchResult.refinements);
-    }
-
-    return allResults;
-  }
-
-  /**
-   * Extract a playbook from a single trajectory
-   */
-  private async extractFromSingleTrajectory(
-    trajectory: Trajectory,
-    analysis: AnalysisResult,
-    existingPlaybooks: Playbook[]
-  ): Promise<LLMExtractionResult> {
-    const prompt = this.buildSingleExtractionPrompt(
-      trajectory,
-      analysis,
-      existingPlaybooks
-    );
-
-    const response = await this.provider.complete(
-      [{ role: 'user', content: prompt }],
-      {
-        temperature: this.config.temperature,
-        maxTokens: this.config.maxTokens,
-      }
-    );
-
-    return this.parseExtractionResponse(response, trajectory);
-  }
-
-  /**
-   * Extract playbooks from a batch of trajectories
-   */
-  private async extractFromBatch(
-    trajectories: Trajectory[],
-    analyses: AnalysisResult[],
-    existingPlaybooks: Playbook[]
-  ): Promise<LLMExtractionResult> {
-    const prompt = this.buildBatchExtractionPrompt(
-      trajectories,
-      analyses,
-      existingPlaybooks
-    );
-
-    const response = await this.provider.complete(
-      [{ role: 'user', content: prompt }],
-      {
-        temperature: this.config.temperature,
-        maxTokens: this.config.maxTokens * 2,
-      }
-    );
-
-    return this.parseExtractionResponse(response);
-  }
-
-  /**
-   * Build prompt for single trajectory extraction
-   */
-  private buildSingleExtractionPrompt(
-    trajectory: Trajectory,
-    analysis: AnalysisResult,
-    existingPlaybooks: Playbook[]
-  ): string {
-    const keyStepsDesc = analysis.keySteps
-      .map((idx) => {
-        const step = trajectory.steps[idx];
-        return step
-          ? `  - Step ${idx + 1}: ${step.action}${step.thought ? ` (reasoning: ${step.thought})` : ''}`
-          : '';
-      })
-      .filter(Boolean)
-      .join('\n');
-
-    const existingPlaybookNames = existingPlaybooks
-      .map((p) => `- ${p.name}: ${p.guidance.strategy.slice(0, 100)}`)
-      .slice(0, 5)
-      .join('\n');
-
-    return `Analyze this successful problem-solving trajectory and extract a reusable playbook.
-
-## Task
-${trajectory.task.description}
-Domain: ${trajectory.task.domain}
-
-## Solution Steps
-${trajectory.steps.map((s, i) => `${i + 1}. ${s.action}${s.observation ? `\n   Observation: ${s.observation.slice(0, 100)}` : ''}`).join('\n')}
-
-## Key Steps (most important for success)
-${keyStepsDesc}
-
-## Outcome
-${trajectory.outcome.success ? 'SUCCESS' : 'FAILURE'}: ${trajectory.outcome.solution ?? trajectory.outcome.errorInfo ?? 'No details'}
-
-${existingPlaybooks.length > 0 ? `## Existing Playbooks (avoid duplicates)
-${existingPlaybookNames}` : ''}
-
-## Instructions
-Extract a reusable playbook that captures the essential approach. Focus on:
-1. What situations trigger this approach
-2. The core strategy/pattern used
-3. Key tactical steps
-4. Success/failure indicators
-
-Respond in JSON format:
-{
-  "playbook": {
-    "name": "kebab-case-name",
-    "situations": ["when X happens", "for Y type problems"],
-    "triggers": ["error messages", "signals that trigger this"],
-    "antiPatterns": ["when NOT to use this"],
-    "strategy": "High-level approach description",
-    "tactics": ["specific technique 1", "specific technique 2"],
-    "steps": ["step 1", "step 2", "step 3"],
-    "successIndicators": ["how to know it worked"],
-    "failureIndicators": ["how to know it failed"],
-    "complexity": "simple|moderate|complex",
-    "estimatedEffort": 5
-  },
-  "isRefinementOf": null
-}
-
-If this is better suited as a refinement to an existing playbook, set isRefinementOf to the playbook name and include only the refinement details.`;
-  }
-
-  /**
-   * Build prompt for batch extraction
-   */
-  private buildBatchExtractionPrompt(
-    trajectories: Trajectory[],
-    analyses: AnalysisResult[],
-    existingPlaybooks: Playbook[]
-  ): string {
-    const trajectorySummaries = trajectories
-      .map((t, i) => {
-        const analysis = analyses[i];
-        const keySteps = analysis?.keySteps ?? [];
-        return `### Trajectory ${i + 1}
-Task: ${t.task.description}
-Domain: ${t.task.domain}
-Steps: ${t.steps.length}
-Key steps: ${keySteps.map((idx) => t.steps[idx]?.action ?? '').join(' → ')}
-Outcome: ${t.outcome.success ? 'SUCCESS' : 'FAILURE'}`;
-      })
-      .join('\n\n');
-
-    const existingPlaybookNames = existingPlaybooks
-      .map((p) => `- ${p.name}`)
-      .slice(0, 10)
-      .join('\n');
-
-    return `Analyze these successful trajectories and identify common patterns that can be extracted as reusable playbooks.
-
-## Trajectories
-${trajectorySummaries}
-
-${existingPlaybooks.length > 0 ? `## Existing Playbooks (avoid duplicates)
-${existingPlaybookNames}` : ''}
-
-## Instructions
-Identify patterns that appear across multiple trajectories and extract them as playbooks.
-Focus on generalizable approaches, not task-specific details.
-
-Respond in JSON format:
-{
-  "playbooks": [
-    {
-      "name": "kebab-case-name",
-      "situations": ["when X happens"],
-      "triggers": [],
-      "antiPatterns": [],
-      "strategy": "Description",
-      "tactics": ["technique 1"],
-      "steps": ["step 1", "step 2"],
-      "successIndicators": ["indicator"],
-      "failureIndicators": ["indicator"],
-      "complexity": "simple|moderate|complex",
-      "estimatedEffort": 5,
-      "sourceTrajectories": [1, 2]
-    }
-  ]
-}`;
-  }
-
-  /**
-   * Parse LLM response into extraction result
-   */
-  private parseExtractionResponse(
-    response: string,
-    sourceTrajectory?: Trajectory
-  ): LLMExtractionResult {
-    const result: LLMExtractionResult = {
-      playbooks: [],
-      refinements: [],
-      rawResponse: response,
-    };
-
-    try {
-      // Extract JSON from response
-      const jsonMatch = response.match(/\{[\s\S]*\}/);
-      if (!jsonMatch) {
-        return result;
-      }
-
-      const parsed = JSON.parse(jsonMatch[0]) as {
-        playbook?: LLMPlaybookData;
-        playbooks?: LLMPlaybookData[];
-        isRefinementOf?: string | null;
-      };
-
-      // Handle single playbook response
-      if (parsed.playbook && !parsed.isRefinementOf) {
-        const playbook = this.createPlaybookFromLLMData(
-          parsed.playbook,
-          sourceTrajectory
-        );
-        result.playbooks.push(playbook);
-      }
-
-      // Handle refinement response
-      if (parsed.playbook && parsed.isRefinementOf) {
-        result.refinements.push({
-          playbookId: parsed.isRefinementOf,
-          refinement: {
-            context: parsed.playbook.situations?.[0] ?? 'General refinement',
-            addition: parsed.playbook.strategy ?? '',
-            source: 'success',
-            addedAt: new Date(),
-          },
-        });
-      }
-
-      // Handle batch response
-      if (parsed.playbooks && Array.isArray(parsed.playbooks)) {
-        for (const pbData of parsed.playbooks) {
-          const playbook = this.createPlaybookFromLLMData(pbData);
-          result.playbooks.push(playbook);
-        }
-      }
-    } catch (error) {
-      // JSON parsing failed, return empty result
-      console.warn(
-        'Failed to parse LLM extraction response:',
-        error instanceof Error ? error.message : error
-      );
-    }
-
-    return result;
-  }
-
-  /**
-   * Create a Playbook from LLM-extracted data
-   */
-  private createPlaybookFromLLMData(
-    data: LLMPlaybookData,
-    sourceTrajectory?: Trajectory
-  ): Playbook {
-    const domain = sourceTrajectory?.task.domain ?? 'general';
-    const sourceIds = sourceTrajectory ? [sourceTrajectory.id] : [];
-
-    return createPlaybook({
-      name: data.name ?? `llm-extracted-${Date.now()}`,
-      applicability: {
-        situations: data.situations ?? [],
-        triggers: data.triggers ?? [],
-        antiPatterns: data.antiPatterns ?? [],
-        domains: [domain],
-      },
-      guidance: {
-        strategy: data.strategy ?? '',
-        tactics: data.tactics ?? [],
-        steps: data.steps ?? [],
-      },
-      verification: {
-        successIndicators: data.successIndicators ?? [],
-        failureIndicators: data.failureIndicators ?? [],
-      },
-      evolution: {
-        version: '1.0.0',
-        createdFrom: sourceIds,
-        failures: [],
-        refinements: [],
-        successCount: 1,
-        failureCount: 0,
-      },
-      confidence: 0.6, // Start with moderate confidence for LLM-extracted
-      complexity: this.parseComplexity(data.complexity),
-      estimatedEffort: data.estimatedEffort ?? 5,
-    });
-  }
-
-  /**
-   * Parse complexity string
-   */
-  private parseComplexity(
-    complexity?: string
-  ): 'simple' | 'moderate' | 'complex' {
-    const normalized = complexity?.toLowerCase();
-    if (normalized === 'simple') return 'simple';
-    if (normalized === 'complex') return 'complex';
-    return 'moderate';
-  }
-
-  /**
-   * Check if two playbooks are similar
-   */
-  private isSimilarPlaybook(a: Playbook, b: Playbook): boolean {
-    // Simple name-based check
-    if (a.name === b.name) return true;
-
-    // Check situation overlap
-    const aSituations = new Set(a.applicability.situations);
-    const bSituations = b.applicability.situations;
-    const overlap = bSituations.filter((s) => aSituations.has(s)).length;
-
-    return overlap >= Math.min(aSituations.size, bSituations.length) * 0.8;
-  }
-
-  /**
-   * Generate a refinement for an existing playbook based on a new trajectory
-   */
-  async generateRefinement(
-    playbook: Playbook,
-    trajectory: Trajectory,
-    context: string
-  ): Promise<Refinement> {
-    const prompt = `Given this existing playbook and a new trajectory, generate a refinement.
-
-## Existing Playbook: ${playbook.name}
-Strategy: ${playbook.guidance.strategy}
-Steps: ${playbook.guidance.steps?.join(' → ') ?? 'None'}
-
-## New Trajectory
-Task: ${trajectory.task.description}
-Steps taken: ${trajectory.steps.map((s) => s.action).join(' → ')}
-Outcome: ${trajectory.outcome.success ? 'SUCCESS' : 'FAILURE'}
-
-## Context
-${context}
-
-## Instructions
-Generate a refinement that captures what's new or different in this trajectory.
-
-Respond in JSON:
-{
-  "context": "when/where this refinement applies",
-  "addition": "what to add/change in the approach"
-}`;
-
-    const response = await this.provider.complete(
-      [{ role: 'user', content: prompt }],
-      {
-        temperature: this.config.temperature,
-        maxTokens: 500,
-      }
-    );
-
-    try {
-      const jsonMatch = response.match(/\{[\s\S]*\}/);
-      if (jsonMatch) {
-        const parsed = JSON.parse(jsonMatch[0]) as {
-          context?: string;
-          addition?: string;
-        };
-
-        return {
-          context: parsed.context ?? context,
-          addition: parsed.addition ?? '',
-          source: trajectory.outcome.success ? 'success' : 'failure',
-          addedAt: new Date(),
-        };
-      }
-    } catch {
-      // Fall back to simple refinement
-    }
-
-    return {
-      context,
-      addition: `Additional approach from trajectory: ${trajectory.steps[0]?.action ?? ''}`,
-      source: trajectory.outcome.success ? 'success' : 'failure',
-      addedAt: new Date(),
-    };
-  }
-}
-
-/**
- * Internal type for LLM-extracted playbook data
- */
-interface LLMPlaybookData {
-  name?: string;
-  situations?: string[];
-  triggers?: string[];
-  antiPatterns?: string[];
-  strategy?: string;
-  tactics?: string[];
-  steps?: string[];
-  successIndicators?: string[];
-  failureIndicators?: string[];
-  complexity?: string;
-  estimatedEffort?: number;
-  sourceTrajectories?: number[];
-}
-
-/**
- * Create an LLM-based playbook extractor
- */
-export function createLLMExtractor(
-  provider: LLMProvider,
-  config?: Partial<LLMExtractorConfig>
-): LLMPlaybookExtractor {
-  return new LLMPlaybookExtractor(provider, config);
-}