cognitive-core 0.0.2 → 0.1.0

package/src/learning/usage-inference.ts

@@ -0,0 +1,372 @@
+/**
+ * Playbook Usage Inference
+ * Infers which playbooks were used and whether they helped
+ * from trajectory analysis (no explicit agent reporting needed)
+ */
+
+import type { Trajectory, Step, Playbook } from '../types/index.js';
+import { textSimilarity } from '../utils/similarity.js';
+
+/**
+ * Result of playbook usage inference
+ */
+export interface PlaybookUsageResult {
+  playbookId: string;
+  playbookName: string;
+  /** Did agent follow this playbook? */
+  wasUsed: boolean;
+  /** How confident are we it was used? */
+  usageConfidence: number;
+  /** Did it lead to success? */
+  wasSuccessful: boolean;
+  /** Which trajectory steps matched */
+  matchedSteps: number[];
+  /** Relevant context for refinement */
+  context?: string;
+}
+
+/**
+ * Internal result of matching steps to playbook
+ */
+interface PlaybookMatchResult {
+  matched: boolean;
+  matchedTactics: number;
+  totalTactics: number;
+  matchedSteps: number[];
+  confidence: number;
+}
+
+/**
+ * Configuration for usage inference
+ */
+export interface UsageInferenceConfig {
+  /** Minimum similarity for tactic matching */
+  tacticMatchThreshold: number;
+  /** Minimum tactics matched to consider playbook used */
+  minTacticsMatchRatio: number;
+  /** Minimum confidence to report as used */
+  minUsageConfidence: number;
+}
+
+const DEFAULT_CONFIG: UsageInferenceConfig = {
+  tacticMatchThreshold: 0.5,
+  minTacticsMatchRatio: 0.3,
+  minUsageConfidence: 0.4,
+};
+
+/**
+ * Playbook Usage Inference
+ * Analyzes trajectories to determine if playbooks were used
+ */
+export class PlaybookUsageInference {
+  private config: UsageInferenceConfig;
+
+  constructor(config?: Partial<UsageInferenceConfig>) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Analyze trajectory to infer playbook usage
+   */
+  async inferUsage(
+    trajectory: Trajectory,
+    injectedPlaybooks: Playbook[]
+  ): Promise<PlaybookUsageResult[]> {
+    const results: PlaybookUsageResult[] = [];
+
+    for (const playbook of injectedPlaybooks) {
+      const matchResult = this.matchStepsToPlaybook(
+        trajectory.steps,
+        playbook
+      );
+
+      const wasUsed =
+        matchResult.matched &&
+        matchResult.confidence >= this.config.minUsageConfidence;
+
+      const wasSuccessful = wasUsed
+        ? this.inferSuccess(trajectory, playbook, matchResult)
+        : false;
+
+      const context = wasUsed
+        ? this.extractRelevantContext(trajectory, playbook, matchResult)
+        : undefined;
+
+      results.push({
+        playbookId: playbook.id,
+        playbookName: playbook.name,
+        wasUsed,
+        usageConfidence: matchResult.confidence,
+        wasSuccessful,
+        matchedSteps: matchResult.matchedSteps,
+        context,
+      });
+    }
+
+    return results;
+  }
+
+  /**
+   * Match trajectory steps against playbook guidance
+   */
+  private matchStepsToPlaybook(
+    steps: Step[],
+    playbook: Playbook
+  ): PlaybookMatchResult {
+    const tactics = playbook.guidance.tactics;
+
+    if (tactics.length === 0) {
+      // No tactics to match - check strategy
+      const strategyMatch = this.matchAgainstStrategy(steps, playbook);
+      return {
+        matched: strategyMatch.confidence > 0.3,
+        matchedTactics: 0,
+        totalTactics: 0,
+        matchedSteps: strategyMatch.matchedSteps,
+        confidence: strategyMatch.confidence,
+      };
+    }
+
+    const matchedSteps: number[] = [];
+    let matchedTactics = 0;
+
+    // Try to match each tactic to trajectory steps
+    for (const tactic of tactics) {
+      const tacticWords = this.extractKeywords(tactic);
+
+      for (let i = 0; i < steps.length; i++) {
+        if (matchedSteps.includes(i)) continue;
+
+        const step = steps[i];
+        const stepText = `${step.thought ?? ''} ${step.action}`.toLowerCase();
+        const stepWords = this.extractKeywords(stepText);
+
+        // Check word overlap
+        const overlap = tacticWords.filter((w) => stepWords.includes(w)).length;
+        const overlapRatio = overlap / Math.max(tacticWords.length, 1);
+
+        // Also check semantic similarity
+        const similarity = textSimilarity(tactic, stepText);
+
+        if (
+          overlapRatio >= this.config.tacticMatchThreshold ||
+          similarity >= this.config.tacticMatchThreshold
+        ) {
+          matchedTactics++;
+          matchedSteps.push(i);
+          break;
+        }
+      }
+    }
+
+    const matchRatio = matchedTactics / tactics.length;
+    const matched = matchRatio >= this.config.minTacticsMatchRatio;
+
+    // Calculate confidence based on match ratio and step coverage
+    const stepCoverage = matchedSteps.length / Math.max(steps.length, 1);
+    const confidence = matched ? (matchRatio * 0.7 + stepCoverage * 0.3) : 0;
+
+    return {
+      matched,
+      matchedTactics,
+      totalTactics: tactics.length,
+      matchedSteps,
+      confidence,
+    };
+  }
+
+  /**
+   * Match steps against playbook strategy (when no tactics)
+   */
+  private matchAgainstStrategy(
+    steps: Step[],
+    playbook: Playbook
+  ): { confidence: number; matchedSteps: number[] } {
+    const strategy = playbook.guidance.strategy;
+    const strategyWords = this.extractKeywords(strategy);
+    const matchedSteps: number[] = [];
+
+    for (let i = 0; i < steps.length; i++) {
+      const step = steps[i];
+      const stepText = `${step.thought ?? ''} ${step.action}`.toLowerCase();
+      const stepWords = this.extractKeywords(stepText);
+
+      const overlap = strategyWords.filter((w) => stepWords.includes(w)).length;
+      const similarity = textSimilarity(strategy, stepText);
+
+      if (overlap >= 2 || similarity >= 0.4) {
+        matchedSteps.push(i);
+      }
+    }
+
+    const confidence =
+      matchedSteps.length > 0
+        ? Math.min(0.6, matchedSteps.length / steps.length + 0.2)
+        : 0;
+
+    return { confidence, matchedSteps };
+  }
+
+  /**
+   * Extract keywords from text
+   */
+  private extractKeywords(text: string): string[] {
+    return text
+      .toLowerCase()
+      .replace(/[^\w\s]/g, ' ')
+      .split(/\s+/)
+      .filter((w) => w.length > 2 && !this.isStopWord(w));
+  }
+
+  /**
+   * Check if word is a stop word
+   */
+  private isStopWord(word: string): boolean {
+    const stopWords = new Set([
+      'the',
+      'a',
+      'an',
+      'and',
+      'or',
+      'but',
+      'is',
+      'are',
+      'was',
+      'were',
+      'be',
+      'been',
+      'have',
+      'has',
+      'had',
+      'do',
+      'does',
+      'did',
+      'will',
+      'would',
+      'could',
+      'should',
+      'can',
+      'to',
+      'of',
+      'in',
+      'for',
+      'on',
+      'with',
+      'at',
+      'by',
+      'from',
+      'as',
+      'then',
+      'that',
+      'this',
+      'these',
+      'those',
+      'you',
+      'your',
+      'they',
+      'their',
+    ]);
+    return stopWords.has(word);
+  }
+
+  /**
+   * Determine if playbook application was successful
+   */
+  private inferSuccess(
+    trajectory: Trajectory,
+    playbook: Playbook,
+    matchResult: PlaybookMatchResult
+  ): boolean {
+    // Primary: Check trajectory outcome
+    if (!trajectory.outcome.success) {
+      return false;
+    }
+
+    // Check if success indicators are present
+    const successIndicators = playbook.verification.successIndicators;
+    if (successIndicators.length > 0) {
+      const outcomeText = JSON.stringify(trajectory.outcome).toLowerCase();
+      const hasIndicator = successIndicators.some((indicator) =>
+        outcomeText.includes(indicator.toLowerCase())
+      );
+      if (hasIndicator) return true;
+    }
+
+    // Check if failure indicators are absent
+    const failureIndicators = playbook.verification.failureIndicators;
+    if (failureIndicators.length > 0) {
+      const trajectoryText = trajectory.steps
+        .map((s: Step) => `${s.action} ${s.observation ?? ''}`)
+        .join(' ')
+        .toLowerCase();
+
+      const hasFailureIndicator = failureIndicators.some((indicator) =>
+        trajectoryText.includes(indicator.toLowerCase())
+      );
+
+      if (hasFailureIndicator) return false;
+    }
+
+    // Default to trajectory success with high match confidence
+    return matchResult.confidence >= 0.5;
+  }
+
+  /**
+   * Extract relevant context for potential refinement
+   */
+  private extractRelevantContext(
+    trajectory: Trajectory,
+    _playbook: Playbook,
+    _matchResult: PlaybookMatchResult
+  ): string | undefined {
+    // Look for context in task metadata
+    const metadata = trajectory.task.metadata ?? {};
+
+    if (metadata.fileType) return `${metadata.fileType} files`;
+    if (metadata.framework) return `${metadata.framework} project`;
+    if (metadata.language) return `${metadata.language} codebase`;
+
+    // Check for specific patterns in steps
+    const allText = trajectory.steps
+      .map((s: Step) => `${s.thought ?? ''} ${s.action} ${s.observation ?? ''}`)
+      .join(' ')
+      .toLowerCase();
+
+    if (allText.includes('monorepo')) return 'monorepo';
+    if (allText.includes('typescript')) return 'typescript';
+    if (allText.includes('react')) return 'react';
+    if (allText.includes('node')) return 'nodejs';
+    if (allText.includes('python')) return 'python';
+
+    // Fall back to domain
+    return trajectory.task.domain || undefined;
+  }
+
+  /**
+   * Batch analyze multiple trajectories
+   */
+  async inferUsageBatch(
+    trajectories: Array<{
+      trajectory: Trajectory;
+      injectedPlaybooks: Playbook[];
+    }>
+  ): Promise<Map<string, PlaybookUsageResult[]>> {
+    const results = new Map<string, PlaybookUsageResult[]>();
+
+    for (const { trajectory, injectedPlaybooks } of trajectories) {
+      const usage = await this.inferUsage(trajectory, injectedPlaybooks);
+      results.set(trajectory.id, usage);
+    }
+
+    return results;
+  }
+}
+
+/**
+ * Create a playbook usage inference instance
+ */
+export function createUsageInference(
+  config?: Partial<UsageInferenceConfig>
+): PlaybookUsageInference {
+  return new PlaybookUsageInference(config);
+}

package/src/mcp/index.ts

@@ -0,0 +1,12 @@
+/**
+ * @deprecated The MCP server is deprecated in favor of the CLI + SKILL.md approach.
+ * See /SKILL.md for agent instructions and use bin/cognitive-core CLI instead.
+ */
+export {
+  PlaybookMCPServer,
+  createPlaybookMCPServer,
+  type MCPTool,
+  type MCPToolCall,
+  type MCPToolResult,
+  type PlaybookSearchResult,
+} from './playbook-server.js';