cognitive-core 0.2.5 → 0.3.1

package/src/workspace/templates/playbook-efficacy-audit.ts

@@ -1,246 +0,0 @@
-/**
- * Playbook Efficacy Audit Template
- *
- * Deep-dive analysis of a single playbook's effectiveness.
- * Uses the efficacy toolkit to pre-compute metrics, then asks an agent
- * to synthesize findings, identify root causes, and recommend actions.
- *
- * Heuristic fallback: for playbooks with very few data points, return
- * a summary without agent analysis.
- */
-
-import type { WorkspaceHandle } from 'agent-workspace';
-import type { Playbook } from '../../types/index.js';
-import type { TaskAnnotation, PlaybookEffectivenessEntry } from '../../learning/effectiveness.js';
-import type {
-  AgenticTaskTemplate,
-  AnalysisComplexity,
-} from '../types.js';
-import {
-  buildEfficacyProfile,
-  type PlaybookEfficacyProfile,
-} from '../efficacy-toolkit.js';
-
-// ============================================================
-// Input / Output Types
-// ============================================================
-
-export interface PlaybookEfficacyAuditInput {
-  playbook: Playbook;
-  annotations: TaskAnnotation[];
-  playbookEffectiveness: PlaybookEffectivenessEntry | undefined;
-  unguidedSuccessRate: number;
-  /** Map of trajectoryId → domain for per-domain breakdown */
-  trajectoryDomainMap: Map<string, string>;
-}
-
-export interface PlaybookEfficacyAuditOutput {
-  /** Pre-computed profile (from toolkit) */
-  profile: PlaybookEfficacyProfile;
-  /** Agent's synthesis and recommendations */
-  assessment: {
-    /** Overall health rating */
-    health: 'healthy' | 'at-risk' | 'underperforming' | 'insufficient-data';
-    /** Root cause analysis for any issues */
-    rootCauses: string[];
-    /** Specific actionable recommendations */
-    recommendations: PlaybookRecommendation[];
-    /** Brief narrative summary */
-    summary: string;
-  };
-}
-
-export interface PlaybookRecommendation {
-  action: 'refine-guidance' | 'add-anti-pattern' | 'narrow-scope' | 'broaden-scope'
-    | 'deprecate' | 'merge-with' | 'split' | 'no-change';
-  description: string;
-  priority: 'high' | 'medium' | 'low';
-  /** For merge-with: target playbook name */
-  targetPlaybook?: string;
-}
-
-// ============================================================
-// Template Implementation
-// ============================================================
-
-export const playbookEfficacyAuditTemplate: AgenticTaskTemplate<
-  PlaybookEfficacyAuditInput,
-  PlaybookEfficacyAuditOutput
-> = {
-  taskType: 'playbook-efficacy-audit',
-  domain: 'meta-learning',
-  description: 'Deep-dive efficacy analysis of a single playbook with recommendations',
-
-  assessComplexity(input: PlaybookEfficacyAuditInput): AnalysisComplexity {
-    const totalUses = input.playbook.evolution.successCount + input.playbook.evolution.failureCount;
-    if (totalUses < 3) return 'heuristic';
-    if (totalUses < 10 && input.playbook.evolution.failures.length === 0) return 'heuristic';
-    if (totalUses > 20 || input.playbook.evolution.failures.length > 3) return 'standard';
-    return 'lightweight';
-  },
-
-  async heuristicFallback(input: PlaybookEfficacyAuditInput): Promise<PlaybookEfficacyAuditOutput> {
-    const profile = buildEfficacyProfile(
-      input.playbook,
-      input.annotations,
-      input.playbookEffectiveness,
-      input.unguidedSuccessRate,
-      input.trajectoryDomainMap,
-    );
-
-    return {
-      profile,
-      assessment: {
-        health: profile.totalUses < 3 ? 'insufficient-data' : (
-          profile.successRate >= 0.7 ? 'healthy' :
-          profile.successRate >= 0.4 ? 'at-risk' : 'underperforming'
-        ),
-        rootCauses: [],
-        recommendations: profile.totalUses < 3
-          ? [{ action: 'no-change', description: 'Insufficient data for assessment', priority: 'low' }]
-          : [],
-        summary: profile.totalUses < 3
-          ? `${profile.playbookName} has only ${profile.totalUses} uses — not enough for reliable assessment.`
-          : `${profile.playbookName}: ${(profile.successRate * 100).toFixed(0)}% success rate, normalized gain ${profile.normalizedGain.toFixed(2)}.`,
-      },
-    };
-  },
-
-  async prepareWorkspace(
-    input: PlaybookEfficacyAuditInput,
-    handle: WorkspaceHandle
-  ): Promise<void> {
-    const profile = buildEfficacyProfile(
-      input.playbook,
-      input.annotations,
-      input.playbookEffectiveness,
-      input.unguidedSuccessRate,
-      input.trajectoryDomainMap,
-    );
-
-    // Pre-computed efficacy profile — the core data the agent analyzes
-    await handle.writeJson('input', 'efficacy-profile.json', profile);
-
-    // Full playbook definition for context
-    await handle.writeJson('input', 'playbook.json', {
-      id: input.playbook.id,
-      name: input.playbook.name,
-      applicability: input.playbook.applicability,
-      guidance: input.playbook.guidance,
-      verification: input.playbook.verification,
-      evolution: {
-        version: input.playbook.evolution.version,
-        failures: input.playbook.evolution.failures.slice(-5),
-        refinements: input.playbook.evolution.refinements,
-        successCount: input.playbook.evolution.successCount,
-        failureCount: input.playbook.evolution.failureCount,
-      },
-      confidence: input.playbook.confidence,
-      complexity: input.playbook.complexity,
-    });
-
-    // Recent trajectory annotations for this playbook
-    const relevantAnnotations = input.annotations
-      .filter(a =>
-        a.knowledgeSurfaced.playbookIds.includes(input.playbook.id)
-      )
-      .slice(-20)
-      .map(a => ({
-        trajectoryId: a.trajectoryId,
-        success: a.outcome.success,
-        stepCount: a.outcome.stepCount,
-        errorRecoveries: a.outcome.errorRecoveries,
-        wasApplied: a.knowledgeApplied.playbookIdsUsed.includes(input.playbook.id),
-        reflection: a.reflection,
-        timestamp: a.timestamp,
-      }));
-
-    await handle.writeJson('input', 'recent-annotations.json', relevantAnnotations);
-  },
-
-  buildTaskPrompt(input: PlaybookEfficacyAuditInput): string {
-    return [
-      `Evaluate the efficacy of playbook "${input.playbook.name}".`,
-      '',
-      'Read the following input files:',
-      '- input/efficacy-profile.json — Pre-computed metrics (normalized gain, temporal trend, domain breakdown, usage profile)',
-      '- input/playbook.json — The playbook definition (guidance, verification, evolution history)',
-      '- input/recent-annotations.json — Recent trajectory outcomes when this playbook was surfaced',
-      '',
-      'Analyze:',
-      '1. Is this playbook providing marginal value above the unguided baseline? (check normalizedGain)',
-      '2. Is efficacy trending up, stable, or decaying? (check temporalTrend)',
-      '3. Does it work equally well across all domains, or is it domain-specific?',
-      '4. Are agents actually adopting it when surfaced? (check usage.adoptionRate)',
-      '5. Are there recurring failure modes that suggest guidance needs updating?',
-      '',
-      'Write your assessment to output/audit.json:',
-      '```json',
-      '{',
-      '  "health": "healthy" | "at-risk" | "underperforming" | "insufficient-data",',
-      '  "rootCauses": ["reason1", "reason2"],',
-      '  "recommendations": [',
-      '    {',
-      '      "action": "refine-guidance" | "add-anti-pattern" | "narrow-scope" | "broaden-scope" | "deprecate" | "merge-with" | "split" | "no-change",',
-      '      "description": "specific actionable recommendation",',
-      '      "priority": "high" | "medium" | "low",',
-      '      "targetPlaybook": "optional: for merge-with actions"',
-      '    }',
-      '  ],',
-      '  "summary": "2-3 sentence narrative summary"',
-      '}',
-      '```',
-    ].join('\n');
-  },
-
-  getSkills() { return []; },
-  getResources() { return []; },
-
-  outputConfig: {
-    files: [
-      {
-        path: 'audit.json',
-        format: 'json' as const,
-        required: true,
-        description: 'Playbook efficacy audit results',
-      },
-    ],
-  },
-
-  async collectOutput(handle: WorkspaceHandle): Promise<PlaybookEfficacyAuditOutput> {
-    const raw = await handle.readJson('output', 'audit.json') as Record<string, unknown>;
-    const profile = await handle.readJson('input', 'efficacy-profile.json') as PlaybookEfficacyProfile;
-
-    const recommendations: PlaybookRecommendation[] = Array.isArray(raw.recommendations)
-      ? (raw.recommendations as Record<string, unknown>[]).map(r => ({
-          action: String(r.action ?? 'no-change') as PlaybookRecommendation['action'],
-          description: String(r.description ?? ''),
-          priority: String(r.priority ?? 'medium') as PlaybookRecommendation['priority'],
-          targetPlaybook: r.targetPlaybook ? String(r.targetPlaybook) : undefined,
-        }))
-      : [];
-
-    return {
-      profile,
-      assessment: {
-        health: String(raw.health ?? 'insufficient-data') as PlaybookEfficacyAuditOutput['assessment']['health'],
-        rootCauses: Array.isArray(raw.rootCauses) ? raw.rootCauses.map(String) : [],
-        recommendations,
-        summary: String(raw.summary ?? ''),
-      },
-    };
-  },
-
-  async processOutput(): Promise<void> {
-    // Caller decides how to act on recommendations
-  },
-
-  computeRequirements: {
-    mode: 'local',
-    complexity: 'lightweight',
-  },
-
-  agentType: 'claude-code',
-  timeout: 120_000,
-  captureToolCalls: true,
-};

package/src/workspace/templates/playbook-extraction.ts

@@ -1,240 +0,0 @@
-/**
- * Playbook Extraction Template
- *
- * Replaces both PlaybookExtractor (heuristic) and LLMPlaybookExtractor
- * with a unified workspace-based agentic extraction.
- *
- * - Small batches (<=3 trajectories) → heuristic fallback
- * - Larger batches → agent semantically identifies patterns and creates playbooks
- */
-
-import type { WorkspaceHandle } from 'agent-workspace';
-import type { Trajectory, Playbook } from '../../types/index.js';
-import type { AnalysisResult } from '../../learning/analyzer.js';
-import type {
-  ExtractedPlaybooks,
-} from '../../learning/playbook-extractor.js';
-import { createPlaybookExtractor } from '../../learning/playbook-extractor.js';
-import type {
-  AgenticTaskTemplate,
-  AnalysisComplexity,
-} from '../types.js';
-import type { ComputeRequirements } from '../../runtime/compute-provider.js';
-
-// ============================================================
-// Input / Output Types
-// ============================================================
-
-export interface PlaybookExtractionInput {
-  trajectories: Trajectory[];
-  analyses: AnalysisResult[];
-  existingPlaybooks: Playbook[];
-}
-
-// Output is ExtractedPlaybooks (reused from learning/playbook-extractor.ts)
-
-// ============================================================
-// Template Implementation
-// ============================================================
-
-export const playbookExtractionTemplate: AgenticTaskTemplate<
-  PlaybookExtractionInput,
-  ExtractedPlaybooks
-> = {
-  taskType: 'playbook-extraction',
-  domain: 'playbook-extraction',
-  description: 'Extract reusable playbooks from trajectory batch',
-
-  assessComplexity(input: PlaybookExtractionInput): AnalysisComplexity {
-    if (input.trajectories.length <= 3) return 'heuristic';
-    if (input.trajectories.length > 10) return 'thorough';
-    return 'standard';
-  },
-
-  async heuristicFallback(input: PlaybookExtractionInput): Promise<ExtractedPlaybooks> {
-    const extractor = createPlaybookExtractor();
-    return extractor.extract(
-      input.trajectories,
-      input.analyses,
-      input.existingPlaybooks
-    );
-  },
-
-  async prepareWorkspace(
-    input: PlaybookExtractionInput,
-    handle: WorkspaceHandle
-  ): Promise<void> {
-    // Write summary
-    const successCount = input.analyses.filter(a => a.success).length;
-    const domains = [...new Set(input.trajectories.map(t => t.task.domain))];
-    await handle.writeJson('input', 'summary.json', {
-      batchSize: input.trajectories.length,
-      domains,
-      successRate: input.trajectories.length > 0
-        ? successCount / input.trajectories.length
-        : 0,
-      existingPlaybookCount: input.existingPlaybooks.length,
-    });
-
-    // Write each trajectory with its analysis
-    for (let i = 0; i < input.trajectories.length; i++) {
-      const traj = input.trajectories[i];
-      const analysis = input.analyses[i];
-      await handle.writeJson('input', `trajectories/${traj.id}.json`, {
-        id: traj.id,
-        domain: traj.task.domain,
-        description: traj.task.description,
-        outcome: traj.outcome,
-        stepCount: traj.steps.length,
-        keySteps: analysis.keySteps,
-        abstractable: analysis.abstractable,
-        errorPatterns: analysis.errorPatterns,
-        // Include only key steps to keep workspace focused
-        steps: analysis.keySteps.map(idx => ({
-          index: idx,
-          thought: traj.steps[idx]?.thought,
-          action: traj.steps[idx]?.action,
-          observation: traj.steps[idx]?.observation,
-        })),
-      });
-    }
-
-    // Write existing playbook summaries for deduplication
-    for (const playbook of input.existingPlaybooks) {
-      await handle.writeJson('input', `existing-playbooks/${playbook.id}.json`, {
-        id: playbook.id,
-        name: playbook.name,
-        strategy: playbook.guidance.strategy,
-        tactics: playbook.guidance.tactics,
-        domains: playbook.applicability.domains,
-        successCount: playbook.evolution.successCount,
-        failureCount: playbook.evolution.failureCount,
-      });
-    }
-  },
-
-  buildTaskPrompt(input: PlaybookExtractionInput): string {
-    return [
-      `Extract reusable playbooks from ${input.trajectories.length} agent trajectories.`,
-      '',
-      'Read input/summary.json for batch overview.',
-      'Read input/trajectories/*.json for individual trajectory data.',
-      'Read input/existing-playbooks/*.json to avoid duplicating existing playbooks.',
-      '',
-      'For each pattern you identify:',
-      '1. Check if it overlaps with an existing playbook (if so, create an update, not a new playbook)',
-      '2. Extract a clear strategy and tactical steps',
-      '3. Identify when the playbook applies and when it should NOT apply',
-      '',
-      'Write results to output/extraction.json with this schema:',
-      '```json',
-      '{',
-      '  "new": [',
-      '    {',
-      '      "name": "descriptive-name",',
-      '      "strategy": "high-level approach",',
-      '      "tactics": ["step1", "step2"],',
-      '      "situations": ["when to apply"],',
-      '      "triggers": ["keywords or patterns"],',
-      '      "antiPatterns": ["when NOT to apply"],',
-      '      "domains": ["applicable domains"],',
-      '      "confidence": 0.0-1.0',
-      '    }',
-      '  ],',
-      '  "updates": [',
-      '    {',
-      '      "id": "existing-playbook-id",',
-      '      "newTrigger": "optional new trigger",',
-      '      "newAntiPattern": "optional new anti-pattern",',
-      '      "refinementContext": "when this refinement applies",',
-      '      "refinementAddition": "what to add to the playbook"',
-      '    }',
-      '  ]',
-      '}',
-      '```',
-    ].join('\n');
-  },
-
-  getSkills() { return []; },
-  getResources() { return []; },
-
-  outputConfig: {
-    files: [
-      {
-        path: 'extraction.json',
-        format: 'json' as const,
-        required: true,
-        description: 'Extracted playbooks and updates to existing playbooks',
-      },
-    ],
-  },
-
-  async collectOutput(handle: WorkspaceHandle): Promise<ExtractedPlaybooks> {
-    const raw = await handle.readJson('output', 'extraction.json') as Record<string, unknown>;
-    const { createPlaybook } = await import('../../types/index.js');
-
-    const newPlaybooks: Playbook[] = [];
-    if (Array.isArray(raw.new)) {
-      for (const item of raw.new as Record<string, unknown>[]) {
-        newPlaybooks.push(createPlaybook({
-          name: String(item.name ?? 'Unnamed Playbook'),
-          guidance: {
-            strategy: String(item.strategy ?? ''),
-            tactics: Array.isArray(item.tactics) ? item.tactics.map(String) : [],
-          },
-          applicability: {
-            situations: Array.isArray(item.situations) ? item.situations.map(String) : [],
-            triggers: Array.isArray(item.triggers) ? item.triggers.map(String) : [],
-            antiPatterns: Array.isArray(item.antiPatterns) ? item.antiPatterns.map(String) : [],
-            domains: Array.isArray(item.domains) ? item.domains.map(String) : [],
-          },
-          confidence: typeof item.confidence === 'number' ? item.confidence : 0.5,
-          complexity: 'moderate',
-        }));
-      }
-    }
-
-    const updates: ExtractedPlaybooks['updates'] = [];
-    if (Array.isArray(raw.updates)) {
-      for (const item of raw.updates as Record<string, unknown>[]) {
-        updates.push({
-          id: String(item.id ?? ''),
-          newTrigger: item.newTrigger ? String(item.newTrigger) : undefined,
-          newAntiPattern: item.newAntiPattern ? String(item.newAntiPattern) : undefined,
-          refinement: item.refinementContext ? {
-            context: String(item.refinementContext),
-            addition: String(item.refinementAddition ?? ''),
-            source: 'success' as const,
-            addedAt: new Date(),
-          } : undefined,
-        });
-      }
-    }
-
-    return { new: newPlaybooks, updates };
-  },
-
-  async processOutput(): Promise<void> {
-    // Pipeline handles storage
-  },
-
-  computeRequirements: {
-    mode: 'local',
-    complexity: 'standard',
-  },
-
-  getComputeRequirements(
-    _input: PlaybookExtractionInput,
-    complexity: AnalysisComplexity
-  ): ComputeRequirements {
-    return {
-      mode: 'local',
-      complexity,
-      timeout: complexity === 'thorough' ? 300_000 : 180_000,
-    };
-  },
-
-  agentType: 'claude-code',
-  timeout: 180_000,
-  captureToolCalls: true,
-};