cognitive-core 0.0.2 → 0.1.2

package/src/workspace/templates/playbook-extraction.ts

@@ -0,0 +1,240 @@
+/**
+ * Playbook Extraction Template
+ *
+ * Replaces both PlaybookExtractor (heuristic) and LLMPlaybookExtractor
+ * with a unified workspace-based agentic extraction.
+ *
+ * - Small batches (<=3 trajectories) → heuristic fallback
+ * - Larger batches → agent semantically identifies patterns and creates playbooks
+ */
+
+import type { WorkspaceHandle } from 'agent-workspace';
+import type { Trajectory, Playbook } from '../../types/index.js';
+import type { AnalysisResult } from '../../learning/analyzer.js';
+import type {
+  ExtractedPlaybooks,
+} from '../../learning/playbook-extractor.js';
+import { createPlaybookExtractor } from '../../learning/playbook-extractor.js';
+import type {
+  AgenticTaskTemplate,
+  AnalysisComplexity,
+} from '../types.js';
+import type { ComputeRequirements } from '../../runtime/compute-provider.js';
+
+// ============================================================
+// Input / Output Types
+// ============================================================
+
+export interface PlaybookExtractionInput {
+  trajectories: Trajectory[];
+  analyses: AnalysisResult[];
+  existingPlaybooks: Playbook[];
+}
+
+// Output is ExtractedPlaybooks (reused from learning/playbook-extractor.ts)
+
+// ============================================================
+// Template Implementation
+// ============================================================
+
+export const playbookExtractionTemplate: AgenticTaskTemplate<
+  PlaybookExtractionInput,
+  ExtractedPlaybooks
+> = {
+  taskType: 'playbook-extraction',
+  domain: 'playbook-extraction',
+  description: 'Extract reusable playbooks from trajectory batch',
+
+  assessComplexity(input: PlaybookExtractionInput): AnalysisComplexity {
+    if (input.trajectories.length <= 3) return 'heuristic';
+    if (input.trajectories.length > 10) return 'thorough';
+    return 'standard';
+  },
+
+  async heuristicFallback(input: PlaybookExtractionInput): Promise<ExtractedPlaybooks> {
+    const extractor = createPlaybookExtractor();
+    return extractor.extract(
+      input.trajectories,
+      input.analyses,
+      input.existingPlaybooks
+    );
+  },
+
+  async prepareWorkspace(
+    input: PlaybookExtractionInput,
+    handle: WorkspaceHandle
+  ): Promise<void> {
+    // Write summary
+    const successCount = input.analyses.filter(a => a.success).length;
+    const domains = [...new Set(input.trajectories.map(t => t.task.domain))];
+    await handle.writeJson('input', 'summary.json', {
+      batchSize: input.trajectories.length,
+      domains,
+      successRate: input.trajectories.length > 0
+        ? successCount / input.trajectories.length
+        : 0,
+      existingPlaybookCount: input.existingPlaybooks.length,
+    });
+
+    // Write each trajectory with its analysis
+    for (let i = 0; i < input.trajectories.length; i++) {
+      const traj = input.trajectories[i];
+      const analysis = input.analyses[i];
+      await handle.writeJson('input', `trajectories/${traj.id}.json`, {
+        id: traj.id,
+        domain: traj.task.domain,
+        description: traj.task.description,
+        outcome: traj.outcome,
+        stepCount: traj.steps.length,
+        keySteps: analysis.keySteps,
+        abstractable: analysis.abstractable,
+        errorPatterns: analysis.errorPatterns,
+        // Include only key steps to keep workspace focused
+        steps: analysis.keySteps.map(idx => ({
+          index: idx,
+          thought: traj.steps[idx]?.thought,
+          action: traj.steps[idx]?.action,
+          observation: traj.steps[idx]?.observation,
+        })),
+      });
+    }
+
+    // Write existing playbook summaries for deduplication
+    for (const playbook of input.existingPlaybooks) {
+      await handle.writeJson('input', `existing-playbooks/${playbook.id}.json`, {
+        id: playbook.id,
+        name: playbook.name,
+        strategy: playbook.guidance.strategy,
+        tactics: playbook.guidance.tactics,
+        domains: playbook.applicability.domains,
+        successCount: playbook.evolution.successCount,
+        failureCount: playbook.evolution.failureCount,
+      });
+    }
+  },
+
+  buildTaskPrompt(input: PlaybookExtractionInput): string {
+    return [
+      `Extract reusable playbooks from ${input.trajectories.length} agent trajectories.`,
+      '',
+      'Read input/summary.json for batch overview.',
+      'Read input/trajectories/*.json for individual trajectory data.',
+      'Read input/existing-playbooks/*.json to avoid duplicating existing playbooks.',
+      '',
+      'For each pattern you identify:',
+      '1. Check if it overlaps with an existing playbook (if so, create an update, not a new playbook)',
+      '2. Extract a clear strategy and tactical steps',
+      '3. Identify when the playbook applies and when it should NOT apply',
+      '',
+      'Write results to output/extraction.json with this schema:',
+      '```json',
+      '{',
+      '  "new": [',
+      '    {',
+      '      "name": "descriptive-name",',
+      '      "strategy": "high-level approach",',
+      '      "tactics": ["step1", "step2"],',
+      '      "situations": ["when to apply"],',
+      '      "triggers": ["keywords or patterns"],',
+      '      "antiPatterns": ["when NOT to apply"],',
+      '      "domains": ["applicable domains"],',
+      '      "confidence": 0.0-1.0',
+      '    }',
+      '  ],',
+      '  "updates": [',
+      '    {',
+      '      "id": "existing-playbook-id",',
+      '      "newTrigger": "optional new trigger",',
+      '      "newAntiPattern": "optional new anti-pattern",',
+      '      "refinementContext": "when this refinement applies",',
+      '      "refinementAddition": "what to add to the playbook"',
+      '    }',
+      '  ]',
+      '}',
+      '```',
+    ].join('\n');
+  },
+
+  getSkills() { return []; },
+  getResources() { return []; },
+
+  outputConfig: {
+    files: [
+      {
+        path: 'extraction.json',
+        format: 'json' as const,
+        required: true,
+        description: 'Extracted playbooks and updates to existing playbooks',
+      },
+    ],
+  },
+
+  async collectOutput(handle: WorkspaceHandle): Promise<ExtractedPlaybooks> {
+    const raw = await handle.readJson('output', 'extraction.json') as Record<string, unknown>;
+    const { createPlaybook } = await import('../../types/index.js');
+
+    const newPlaybooks: Playbook[] = [];
+    if (Array.isArray(raw.new)) {
+      for (const item of raw.new as Record<string, unknown>[]) {
+        newPlaybooks.push(createPlaybook({
+          name: String(item.name ?? 'Unnamed Playbook'),
+          guidance: {
+            strategy: String(item.strategy ?? ''),
+            tactics: Array.isArray(item.tactics) ? item.tactics.map(String) : [],
+          },
+          applicability: {
+            situations: Array.isArray(item.situations) ? item.situations.map(String) : [],
+            triggers: Array.isArray(item.triggers) ? item.triggers.map(String) : [],
+            antiPatterns: Array.isArray(item.antiPatterns) ? item.antiPatterns.map(String) : [],
+            domains: Array.isArray(item.domains) ? item.domains.map(String) : [],
+          },
+          confidence: typeof item.confidence === 'number' ? item.confidence : 0.5,
+          complexity: 'moderate',
+        }));
+      }
+    }
+
+    const updates: ExtractedPlaybooks['updates'] = [];
+    if (Array.isArray(raw.updates)) {
+      for (const item of raw.updates as Record<string, unknown>[]) {
+        updates.push({
+          id: String(item.id ?? ''),
+          newTrigger: item.newTrigger ? String(item.newTrigger) : undefined,
+          newAntiPattern: item.newAntiPattern ? String(item.newAntiPattern) : undefined,
+          refinement: item.refinementContext ? {
+            context: String(item.refinementContext),
+            addition: String(item.refinementAddition ?? ''),
+            source: 'success' as const,
+            addedAt: new Date(),
+          } : undefined,
+        });
+      }
+    }
+
+    return { new: newPlaybooks, updates };
+  },
+
+  async processOutput(): Promise<void> {
+    // Pipeline handles storage
+  },
+
+  computeRequirements: {
+    mode: 'local',
+    complexity: 'standard',
+  },
+
+  getComputeRequirements(
+    _input: PlaybookExtractionInput,
+    complexity: AnalysisComplexity
+  ): ComputeRequirements {
+    return {
+      mode: 'local',
+      complexity,
+      timeout: complexity === 'thorough' ? 300_000 : 180_000,
+    };
+  },
+
+  agentType: 'claude-code',
+  timeout: 180_000,
+  captureToolCalls: true,
+};

package/src/workspace/templates/refinement-analysis.ts

@@ -0,0 +1,162 @@
+/**
+ * Refinement Analysis Template
+ *
+ * Enhances the RefinementLoop's failure analysis with workspace-based
+ * root cause analysis. Instead of just retrying, the agent analyzes
+ * WHY the previous attempt failed and produces a targeted strategy.
+ *
+ * - First failure → lightweight analysis
+ * - Repeated failures → standard deeper analysis
+ */
+
+import type { WorkspaceHandle } from 'agent-workspace';
+import type { Trajectory, Task } from '../../types/index.js';
+import type { EvaluationResult } from '../../search/refinement-types.js';
+import type {
+  AgenticTaskTemplate,
+  AnalysisComplexity,
+} from '../types.js';
+import type { ComputeRequirements } from '../../runtime/compute-provider.js';
+
+// ============================================================
+// Input / Output Types
+// ============================================================
+
+export interface RefinementAnalysisInput {
+  trajectory: Trajectory;
+  task: Task;
+  evaluation: EvaluationResult;
+  previousAttempts: number;
+}
+
+export interface RefinementPlan {
+  /** Root causes identified for the failure */
+  rootCauses: string[];
+  /** Recommended strategy for the next attempt */
+  strategy: string;
+  /** Specific steps to take */
+  steps: string[];
+  /** Patterns to avoid (from previous failure) */
+  avoidPatterns: string[];
+}
+
+// ============================================================
+// Template Implementation
+// ============================================================
+
+export const refinementAnalysisTemplate: AgenticTaskTemplate<
+  RefinementAnalysisInput,
+  RefinementPlan
+> = {
+  taskType: 'refinement-analysis',
+  domain: 'refinement-analysis',
+  description: 'Analyze failure and plan refinement strategy',
+
+  assessComplexity(input: RefinementAnalysisInput): AnalysisComplexity {
+    // First failure → lightweight
+    if (input.previousAttempts <= 1) return 'lightweight';
+    // Repeated failures → deeper analysis
+    return 'standard';
+  },
+
+  // No heuristic fallback — refinement always benefits from semantic reasoning
+
+  async prepareWorkspace(
+    input: RefinementAnalysisInput,
+    handle: WorkspaceHandle
+  ): Promise<void> {
+    const { trajectory, task, evaluation, previousAttempts } = input;
+
+    await handle.writeJson('input', 'context.json', {
+      task: {
+        description: task.description,
+        domain: task.domain,
+      },
+      evaluation: {
+        quality: evaluation.quality,
+        score: evaluation.score,
+        issues: evaluation.issues,
+      },
+      previousAttempts,
+      outcome: trajectory.outcome,
+    });
+
+    await handle.writeJsonl('input', 'steps.jsonl',
+      trajectory.steps.map((step, i) => ({
+        index: i,
+        thought: step.thought?.slice(0, 300),
+        action: step.action.slice(0, 300),
+        observation: step.observation?.slice(0, 300),
+      }))
+    );
+  },
+
+  buildTaskPrompt(input: RefinementAnalysisInput): string {
+    const attemptNum = input.previousAttempts + 1;
+    return [
+      `This is attempt #${attemptNum}. The previous attempt failed with quality "${input.evaluation.quality}" (score: ${input.evaluation.score}).`,
+      '',
+      'Read input/context.json for task, evaluation, and outcome details.',
+      'Read input/steps.jsonl for the execution trace.',
+      '',
+      'Perform root cause analysis:',
+      '1. Why did the previous attempt fail? Identify specific root causes.',
+      '2. What strategy should the next attempt use?',
+      '3. What specific steps should be taken?',
+      '4. What patterns from the failed attempt should be avoided?',
+      '',
+      'Write to output/plan.json:',
+      '```json',
+      '{',
+      '  "rootCauses": ["cause1", "cause2"],',
+      '  "strategy": "high-level approach for next attempt",',
+      '  "steps": ["specific step 1", "specific step 2"],',
+      '  "avoidPatterns": ["pattern to avoid 1", "pattern to avoid 2"]',
+      '}',
+      '```',
+    ].join('\n');
+  },
+
+  getSkills() { return []; },
+  getResources() { return []; },
+
+  outputConfig: {
+    files: [{
+      path: 'plan.json',
+      format: 'json' as const,
+      required: true,
+      description: 'Refinement plan with root causes and strategy',
+    }],
+  },
+
+  async collectOutput(handle: WorkspaceHandle): Promise<RefinementPlan> {
+    const raw = await handle.readJson('output', 'plan.json') as Record<string, unknown>;
+    return {
+      rootCauses: Array.isArray(raw.rootCauses) ? raw.rootCauses.map(String) : [],
+      strategy: String(raw.strategy ?? ''),
+      steps: Array.isArray(raw.steps) ? raw.steps.map(String) : [],
+      avoidPatterns: Array.isArray(raw.avoidPatterns) ? raw.avoidPatterns.map(String) : [],
+    };
+  },
+
+  async processOutput(): Promise<void> {},
+
+  computeRequirements: {
+    mode: 'local',
+    complexity: 'lightweight',
+  },
+
+  getComputeRequirements(
+    _input: RefinementAnalysisInput,
+    complexity: AnalysisComplexity
+  ): ComputeRequirements {
+    return {
+      mode: 'local',
+      complexity,
+    };
+  },
+
+  agentType: 'claude-code',
+  timeout: 120_000,
+  captureToolCalls: true,
+};

package/src/workspace/templates/solution-evaluation.ts

@@ -0,0 +1,199 @@
+/**
+ * Solution Evaluation Template
+ *
+ * Enhances the SolutionEvaluator with workspace-based agentic evaluation
+ * for cases where programmatic verification is insufficient.
+ *
+ * - Tasks with verification spec → heuristic (programmatic verification)
+ * - Short successful trajectories → lightweight agent evaluation
+ * - Complex or failed cases → standard agent evaluation
+ */
+
+import type { WorkspaceHandle } from 'agent-workspace';
+import type { Trajectory, Task } from '../../types/index.js';
+import type { EvaluationResult } from '../../search/refinement-types.js';
+import { createEvaluationResult, scoreToQuality } from '../../search/refinement-types.js';
+import type {
+  AgenticTaskTemplate,
+  AnalysisComplexity,
+  ResourceSpec,
+} from '../types.js';
+import type { ComputeRequirements } from '../../runtime/compute-provider.js';
+
+// ============================================================
+// Input / Output Types
+// ============================================================
+
+export interface SolutionEvaluationInput {
+  trajectory: Trajectory;
+  task: Task;
+  /** Optional path to the codebase for agent to inspect */
+  codebasePath?: string;
+}
+
+// Output: EvaluationResult (reused from search/refinement-types.ts)
+
+// ============================================================
+// Template Implementation
+// ============================================================
+
+export const solutionEvaluationTemplate: AgenticTaskTemplate<
+  SolutionEvaluationInput,
+  EvaluationResult
+> = {
+  taskType: 'solution-evaluation',
+  domain: 'solution-evaluation',
+  description: 'Evaluate solution quality from trajectory',
+
+  assessComplexity(input: SolutionEvaluationInput): AnalysisComplexity {
+    // Programmatic verification available → heuristic path
+    if (input.task.verification) return 'heuristic';
+    // Short successful trajectories → lightweight
+    if (input.trajectory.outcome.success && input.trajectory.steps.length <= 10) {
+      return 'lightweight';
+    }
+    return 'standard';
+  },
+
+  async heuristicFallback(input: SolutionEvaluationInput): Promise<EvaluationResult> {
+    // Simple heuristic evaluation based on trajectory outcome
+    const score = input.trajectory.outcome.success
+      ? Math.max(0.7, input.trajectory.outcome.partialScore)
+      : input.trajectory.outcome.partialScore;
+
+    return createEvaluationResult({
+      quality: scoreToQuality(score),
+      score,
+      acceptable: score >= 0.7,
+      method: 'heuristic',
+    });
+  },
+
+  async prepareWorkspace(
+    input: SolutionEvaluationInput,
+    handle: WorkspaceHandle
+  ): Promise<void> {
+    const { trajectory, task } = input;
+
+    await handle.writeJson('input', 'task.json', {
+      description: task.description,
+      domain: task.domain,
+      verification: task.verification,
+    });
+
+    await handle.writeJson('input', 'trajectory.json', {
+      outcome: trajectory.outcome,
+      stepCount: trajectory.steps.length,
+      wallTimeSeconds: trajectory.wallTimeSeconds,
+      solution: trajectory.outcome.solution,
+    });
+
+    await handle.writeJsonl('input', 'steps.jsonl',
+      trajectory.steps.map((step, i) => ({
+        index: i,
+        action: step.action,
+        observation: step.observation?.slice(0, 300),
+      }))
+    );
+  },
+
+  buildTaskPrompt(input: SolutionEvaluationInput): string {
+    const lines = [
+      `Evaluate the quality of this solution for: "${input.task.description.slice(0, 200)}"`,
+      '',
+      'Read input/task.json for the task requirements.',
+      'Read input/trajectory.json for the outcome and solution.',
+      'Read input/steps.jsonl for execution details.',
+    ];
+
+    if (input.codebasePath) {
+      lines.push('Browse resources/codebase/ to inspect the actual code changes if relevant.');
+    }
+
+    lines.push(
+      '',
+      'Evaluate:',
+      '1. Does the solution address the task requirements?',
+      '2. Are there any issues (incomplete, incorrect, inefficient, errors)?',
+      '3. Overall quality score (0.0 to 1.0)',
+      '',
+      'Write to output/evaluation.json:',
+      '```json',
+      '{',
+      '  "quality": "excellent" | "good" | "needs_work" | "poor",',
+      '  "score": 0.0-1.0,',
+      '  "acceptable": boolean,',
+      '  "issues": [{ "type": "incomplete"|"incorrect"|"inefficient"|"error", "description": "...", "severity": "critical"|"major"|"minor", "suggestion": "..." }]',
+      '}',
+      '```',
+    );
+    return lines.join('\n');
+  },
+
+  getSkills() { return []; },
+
+  getResources(input: SolutionEvaluationInput): ResourceSpec[] {
+    if (!input.codebasePath) return [];
+    return [{
+      path: 'codebase',
+      type: 'symlink',
+      source: input.codebasePath,
+      description: 'Codebase for solution verification',
+    }];
+  },
+
+  outputConfig: {
+    files: [{
+      path: 'evaluation.json',
+      format: 'json' as const,
+      required: true,
+      description: 'Solution quality evaluation',
+    }],
+  },
+
+  async collectOutput(handle: WorkspaceHandle): Promise<EvaluationResult> {
+    const raw = await handle.readJson('output', 'evaluation.json') as Record<string, unknown>;
+
+    const score = typeof raw.score === 'number' ? Math.min(1, Math.max(0, raw.score)) : 0.5;
+    const quality = ['excellent', 'good', 'needs_work', 'poor'].includes(String(raw.quality))
+      ? String(raw.quality) as EvaluationResult['quality']
+      : scoreToQuality(score);
+
+    return createEvaluationResult({
+      quality,
+      score,
+      acceptable: raw.acceptable !== undefined ? Boolean(raw.acceptable) : score >= 0.7,
+      issues: Array.isArray(raw.issues)
+        ? (raw.issues as Record<string, unknown>[]).map(issue => ({
+            type: String(issue.type ?? 'error') as 'incomplete' | 'incorrect' | 'inefficient' | 'error',
+            description: String(issue.description ?? ''),
+            severity: (['critical', 'major', 'minor'].includes(String(issue.severity))
+              ? String(issue.severity) : 'minor') as 'critical' | 'major' | 'minor',
+            suggestion: issue.suggestion ? String(issue.suggestion) : undefined,
+          }))
+        : [],
+      method: 'agent',
+    });
+  },
+
+  async processOutput(): Promise<void> {},
+
+  computeRequirements: {
+    mode: 'local',
+    complexity: 'standard',
+  },
+
+  getComputeRequirements(
+    _input: SolutionEvaluationInput,
+    complexity: AnalysisComplexity
+  ): ComputeRequirements {
+    return {
+      mode: 'local',
+      complexity,
+    };
+  },
+
+  agentType: 'claude-code',
+  timeout: 120_000,
+  captureToolCalls: true,
+};