cognitive-core 0.0.2 → 0.1.2

package/src/workspace/templates/trajectory-analysis.ts

@@ -0,0 +1,234 @@
+/**
+ * Trajectory Analysis Template
+ *
+ * Replaces the heuristic TrajectoryAnalyzer with workspace-based agentic analysis.
+ * Consolidates credit assignment, key step identification, error pattern detection,
+ * and abstractability assessment into a single template.
+ *
+ * - Simple, short successful trajectories → heuristic fallback (zero cost)
+ * - Failed or complex trajectories → agent analyzes semantically
+ */
+
+import type { WorkspaceHandle } from 'agent-workspace';
+import type { Trajectory, CreditStrategy } from '../../types/index.js';
+import type { AnalysisResult, ErrorPattern, TrainingExample } from '../../learning/analyzer.js';
+import { createAnalyzer } from '../../learning/analyzer.js';
+import type {
+  AgenticTaskTemplate,
+  AnalysisComplexity,
+  ProcessingContext,
+} from '../types.js';
+import type { ComputeRequirements } from '../../runtime/compute-provider.js';
+
+// ============================================================
+// Input / Output Types
+// ============================================================
+
+export interface TrajectoryAnalysisInput {
+  trajectory: Trajectory;
+  creditStrategy: CreditStrategy;
+}
+
+// Output is AnalysisResult (reused from learning/analyzer.ts)
+// No new types needed.
+
+// ============================================================
+// Template Implementation
+// ============================================================
+
+export const trajectoryAnalysisTemplate: AgenticTaskTemplate<
+  TrajectoryAnalysisInput,
+  AnalysisResult
+> = {
+  taskType: 'trajectory-analysis',
+  domain: 'trajectory-analysis',
+  description: 'Analyze agent trajectory for credit assignment, key steps, error patterns, and abstractability',
+
+  // === COMPLEXITY ASSESSMENT ===
+
+  assessComplexity(input: TrajectoryAnalysisInput): AnalysisComplexity {
+    const { trajectory } = input;
+
+    // Short successful trajectories → heuristic (existing analyzer is fine)
+    if (trajectory.steps.length <= 5 && trajectory.outcome.success) {
+      return 'heuristic';
+    }
+
+    // Very short trajectories (even if failed) → lightweight
+    if (trajectory.steps.length <= 3) {
+      return 'lightweight';
+    }
+
+    // Failed trajectories benefit most from semantic analysis
+    if (!trajectory.outcome.success) {
+      return 'standard';
+    }
+
+    // Long trajectories need agent to identify what actually mattered
+    if (trajectory.steps.length > 20) {
+      return 'standard';
+    }
+
+    return 'lightweight';
+  },
+
+  // === HEURISTIC FALLBACK ===
+
+  async heuristicFallback(input: TrajectoryAnalysisInput): Promise<AnalysisResult> {
+    const analyzer = createAnalyzer(input.creditStrategy);
+    return analyzer.analyze(input.trajectory);
+  },
+
+  // === WORKSPACE SETUP ===
+
+  async prepareWorkspace(
+    input: TrajectoryAnalysisInput,
+    handle: WorkspaceHandle
+  ): Promise<void> {
+    // Write trajectory context
+    const { trajectory } = input;
+    await handle.writeJson('input', 'context.json', {
+      taskId: trajectory.task.id,
+      taskDomain: trajectory.task.domain,
+      taskDescription: trajectory.task.description,
+      outcome: {
+        success: trajectory.outcome.success,
+        partialScore: trajectory.outcome.partialScore,
+        errorInfo: trajectory.outcome.errorInfo,
+      },
+      stepCount: trajectory.steps.length,
+      wallTimeSeconds: trajectory.wallTimeSeconds,
+      llmCalls: trajectory.llmCalls,
+      totalTokens: trajectory.totalTokens,
+    });
+
+    // Write steps as JSONL for efficient processing
+    await handle.writeJsonl('input', 'steps.jsonl',
+      trajectory.steps.map((step, i) => ({
+        index: i,
+        thought: step.thought,
+        action: step.action,
+        observation: step.observation,
+        metadata: step.metadata,
+      }))
+    );
+  },
+
+  // === PROMPT ===
+
+  buildTaskPrompt(input: TrajectoryAnalysisInput): string {
+    const { trajectory } = input;
+    const outcomeStr = trajectory.outcome.success ? 'SUCCESSFUL' : 'FAILED';
+
+    return [
+      `Analyze this ${outcomeStr} agent trajectory (${trajectory.steps.length} steps, ${trajectory.wallTimeSeconds}s wall time).`,
+      '',
+      'Read the trajectory from input/context.json and input/steps.jsonl.',
+      '',
+      'Perform the following analyses:',
+      '',
+      '1. **Credit Assignment**: For each step, assign an attribution score (0.0 to 1.0)',
+      '   indicating how much that step contributed to the final outcome.',
+      '   Consider causal impact, not just proximity to the end.',
+      '',
+      '2. **Key Steps**: Identify the indices of pivotal steps (attribution >= 0.15).',
+      '',
+      '3. **Error Patterns**: Detect any error patterns (type, signature, description, optional fix).',
+      '   Look for: timeouts, missing resources, syntax errors, type errors, repeated actions, loops.',
+      '',
+      '4. **Abstractability**: Is this trajectory worth extracting patterns from?',
+      '   Criteria: successful with >= 2 key steps, OR failed with clear error pattern and partial progress.',
+      '',
+      '5. **Training Examples**: Extract thought→action pairs from key steps.',
+      '',
+      'Write your analysis to output/analysis.json with this exact schema:',
+      '```json',
+      '{',
+      '  "success": boolean,',
+      '  "keySteps": number[],',
+      '  "stepAttribution": number[],',
+      '  "errorPatterns": [{ "type": string, "signature": string, "description": string, "suggestedFix"?: string }],',
+      '  "abstractable": boolean,',
+      '  "trainingExamples": [{ "input": string, "output": string, "stepIndex": number }]',
+      '}',
+      '```',
+    ].join('\n');
+  },
+
+  // === SKILLS & RESOURCES ===
+
+  getSkills() {
+    return [];
+  },
+
+  getResources() {
+    return [];
+  },
+
+  // === OUTPUT ===
+
+  outputConfig: {
+    files: [
+      {
+        path: 'analysis.json',
+        format: 'json' as const,
+        required: true,
+        description: 'Trajectory analysis result with credit assignment, key steps, error patterns',
+      },
+    ],
+  },
+
+  async collectOutput(handle: WorkspaceHandle): Promise<AnalysisResult> {
+    const raw = await handle.readJson('output', 'analysis.json');
+    const data = raw as Record<string, unknown>;
+
+    // Validate and coerce the output into AnalysisResult
+    return {
+      success: Boolean(data.success),
+      keySteps: Array.isArray(data.keySteps) ? data.keySteps.map(Number) : [],
+      stepAttribution: Array.isArray(data.stepAttribution)
+        ? data.stepAttribution.map(Number)
+        : [],
+      errorPatterns: Array.isArray(data.errorPatterns)
+        ? (data.errorPatterns as ErrorPattern[])
+        : [],
+      abstractable: Boolean(data.abstractable),
+      trainingExamples: Array.isArray(data.trainingExamples)
+        ? (data.trainingExamples as TrainingExample[])
+        : [],
+    };
+  },
+
+  async processOutput(
+    _output: AnalysisResult,
+    _input: TrajectoryAnalysisInput,
+    _context: ProcessingContext
+  ): Promise<void> {
+    // No side effects during analysis — the pipeline handles storage
+  },
+
+  // === COMPUTE ===
+
+  computeRequirements: {
+    mode: 'local',
+    complexity: 'lightweight',
+  },
+
+  getComputeRequirements(
+    _input: TrajectoryAnalysisInput,
+    complexity: AnalysisComplexity
+  ): ComputeRequirements {
+    return {
+      mode: 'local',
+      complexity,
+      // Standard analysis can take a bit longer
+      timeout: complexity === 'standard' ? 180_000 : 120_000,
+    };
+  },
+
+  // === CONFIG ===
+
+  agentType: 'claude-code',
+  timeout: 120_000,
+  captureToolCalls: true,
+};

package/src/workspace/templates/usage-inference.ts

@@ -0,0 +1,163 @@
+/**
+ * Usage Inference Template
+ *
+ * Replaces the heuristic PlaybookUsageInference with workspace-based
+ * semantic matching that determines which injected playbooks were
+ * actually followed by the agent.
+ *
+ * - No playbooks or very few steps → heuristic fallback
+ * - Otherwise → agent reads trajectory and judges playbook adherence
+ */
+
+import type { WorkspaceHandle } from 'agent-workspace';
+import type { Trajectory, Playbook } from '../../types/index.js';
+import type { PlaybookUsageResult } from '../../learning/usage-inference.js';
+import { createUsageInference } from '../../learning/usage-inference.js';
+import type {
+  AgenticTaskTemplate,
+  AnalysisComplexity,
+} from '../types.js';
+
+// ============================================================
+// Input / Output Types
+// ============================================================
+
+export interface UsageInferenceInput {
+  trajectory: Trajectory;
+  injectedPlaybooks: Playbook[];
+}
+
+// Output: PlaybookUsageResult[] (reused from learning/usage-inference.ts)
+
+// ============================================================
+// Template Implementation
+// ============================================================
+
+export const usageInferenceTemplate: AgenticTaskTemplate<
+  UsageInferenceInput,
+  PlaybookUsageResult[]
+> = {
+  taskType: 'usage-inference',
+  domain: 'usage-inference',
+  description: 'Determine which injected playbooks were actually followed',
+
+  assessComplexity(input: UsageInferenceInput): AnalysisComplexity {
+    if (input.injectedPlaybooks.length === 0) return 'heuristic';
+    if (input.injectedPlaybooks.length <= 2 && input.trajectory.steps.length <= 10) {
+      return 'heuristic';
+    }
+    return 'lightweight';
+  },
+
+  async heuristicFallback(input: UsageInferenceInput): Promise<PlaybookUsageResult[]> {
+    const inference = createUsageInference();
+    return inference.inferUsage(input.trajectory, input.injectedPlaybooks);
+  },
+
+  async prepareWorkspace(
+    input: UsageInferenceInput,
+    handle: WorkspaceHandle
+  ): Promise<void> {
+    const { trajectory, injectedPlaybooks } = input;
+
+    // Write trajectory summary + steps
+    await handle.writeJson('input', 'trajectory.json', {
+      taskDescription: trajectory.task.description,
+      domain: trajectory.task.domain,
+      success: trajectory.outcome.success,
+      stepCount: trajectory.steps.length,
+      steps: trajectory.steps.map((step, i) => ({
+        index: i,
+        thought: step.thought,
+        action: step.action,
+        observation: step.observation?.slice(0, 300),
+      })),
+    });
+
+    // Write injected playbooks
+    await handle.writeJson('input', 'playbooks.json',
+      injectedPlaybooks.map(pb => ({
+        id: pb.id,
+        name: pb.name,
+        strategy: pb.guidance.strategy,
+        tactics: pb.guidance.tactics,
+        steps: pb.guidance.steps,
+        successIndicators: pb.verification.successIndicators,
+      }))
+    );
+  },
+
+  buildTaskPrompt(input: UsageInferenceInput): string {
+    return [
+      `Analyze whether the agent followed ${input.injectedPlaybooks.length} injected playbooks.`,
+      '',
+      'Read input/trajectory.json for the agent execution trace.',
+      'Read input/playbooks.json for the playbooks that were injected.',
+      '',
+      'For each playbook, determine:',
+      '1. Was the playbook guidance followed? Look for tactical alignment, not literal matches.',
+      '2. How confident are you? (0.0-1.0)',
+      '3. Was the outcome successful when following this guidance?',
+      '4. Which trajectory steps aligned with the playbook?',
+      '',
+      'Write results to output/usage.json:',
+      '```json',
+      '[',
+      '  {',
+      '    "playbookId": "id",',
+      '    "playbookName": "name",',
+      '    "wasUsed": boolean,',
+      '    "usageConfidence": 0.0-1.0,',
+      '    "wasSuccessful": boolean,',
+      '    "matchedSteps": [step indices],',
+      '    "context": "brief explanation of usage or non-usage"',
+      '  }',
+      ']',
+      '```',
+    ].join('\n');
+  },
+
+  getSkills() { return []; },
+  getResources() { return []; },
+
+  outputConfig: {
+    files: [
+      {
+        path: 'usage.json',
+        format: 'json' as const,
+        required: true,
+        description: 'Playbook usage inference results',
+      },
+    ],
+  },
+
+  async collectOutput(handle: WorkspaceHandle): Promise<PlaybookUsageResult[]> {
+    const raw = await handle.readJson('output', 'usage.json');
+    if (!Array.isArray(raw)) return [];
+
+    return (raw as Record<string, unknown>[]).map(item => ({
+      playbookId: String(item.playbookId ?? ''),
+      playbookName: String(item.playbookName ?? ''),
+      wasUsed: Boolean(item.wasUsed),
+      usageConfidence: typeof item.usageConfidence === 'number' ? item.usageConfidence : 0,
+      wasSuccessful: Boolean(item.wasSuccessful),
+      matchedSteps: Array.isArray(item.matchedSteps)
+        ? item.matchedSteps.map(Number)
+        : [],
+      context: item.context ? String(item.context) : undefined,
+    }));
+  },
+
+  async processOutput(): Promise<void> {
+    // Atlas handles storage and skill library recording
+  },
+
+  computeRequirements: {
+    mode: 'local',
+    complexity: 'lightweight',
+  },
+
+  agentType: 'claude-code',
+  timeout: 120_000,
+  captureToolCalls: true,
+};

package/src/workspace/types.ts

@@ -0,0 +1,233 @@
+import type { WorkspaceHandle, OutputSpec } from 'agent-workspace';
+import type { MemorySystem, MemoryQueryResultV2 } from '../memory/system.js';
+import type { LearningPipeline } from '../learning/pipeline.js';
+import type { ComputeRequirements } from '../runtime/compute-provider.js';
+
+// Re-export agent-workspace types used by consumers
+export type { WorkspaceHandle, OutputSpec, OutputFileSpec, ValidationResult } from 'agent-workspace';
+
+// ============================================================
+// Agent Skills
+// ============================================================
+
+/**
+ * Skill to load into workspace skills/ directory.
+ * Agent Skills open standard format (SKILL.md).
+ */
+export interface AgentSkillSpec {
+  /** Skill directory name */
+  name: string;
+  /** SKILL.md content */
+  content: string;
+  /** Source of this skill */
+  source: 'playbook-conversion' | 'curated' | 'extracted';
+}
+
+// ============================================================
+// Resources
+// ============================================================
+
+/**
+ * Supplementary resource to load into workspace resources/ directory.
+ */
+export interface ResourceSpec {
+  /** Path relative to resources/ directory */
+  path: string;
+  /** How to populate this resource */
+  type: 'directory' | 'file' | 'symlink';
+  /** Source path (for symlink/copy) or content (for file) */
+  source: string;
+  /** Description for the agent */
+  description?: string;
+}
+
+// ============================================================
+// Complexity Assessment
+// ============================================================
+
+/**
+ * Complexity level determines whether to use heuristic or agentic analysis.
+ */
+export type AnalysisComplexity =
+  | 'heuristic'    // Use heuristicFallback(), no agent needed
+  | 'lightweight'  // Agent with small budget, simple workspace
+  | 'standard'     // Agent with moderate budget, full workspace
+  | 'thorough';    // Agent with large budget, full workspace, more time
+
+// ============================================================
+// Processing Context
+// ============================================================
+
+/**
+ * Context passed to processOutput for storing/accumulating results.
+ */
+export interface ProcessingContext {
+  memory: MemorySystem;
+  learningPipeline: LearningPipeline;
+}
+
+// ============================================================
+// AgenticTaskTemplate
+// ============================================================
+
+/**
+ * Declarative recipe for an agentic analysis task.
+ *
+ * TInput: the typed input data for this analysis
+ * TOutput: the typed output the agent produces
+ */
+export interface AgenticTaskTemplate<TInput, TOutput> {
+  /** Unique identifier for this task type */
+  taskType: string;
+
+  /** Domain for knowledge injection and meta-learning */
+  domain: string;
+
+  /** Human-readable description */
+  description: string;
+
+  // === WORKSPACE SETUP ===
+
+  /**
+   * Prepare the workspace filesystem from typed input.
+   * Uses the WorkspaceHandle to write input files via typed I/O methods.
+   */
+  prepareWorkspace(input: TInput, handle: WorkspaceHandle): Promise<void>;
+
+  /**
+   * Build the task description for the agent.
+   * Natural language prompt explaining what to do and what output to produce.
+   */
+  buildTaskPrompt(input: TInput): string;
+
+  // === SKILLS ===
+
+  /**
+   * Skills to load into the workspace skills/ directory.
+   * Playbooks from memory are converted to skills via the converter.
+   */
+  getSkills(input: TInput): AgentSkillSpec[];
+
+  // === RESOURCES ===
+
+  /**
+   * Supplementary resources for the workspace resources/ directory.
+   */
+  getResources(input: TInput): ResourceSpec[];
+
+  // === KNOWLEDGE ===
+
+  /**
+   * Additional memory queries beyond the automatic domain-scoped query.
+   */
+  getAdditionalKnowledge?(
+    input: TInput,
+    memory: MemorySystem
+  ): Promise<MemoryQueryResultV2>;
+
+  // === OUTPUT ===
+
+  /** Configuration defining the expected output shape */
+  outputConfig: OutputSpec;
+
+  /**
+   * Read and validate agent output from workspace.
+   * Uses the WorkspaceHandle to read output via typed I/O methods.
+   * Returns typed output or throws if validation fails.
+   */
+  collectOutput(handle: WorkspaceHandle): Promise<TOutput>;
+
+  /**
+   * Process the validated output — store results, accumulate for batch learning, etc.
+   */
+  processOutput(
+    output: TOutput,
+    input: TInput,
+    context: ProcessingContext
+  ): Promise<void>;
+
+  // === COMPLEXITY ASSESSMENT ===
+
+  /**
+   * Assess whether this input needs agentic analysis or heuristic fallback.
+   */
+  assessComplexity(input: TInput): AnalysisComplexity;
+
+  /**
+   * Heuristic fallback for simple cases.
+   * Called when assessComplexity returns 'heuristic'.
+   */
+  heuristicFallback?(input: TInput): Promise<TOutput>;
+
+  // === CONFIGURATION ===
+
+  /** Agent type to spawn (default: 'claude-code') */
+  agentType?: string;
+
+  /** Timeout for the analysis agent (default: 120_000ms) */
+  timeout?: number;
+
+  /** Whether to capture detailed tool calls (default: true) */
+  captureToolCalls?: boolean;
+
+  // === COMPUTE ===
+
+  /**
+   * Static compute requirements for this template.
+   * Used by ComputeProvider to select appropriate backend.
+   */
+  computeRequirements?: ComputeRequirements;
+
+  /**
+   * Dynamic compute requirements based on input and assessed complexity.
+   * Overrides static computeRequirements when provided.
+   */
+  getComputeRequirements?(
+    input: TInput,
+    complexity: AnalysisComplexity
+  ): ComputeRequirements;
+}
+
+// ============================================================
+// Task Result
+// ============================================================
+
+/**
+ * Result from running an agentic task.
+ */
+export interface AgenticTaskResult<TOutput> {
+  /** The validated output */
+  output: TOutput;
+  /** Whether heuristic or agentic path was used */
+  method: 'heuristic' | 'agentic';
+  /** Trajectory ID of the analysis agent (if agentic) */
+  agentTrajectoryId?: string;
+  /** Execution metrics */
+  metrics: AgenticTaskMetrics;
+}
+
+/**
+ * Metrics from an agentic task execution.
+ */
+export interface AgenticTaskMetrics {
+  totalTimeMs: number;
+  tokensUsed: number;
+  toolCalls: number;
+}
+
+// ============================================================
+// Errors
+// ============================================================
+
+/**
+ * Error thrown when agent output validation fails.
+ */
+export class AgenticTaskOutputError extends Error {
+  constructor(
+    public readonly taskType: string,
+    public readonly cause: Error
+  ) {
+    super(`Agentic task '${taskType}' output validation failed: ${cause.message}`);
+    this.name = 'AgenticTaskOutputError';
+  }
+}