cognitive-core 0.2.0 → 0.2.1

package/src/workspace/templates/playbook-lifecycle-review.ts

@@ -0,0 +1,274 @@
+/**
+ * Playbook Lifecycle Review Template
+ *
+ * Portfolio-wide health assessment with lifecycle recommendations.
+ * Uses the efficacy toolkit's portfolio snapshot to give the agent
+ * a pre-computed view of the entire playbook portfolio, then asks
+ * for promotion, deprecation, merge, and evolution recommendations.
+ *
+ * This is the "skill designer" review step (inspired by MemSkill)
+ * that periodically evaluates the whole portfolio.
+ */
+
+import type { WorkspaceHandle } from 'agent-workspace';
+import type { Playbook } from '../../types/index.js';
+import type { TaskAnnotation, PlaybookEffectivenessEntry } from '../../learning/effectiveness.js';
+import type {
+  AgenticTaskTemplate,
+  AnalysisComplexity,
+} from '../types.js';
+import {
+  buildPortfolioSnapshot,
+  type PortfolioSnapshot,
+} from '../efficacy-toolkit.js';
+
+// ============================================================
+// Input / Output Types
+// ============================================================
+
+export interface PlaybookLifecycleReviewInput {
+  playbooks: Playbook[];
+  annotations: TaskAnnotation[];
+  playbookEffectiveness: PlaybookEffectivenessEntry[];
+  unguidedSuccessRate: number;
+  trajectoryDomainMap: Map<string, string>;
+  /** Current core tier playbook IDs (from SkillLibrary) */
+  corePlaybookIds?: string[];
+}
+
+export interface PlaybookLifecycleReviewOutput {
+  /** Pre-computed portfolio snapshot */
+  snapshot: PortfolioSnapshot;
+  /** Agent's lifecycle recommendations */
+  recommendations: LifecycleRecommendation[];
+  /** Portfolio-level strategic insights */
+  insights: string[];
+  /** Priority-ordered action plan */
+  actionPlan: string[];
+}
+
+export interface LifecycleRecommendation {
+  playbookId: string;
+  playbookName: string;
+  currentState: 'active' | 'core' | 'stale' | 'underperforming';
+  proposedState: 'promote-to-core' | 'keep-active' | 'deprecate' | 'archive'
+    | 'merge' | 'evolve' | 'split';
+  rationale: string;
+  /** For merge: target playbook */
+  mergeTarget?: { id: string; name: string };
+  /** For evolve: suggested guidance changes */
+  evolutionNotes?: string;
+  priority: 'high' | 'medium' | 'low';
+}
+
+// ============================================================
+// Template Implementation
+// ============================================================
+
+export const playbookLifecycleReviewTemplate: AgenticTaskTemplate<
+  PlaybookLifecycleReviewInput,
+  PlaybookLifecycleReviewOutput
+> = {
+  taskType: 'playbook-lifecycle-review',
+  domain: 'meta-learning',
+  description: 'Portfolio-wide playbook health assessment with lifecycle recommendations',
+
+  assessComplexity(input: PlaybookLifecycleReviewInput): AnalysisComplexity {
+    if (input.playbooks.length === 0) return 'heuristic';
+    if (input.playbooks.length <= 3 && input.annotations.length < 10) return 'heuristic';
+    if (input.playbooks.length > 30) return 'thorough';
+    if (input.playbooks.length > 10) return 'standard';
+    return 'lightweight';
+  },
+
+  async heuristicFallback(input: PlaybookLifecycleReviewInput): Promise<PlaybookLifecycleReviewOutput> {
+    const snapshot = buildPortfolioSnapshot(
+      input.playbooks,
+      input.annotations,
+      input.playbookEffectiveness,
+      input.unguidedSuccessRate,
+      input.trajectoryDomainMap,
+    );
+
+    // Simple heuristic recommendations
+    const recommendations: LifecycleRecommendation[] = [];
+
+    for (const flag of snapshot.decaying) {
+      recommendations.push({
+        playbookId: flag.playbookId,
+        playbookName: flag.playbookName,
+        currentState: 'active',
+        proposedState: 'evolve',
+        rationale: flag.reason,
+        priority: 'high',
+      });
+    }
+
+    for (const flag of snapshot.stale) {
+      recommendations.push({
+        playbookId: flag.playbookId,
+        playbookName: flag.playbookName,
+        currentState: 'stale',
+        proposedState: 'archive',
+        rationale: flag.reason,
+        priority: 'medium',
+      });
+    }
+
+    return {
+      snapshot,
+      recommendations,
+      insights: [],
+      actionPlan: recommendations.map(r =>
+        `${r.proposedState} "${r.playbookName}" — ${r.rationale}`
+      ),
+    };
+  },
+
+  async prepareWorkspace(
+    input: PlaybookLifecycleReviewInput,
+    handle: WorkspaceHandle
+  ): Promise<void> {
+    const snapshot = buildPortfolioSnapshot(
+      input.playbooks,
+      input.annotations,
+      input.playbookEffectiveness,
+      input.unguidedSuccessRate,
+      input.trajectoryDomainMap,
+    );
+
+    // Portfolio snapshot — the core data
+    await handle.writeJson('input', 'portfolio-snapshot.json', snapshot);
+
+    // Per-playbook summaries for the agent to reference
+    const playbookSummaries = input.playbooks.map(pb => ({
+      id: pb.id,
+      name: pb.name,
+      confidence: pb.confidence,
+      successCount: pb.evolution.successCount,
+      failureCount: pb.evolution.failureCount,
+      successRate: pb.evolution.successCount + pb.evolution.failureCount > 0
+        ? pb.evolution.successCount / (pb.evolution.successCount + pb.evolution.failureCount)
+        : 0,
+      domains: pb.applicability.domains,
+      strategy: pb.guidance.strategy,
+      tacticsCount: pb.guidance.tactics.length,
+      refinementsCount: pb.evolution.refinements.length,
+      version: pb.evolution.version,
+      lastUsed: pb.evolution.lastUsed,
+      isCore: input.corePlaybookIds?.includes(pb.id) ?? false,
+      consolidationStrength: pb.evolution.consolidationStrength ?? 0,
+    }));
+
+    await handle.writeJson('input', 'playbook-summaries.json', playbookSummaries);
+
+    // Redundancy pairs (pre-computed by toolkit)
+    if (snapshot.redundant.length > 0) {
+      await handle.writeJson('input', 'redundancy-pairs.json', snapshot.redundant);
+    }
+  },
+
+  buildTaskPrompt(input: PlaybookLifecycleReviewInput): string {
+    return [
+      `Review the health of the entire playbook portfolio (${input.playbooks.length} playbooks).`,
+      '',
+      'Read:',
+      '- input/portfolio-snapshot.json — Portfolio health metrics, flagged issues (decaying, stale, low adoption, redundant)',
+      '- input/playbook-summaries.json — Per-playbook summaries with confidence, success rates, domains',
+      '- input/redundancy-pairs.json — (if present) Playbook pairs with high content overlap',
+      '',
+      'For each playbook, recommend a lifecycle action:',
+      '- **promote-to-core**: High confidence + success rate, proven across tasks',
+      '- **keep-active**: Performing well, no changes needed',
+      '- **evolve**: Decent foundation but guidance needs updating based on failure patterns',
+      '- **merge**: Two playbooks cover the same ground — merge into one stronger playbook',
+      '- **split**: One playbook covers too many domains — split into domain-specific variants',
+      '- **deprecate**: Low value, actively misleading, or superseded',
+      '- **archive**: No longer relevant but worth preserving for reference',
+      '',
+      'Also provide:',
+      '- Strategic insights about the portfolio (coverage gaps, over-invested domains, etc.)',
+      '- A priority-ordered action plan (most impactful changes first)',
+      '',
+      'Write to output/lifecycle-review.json:',
+      '```json',
+      '{',
+      '  "recommendations": [',
+      '    {',
+      '      "playbookId": "id",',
+      '      "playbookName": "name",',
+      '      "currentState": "active" | "core" | "stale" | "underperforming",',
+      '      "proposedState": "promote-to-core" | "keep-active" | "deprecate" | "archive" | "merge" | "evolve" | "split",',
+      '      "rationale": "why this change",',
+      '      "mergeTarget": { "id": "...", "name": "..." },',
+      '      "evolutionNotes": "optional: what to change in guidance",',
+      '      "priority": "high" | "medium" | "low"',
+      '    }',
+      '  ],',
+      '  "insights": ["insight1", "insight2"],',
+      '  "actionPlan": ["step1", "step2"]',
+      '}',
+      '```',
+    ].join('\n');
+  },
+
+  getSkills() { return []; },
+  getResources() { return []; },
+
+  outputConfig: {
+    files: [
+      {
+        path: 'lifecycle-review.json',
+        format: 'json' as const,
+        required: true,
+        description: 'Playbook lifecycle review results',
+      },
+    ],
+  },
+
+  async collectOutput(handle: WorkspaceHandle): Promise<PlaybookLifecycleReviewOutput> {
+    const raw = await handle.readJson('output', 'lifecycle-review.json') as Record<string, unknown>;
+    const snapshot = await handle.readJson('input', 'portfolio-snapshot.json') as PortfolioSnapshot;
+
+    const recommendations: LifecycleRecommendation[] = Array.isArray(raw.recommendations)
+      ? (raw.recommendations as Record<string, unknown>[]).map(r => {
+          const mergeTarget = r.mergeTarget as Record<string, unknown> | undefined;
+          return {
+            playbookId: String(r.playbookId ?? ''),
+            playbookName: String(r.playbookName ?? ''),
+            currentState: String(r.currentState ?? 'active') as LifecycleRecommendation['currentState'],
+            proposedState: String(r.proposedState ?? 'keep-active') as LifecycleRecommendation['proposedState'],
+            rationale: String(r.rationale ?? ''),
+            mergeTarget: mergeTarget
+              ? { id: String(mergeTarget.id ?? ''), name: String(mergeTarget.name ?? '') }
+              : undefined,
+            evolutionNotes: r.evolutionNotes ? String(r.evolutionNotes) : undefined,
+            priority: String(r.priority ?? 'medium') as LifecycleRecommendation['priority'],
+          };
+        })
+      : [];
+
+    return {
+      snapshot,
+      recommendations,
+      insights: Array.isArray(raw.insights) ? raw.insights.map(String) : [],
+      actionPlan: Array.isArray(raw.actionPlan) ? raw.actionPlan.map(String) : [],
+    };
+  },
+
+  async processOutput(): Promise<void> {
+    // Caller decides how to act on lifecycle recommendations
+  },
+
+  getComputeRequirements(_input, complexity) {
+    return {
+      mode: 'local' as const,
+      complexity,
+      timeout: complexity === 'thorough' ? 240_000 : 120_000,
+    };
+  },
+
+  agentType: 'claude-code',
+  timeout: 180_000,
+  captureToolCalls: true,
+};

package/tests/fixtures/behavioral-trajectories.ts

@@ -0,0 +1,210 @@
+/**
+ * Story-based trajectory fixtures for behavioral tests.
+ *
+ * Each factory produces a realistic trajectory with semantically meaningful
+ * steps — the kind of thing an agent actually does when solving a task.
+ */
+
+import { createTrajectory, type Trajectory } from '../../src/types/trajectory.js';
+import { createTask } from '../../src/types/task.js';
+import { createStep } from '../../src/types/step.js';
+import { successOutcome, failureOutcome } from '../../src/types/outcome.js';
+
+/**
+ * Successful "fix broken TypeScript import" trajectory.
+ * Steps: read file → identify wrong path → edit import → run tsc → pass.
+ */
+export function makeTypescriptImportFix(variant: number): Trajectory {
+  const files = ['routes.ts', 'utils.ts', 'handler.ts', 'service.ts', 'controller.ts', 'middleware.ts'];
+  const file = files[variant % files.length];
+  return createTrajectory({
+    task: createTask({
+      domain: 'typescript',
+      description: `Fix broken TypeScript import in src/${file}`,
+    }),
+    steps: [
+      createStep({
+        thought: `Check the import statements in ${file}`,
+        action: `ReadFile src/${file}`,
+        observation: `import { UserService } from "./services/user"\n// TS2307: Cannot find module './services/user'`,
+      }),
+      createStep({
+        thought: 'The import path is missing the .js extension required by ESM',
+        action: `EditFile src/${file}`,
+        observation: `Updated: import { UserService } from "./services/user.js"`,
+      }),
+      createStep({
+        thought: 'Verify the fix compiles',
+        action: 'Bash tsc --noEmit',
+        observation: 'Compilation successful. No errors found.',
+      }),
+    ],
+    outcome: successOutcome(`Fixed ESM import extension in src/${file}`),
+    agentId: 'test-agent',
+  });
+}
+
+/**
+ * Successful "fix Python type error" trajectory.
+ * Steps: read traceback → identify wrong type → edit function → run pytest → pass.
+ */
+export function makePythonTypeFix(variant: number): Trajectory {
+  const files = ['api.py', 'models.py', 'views.py', 'serializers.py', 'utils.py'];
+  const file = files[variant % files.length];
+  return createTrajectory({
+    task: createTask({
+      domain: 'python',
+      description: `Fix Python type error in ${file}`,
+    }),
+    steps: [
+      createStep({
+        thought: `Read the failing file ${file}`,
+        action: `ReadFile ${file}`,
+        observation: `def process_items(items: list[str]) -> int:\n    return items.count`,
+      }),
+      createStep({
+        thought: 'The function returns the method reference instead of calling it',
+        action: `EditFile ${file}`,
+        observation: `Updated: return items.count()  # was missing parentheses`,
+      }),
+      createStep({
+        thought: 'Run the tests to verify',
+        action: 'Bash pytest -x',
+        observation: '5 passed in 0.3s',
+      }),
+    ],
+    outcome: successOutcome(`Fixed missing parentheses in ${file}`),
+    agentId: 'test-agent',
+  });
+}
+
+/**
+ * Failed "fix authentication bypass" trajectory.
+ * Steps: read auth code → attempt fix → tests still fail.
+ */
+export function makeFailedAuthFix(errorDetail: string): Trajectory {
+  return createTrajectory({
+    task: createTask({
+      domain: 'security',
+      description: 'Fix authentication bypass vulnerability in auth middleware',
+    }),
+    steps: [
+      createStep({
+        thought: 'Read the auth middleware to understand the vulnerability',
+        action: 'ReadFile src/middleware/auth.ts',
+        observation: 'JWT token validation skips expiry check when token has admin role',
+      }),
+      createStep({
+        thought: 'Add expiry validation for all tokens regardless of role',
+        action: 'EditFile src/middleware/auth.ts',
+        observation: 'Added: if (isTokenExpired(token)) throw new AuthError("expired")',
+      }),
+      createStep({
+        thought: 'Run the security test suite',
+        action: 'Bash npm run test:security',
+        observation: `FAIL: ${errorDetail}`,
+      }),
+    ],
+    outcome: failureOutcome(errorDetail),
+    agentId: 'test-agent',
+  });
+}
+
+/**
+ * Successful Kubernetes deployment — completely different domain.
+ */
+export function makeKubernetesDeploy(): Trajectory {
+  return createTrajectory({
+    task: createTask({
+      domain: 'devops',
+      description: 'Deploy updated API service to Kubernetes staging cluster',
+    }),
+    steps: [
+      createStep({
+        thought: 'Check current deployment status',
+        action: 'Bash kubectl get pods -n staging',
+        observation: 'api-service-7f8d9c 1/1 Running 0 12h',
+      }),
+      createStep({
+        thought: 'Apply the new deployment manifest',
+        action: 'Bash kubectl apply -f k8s/api-service.yaml',
+        observation: 'deployment.apps/api-service configured',
+      }),
+      createStep({
+        thought: 'Wait for rollout to complete',
+        action: 'Bash kubectl rollout status deployment/api-service -n staging',
+        observation: 'deployment "api-service" successfully rolled out',
+      }),
+    ],
+    outcome: successOutcome('Deployed api-service v2.3.1 to staging'),
+    agentId: 'test-agent',
+  });
+}
+
+/**
+ * Inefficient trajectory with many repeated read actions.
+ * Useful for testing reflexion critique of wasted steps.
+ */
+export function makeInefficient(stepCount: number): Trajectory {
+  const steps = [];
+  for (let i = 0; i < stepCount; i++) {
+    steps.push(
+      createStep({
+        thought: `Check config again (attempt ${i + 1})`,
+        action: 'ReadFile src/config.ts',
+        observation: 'export const config = { debug: false, port: 3000 }',
+      }),
+    );
+  }
+  // Finally do the actual fix
+  steps.push(
+    createStep({
+      thought: 'After re-reading many times, change the debug flag',
+      action: 'EditFile src/config.ts',
+      observation: 'Updated: debug: true',
+    }),
+  );
+
+  return createTrajectory({
+    task: createTask({
+      domain: 'typescript',
+      description: 'Enable debug mode in application config',
+    }),
+    steps,
+    outcome: failureOutcome('Timed out after too many read operations'),
+    agentId: 'test-agent',
+  });
+}
+
+/**
+ * Successful trajectory with a specific solution description,
+ * useful for verifying causal edge extraction.
+ */
+export function makeSuccessfulWithSolution(
+  domain: string,
+  description: string,
+  solution: string,
+): Trajectory {
+  return createTrajectory({
+    task: createTask({ domain, description }),
+    steps: [
+      createStep({
+        thought: 'Analyze the problem',
+        action: 'ReadFile src/main.ts',
+        observation: 'Found the issue',
+      }),
+      createStep({
+        thought: 'Apply the fix',
+        action: 'EditFile src/main.ts',
+        observation: 'File updated',
+      }),
+      createStep({
+        thought: 'Verify',
+        action: 'Bash npm test',
+        observation: 'All tests passed',
+      }),
+    ],
+    outcome: successOutcome(solution),
+    agentId: 'test-agent',
+  });
+}