cognitive-core 0.2.4 → 0.3.1

package/src/workspace/efficacy-toolkit.ts

@@ -1,496 +0,0 @@
-/**
- * Efficacy Toolkit
- *
- * Pre-computes playbook efficacy data for agentic analysis tasks.
- * Agents receive the computed data as workspace input files rather than
- * needing to query raw stores and reinvent metric calculations.
- *
- * Key metrics from the literature:
- * - Normalized gain (SkillsBench): measures marginal value above baseline
- * - Temporal trend analysis: sliding-window regression to detect decay
- * - Per-domain breakdown: avoids cross-domain interference in confidence
- * - Portfolio health: aggregate view of the skill library
- */
-
-import type { Playbook } from '../types/index.js';
-import type { TaskAnnotation, PlaybookEffectivenessEntry } from '../learning/effectiveness.js';
-import { getPlaybookSuccessRate } from '../types/index.js';
-
-// ============================================================
-// Output Types — these become workspace input files for agents
-// ============================================================
-
-/**
- * Complete efficacy profile for a single playbook.
- * Written to workspace as input for the playbook-efficacy-audit template.
- */
-export interface PlaybookEfficacyProfile {
-  playbookId: string;
-  playbookName: string;
-  confidence: number;
-  successRate: number;
-  totalUses: number;
-  domains: string[];
-
-  /** Raw guided - unguided success delta */
-  successRateDelta: number;
-  /** (guided - unguided) / (1 - unguided) — measures marginal value */
-  normalizedGain: number;
-
-  /** Per-domain success breakdown */
-  domainBreakdown: DomainBreakdown[];
-
-  /** Temporal trend (slope of success rate over time windows) */
-  temporalTrend: TemporalTrend;
-
-  /** Usage patterns */
-  usage: UsageProfile;
-
-  /** Evolution history summary */
-  evolution: EvolutionSummary;
-}
-
-/**
- * Per-domain performance for a playbook
- */
-export interface DomainBreakdown {
-  domain: string;
-  appliedCount: number;
-  successCount: number;
-  successRate: number;
-  avgSteps: number;
-}
-
-/**
- * Temporal trend computed via sliding-window regression
- */
-export interface TemporalTrend {
-  /** Slope of success rate over time (positive = improving, negative = decaying) */
-  slope: number;
-  /** Categorical assessment */
-  direction: 'improving' | 'stable' | 'decaying';
-  /** Number of data points used */
-  dataPoints: number;
-  /** Success rate in most recent window */
-  recentSuccessRate: number;
-  /** Success rate in oldest window */
-  oldestSuccessRate: number;
-  /** Days since last usage */
-  daysSinceLastUse: number | null;
-}
-
-/**
- * How the playbook is being used
- */
-export interface UsageProfile {
-  /** Times surfaced to agents */
-  surfacedCount: number;
-  /** Times agents actually followed it */
-  appliedCount: number;
-  /** surfaced → applied conversion */
-  adoptionRate: number;
-  /** Of applied, how often it led to success */
-  appliedSuccessRate: number;
-  /** Relevance ratings from reflections */
-  relevanceRate: number;
-}
-
-/**
- * Evolution history digest
- */
-export interface EvolutionSummary {
-  version: string;
-  totalRefinements: number;
-  recentFailureModes: string[];
-  consolidationStrength: number;
-  domainBreadth: number;
-}
-
-/**
- * Portfolio-wide health snapshot
- */
-export interface PortfolioSnapshot {
-  totalPlaybooks: number;
-  avgConfidence: number;
-  avgSuccessRate: number;
-
-  /** Distribution by confidence bands */
-  confidenceBands: {
-    high: number;    // >= 0.7
-    medium: number;  // 0.4 - 0.7
-    low: number;     // < 0.4
-  };
-
-  /** Playbooks needing attention */
-  decaying: PlaybookHealthFlag[];
-  stale: PlaybookHealthFlag[];
-  lowAdoption: PlaybookHealthFlag[];
-  redundant: RedundancyPair[];
-
-  /** Overall system effectiveness */
-  systemMetrics: {
-    guidedSuccessRate: number;
-    unguidedSuccessRate: number;
-    normalizedGain: number;
-    knowledgeApplicationRate: number;
-  };
-}
-
-export interface PlaybookHealthFlag {
-  playbookId: string;
-  playbookName: string;
-  reason: string;
-  metric: number;
-}
-
-export interface RedundancyPair {
-  playbookA: { id: string; name: string };
-  playbookB: { id: string; name: string };
-  overlapScore: number;
-}
-
-// ============================================================
-// Computation Functions
-// ============================================================
-
-/**
- * Compute normalized gain: (guided - unguided) / (1 - unguided)
- * Measures marginal improvement above baseline, handling ceiling effects.
- * Returns 0 if no unguided baseline exists or unguided rate is 1.0.
- */
-export function computeNormalizedGain(
-  guidedSuccessRate: number,
-  unguidedSuccessRate: number
-): number {
-  if (unguidedSuccessRate >= 1.0) return 0;
-  if (unguidedSuccessRate <= 0 && guidedSuccessRate <= 0) return 0;
-  return (guidedSuccessRate - unguidedSuccessRate) / (1 - unguidedSuccessRate);
-}
-
-/**
- * Compute temporal trend via simple linear regression on windowed success rates.
- * Divides the annotation history into windows and measures the slope.
- */
-export function computeTemporalTrend(
-  annotations: TaskAnnotation[],
-  playbookId: string,
-  windowCount: number = 4
-): TemporalTrend {
-  // Filter to annotations where this playbook was applied
-  const relevant = annotations
-    .filter(a =>
-      a.knowledgeApplied.playbookIdsUsed.includes(playbookId)
-    )
-    .sort((a, b) =>
-      new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
-    );
-
-  if (relevant.length < 2) {
-    return {
-      slope: 0,
-      direction: 'stable',
-      dataPoints: relevant.length,
-      recentSuccessRate: relevant.length === 1 ? (relevant[0].outcome.success ? 1 : 0) : 0,
-      oldestSuccessRate: relevant.length === 1 ? (relevant[0].outcome.success ? 1 : 0) : 0,
-      daysSinceLastUse: relevant.length > 0
-        ? (Date.now() - new Date(relevant[relevant.length - 1].timestamp).getTime()) / 86400000
-        : null,
-    };
-  }
-
-  // Divide into windows
-  const actualWindows = Math.min(windowCount, relevant.length);
-  const windowSize = Math.ceil(relevant.length / actualWindows);
-  const windowRates: number[] = [];
-
-  for (let i = 0; i < relevant.length; i += windowSize) {
-    const window = relevant.slice(i, i + windowSize);
-    const successCount = window.filter(a => a.outcome.success).length;
-    windowRates.push(successCount / window.length);
-  }
-
-  // Simple linear regression on window indices
-  const slope = linearRegressionSlope(windowRates);
-
-  const direction: TemporalTrend['direction'] =
-    slope > 0.05 ? 'improving' :
-    slope < -0.05 ? 'decaying' :
-    'stable';
-
-  const lastAnnotation = relevant[relevant.length - 1];
-  const daysSinceLastUse = (Date.now() - new Date(lastAnnotation.timestamp).getTime()) / 86400000;
-
-  return {
-    slope,
-    direction,
-    dataPoints: relevant.length,
-    recentSuccessRate: windowRates[windowRates.length - 1],
-    oldestSuccessRate: windowRates[0],
-    daysSinceLastUse,
-  };
-}
-
-/**
- * Compute per-domain breakdown from annotations
- */
-export function computeDomainBreakdown(
-  annotations: TaskAnnotation[],
-  playbookId: string,
-  trajectoryDomainMap: Map<string, string>
-): DomainBreakdown[] {
-  const domainStats = new Map<string, {
-    applied: number;
-    successes: number;
-    stepSum: number;
-  }>();
-
-  for (const ann of annotations) {
-    if (!ann.knowledgeApplied.playbookIdsUsed.includes(playbookId)) continue;
-
-    const domain = trajectoryDomainMap.get(ann.trajectoryId) ?? 'unknown';
-
-    if (!domainStats.has(domain)) {
-      domainStats.set(domain, { applied: 0, successes: 0, stepSum: 0 });
-    }
-
-    const stats = domainStats.get(domain)!;
-    stats.applied++;
-    if (ann.outcome.success) stats.successes++;
-    stats.stepSum += ann.outcome.stepCount;
-  }
-
-  return Array.from(domainStats.entries())
-    .map(([domain, stats]) => ({
-      domain,
-      appliedCount: stats.applied,
-      successCount: stats.successes,
-      successRate: stats.applied > 0 ? stats.successes / stats.applied : 0,
-      avgSteps: stats.applied > 0 ? stats.stepSum / stats.applied : 0,
-    }))
-    .sort((a, b) => b.appliedCount - a.appliedCount);
-}
-
-/**
- * Build a complete efficacy profile for a single playbook.
- * This is the main entry point for the playbook-efficacy-audit template.
- */
-export function buildEfficacyProfile(
-  playbook: Playbook,
-  annotations: TaskAnnotation[],
-  playbookEffectiveness: PlaybookEffectivenessEntry | undefined,
-  unguidedSuccessRate: number,
-  trajectoryDomainMap: Map<string, string>
-): PlaybookEfficacyProfile {
-  const successRate = getPlaybookSuccessRate(playbook);
-  const appliedSuccessRate = playbookEffectiveness?.appliedSuccessRate ?? successRate;
-  const normalizedGain = computeNormalizedGain(appliedSuccessRate, unguidedSuccessRate);
-
-  const temporalTrend = computeTemporalTrend(annotations, playbook.id);
-  const domainBreakdown = computeDomainBreakdown(annotations, playbook.id, trajectoryDomainMap);
-
-  return {
-    playbookId: playbook.id,
-    playbookName: playbook.name,
-    confidence: playbook.confidence,
-    successRate,
-    totalUses: playbook.evolution.successCount + playbook.evolution.failureCount,
-    domains: playbook.applicability.domains,
-    successRateDelta: appliedSuccessRate - unguidedSuccessRate,
-    normalizedGain,
-    domainBreakdown,
-    temporalTrend,
-    usage: {
-      surfacedCount: playbookEffectiveness?.surfacedCount ?? 0,
-      appliedCount: playbookEffectiveness?.appliedCount ?? 0,
-      adoptionRate: playbookEffectiveness
-        ? (playbookEffectiveness.surfacedCount > 0
-          ? playbookEffectiveness.appliedCount / playbookEffectiveness.surfacedCount
-          : 0)
-        : 0,
-      appliedSuccessRate,
-      relevanceRate: playbookEffectiveness?.relevanceRate ?? 0,
-    },
-    evolution: {
-      version: playbook.evolution.version,
-      totalRefinements: playbook.evolution.refinements.length,
-      recentFailureModes: playbook.evolution.failures
-        .slice(-3)
-        .map(f => f.failureMode),
-      consolidationStrength: playbook.evolution.consolidationStrength ?? 0,
-      domainBreadth: playbook.evolution.domainBreadth ?? 0,
-    },
-  };
-}
-
-/**
- * Build a portfolio-wide health snapshot.
- * Main entry point for the playbook-lifecycle-review template.
- */
-export function buildPortfolioSnapshot(
-  playbooks: Playbook[],
-  annotations: TaskAnnotation[],
-  playbookEffectiveness: PlaybookEffectivenessEntry[],
-  unguidedSuccessRate: number,
-  _trajectoryDomainMap: Map<string, string>,
-  options?: {
-    staleThresholdDays?: number;
-    lowAdoptionThreshold?: number;
-    redundancyThreshold?: number;
-  }
-): PortfolioSnapshot {
-  const staleThresholdDays = options?.staleThresholdDays ?? 30;
-  const lowAdoptionThreshold = options?.lowAdoptionThreshold ?? 0.2;
-  const redundancyThreshold = options?.redundancyThreshold ?? 0.8;
-
-  const effectivenessMap = new Map(
-    playbookEffectiveness.map(e => [e.playbookId, e])
-  );
-
-  // Confidence bands
-  const high = playbooks.filter(p => p.confidence >= 0.7).length;
-  const low = playbooks.filter(p => p.confidence < 0.4).length;
-  const medium = playbooks.length - high - low;
-
-  // Detect decaying playbooks
-  const decaying: PlaybookHealthFlag[] = [];
-  const stale: PlaybookHealthFlag[] = [];
-  const lowAdoption: PlaybookHealthFlag[] = [];
-
-  for (const pb of playbooks) {
-    const trend = computeTemporalTrend(annotations, pb.id);
-
-    if (trend.direction === 'decaying' && trend.dataPoints >= 4) {
-      decaying.push({
-        playbookId: pb.id,
-        playbookName: pb.name,
-        reason: `Success rate declining (slope: ${trend.slope.toFixed(3)})`,
-        metric: trend.slope,
-      });
-    }
-
-    if (trend.daysSinceLastUse !== null && trend.daysSinceLastUse > staleThresholdDays) {
-      stale.push({
-        playbookId: pb.id,
-        playbookName: pb.name,
-        reason: `No usage in ${Math.round(trend.daysSinceLastUse)} days`,
-        metric: trend.daysSinceLastUse,
-      });
-    }
-
-    const eff = effectivenessMap.get(pb.id);
-    if (eff && eff.surfacedCount >= 5) {
-      const adoptionRate = eff.appliedCount / eff.surfacedCount;
-      if (adoptionRate < lowAdoptionThreshold) {
-        lowAdoption.push({
-          playbookId: pb.id,
-          playbookName: pb.name,
-          reason: `Surfaced ${eff.surfacedCount} times but only applied ${eff.appliedCount} times`,
-          metric: adoptionRate,
-        });
-      }
-    }
-  }
-
-  // Detect redundant playbook pairs via strategy + tactic overlap
-  const redundant = detectRedundancy(playbooks, redundancyThreshold);
-
-  // System-level metrics
-  const guided = annotations.filter(a => a.knowledgeSurfaced.totalItems > 0);
-  const unguided = annotations.filter(a => a.knowledgeSurfaced.totalItems === 0);
-  const guidedSuccessRate = guided.length > 0
-    ? guided.filter(a => a.outcome.success).length / guided.length
-    : 0;
-  const actualUnguidedRate = unguided.length > 0
-    ? unguided.filter(a => a.outcome.success).length / unguided.length
-    : unguidedSuccessRate;
-  const applied = guided.filter(a => a.knowledgeApplied.anyKnowledgeUsed);
-  const applicationRate = guided.length > 0 ? applied.length / guided.length : 0;
-
-  const avgConfidence = playbooks.length > 0
-    ? playbooks.reduce((s, p) => s + p.confidence, 0) / playbooks.length
-    : 0;
-  const avgSuccessRate = playbooks.length > 0
-    ? playbooks.reduce((s, p) => s + getPlaybookSuccessRate(p), 0) / playbooks.length
-    : 0;
-
-  return {
-    totalPlaybooks: playbooks.length,
-    avgConfidence,
-    avgSuccessRate,
-    confidenceBands: { high, medium, low },
-    decaying,
-    stale,
-    lowAdoption,
-    redundant,
-    systemMetrics: {
-      guidedSuccessRate,
-      unguidedSuccessRate: actualUnguidedRate,
-      normalizedGain: computeNormalizedGain(guidedSuccessRate, actualUnguidedRate),
-      knowledgeApplicationRate: applicationRate,
-    },
-  };
-}
-
-// ============================================================
-// Internal Helpers
-// ============================================================
-
-/**
- * Simple linear regression slope on evenly-spaced data points.
- */
-function linearRegressionSlope(values: number[]): number {
-  const n = values.length;
-  if (n < 2) return 0;
-
-  const meanX = (n - 1) / 2;
-  const meanY = values.reduce((s, v) => s + v, 0) / n;
-
-  let num = 0;
-  let den = 0;
-  for (let i = 0; i < n; i++) {
-    num += (i - meanX) * (values[i] - meanY);
-    den += (i - meanX) * (i - meanX);
-  }
-
-  return den === 0 ? 0 : num / den;
-}
-
-/**
- * Detect redundant playbook pairs using text overlap of strategies + tactics.
- */
-function detectRedundancy(
-  playbooks: Playbook[],
-  threshold: number
-): RedundancyPair[] {
-  const pairs: RedundancyPair[] = [];
-
-  for (let i = 0; i < playbooks.length; i++) {
-    for (let j = i + 1; j < playbooks.length; j++) {
-      const a = playbooks[i];
-      const b = playbooks[j];
-
-      // Compare strategy + tactics text
-      const textA = [a.guidance.strategy, ...a.guidance.tactics].join(' ').toLowerCase();
-      const textB = [b.guidance.strategy, ...b.guidance.tactics].join(' ').toLowerCase();
-
-      const wordsA = new Set(textA.split(/\s+/).filter(w => w.length > 2));
-      const wordsB = new Set(textB.split(/\s+/).filter(w => w.length > 2));
-
-      if (wordsA.size === 0 || wordsB.size === 0) continue;
-
-      const intersection = [...wordsA].filter(w => wordsB.has(w)).length;
-      const union = new Set([...wordsA, ...wordsB]).size;
-      const jaccard = intersection / union;
-
-      if (jaccard >= threshold) {
-        pairs.push({
-          playbookA: { id: a.id, name: a.name },
-          playbookB: { id: b.id, name: b.name },
-          overlapScore: jaccard,
-        });
-      }
-    }
-  }
-
-  return pairs.sort((a, b) => b.overlapScore - a.overlapScore);
-}

package/src/workspace/index.ts

@@ -1,77 +0,0 @@
-// Workspace management (from agent-workspace package)
-export { WorkspaceManager, WorkspaceHandle } from 'agent-workspace';
-
-// Workspace types
-export {
-  type OutputSpec,
-  type OutputFileSpec,
-  type ValidationResult,
-  type AgentSkillSpec,
-  type ResourceSpec,
-  type AnalysisComplexity,
-  type ProcessingContext,
-  type AgenticTaskTemplate,
-  type AgenticTaskResult,
-  type AgenticTaskMetrics,
-  AgenticTaskOutputError,
-} from './types.js';
-
-// AgenticTaskRunner
-export {
-  AgenticTaskRunner,
-  createAgenticTaskRunner,
-} from './runner.js';
-
-// Playbook ↔ Skill converter
-export {
-  convertPlaybooksToSkills,
-  generateSkillMd,
-  parseSkillMdToPlaybook,
-} from './skill-converter.js';
-
-// Templates
-export {
-  trajectoryAnalysisTemplate,
-  type TrajectoryAnalysisInput,
-  playbookExtractionTemplate,
-  type PlaybookExtractionInput,
-  usageInferenceTemplate,
-  type UsageInferenceInput,
-  metaReflectionTemplate,
-  type MetaReflectionInput,
-  type MetaReflectionOutput,
-  solutionEvaluationTemplate,
-  type SolutionEvaluationInput,
-  refinementAnalysisTemplate,
-  type RefinementAnalysisInput,
-  type RefinementPlan,
-  playbookEfficacyAuditTemplate,
-  type PlaybookEfficacyAuditInput,
-  type PlaybookEfficacyAuditOutput,
-  type PlaybookRecommendation,
-  playbookDecayDetectionTemplate,
-  type PlaybookDecayDetectionInput,
-  type PlaybookDecayDetectionOutput,
-  type DecaySignal,
-  playbookLifecycleReviewTemplate,
-  type PlaybookLifecycleReviewInput,
-  type PlaybookLifecycleReviewOutput,
-  type LifecycleRecommendation,
-} from './templates/index.js';
-
-// Efficacy Toolkit
-export {
-  computeNormalizedGain,
-  computeTemporalTrend,
-  computeDomainBreakdown,
-  buildEfficacyProfile,
-  buildPortfolioSnapshot,
-  type PlaybookEfficacyProfile,
-  type DomainBreakdown,
-  type TemporalTrend,
-  type UsageProfile,
-  type EvolutionSummary,
-  type PortfolioSnapshot,
-  type PlaybookHealthFlag,
-  type RedundancyPair,
-} from './efficacy-toolkit.js';