cognitive-core 0.2.0 → 0.2.2

package/src/workspace/efficacy-toolkit.ts

@@ -0,0 +1,496 @@
+/**
+ * Efficacy Toolkit
+ *
+ * Pre-computes playbook efficacy data for agentic analysis tasks.
+ * Agents receive the computed data as workspace input files rather than
+ * needing to query raw stores and reinvent metric calculations.
+ *
+ * Key metrics from the literature:
+ * - Normalized gain (SkillsBench): measures marginal value above baseline
+ * - Temporal trend analysis: sliding-window regression to detect decay
+ * - Per-domain breakdown: avoids cross-domain interference in confidence
+ * - Portfolio health: aggregate view of the skill library
+ */
+
+import type { Playbook } from '../types/index.js';
+import type { TaskAnnotation, PlaybookEffectivenessEntry } from '../learning/effectiveness.js';
+import { getPlaybookSuccessRate } from '../types/index.js';
+
+// ============================================================
+// Output Types — these become workspace input files for agents
+// ============================================================
+
+/**
+ * Complete efficacy profile for a single playbook.
+ * Written to workspace as input for the playbook-efficacy-audit template.
+ */
+export interface PlaybookEfficacyProfile {
+  playbookId: string;
+  playbookName: string;
+  confidence: number;
+  successRate: number;
+  totalUses: number;
+  domains: string[];
+
+  /** Raw guided - unguided success delta */
+  successRateDelta: number;
+  /** (guided - unguided) / (1 - unguided) — measures marginal value */
+  normalizedGain: number;
+
+  /** Per-domain success breakdown */
+  domainBreakdown: DomainBreakdown[];
+
+  /** Temporal trend (slope of success rate over time windows) */
+  temporalTrend: TemporalTrend;
+
+  /** Usage patterns */
+  usage: UsageProfile;
+
+  /** Evolution history summary */
+  evolution: EvolutionSummary;
+}
+
+/**
+ * Per-domain performance for a playbook
+ */
+export interface DomainBreakdown {
+  domain: string;
+  appliedCount: number;
+  successCount: number;
+  successRate: number;
+  avgSteps: number;
+}
+
+/**
+ * Temporal trend computed via sliding-window regression
+ */
+export interface TemporalTrend {
+  /** Slope of success rate over time (positive = improving, negative = decaying) */
+  slope: number;
+  /** Categorical assessment */
+  direction: 'improving' | 'stable' | 'decaying';
+  /** Number of data points used */
+  dataPoints: number;
+  /** Success rate in most recent window */
+  recentSuccessRate: number;
+  /** Success rate in oldest window */
+  oldestSuccessRate: number;
+  /** Days since last usage */
+  daysSinceLastUse: number | null;
+}
+
+/**
+ * How the playbook is being used
+ */
+export interface UsageProfile {
+  /** Times surfaced to agents */
+  surfacedCount: number;
+  /** Times agents actually followed it */
+  appliedCount: number;
+  /** surfaced → applied conversion */
+  adoptionRate: number;
+  /** Of applied, how often it led to success */
+  appliedSuccessRate: number;
+  /** Relevance ratings from reflections */
+  relevanceRate: number;
+}
+
+/**
+ * Evolution history digest
+ */
+export interface EvolutionSummary {
+  version: string;
+  totalRefinements: number;
+  recentFailureModes: string[];
+  consolidationStrength: number;
+  domainBreadth: number;
+}
+
+/**
+ * Portfolio-wide health snapshot
+ */
+export interface PortfolioSnapshot {
+  totalPlaybooks: number;
+  avgConfidence: number;
+  avgSuccessRate: number;
+
+  /** Distribution by confidence bands */
+  confidenceBands: {
+    high: number;    // >= 0.7
+    medium: number;  // 0.4 - 0.7
+    low: number;     // < 0.4
+  };
+
+  /** Playbooks needing attention */
+  decaying: PlaybookHealthFlag[];
+  stale: PlaybookHealthFlag[];
+  lowAdoption: PlaybookHealthFlag[];
+  redundant: RedundancyPair[];
+
+  /** Overall system effectiveness */
+  systemMetrics: {
+    guidedSuccessRate: number;
+    unguidedSuccessRate: number;
+    normalizedGain: number;
+    knowledgeApplicationRate: number;
+  };
+}
+
+export interface PlaybookHealthFlag {
+  playbookId: string;
+  playbookName: string;
+  reason: string;
+  metric: number;
+}
+
+export interface RedundancyPair {
+  playbookA: { id: string; name: string };
+  playbookB: { id: string; name: string };
+  overlapScore: number;
+}
+
+// ============================================================
+// Computation Functions
+// ============================================================
+
+/**
+ * Compute normalized gain: (guided - unguided) / (1 - unguided)
+ * Measures marginal improvement above baseline, handling ceiling effects.
+ * Returns 0 if no unguided baseline exists or unguided rate is 1.0.
+ */
+export function computeNormalizedGain(
+  guidedSuccessRate: number,
+  unguidedSuccessRate: number
+): number {
+  if (unguidedSuccessRate >= 1.0) return 0;
+  if (unguidedSuccessRate <= 0 && guidedSuccessRate <= 0) return 0;
+  return (guidedSuccessRate - unguidedSuccessRate) / (1 - unguidedSuccessRate);
+}
+
+/**
+ * Compute temporal trend via simple linear regression on windowed success rates.
+ * Divides the annotation history into windows and measures the slope.
+ */
+export function computeTemporalTrend(
+  annotations: TaskAnnotation[],
+  playbookId: string,
+  windowCount: number = 4
+): TemporalTrend {
+  // Filter to annotations where this playbook was applied
+  const relevant = annotations
+    .filter(a =>
+      a.knowledgeApplied.playbookIdsUsed.includes(playbookId)
+    )
+    .sort((a, b) =>
+      new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
+    );
+
+  if (relevant.length < 2) {
+    return {
+      slope: 0,
+      direction: 'stable',
+      dataPoints: relevant.length,
+      recentSuccessRate: relevant.length === 1 ? (relevant[0].outcome.success ? 1 : 0) : 0,
+      oldestSuccessRate: relevant.length === 1 ? (relevant[0].outcome.success ? 1 : 0) : 0,
+      daysSinceLastUse: relevant.length > 0
+        ? (Date.now() - new Date(relevant[relevant.length - 1].timestamp).getTime()) / 86400000
+        : null,
+    };
+  }
+
+  // Divide into windows
+  const actualWindows = Math.min(windowCount, relevant.length);
+  const windowSize = Math.ceil(relevant.length / actualWindows);
+  const windowRates: number[] = [];
+
+  for (let i = 0; i < relevant.length; i += windowSize) {
+    const window = relevant.slice(i, i + windowSize);
+    const successCount = window.filter(a => a.outcome.success).length;
+    windowRates.push(successCount / window.length);
+  }
+
+  // Simple linear regression on window indices
+  const slope = linearRegressionSlope(windowRates);
+
+  const direction: TemporalTrend['direction'] =
+    slope > 0.05 ? 'improving' :
+    slope < -0.05 ? 'decaying' :
+    'stable';
+
+  const lastAnnotation = relevant[relevant.length - 1];
+  const daysSinceLastUse = (Date.now() - new Date(lastAnnotation.timestamp).getTime()) / 86400000;
+
+  return {
+    slope,
+    direction,
+    dataPoints: relevant.length,
+    recentSuccessRate: windowRates[windowRates.length - 1],
+    oldestSuccessRate: windowRates[0],
+    daysSinceLastUse,
+  };
+}
+
+/**
+ * Compute per-domain breakdown from annotations
+ */
+export function computeDomainBreakdown(
+  annotations: TaskAnnotation[],
+  playbookId: string,
+  trajectoryDomainMap: Map<string, string>
+): DomainBreakdown[] {
+  const domainStats = new Map<string, {
+    applied: number;
+    successes: number;
+    stepSum: number;
+  }>();
+
+  for (const ann of annotations) {
+    if (!ann.knowledgeApplied.playbookIdsUsed.includes(playbookId)) continue;
+
+    const domain = trajectoryDomainMap.get(ann.trajectoryId) ?? 'unknown';
+
+    if (!domainStats.has(domain)) {
+      domainStats.set(domain, { applied: 0, successes: 0, stepSum: 0 });
+    }
+
+    const stats = domainStats.get(domain)!;
+    stats.applied++;
+    if (ann.outcome.success) stats.successes++;
+    stats.stepSum += ann.outcome.stepCount;
+  }
+
+  return Array.from(domainStats.entries())
+    .map(([domain, stats]) => ({
+      domain,
+      appliedCount: stats.applied,
+      successCount: stats.successes,
+      successRate: stats.applied > 0 ? stats.successes / stats.applied : 0,
+      avgSteps: stats.applied > 0 ? stats.stepSum / stats.applied : 0,
+    }))
+    .sort((a, b) => b.appliedCount - a.appliedCount);
+}
+
+/**
+ * Build a complete efficacy profile for a single playbook.
+ * This is the main entry point for the playbook-efficacy-audit template.
+ */
+export function buildEfficacyProfile(
+  playbook: Playbook,
+  annotations: TaskAnnotation[],
+  playbookEffectiveness: PlaybookEffectivenessEntry | undefined,
+  unguidedSuccessRate: number,
+  trajectoryDomainMap: Map<string, string>
+): PlaybookEfficacyProfile {
+  const successRate = getPlaybookSuccessRate(playbook);
+  const appliedSuccessRate = playbookEffectiveness?.appliedSuccessRate ?? successRate;
+  const normalizedGain = computeNormalizedGain(appliedSuccessRate, unguidedSuccessRate);
+
+  const temporalTrend = computeTemporalTrend(annotations, playbook.id);
+  const domainBreakdown = computeDomainBreakdown(annotations, playbook.id, trajectoryDomainMap);
+
+  return {
+    playbookId: playbook.id,
+    playbookName: playbook.name,
+    confidence: playbook.confidence,
+    successRate,
+    totalUses: playbook.evolution.successCount + playbook.evolution.failureCount,
+    domains: playbook.applicability.domains,
+    successRateDelta: appliedSuccessRate - unguidedSuccessRate,
+    normalizedGain,
+    domainBreakdown,
+    temporalTrend,
+    usage: {
+      surfacedCount: playbookEffectiveness?.surfacedCount ?? 0,
+      appliedCount: playbookEffectiveness?.appliedCount ?? 0,
+      adoptionRate: playbookEffectiveness
+        ? (playbookEffectiveness.surfacedCount > 0
+          ? playbookEffectiveness.appliedCount / playbookEffectiveness.surfacedCount
+          : 0)
+        : 0,
+      appliedSuccessRate,
+      relevanceRate: playbookEffectiveness?.relevanceRate ?? 0,
+    },
+    evolution: {
+      version: playbook.evolution.version,
+      totalRefinements: playbook.evolution.refinements.length,
+      recentFailureModes: playbook.evolution.failures
+        .slice(-3)
+        .map(f => f.failureMode),
+      consolidationStrength: playbook.evolution.consolidationStrength ?? 0,
+      domainBreadth: playbook.evolution.domainBreadth ?? 0,
+    },
+  };
+}
+
+/**
+ * Build a portfolio-wide health snapshot.
+ * Main entry point for the playbook-lifecycle-review template.
+ */
+export function buildPortfolioSnapshot(
+  playbooks: Playbook[],
+  annotations: TaskAnnotation[],
+  playbookEffectiveness: PlaybookEffectivenessEntry[],
+  unguidedSuccessRate: number,
+  _trajectoryDomainMap: Map<string, string>,
+  options?: {
+    staleThresholdDays?: number;
+    lowAdoptionThreshold?: number;
+    redundancyThreshold?: number;
+  }
+): PortfolioSnapshot {
+  const staleThresholdDays = options?.staleThresholdDays ?? 30;
+  const lowAdoptionThreshold = options?.lowAdoptionThreshold ?? 0.2;
+  const redundancyThreshold = options?.redundancyThreshold ?? 0.8;
+
+  const effectivenessMap = new Map(
+    playbookEffectiveness.map(e => [e.playbookId, e])
+  );
+
+  // Confidence bands
+  const high = playbooks.filter(p => p.confidence >= 0.7).length;
+  const low = playbooks.filter(p => p.confidence < 0.4).length;
+  const medium = playbooks.length - high - low;
+
+  // Detect decaying playbooks
+  const decaying: PlaybookHealthFlag[] = [];
+  const stale: PlaybookHealthFlag[] = [];
+  const lowAdoption: PlaybookHealthFlag[] = [];
+
+  for (const pb of playbooks) {
+    const trend = computeTemporalTrend(annotations, pb.id);
+
+    if (trend.direction === 'decaying' && trend.dataPoints >= 4) {
+      decaying.push({
+        playbookId: pb.id,
+        playbookName: pb.name,
+        reason: `Success rate declining (slope: ${trend.slope.toFixed(3)})`,
+        metric: trend.slope,
+      });
+    }
+
+    if (trend.daysSinceLastUse !== null && trend.daysSinceLastUse > staleThresholdDays) {
+      stale.push({
+        playbookId: pb.id,
+        playbookName: pb.name,
+        reason: `No usage in ${Math.round(trend.daysSinceLastUse)} days`,
+        metric: trend.daysSinceLastUse,
+      });
+    }
+
+    const eff = effectivenessMap.get(pb.id);
+    if (eff && eff.surfacedCount >= 5) {
+      const adoptionRate = eff.appliedCount / eff.surfacedCount;
+      if (adoptionRate < lowAdoptionThreshold) {
+        lowAdoption.push({
+          playbookId: pb.id,
+          playbookName: pb.name,
+          reason: `Surfaced ${eff.surfacedCount} times but only applied ${eff.appliedCount} times`,
+          metric: adoptionRate,
+        });
+      }
+    }
+  }
+
+  // Detect redundant playbook pairs via strategy + tactic overlap
+  const redundant = detectRedundancy(playbooks, redundancyThreshold);
+
+  // System-level metrics
+  const guided = annotations.filter(a => a.knowledgeSurfaced.totalItems > 0);
+  const unguided = annotations.filter(a => a.knowledgeSurfaced.totalItems === 0);
+  const guidedSuccessRate = guided.length > 0
+    ? guided.filter(a => a.outcome.success).length / guided.length
+    : 0;
+  const actualUnguidedRate = unguided.length > 0
+    ? unguided.filter(a => a.outcome.success).length / unguided.length
+    : unguidedSuccessRate;
+  const applied = guided.filter(a => a.knowledgeApplied.anyKnowledgeUsed);
+  const applicationRate = guided.length > 0 ? applied.length / guided.length : 0;
+
+  const avgConfidence = playbooks.length > 0
+    ? playbooks.reduce((s, p) => s + p.confidence, 0) / playbooks.length
+    : 0;
+  const avgSuccessRate = playbooks.length > 0
+    ? playbooks.reduce((s, p) => s + getPlaybookSuccessRate(p), 0) / playbooks.length
+    : 0;
+
+  return {
+    totalPlaybooks: playbooks.length,
+    avgConfidence,
+    avgSuccessRate,
+    confidenceBands: { high, medium, low },
+    decaying,
+    stale,
+    lowAdoption,
+    redundant,
+    systemMetrics: {
+      guidedSuccessRate,
+      unguidedSuccessRate: actualUnguidedRate,
+      normalizedGain: computeNormalizedGain(guidedSuccessRate, actualUnguidedRate),
+      knowledgeApplicationRate: applicationRate,
+    },
+  };
+}
+
+// ============================================================
+// Internal Helpers
+// ============================================================
+
+/**
+ * Simple linear regression slope on evenly-spaced data points.
+ */
+function linearRegressionSlope(values: number[]): number {
+  const n = values.length;
+  if (n < 2) return 0;
+
+  const meanX = (n - 1) / 2;
+  const meanY = values.reduce((s, v) => s + v, 0) / n;
+
+  let num = 0;
+  let den = 0;
+  for (let i = 0; i < n; i++) {
+    num += (i - meanX) * (values[i] - meanY);
+    den += (i - meanX) * (i - meanX);
+  }
+
+  return den === 0 ? 0 : num / den;
+}
+
+/**
+ * Detect redundant playbook pairs using text overlap of strategies + tactics.
+ */
+function detectRedundancy(
+  playbooks: Playbook[],
+  threshold: number
+): RedundancyPair[] {
+  const pairs: RedundancyPair[] = [];
+
+  for (let i = 0; i < playbooks.length; i++) {
+    for (let j = i + 1; j < playbooks.length; j++) {
+      const a = playbooks[i];
+      const b = playbooks[j];
+
+      // Compare strategy + tactics text
+      const textA = [a.guidance.strategy, ...a.guidance.tactics].join(' ').toLowerCase();
+      const textB = [b.guidance.strategy, ...b.guidance.tactics].join(' ').toLowerCase();
+
+      const wordsA = new Set(textA.split(/\s+/).filter(w => w.length > 2));
+      const wordsB = new Set(textB.split(/\s+/).filter(w => w.length > 2));
+
+      if (wordsA.size === 0 || wordsB.size === 0) continue;
+
+      const intersection = [...wordsA].filter(w => wordsB.has(w)).length;
+      const union = new Set([...wordsA, ...wordsB]).size;
+      const jaccard = intersection / union;
+
+      if (jaccard >= threshold) {
+        pairs.push({
+          playbookA: { id: a.id, name: a.name },
+          playbookB: { id: b.id, name: b.name },
+          overlapScore: jaccard,
+        });
+      }
+    }
+  }
+
+  return pairs.sort((a, b) => b.overlapScore - a.overlapScore);
+}

package/src/workspace/index.ts

@@ -45,4 +45,33 @@ export {
   refinementAnalysisTemplate,
   type RefinementAnalysisInput,
   type RefinementPlan,
+  playbookEfficacyAuditTemplate,
+  type PlaybookEfficacyAuditInput,
+  type PlaybookEfficacyAuditOutput,
+  type PlaybookRecommendation,
+  playbookDecayDetectionTemplate,
+  type PlaybookDecayDetectionInput,
+  type PlaybookDecayDetectionOutput,
+  type DecaySignal,
+  playbookLifecycleReviewTemplate,
+  type PlaybookLifecycleReviewInput,
+  type PlaybookLifecycleReviewOutput,
+  type LifecycleRecommendation,
 } from './templates/index.js';
+
+// Efficacy Toolkit
+export {
+  computeNormalizedGain,
+  computeTemporalTrend,
+  computeDomainBreakdown,
+  buildEfficacyProfile,
+  buildPortfolioSnapshot,
+  type PlaybookEfficacyProfile,
+  type DomainBreakdown,
+  type TemporalTrend,
+  type UsageProfile,
+  type EvolutionSummary,
+  type PortfolioSnapshot,
+  type PlaybookHealthFlag,
+  type RedundancyPair,
+} from './efficacy-toolkit.js';

package/src/workspace/templates/index.ts

@@ -62,3 +62,27 @@ export {
   type TeamPlaybookUpdate,
   type RolePlaybookUpdate,
 } from './team-playbook-extraction.js';
+
+// Playbook Efficacy Audit (meta-learning — skill evaluation)
+export {
+  playbookEfficacyAuditTemplate,
+  type PlaybookEfficacyAuditInput,
+  type PlaybookEfficacyAuditOutput,
+  type PlaybookRecommendation,
+} from './playbook-efficacy-audit.js';
+
+// Playbook Decay Detection (meta-learning — temporal trend analysis)
+export {
+  playbookDecayDetectionTemplate,
+  type PlaybookDecayDetectionInput,
+  type PlaybookDecayDetectionOutput,
+  type DecaySignal,
+} from './playbook-decay-detection.js';
+
+// Playbook Lifecycle Review (meta-learning — portfolio health)
+export {
+  playbookLifecycleReviewTemplate,
+  type PlaybookLifecycleReviewInput,
+  type PlaybookLifecycleReviewOutput,
+  type LifecycleRecommendation,
+} from './playbook-lifecycle-review.js';