cognitive-core 0.0.2 → 0.1.2

package/dist/learning/analyzer.d.ts

@@ -0,0 +1,110 @@
+import type { Trajectory, CreditStrategy } from '../types/index.js';
+/**
+ * Analysis result from trajectory examination
+ */
+export interface AnalysisResult {
+    /** Whether the trajectory succeeded */
+    success: boolean;
+    /** Indices of key steps (above attribution threshold) */
+    keySteps: number[];
+    /** Attribution score for each step */
+    stepAttribution: number[];
+    /** Detected error patterns */
+    errorPatterns: ErrorPattern[];
+    /** Whether this trajectory is worth extracting patterns from */
+    abstractable: boolean;
+    /** Suggested training examples */
+    trainingExamples: TrainingExample[];
+}
+/**
+ * Detected error pattern
+ */
+export interface ErrorPattern {
+    type: string;
+    signature: string;
+    description: string;
+    suggestedFix?: string;
+}
+/**
+ * Training example extracted from trajectory
+ */
+export interface TrainingExample {
+    input: string;
+    output: string;
+    stepIndex: number;
+}
+/**
+ * Configuration for trajectory analyzer
+ */
+export interface AnalyzerConfig {
+    /** Credit decay factor for exponential decay (default: 0.5) */
+    creditDecayFactor: number;
+    /** Minimum action repetitions to flag as repetition pattern (default: 3) */
+    actionRepetitionThreshold: number;
+    /** Attribution threshold for key step identification (default: 0.15) */
+    attributionThreshold: number;
+}
+/**
+ * Credit assignment strategies
+ */
+export type CreditAssignmentFn = (trajectory: Trajectory, config: AnalyzerConfig) => Promise<number[]>;
+/**
+ * Simple exponential decay credit assignment
+ * Last step gets 1.0, decays exponentially backwards
+ */
+export declare function simpleCreditAssignment(trajectory: Trajectory, config: AnalyzerConfig): Promise<number[]>;
+/**
+ * Outcome-based credit assignment
+ * Successful trajectories get higher credit for all steps
+ */
+export declare function outcomeCreditAssignment(trajectory: Trajectory, _config: AnalyzerConfig): Promise<number[]>;
+/**
+ * Get credit assignment function by strategy type
+ */
+export declare function getCreditAssignmentFn(strategy: CreditStrategy): CreditAssignmentFn;
+/**
+ * Trajectory Analyzer
+ * Analyzes trajectories to extract learning signals
+ */
+export declare class TrajectoryAnalyzer {
+    private creditFn;
+    private config;
+    constructor(strategy?: CreditStrategy, config?: Partial<AnalyzerConfig>);
+    /**
+     * Analyze a trajectory
+     */
+    analyze(trajectory: Trajectory): Promise<AnalysisResult>;
+    /**
+     * Detect error patterns in failed trajectories
+     */
+    private detectErrorPatterns;
+    /**
+     * Assess whether a trajectory is worth abstracting patterns from
+     */
+    private assessAbstractability;
+    /**
+     * Extract training examples from trajectory
+     */
+    private extractTrainingExamples;
+    /**
+     * Set credit assignment strategy
+     */
+    setStrategy(strategy: CreditStrategy): void;
+    /**
+     * Set attribution threshold
+     */
+    setThreshold(threshold: number): void;
+    /**
+     * Update configuration
+     */
+    updateConfig(config: Partial<AnalyzerConfig>): void;
+    /**
+     * Get current configuration
+     */
+    getConfig(): AnalyzerConfig;
+}
+/**
+ * Create a trajectory analyzer
+ */
+export declare function createAnalyzer(strategy?: CreditStrategy, config?: Partial<AnalyzerConfig>): TrajectoryAnalyzer;
+//# sourceMappingURL=analyzer.d.ts.map

package/dist/learning/analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../../src/learning/analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEpE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,sCAAsC;IACtC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,8BAA8B;IAC9B,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,gEAAgE;IAChE,YAAY,EAAE,OAAO,CAAC;IACtB,kCAAkC;IAClC,gBAAgB,EAAE,eAAe,EAAE,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,+DAA+D;IAC/D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,4EAA4E;IAC5E,yBAAyB,EAAE,MAAM,CAAC;IAClC,wEAAwE;IACxE,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAQD;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,cAAc,KACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AAEvB;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC,MAAM,EAAE,CAAC,CAWnB;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC,CAWnB;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,cAAc,GACvB,kBAAkB,CAepB;AAED;;;GAGG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,MAAM,CAAiB;gBAEnB,QAAQ,GAAE,cAAyB,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC;IAKjF;;OAEG;IACG,OAAO,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,CAAC;IAyC9D;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmD3B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAuB7B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA8B/B;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IAI3C;;OAEG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAIrC;;OAEG;IACH,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,IAAI;IAInD;;OAEG;IACH,SAAS,IAAI,cAAc;CAG5B;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,GAAE,cAAyB,EACnC,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAC/B,kBAAkB,CAEpB"}

package/dist/learning/analyzer.js

@@ -0,0 +1,213 @@
+const DEFAULT_ANALYZER_CONFIG = {
+    creditDecayFactor: 0.5,
+    actionRepetitionThreshold: 3,
+    attributionThreshold: 0.15,
+};
+/**
+ * Simple exponential decay credit assignment
+ * Last step gets 1.0, decays exponentially backwards
+ */
+export function simpleCreditAssignment(trajectory, config) {
+    const n = trajectory.steps.length;
+    const scores = [];
+    const decay = config.creditDecayFactor;
+    for (let i = 0; i < n; i++) {
+        const distanceFromEnd = n - 1 - i;
+        scores.push(Math.pow(decay, distanceFromEnd));
+    }
+    return Promise.resolve(scores);
+}
+/**
+ * Outcome-based credit assignment
+ * Successful trajectories get higher credit for all steps
+ */
+export function outcomeCreditAssignment(trajectory, _config) {
+    const baseScores = trajectory.steps.map(() => trajectory.outcome.success ? 0.7 : 0.3);
+    // Boost last step for successful trajectories
+    if (trajectory.outcome.success && baseScores.length > 0) {
+        baseScores[baseScores.length - 1] = 1.0;
+    }
+    return Promise.resolve(baseScores);
+}
+/**
+ * Get credit assignment function by strategy type
+ */
+export function getCreditAssignmentFn(strategy) {
+    switch (strategy) {
+        case 'simple':
+            return simpleCreditAssignment;
+        case 'llm':
+            // LLM-based credit assignment would require an LLM client
+            // Fall back to simple for now
+            return simpleCreditAssignment;
+        case 'counterfactual':
+            // Counterfactual requires environment replay
+            // Fall back to outcome-based
+            return outcomeCreditAssignment;
+        default:
+            return simpleCreditAssignment;
+    }
+}
+/**
+ * Trajectory Analyzer
+ * Analyzes trajectories to extract learning signals
+ */
+export class TrajectoryAnalyzer {
+    creditFn;
+    config;
+    constructor(strategy = 'simple', config) {
+        this.creditFn = getCreditAssignmentFn(strategy);
+        this.config = { ...DEFAULT_ANALYZER_CONFIG, ...config };
+    }
+    /**
+     * Analyze a trajectory
+     */
+    async analyze(trajectory) {
+        // Assign credit to steps
+        const stepAttribution = await this.creditFn(trajectory, this.config);
+        // Apply attribution to steps
+        for (let i = 0; i < trajectory.steps.length; i++) {
+            trajectory.steps[i].attributionScore = stepAttribution[i];
+        }
+        // Identify key steps
+        const keySteps = stepAttribution
+            .map((score, index) => ({ score, index }))
+            .filter(({ score }) => score >= this.config.attributionThreshold)
+            .map(({ index }) => index);
+        // Detect error patterns
+        const errorPatterns = this.detectErrorPatterns(trajectory);
+        // Assess abstractability
+        const abstractable = this.assessAbstractability(trajectory, keySteps, errorPatterns);
+        // Extract training examples
+        const trainingExamples = this.extractTrainingExamples(trajectory, keySteps);
+        return {
+            success: trajectory.outcome.success,
+            keySteps,
+            stepAttribution,
+            errorPatterns,
+            abstractable,
+            trainingExamples,
+        };
+    }
+    /**
+     * Detect error patterns in failed trajectories
+     */
+    detectErrorPatterns(trajectory) {
+        const patterns = [];
+        if (!trajectory.outcome.success && trajectory.outcome.errorInfo) {
+            // Extract error type from error info
+            const errorInfo = trajectory.outcome.errorInfo;
+            // Common error patterns
+            const errorTypes = [
+                [/timeout/i, 'timeout', 'Operation timed out'],
+                [/not found/i, 'not_found', 'Resource not found'],
+                [/permission denied/i, 'permission', 'Permission denied'],
+                [/syntax error/i, 'syntax', 'Syntax error in code'],
+                [/type error/i, 'type', 'Type mismatch error'],
+                [/assertion|assert/i, 'assertion', 'Assertion failed'],
+                [/exception|error/i, 'general', 'General error'],
+            ];
+            for (const [pattern, type, desc] of errorTypes) {
+                if (pattern.test(errorInfo)) {
+                    patterns.push({
+                        type,
+                        signature: errorInfo.slice(0, 100),
+                        description: desc,
+                    });
+                    break;
+                }
+            }
+        }
+        // Check for repeated failed actions
+        const actionCounts = new Map();
+        for (const step of trajectory.steps) {
+            const count = (actionCounts.get(step.action) ?? 0) + 1;
+            actionCounts.set(step.action, count);
+        }
+        for (const [action, count] of actionCounts) {
+            if (count >= this.config.actionRepetitionThreshold) {
+                patterns.push({
+                    type: 'repetition',
+                    signature: action,
+                    description: `Action repeated ${count} times`,
+                    suggestedFix: 'Try a different approach',
+                });
+            }
+        }
+        return patterns;
+    }
+    /**
+     * Assess whether a trajectory is worth abstracting patterns from
+     */
+    assessAbstractability(trajectory, keySteps, errorPatterns) {
+        // Successful trajectories are generally abstractable
+        if (trajectory.outcome.success) {
+            // Must have enough meaningful steps
+            return keySteps.length >= 2;
+        }
+        // Failed trajectories with clear error patterns might be useful
+        // for learning what NOT to do
+        if (errorPatterns.length > 0) {
+            return (errorPatterns.some((p) => p.suggestedFix) ||
+                trajectory.outcome.partialScore >= 0.3);
+        }
+        return false;
+    }
+    /**
+     * Extract training examples from trajectory
+     */
+    extractTrainingExamples(trajectory, keySteps) {
+        const examples = [];
+        // Extract from key steps
+        for (const index of keySteps) {
+            const step = trajectory.steps[index];
+            if (step.thought && step.action) {
+                examples.push({
+                    input: step.thought,
+                    output: step.action,
+                    stepIndex: index,
+                });
+            }
+        }
+        // Add final step as input-output pair
+        if (trajectory.steps.length > 0 && trajectory.outcome.success) {
+            examples.push({
+                input: trajectory.task.description,
+                output: JSON.stringify(trajectory.outcome.solution),
+                stepIndex: trajectory.steps.length - 1,
+            });
+        }
+        return examples;
+    }
+    /**
+     * Set credit assignment strategy
+     */
+    setStrategy(strategy) {
+        this.creditFn = getCreditAssignmentFn(strategy);
+    }
+    /**
+     * Set attribution threshold
+     */
+    setThreshold(threshold) {
+        this.config.attributionThreshold = threshold;
+    }
+    /**
+     * Update configuration
+     */
+    updateConfig(config) {
+        this.config = { ...this.config, ...config };
+    }
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+}
+/**
+ * Create a trajectory analyzer
+ */
+export function createAnalyzer(strategy = 'simple', config) {
+    return new TrajectoryAnalyzer(strategy, config);
+}
+//# sourceMappingURL=analyzer.js.map

package/dist/learning/analyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzer.js","sourceRoot":"","sources":["../../src/learning/analyzer.ts"],"names":[],"mappings":"AAmDA,MAAM,uBAAuB,GAAmB;IAC9C,iBAAiB,EAAE,GAAG;IACtB,yBAAyB,EAAE,CAAC;IAC5B,oBAAoB,EAAE,IAAI;CAC3B,CAAC;AAUF;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,UAAsB,EACtB,MAAsB;IAEtB,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC;IAClC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CACrC,UAAsB,EACtB,OAAuB;IAEvB,MAAM,UAAU,GAAa,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CACrD,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CACvC,CAAC;IAEF,8CAA8C;IAC9C,IAAI,UAAU,CAAC,OAAO,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxD,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;IAC1C,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,QAAwB;IAExB,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO,sBAAsB,CAAC;QAChC,KAAK,KAAK;YACR,0DAA0D;YAC1D,8BAA8B;YAC9B,OAAO,sBAAsB,CAAC;QAChC,KAAK,gBAAgB;YACnB,6CAA6C;YAC7C,6BAA6B;YAC7B,OAAO,uBAAuB,CAAC;QACjC;YACE,OAAO,sBAAsB,CAAC;IAClC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IACrB,QAAQ,CAAqB;IAC7B,MAAM,CAAiB;IAE/B,YAAY,WAA2B,QAAQ,EAAE,MAAgC;QAC/E,IAAI,CAAC,QAAQ,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,uBAAuB,EAAE,GAAG,MAAM,EAAE,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAAsB;QAClC,yBAAyB;QACzB,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAErE,6BAA6B;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,qBAAqB;QACrB,MAAM,QAAQ,GAAG,eAAe;aAC7B,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;aACzC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;aAChE,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QAE7B,wBAAwB;QACxB,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAE3D,yBAAyB;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAC7C,UAAU,EACV,QAAQ,EACR,aAAa,CACd,CAAC;QAEF,4BAA4B;QAC5B,MAAM,gBAAgB,GAAG,IAAI,CAAC,uBAAuB,CACnD,UAAU,EACV,QAAQ,CACT,CAAC;QAEF,OAAO;YACL,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,OAAO;YACnC,QAAQ;YACR,eAAe;YACf,aAAa;YACb,YAAY;YACZ,gBAAgB;SACjB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,UAAsB;QAChD,MAAM,QAAQ,GAAmB,EAAE,CAAC;QAEpC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;YAChE,qCAAqC;YACrC,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC;YAE/C,wBAAwB;YACxB,MAAM,UAAU,GAAoC;gBAClD,CAAC,UAAU,EAAE,SAAS,EAAE,qBAAqB,CAAC;gBAC9C,CAAC,YAAY,EAAE,WAAW,EAAE,oBAAoB,CAAC;gBACjD,CAAC,oBAAoB,EAAE,YAAY,EAAE,mBAAmB,CAAC;gBACzD,CAAC,eAAe,EAAE,QAAQ,EAAE,sBAAsB,CAAC;gBACnD,CAAC,aAAa,EAAE,MAAM,EAAE,qBAAqB,CAAC;gBAC9C,CAAC,mBAAmB,EAAE,WAAW,EAAE,kBAAkB,CAAC;gBACtD,CAAC,kBAAkB,EAAE,SAAS,EAAE,eAAe,CAAC;aACjD,CAAC;YAEF,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC;gBAC/C,IAAI,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC5B,QAAQ,CAAC,IAAI,CAAC;wBACZ,IAAI;wBACJ,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;wBAClC,WAAW,EAAE,IAAI;qBAClB,CAAC,CAAC;oBACH,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,oCAAoC;QACpC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACvD,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;QAED,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,YAAY,EAAE,CAAC;YAC3C,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,yBAAyB,EAAE,CAAC;gBACnD,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,YAAY;oBAClB,SAAS,EAAE,MAAM;oBACjB,WAAW,EAAE,mBAAmB,KAAK,QAAQ;oBAC7C,YAAY,EAAE,0BAA0B;iBACzC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,qBAAqB,CAC3B,UAAsB,EACtB,QAAkB,EAClB,aAA6B;QAE7B,qDAAqD;QACrD,IAAI,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/B,oCAAoC;YACpC,OAAO,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,gEAAgE;QAChE,8BAA8B;QAC9B,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,CACL,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;gBACzC,UAAU,CAAC,OAAO,CAAC,YAAY,IAAI,GAAG,CACvC,CAAC;QACJ,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,uBAAuB,CAC7B,UAAsB,EACtB,QAAkB;QAElB,MAAM,QAAQ,GAAsB,EAAE,CAAC;QAEvC,yBAAyB;QACzB,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,QAAQ,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,IAAI,CAAC,OAAO;oBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,SAAS,EAAE,KAAK;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC9D,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,WAAW;gBAClC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC;gBACnD,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAwB;QAClC,IAAI,CAAC,QAAQ,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,SAAiB;QAC5B,IAAI,CAAC,MAAM,CAAC,oBAAoB,GAAG,SAAS,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,MAA+B;QAC1C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,WAA2B,QAAQ,EACnC,MAAgC;IAEhC,OAAO,IAAI,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC"}

package/dist/learning/effectiveness.d.ts

@@ -0,0 +1,158 @@
+/**
+ * Learning Effectiveness Tracker
+ *
+ * Measures whether the learning system is actually improving agent outcomes.
+ * Uses post-task reflection annotations rather than A/B replay evaluation.
+ *
+ * After each trajectory completes, the tracker annotates it with:
+ * - What knowledge was surfaced (playbooks, experiences)
+ * - Whether the surfaced knowledge was applied (inferred from trajectory)
+ * - Whether the outcome improved relative to similar unaided tasks
+ *
+ * These annotations accumulate into aggregate metrics that answer:
+ * "Is learning making agents better over time?"
+ *
+ * Inspired by Dash's eval framework but adapted for domain-agnostic
+ * trajectory learning rather than text-to-SQL.
+ */
+import type { Trajectory } from '../types/index.js';
+import type { PlaybookMatch } from '../memory/playbook.js';
+/**
+ * Annotation attached to a trajectory after completion.
+ * Records what knowledge was available and whether it helped.
+ */
+export interface TaskAnnotation {
+    id: string;
+    trajectoryId: string;
+    timestamp: Date;
+    /** What was surfaced before/during execution */
+    knowledgeSurfaced: {
+        playbookIds: string[];
+        playbookNames: string[];
+        experienceIds: string[];
+        /** Total number of knowledge items retrieved */
+        totalItems: number;
+    };
+    /** What was actually applied (inferred from trajectory analysis) */
+    knowledgeApplied: {
+        /** Playbook IDs whose tactics appeared in the trajectory steps */
+        playbookIdsUsed: string[];
+        /** Whether any surfaced knowledge was used */
+        anyKnowledgeUsed: boolean;
+    };
+    /** Outcome metrics for this task */
+    outcome: {
+        success: boolean;
+        stepCount: number;
+        /** Number of error-recovery cycles in the trajectory */
+        errorRecoveries: number;
+        /** Whether the task domain had prior playbooks */
+        domainHadPlaybooks: boolean;
+    };
+    /** Optional human or agent reflection after task completion */
+    reflection?: {
+        /** Was the surfaced knowledge relevant? */
+        knowledgeRelevance: 'helpful' | 'irrelevant' | 'misleading' | 'not_assessed';
+        /** Free-form notes on what worked or didn't */
+        notes?: string;
+    };
+}
+/**
+ * Aggregate effectiveness metrics computed from annotations
+ */
+export interface EffectivenessMetrics {
+    /** Total tasks tracked */
+    totalTasks: number;
+    /** Tasks where knowledge was surfaced vs not */
+    guidedTasks: number;
+    unguidedTasks: number;
+    /** Success rates */
+    guidedSuccessRate: number;
+    unguidedSuccessRate: number;
+    /** Difference: positive means guidance helps */
+    successRateDelta: number;
+    /** Average step counts */
+    guidedAvgSteps: number;
+    unguidedAvgSteps: number;
+    /** Negative means guidance reduces steps (good) */
+    stepCountDelta: number;
+    /** Knowledge application rates */
+    knowledgeApplicationRate: number;
+    /** Of applied knowledge, how often did the task succeed? */
+    appliedKnowledgeSuccessRate: number;
+    /** Error recovery */
+    guidedAvgErrorRecoveries: number;
+    unguidedAvgErrorRecoveries: number;
+    /** Per-playbook effectiveness */
+    playbookEffectiveness: PlaybookEffectivenessEntry[];
+    /** Time window these metrics cover */
+    windowStart: Date;
+    windowEnd: Date;
+}
+/**
+ * Effectiveness metrics for a single playbook
+ */
+export interface PlaybookEffectivenessEntry {
+    playbookId: string;
+    playbookName: string;
+    /** Times this playbook was surfaced */
+    surfacedCount: number;
+    /** Times this playbook was actually applied */
+    appliedCount: number;
+    /** Success rate when applied */
+    appliedSuccessRate: number;
+    /** Average step count when applied */
+    appliedAvgSteps: number;
+    /** How often agents found this knowledge relevant */
+    relevanceRate: number;
+}
+/**
+ * Tracks learning effectiveness through post-task annotations
+ */
+export declare class LearningEffectivenessTracker {
+    private store;
+    private initialized;
+    constructor(baseDir: string);
+    init(): Promise<void>;
+    /**
+     * Annotate a completed trajectory with knowledge usage data.
+     * Call this after a task finishes and usage inference has run.
+     */
+    annotate(trajectory: Trajectory, surfacedPlaybooks: PlaybookMatch[], surfacedExperienceIds: string[], appliedPlaybookIds: string[]): Promise<TaskAnnotation>;
+    /**
+     * Add a reflection to an existing annotation.
+     * Called after human review or agent self-reflection.
+     */
+    addReflection(annotationId: string, reflection: TaskAnnotation['reflection']): Promise<void>;
+    /**
+     * Compute aggregate effectiveness metrics over a time window.
+     * Defaults to all time if no window specified.
+     */
+    computeMetrics(options?: {
+        since?: Date;
+        until?: Date;
+        domain?: string;
+    }): Promise<EffectivenessMetrics>;
+    /**
+     * Get all annotations (for export/inspection)
+     */
+    getAll(): Promise<TaskAnnotation[]>;
+    /**
+     * Get annotation by trajectory ID
+     */
+    getByTrajectoryId(trajectoryId: string): Promise<TaskAnnotation | undefined>;
+    /**
+     * Get annotation count
+     */
+    count(): Promise<number>;
+    /**
+     * Close the store
+     */
+    close(): Promise<void>;
+    private computePlaybookEffectiveness;
+}
+/**
+ * Create a learning effectiveness tracker
+ */
+export declare function createEffectivenessTracker(baseDir: string): LearningEffectivenessTracker;
+//# sourceMappingURL=effectiveness.d.ts.map

package/dist/learning/effectiveness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"effectiveness.d.ts","sourceRoot":"","sources":["../../src/learning/effectiveness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAK3D;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,IAAI,CAAC;IAEhB,gDAAgD;IAChD,iBAAiB,EAAE;QACjB,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,gDAAgD;QAChD,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,oEAAoE;IACpE,gBAAgB,EAAE;QAChB,kEAAkE;QAClE,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,8CAA8C;QAC9C,gBAAgB,EAAE,OAAO,CAAC;KAC3B,CAAC;IAEF,oCAAoC;IACpC,OAAO,EAAE;QACP,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,wDAAwD;QACxD,eAAe,EAAE,MAAM,CAAC;QACxB,kDAAkD;QAClD,kBAAkB,EAAE,OAAO,CAAC;KAC7B,CAAC;IAEF,+DAA+D;IAC/D,UAAU,CAAC,EAAE;QACX,2CAA2C;QAC3C,kBAAkB,EAAE,SAAS,GAAG,YAAY,GAAG,YAAY,GAAG,cAAc,CAAC;QAC7E,+CAA+C;QAC/C,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,0BAA0B;IAC1B,UAAU,EAAE,MAAM,CAAC;IAEnB,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IAEtB,oBAAoB;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gDAAgD;IAChD,gBAAgB,EAAE,MAAM,CAAC;IAEzB,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,cAAc,EAAE,MAAM,CAAC;IAEvB,kCAAkC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,4DAA4D;IAC5D,2BAA2B,EAAE,MAAM,CAAC;IAEpC,qBAAqB;IACrB,wBAAwB,EAAE,MAAM,CAAC;IACjC,0BAA0B,EAAE,MAAM,CAAC;IAEnC,iCAAiC;IACjC,qBAAqB,EAAE,0BAA0B,EAAE,CAAC;IAEpD,sCAAsC;IACtC,WAAW,EAAE,IAAI,CAAC;IAClB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,gCAAgC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,sCAAsC;IACtC,eAAe,EAAE,MAAM,CAAC;IACxB,qDAAqD;IACrD,aAAa,EAAE,MAAM,CAAC;CACvB;AAID;;GAEG;AACH,qBAAa,4BAA4B;IACvC,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,WAAW,CAAS;gBAEhB,OAAO,EAAE,MAAM;IAOrB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;OAGG;IACG,QAAQ,CACZ,UAAU,EAAE,UAAU,EACtB,iBAAiB,EAAE,aAAa,EAAE,EAClC,qBAAqB,EAAE,MAAM,EAAE,EAC/B,kBAAkB,EAAE,MAAM,EAAE,GAC3B,OAAO,CAAC,cAAc,CAAC;IAiC1B;;;OAGG;IACG,aAAa,CACjB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,cAAc,CAAC,YAAY,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAWhB;;;OAGG;IACG,cAAc,CAAC,OAAO,CAAC,EAAE;QAC7B,KAAK,CAAC,EAAE,IAAI,CAAC;QACb,KAAK,CAAC,EAAE,IAAI,CAAC;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IA+EjC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAKzC;;OAEG;IACG,iBAAiB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAKlF;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAK9B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B,OAAO,CAAC,4BAA4B;CAoErC;AAuBD;;GAEG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,MAAM,GACd,4BAA4B,CAE9B"}