cognitive-core 0.0.1 → 0.1.0

package/dist/learning/effectiveness.js

@@ -0,0 +1,251 @@
+/**
+ * Learning Effectiveness Tracker
+ *
+ * Measures whether the learning system is actually improving agent outcomes.
+ * Uses post-task reflection annotations rather than A/B replay evaluation.
+ *
+ * After each trajectory completes, the tracker annotates it with:
+ * - What knowledge was surfaced (playbooks, experiences)
+ * - Whether the surfaced knowledge was applied (inferred from trajectory)
+ * - Whether the outcome improved relative to similar unaided tasks
+ *
+ * These annotations accumulate into aggregate metrics that answer:
+ * "Is learning making agents better over time?"
+ *
+ * Inspired by Dash's eval framework but adapted for domain-agnostic
+ * trajectory learning rather than text-to-SQL.
+ */
+import { JsonStore } from '../utils/storage.js';
+// === TRACKER ===
+/**
+ * Tracks learning effectiveness through post-task annotations
+ */
+export class LearningEffectivenessTracker {
+    store;
+    initialized = false;
+    constructor(baseDir) {
+        this.store = new JsonStore(baseDir, 'effectiveness', {
+            autoSaveInterval: 30000,
+            pretty: true,
+        });
+    }
+    async init() {
+        if (this.initialized)
+            return;
+        await this.store.init();
+        this.initialized = true;
+    }
+    /**
+     * Annotate a completed trajectory with knowledge usage data.
+     * Call this after a task finishes and usage inference has run.
+     */
+    async annotate(trajectory, surfacedPlaybooks, surfacedExperienceIds, appliedPlaybookIds) {
+        await this.init();
+        const errorRecoveries = countErrorRecoveries(trajectory);
+        const annotation = {
+            id: `ann-${trajectory.id}`,
+            trajectoryId: trajectory.id,
+            timestamp: new Date(),
+            knowledgeSurfaced: {
+                playbookIds: surfacedPlaybooks.map((m) => m.playbook.id),
+                playbookNames: surfacedPlaybooks.map((m) => m.playbook.name),
+                experienceIds: surfacedExperienceIds,
+                totalItems: surfacedPlaybooks.length + surfacedExperienceIds.length,
+            },
+            knowledgeApplied: {
+                playbookIdsUsed: appliedPlaybookIds,
+                anyKnowledgeUsed: appliedPlaybookIds.length > 0,
+            },
+            outcome: {
+                success: trajectory.outcome.success,
+                stepCount: trajectory.steps.length,
+                errorRecoveries,
+                domainHadPlaybooks: surfacedPlaybooks.length > 0,
+            },
+        };
+        this.store.set(annotation.id, annotation);
+        await this.store.save(annotation.id);
+        return annotation;
+    }
+    /**
+     * Add a reflection to an existing annotation.
+     * Called after human review or agent self-reflection.
+     */
+    async addReflection(annotationId, reflection) {
+        await this.init();
+        const annotation = this.store.get(annotationId);
+        if (!annotation)
+            return;
+        annotation.reflection = reflection;
+        this.store.set(annotationId, annotation);
+        await this.store.save(annotationId);
+    }
+    /**
+     * Compute aggregate effectiveness metrics over a time window.
+     * Defaults to all time if no window specified.
+     */
+    async computeMetrics(options) {
+        await this.init();
+        let annotations = this.store.values();
+        // Filter by time window
+        if (options?.since) {
+            const since = options.since.getTime();
+            annotations = annotations.filter((a) => new Date(a.timestamp).getTime() >= since);
+        }
+        if (options?.until) {
+            const until = options.until.getTime();
+            annotations = annotations.filter((a) => new Date(a.timestamp).getTime() <= until);
+        }
+        const guided = annotations.filter((a) => a.knowledgeSurfaced.totalItems > 0);
+        const unguided = annotations.filter((a) => a.knowledgeSurfaced.totalItems === 0);
+        const guidedSuccesses = guided.filter((a) => a.outcome.success);
+        const unguidedSuccesses = unguided.filter((a) => a.outcome.success);
+        const guidedSuccessRate = guided.length > 0
+            ? guidedSuccesses.length / guided.length
+            : 0;
+        const unguidedSuccessRate = unguided.length > 0
+            ? unguidedSuccesses.length / unguided.length
+            : 0;
+        const guidedAvgSteps = guided.length > 0
+            ? guided.reduce((sum, a) => sum + a.outcome.stepCount, 0) / guided.length
+            : 0;
+        const unguidedAvgSteps = unguided.length > 0
+            ? unguided.reduce((sum, a) => sum + a.outcome.stepCount, 0) / unguided.length
+            : 0;
+        const applied = annotations.filter((a) => a.knowledgeApplied.anyKnowledgeUsed);
+        const appliedSuccesses = applied.filter((a) => a.outcome.success);
+        const knowledgeApplicationRate = guided.length > 0
+            ? applied.length / guided.length
+            : 0;
+        const appliedKnowledgeSuccessRate = applied.length > 0
+            ? appliedSuccesses.length / applied.length
+            : 0;
+        const guidedAvgErrors = guided.length > 0
+            ? guided.reduce((sum, a) => sum + a.outcome.errorRecoveries, 0) / guided.length
+            : 0;
+        const unguidedAvgErrors = unguided.length > 0
+            ? unguided.reduce((sum, a) => sum + a.outcome.errorRecoveries, 0) / unguided.length
+            : 0;
+        // Per-playbook effectiveness
+        const playbookEffectiveness = this.computePlaybookEffectiveness(annotations);
+        return {
+            totalTasks: annotations.length,
+            guidedTasks: guided.length,
+            unguidedTasks: unguided.length,
+            guidedSuccessRate,
+            unguidedSuccessRate,
+            successRateDelta: guidedSuccessRate - unguidedSuccessRate,
+            guidedAvgSteps,
+            unguidedAvgSteps,
+            stepCountDelta: guidedAvgSteps - unguidedAvgSteps,
+            knowledgeApplicationRate,
+            appliedKnowledgeSuccessRate,
+            guidedAvgErrorRecoveries: guidedAvgErrors,
+            unguidedAvgErrorRecoveries: unguidedAvgErrors,
+            playbookEffectiveness,
+            windowStart: options?.since ?? new Date(0),
+            windowEnd: options?.until ?? new Date(),
+        };
+    }
+    /**
+     * Get all annotations (for export/inspection)
+     */
+    async getAll() {
+        await this.init();
+        return this.store.values();
+    }
+    /**
+     * Get annotation by trajectory ID
+     */
+    async getByTrajectoryId(trajectoryId) {
+        await this.init();
+        return this.store.values().find((a) => a.trajectoryId === trajectoryId);
+    }
+    /**
+     * Get annotation count
+     */
+    async count() {
+        await this.init();
+        return this.store.size();
+    }
+    /**
+     * Close the store
+     */
+    async close() {
+        await this.store.close();
+    }
+    // === PRIVATE ===
+    computePlaybookEffectiveness(annotations) {
+        const playbookStats = new Map();
+        for (const ann of annotations) {
+            // Track surfaced
+            for (let i = 0; i < ann.knowledgeSurfaced.playbookIds.length; i++) {
+                const id = ann.knowledgeSurfaced.playbookIds[i];
+                const name = ann.knowledgeSurfaced.playbookNames[i] ?? id;
+                if (!playbookStats.has(id)) {
+                    playbookStats.set(id, {
+                        name,
+                        surfaced: 0,
+                        applied: 0,
+                        appliedSuccesses: 0,
+                        appliedStepSum: 0,
+                        relevantCount: 0,
+                        assessedCount: 0,
+                    });
+                }
+                const stats = playbookStats.get(id);
+                stats.surfaced++;
+                const wasApplied = ann.knowledgeApplied.playbookIdsUsed.includes(id);
+                if (wasApplied) {
+                    stats.applied++;
+                    if (ann.outcome.success)
+                        stats.appliedSuccesses++;
+                    stats.appliedStepSum += ann.outcome.stepCount;
+                }
+                if (ann.reflection) {
+                    stats.assessedCount++;
+                    if (ann.reflection.knowledgeRelevance === 'helpful') {
+                        stats.relevantCount++;
+                    }
+                }
+            }
+        }
+        return Array.from(playbookStats.entries())
+            .map(([id, stats]) => ({
+            playbookId: id,
+            playbookName: stats.name,
+            surfacedCount: stats.surfaced,
+            appliedCount: stats.applied,
+            appliedSuccessRate: stats.applied > 0 ? stats.appliedSuccesses / stats.applied : 0,
+            appliedAvgSteps: stats.applied > 0 ? stats.appliedStepSum / stats.applied : 0,
+            relevanceRate: stats.assessedCount > 0 ? stats.relevantCount / stats.assessedCount : 0,
+        }))
+            .sort((a, b) => b.surfacedCount - a.surfacedCount);
+    }
+}
+// === HELPERS ===
+/**
+ * Count error-recovery cycles in a trajectory.
+ * An error recovery is when a step has an error observation
+ * followed by a subsequent step that doesn't.
+ */
+function countErrorRecoveries(trajectory) {
+    let recoveries = 0;
+    for (let i = 0; i < trajectory.steps.length - 1; i++) {
+        const current = trajectory.steps[i];
+        const next = trajectory.steps[i + 1];
+        const hasError = current.observation?.toLowerCase().includes('error');
+        const nextOk = !next.observation?.toLowerCase().includes('error');
+        if (hasError && nextOk) {
+            recoveries++;
+        }
+    }
+    return recoveries;
+}
+/**
+ * Create a learning effectiveness tracker
+ */
+export function createEffectivenessTracker(baseDir) {
+    return new LearningEffectivenessTracker(baseDir);
+}
+//# sourceMappingURL=effectiveness.js.map

package/dist/learning/effectiveness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"effectiveness.js","sourceRoot":"","sources":["../../src/learning/effectiveness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AA2GhD,kBAAkB;AAElB;;GAEG;AACH,MAAM,OAAO,4BAA4B;IAC/B,KAAK,CAA4B;IACjC,WAAW,GAAG,KAAK,CAAC;IAE5B,YAAY,OAAe;QACzB,IAAI,CAAC,KAAK,GAAG,IAAI,SAAS,CAAiB,OAAO,EAAE,eAAe,EAAE;YACnE,gBAAgB,EAAE,KAAK;YACvB,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CACZ,UAAsB,EACtB,iBAAkC,EAClC,qBAA+B,EAC/B,kBAA4B;QAE5B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAElB,MAAM,eAAe,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;QAEzD,MAAM,UAAU,GAAmB;YACjC,EAAE,EAAE,OAAO,UAAU,CAAC,EAAE,EAAE;YAC1B,YAAY,EAAE,UAAU,CAAC,EAAE;YAC3B,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,iBAAiB,EAAE;gBACjB,WAAW,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxD,aAAa,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC5D,aAAa,EAAE,qBAAqB;gBACpC,UAAU,EAAE,iBAAiB,CAAC,MAAM,GAAG,qBAAqB,CAAC,MAAM;aACpE;YACD,gBAAgB,EAAE;gBAChB,eAAe,EAAE,kBAAkB;gBACnC,gBAAgB,EAAE,kBAAkB,CAAC,MAAM,GAAG,CAAC;aAChD;YACD,OAAO,EAAE;gBACP,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,OAAO;gBACnC,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM;gBAClC,eAAe;gBACf,kBAAkB,EAAE,iBAAiB,CAAC,MAAM,GAAG,CAAC;aACjD;SACF,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAErC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa,CACjB,YAAoB,EACpB,UAAwC;QAExC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAElB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,UAAU,CAAC,UAAU,GAAG,UAAU,CAAC;QACnC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,OAIpB;QACC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAElB,IAAI,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAEtC,wBAAwB;QACxB,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACtC,WAAW,GAAG,WAAW,CAAC,MAAM,CAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,IAAI,KAAK,CAChD,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACtC,WAAW,GAAG,WAAW,CAAC,MAAM,CAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,IAAI,KAAK,CAChD,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC;QAEjF,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChE,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;YACzC,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;YACxC,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,mBAAmB,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC;YAC7C,CAAC,CAAC,iBAAiB,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM;YAC5C,CAAC,CAAC,CAAC,CAAC;QAEN,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;YACtC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;YACzE,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC;YAC1C,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM;YAC7E,CAAC,CAAC,CAAC,CAAC;QAEN,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;QAC/E,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAElE,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;YAChD,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;YAChC,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC;YACpD,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;YAC1C,CAAC,CAAC,CAAC,CAAC;QAEN,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;YACvC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;YAC/E,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC;YAC3C,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM;YACnF,CAAC,CAAC,CAAC,CAAC;QAEN,6BAA6B;QAC7B,MAAM,qBAAqB,GAAG,IAAI,CAAC,4BAA4B,CAAC,WAAW,CAAC,CAAC;QAE7E,OAAO;YACL,UAAU,EAAE,WAAW,CAAC,MAAM;YAC9B,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,iBAAiB;YACjB,mBAAmB;YACnB,gBAAgB,EAAE,iBAAiB,GAAG,mBAAmB;YACzD,cAAc;YACd,gBAAgB;YAChB,cAAc,EAAE,cAAc,GAAG,gBAAgB;YACjD,wBAAwB;YACxB,2BAA2B;YAC3B,wBAAwB,EAAE,eAAe;YACzC,0BAA0B,EAAE,iBAAiB;YAC7C,qBAAqB;YACrB,WAAW,EAAE,OAAO,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC;YAC1C,SAAS,EAAE,OAAO,EAAE,KAAK,IAAI,IAAI,IAAI,EAAE;SACxC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,YAAoB;QAC1C,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,YAAY,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED,kBAAkB;IAEV,4BAA4B,CAClC,WAA6B;QAE7B,MAAM,aAAa,GAAG,IAAI,GAAG,EAW1B,CAAC;QAEJ,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,iBAAiB;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,iBAAiB,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClE,MAAM,EAAE,GAAG,GAAG,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;gBAChD,MAAM,IAAI,GAAG,GAAG,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE1D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC3B,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;wBACpB,IAAI;wBACJ,QAAQ,EAAE,CAAC;wBACX,OAAO,EAAE,CAAC;wBACV,gBAAgB,EAAE,CAAC;wBACnB,cAAc,EAAE,CAAC;wBACjB,aAAa,EAAE,CAAC;wBAChB,aAAa,EAAE,CAAC;qBACjB,CAAC,CAAC;gBACL,CAAC;gBAED,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;gBACrC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAEjB,MAAM,UAAU,GAAG,GAAG,CAAC,gBAAgB,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrE,IAAI,UAAU,EAAE,CAAC;oBACf,KAAK,CAAC,OAAO,EAAE,CAAC;oBAChB,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO;wBAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;oBAClD,KAAK,CAAC,cAAc,IAAI,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC;gBAChD,CAAC;gBAED,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;oBACnB,KAAK,CAAC,aAAa,EAAE,CAAC;oBACtB,IAAI,GAAG,CAAC,UAAU,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;wBACpD,KAAK,CAAC,aAAa,EAAE,CAAC;oBACxB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;aACvC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;YACrB,UAAU,EAAE,EAAE;YACd,YAAY,EAAE,KAAK,CAAC,IAAI;YACxB,aAAa,EAAE,KAAK,CAAC,QAAQ;YAC7B,YAAY,EAAE,KAAK,CAAC,OAAO;YAC3B,kBAAkB,EAChB,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChE,eAAe,EACb,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC9D,aAAa,EACX,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SAC1E,CAAC,CAAC;aACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;IACvD,CAAC;CACF;AAED,kBAAkB;AAElB;;;;GAIG;AACH,SAAS,oBAAoB,CAAC,UAAsB;IAClD,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;YACvB,UAAU,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CACxC,OAAe;IAEf,OAAO,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAAC;AACnD,CAAC"}

package/dist/learning/index.d.ts

@@ -0,0 +1,8 @@
+export { TrajectoryAnalyzer, createAnalyzer, simpleCreditAssignment, outcomeCreditAssignment, getCreditAssignmentFn, type AnalysisResult, type ErrorPattern, type TrainingExample, type CreditAssignmentFn, type AnalyzerConfig, } from './analyzer.js';
+export { LearningPipeline, createLearningPipeline, type ProcessResult, type BatchResult, } from './pipeline.js';
+export { PlaybookExtractor, createPlaybookExtractor, type ExtractedPlaybooks, type PlaybookUpdate, type PlaybookExtractorConfig, } from './playbook-extractor.js';
+export { MetaLearner, createMetaLearner, type MetaLearnerConfig, } from './meta-learner.js';
+export { PlaybookUsageInference, createUsageInference, type PlaybookUsageResult, type UsageInferenceConfig, } from './usage-inference.js';
+export { LLMPlaybookExtractor, createLLMExtractor, type LLMExtractorConfig, type LLMExtractionResult, } from './llm-extractor.js';
+export { LearningEffectivenessTracker, createEffectivenessTracker, type TaskAnnotation, type EffectivenessMetrics, type PlaybookEffectivenessEntry, } from './effectiveness.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/learning/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/learning/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,cAAc,EACd,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,EACrB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,cAAc,GACpB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,KAAK,aAAa,EAClB,KAAK,WAAW,GACjB,MAAM,eAAe,CAAC;AAGvB,OAAO,EACL,iBAAiB,EACjB,uBAAuB,EACvB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,uBAAuB,GAC7B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,KAAK,iBAAiB,GACvB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,GAC1B,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,GACzB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EACL,4BAA4B,EAC5B,0BAA0B,EAC1B,KAAK,cAAc,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,GAChC,MAAM,oBAAoB,CAAC"}

package/dist/learning/index.js

@@ -0,0 +1,11 @@
+export { TrajectoryAnalyzer, createAnalyzer, simpleCreditAssignment, outcomeCreditAssignment, getCreditAssignmentFn, } from './analyzer.js';
+export { LearningPipeline, createLearningPipeline, } from './pipeline.js';
+// Playbook-based learning
+export { PlaybookExtractor, createPlaybookExtractor, } from './playbook-extractor.js';
+export { MetaLearner, createMetaLearner, } from './meta-learner.js';
+export { PlaybookUsageInference, createUsageInference, } from './usage-inference.js';
+// LLM-based extraction
+export { LLMPlaybookExtractor, createLLMExtractor, } from './llm-extractor.js';
+// Effectiveness tracking
+export { LearningEffectivenessTracker, createEffectivenessTracker, } from './effectiveness.js';
+//# sourceMappingURL=index.js.map

package/dist/learning/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/learning/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,cAAc,EACd,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GAMtB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,gBAAgB,EAChB,sBAAsB,GAGvB,MAAM,eAAe,CAAC;AAEvB,0BAA0B;AAC1B,OAAO,EACL,iBAAiB,EACjB,uBAAuB,GAIxB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,WAAW,EACX,iBAAiB,GAElB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,GAGrB,MAAM,sBAAsB,CAAC;AAE9B,uBAAuB;AACvB,OAAO,EACL,oBAAoB,EACpB,kBAAkB,GAGnB,MAAM,oBAAoB,CAAC;AAE5B,yBAAyB;AACzB,OAAO,EACL,4BAA4B,EAC5B,0BAA0B,GAI3B,MAAM,oBAAoB,CAAC"}

package/dist/learning/llm-extractor.d.ts

@@ -0,0 +1,88 @@
+/**
+ * LLM-Based Playbook Extractor
+ *
+ * Uses language models to generate higher-quality playbooks
+ * from trajectories. Complements the pattern-based extractor
+ * with deeper semantic analysis.
+ */
+import type { Playbook, Trajectory, Refinement } from '../types/index.js';
+import type { AnalysisResult } from './analyzer.js';
+import type { LLMProvider } from '../agents/types.js';
+/**
+ * Configuration for LLM-based extraction
+ */
+export interface LLMExtractorConfig {
+    /** Temperature for LLM calls (default: 0.2 for determinism) */
+    temperature: number;
+    /** Maximum tokens for response */
+    maxTokens: number;
+    /** Whether to extract from single trajectories (vs only batches) */
+    extractFromSingle: boolean;
+    /** Minimum complexity for LLM extraction */
+    minComplexity: number;
+}
+/**
+ * Result from LLM extraction
+ */
+export interface LLMExtractionResult {
+    playbooks: Playbook[];
+    refinements: Array<{
+        playbookId: string;
+        refinement: Refinement;
+    }>;
+    /** Raw LLM response for debugging */
+    rawResponse?: string;
+}
+/**
+ * LLM-Based Playbook Extractor
+ */
+export declare class LLMPlaybookExtractor {
+    private provider;
+    private config;
+    constructor(provider: LLMProvider, config?: Partial<LLMExtractorConfig>);
+    /**
+     * Extract playbooks from trajectories using LLM
+     */
+    extract(trajectories: Trajectory[], analyses: AnalysisResult[], existingPlaybooks?: Playbook[]): Promise<LLMExtractionResult>;
+    /**
+     * Extract a playbook from a single trajectory
+     */
+    private extractFromSingleTrajectory;
+    /**
+     * Extract playbooks from a batch of trajectories
+     */
+    private extractFromBatch;
+    /**
+     * Build prompt for single trajectory extraction
+     */
+    private buildSingleExtractionPrompt;
+    /**
+     * Build prompt for batch extraction
+     */
+    private buildBatchExtractionPrompt;
+    /**
+     * Parse LLM response into extraction result
+     */
+    private parseExtractionResponse;
+    /**
+     * Create a Playbook from LLM-extracted data
+     */
+    private createPlaybookFromLLMData;
+    /**
+     * Parse complexity string
+     */
+    private parseComplexity;
+    /**
+     * Check if two playbooks are similar
+     */
+    private isSimilarPlaybook;
+    /**
+     * Generate a refinement for an existing playbook based on a new trajectory
+     */
+    generateRefinement(playbook: Playbook, trajectory: Trajectory, context: string): Promise<Refinement>;
+}
+/**
+ * Create an LLM-based playbook extractor
+ */
+export declare function createLLMExtractor(provider: LLMProvider, config?: Partial<LLMExtractorConfig>): LLMPlaybookExtractor;
+//# sourceMappingURL=llm-extractor.d.ts.map

package/dist/learning/llm-extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-extractor.d.ts","sourceRoot":"","sources":["../../src/learning/llm-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EACV,QAAQ,EACR,UAAU,EACV,UAAU,EACX,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,iBAAiB,EAAE,OAAO,CAAC;IAC3B,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;CACvB;AASD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,WAAW,EAAE,KAAK,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC,CAAC;IACH,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,MAAM,CAAqB;gBAEvB,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IAKvE;;OAEG;IACG,OAAO,CACX,YAAY,EAAE,UAAU,EAAE,EAC1B,QAAQ,EAAE,cAAc,EAAE,EAC1B,iBAAiB,GAAE,QAAQ,EAAO,GACjC,OAAO,CAAC,mBAAmB,CAAC;IAkE/B;;OAEG;YACW,2BAA2B;IAsBzC;;OAEG;YACW,gBAAgB;IAsB9B;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAwDlC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA+D/B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAsCjC;;OAEG;IACH,OAAO,CAAC,eAAe;IASvB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;OAEG;IACG,kBAAkB,CACtB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,UAAU,CAAC;CA0DvB;AAoBD;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,WAAW,EACrB,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,GACnC,oBAAoB,CAEtB"}