@cogitator-ai/core 0.1.0 → 0.3.0

package/dist/learning/instruction-optimizer.js

@@ -0,0 +1,175 @@
+import { buildFailureAnalysisPrompt, buildInstructionCandidatePrompt, buildInstructionEvaluationPrompt, buildInstructionRefinementPrompt, parseFailureAnalysisResponse, parseInstructionCandidatesResponse, parseInstructionEvaluationResponse, parseInstructionRefinementResponse, } from './prompts';
+export class InstructionOptimizer {
+    llm;
+    model;
+    traceStore;
+    insightStore;
+    candidateCount;
+    refinementRounds;
+    constructor(options) {
+        this.llm = options.llm;
+        this.model = options.model;
+        this.traceStore = options.traceStore;
+        this.insightStore = options.insightStore;
+        this.candidateCount = options.candidateCount ?? 3;
+        this.refinementRounds = options.refinementRounds ?? 1;
+    }
+    async optimize(agentId, currentInstructions, options) {
+        const traces = options?.traces ?? (await this.traceStore.getAll(agentId));
+        const maxTraces = options?.maxTraces ?? 20;
+        const relevantTraces = traces.slice(0, maxTraces);
+        if (relevantTraces.length === 0) {
+            return {
+                originalInstructions: currentInstructions,
+                optimizedInstructions: currentInstructions,
+                improvement: 0,
+                gapsAddressed: [],
+                candidatesEvaluated: 0,
+                reasoning: 'No traces available for optimization',
+            };
+        }
+        const gaps = await this.analyzeFailures(relevantTraces, currentInstructions);
+        if (gaps.length === 0) {
+            return {
+                originalInstructions: currentInstructions,
+                optimizedInstructions: currentInstructions,
+                improvement: 0,
+                gapsAddressed: [],
+                candidatesEvaluated: 0,
+                reasoning: 'No instruction gaps identified',
+            };
+        }
+        const insights = this.insightStore
+            ? await this.insightStore.findRelevant(agentId, currentInstructions, 10)
+            : [];
+        const candidates = await this.generateCandidates(currentInstructions, gaps, insights);
+        if (candidates.length === 0) {
+            return {
+                originalInstructions: currentInstructions,
+                optimizedInstructions: currentInstructions,
+                improvement: 0,
+                gapsAddressed: gaps,
+                candidatesEvaluated: 0,
+                reasoning: 'Failed to generate instruction candidates',
+            };
+        }
+        const evaluations = await this.evaluateCandidates(candidates, relevantTraces);
+        let bestCandidate = candidates[0];
+        let bestScore = 0;
+        let bestEvaluation = evaluations.get(candidates[0]) ?? { score: 0, weaknesses: [] };
+        for (const [candidate, evaluation] of evaluations) {
+            if (evaluation.score > bestScore) {
+                bestScore = evaluation.score;
+                bestCandidate = candidate;
+                bestEvaluation = evaluation;
+            }
+        }
+        let finalInstructions = bestCandidate;
+        for (let round = 0; round < this.refinementRounds; round++) {
+            if (bestEvaluation.weaknesses.length === 0)
+                break;
+            const refined = await this.refineInstructions(finalInstructions, bestEvaluation.weaknesses);
+            if (refined) {
+                finalInstructions = refined;
+            }
+        }
+        const originalScore = await this.estimateInstructionScore(currentInstructions, relevantTraces);
+        const newScore = await this.estimateInstructionScore(finalInstructions, relevantTraces);
+        return {
+            originalInstructions: currentInstructions,
+            optimizedInstructions: finalInstructions,
+            improvement: newScore - originalScore,
+            gapsAddressed: gaps,
+            candidatesEvaluated: candidates.length,
+            reasoning: `Identified ${gaps.length} gaps, evaluated ${candidates.length} candidates, best score: ${bestScore.toFixed(2)}`,
+        };
+    }
+    async analyzeFailures(traces, currentInstructions) {
+        const prompt = buildFailureAnalysisPrompt(traces, currentInstructions);
+        try {
+            const response = await this.llm.chat({
+                model: this.model,
+                messages: [{ role: 'user', content: prompt }],
+                temperature: 0.5,
+                maxTokens: 1000,
+            });
+            const parsed = parseFailureAnalysisResponse(response.content);
+            return parsed?.gaps ?? [];
+        }
+        catch {
+            return [];
+        }
+    }
+    async generateCandidates(currentInstructions, gaps, insights) {
+        const prompt = buildInstructionCandidatePrompt(currentInstructions, gaps, insights);
+        try {
+            const response = await this.llm.chat({
+                model: this.model,
+                messages: [{ role: 'user', content: prompt }],
+                temperature: 0.7,
+                maxTokens: 2000,
+            });
+            const parsed = parseInstructionCandidatesResponse(response.content);
+            return parsed
+                .map((c) => c.instructions)
+                .filter((i) => i.length > 0)
+                .slice(0, this.candidateCount);
+        }
+        catch {
+            return [];
+        }
+    }
+    async evaluateCandidates(candidates, traces) {
+        const evaluations = new Map();
+        for (const candidate of candidates) {
+            const prompt = buildInstructionEvaluationPrompt(candidate, traces);
+            try {
+                const response = await this.llm.chat({
+                    model: this.model,
+                    messages: [{ role: 'user', content: prompt }],
+                    temperature: 0.3,
+                    maxTokens: 500,
+                });
+                const parsed = parseInstructionEvaluationResponse(response.content);
+                if (parsed) {
+                    evaluations.set(candidate, {
+                        score: parsed.score,
+                        weaknesses: parsed.weaknesses,
+                    });
+                }
+                else {
+                    evaluations.set(candidate, { score: 0.5, weaknesses: [] });
+                }
+            }
+            catch {
+                evaluations.set(candidate, { score: 0.5, weaknesses: [] });
+            }
+        }
+        return evaluations;
+    }
+    async refineInstructions(candidate, weaknesses) {
+        const prompt = buildInstructionRefinementPrompt(candidate, weaknesses);
+        try {
+            const response = await this.llm.chat({
+                model: this.model,
+                messages: [{ role: 'user', content: prompt }],
+                temperature: 0.5,
+                maxTokens: 1500,
+            });
+            const parsed = parseInstructionRefinementResponse(response.content);
+            return parsed?.instructions ?? null;
+        }
+        catch {
+            return null;
+        }
+    }
+    async estimateInstructionScore(instructions, traces) {
+        const successfulTraces = traces.filter((t) => t.metrics.success);
+        const baseScore = traces.length > 0 ? successfulTraces.length / traces.length : 0.5;
+        const avgTraceScore = traces.length > 0 ? traces.reduce((sum, t) => sum + t.score, 0) / traces.length : 0.5;
+        const instructionLength = instructions.length;
+        const conciseBonus = instructionLength < 500 ? 0.1 : instructionLength > 1500 ? -0.1 : 0;
+        return Math.max(0, Math.min(1, baseScore * 0.4 + avgTraceScore * 0.5 + 0.5 + conciseBonus));
+    }
+}
+//# sourceMappingURL=instruction-optimizer.js.map

package/dist/learning/instruction-optimizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"instruction-optimizer.js","sourceRoot":"","sources":["../../src/learning/instruction-optimizer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,0BAA0B,EAC1B,+BAA+B,EAC/B,gCAAgC,EAChC,gCAAgC,EAChC,4BAA4B,EAC5B,kCAAkC,EAClC,kCAAkC,EAClC,kCAAkC,GACnC,MAAM,WAAW,CAAC;AAWnB,MAAM,OAAO,oBAAoB;IACvB,GAAG,CAAa;IAChB,KAAK,CAAS;IACd,UAAU,CAAa;IACvB,YAAY,CAAgB;IAC5B,cAAc,CAAS;IACvB,gBAAgB,CAAS;IAEjC,YAAY,OAAoC;QAC9C,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,OAAe,EACf,mBAA2B,EAC3B,OAIC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;QAC3C,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAElD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO;gBACL,oBAAoB,EAAE,mBAAmB;gBACzC,qBAAqB,EAAE,mBAAmB;gBAC1C,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,EAAE;gBACjB,mBAAmB,EAAE,CAAC;gBACtB,SAAS,EAAE,sCAAsC;aAClD,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;QAE7E,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO;gBACL,oBAAoB,EAAE,mBAAmB;gBACzC,qBAAqB,EAAE,mBAAmB;gBAC1C,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,EAAE;gBACjB,mBAAmB,EAAE,CAAC;gBACtB,SAAS,EAAE,gCAAgC;aAC5C,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY;YAChC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,OAAO,EAAE,mBAAmB,EAAE,EAAE,CAAC;YACxE,CAAC,CAAC,EAAE,CAAC;QAEP,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,mBAAmB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QAEtF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,oBAAoB,EAAE,mBAAmB;gBACzC,qBAAqB,EAAE,mBAAmB;gBAC1C,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,IAAI;gBACnB,mBAAmB,EAAE,CAAC;gBACtB,SAAS,EAAE,2CAA2C;aACvD,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAE9E,IAAI,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,cAAc,GAAG,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;QAEpF,KAAK,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,WAAW,EAAE,CAAC;YAClD,IAAI,UAAU,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;gBACjC,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC;gBAC7B,aAAa,GAAG,SAAS,CAAC;gBAC1B,cAAc,GAAG,UAAU,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,IAAI,iBAAiB,GAAG,aAAa,CAAC;QACtC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3D,IAAI,cAAc,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAElD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;YAE5F,IAAI,OAAO,EAAE,CAAC;gBACZ,iBAAiB,GAAG,OAAO,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;QAC/F,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;QAExF,OAAO;YACL,oBAAoB,EAAE,mBAAmB;YACzC,qBAAqB,EAAE,iBAAiB;YACxC,WAAW,EAAE,QAAQ,GAAG,aAAa;YACrC,aAAa,EAAE,IAAI;YACnB,mBAAmB,EAAE,UAAU,CAAC,MAAM;YACtC,SAAS,EAAE,cAAc,IAAI,CAAC,MAAM,oBAAoB,UAAU,CAAC,MAAM,4BAA4B,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;SAC5H,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,MAAwB,EACxB,mBAA2B;QAE3B,MAAM,MAAM,GAAG,0BAA0B,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;QAEvE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7C,WAAW,EAAE,GAAG;gBAChB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,4BAA4B,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC9D,OAAO,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,mBAA2B,EAC3B,IAAsB,EACtB,QAAmB;QAEnB,MAAM,MAAM,GAAG,+BAA+B,CAAC,mBAAmB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QAEpF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7C,WAAW,EAAE,GAAG;gBAChB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kCAAkC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpE,OAAO,MAAM;iBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;iBAC1B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;iBAC3B,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,UAAoB,EACpB,MAAwB;QAExB,MAAM,WAAW,GAAG,IAAI,GAAG,EAAmD,CAAC;QAE/E,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAEnE,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;oBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;oBAC7C,WAAW,EAAE,GAAG;oBAChB,SAAS,EAAE,GAAG;iBACf,CAAC,CAAC;gBAEH,MAAM,MAAM,GAAG,kCAAkC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpE,IAAI,MAAM,EAAE,CAAC;oBACX,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE;wBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;wBACnB,UAAU,EAAE,MAAM,CAAC,UAAU;qBAC9B,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAAiB,EACjB,UAAoB;QAEpB,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAEvE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7C,WAAW,EAAE,GAAG;gBAChB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kCAAkC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpE,OAAO,MAAM,EAAE,YAAY,IAAI,IAAI,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,YAAoB,EACpB,MAAwB;QAExB,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEjE,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;QAEpF,MAAM,aAAa,GACjB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;QAExF,MAAM,iBAAiB,GAAG,YAAY,CAAC,MAAM,CAAC;QAC9C,MAAM,YAAY,GAAG,iBAAiB,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEzF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,GAAG,GAAG,aAAa,GAAG,GAAG,GAAG,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC;IAC9F,CAAC;CACF"}

package/dist/learning/metrics.d.ts

@@ -0,0 +1,37 @@
+import type { ExecutionTrace, MetricResult, MetricFn, MetricEvaluatorConfig, LLMBackend } from '@cogitator-ai/types';
+export interface MetricEvaluatorOptions {
+    llm?: LLMBackend;
+    model?: string;
+    config?: Partial<MetricEvaluatorConfig>;
+}
+export declare class MetricEvaluator {
+    private llm?;
+    private model?;
+    private config;
+    private customMetrics;
+    constructor(options?: MetricEvaluatorOptions);
+    registerMetric(name: string, fn: MetricFn): void;
+    evaluate(trace: ExecutionTrace, expected?: unknown): Promise<{
+        results: MetricResult[];
+        score: number;
+        passed: boolean;
+    }>;
+    evaluateBatch(traces: ExecutionTrace[], expectedList?: unknown[]): Promise<Map<string, {
+        results: MetricResult[];
+        score: number;
+        passed: boolean;
+    }>>;
+    private evaluateMetric;
+    successMetric(trace: ExecutionTrace): MetricResult;
+    toolAccuracyMetric(trace: ExecutionTrace, expected?: unknown): MetricResult;
+    efficiencyMetric(trace: ExecutionTrace): MetricResult;
+    completenessMetric(trace: ExecutionTrace, expected?: unknown): Promise<MetricResult>;
+    coherenceMetric(trace: ExecutionTrace): Promise<MetricResult>;
+    private parseMetricResponse;
+    private aggregateScores;
+    getConfig(): MetricEvaluatorConfig;
+}
+export declare function createSuccessMetric(): MetricFn;
+export declare function createExactMatchMetric(fieldPath?: string): MetricFn;
+export declare function createContainsMetric(keywords: string[]): MetricFn;
+//# sourceMappingURL=metrics.d.ts.map

package/dist/learning/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/learning/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,QAAQ,EACR,qBAAqB,EAErB,UAAU,EACX,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,sBAAsB;IACrC,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;CACzC;AAsBD,qBAAa,eAAe;IAC1B,OAAO,CAAC,GAAG,CAAC,CAAa;IACzB,OAAO,CAAC,KAAK,CAAC,CAAS;IACvB,OAAO,CAAC,MAAM,CAAwB;IACtC,OAAO,CAAC,aAAa,CAA+B;gBAExC,OAAO,GAAE,sBAA2B;IAMhD,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,GAAG,IAAI;IAI1C,QAAQ,CACZ,KAAK,EAAE,cAAc,EACrB,QAAQ,CAAC,EAAE,OAAO,GACjB,OAAO,CAAC;QAAE,OAAO,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IAcjE,aAAa,CACjB,MAAM,EAAE,cAAc,EAAE,EACxB,YAAY,CAAC,EAAE,OAAO,EAAE,GACvB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;YAatE,cAAc;IA0B5B,aAAa,CAAC,KAAK,EAAE,cAAc,GAAG,YAAY;IAmBlD,kBAAkB,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,YAAY;IAwC3E,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,YAAY;IAiB/C,kBAAkB,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC;IAiDpF,eAAe,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;IAgDnE,OAAO,CAAC,mBAAmB;IAkB3B,OAAO,CAAC,eAAe;IA6BvB,SAAS,IAAI,qBAAqB;CAGnC;AAED,wBAAgB,mBAAmB,IAAI,QAAQ,CAS9C;AAED,wBAAgB,sBAAsB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,CAkBnE;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,QAAQ,CAoBjE"}

package/dist/learning/metrics.js

@@ -0,0 +1,310 @@
+const DEFAULT_CONFIG = {
+    metrics: [
+        {
+            name: 'success',
+            type: 'boolean',
+            description: 'Did the run complete without errors?',
+            weight: 0.4,
+        },
+        {
+            name: 'tool_accuracy',
+            type: 'numeric',
+            description: 'Did tools produce expected results?',
+            weight: 0.3,
+        },
+        { name: 'efficiency', type: 'numeric', description: 'Token/time efficiency', weight: 0.3 },
+    ],
+    aggregation: 'weighted-average',
+    passThreshold: 0.7,
+};
+export class MetricEvaluator {
+    llm;
+    model;
+    config;
+    customMetrics = new Map();
+    constructor(options = {}) {
+        this.llm = options.llm;
+        this.model = options.model;
+        this.config = { ...DEFAULT_CONFIG, ...options.config };
+    }
+    registerMetric(name, fn) {
+        this.customMetrics.set(name, fn);
+    }
+    async evaluate(trace, expected) {
+        const results = [];
+        for (const metricDef of this.config.metrics) {
+            const result = await this.evaluateMetric(metricDef.name, trace, expected);
+            results.push(result);
+        }
+        const score = this.aggregateScores(results);
+        const passed = score >= this.config.passThreshold;
+        return { results, score, passed };
+    }
+    async evaluateBatch(traces, expectedList) {
+        const results = new Map();
+        for (let i = 0; i < traces.length; i++) {
+            const trace = traces[i];
+            const expected = expectedList?.[i];
+            const evaluation = await this.evaluate(trace, expected);
+            results.set(trace.id, evaluation);
+        }
+        return results;
+    }
+    async evaluateMetric(name, trace, expected) {
+        if (this.customMetrics.has(name)) {
+            const fn = this.customMetrics.get(name);
+            return fn(trace, expected);
+        }
+        switch (name) {
+            case 'success':
+                return this.successMetric(trace);
+            case 'tool_accuracy':
+                return this.toolAccuracyMetric(trace, expected);
+            case 'efficiency':
+                return this.efficiencyMetric(trace);
+            case 'completeness':
+                return this.completenessMetric(trace, expected);
+            case 'coherence':
+                return this.coherenceMetric(trace);
+            default:
+                return { name, value: 0.5, passed: true, reasoning: 'Unknown metric' };
+        }
+    }
+    successMetric(trace) {
+        const hasErrors = trace.steps.some((step) => step.toolResult?.error ||
+            (step.type === 'reflection' && step.reflection?.analysis?.wasSuccessful === false));
+        const value = hasErrors ? 0 : 1;
+        return {
+            name: 'success',
+            value,
+            passed: value === 1,
+            reasoning: hasErrors
+                ? 'Run had errors or failed reflections'
+                : 'Run completed without errors',
+        };
+    }
+    toolAccuracyMetric(trace, expected) {
+        const toolSteps = trace.steps.filter((s) => s.type === 'tool_call');
+        if (toolSteps.length === 0) {
+            return {
+                name: 'tool_accuracy',
+                value: 1,
+                passed: true,
+                reasoning: 'No tool calls to evaluate',
+            };
+        }
+        let successfulCalls = 0;
+        for (const step of toolSteps) {
+            if (step.toolResult && !step.toolResult.error) {
+                successfulCalls++;
+            }
+        }
+        const value = successfulCalls / toolSteps.length;
+        if (expected !== undefined && typeof expected === 'string') {
+            const outputMatches = trace.output.toLowerCase().includes(expected.toString().toLowerCase());
+            const adjustedValue = outputMatches ? Math.min(value + 0.2, 1) : Math.max(value - 0.2, 0);
+            return {
+                name: 'tool_accuracy',
+                value: adjustedValue,
+                passed: adjustedValue >= 0.7,
+                reasoning: `${successfulCalls}/${toolSteps.length} successful tool calls, output ${outputMatches ? 'matches' : 'does not match'} expected`,
+            };
+        }
+        return {
+            name: 'tool_accuracy',
+            value,
+            passed: value >= 0.7,
+            reasoning: `${successfulCalls}/${toolSteps.length} successful tool calls`,
+        };
+    }
+    efficiencyMetric(trace) {
+        const totalTokens = trace.usage.inputTokens + trace.usage.outputTokens;
+        const duration = trace.duration;
+        const tokenEfficiency = Math.min(1, 10000 / Math.max(totalTokens, 1));
+        const timeEfficiency = Math.min(1, 30000 / Math.max(duration, 1));
+        const value = tokenEfficiency * 0.6 + timeEfficiency * 0.4;
+        return {
+            name: 'efficiency',
+            value,
+            passed: value >= 0.5,
+            reasoning: `${totalTokens} tokens in ${duration}ms`,
+        };
+    }
+    async completenessMetric(trace, expected) {
+        if (!this.llm || !this.model) {
+            const hasOutput = !!trace.output && trace.output.length > 10;
+            return {
+                name: 'completeness',
+                value: hasOutput ? 0.7 : 0.3,
+                passed: hasOutput,
+                reasoning: 'Basic output length check (no LLM available)',
+            };
+        }
+        const prompt = `Evaluate how completely this output addresses the input.
+
+Input: ${trace.input}
+Output: ${trace.output}
+${expected ? `Expected: ${JSON.stringify(expected)}` : ''}
+
+Rate completeness from 0.0 to 1.0 where:
+- 0.0 = completely misses the point
+- 0.5 = partially addresses input
+- 1.0 = fully and thoroughly addresses input
+
+Respond with JSON: { "score": 0.X, "reasoning": "..." }`;
+        try {
+            const response = await this.llm.chat({
+                model: this.model,
+                messages: [{ role: 'user', content: prompt }],
+                temperature: 0.3,
+                maxTokens: 200,
+            });
+            const parsed = this.parseMetricResponse(response.content);
+            return {
+                name: 'completeness',
+                value: parsed.score,
+                passed: parsed.score >= 0.7,
+                reasoning: parsed.reasoning,
+            };
+        }
+        catch {
+            return {
+                name: 'completeness',
+                value: 0.5,
+                passed: true,
+                reasoning: 'Evaluation failed, using default',
+            };
+        }
+    }
+    async coherenceMetric(trace) {
+        if (!this.llm || !this.model) {
+            return {
+                name: 'coherence',
+                value: 0.7,
+                passed: true,
+                reasoning: 'No LLM available for coherence check',
+            };
+        }
+        const prompt = `Evaluate the logical coherence of this agent execution.
+
+Input: ${trace.input}
+Steps taken: ${trace.steps.map((s) => (s.type === 'tool_call' ? `Tool: ${s.toolCall?.name}` : s.type)).join(' → ')}
+Output: ${trace.output}
+
+Rate coherence from 0.0 to 1.0 where:
+- 0.0 = completely incoherent, steps don't make sense
+- 0.5 = somewhat logical but with issues
+- 1.0 = perfectly logical and well-structured
+
+Respond with JSON: { "score": 0.X, "reasoning": "..." }`;
+        try {
+            const response = await this.llm.chat({
+                model: this.model,
+                messages: [{ role: 'user', content: prompt }],
+                temperature: 0.3,
+                maxTokens: 200,
+            });
+            const parsed = this.parseMetricResponse(response.content);
+            return {
+                name: 'coherence',
+                value: parsed.score,
+                passed: parsed.score >= 0.6,
+                reasoning: parsed.reasoning,
+            };
+        }
+        catch {
+            return {
+                name: 'coherence',
+                value: 0.6,
+                passed: true,
+                reasoning: 'Evaluation failed, using default',
+            };
+        }
+    }
+    parseMetricResponse(content) {
+        try {
+            let jsonStr = content;
+            const codeBlockMatch = /```(?:json)?\s*([\s\S]*?)\s*```/.exec(content);
+            if (codeBlockMatch) {
+                jsonStr = codeBlockMatch[1];
+            }
+            const parsed = JSON.parse(jsonStr);
+            const score = Math.max(0, Math.min(1, Number(parsed.score) || 0.5));
+            const reasoning = String(parsed.reasoning || 'No reasoning provided');
+            return { score, reasoning };
+        }
+        catch {
+            return { score: 0.5, reasoning: 'Failed to parse metric response' };
+        }
+    }
+    aggregateScores(results) {
+        if (results.length === 0)
+            return 0;
+        switch (this.config.aggregation) {
+            case 'weighted-average': {
+                let totalWeight = 0;
+                let weightedSum = 0;
+                for (const result of results) {
+                    const metricDef = this.config.metrics.find((m) => m.name === result.name);
+                    const weight = metricDef?.weight ?? 1;
+                    weightedSum += result.value * weight;
+                    totalWeight += weight;
+                }
+                return totalWeight > 0 ? weightedSum / totalWeight : 0;
+            }
+            case 'min':
+                return Math.min(...results.map((r) => r.value));
+            case 'product':
+                return results.reduce((acc, r) => acc * r.value, 1);
+            default:
+                return results.reduce((sum, r) => sum + r.value, 0) / results.length;
+        }
+    }
+    getConfig() {
+        return { ...this.config };
+    }
+}
+export function createSuccessMetric() {
+    return (trace) => {
+        const hasErrors = trace.steps.some((step) => step.toolResult?.error);
+        return {
+            name: 'success',
+            value: hasErrors ? 0 : 1,
+            passed: !hasErrors,
+        };
+    };
+}
+export function createExactMatchMetric(fieldPath) {
+    return (trace, expected) => {
+        if (expected === undefined) {
+            return { name: 'exact_match', value: 1, passed: true, reasoning: 'No expected value' };
+        }
+        const outputValue = fieldPath ? trace.output : trace.output;
+        const matches = String(outputValue).toLowerCase().trim() === String(expected).toLowerCase().trim();
+        return {
+            name: 'exact_match',
+            value: matches ? 1 : 0,
+            passed: matches,
+            reasoning: matches ? 'Output matches expected' : 'Output does not match expected',
+        };
+    };
+}
+export function createContainsMetric(keywords) {
+    return (trace) => {
+        const outputLower = trace.output.toLowerCase();
+        let found = 0;
+        for (const keyword of keywords) {
+            if (outputLower.includes(keyword.toLowerCase())) {
+                found++;
+            }
+        }
+        const value = keywords.length > 0 ? found / keywords.length : 1;
+        return {
+            name: 'contains',
+            value,
+            passed: value >= 0.5,
+            reasoning: `Found ${found}/${keywords.length} keywords`,
+        };
+    };
+}
+//# sourceMappingURL=metrics.js.map

package/dist/learning/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/learning/metrics.ts"],"names":[],"mappings":"AAeA,MAAM,cAAc,GAA0B;IAC5C,OAAO,EAAE;QACP;YACE,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,sCAAsC;YACnD,MAAM,EAAE,GAAG;SACZ;QACD;YACE,IAAI,EAAE,eAAe;YACrB,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,qCAAqC;YAClD,MAAM,EAAE,GAAG;SACZ;QACD,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,EAAE,GAAG,EAAE;KAC3F;IACD,WAAW,EAAE,kBAAkB;IAC/B,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF,MAAM,OAAO,eAAe;IAClB,GAAG,CAAc;IACjB,KAAK,CAAU;IACf,MAAM,CAAwB;IAC9B,aAAa,GAAG,IAAI,GAAG,EAAoB,CAAC;IAEpD,YAAY,UAAkC,EAAE;QAC9C,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IACzD,CAAC;IAED,cAAc,CAAC,IAAY,EAAE,EAAY;QACvC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,KAAqB,EACrB,QAAkB;QAElB,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;YAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;QAElD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,MAAwB,EACxB,YAAwB;QAExB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuE,CAAC;QAE/F,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACxD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QACpC,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,IAAY,EACZ,KAAqB,EACrB,QAAkB;QAElB,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;YACzC,OAAO,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC7B,CAAC;QAED,QAAQ,IAAqB,EAAE,CAAC;YAC9B,KAAK,SAAS;gBACZ,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACnC,KAAK,eAAe;gBAClB,OAAO,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClD,KAAK,YAAY;gBACf,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACtC,KAAK,cAAc;gBACjB,OAAO,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClD,KAAK,WAAW;gBACd,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACrC;gBACE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,aAAa,CAAC,KAAqB;QACjC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAChC,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,UAAU,EAAE,KAAK;YACtB,CAAC,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,aAAa,KAAK,KAAK,CAAC,CACrF,CAAC;QAEF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEhC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK;YACL,MAAM,EAAE,KAAK,KAAK,CAAC;YACnB,SAAS,EAAE,SAAS;gBAClB,CAAC,CAAC,sCAAsC;gBACxC,CAAC,CAAC,8BAA8B;SACnC,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,KAAqB,EAAE,QAAkB;QAC1D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;QAEpE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,2BAA2B;aACvC,CAAC;QACJ,CAAC;QAED,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;gBAC9C,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC;QAEjD,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;YAC7F,MAAM,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;YAC1F,OAAO;gBACL,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,aAAa;gBACpB,MAAM,EAAE,aAAa,IAAI,GAAG;gBAC5B,SAAS,EAAE,GAAG,eAAe,IAAI,SAAS,CAAC,MAAM,kCAAkC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,WAAW;aAC3I,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,eAAe;YACrB,KAAK;YACL,MAAM,EAAE,KAAK,IAAI,GAAG;YACpB,SAAS,EAAE,GAAG,eAAe,IAAI,SAAS,CAAC,MAAM,wBAAwB;SAC1E,CAAC;IACJ,CAAC;IAED,gBAAgB,CAAC,KAAqB;QACpC,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC;QACvE,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAEhC,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;QAElE,MAAM,KAAK,GAAG,eAAe,GAAG,GAAG,GAAG,cAAc,GAAG,GAAG,CAAC;QAE3D,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK;YACL,MAAM,EAAE,KAAK,IAAI,GAAG;YACpB,SAAS,EAAE,GAAG,WAAW,cAAc,QAAQ,IAAI;SACpD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,KAAqB,EAAE,QAAkB;QAChE,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC;YAC7D,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;gBAC5B,MAAM,EAAE,SAAS;gBACjB,SAAS,EAAE,8CAA8C;aAC1D,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG;;SAEV,KAAK,CAAC,KAAK;UACV,KAAK,CAAC,MAAM;EACpB,QAAQ,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;;;;;;;wDAOD,CAAC;QAErD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7C,WAAW,EAAE,GAAG;gBAChB,SAAS,EAAE,GAAG;aACf,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC1D,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM,EAAE,MAAM,CAAC,KAAK,IAAI,GAAG;gBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,kCAAkC;aAC9C,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAqB;QACzC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7B,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,sCAAsC;aAClD,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG;;SAEV,KAAK,CAAC,KAAK;eACL,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;UACxG,KAAK,CAAC,MAAM;;;;;;;wDAOkC,CAAC;QAErD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7C,WAAW,EAAE,GAAG;gBAChB,SAAS,EAAE,GAAG;aACf,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC1D,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM,EAAE,MAAM,CAAC,KAAK,IAAI,GAAG;gBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,kCAAkC;aAC9C,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,OAAe;QACzC,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,OAAO,CAAC;YACtB,MAAM,cAAc,GAAG,iCAAiC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvE,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;YACpE,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,IAAI,uBAAuB,CAAC,CAAC;YAEtE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,iCAAiC,EAAE,CAAC;QACtE,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,OAAuB;QAC7C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEnC,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAChC,KAAK,kBAAkB,CAAC,CAAC,CAAC;gBACxB,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,IAAI,WAAW,GAAG,CAAC,CAAC;gBAEpB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC1E,MAAM,MAAM,GAAG,SAAS,EAAE,MAAM,IAAI,CAAC,CAAC;oBACtC,WAAW,IAAI,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;oBACrC,WAAW,IAAI,MAAM,CAAC;gBACxB,CAAC;gBAED,OAAO,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,KAAK,KAAK;gBACR,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAElD,KAAK,SAAS;gBACZ,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAEtD;gBACE,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QACzE,CAAC;IACH,CAAC;IAED,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;CACF;AAED,MAAM,UAAU,mBAAmB;IACjC,OAAO,CAAC,KAAqB,EAAE,EAAE;QAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QACrE,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,EAAE,CAAC,SAAS;SACnB,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,SAAkB;IACvD,OAAO,CAAC,KAAqB,EAAE,QAAkB,EAAE,EAAE;QACnD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;QACzF,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;QAE5D,MAAM,OAAO,GACX,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAErF,OAAO;YACL,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,EAAE,OAAO;YACf,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,gCAAgC;SAClF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAkB;IACrD,OAAO,CAAC,KAAqB,EAAE,EAAE;QAC/B,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC/C,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;gBAChD,KAAK,EAAE,CAAC;YACV,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAEhE,OAAO;YACL,IAAI,EAAE,UAAU;YAChB,KAAK;YACL,MAAM,EAAE,KAAK,IAAI,GAAG;YACpB,SAAS,EAAE,SAAS,KAAK,IAAI,QAAQ,CAAC,MAAM,WAAW;SACxD,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/learning/postgres-trace-store.d.ts

@@ -0,0 +1,53 @@
+import type { ExecutionTrace, TraceQuery, TraceStoreStats, CapturedPrompt, PromptQuery, ABTest, ABTestStatus, ABTestVariant, InstructionVersion, InstructionVersionMetrics, CombinedPersistentStore } from '@cogitator-ai/types';
+export interface PostgresTraceStoreConfig {
+    connectionString: string;
+    schema?: string;
+    poolSize?: number;
+}
+export declare class PostgresTraceStore implements CombinedPersistentStore {
+    private pool;
+    private config;
+    private schema;
+    constructor(config: PostgresTraceStoreConfig);
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    private initSchema;
+    private generateId;
+    storeTrace(trace: ExecutionTrace): Promise<void>;
+    storeTraceMany(traces: ExecutionTrace[]): Promise<void>;
+    getTrace(id: string): Promise<ExecutionTrace | null>;
+    getTraceByRunId(runId: string): Promise<ExecutionTrace | null>;
+    queryTraces(query: TraceQuery): Promise<ExecutionTrace[]>;
+    getAllTraces(agentId: string): Promise<ExecutionTrace[]>;
+    getDemos(agentId: string, limit?: number): Promise<ExecutionTrace[]>;
+    markAsDemo(id: string): Promise<void>;
+    unmarkAsDemo(id: string): Promise<void>;
+    deleteTrace(id: string): Promise<boolean>;
+    pruneTraces(agentId: string, maxTraces: number): Promise<number>;
+    clearTraces(agentId: string): Promise<void>;
+    getTraceStats(agentId: string): Promise<TraceStoreStats>;
+    private rowToTrace;
+    capture(prompt: CapturedPrompt): Promise<void>;
+    getPrompt(id: string): Promise<CapturedPrompt | null>;
+    getByRun(runId: string): Promise<CapturedPrompt[]>;
+    query(query: PromptQuery): Promise<CapturedPrompt[]>;
+    deletePrompt(id: string): Promise<boolean>;
+    prune(beforeDate: Date): Promise<number>;
+    private rowToPrompt;
+    create(test: Omit<ABTest, 'id' | 'createdAt'>): Promise<ABTest>;
+    getABTest(id: string): Promise<ABTest | null>;
+    getActive(agentId: string): Promise<ABTest | null>;
+    update(id: string, updates: Partial<ABTest>): Promise<ABTest>;
+    recordResult(testId: string, variant: ABTestVariant, score: number, latency: number, cost: number): Promise<void>;
+    list(agentId?: string, status?: ABTestStatus): Promise<ABTest[]>;
+    deleteABTest(id: string): Promise<boolean>;
+    private rowToABTest;
+    save(version: Omit<InstructionVersion, 'id'>): Promise<InstructionVersion>;
+    getVersion(id: string): Promise<InstructionVersion | null>;
+    getCurrent(agentId: string): Promise<InstructionVersion | null>;
+    getHistory(agentId: string, limit?: number): Promise<InstructionVersion[]>;
+    retire(id: string): Promise<void>;
+    updateMetrics(id: string, metrics: Partial<InstructionVersionMetrics>): Promise<void>;
+    private rowToVersion;
+}
+//# sourceMappingURL=postgres-trace-store.d.ts.map

package/dist/learning/postgres-trace-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"postgres-trace-store.d.ts","sourceRoot":"","sources":["../../src/learning/postgres-trace-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,UAAU,EACV,eAAe,EACf,cAAc,EACd,WAAW,EACX,MAAM,EACN,YAAY,EACZ,aAAa,EAEb,kBAAkB,EAClB,yBAAyB,EACzB,uBAAuB,EACxB,MAAM,qBAAqB,CAAC;AAS7B,MAAM,WAAW,wBAAwB;IACvC,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,kBAAmB,YAAW,uBAAuB;IAChE,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,wBAAwB;IAKtC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAexB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAOnB,UAAU;IA+HxB,OAAO,CAAC,UAAU;IAIZ,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IA+BhD,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAMvD,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAapD,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAa9D,WAAW,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IA4CzD,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAWxD,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAWhE,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASrC,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASvC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWzC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAiBhE,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS3C,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAoB9D,OAAO,CAAC,UAAU;IAwBZ,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAkC9C,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAarD,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAWlD,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAgDpD,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAW1C,KAAK,CAAC,UAAU,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;IAW9C,OAAO,CAAC,WAAW;IAiCb,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,GAAG,WAAW,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAkC/D,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAa7C,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAalD,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAsC7D,YAAY,CAChB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC;IAuBV,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAuBhE,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWhD,OAAO,CAAC,WAAW;IAsBb,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA8B1E,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAa1D,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAe/D,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAatE,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASjC,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,yBAAyB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAsC3F,OAAO,CAAC,YAAY;CAoBrB"}