kagent-ts 0.1.4 → 0.1.5

package/dist/eval/index.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Benchmark = exports.EvalRunner = exports.ToolCallEvaluator = void 0;
+// Tool Call Evaluator — per-tool metrics via AgentHooks
+var tool_call_evaluator_1 = require("./tool-call-evaluator");
+Object.defineProperty(exports, "ToolCallEvaluator", { enumerable: true, get: function () { return tool_call_evaluator_1.ToolCallEvaluator; } });
+// Eval Runner — end-to-end test case execution
+var eval_runner_1 = require("./eval-runner");
+Object.defineProperty(exports, "EvalRunner", { enumerable: true, get: function () { return eval_runner_1.EvalRunner; } });
+// Benchmark — regression testing & baseline comparison
+var benchmark_1 = require("./benchmark");
+Object.defineProperty(exports, "Benchmark", { enumerable: true, get: function () { return benchmark_1.Benchmark; } });
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":";;;AAAA,wDAAwD;AACxD,6DAA0D;AAAjD,wHAAA,iBAAiB,OAAA;AAE1B,+CAA+C;AAC/C,6CAA2C;AAAlC,yGAAA,UAAU,OAAA;AAGnB,uDAAuD;AACvD,yCAAwC;AAA/B,sGAAA,SAAS,OAAA"}

package/dist/eval/tool-call-evaluator.d.ts

@@ -0,0 +1,72 @@
+import { AgentHooks } from "../core/hooks";
+import type { LLMResponse } from "../llm/interface";
+import type { LLMNetworkError } from "../llm/errors";
+import type { MessageData } from "../messages/types";
+import type { Tool } from "../tools/types";
+import type { ToolCallRecord, ToolCallScorecard } from "./types";
+/**
+ * ToolCallEvaluator — collects per-tool-call metrics via AgentHooks.
+ *
+ * Attach to any agent to track tool call performance:
+ * ```ts
+ * const evaluator = new ToolCallEvaluator();
+ * const agent = new ReActAgent({ llm, hooks: [evaluator, ...otherHooks] });
+ * await agent.run("...");
+ * console.log(evaluator.generateReport());
+ * ```
+ */
+export declare class ToolCallEvaluator implements AgentHooks {
+    /** All recorded tool calls in chronological order. */
+    private records;
+    /** Per-tool attempt counters (reset on success). */
+    private attemptCounters;
+    /** Per-tool circuit breaker trip counters. */
+    private circuitBreakerTripCounts;
+    onLLMStart?: ((messages: MessageData[], tools: Tool[]) => void) | undefined;
+    onLLMEnd?: ((response: LLMResponse) => void) | undefined;
+    onLLMError?: ((error: LLMNetworkError) => void) | undefined;
+    onToolStart(toolName: string, args: Record<string, unknown>, toolCallId?: string): void;
+    onToolEnd(toolName: string, result: string, toolCallId?: string): void;
+    onToolError(toolName: string, error: string, toolCallId?: string): void;
+    onThought?: ((thought: string) => void) | undefined;
+    onPlanCreated?: ((plan: string[]) => void) | undefined;
+    onPlanRevised?: ((plan: string[]) => void) | undefined;
+    onFinish?: ((answer: string) => void) | undefined;
+    /**
+     * Get all raw tool call records.
+     */
+    getRecords(): ToolCallRecord[];
+    /**
+     * Compute the aggregated scorecard from all recorded calls.
+     */
+    getScorecard(): ToolCallScorecard;
+    /**
+     * Generate a Markdown report of tool call metrics.
+     */
+    generateReport(): string;
+    /**
+     * Reset all counters and records.
+     */
+    reset(): void;
+    /**
+     * Find the matching uncompleted record for a tool call.
+     *
+     * When `toolCallId` is provided, performs an exact match (preferred:
+     * handles parallel calls to the same tool correctly). Falls back to
+     * reverse-scan by tool name when the ID is not available (legacy
+     * hooks that don't pass `toolCallId`).
+     */
+    private findRecord;
+    /**
+     * Extract a ToolErrorCode from the structured error message format.
+     *
+     * Error messages follow the pattern:
+     *   [SEVERITY:ERROR_CODE] Human-readable message...
+     */
+    private extractErrorCode;
+    /**
+     * Compute per-tool statistics from raw records.
+     */
+    private computePerToolStats;
+}
+//# sourceMappingURL=tool-call-evaluator.d.ts.map

package/dist/eval/tool-call-evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-call-evaluator.d.ts","sourceRoot":"","sources":["../../src/eval/tool-call-evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,KAAK,EACV,cAAc,EAEd,iBAAiB,EAClB,MAAM,SAAS,CAAC;AAEjB;;;;;;;;;;GAUG;AACH,qBAAa,iBAAkB,YAAW,UAAU;IAClD,sDAAsD;IACtD,OAAO,CAAC,OAAO,CAAwB;IAEvC,oDAAoD;IACpD,OAAO,CAAC,eAAe,CAAkC;IAEzD,8CAA8C;IAC9C,OAAO,CAAC,wBAAwB,CAAkC;IAIlE,UAAU,CAAC,EACP,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC,GAClD,SAAS,CAAa;IAE1B,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,WAAW,KAAK,IAAI,CAAC,GAAG,SAAS,CAAa;IAErE,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,CAAC,GAAG,SAAS,CAAa;IAExE,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI;IAavF,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI;IAetE,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI;IA2BvE,SAAS,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAa;IAChE,aAAa,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC,GAAG,SAAS,CAAa;IACnE,aAAa,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC,GAAG,SAAS,CAAa;IACnE,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAa;IAI9D;;OAEG;IACH,UAAU,IAAI,cAAc,EAAE;IAI9B;;OAEG;IACH,YAAY,IAAI,iBAAiB;IA6BjC;;OAEG;IACH,cAAc,IAAI,MAAM;IA+CxB;;OAEG;IACH,KAAK,IAAI,IAAI;IAQb;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IAwBlB;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAiD5B"}

package/dist/eval/tool-call-evaluator.js

@@ -0,0 +1,265 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ToolCallEvaluator = void 0;
+const types_1 = require("../tools/types");
+/**
+ * ToolCallEvaluator — collects per-tool-call metrics via AgentHooks.
+ *
+ * Attach to any agent to track tool call performance:
+ * ```ts
+ * const evaluator = new ToolCallEvaluator();
+ * const agent = new ReActAgent({ llm, hooks: [evaluator, ...otherHooks] });
+ * await agent.run("...");
+ * console.log(evaluator.generateReport());
+ * ```
+ */
+class ToolCallEvaluator {
+    /** All recorded tool calls in chronological order. */
+    records = [];
+    /** Per-tool attempt counters (reset on success). */
+    attemptCounters = new Map();
+    /** Per-tool circuit breaker trip counters. */
+    circuitBreakerTripCounts = new Map();
+    // ─── AgentHooks Implementation ─────────────────────────────────────────
+    onLLMStart = undefined;
+    onLLMEnd = undefined;
+    onLLMError = undefined;
+    onToolStart(toolName, args, toolCallId) {
+        const attempt = this.attemptCounters.get(toolName) ?? 0;
+        this.records.push({
+            toolCallId,
+            toolName,
+            args,
+            startTime: new Date().toISOString(),
+            success: false, // set on end/error
+            attemptNumber: attempt + 1,
+        });
+    }
+    onToolEnd(toolName, result, toolCallId) {
+        this.attemptCounters.set(toolName, 0); // reset on success
+        const record = this.findRecord(toolName, toolCallId);
+        if (!record)
+            return;
+        record.endTime = new Date().toISOString();
+        record.latencyMs =
+            new Date(record.endTime).getTime() -
+                new Date(record.startTime).getTime();
+        record.success = true;
+        record.errorCode = types_1.ToolErrorCode.SUCCESS;
+        record.resultLength = result.length;
+    }
+    onToolError(toolName, error, toolCallId) {
+        const attempt = (this.attemptCounters.get(toolName) ?? 0) + 1;
+        this.attemptCounters.set(toolName, attempt);
+        const record = this.findRecord(toolName, toolCallId);
+        if (!record)
+            return;
+        record.endTime = new Date().toISOString();
+        record.latencyMs =
+            new Date(record.endTime).getTime() -
+                new Date(record.startTime).getTime();
+        record.success = false;
+        record.error = error;
+        record.attemptNumber = attempt;
+        // Extract error code from the structured error message
+        record.errorCode = this.extractErrorCode(error);
+        // Track circuit breaker trips
+        if (record.errorCode === types_1.ToolErrorCode.CIRCUIT_OPEN) {
+            this.circuitBreakerTripCounts.set(toolName, (this.circuitBreakerTripCounts.get(toolName) ?? 0) + 1);
+        }
+    }
+    onThought = undefined;
+    onPlanCreated = undefined;
+    onPlanRevised = undefined;
+    onFinish = undefined;
+    // ─── Public API ────────────────────────────────────────────────────────
+    /**
+     * Get all raw tool call records.
+     */
+    getRecords() {
+        return [...this.records];
+    }
+    /**
+     * Compute the aggregated scorecard from all recorded calls.
+     */
+    getScorecard() {
+        const allStats = this.computePerToolStats();
+        const totalCalls = allStats.reduce((s, t) => s + t.totalCalls, 0);
+        const totalSuccesses = allStats.reduce((s, t) => s + t.successCount, 0);
+        const totalFailures = totalCalls - totalSuccesses;
+        const allLatencies = allStats.flatMap((t) => t.latencySamples);
+        const avgLatencyMs = allLatencies.length > 0
+            ? Math.round(allLatencies.reduce((a, b) => a + b, 0) / allLatencies.length)
+            : 0;
+        const circuitBreakerTrips = allStats.reduce((s, t) => s + t.circuitBreakerTrips, 0);
+        return {
+            totalCalls,
+            totalSuccesses,
+            totalFailures,
+            overallSuccessRate: totalCalls > 0 ? totalSuccesses / totalCalls : 1,
+            avgLatencyMs,
+            uniqueToolsUsed: allStats.length,
+            circuitBreakerTrips,
+            perTool: allStats.sort((a, b) => b.totalCalls - a.totalCalls),
+        };
+    }
+    /**
+     * Generate a Markdown report of tool call metrics.
+     */
+    generateReport() {
+        const scorecard = this.getScorecard();
+        if (scorecard.totalCalls === 0) {
+            return "# Tool Call Evaluation Report\n\n*No tool calls recorded.*\n";
+        }
+        const ratePercent = (scorecard.overallSuccessRate * 100).toFixed(1);
+        let report = `# Tool Call Evaluation Report\n\n`;
+        report += `## Summary\n\n`;
+        report += `| Metric | Value |\n`;
+        report += `|--------|-------|\n`;
+        report += `| Total Calls | ${scorecard.totalCalls} |\n`;
+        report += `| Successes | ${scorecard.totalSuccesses} |\n`;
+        report += `| Failures | ${scorecard.totalFailures} |\n`;
+        report += `| Success Rate | ${ratePercent}% |\n`;
+        report += `| Avg Latency | ${scorecard.avgLatencyMs}ms |\n`;
+        report += `| Unique Tools Used | ${scorecard.uniqueToolsUsed} |\n`;
+        report += `| Circuit Breaker Trips | ${scorecard.circuitBreakerTrips} |\n\n`;
+        report += `## Per-Tool Breakdown\n\n`;
+        report += `| Tool | Calls | Success Rate | Avg Latency | P50 | P99 | Avg Retries | CB Trips |\n`;
+        report += `|------|-------|-------------|-------------|-----|-----|-------------|----------|\n`;
+        for (const stat of scorecard.perTool) {
+            const sr = (stat.successRate * 100).toFixed(1);
+            report += `| \`${stat.toolName}\` | ${stat.totalCalls} | ${sr}% | ${stat.avgLatencyMs}ms | ${stat.p50LatencyMs}ms | ${stat.p99LatencyMs}ms | ${stat.avgRetries.toFixed(1)} | ${stat.circuitBreakerTrips} |\n`;
+        }
+        report += `\n## Error Distribution\n\n`;
+        for (const stat of scorecard.perTool) {
+            if (Object.keys(stat.errorDistribution).length === 0)
+                continue;
+            report += `### ${stat.toolName}\n\n`;
+            report += `| Error Code | Count |\n`;
+            report += `|------------|-------|\n`;
+            for (const [code, count] of Object.entries(stat.errorDistribution)) {
+                report += `| \`${code}\` | ${count} |\n`;
+            }
+            report += `\n`;
+        }
+        report += `---\n*Generated at ${new Date().toISOString()}*\n`;
+        return report;
+    }
+    /**
+     * Reset all counters and records.
+     */
+    reset() {
+        this.records = [];
+        this.attemptCounters.clear();
+        this.circuitBreakerTripCounts.clear();
+    }
+    // ─── Private Helpers ───────────────────────────────────────────────────
+    /**
+     * Find the matching uncompleted record for a tool call.
+     *
+     * When `toolCallId` is provided, performs an exact match (preferred:
+     * handles parallel calls to the same tool correctly). Falls back to
+     * reverse-scan by tool name when the ID is not available (legacy
+     * hooks that don't pass `toolCallId`).
+     */
+    findRecord(toolName, toolCallId) {
+        // Exact ID match — correct even when the same tool is called
+        // multiple times within one LLM response batch.
+        if (toolCallId) {
+            for (let i = this.records.length - 1; i >= 0; i--) {
+                if (this.records[i].toolCallId === toolCallId && !this.records[i].endTime) {
+                    return this.records[i];
+                }
+            }
+            return undefined;
+        }
+        // Legacy fallback — reverse scan by tool name only.
+        for (let i = this.records.length - 1; i >= 0; i--) {
+            if (this.records[i].toolName === toolName && !this.records[i].endTime) {
+                return this.records[i];
+            }
+        }
+        return undefined;
+    }
+    /**
+     * Extract a ToolErrorCode from the structured error message format.
+     *
+     * Error messages follow the pattern:
+     *   [SEVERITY:ERROR_CODE] Human-readable message...
+     */
+    extractErrorCode(error) {
+        const match = error.match(/\[(?:RETRYABLE|FATAL):([A-Z_]+)\]/);
+        if (match) {
+            // Error format uses uppercase (CIRCUIT_OPEN), but enum values are
+            // lowercase ("circuit_open"). Normalise before lookup.
+            const code = match[1].toLowerCase();
+            if (Object.values(types_1.ToolErrorCode).includes(code)) {
+                return code;
+            }
+        }
+        return types_1.ToolErrorCode.EXECUTION_FAILURE; // default for unparseable errors
+    }
+    /**
+     * Compute per-tool statistics from raw records.
+     */
+    computePerToolStats() {
+        const byTool = new Map();
+        for (const r of this.records) {
+            if (!byTool.has(r.toolName))
+                byTool.set(r.toolName, []);
+            byTool.get(r.toolName).push(r);
+        }
+        const stats = [];
+        for (const [toolName, records] of byTool) {
+            const completed = records.filter((r) => r.endTime);
+            const successes = completed.filter((r) => r.success);
+            const failures = completed.filter((r) => !r.success);
+            const latencies = completed
+                .filter((r) => r.latencyMs !== undefined)
+                .map((r) => r.latencyMs);
+            // Error distribution
+            const errorDist = {};
+            for (const f of failures) {
+                const code = f.errorCode ?? types_1.ToolErrorCode.EXECUTION_FAILURE;
+                errorDist[code] = (errorDist[code] ?? 0) + 1;
+            }
+            stats.push({
+                toolName,
+                totalCalls: completed.length,
+                successCount: successes.length,
+                failureCount: failures.length,
+                successRate: completed.length > 0 ? successes.length / completed.length : 1,
+                avgLatencyMs: latencies.length > 0
+                    ? Math.round(latencies.reduce((a, b) => a + b, 0) / latencies.length)
+                    : 0,
+                p50LatencyMs: percentile(latencies, 50),
+                p99LatencyMs: percentile(latencies, 99),
+                avgRetries: successes.length > 0
+                    ? failures.length / successes.length
+                    : failures.length,
+                circuitBreakerTrips: this.circuitBreakerTripCounts.get(toolName) ?? 0,
+                errorDistribution: errorDist,
+                latencySamples: latencies,
+            });
+        }
+        return stats;
+    }
+}
+exports.ToolCallEvaluator = ToolCallEvaluator;
+/**
+ * Compute a percentile from an array of numbers.
+ * Uses linear interpolation between closest ranks.
+ */
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const arr = [...sorted].sort((a, b) => a - b);
+    const index = (p / 100) * (arr.length - 1);
+    const lower = Math.floor(index);
+    const upper = Math.ceil(index);
+    if (lower === upper)
+        return Math.round(arr[lower]);
+    const weight = index - lower;
+    return Math.round(arr[lower] * (1 - weight) + arr[upper] * weight);
+}
+//# sourceMappingURL=tool-call-evaluator.js.map

package/dist/eval/tool-call-evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-call-evaluator.js","sourceRoot":"","sources":["../../src/eval/tool-call-evaluator.ts"],"names":[],"mappings":";;;AAKA,0CAA+C;AAO/C;;;;;;;;;;GAUG;AACH,MAAa,iBAAiB;IAC5B,sDAAsD;IAC9C,OAAO,GAAqB,EAAE,CAAC;IAEvC,oDAAoD;IAC5C,eAAe,GAAwB,IAAI,GAAG,EAAE,CAAC;IAEzD,8CAA8C;IACtC,wBAAwB,GAAwB,IAAI,GAAG,EAAE,CAAC;IAElE,0EAA0E;IAE1E,UAAU,GAEM,SAAS,CAAC;IAE1B,QAAQ,GAAmD,SAAS,CAAC;IAErE,UAAU,GAAoD,SAAS,CAAC;IAExE,WAAW,CAAC,QAAgB,EAAE,IAA6B,EAAE,UAAmB;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAExD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,UAAU;YACV,QAAQ;YACR,IAAI;YACJ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,KAAK,EAAE,mBAAmB;YACnC,aAAa,EAAE,OAAO,GAAG,CAAC;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,SAAS,CAAC,QAAgB,EAAE,MAAc,EAAE,UAAmB;QAC7D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,mBAAmB;QAE1D,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,MAAM,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,CAAC,SAAS;YACd,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE;gBAClC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,MAAM,CAAC,SAAS,GAAG,qBAAa,CAAC,OAAO,CAAC;QACzC,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;IACtC,CAAC;IAED,WAAW,CAAC,QAAgB,EAAE,KAAa,EAAE,UAAmB;QAC9D,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,MAAM,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,CAAC,SAAS;YACd,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE;gBAClC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC;QAE/B,uDAAuD;QACvD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAEhD,8BAA8B;QAC9B,IAAI,MAAM,CAAC,SAAS,KAAK,qBAAa,CAAC,YAAY,EAAE,CAAC;YACpD,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAC/B,QAAQ,EACR,CAAC,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CACvD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,SAAS,GAA6C,SAAS,CAAC;IAChE,aAAa,GAA4C,SAAS,CAAC;IACnE,aAAa,GAA4C,SAAS,CAAC;IACnE,QAAQ,GAA4C,SAAS,CAAC;IAE9D,0EAA0E;IAE1E;;OAEG;IACH,UAAU;QACR,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY;QACV,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,aAAa,GAAG,UAAU,GAAG,cAAc,CAAC;QAElD,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QAC/D,MAAM,YAAY,GAChB,YAAY,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC;YAC3E,CAAC,CAAC,CAAC,CAAC;QAER,MAAM,mBAAmB,GAAG,QAAQ,CAAC,MAAM,CACzC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,mBAAmB,EACnC,CAAC,CACF,CAAC;QAEF,OAAO;YACL,UAAU;YACV,cAAc;YACd,aAAa;YACb,kBAAkB,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YACpE,YAAY;YACZ,eAAe,EAAE,QAAQ,CAAC,MAAM;YAChC,mBAAmB;YACnB,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;SAC9D,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,IAAI,SAAS,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,8DAA8D,CAAC;QACxE,CAAC;QAED,MAAM,WAAW,GAAG,CAAC,SAAS,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEpE,IAAI,MAAM,GAAG,mCAAmC,CAAC;QACjD,MAAM,IAAI,gBAAgB,CAAC;QAC3B,MAAM,IAAI,sBAAsB,CAAC;QACjC,MAAM,IAAI,sBAAsB,CAAC;QACjC,MAAM,IAAI,mBAAmB,SAAS,CAAC,UAAU,MAAM,CAAC;QACxD,MAAM,IAAI,iBAAiB,SAAS,CAAC,cAAc,MAAM,CAAC;QAC1D,MAAM,IAAI,gBAAgB,SAAS,CAAC,aAAa,MAAM,CAAC;QACxD,MAAM,IAAI,oBAAoB,WAAW,OAAO,CAAC;QACjD,MAAM,IAAI,mBAAmB,SAAS,CAAC,YAAY,QAAQ,CAAC;QAC5D,MAAM,IAAI,yBAAyB,SAAS,CAAC,eAAe,MAAM,CAAC;QACnE,MAAM,IAAI,6BAA6B,SAAS,CAAC,mBAAmB,QAAQ,CAAC;QAE7E,MAAM,IAAI,2BAA2B,CAAC;QACtC,MAAM,IAAI,sFAAsF,CAAC;QACjG,MAAM,IAAI,qFAAqF,CAAC;QAEhG,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,IAAI,OAAO,IAAI,CAAC,QAAQ,QAAQ,IAAI,CAAC,UAAU,MAAM,EAAE,OAAO,IAAI,CAAC,YAAY,QAAQ,IAAI,CAAC,YAAY,QAAQ,IAAI,CAAC,YAAY,QAAQ,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,mBAAmB,MAAM,CAAC;QAChN,CAAC;QAED,MAAM,IAAI,6BAA6B,CAAC;QAExC,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC/D,MAAM,IAAI,OAAO,IAAI,CAAC,QAAQ,MAAM,CAAC;YACrC,MAAM,IAAI,0BAA0B,CAAC;YACrC,MAAM,IAAI,0BAA0B,CAAC;YACrC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACnE,MAAM,IAAI,OAAO,IAAI,QAAQ,KAAK,MAAM,CAAC;YAC3C,CAAC;YACD,MAAM,IAAI,IAAI,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,sBAAsB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC;QAE9D,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,CAAC;IACxC,CAAC;IAED,0EAA0E;IAE1E;;;;;;;OAOG;IACK,UAAU,CAChB,QAAgB,EAChB,UAAmB;QAEnB,6DAA6D;QAC7D,gDAAgD;QAChD,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClD,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;oBAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,oDAAoD;QACpD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;gBACtE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACK,gBAAgB,CAAC,KAAa;QACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC/D,IAAI,KAAK,EAAE,CAAC;YACV,kEAAkE;YAClE,uDAAuD;YACvD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAmB,CAAC;YACrD,IAAI,MAAM,CAAC,MAAM,CAAC,qBAAa,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,qBAAa,CAAC,iBAAiB,CAAC,CAAC,iCAAiC;IAC3E,CAAC;IAED;;OAEG;IACK,mBAAmB;QACzB,MAAM,MAAM,GAAG,IAAI,GAAG,EAA4B,CAAC;QACnD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACxD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,KAAK,GAAoB,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,SAAS,GAAG,SAAS;iBACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC;iBACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC,CAAC;YAE5B,qBAAqB;YACrB,MAAM,SAAS,GAA2B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,CAAC,CAAC,SAAS,IAAI,qBAAa,CAAC,iBAAiB,CAAC;gBAC5D,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC/C,CAAC;YAED,KAAK,CAAC,IAAI,CAAC;gBACT,QAAQ;gBACR,UAAU,EAAE,SAAS,CAAC,MAAM;gBAC5B,YAAY,EAAE,SAAS,CAAC,MAAM;gBAC9B,YAAY,EAAE,QAAQ,CAAC,MAAM;gBAC7B,WAAW,EACT,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChE,YAAY,EACV,SAAS,CAAC,MAAM,GAAG,CAAC;oBAClB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;oBACrE,CAAC,CAAC,CAAC;gBACP,YAAY,EAAE,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC;gBACvC,YAAY,EAAE,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC;gBACvC,UAAU,EACR,SAAS,CAAC,MAAM,GAAG,CAAC;oBAClB,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;oBACpC,CAAC,CAAC,QAAQ,CAAC,MAAM;gBACrB,mBAAmB,EACjB,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAClD,iBAAiB,EAAE,SAAS;gBAC5B,cAAc,EAAE,SAAS;aAC1B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AA7RD,8CA6RC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,KAAK,GAAG,KAAK,CAAC;IAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;AACrE,CAAC"}

package/dist/eval/types.d.ts

@@ -0,0 +1,219 @@
+import type { ToolErrorCode } from "../tools/types";
+/**
+ * Types for the agent evaluation framework.
+ *
+ * Three layers of evaluation:
+ * 1. ToolCallEvaluator  — per-tool-call metrics (observed via AgentHooks)
+ * 2. EvalRunner         — end-to-end test cases with pass/fail criteria
+ * 3. Benchmark          — regression testing across runs
+ */
+/**
+ * A single tool call record captured by the ToolCallEvaluator hook.
+ */
+export interface ToolCallRecord {
+    /** The tool being called. */
+    toolName: string;
+    /** The unique ID assigned by the LLM to this tool call
+     *  (from `response.tool_calls[].id`). Enables exact matching
+     *  when the same tool is called multiple times in one batch. */
+    toolCallId?: string;
+    /** Arguments passed to the tool. */
+    args: Record<string, unknown>;
+    /** ISO-8601 timestamp when execution started. */
+    startTime: string;
+    /** ISO-8601 timestamp when execution ended (set on success or error). */
+    endTime?: string;
+    /** Wall-clock duration in milliseconds. */
+    latencyMs?: number;
+    /** Whether the tool executed successfully. */
+    success: boolean;
+    /** Error message if the tool failed. */
+    error?: string;
+    /** Machine-readable error code (SUCCESS if no error). */
+    errorCode?: ToolErrorCode;
+    /** Attempt number within the current circuit-breaker cycle (1-based). */
+    attemptNumber: number;
+    /** Length of the result content in characters. */
+    resultLength?: number;
+}
+/**
+ * Aggregated per-tool statistics computed from all recorded calls.
+ */
+export interface ToolCallStats {
+    /** Tool name. */
+    toolName: string;
+    /** Total number of call attempts (successes + failures). */
+    totalCalls: number;
+    /** Number of successful calls. */
+    successCount: number;
+    /** Number of failed calls. */
+    failureCount: number;
+    /** Success rate (0–1). */
+    successRate: number;
+    /** Average latency in milliseconds. */
+    avgLatencyMs: number;
+    /** Median (P50) latency in milliseconds. */
+    p50LatencyMs: number;
+    /** P99 latency in milliseconds. */
+    p99LatencyMs: number;
+    /** Average number of retries before success or circuit open. */
+    avgRetries: number;
+    /** How many times the circuit breaker opened for this tool. */
+    circuitBreakerTrips: number;
+    /** Distribution of error codes for failed calls. */
+    errorDistribution: Record<string, number>;
+    /** Raw latency samples (for percentile computation). */
+    latencySamples: number[];
+}
+/**
+ * Overall scorecard produced at the end of a session.
+ */
+export interface ToolCallScorecard {
+    /** Total tool calls across all tools. */
+    totalCalls: number;
+    /** Total successful calls. */
+    totalSuccesses: number;
+    /** Total failed calls. */
+    totalFailures: number;
+    /** Overall success rate (0–1). */
+    overallSuccessRate: number;
+    /** Overall average latency in milliseconds. */
+    avgLatencyMs: number;
+    /** How many distinct tools were called. */
+    uniqueToolsUsed: number;
+    /** How many tools had their circuit breaker open. */
+    circuitBreakerTrips: number;
+    /** Per-tool statistics, sorted by call count descending. */
+    perTool: ToolCallStats[];
+}
+/**
+ * A single evaluation test case.
+ */
+export interface EvalCase {
+    /** Human-readable name for this test case. */
+    name: string;
+    /** The user input / prompt to send to the agent. */
+    input: string;
+    /**
+     * Tools that SHOULD be called during execution.
+     * The test passes if ALL expected tools are called at least once.
+     * Leave empty to skip this check.
+     */
+    expectedTools?: string[];
+    /**
+     * Tools that should NOT be called during execution.
+     * The test fails if ANY forbidden tool is called.
+     */
+    forbiddenTools?: string[];
+    /**
+     * Pattern that the final answer should match (string substring or RegExp).
+     * The test passes if the answer contains this pattern.
+     * Leave undefined to skip output validation.
+     */
+    expectedOutput?: string | RegExp;
+    /**
+     * Maximum iterations for this case (overrides agent default).
+     */
+    maxIterations?: number;
+    /**
+     * Timeout in milliseconds (default: 120_000).
+     */
+    timeoutMs?: number;
+}
+/**
+ * LLM-based quality judgment of an agent's final answer.
+ */
+export interface LLMEvalJudgment {
+    /** Whether the LLM judge considers the answer satisfactory. */
+    passed: boolean;
+    /** 0–100 quality score. */
+    score: number;
+    /** Brief explanation of the judgment. */
+    reasoning: string;
+    /** Specific issues identified (empty if none). */
+    issues: string[];
+}
+/**
+ * Result of running a single evaluation case.
+ */
+export interface EvalResult {
+    /** The case name (from EvalCase). */
+    caseName: string;
+    /** Whether the case passed all checks. */
+    passed: boolean;
+    /** The agent's final answer. */
+    answer: string;
+    /** Tool names called during execution (in order). */
+    toolCalls: string[];
+    /** Number of ReAct/PlanSolve iterations consumed. */
+    iterations: number;
+    /** Wall-clock duration in milliseconds. */
+    durationMs: number;
+    /** Tool call scorecard for this run. */
+    scorecard: ToolCallScorecard;
+    /** Optional LLM-based quality judgment. */
+    llmJudgment?: LLMEvalJudgment;
+    /** Failure reasons (empty if passed). */
+    failures: string[];
+}
+/**
+ * A regression — something that got worse compared to the baseline.
+ */
+export interface Regression {
+    /** What regressed (tool name, metric name, or case name). */
+    target: string;
+    /** The metric that changed. */
+    metric: string;
+    /** Baseline value. */
+    baseline: number | string;
+    /** Current value. */
+    current: number | string;
+    /** Human-readable description. */
+    description: string;
+}
+/**
+ * An improvement — something that got better compared to the baseline.
+ */
+export interface Improvement {
+    /** What improved. */
+    target: string;
+    /** The metric that changed. */
+    metric: string;
+    /** Baseline value. */
+    baseline: number | string;
+    /** Current value. */
+    current: number | string;
+}
+/**
+ * Summary of a benchmark run.
+ */
+export interface BenchmarkSummary {
+    /** Benchmark name. */
+    name: string;
+    /** ISO-8601 timestamp of the run. */
+    timestamp: string;
+    /** Number of cases that passed. */
+    passed: number;
+    /** Total number of cases. */
+    total: number;
+    /** Pass rate (0–1). */
+    passRate: number;
+    /** Average tool calls per case. */
+    avgToolCallsPerCase: number;
+    /** Average latency per case in milliseconds. */
+    avgLatencyMs: number;
+    /** Regressions vs baseline (empty if no baseline). */
+    regressions: Regression[];
+    /** Improvements vs baseline (empty if no baseline). */
+    improvements: Improvement[];
+}
+/**
+ * Full benchmark result including per-case details.
+ */
+export interface BenchmarkResult {
+    /** Summary stats. */
+    summary: BenchmarkSummary;
+    /** Per-case results. */
+    cases: EvalResult[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/eval/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;;;;;;;GAOG;AAIH;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB;;oEAEgE;IAChE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2CAA2C;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,OAAO,EAAE,OAAO,CAAC;IACjB,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B,yEAAyE;IACzE,aAAa,EAAE,MAAM,CAAC;IACtB,kDAAkD;IAClD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,iBAAiB;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,4DAA4D;IAC5D,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,YAAY,EAAE,MAAM,CAAC;IACrB,4CAA4C;IAC5C,YAAY,EAAE,MAAM,CAAC;IACrB,mCAAmC;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,gEAAgE;IAChE,UAAU,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oDAAoD;IACpD,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,wDAAwD;IACxD,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,+CAA+C;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,2CAA2C;IAC3C,eAAe,EAAE,MAAM,CAAC;IACxB,qDAAqD;IACrD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4DAA4D;IAC5D,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B;AAID;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACjC;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+DAA+D;IAC/D,MAAM,EAAE,OAAO,CAAC;IAChB,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,MAAM,EAAE,OAAO,CAAC;IAChB,gCAAgC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,wCAAwC;IACxC,SAAS,EAAE,iBAAiB,CAAC;IAC7B,2CAA2C;IAC3C,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,yCAAyC;IACzC,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAID;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB;IACtB,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,qBAAqB;IACrB,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB;IACtB,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,qBAAqB;IACrB,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,uBAAuB;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,mCAAmC;IACnC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gDAAgD;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,sDAAsD;IACtD,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,uDAAuD;IACvD,YAAY,EAAE,WAAW,EAAE,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qBAAqB;IACrB,OAAO,EAAE,gBAAgB,CAAC;IAC1B,wBAAwB;IACxB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB"}

package/dist/eval/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/eval/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":""}