@aigne/cli 1.48.4-beta.5 → 1.49.0-beta.5

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## [1.49.0-beta.5](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.48.4-beta.5...cli-v1.49.0-beta.5) (2025-09-25)
+
+
+### Features
+
+* **cli:** add new eval command for assessing AI agent performance using custom datasets ([#535](https://github.com/AIGNE-io/aigne-framework/issues/535)) ([9da967b](https://github.com/AIGNE-io/aigne-framework/commit/9da967b01ef9eeee4c5e1242934cf08e14815753))
+* **cli:** add new eval command for assessing AI agent performance using custom datasets ([#535](https://github.com/AIGNE-io/aigne-framework/issues/535)) ([9da967b](https://github.com/AIGNE-io/aigne-framework/commit/9da967b01ef9eeee4c5e1242934cf08e14815753))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/agent-library bumped to 1.21.46-beta.5
+    * @aigne/agentic-memory bumped to 1.0.46-beta.5
+    * @aigne/aigne-hub bumped to 0.10.0-beta.5
+    * @aigne/core bumped to 1.61.0-beta.4
+    * @aigne/default-memory bumped to 1.2.9-beta.5
+    * @aigne/observability-api bumped to 0.10.5-beta
+    * @aigne/openai bumped to 0.16.0-beta.5
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.53-beta.4
+
 ## [1.48.4-beta.5](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.48.4-beta.4...cli-v1.48.4-beta.5) (2025-09-24)
 
 

package/dist/commands/aigne.js

@@ -4,6 +4,7 @@ import { asciiLogo } from "../utils/ascii-logo.js";
 import { createAppCommands } from "./app.js";
 import { createCreateCommand } from "./create.js";
 import { createDeployCommands } from "./deploy.js";
+import { createEvalCommand } from "./eval.js";
 import { createHubCommand } from "./hub.js";
 import { createObservabilityCommand } from "./observe.js";
 import { createRunCommand } from "./run.js";
@@ -15,6 +16,7 @@ export function createAIGNECommand(options) {
         .usage(`${asciiLogo}\n$0 <command> [options]`)
         .version(AIGNE_CLI_VERSION)
         .command(createRunCommand(options))
+        .command(createEvalCommand(options))
         .command(createTestCommand(options))
         .command(createCreateCommand())
         .command(createServeMCPCommand(options))

package/dist/commands/eval.d.ts

@@ -0,0 +1,11 @@
+import type { CommandModule } from "yargs";
+export declare function createEvalCommand({ aigneFilePath, }?: {
+    aigneFilePath?: string;
+}): CommandModule<unknown, {
+    path?: string;
+    agent?: string;
+    dataset?: string;
+    evaluator?: string;
+    concurrency?: number;
+    output?: string;
+}>;

package/dist/commands/eval.js

@@ -0,0 +1,109 @@
+import { isAbsolute, resolve } from "node:path";
+import { exists } from "@aigne/agent-library/utils/fs.js";
+import { z } from "zod";
+import { runEvaluationPipeline } from "../utils/evaluation/core.js";
+import { FileDataset } from "../utils/evaluation/dataset.js";
+import { LLMEvaluator } from "../utils/evaluation/evaluator.js";
+import { ConsoleReporter, CsvReporter } from "../utils/evaluation/reporter.js";
+import { DefaultRunnerWithConcurrency } from "../utils/evaluation/runner.js";
+import { loadAIGNE } from "../utils/load-aigne.js";
+const schema = z.object({
+    path: z.string().optional(),
+    agent: z.string(),
+    dataset: z.string(),
+    evaluator: z.string().optional(),
+    concurrency: z.number().optional(),
+    output: z.string().optional(),
+});
+const getResolvePath = (path) => {
+    return isAbsolute(path) ? path : resolve(process.cwd(), path);
+};
+export function createEvalCommand({ aigneFilePath, } = {}) {
+    return {
+        command: "eval [path] [agent]",
+        describe: "Evaluate AIGNE for the specified path",
+        builder: async (yargs) => {
+            return yargs
+                .positional("path", {
+                type: "string",
+                describe: "Path to the agents directory or URL to an aigne project",
+                default: ".",
+            })
+                .positional("agent", {
+                type: "string",
+                describe: "Name of the agent to evaluate",
+            })
+                .positional("dataset", {
+                type: "string",
+                describe: "Path to the dataset file",
+            })
+                .positional("evaluator", {
+                type: "string",
+                describe: "Name of the evaluator to use",
+            })
+                .positional("output", {
+                alias: "o",
+                type: "string",
+                describe: "Path to the output file",
+            })
+                .positional("concurrency", {
+                type: "number",
+                describe: "Concurrency level",
+                default: 1,
+            })
+                .help(false)
+                .version(false)
+                .strict(false);
+        },
+        handler: async (options) => {
+            const parsedOptions = await schema.safeParseAsync(options);
+            if (!parsedOptions.success) {
+                throw new Error(`Invalid options: ${JSON.stringify(parsedOptions.error.format())}`);
+            }
+            const { agent: entryAgent, dataset: datasetPath, evaluator: evaluatorName, concurrency, } = parsedOptions.data;
+            const path = parsedOptions.data?.path;
+            const aigne = await loadAIGNE({ path: aigneFilePath || path || "." });
+            const resolvedDatasetPath = getResolvePath(datasetPath);
+            if (!(await exists(resolvedDatasetPath))) {
+                throw new Error("Dataset file does not exist");
+            }
+            const { chat } = aigne.cli;
+            const agent = chat && chat.name === entryAgent
+                ? chat
+                : aigne.cli.agents[entryAgent] ||
+                    aigne.agents[entryAgent] ||
+                    aigne.skills[entryAgent] ||
+                    aigne.mcpServer.agents[entryAgent];
+            if (!agent)
+                throw new Error("Entry agent does not exist");
+            agent.model = agent.model ?? aigne.model;
+            let evaluatorAgent;
+            if (evaluatorName) {
+                evaluatorAgent =
+                    aigne.cli.agents[evaluatorName] ||
+                        aigne.agents[evaluatorName] ||
+                        aigne.skills[evaluatorName] ||
+                        aigne.mcpServer.agents[evaluatorName];
+            }
+            if (evaluatorAgent) {
+                evaluatorAgent.model = evaluatorAgent.model ?? aigne.model;
+            }
+            const dataset = new FileDataset(resolvedDatasetPath);
+            const runner = new DefaultRunnerWithConcurrency(agent, aigne);
+            const evaluator = new LLMEvaluator(aigne, evaluatorAgent);
+            const reporters = [new ConsoleReporter()];
+            if (options.output) {
+                const resolvedReporterPath = getResolvePath(options.output);
+                const reporter = new CsvReporter(resolvedReporterPath);
+                reporters.push(reporter);
+            }
+            await runEvaluationPipeline({
+                dataset,
+                runner,
+                evaluators: [evaluator],
+                reporters: reporters,
+                options: { concurrency },
+            });
+        },
+    };
+}

package/dist/utils/evaluation/core.d.ts

@@ -0,0 +1,8 @@
+import type { Dataset, Evaluator, Reporter, Runner, RunOptions } from "./type.js";
+export declare function runEvaluationPipeline(params: {
+    dataset: Dataset;
+    runner: Runner;
+    evaluators: Evaluator[];
+    reporters?: Reporter[];
+    options?: RunOptions;
+}): Promise<void>;

package/dist/utils/evaluation/core.js

@@ -0,0 +1,80 @@
+import { Listr } from "@aigne/listr2";
+import { ConsoleReporter } from "./reporter.js";
+function aggregateSummary(results, duration) {
+    const total = results.length;
+    const scores = results.flatMap((r) => r.evaluations.map((e) => e.score));
+    const successRate = Number((scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0).toFixed(2));
+    const latencies = results.map((r) => r.latency || 0);
+    const totalTokens = results.reduce((a, r) => a + (r.usage?.inputTokens || 0) + (r.usage?.outputTokens || 0), 0);
+    const errors = results.filter((r) => r.error).length;
+    return {
+        total,
+        successRate,
+        duration: Number(duration.toFixed(3)),
+        avgLatency: latencies.reduce((a, b) => a + b, 0) / (latencies.length || 1),
+        maxLatency: Math.max(...latencies, 0),
+        minLatency: Math.min(...latencies, 0),
+        totalTokens,
+        errorCount: errors,
+        scoreDistribution: {
+            min: Math.min(...scores, 0),
+            max: Math.max(...scores, 0),
+            mean: successRate,
+            median: scores.length ? scores.sort((a, b) => a - b)[Math.floor(scores.length / 2)] : 0,
+            variance: scores.length > 1
+                ? scores.reduce((a, b) => a + (b - successRate) ** 2, 0) / scores.length
+                : 0,
+        },
+    };
+}
+export async function runEvaluationPipeline(params) {
+    const now = Date.now();
+    const { dataset, runner, evaluators, reporters = [new ConsoleReporter()], options } = params;
+    const results = [];
+    const task1 = new Listr([
+        {
+            title: "Load dataset",
+            task: async (ctx, _task) => {
+                ctx.items = await dataset.loadWithOptions();
+            },
+        },
+    ]);
+    const { items } = await task1.run();
+    const task2 = new Listr(items.map((item) => {
+        const input = JSON.stringify(item.input);
+        return {
+            title: `Run evaluations with input: ${input.length > 100 ? `${input.slice(0, 100)}...` : input}`,
+            task: async (ctx, task) => {
+                task.output = `Start running agent with input: ${JSON.stringify(item.input, null, 2)}`;
+                const runnerResults = await runner.run([item], options);
+                for await (const result of runnerResults) {
+                    task.output = `Start running evaluation with: ${JSON.stringify({
+                        input: result.input,
+                        output: result.output,
+                        expected: result.expected,
+                    }, null, 2)}`;
+                    const evaluations = [];
+                    for (const evaluator of evaluators) {
+                        const evals = await evaluator.evaluate(result);
+                        evaluations.push(...evals);
+                    }
+                    results.push({ ...result, evaluations });
+                    task.output = `Finish running evaluation`;
+                }
+                ctx.results = results;
+            },
+        };
+    }), {
+        concurrent: options?.concurrency ? Math.min(items.length, options?.concurrency) : false,
+        exitOnError: true,
+        rendererOptions: {
+            collapseSubtasks: false,
+        },
+    });
+    await task2.run();
+    const summary = aggregateSummary(results, (Date.now() - now) / 1000);
+    const report = { dataset: dataset.name, results, summary };
+    for (const reporter of reporters) {
+        await reporter.report(report);
+    }
+}

package/dist/utils/evaluation/dataset.d.ts

@@ -0,0 +1,15 @@
+import type { Dataset, DatasetItem } from "./type.js";
+export declare class FileDataset implements Dataset {
+    name: string;
+    private filePath;
+    constructor(filePath: string);
+    load(): Promise<DatasetItem[]>;
+    loadWithOptions(): Promise<DatasetItem[]>;
+}
+export declare class JsonDataset implements Dataset {
+    name: string;
+    private data;
+    constructor(data: DatasetItem[]);
+    load(): Promise<DatasetItem[]>;
+    loadWithOptions(): Promise<DatasetItem[]>;
+}

package/dist/utils/evaluation/dataset.js

@@ -0,0 +1,61 @@
+import fs from "node:fs/promises";
+import { z } from "zod";
+const recordSchema = z.record(z.any());
+const datasetItemSchema = z.object({
+    id: z.union([z.string(), z.number()]),
+    input: recordSchema,
+    output: recordSchema.optional(),
+    expected: recordSchema.optional(),
+    metadata: recordSchema.optional(),
+    tags: z.array(z.string()).optional(),
+    selected: z.boolean().optional(),
+});
+const datasetSchema = z.array(datasetItemSchema);
+export class FileDataset {
+    name = "file-dataset";
+    filePath;
+    constructor(filePath) {
+        this.filePath = filePath;
+    }
+    async load() {
+        let list;
+        try {
+            list = await fs.readFile(this.filePath, "utf-8");
+        }
+        catch (err) {
+            throw new Error(`Failed to read dataset file: ${err.message}`);
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(list);
+        }
+        catch (err) {
+            throw new Error(`Invalid JSON in dataset file: ${err.message}`);
+        }
+        const result = await datasetSchema.safeParseAsync(parsed);
+        if (!result.success) {
+            throw new Error(`Invalid dataset file: ${JSON.stringify(result.error.format())}`);
+        }
+        return result.data;
+    }
+    async loadWithOptions() {
+        return this.load();
+    }
+}
+export class JsonDataset {
+    name = "json-dataset";
+    data;
+    constructor(data) {
+        this.data = data;
+    }
+    async load() {
+        const result = await datasetSchema.safeParseAsync(this.data);
+        if (!result.success) {
+            throw new Error(`Invalid dataset file: ${JSON.stringify(result.error.format())}`);
+        }
+        return result.data;
+    }
+    async loadWithOptions() {
+        return this.load();
+    }
+}

package/dist/utils/evaluation/evaluator.d.ts

@@ -0,0 +1,9 @@
+import { type Agent, AIGNE } from "@aigne/core";
+import type { DatasetItem, Evaluation, Evaluator } from "./type.js";
+export declare class LLMEvaluator implements Evaluator {
+    private readonly aigne;
+    private readonly agent;
+    name: string;
+    constructor(aigne?: AIGNE, agent?: Agent);
+    evaluate(dataset: DatasetItem): Promise<Evaluation[]>;
+}

package/dist/utils/evaluation/evaluator.js

@@ -0,0 +1,107 @@
+import { AIAgent, AIGNE } from "@aigne/core";
+import { z } from "zod";
+const EVALUATOR_PROMPT = `
+# Instructions
+You are an expert evaluator. Your task is to evaluate the quality of AI-generated responses.
+You will be given:
+1. User Input (Prompt)
+2. AI-generated Output
+3. Expected Output
+
+## Evaluation Methods
+Follow these three correlation checks before assigning a score:
+1. **AI Output vs User Input**: Check if the AI response is relevant to the user input.
+2. **Expected Output vs User Input**: Check if the expected output is relevant to the user input.
+3. **AI Output vs Expected Output**: Check the similarity and alignment between the AI output and the expected output.
+
+Then assign a rating and a score based on the overall quality.
+
+## Criteria
+- **Instruction following**: Does the AI response follow the prompt’s requirements?
+- **Groundedness**: Is the AI response consistent with the expected output and free from irrelevant information?
+- **Completeness**: Does the AI response fully address the task?
+- **Accuracy/Correctness**: Is the AI response factually correct and logically consistent?
+- **Fluency**: Is the AI response clear, structured, and easy to read?
+
+## Rating Rubric (1–5)
+- **5 - Very Good**: Highly relevant, closely aligned with the expected output, accurate, complete, and fluent.  
+- **4 - Good**: Relevant, mostly aligned with the expected output, generally accurate and complete, only minor issues.  
+- **3 - Ok**: Somewhat relevant, partially aligned, or missing important details.  
+- **2 - Bad**: Weak relevance, low similarity with expected output, contains significant errors or omissions.  
+- **1 - Very Bad**: Irrelevant, fails to align with expected output, or completely incorrect.  
+
+## Evaluation Steps
+1. Compare the **semantic content** of AI Output vs Expected Output.  
+  - Ignore JSON keys, object structure, formatting, whitespace, capitalization, and minor punctuation differences.  
+  - If meaning is the same but phrasing differs slightly, assign a higher score (4–5).  
+  - If AI output deviates significantly, assign a lower score (1–2).  
+  - If AI output is empty, assign a lower score (1–2).
+2. Assess against criteria: instruction following, groundedness, completeness, correctness, fluency.  
+3. Assign a 1–5 integer score.  
+4. Provide reasoning, and explicitly justify why this result is **not** a 1/2/3 case (why it avoids being a negative example).  
+
+# Response Output Format
+Your output must strictly follow this three-line format:
+- First line: rating (Very Good, Good, Ok, Bad, Very Bad)
+- Second line: reasoning (must include justification why it is not a 1, 2, or 3 if scored higher)
+- Third line: SCORE: [1-5]
+
+Example:
+Good
+The response follows most instructions and is largely consistent with the expected output, but it omits one detail. This prevents it from being 5. However, it is more accurate and complete than an "Ok" response, so it deserves 4.
+SCORE: 4
+
+# User Inputs and AI-generated Response
+### Input
+{{input}}
+
+### AI-generated Output
+{{output}}
+
+### Expected Output
+{{expectedOutput}}
+`;
+const defaultAgent = AIAgent.from({
+    name: "LLMEvaluator",
+    instructions: EVALUATOR_PROMPT,
+    inputSchema: z.object({
+        input: z.string().describe("The input content to analyze"),
+        output: z.string().describe("The output content to analyze"),
+        expectedOutput: z.string().describe("The expected output content to analyze"),
+    }),
+    outputSchema: z.object({
+        rating: z
+            .enum(["Very Good", "Good", "Ok", "Bad", "Very Bad"])
+            .describe("The rating of the output"),
+        reasoning: z.string().describe("The reasoning of the rating, including justification"),
+        score: z.number().int().min(1).max(5).describe("The score of the output, 1–5, 5 is the best"),
+    }),
+});
+const defaultAigne = new AIGNE();
+export class LLMEvaluator {
+    aigne;
+    agent;
+    name = "llm-as-judge";
+    constructor(aigne = defaultAigne, agent = defaultAgent) {
+        this.aigne = aigne;
+        this.agent = agent;
+    }
+    async evaluate(dataset) {
+        const result = await this.aigne.invoke(this.agent, {
+            input: typeof dataset.input === "string"
+                ? dataset.input
+                : JSON.stringify(dataset.input, null, 2),
+            output: dataset.output ? JSON.stringify(dataset.output, null, 2) : "",
+            expectedOutput: JSON.stringify(dataset.expected, null, 2),
+        }, { returnMetadata: true });
+        return [
+            {
+                name: this.name,
+                rating: result.rating,
+                score: result.score,
+                reason: result.reasoning,
+                usage: result?.$meta?.usage ?? {},
+            },
+        ];
+    }
+}

package/dist/utils/evaluation/reporter.d.ts

@@ -0,0 +1,28 @@
+import type { Report, Reporter } from "./type.js";
+export declare class BaseReporter implements Reporter {
+    name: string;
+    report(_report: Report): Promise<void>;
+    protected formatReport(report: Report): {
+        header: string;
+        key: string;
+        width: number;
+        value: string | number;
+    }[][];
+    protected formatSummary(summary: Report["summary"]): {
+        header: string;
+        key: string;
+        width: number;
+        value: string | number;
+    }[];
+}
+export declare class ConsoleReporter extends BaseReporter {
+    name: string;
+    report(report: Report): Promise<void>;
+}
+export declare class CsvReporter extends BaseReporter {
+    private filePath;
+    name: string;
+    constructor(filePath: string);
+    private writeCsv;
+    report(report: Report): Promise<void>;
+}

package/dist/utils/evaluation/reporter.js

@@ -0,0 +1,202 @@
+import fs from "node:fs";
+import path from "node:path";
+import { format } from "@fast-csv/format";
+import chalk from "chalk";
+import Table from "cli-table3";
+const borderColor = chalk.green;
+const chars = {
+    top: borderColor("─"),
+    "top-mid": borderColor("┬"),
+    "top-left": borderColor("┌"),
+    "top-right": borderColor("┐"),
+    bottom: borderColor("─"),
+    "bottom-mid": borderColor("┴"),
+    "bottom-left": borderColor("└"),
+    "bottom-right": borderColor("┘"),
+    left: borderColor("│"),
+    "left-mid": borderColor("├"),
+    mid: borderColor("─"),
+    "mid-mid": borderColor("┼"),
+    right: borderColor("│"),
+    "right-mid": borderColor("┤"),
+    middle: borderColor("│"),
+};
+export class BaseReporter {
+    name = "base";
+    async report(_report) {
+        throw new Error("Not implemented");
+    }
+    formatReport(report) {
+        return report.results.map((r) => {
+            return [
+                { header: "ID", key: "ID", width: 10, value: r.id },
+                { header: "Input", key: "Input", width: 40, value: JSON.stringify(r.input) },
+                {
+                    header: "Output",
+                    key: "Output",
+                    width: 40,
+                    value: r.output ? JSON.stringify(r.output) : "-",
+                },
+                {
+                    header: "Expected",
+                    key: "Expected",
+                    width: 40,
+                    value: r.expected ? JSON.stringify(r.expected) : "-",
+                },
+                { header: "Error", key: "Error", width: 20, value: r.error ?? "-" },
+                {
+                    header: "Evaluations",
+                    key: "Evaluations",
+                    width: 30,
+                    value: r.evaluations.map((e) => `${e.name}:${e.score}`).join(", "),
+                },
+                {
+                    header: "Rating",
+                    key: "Rating",
+                    width: 20,
+                    value: r.evaluations.map((e) => `${e.rating}`).join(", "),
+                },
+                {
+                    header: "Reason",
+                    key: "Reason",
+                    width: 50,
+                    value: r.evaluations
+                        .map((e) => e.reason ?? "")
+                        .filter(Boolean)
+                        .join(" | "),
+                },
+                {
+                    header: "Latency",
+                    key: "Latency",
+                    width: 15,
+                    value: r.latency ? `${r.latency.toFixed(2)}s` : "-",
+                },
+                {
+                    header: "Tokens",
+                    key: "Tokens",
+                    width: 40,
+                    value: r.usage
+                        ? `${(r.usage.inputTokens || 0) + (r.usage.outputTokens || 0)} (input:${r.usage.inputTokens || 0}, output:${r.usage.outputTokens || 0})`
+                        : "-",
+                },
+            ];
+        });
+    }
+    formatSummary(summary) {
+        return [
+            {
+                header: "Total",
+                key: "Total",
+                width: 10,
+                value: summary.total,
+            },
+            {
+                header: "Success Rate",
+                key: "SuccessRate",
+                width: 15,
+                value: summary.successRate,
+            },
+            {
+                header: "Total Duration",
+                key: "Duration",
+                width: 15,
+                value: summary.duration ? `${summary.duration.toFixed(3)}s` : "-",
+            },
+            {
+                header: "Avg Latency",
+                key: "AvgLatency",
+                width: 15,
+                value: summary.avgLatency ? `${summary.avgLatency.toFixed(3)}s` : "-",
+            },
+            {
+                header: "Total Tokens",
+                key: "TotalTokens",
+                width: 15,
+                value: summary.totalTokens ?? "-",
+            },
+            {
+                header: "Errors",
+                key: "Errors",
+                width: 8,
+                value: summary.errorCount ?? 0,
+            },
+        ];
+    }
+}
+export class ConsoleReporter extends BaseReporter {
+    name = "console";
+    async report(report) {
+        const summary = report.summary;
+        console.log("\n=== 📊 Evaluation Summary ===");
+        const summaryList = this.formatSummary(summary);
+        const summaryTable = new Table({
+            head: summaryList.map((h) => h.header),
+            colWidths: summaryList.map((h) => h.width),
+            chars,
+        });
+        summaryTable.push(summaryList.map((h) => h.value));
+        console.log(summaryTable.toString());
+        const list = this.formatReport(report);
+        if (!list.length)
+            return;
+        console.log("\n=== 📋 Detailed Results ===");
+        const head = list[0]?.map((h) => h.header) ?? [];
+        const colWidths = list[0]?.map((h) => h.width) ?? [];
+        const detailTable = new Table({
+            head,
+            colWidths,
+            wordWrap: true,
+            chars,
+        });
+        for (const r of list) {
+            detailTable.push(r.map((h) => h.value));
+        }
+        console.log(detailTable.toString());
+        const failed = report.results.filter((r) => r.error);
+        if (failed.length) {
+            console.log(chalk.red("\n=== ❌ Failed Cases ==="));
+            for (const f of failed) {
+                console.log(`#${f.id} Input: ${JSON.stringify(f.input)}\n  Expected: ${f.expected ? JSON.stringify(f.expected) : "-"}\n  Output: ${f.output ? JSON.stringify(f.output) : "-"}\n  Error: ${f.error ?? "-"}\n`);
+            }
+        }
+    }
+}
+export class CsvReporter extends BaseReporter {
+    filePath;
+    name = "csv";
+    constructor(filePath) {
+        super();
+        this.filePath = filePath;
+    }
+    async writeCsv(filePath, data, headers) {
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        const stream = format({ headers });
+        const writeStream = fs.createWriteStream(filePath);
+        stream.pipe(writeStream);
+        for (const row of data) {
+            stream.write(row);
+        }
+        stream.end();
+        await new Promise((resolve, reject) => {
+            writeStream.on("finish", resolve);
+            writeStream.on("error", reject);
+        });
+    }
+    async report(report) {
+        const list = this.formatReport(report);
+        if (list.length > 0) {
+            const resultsHeaders = list[0]?.map((h) => h.header) ?? [];
+            const resultsRows = list.map((row) => {
+                const record = {};
+                for (const item of row) {
+                    record[item.header] = item.value;
+                }
+                return record;
+            });
+            const ext = path.extname(this.filePath).toLowerCase();
+            const outputFile = ext ? this.filePath : `${this.filePath}.csv`;
+            await this.writeCsv(outputFile, resultsRows, resultsHeaders);
+            console.log(`✅ Results CSV saved to ${outputFile}`);
+        }
+    }
+}

package/dist/utils/evaluation/runner.d.ts

@@ -0,0 +1,16 @@
+import { type Agent, AIGNE } from "@aigne/core";
+import type { DatasetItem, Runner, RunOptions, RunResult } from "./type.js";
+export declare class DefaultRunner implements Runner {
+    private agent;
+    private aigne;
+    name: string;
+    constructor(agent: Agent, aigne?: AIGNE);
+    run(dataset: DatasetItem[], options?: RunOptions): AsyncGenerator<RunResult>;
+}
+export declare class DefaultRunnerWithConcurrency implements Runner {
+    private agent;
+    private aigne;
+    name: string;
+    constructor(agent: Agent, aigne?: AIGNE);
+    run(dataset: DatasetItem[], options?: RunOptions): AsyncGenerator<RunResult>;
+}

package/dist/utils/evaluation/runner.js

@@ -0,0 +1,129 @@
+import { AIGNE } from "@aigne/core";
+export class DefaultRunner {
+    agent;
+    aigne;
+    name = "default-runner";
+    constructor(agent, aigne = new AIGNE()) {
+        this.agent = agent;
+        this.aigne = aigne;
+    }
+    async *run(dataset, options) {
+        const timeoutMs = options?.timeoutMs ?? 0;
+        const runTask = async (item) => {
+            const start = Date.now();
+            options?.hooks?.onBeforeRun?.(item);
+            try {
+                const execPromise = this.aigne.invoke(this.agent, item.input, { returnMetadata: true });
+                const result = timeoutMs > 0 ? await withTimeout(execPromise, timeoutMs, item.id) : await execPromise;
+                const { $meta, ...output } = result;
+                options?.hooks?.onAfterRun?.(result);
+                return {
+                    ...item,
+                    output,
+                    latency: (Date.now() - start) / 1000,
+                    usage: $meta?.usage || {},
+                };
+            }
+            catch (err) {
+                options?.hooks?.onError?.(err);
+                return {
+                    ...item,
+                    error: err.message,
+                };
+            }
+        };
+        for (const item of dataset) {
+            yield await runTask(item);
+        }
+    }
+}
+function withTimeout(promise, ms, id) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            reject(new Error(`Task ${id} timed out after ${ms}ms`));
+        }, ms);
+        promise
+            .then((res) => {
+            clearTimeout(timer);
+            resolve(res);
+        })
+            .catch((err) => {
+            clearTimeout(timer);
+            reject(err);
+        });
+    });
+}
+export class DefaultRunnerWithConcurrency {
+    agent;
+    aigne;
+    name = "default-runner-with-concurrency";
+    constructor(agent, aigne = new AIGNE()) {
+        this.agent = agent;
+        this.aigne = aigne;
+    }
+    async *run(dataset, options) {
+        const concurrency = options?.concurrency ?? 1;
+        const timeoutMs = options?.timeoutMs ?? 0;
+        let index = 0;
+        const yieldQueue = [];
+        let waitingResolve = null;
+        let activeWorkers = 0;
+        const runTask = async (item) => {
+            const start = Date.now();
+            options?.hooks?.onBeforeRun?.(item);
+            try {
+                const execPromise = this.aigne.invoke(this.agent, item.input, { returnMetadata: true });
+                const result = timeoutMs > 0 ? await withTimeout(execPromise, timeoutMs, item.id) : await execPromise;
+                const { $meta, ...output } = result;
+                options?.hooks?.onAfterRun?.(result);
+                return {
+                    ...item,
+                    output,
+                    latency: (Date.now() - start) / 1000,
+                    usage: $meta?.usage || {},
+                };
+            }
+            catch (err) {
+                options?.hooks?.onError?.(err);
+                return {
+                    ...item,
+                    error: err.message,
+                };
+            }
+        };
+        const worker = async () => {
+            activeWorkers++;
+            try {
+                while (true) {
+                    const currentIndex = index++;
+                    if (currentIndex >= dataset.length)
+                        break;
+                    const item = dataset[currentIndex];
+                    if (!item)
+                        continue;
+                    const res = await runTask(item);
+                    yieldQueue.push(res);
+                    waitingResolve?.();
+                }
+            }
+            finally {
+                activeWorkers--;
+                waitingResolve?.();
+            }
+        };
+        Array.from({ length: Math.min(concurrency, dataset.length) }, () => worker());
+        while (yieldQueue.length > 0 || activeWorkers > 0) {
+            if (yieldQueue.length > 0) {
+                const result = yieldQueue.shift();
+                if (result)
+                    yield result;
+            }
+            else {
+                await new Promise((resolve) => {
+                    waitingResolve = resolve;
+                });
+                waitingResolve = null;
+            }
+        }
+    }
+}

package/dist/utils/evaluation/type.d.ts

@@ -0,0 +1,68 @@
+export interface DatasetItem {
+    id: string | number;
+    input: Record<string, any>;
+    output?: Record<string, any>;
+    expected?: Record<string, any>;
+    metadata?: Record<string, any>;
+    tags?: string[];
+    selected?: boolean;
+}
+export interface Dataset {
+    name: string;
+    load(): Promise<DatasetItem[]>;
+    loadWithOptions(options?: {
+        filter?: (item: DatasetItem) => boolean;
+        limit?: number;
+    }): Promise<DatasetItem[]>;
+}
+export interface RunOptions {
+    timeoutMs?: number;
+    concurrency?: number;
+    hooks?: {
+        onBeforeRun?: (item: DatasetItem) => void;
+        onAfterRun?: (result: RunResult) => void;
+        onError?: (err: Error) => void;
+    };
+}
+export interface RunResult extends DatasetItem {
+    error?: string;
+    latency?: number;
+    usage?: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+}
+export interface Runner {
+    name: string;
+    run(dataset: DatasetItem[], options?: RunOptions): AsyncGenerator<RunResult>;
+}
+export interface Evaluation {
+    name: string;
+    score: number;
+    reason?: string;
+    [key: string]: any;
+}
+export interface Evaluator {
+    name: string;
+    evaluate(result: RunResult): Promise<Evaluation[]>;
+}
+export interface EvaluationSummary {
+    total: number;
+    successRate: number;
+    avgLatency?: number;
+    totalTokens?: number;
+    errorCount?: number;
+    [key: string]: any;
+}
+export interface EvaluationResult extends RunResult {
+    evaluations: Evaluation[];
+}
+export interface Report {
+    dataset: string;
+    results: EvaluationResult[];
+    summary: EvaluationSummary;
+}
+export interface Reporter {
+    name: string;
+    report(report: Report): Promise<void>;
+}

package/dist/utils/evaluation/type.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/cli",
-  "version": "1.48.4-beta.5",
+  "version": "1.49.0-beta.5",
   "description": "Your command center for agent development",
   "publishConfig": {
     "access": "public"
@@ -47,6 +47,7 @@
   "dependencies": {
     "@aigne/listr2": "^1.0.10",
     "@aigne/marked-terminal": "^7.3.2",
+    "@fast-csv/format": "^5.0.5",
     "@inquirer/core": "^10.2.2",
     "@inquirer/figures": "^1.0.13",
     "@inquirer/prompts": "^7.8.6",
@@ -81,13 +82,13 @@
     "yargs": "^18.0.0",
     "yoctocolors-cjs": "^2.1.3",
     "zod": "^3.25.67",
-    "@aigne/agent-library": "^1.21.46-beta.4",
-    "@aigne/agentic-memory": "^1.0.46-beta.4",
-    "@aigne/aigne-hub": "^0.10.0-beta.4",
-    "@aigne/core": "^1.61.0-beta.3",
-    "@aigne/default-memory": "^1.2.9-beta.4",
-    "@aigne/openai": "^0.16.0-beta.4",
-    "@aigne/observability-api": "^0.10.4"
+    "@aigne/agent-library": "^1.21.46-beta.5",
+    "@aigne/agentic-memory": "^1.0.46-beta.5",
+    "@aigne/aigne-hub": "^0.10.0-beta.5",
+    "@aigne/core": "^1.61.0-beta.4",
+    "@aigne/default-memory": "^1.2.9-beta.5",
+    "@aigne/observability-api": "^0.10.5-beta",
+    "@aigne/openai": "^0.16.0-beta.5"
   },
   "devDependencies": {
     "@inquirer/testing": "^2.1.50",
@@ -104,7 +105,7 @@
     "rimraf": "^6.0.1",
     "typescript": "^5.9.2",
     "ufo": "^1.6.1",
-    "@aigne/test-utils": "^0.5.53-beta.3"
+    "@aigne/test-utils": "^0.5.53-beta.4"
   },
   "scripts": {
     "lint": "tsc --noEmit",