@aigne/cli 1.48.4-beta.5 → 1.49.0-beta.10

package/CHANGELOG.md

@@ -1,5 +1,121 @@
 # Changelog
 
+## [1.49.0-beta.10](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.49.0-beta.9...cli-v1.49.0-beta.10) (2025-09-27)
+
+
+### Bug Fixes
+
+* **cli:** resolve input schema not working for run command ([#557](https://github.com/AIGNE-io/aigne-framework/issues/557)) ([6fa12b3](https://github.com/AIGNE-io/aigne-framework/commit/6fa12b3e068aaec28cf204c3a3f7c471bd2827ae))
+
+## [1.49.0-beta.9](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.49.0-beta.8...cli-v1.49.0-beta.9) (2025-09-26)
+
+
+### Features
+
+* **cli:** add `--beta` `--target-version` `--force` options for `upgrade` command ([#555](https://github.com/AIGNE-io/aigne-framework/issues/555)) ([f9f0471](https://github.com/AIGNE-io/aigne-framework/commit/f9f04719020cca00bc3adbe8169c42422201df49))
+
+
+### Bug Fixes
+
+* **blocklet:** observability blocklet start failure ([#554](https://github.com/AIGNE-io/aigne-framework/issues/554)) ([8431d4d](https://github.com/AIGNE-io/aigne-framework/commit/8431d4d89a4b96f735f23e774e9545bbe1fd811c))
+* **blocklet:** observability blocklet start failure ([#554](https://github.com/AIGNE-io/aigne-framework/issues/554)) ([8431d4d](https://github.com/AIGNE-io/aigne-framework/commit/8431d4d89a4b96f735f23e774e9545bbe1fd811c))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/agent-library bumped to 1.21.46-beta.9
+    * @aigne/agentic-memory bumped to 1.0.46-beta.9
+    * @aigne/aigne-hub bumped to 0.10.0-beta.9
+    * @aigne/core bumped to 1.61.0-beta.8
+    * @aigne/default-memory bumped to 1.2.9-beta.9
+    * @aigne/observability-api bumped to 0.11.0-beta.1
+    * @aigne/openai bumped to 0.16.0-beta.9
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.53-beta.8
+
+## [1.49.0-beta.8](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.49.0-beta.7...cli-v1.49.0-beta.8) (2025-09-26)
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/agent-library bumped to 1.21.46-beta.8
+    * @aigne/agentic-memory bumped to 1.0.46-beta.8
+    * @aigne/aigne-hub bumped to 0.10.0-beta.8
+    * @aigne/core bumped to 1.61.0-beta.7
+    * @aigne/default-memory bumped to 1.2.9-beta.8
+    * @aigne/openai bumped to 0.16.0-beta.8
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.53-beta.7
+
+## [1.49.0-beta.7](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.49.0-beta.6...cli-v1.49.0-beta.7) (2025-09-26)
+
+
+### Bug Fixes
+
+* **cli:** options.prompts should always available ([44ca0a6](https://github.com/AIGNE-io/aigne-framework/commit/44ca0a65d910fbd327b89d2f3dfe38ab7d1be7df))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/agent-library bumped to 1.21.46-beta.7
+    * @aigne/agentic-memory bumped to 1.0.46-beta.7
+    * @aigne/aigne-hub bumped to 0.10.0-beta.7
+    * @aigne/core bumped to 1.61.0-beta.6
+    * @aigne/default-memory bumped to 1.2.9-beta.7
+    * @aigne/openai bumped to 0.16.0-beta.7
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.53-beta.6
+
+## [1.49.0-beta.6](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.49.0-beta.5...cli-v1.49.0-beta.6) (2025-09-25)
+
+
+### Features
+
+* **blocklet:** add token and cost summary for tracing list ([#543](https://github.com/AIGNE-io/aigne-framework/issues/543)) ([5e78919](https://github.com/AIGNE-io/aigne-framework/commit/5e789199b8183cf9c48339ec8163faec001ca64c))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/agent-library bumped to 1.21.46-beta.6
+    * @aigne/agentic-memory bumped to 1.0.46-beta.6
+    * @aigne/aigne-hub bumped to 0.10.0-beta.6
+    * @aigne/core bumped to 1.61.0-beta.5
+    * @aigne/default-memory bumped to 1.2.9-beta.6
+    * @aigne/observability-api bumped to 0.11.0-beta
+    * @aigne/openai bumped to 0.16.0-beta.6
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.53-beta.5
+
+## [1.49.0-beta.5](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.48.4-beta.5...cli-v1.49.0-beta.5) (2025-09-25)
+
+
+### Features
+
+* **cli:** add new eval command for assessing AI agent performance using custom datasets ([#535](https://github.com/AIGNE-io/aigne-framework/issues/535)) ([9da967b](https://github.com/AIGNE-io/aigne-framework/commit/9da967b01ef9eeee4c5e1242934cf08e14815753))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/agent-library bumped to 1.21.46-beta.5
+    * @aigne/agentic-memory bumped to 1.0.46-beta.5
+    * @aigne/aigne-hub bumped to 0.10.0-beta.5
+    * @aigne/core bumped to 1.61.0-beta.4
+    * @aigne/default-memory bumped to 1.2.9-beta.5
+    * @aigne/observability-api bumped to 0.10.5-beta
+    * @aigne/openai bumped to 0.16.0-beta.5
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.53-beta.4
+
 ## [1.48.4-beta.5](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.48.4-beta.4...cli-v1.48.4-beta.5) (2025-09-24)
 
 

package/dist/commands/aigne.js

@@ -4,6 +4,7 @@ import { asciiLogo } from "../utils/ascii-logo.js";
 import { createAppCommands } from "./app.js";
 import { createCreateCommand } from "./create.js";
 import { createDeployCommands } from "./deploy.js";
+import { createEvalCommand } from "./eval.js";
 import { createHubCommand } from "./hub.js";
 import { createObservabilityCommand } from "./observe.js";
 import { createRunCommand } from "./run.js";
@@ -15,6 +16,7 @@ export function createAIGNECommand(options) {
         .usage(`${asciiLogo}\n$0 <command> [options]`)
         .version(AIGNE_CLI_VERSION)
         .command(createRunCommand(options))
+        .command(createEvalCommand(options))
         .command(createTestCommand(options))
         .command(createCreateCommand())
         .command(createServeMCPCommand(options))

package/dist/commands/app.d.ts

@@ -1,27 +1,41 @@
-import type { Agent, AIGNE, Message } from "@aigne/core";
 import type { CommandModule } from "yargs";
+import { type AgentInChildProcess, type LoadAIGNEInChildProcessResult } from "../utils/workers/run-aigne-in-child-process.js";
 import { type AgentRunCommonOptions } from "../utils/yargs.js";
 export declare function createAppCommands(): CommandModule[];
 export declare const agentCommandModule: ({ dir, agent, }: {
     dir: string;
-    agent: Agent;
+    agent: AgentInChildProcess;
 }) => CommandModule<unknown, AgentRunCommonOptions>;
-export declare function invokeCLIAgentFromDir(options: {
-    dir: string;
-    agent: string;
-    input: Message & AgentRunCommonOptions;
-}): Promise<void>;
-export declare function loadApplication({ name, dir, forceUpgrade, }: {
-    name: string;
-    dir?: string;
-    forceUpgrade?: boolean;
-}): Promise<{
-    aigne: AIGNE;
+interface LoadApplicationOptions {
+    packageName: string;
     dir: string;
+    install?: boolean;
+}
+interface LoadApplicationResult {
+    aigne: LoadAIGNEInChildProcessResult;
     version: string;
     isCache?: boolean;
+}
+export declare function loadApplication(options: LoadApplicationOptions & {
+    install: true;
+}): Promise<LoadApplicationResult>;
+export declare function loadApplication(options: LoadApplicationOptions & {
+    install?: false;
+}): Promise<LoadApplicationResult | null>;
+export declare function installApp({ dir, packageName, beta, version, }: {
+    dir: string;
+    packageName: string;
+    beta?: boolean;
+    version?: string;
+}): Promise<{
+    url: string;
+    version: string;
 }>;
-export declare function getNpmTgzInfo(name: string): Promise<{
+export declare function getNpmTgzInfo(name: string, { version, beta }?: {
+    version?: string;
+    beta?: boolean;
+}): Promise<{
     version: string;
-    url: any;
+    url: string;
 }>;
+export {};

package/dist/commands/app.js

@@ -1,16 +1,14 @@
-import assert from "node:assert";
 import { spawn } from "node:child_process";
 import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { logger } from "@aigne/core/utils/logger.js";
+import { jsonSchemaToZod } from "@aigne/json-schema-to-zod";
 import { Listr, PRESET_TIMER } from "@aigne/listr2";
 import { joinURL } from "ufo";
 import { downloadAndExtract } from "../utils/download.js";
-import { loadAIGNE } from "../utils/load-aigne.js";
-import { runAgentWithAIGNE } from "../utils/run-with-aigne.js";
-import { safeLoadAIGNE } from "../utils/workers/load-aigne.js";
-import { parseAgentInput, withAgentInputSchema, } from "../utils/yargs.js";
+import { runAIGNEInChildProcess, } from "../utils/workers/run-aigne-in-child-process.js";
+import { withAgentInputSchema } from "../utils/yargs.js";
 import { serveMCPServerFromDir } from "./serve-mcp.js";
 const NPM_PACKAGE_CACHE_TIME_MS = 1000 * 60 * 60 * 24; // 1 day
 /**
@@ -23,11 +21,13 @@ function shouldUseBetaApps() {
 const builtinApps = [
     {
         name: "doc-smith",
+        packageName: "@aigne/doc-smith",
         describe: "Generate and maintain project docs — powered by agents.",
         aliases: ["docsmith", "doc"],
     },
     {
         name: "web-smith",
+        packageName: "@aigne/web-smith",
         describe: "Generate and maintain project website pages — powered by agents.",
         aliases: ["websmith", "web"],
     },
@@ -38,30 +38,28 @@ export function createAppCommands() {
         describe: app.describe,
         aliases: app.aliases,
         builder: async (yargs) => {
-            const { aigne, dir, version, isCache } = await loadApplication({
-                name: app.name,
-            });
-            yargs
-                .option("model", {
-                type: "string",
-                description: "Model to use for the application, example: openai:gpt-4.1 or google:gemini-2.5-flash",
-            })
-                .command(serveMcpCommandModule({ name: app.name, dir }))
-                .command(upgradeCommandModule({
-                name: app.name,
+            const dir = join(homedir(), ".aigne", "registry.npmjs.org", app.packageName);
+            const { aigne, version } = await loadApplication({
                 dir,
-                isLatest: !isCache,
-                version,
-            }));
-            if (aigne.cli.chat) {
+                packageName: app.packageName,
+                install: true,
+            });
+            if (aigne.cli?.chat) {
                 yargs.command({
                     ...agentCommandModule({ dir, agent: aigne.cli.chat }),
                     command: "$0",
                 });
             }
-            for (const agent of aigne.cli.agents) {
+            for (const agent of aigne.cli?.agents ?? []) {
                 yargs.command(agentCommandModule({ dir, agent }));
             }
+            yargs
+                .option("model", {
+                type: "string",
+                description: "Model to use for the application, example: openai:gpt-4.1 or google:gemini-2.5-flash",
+            })
+                .command(serveMcpCommandModule({ name: app.name, dir }))
+                .command(upgradeCommandModule({ packageName: app.packageName, dir }));
             yargs.version(`${app.name} v${version}`).alias("version", "v");
             return yargs.demandCommand();
         },
@@ -92,18 +90,39 @@ const serveMcpCommandModule = ({ name, dir, }) => ({
         await serveMCPServerFromDir({ ...options, dir });
     },
 });
-const upgradeCommandModule = ({ name, dir, isLatest, version, }) => ({
+const upgradeCommandModule = ({ packageName, dir, }) => ({
     command: "upgrade",
-    describe: `Upgrade ${name} to the latest version`,
-    handler: async () => {
-        if (!isLatest) {
-            const result = await loadApplication({ name, dir, forceUpgrade: true });
-            if (version !== result.version) {
-                console.log(`\n✅ Upgraded ${name} to version ${result.version}`);
-                return;
-            }
+    describe: `Upgrade ${packageName} to the latest version`,
+    builder: (argv) => {
+        return argv
+            .option("beta", {
+            type: "boolean",
+            describe: "Use beta versions if available",
+        })
+            .option("target-version", {
+            type: "string",
+            describe: "Specify a version to upgrade to (default is latest)",
+            alias: ["to", "target"],
+        })
+            .option("force", {
+            type: "boolean",
+            describe: "Force upgrade even if already at latest version",
+            default: false,
+        });
+    },
+    handler: async ({ beta, targetVersion, force }) => {
+        beta ??= shouldUseBetaApps();
+        let app = await loadApplication({ packageName, dir });
+        const npm = await getNpmTgzInfo(packageName, { beta, version: targetVersion });
+        if (!app || force || npm.version !== app.version) {
+            if (force)
+                await rm(dir, { force: true, recursive: true });
+            await installApp({ packageName, dir, beta, version: targetVersion });
+            app = await loadApplication({ dir, packageName, install: true });
+            console.log(`\n✅ Upgraded ${packageName} to version ${app.version}`);
+            return;
         }
-        console.log(`\n✅ ${name} is already at the latest version (${version})`);
+        console.log(`\n✅ ${packageName} is already at the latest version (${app.version})`);
     },
 });
 export const agentCommandModule = ({ dir, agent, }) => {
@@ -111,76 +130,76 @@ export const agentCommandModule = ({ dir, agent, }) => {
         command: agent.name,
         aliases: agent.alias || [],
         describe: agent.description || "",
-        builder: async (yargs) => withAgentInputSchema(yargs, agent),
+        builder: async (yargs) => {
+            return withAgentInputSchema(yargs, { inputSchema: jsonSchemaToZod(agent.inputSchema) });
+        },
         handler: async (options) => {
             if (options.logLevel)
                 logger.level = options.logLevel;
-            await invokeCLIAgentFromDir({ dir, agent: agent.name, input: options });
+            await runAIGNEInChildProcess("invokeCLIAgentFromDir", {
+                dir,
+                agent: agent.name,
+                input: options,
+            });
             process.exit(0);
         },
     };
 };
-export async function invokeCLIAgentFromDir(options) {
-    const aigne = await loadAIGNE({
-        path: options.dir,
-        modelOptions: options.input,
-    });
-    try {
-        const { chat } = aigne.cli;
-        const agent = chat && chat.name === options.agent
-            ? chat
-            : aigne.cli.agents[options.agent] ||
-                aigne.agents[options.agent] ||
-                aigne.skills[options.agent] ||
-                aigne.mcpServer.agents[options.agent];
-        assert(agent, `Agent ${options.agent} not found in ${options.dir}`);
-        const input = await parseAgentInput(options.input, agent);
-        await runAgentWithAIGNE(aigne, agent, {
-            ...options.input,
-            input,
-            chat: agent === chat || options.input.chat,
-        });
-    }
-    finally {
-        await aigne.shutdown();
-    }
-}
-export async function loadApplication({ name, dir, forceUpgrade = false, }) {
-    name = `@aigne/${name}`;
-    dir ??= join(homedir(), ".aigne", "registry.npmjs.org", name);
-    let check = forceUpgrade ? undefined : await isInstallationAvailable(dir);
-    if (check?.available) {
-        const aigne = await safeLoadAIGNE(dir).catch((error) => {
-            console.warn(`⚠️ Failed to load ${name}, trying to reinstall:`, error.message);
+export async function loadApplication(options) {
+    const { dir, packageName } = options;
+    const check = await checkInstallation(dir);
+    if (check && !check.expired) {
+        const aigne = await runAIGNEInChildProcess("loadAIGNE", dir).catch((error) => {
+            console.warn(`⚠️ Failed to load ${packageName}, trying to reinstall:`, error.message);
         });
         if (aigne) {
-            return {
-                aigne,
-                dir,
-                version: check.version,
-                isCache: true,
-            };
+            return { aigne, version: check.version, isCache: true };
         }
-        check = undefined;
     }
-    const result = await new Listr([
+    if (!options.install)
+        return null;
+    const result = await installApp({ dir, packageName, beta: check?.version?.includes("beta") });
+    return {
+        aigne: await runAIGNEInChildProcess("loadAIGNE", dir),
+        version: result.version,
+    };
+}
+async function readInstallationMetadata(dir) {
+    return safeParseJSON(await readFile(join(dir, ".aigne-cli.json"), "utf-8").catch(() => "{}"));
+}
+async function writeInstallationMetadata(dir, metadata) {
+    await writeFile(join(dir, ".aigne-cli.json"), JSON.stringify(metadata, null, 2));
+}
+async function checkInstallation(dir, { cacheTimeMs = NPM_PACKAGE_CACHE_TIME_MS } = {}) {
+    const s = await stat(join(dir, "package.json")).catch(() => null);
+    if (!s)
+        return null;
+    const version = safeParseJSON(await readFile(join(dir, "package.json"), "utf-8"))?.version;
+    if (!version)
+        return null;
+    const installedAt = (await readInstallationMetadata(dir))?.installedAt;
+    if (!installedAt)
+        return null;
+    const now = Date.now();
+    const expired = now - installedAt > cacheTimeMs;
+    return { version, expired };
+}
+export async function installApp({ dir, packageName, beta, version, }) {
+    return await new Listr([
         {
-            title: `Fetching ${name} metadata`,
+            title: `Fetching ${packageName} metadata`,
             task: async (ctx, task) => {
-                const info = await getNpmTgzInfo(name);
-                Object.assign(ctx, info);
-                const useBeta = shouldUseBetaApps();
-                if (useBeta && ctx.version.includes("beta")) {
-                    task.title = `Fetching ${name} metadata (using beta version)`;
+                if (beta) {
+                    task.title = `Fetching ${packageName} metadata (using beta version)`;
                 }
+                const info = await getNpmTgzInfo(packageName, { beta, version });
+                Object.assign(ctx, info);
             },
         },
         {
-            title: `Downloading ${name}`,
-            skip: (ctx) => ctx.version === check?.version,
+            title: `Downloading ${packageName}`,
             task: async (ctx, task) => {
-                task.title = `Downloading ${name}(v${ctx.version})`;
-                await rm(dir, { force: true, recursive: true });
+                task.title = `Downloading ${packageName} (v${ctx.version})`;
                 await mkdir(dir, { recursive: true });
                 await downloadAndExtract(ctx.url, dir, { strip: 1 });
             },
@@ -195,6 +214,7 @@ export async function loadApplication({ name, dir, forceUpgrade = false, }) {
                             task.output = last;
                     },
                 });
+                await writeInstallationMetadata(dir, { installedAt: Date.now() });
             },
         },
     ], {
@@ -204,25 +224,6 @@ export async function loadApplication({ name, dir, forceUpgrade = false, }) {
             timer: PRESET_TIMER,
         },
     }).run();
-    return {
-        aigne: await safeLoadAIGNE(dir),
-        dir,
-        version: result.version,
-    };
-}
-async function isInstallationAvailable(dir, { cacheTimeMs = NPM_PACKAGE_CACHE_TIME_MS } = {}) {
-    const s = await stat(join(dir, "package.json")).catch(() => null);
-    if (!s)
-        return null;
-    const version = safeParseJSON(await readFile(join(dir, "package.json"), "utf-8"))?.version;
-    if (!version)
-        return null;
-    const installedAt = safeParseJSON(await readFile(join(dir, ".aigne-cli.json"), "utf-8").catch(() => "{}"))?.installedAt;
-    if (!installedAt)
-        return null;
-    const now = Date.now();
-    const available = installedAt ? now - installedAt < cacheTimeMs : false;
-    return { version, available };
 }
 async function installDependencies(dir, { log } = {}) {
     await new Promise((resolve, reject) => {
@@ -250,16 +251,20 @@ async function installDependencies(dir, { log } = {}) {
             }
         });
     });
-    await writeFile(join(dir, ".aigne-cli.json"), JSON.stringify({ installedAt: Date.now() }, null, 2));
 }
-export async function getNpmTgzInfo(name) {
+export async function getNpmTgzInfo(name, { version, beta } = {}) {
     const res = await fetch(joinURL("https://registry.npmjs.org", name));
     if (!res.ok)
         throw new Error(`Failed to fetch package info for ${name}: ${res.statusText}`);
     const data = await res.json();
-    const useBeta = shouldUseBetaApps();
     let targetVersion;
-    if (useBeta && data["dist-tags"].beta) {
+    if (version) {
+        if (!data.versions[version]) {
+            throw new Error(`Version ${version} of package ${name} not found`);
+        }
+        targetVersion = version;
+    }
+    else if (beta && data["dist-tags"].beta) {
         // Use beta version if available and beta flag is set
         targetVersion = data["dist-tags"].beta;
     }

package/dist/commands/eval.d.ts

@@ -0,0 +1,11 @@
+import type { CommandModule } from "yargs";
+export declare function createEvalCommand({ aigneFilePath, }?: {
+    aigneFilePath?: string;
+}): CommandModule<unknown, {
+    path?: string;
+    agent?: string;
+    dataset?: string;
+    evaluator?: string;
+    concurrency?: number;
+    output?: string;
+}>;

package/dist/commands/eval.js

@@ -0,0 +1,109 @@
+import { isAbsolute, resolve } from "node:path";
+import { exists } from "@aigne/agent-library/utils/fs.js";
+import { z } from "zod";
+import { runEvaluationPipeline } from "../utils/evaluation/core.js";
+import { FileDataset } from "../utils/evaluation/dataset.js";
+import { LLMEvaluator } from "../utils/evaluation/evaluator.js";
+import { ConsoleReporter, CsvReporter } from "../utils/evaluation/reporter.js";
+import { DefaultRunnerWithConcurrency } from "../utils/evaluation/runner.js";
+import { loadAIGNE } from "../utils/load-aigne.js";
+const schema = z.object({
+    path: z.string().optional(),
+    agent: z.string(),
+    dataset: z.string(),
+    evaluator: z.string().optional(),
+    concurrency: z.number().optional(),
+    output: z.string().optional(),
+});
+const getResolvePath = (path) => {
+    return isAbsolute(path) ? path : resolve(process.cwd(), path);
+};
+export function createEvalCommand({ aigneFilePath, } = {}) {
+    return {
+        command: "eval [path] [agent]",
+        describe: "Evaluate AIGNE for the specified path",
+        builder: async (yargs) => {
+            return yargs
+                .positional("path", {
+                type: "string",
+                describe: "Path to the agents directory or URL to an aigne project",
+                default: ".",
+            })
+                .positional("agent", {
+                type: "string",
+                describe: "Name of the agent to evaluate",
+            })
+                .positional("dataset", {
+                type: "string",
+                describe: "Path to the dataset file",
+            })
+                .positional("evaluator", {
+                type: "string",
+                describe: "Name of the evaluator to use",
+            })
+                .positional("output", {
+                alias: "o",
+                type: "string",
+                describe: "Path to the output file",
+            })
+                .positional("concurrency", {
+                type: "number",
+                describe: "Concurrency level",
+                default: 1,
+            })
+                .help(false)
+                .version(false)
+                .strict(false);
+        },
+        handler: async (options) => {
+            const parsedOptions = await schema.safeParseAsync(options);
+            if (!parsedOptions.success) {
+                throw new Error(`Invalid options: ${JSON.stringify(parsedOptions.error.format())}`);
+            }
+            const { agent: entryAgent, dataset: datasetPath, evaluator: evaluatorName, concurrency, } = parsedOptions.data;
+            const path = parsedOptions.data?.path;
+            const aigne = await loadAIGNE({ path: aigneFilePath || path || "." });
+            const resolvedDatasetPath = getResolvePath(datasetPath);
+            if (!(await exists(resolvedDatasetPath))) {
+                throw new Error("Dataset file does not exist");
+            }
+            const { chat } = aigne.cli;
+            const agent = chat && chat.name === entryAgent
+                ? chat
+                : aigne.cli.agents[entryAgent] ||
+                    aigne.agents[entryAgent] ||
+                    aigne.skills[entryAgent] ||
+                    aigne.mcpServer.agents[entryAgent];
+            if (!agent)
+                throw new Error("Entry agent does not exist");
+            agent.model = agent.model ?? aigne.model;
+            let evaluatorAgent;
+            if (evaluatorName) {
+                evaluatorAgent =
+                    aigne.cli.agents[evaluatorName] ||
+                        aigne.agents[evaluatorName] ||
+                        aigne.skills[evaluatorName] ||
+                        aigne.mcpServer.agents[evaluatorName];
+            }
+            if (evaluatorAgent) {
+                evaluatorAgent.model = evaluatorAgent.model ?? aigne.model;
+            }
+            const dataset = new FileDataset(resolvedDatasetPath);
+            const runner = new DefaultRunnerWithConcurrency(agent, aigne);
+            const evaluator = new LLMEvaluator(aigne, evaluatorAgent);
+            const reporters = [new ConsoleReporter()];
+            if (options.output) {
+                const resolvedReporterPath = getResolvePath(options.output);
+                const reporter = new CsvReporter(resolvedReporterPath);
+                reporters.push(reporter);
+            }
+            await runEvaluationPipeline({
+                dataset,
+                runner,
+                evaluators: [evaluator],
+                reporters: reporters,
+                options: { concurrency },
+            });
+        },
+    };
+}

package/dist/commands/run.js

@@ -10,6 +10,7 @@ import { isV1Package, toAIGNEPackage } from "../utils/agent-v1.js";
 import { downloadAndExtract } from "../utils/download.js";
 import { loadAIGNE } from "../utils/load-aigne.js";
 import { isUrl } from "../utils/url.js";
+import { serializeAgent } from "../utils/workers/run-aigne-in-child-process.js";
 import { agentCommandModule } from "./app.js";
 export function createRunCommand({ aigneFilePath, } = {}) {
     return {
@@ -41,14 +42,14 @@ export function createRunCommand({ aigneFilePath, } = {}) {
             const subYargs = yargs().scriptName("").usage("aigne run <path> <agent> [...options]");
             if (aigne.cli.chat) {
                 subYargs.command({
-                    ...agentCommandModule({ dir: path, agent: aigne.cli.chat }),
+                    ...agentCommandModule({ dir: path, agent: serializeAgent(aigne.cli.chat) }),
                     command: "$0",
                 });
             }
             // Allow user to run all of agents in the AIGNE instances
             const allAgents = flat(aigne.cli.agents, aigne.agents, aigne.skills, aigne.cli.chat, aigne.mcpServer.agents);
             for (const agent of allAgents) {
-                subYargs.command(agentCommandModule({ dir: path, agent }));
+                subYargs.command(agentCommandModule({ dir: path, agent: serializeAgent(agent) }));
             }
             const argv = process.argv.slice(aigneFilePath ? 3 : 2);
             if (argv[0] === "run")
@@ -102,7 +103,7 @@ async function loadApplication(path) {
     }
     // Load env files in the aigne directory
     config({ path: dir, silent: true });
-    const aigne = await loadAIGNE({ path: dir });
+    const aigne = await loadAIGNE({ path: dir, printTips: false });
     return { aigne, path: dir };
 }
 async function downloadPackage(url, cacheDir) {