@aigne/cli 1.60.0-beta → 1.74.0-beta

package/dist/utils/aigne-hub-user.d.ts

@@ -1,16 +0,0 @@
-export interface UserInfoResult {
-    user: Record<string, any>;
-    enableCredit: boolean;
-    creditBalance: {
-        balance: string;
-        total: string;
-        grantCount: number;
-        pendingCredit: string;
-    } | null;
-    paymentLink: string | null;
-    profileLink: string;
-}
-export declare function getUserInfo({ baseUrl, apiKey, }: {
-    baseUrl: string;
-    apiKey: string;
-}): Promise<UserInfoResult>;

package/dist/utils/aigne-hub-user.js

@@ -1,10 +0,0 @@
-import { fetch } from "@aigne/core/utils/fetch.js";
-import { joinURL } from "ufo";
-export async function getUserInfo({ baseUrl, apiKey, }) {
-    const secureBaseUrl = baseUrl.replace(/^http:/, "https:");
-    const response = await fetch(joinURL(secureBaseUrl, "/api/user/info"), {
-        headers: { Authorization: `Bearer ${apiKey}` },
-    });
-    const data = await response.json();
-    return data;
-}

package/dist/utils/ascii-logo.d.ts

@@ -1 +0,0 @@
-export declare const asciiLogo: string;

package/dist/utils/download.d.ts

@@ -1,3 +0,0 @@
-export declare function downloadAndExtract(url: string, dir: string, options?: {
-    strip?: number;
-}): Promise<void>;

package/dist/utils/download.js

@@ -1,19 +0,0 @@
-import { mkdir } from "node:fs/promises";
-import { Readable } from "node:stream";
-import { finished } from "node:stream/promises";
-import { fetch } from "@aigne/core/utils/fetch.js";
-import { x } from "tar";
-export async function downloadAndExtract(url, dir, options = {}) {
-    const response = await fetch(url);
-    if (!response.body) {
-        throw new Error(`Failed to download package from ${url}: Unexpected to get empty response`);
-    }
-    try {
-        await mkdir(dir, { recursive: true });
-        await finished(Readable.fromWeb(response.body).pipe(x({ C: dir, ...options })));
-    }
-    catch (error) {
-        error.message = `Failed to extract package from ${url}: ${error.message}`;
-        throw error;
-    }
-}

package/dist/utils/evaluation/core.d.ts

@@ -1,8 +0,0 @@
-import type { Dataset, Evaluator, Reporter, Runner, RunOptions } from "./type.js";
-export declare function runEvaluationPipeline(params: {
-    dataset: Dataset;
-    runner: Runner;
-    evaluators: Evaluator[];
-    reporters?: Reporter[];
-    options?: RunOptions;
-}): Promise<void>;

package/dist/utils/evaluation/core.js

@@ -1,83 +0,0 @@
-import { Listr } from "@aigne/listr2";
-import { ConsoleReporter } from "./reporter.js";
-function aggregateSummary(results, duration) {
-    const total = results.length;
-    const scores = results.flatMap((r) => r.evaluations.map((e) => e.score));
-    const successRate = Number((scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0).toFixed(2));
-    const latencies = results.map((r) => r.latency || 0);
-    const totalTokens = results.reduce((a, r) => a + (r.usage?.inputTokens || 0) + (r.usage?.outputTokens || 0), 0);
-    const errors = results.filter((r) => r.error).length;
-    return {
-        total,
-        successRate,
-        duration: Number(duration.toFixed(3)),
-        avgLatency: latencies.reduce((a, b) => a + b, 0) / (latencies.length || 1),
-        maxLatency: Math.max(...latencies, 0),
-        minLatency: Math.min(...latencies, 0),
-        totalTokens,
-        errorCount: errors,
-        scoreDistribution: {
-            min: Math.min(...scores, 0),
-            max: Math.max(...scores, 0),
-            mean: successRate,
-            median: scores.length ? scores.sort((a, b) => a - b)[Math.floor(scores.length / 2)] : 0,
-            variance: scores.length > 1
-                ? scores.reduce((a, b) => a + (b - successRate) ** 2, 0) / scores.length
-                : 0,
-        },
-    };
-}
-export async function runEvaluationPipeline(params) {
-    const now = Date.now();
-    const { dataset, runner, evaluators, reporters = [new ConsoleReporter()], options } = params;
-    const results = [];
-    const task1 = new Listr([
-        {
-            title: "Load dataset",
-            task: async (ctx, _task) => {
-                ctx.items = await dataset.loadWithOptions();
-            },
-        },
-    ], {
-        registerSignalListeners: false,
-    });
-    const { items } = await task1.run();
-    const task2 = new Listr(items.map((item) => {
-        const input = JSON.stringify(item.input);
-        return {
-            title: `Run evaluations with input: ${input.length > 100 ? `${input.slice(0, 100)}...` : input}`,
-            task: async (ctx, task) => {
-                task.output = `Start running agent with input: ${JSON.stringify(item.input, null, 2)}`;
-                const runnerResults = await runner.run([item], options);
-                for await (const result of runnerResults) {
-                    task.output = `Start running evaluation with: ${JSON.stringify({
-                        input: result.input,
-                        output: result.output,
-                        expected: result.expected,
-                    }, null, 2)}`;
-                    const evaluations = [];
-                    for (const evaluator of evaluators) {
-                        const evals = await evaluator.evaluate(result);
-                        evaluations.push(...evals);
-                    }
-                    results.push({ ...result, evaluations });
-                    task.output = `Finish running evaluation`;
-                }
-                ctx.results = results;
-            },
-        };
-    }), {
-        concurrent: options?.concurrency ? Math.min(items.length, options?.concurrency) : false,
-        exitOnError: true,
-        rendererOptions: {
-            collapseSubtasks: false,
-        },
-        registerSignalListeners: false,
-    });
-    await task2.run();
-    const summary = aggregateSummary(results, (Date.now() - now) / 1000);
-    const report = { dataset: dataset.name, results, summary };
-    for (const reporter of reporters) {
-        await reporter.report(report);
-    }
-}

package/dist/utils/evaluation/dataset.d.ts

@@ -1,15 +0,0 @@
-import type { Dataset, DatasetItem } from "./type.js";
-export declare class FileDataset implements Dataset {
-    name: string;
-    private filePath;
-    constructor(filePath: string);
-    load(): Promise<DatasetItem[]>;
-    loadWithOptions(): Promise<DatasetItem[]>;
-}
-export declare class JsonDataset implements Dataset {
-    name: string;
-    private data;
-    constructor(data: DatasetItem[]);
-    load(): Promise<DatasetItem[]>;
-    loadWithOptions(): Promise<DatasetItem[]>;
-}

package/dist/utils/evaluation/dataset.js

@@ -1,61 +0,0 @@
-import fs from "node:fs/promises";
-import { z } from "zod";
-const recordSchema = z.record(z.any());
-const datasetItemSchema = z.object({
-    id: z.union([z.string(), z.number()]),
-    input: recordSchema,
-    output: recordSchema.optional(),
-    expected: recordSchema.optional(),
-    metadata: recordSchema.optional(),
-    tags: z.array(z.string()).optional(),
-    selected: z.boolean().optional(),
-});
-const datasetSchema = z.array(datasetItemSchema);
-export class FileDataset {
-    name = "file-dataset";
-    filePath;
-    constructor(filePath) {
-        this.filePath = filePath;
-    }
-    async load() {
-        let list;
-        try {
-            list = await fs.readFile(this.filePath, "utf-8");
-        }
-        catch (err) {
-            throw new Error(`Failed to read dataset file: ${err.message}`);
-        }
-        let parsed;
-        try {
-            parsed = JSON.parse(list);
-        }
-        catch (err) {
-            throw new Error(`Invalid JSON in dataset file: ${err.message}`);
-        }
-        const result = await datasetSchema.safeParseAsync(parsed);
-        if (!result.success) {
-            throw new Error(`Invalid dataset file: ${JSON.stringify(result.error.format())}`);
-        }
-        return result.data;
-    }
-    async loadWithOptions() {
-        return this.load();
-    }
-}
-export class JsonDataset {
-    name = "json-dataset";
-    data;
-    constructor(data) {
-        this.data = data;
-    }
-    async load() {
-        const result = await datasetSchema.safeParseAsync(this.data);
-        if (!result.success) {
-            throw new Error(`Invalid dataset file: ${JSON.stringify(result.error.format())}`);
-        }
-        return result.data;
-    }
-    async loadWithOptions() {
-        return this.load();
-    }
-}

package/dist/utils/evaluation/evaluator.d.ts

@@ -1,9 +0,0 @@
-import { type Agent, AIGNE } from "@aigne/core";
-import type { DatasetItem, Evaluation, Evaluator } from "./type.js";
-export declare class LLMEvaluator implements Evaluator {
-    private readonly aigne;
-    private readonly agent;
-    name: string;
-    constructor(aigne?: AIGNE, agent?: Agent);
-    evaluate(dataset: DatasetItem): Promise<Evaluation[]>;
-}

package/dist/utils/evaluation/reporter.d.ts

@@ -1,28 +0,0 @@
-import type { Report, Reporter } from "./type.js";
-export declare class BaseReporter implements Reporter {
-    name: string;
-    report(_report: Report): Promise<void>;
-    protected formatReport(report: Report): {
-        header: string;
-        key: string;
-        width: number;
-        value: string | number;
-    }[][];
-    protected formatSummary(summary: Report["summary"]): {
-        header: string;
-        key: string;
-        width: number;
-        value: string | number;
-    }[];
-}
-export declare class ConsoleReporter extends BaseReporter {
-    name: string;
-    report(report: Report): Promise<void>;
-}
-export declare class CsvReporter extends BaseReporter {
-    private filePath;
-    name: string;
-    constructor(filePath: string);
-    private writeCsv;
-    report(report: Report): Promise<void>;
-}

package/dist/utils/evaluation/reporter.js

@@ -1,221 +0,0 @@
-import fs from "node:fs";
-import path from "node:path";
-import { format } from "@fast-csv/format";
-import chalk from "chalk";
-import Table from "cli-table3";
-const borderColor = chalk.green;
-const chars = {
-    top: borderColor("─"),
-    "top-mid": borderColor("┬"),
-    "top-left": borderColor("┌"),
-    "top-right": borderColor("┐"),
-    bottom: borderColor("─"),
-    "bottom-mid": borderColor("┴"),
-    "bottom-left": borderColor("└"),
-    "bottom-right": borderColor("┘"),
-    left: borderColor("│"),
-    "left-mid": borderColor("├"),
-    mid: borderColor("─"),
-    "mid-mid": borderColor("┼"),
-    right: borderColor("│"),
-    "right-mid": borderColor("┤"),
-    middle: borderColor("│"),
-};
-const MAX_CELL_LENGTH = 300;
-function renderPagedTable(list, { maxCols = 6, chars }) {
-    if (!list.length)
-        return [];
-    const tables = [];
-    const head = list[0]?.map((h) => h.header) ?? [];
-    const widths = list[0]?.map((h) => h.width ?? 20) ?? [];
-    for (let i = 0; i < head.length; i += maxCols) {
-        const subHead = head.slice(i, i + maxCols);
-        const subWidths = widths.slice(i, i + maxCols);
-        const table = new Table({
-            head: subHead,
-            colWidths: subWidths,
-            wordWrap: true,
-            chars,
-        });
-        for (const row of list) {
-            table.push(row
-                .slice(i, i + maxCols)
-                .map((h) => String(h.value).length > MAX_CELL_LENGTH
-                ? `${String(h.value).slice(0, MAX_CELL_LENGTH)}...`
-                : h.value));
-        }
-        tables.push(table.toString());
-    }
-    return tables;
-}
-export class BaseReporter {
-    name = "base";
-    async report(_report) {
-        throw new Error("Not implemented");
-    }
-    formatReport(report) {
-        return report.results.map((r) => {
-            return [
-                { header: "ID", key: "ID", width: 10, value: r.id },
-                { header: "Input", key: "Input", width: 40, value: JSON.stringify(r.input) },
-                {
-                    header: "Output",
-                    key: "Output",
-                    width: 40,
-                    value: r.output ? JSON.stringify(r.output) : "-",
-                },
-                {
-                    header: "Expected",
-                    key: "Expected",
-                    width: 40,
-                    value: r.expected ? JSON.stringify(r.expected) : "-",
-                },
-                {
-                    header: "Evaluations",
-                    key: "Evaluations",
-                    width: 30,
-                    value: r.evaluations.map((e) => `${e.name}:${e.score}`).join(", "),
-                },
-                {
-                    header: "Rating",
-                    key: "Rating",
-                    width: 20,
-                    value: r.evaluations.map((e) => `${e.rating}`).join(", "),
-                },
-                { header: "Error", key: "Error", width: 20, value: r.error ?? "-" },
-                {
-                    header: "Reason",
-                    key: "Reason",
-                    width: 50,
-                    value: r.evaluations
-                        .map((e) => e.reason ?? "")
-                        .filter(Boolean)
-                        .join(" | "),
-                },
-                {
-                    header: "Latency",
-                    key: "Latency",
-                    width: 15,
-                    value: r.latency ? `${r.latency.toFixed(2)}s` : "-",
-                },
-                {
-                    header: "Tokens",
-                    key: "Tokens",
-                    width: 40,
-                    value: r.usage
-                        ? `${(r.usage.inputTokens || 0) + (r.usage.outputTokens || 0)} (input:${r.usage.inputTokens || 0}, output:${r.usage.outputTokens || 0})`
-                        : "-",
-                },
-            ];
-        });
-    }
-    formatSummary(summary) {
-        return [
-            {
-                header: "Total",
-                key: "Total",
-                width: 10,
-                value: summary.total,
-            },
-            {
-                header: "Success Rate",
-                key: "SuccessRate",
-                width: 15,
-                value: summary.successRate,
-            },
-            {
-                header: "Total Duration",
-                key: "Duration",
-                width: 15,
-                value: summary.duration ? `${summary.duration.toFixed(3)}s` : "-",
-            },
-            {
-                header: "Avg Latency",
-                key: "AvgLatency",
-                width: 15,
-                value: summary.avgLatency ? `${summary.avgLatency.toFixed(3)}s` : "-",
-            },
-            {
-                header: "Total Tokens",
-                key: "TotalTokens",
-                width: 15,
-                value: summary.totalTokens ?? "-",
-            },
-            {
-                header: "Errors",
-                key: "Errors",
-                width: 8,
-                value: summary.errorCount ?? 0,
-            },
-        ];
-    }
-}
-export class ConsoleReporter extends BaseReporter {
-    name = "console";
-    async report(report) {
-        const summary = report.summary;
-        console.log("\n=== 📊 Evaluation Summary ===");
-        const summaryList = this.formatSummary(summary);
-        const summaryTable = new Table({
-            head: summaryList.map((h) => h.header),
-            colWidths: summaryList.map((h) => h.width),
-            chars,
-        });
-        summaryTable.push(summaryList.map((h) => h.value));
-        console.log(summaryTable.toString());
-        const list = this.formatReport(report);
-        if (!list.length)
-            return;
-        console.log("\n=== 📋 Detailed Results ===");
-        const pages = renderPagedTable(list, { maxCols: 5, chars });
-        for (const page of pages) {
-            console.log(page);
-        }
-        const failed = report.results.filter((r) => r.error);
-        if (failed.length) {
-            console.log(chalk.red("\n=== ❌ Failed Cases ==="));
-            for (const f of failed) {
-                console.log(`#${f.id} Input: ${JSON.stringify(f.input)}\n  Expected: ${f.expected ? JSON.stringify(f.expected) : "-"}\n  Output: ${f.output ? JSON.stringify(f.output) : "-"}\n  Error: ${f.error ?? "-"}\n`);
-            }
-        }
-    }
-}
-export class CsvReporter extends BaseReporter {
-    filePath;
-    name = "csv";
-    constructor(filePath) {
-        super();
-        this.filePath = filePath;
-    }
-    async writeCsv(filePath, data, headers) {
-        fs.mkdirSync(path.dirname(filePath), { recursive: true });
-        const stream = format({ headers });
-        const writeStream = fs.createWriteStream(filePath);
-        stream.pipe(writeStream);
-        for (const row of data) {
-            stream.write(row);
-        }
-        stream.end();
-        await new Promise((resolve, reject) => {
-            writeStream.on("finish", resolve);
-            writeStream.on("error", reject);
-        });
-    }
-    async report(report) {
-        const list = this.formatReport(report);
-        if (list.length > 0) {
-            const resultsHeaders = list[0]?.map((h) => h.header) ?? [];
-            const resultsRows = list.map((row) => {
-                const record = {};
-                for (const item of row) {
-                    record[item.header] = item.value;
-                }
-                return record;
-            });
-            const ext = path.extname(this.filePath).toLowerCase();
-            const outputFile = ext ? this.filePath : `${this.filePath}.csv`;
-            await this.writeCsv(outputFile, resultsRows, resultsHeaders);
-            console.log(`✅ Results CSV saved to ${outputFile}`);
-        }
-    }
-}

package/dist/utils/evaluation/runner.d.ts

@@ -1,16 +0,0 @@
-import { type Agent, AIGNE } from "@aigne/core";
-import type { DatasetItem, Runner, RunOptions, RunResult } from "./type.js";
-export declare class DefaultRunner implements Runner {
-    private agent;
-    private aigne;
-    name: string;
-    constructor(agent: Agent, aigne?: AIGNE);
-    run(dataset: DatasetItem[], options?: RunOptions): AsyncGenerator<RunResult>;
-}
-export declare class DefaultRunnerWithConcurrency implements Runner {
-    private agent;
-    private aigne;
-    name: string;
-    constructor(agent: Agent, aigne?: AIGNE);
-    run(dataset: DatasetItem[], options?: RunOptions): AsyncGenerator<RunResult>;
-}

package/dist/utils/evaluation/runner.js

@@ -1,129 +0,0 @@
-import { AIGNE } from "@aigne/core";
-export class DefaultRunner {
-    agent;
-    aigne;
-    name = "default-runner";
-    constructor(agent, aigne = new AIGNE()) {
-        this.agent = agent;
-        this.aigne = aigne;
-    }
-    async *run(dataset, options) {
-        const timeoutMs = options?.timeoutMs ?? 0;
-        const runTask = async (item) => {
-            const start = Date.now();
-            options?.hooks?.onBeforeRun?.(item);
-            try {
-                const execPromise = this.aigne.invoke(this.agent, item.input, { returnMetadata: true });
-                const result = timeoutMs > 0 ? await withTimeout(execPromise, timeoutMs, item.id) : await execPromise;
-                const { $meta, ...output } = result;
-                options?.hooks?.onAfterRun?.(result);
-                return {
-                    ...item,
-                    output,
-                    latency: (Date.now() - start) / 1000,
-                    usage: $meta?.usage || {},
-                };
-            }
-            catch (err) {
-                options?.hooks?.onError?.(err);
-                return {
-                    ...item,
-                    error: err.message,
-                };
-            }
-        };
-        for (const item of dataset) {
-            yield await runTask(item);
-        }
-    }
-}
-function withTimeout(promise, ms, id) {
-    return new Promise((resolve, reject) => {
-        const timer = setTimeout(() => {
-            reject(new Error(`Task ${id} timed out after ${ms}ms`));
-        }, ms);
-        promise
-            .then((res) => {
-            clearTimeout(timer);
-            resolve(res);
-        })
-            .catch((err) => {
-            clearTimeout(timer);
-            reject(err);
-        });
-    });
-}
-export class DefaultRunnerWithConcurrency {
-    agent;
-    aigne;
-    name = "default-runner-with-concurrency";
-    constructor(agent, aigne = new AIGNE()) {
-        this.agent = agent;
-        this.aigne = aigne;
-    }
-    async *run(dataset, options) {
-        const concurrency = options?.concurrency ?? 1;
-        const timeoutMs = options?.timeoutMs ?? 0;
-        let index = 0;
-        const yieldQueue = [];
-        let waitingResolve = null;
-        let activeWorkers = 0;
-        const runTask = async (item) => {
-            const start = Date.now();
-            options?.hooks?.onBeforeRun?.(item);
-            try {
-                const execPromise = this.aigne.invoke(this.agent, item.input, { returnMetadata: true });
-                const result = timeoutMs > 0 ? await withTimeout(execPromise, timeoutMs, item.id) : await execPromise;
-                const { $meta, ...output } = result;
-                options?.hooks?.onAfterRun?.(result);
-                return {
-                    ...item,
-                    output,
-                    latency: (Date.now() - start) / 1000,
-                    usage: $meta?.usage || {},
-                };
-            }
-            catch (err) {
-                options?.hooks?.onError?.(err);
-                return {
-                    ...item,
-                    error: err.message,
-                };
-            }
-        };
-        const worker = async () => {
-            activeWorkers++;
-            try {
-                while (true) {
-                    const currentIndex = index++;
-                    if (currentIndex >= dataset.length)
-                        break;
-                    const item = dataset[currentIndex];
-                    if (!item)
-                        continue;
-                    const res = await runTask(item);
-                    yieldQueue.push(res);
-                    waitingResolve?.();
-                }
-            }
-            finally {
-                activeWorkers--;
-                waitingResolve?.();
-            }
-        };
-        Array.from({ length: Math.min(concurrency, dataset.length) }, () => worker());
-        while (yieldQueue.length > 0 || activeWorkers > 0) {
-            if (yieldQueue.length > 0) {
-                const result = yieldQueue.shift();
-                if (result)
-                    yield result;
-            }
-            else {
-                await new Promise((resolve) => {
-                    waitingResolve = resolve;
-                });
-                waitingResolve = null;
-            }
-        }
-    }
-}