@aigne/cli 1.60.0-beta → 1.74.0-beta

package/dist/utils/evaluation/reporter.cjs

@@ -0,0 +1,225 @@
+const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
+let node_fs = require("node:fs");
+node_fs = require_rolldown_runtime.__toESM(node_fs);
+let chalk = require("chalk");
+chalk = require_rolldown_runtime.__toESM(chalk);
+let node_path = require("node:path");
+node_path = require_rolldown_runtime.__toESM(node_path);
+let _fast_csv_format = require("@fast-csv/format");
+let cli_table3 = require("cli-table3");
+cli_table3 = require_rolldown_runtime.__toESM(cli_table3);
+
+//#region src/utils/evaluation/reporter.ts
+const borderColor = chalk.default.green;
+const chars = {
+	top: borderColor("─"),
+	"top-mid": borderColor("┬"),
+	"top-left": borderColor("┌"),
+	"top-right": borderColor("┐"),
+	bottom: borderColor("─"),
+	"bottom-mid": borderColor("┴"),
+	"bottom-left": borderColor("└"),
+	"bottom-right": borderColor("┘"),
+	left: borderColor("│"),
+	"left-mid": borderColor("├"),
+	mid: borderColor("─"),
+	"mid-mid": borderColor("┼"),
+	right: borderColor("│"),
+	"right-mid": borderColor("┤"),
+	middle: borderColor("│")
+};
+const MAX_CELL_LENGTH = 300;
+function renderPagedTable(list, { maxCols = 6, chars }) {
+	if (!list.length) return [];
+	const tables = [];
+	const head = list[0]?.map((h) => h.header) ?? [];
+	const widths = list[0]?.map((h) => h.width ?? 20) ?? [];
+	for (let i = 0; i < head.length; i += maxCols) {
+		const table = new cli_table3.default({
+			head: head.slice(i, i + maxCols),
+			colWidths: widths.slice(i, i + maxCols),
+			wordWrap: true,
+			chars
+		});
+		for (const row of list) table.push(row.slice(i, i + maxCols).map((h) => String(h.value).length > MAX_CELL_LENGTH ? `${String(h.value).slice(0, MAX_CELL_LENGTH)}...` : h.value));
+		tables.push(table.toString());
+	}
+	return tables;
+}
+var BaseReporter = class {
+	name = "base";
+	async report(_report) {
+		throw new Error("Not implemented");
+	}
+	formatReport(report) {
+		return report.results.map((r) => {
+			return [
+				{
+					header: "ID",
+					key: "ID",
+					width: 10,
+					value: r.id
+				},
+				{
+					header: "Input",
+					key: "Input",
+					width: 40,
+					value: JSON.stringify(r.input)
+				},
+				{
+					header: "Output",
+					key: "Output",
+					width: 40,
+					value: r.output ? JSON.stringify(r.output) : "-"
+				},
+				{
+					header: "Expected",
+					key: "Expected",
+					width: 40,
+					value: r.expected ? JSON.stringify(r.expected) : "-"
+				},
+				{
+					header: "Evaluations",
+					key: "Evaluations",
+					width: 30,
+					value: r.evaluations.map((e) => `${e.name}:${e.score}`).join(", ")
+				},
+				{
+					header: "Rating",
+					key: "Rating",
+					width: 20,
+					value: r.evaluations.map((e) => `${e.rating}`).join(", ")
+				},
+				{
+					header: "Error",
+					key: "Error",
+					width: 20,
+					value: r.error ?? "-"
+				},
+				{
+					header: "Reason",
+					key: "Reason",
+					width: 50,
+					value: r.evaluations.map((e) => e.reason ?? "").filter(Boolean).join(" | ")
+				},
+				{
+					header: "Latency",
+					key: "Latency",
+					width: 15,
+					value: r.latency ? `${r.latency.toFixed(2)}s` : "-"
+				},
+				{
+					header: "Tokens",
+					key: "Tokens",
+					width: 40,
+					value: r.usage ? `${(r.usage.inputTokens || 0) + (r.usage.outputTokens || 0)} (input:${r.usage.inputTokens || 0}, output:${r.usage.outputTokens || 0})` : "-"
+				}
+			];
+		});
+	}
+	formatSummary(summary) {
+		return [
+			{
+				header: "Total",
+				key: "Total",
+				width: 10,
+				value: summary.total
+			},
+			{
+				header: "Success Rate",
+				key: "SuccessRate",
+				width: 15,
+				value: summary.successRate
+			},
+			{
+				header: "Total Duration",
+				key: "Duration",
+				width: 15,
+				value: summary.duration ? `${summary.duration.toFixed(3)}s` : "-"
+			},
+			{
+				header: "Avg Latency",
+				key: "AvgLatency",
+				width: 15,
+				value: summary.avgLatency ? `${summary.avgLatency.toFixed(3)}s` : "-"
+			},
+			{
+				header: "Total Tokens",
+				key: "TotalTokens",
+				width: 15,
+				value: summary.totalTokens ?? "-"
+			},
+			{
+				header: "Errors",
+				key: "Errors",
+				width: 8,
+				value: summary.errorCount ?? 0
+			}
+		];
+	}
+};
+var ConsoleReporter = class extends BaseReporter {
+	name = "console";
+	async report(report) {
+		const summary = report.summary;
+		console.log("\n=== 📊 Evaluation Summary ===");
+		const summaryList = this.formatSummary(summary);
+		const summaryTable = new cli_table3.default({
+			head: summaryList.map((h) => h.header),
+			colWidths: summaryList.map((h) => h.width),
+			chars
+		});
+		summaryTable.push(summaryList.map((h) => h.value));
+		console.log(summaryTable.toString());
+		const list = this.formatReport(report);
+		if (!list.length) return;
+		console.log("\n=== 📋 Detailed Results ===");
+		const pages = renderPagedTable(list, {
+			maxCols: 5,
+			chars
+		});
+		for (const page of pages) console.log(page);
+		const failed = report.results.filter((r) => r.error);
+		if (failed.length) {
+			console.log(chalk.default.red("\n=== ❌ Failed Cases ==="));
+			for (const f of failed) console.log(`#${f.id} Input: ${JSON.stringify(f.input)}\n  Expected: ${f.expected ? JSON.stringify(f.expected) : "-"}\n  Output: ${f.output ? JSON.stringify(f.output) : "-"}\n  Error: ${f.error ?? "-"}\n`);
+		}
+	}
+};
+var CsvReporter = class extends BaseReporter {
+	name = "csv";
+	constructor(filePath) {
+		super();
+		this.filePath = filePath;
+	}
+	async writeCsv(filePath, data, headers) {
+		node_fs.default.mkdirSync(node_path.default.dirname(filePath), { recursive: true });
+		const stream = (0, _fast_csv_format.format)({ headers });
+		const writeStream = node_fs.default.createWriteStream(filePath);
+		stream.pipe(writeStream);
+		for (const row of data) stream.write(row);
+		stream.end();
+		await new Promise((resolve, reject) => {
+			writeStream.on("finish", resolve);
+			writeStream.on("error", reject);
+		});
+	}
+	async report(report) {
+		const list = this.formatReport(report);
+		if (list.length > 0) {
+			const resultsHeaders = list[0]?.map((h) => h.header) ?? [];
+			const resultsRows = list.map((row) => {
+				const record = {};
+				for (const item of row) record[item.header] = item.value;
+				return record;
+			});
+			const outputFile = node_path.default.extname(this.filePath).toLowerCase() ? this.filePath : `${this.filePath}.csv`;
+			await this.writeCsv(outputFile, resultsRows, resultsHeaders);
+			console.log(`✅ Results CSV saved to ${outputFile}`);
+		}
+	}
+};
+
+//#endregion
+exports.ConsoleReporter = ConsoleReporter;
+exports.CsvReporter = CsvReporter;

package/dist/utils/evaluation/reporter.mjs

@@ -0,0 +1,220 @@
+import fs from "node:fs";
+import chalk from "chalk";
+import path from "node:path";
+import { format } from "@fast-csv/format";
+import Table from "cli-table3";
+
+//#region src/utils/evaluation/reporter.ts
+const borderColor = chalk.green;
+const chars = {
+	top: borderColor("─"),
+	"top-mid": borderColor("┬"),
+	"top-left": borderColor("┌"),
+	"top-right": borderColor("┐"),
+	bottom: borderColor("─"),
+	"bottom-mid": borderColor("┴"),
+	"bottom-left": borderColor("└"),
+	"bottom-right": borderColor("┘"),
+	left: borderColor("│"),
+	"left-mid": borderColor("├"),
+	mid: borderColor("─"),
+	"mid-mid": borderColor("┼"),
+	right: borderColor("│"),
+	"right-mid": borderColor("┤"),
+	middle: borderColor("│")
+};
+const MAX_CELL_LENGTH = 300;
+function renderPagedTable(list, { maxCols = 6, chars }) {
+	if (!list.length) return [];
+	const tables = [];
+	const head = list[0]?.map((h) => h.header) ?? [];
+	const widths = list[0]?.map((h) => h.width ?? 20) ?? [];
+	for (let i = 0; i < head.length; i += maxCols) {
+		const table = new Table({
+			head: head.slice(i, i + maxCols),
+			colWidths: widths.slice(i, i + maxCols),
+			wordWrap: true,
+			chars
+		});
+		for (const row of list) table.push(row.slice(i, i + maxCols).map((h) => String(h.value).length > MAX_CELL_LENGTH ? `${String(h.value).slice(0, MAX_CELL_LENGTH)}...` : h.value));
+		tables.push(table.toString());
+	}
+	return tables;
+}
+var BaseReporter = class {
+	name = "base";
+	async report(_report) {
+		throw new Error("Not implemented");
+	}
+	formatReport(report) {
+		return report.results.map((r) => {
+			return [
+				{
+					header: "ID",
+					key: "ID",
+					width: 10,
+					value: r.id
+				},
+				{
+					header: "Input",
+					key: "Input",
+					width: 40,
+					value: JSON.stringify(r.input)
+				},
+				{
+					header: "Output",
+					key: "Output",
+					width: 40,
+					value: r.output ? JSON.stringify(r.output) : "-"
+				},
+				{
+					header: "Expected",
+					key: "Expected",
+					width: 40,
+					value: r.expected ? JSON.stringify(r.expected) : "-"
+				},
+				{
+					header: "Evaluations",
+					key: "Evaluations",
+					width: 30,
+					value: r.evaluations.map((e) => `${e.name}:${e.score}`).join(", ")
+				},
+				{
+					header: "Rating",
+					key: "Rating",
+					width: 20,
+					value: r.evaluations.map((e) => `${e.rating}`).join(", ")
+				},
+				{
+					header: "Error",
+					key: "Error",
+					width: 20,
+					value: r.error ?? "-"
+				},
+				{
+					header: "Reason",
+					key: "Reason",
+					width: 50,
+					value: r.evaluations.map((e) => e.reason ?? "").filter(Boolean).join(" | ")
+				},
+				{
+					header: "Latency",
+					key: "Latency",
+					width: 15,
+					value: r.latency ? `${r.latency.toFixed(2)}s` : "-"
+				},
+				{
+					header: "Tokens",
+					key: "Tokens",
+					width: 40,
+					value: r.usage ? `${(r.usage.inputTokens || 0) + (r.usage.outputTokens || 0)} (input:${r.usage.inputTokens || 0}, output:${r.usage.outputTokens || 0})` : "-"
+				}
+			];
+		});
+	}
+	formatSummary(summary) {
+		return [
+			{
+				header: "Total",
+				key: "Total",
+				width: 10,
+				value: summary.total
+			},
+			{
+				header: "Success Rate",
+				key: "SuccessRate",
+				width: 15,
+				value: summary.successRate
+			},
+			{
+				header: "Total Duration",
+				key: "Duration",
+				width: 15,
+				value: summary.duration ? `${summary.duration.toFixed(3)}s` : "-"
+			},
+			{
+				header: "Avg Latency",
+				key: "AvgLatency",
+				width: 15,
+				value: summary.avgLatency ? `${summary.avgLatency.toFixed(3)}s` : "-"
+			},
+			{
+				header: "Total Tokens",
+				key: "TotalTokens",
+				width: 15,
+				value: summary.totalTokens ?? "-"
+			},
+			{
+				header: "Errors",
+				key: "Errors",
+				width: 8,
+				value: summary.errorCount ?? 0
+			}
+		];
+	}
+};
+var ConsoleReporter = class extends BaseReporter {
+	name = "console";
+	async report(report) {
+		const summary = report.summary;
+		console.log("\n=== 📊 Evaluation Summary ===");
+		const summaryList = this.formatSummary(summary);
+		const summaryTable = new Table({
+			head: summaryList.map((h) => h.header),
+			colWidths: summaryList.map((h) => h.width),
+			chars
+		});
+		summaryTable.push(summaryList.map((h) => h.value));
+		console.log(summaryTable.toString());
+		const list = this.formatReport(report);
+		if (!list.length) return;
+		console.log("\n=== 📋 Detailed Results ===");
+		const pages = renderPagedTable(list, {
+			maxCols: 5,
+			chars
+		});
+		for (const page of pages) console.log(page);
+		const failed = report.results.filter((r) => r.error);
+		if (failed.length) {
+			console.log(chalk.red("\n=== ❌ Failed Cases ==="));
+			for (const f of failed) console.log(`#${f.id} Input: ${JSON.stringify(f.input)}\n  Expected: ${f.expected ? JSON.stringify(f.expected) : "-"}\n  Output: ${f.output ? JSON.stringify(f.output) : "-"}\n  Error: ${f.error ?? "-"}\n`);
+		}
+	}
+};
+var CsvReporter = class extends BaseReporter {
+	name = "csv";
+	constructor(filePath) {
+		super();
+		this.filePath = filePath;
+	}
+	async writeCsv(filePath, data, headers) {
+		fs.mkdirSync(path.dirname(filePath), { recursive: true });
+		const stream = format({ headers });
+		const writeStream = fs.createWriteStream(filePath);
+		stream.pipe(writeStream);
+		for (const row of data) stream.write(row);
+		stream.end();
+		await new Promise((resolve$1, reject) => {
+			writeStream.on("finish", resolve$1);
+			writeStream.on("error", reject);
+		});
+	}
+	async report(report) {
+		const list = this.formatReport(report);
+		if (list.length > 0) {
+			const resultsHeaders = list[0]?.map((h) => h.header) ?? [];
+			const resultsRows = list.map((row) => {
+				const record = {};
+				for (const item of row) record[item.header] = item.value;
+				return record;
+			});
+			const outputFile = path.extname(this.filePath).toLowerCase() ? this.filePath : `${this.filePath}.csv`;
+			await this.writeCsv(outputFile, resultsRows, resultsHeaders);
+			console.log(`✅ Results CSV saved to ${outputFile}`);
+		}
+	}
+};
+
+//#endregion
+export { ConsoleReporter, CsvReporter };
+//# sourceMappingURL=reporter.mjs.map

package/dist/utils/evaluation/reporter.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.mjs","names":["resolve"],"sources":["../../../src/utils/evaluation/reporter.ts"],"sourcesContent":["import fs from \"node:fs\";\nimport path from \"node:path\";\nimport { format } from \"@fast-csv/format\";\nimport chalk from \"chalk\";\nimport Table from \"cli-table3\";\nimport type { Report, Reporter } from \"./type.js\";\n\nconst borderColor = chalk.green;\nconst chars = {\n  top: borderColor(\"─\"),\n  \"top-mid\": borderColor(\"┬\"),\n  \"top-left\": borderColor(\"┌\"),\n  \"top-right\": borderColor(\"┐\"),\n  bottom: borderColor(\"─\"),\n  \"bottom-mid\": borderColor(\"┴\"),\n  \"bottom-left\": borderColor(\"└\"),\n  \"bottom-right\": borderColor(\"┘\"),\n  left: borderColor(\"│\"),\n  \"left-mid\": borderColor(\"├\"),\n  mid: borderColor(\"─\"),\n  \"mid-mid\": borderColor(\"┼\"),\n  right: borderColor(\"│\"),\n  \"right-mid\": borderColor(\"┤\"),\n  middle: borderColor(\"│\"),\n};\n\nconst MAX_CELL_LENGTH = 300;\n\nfunction renderPagedTable(\n  list: { header: string; width?: number; value: any }[][],\n  { maxCols = 6, chars }: { maxCols?: number; chars: Record<string, string> },\n) {\n  if (!list.length) return [];\n\n  const tables: string[] = [];\n  const head = list[0]?.map((h) => h.header) ?? [];\n  const widths = list[0]?.map((h) => h.width ?? 20) ?? [];\n\n  for (let i = 0; i < head.length; i += maxCols) {\n    const subHead = head.slice(i, i + maxCols);\n    const subWidths = widths.slice(i, i + maxCols);\n\n    const table = new Table({\n      head: subHead,\n      colWidths: subWidths,\n      wordWrap: true,\n      chars,\n    });\n\n    for (const row of list) {\n      table.push(\n        row\n          .slice(i, i + maxCols)\n          .map((h) =>\n            String(h.value).length > MAX_CELL_LENGTH\n              ? `${String(h.value).slice(0, MAX_CELL_LENGTH)}...`\n              : h.value,\n          ),\n      );\n    }\n\n    tables.push(table.toString());\n  }\n\n  return tables;\n}\n\nexport class BaseReporter implements Reporter {\n  name = \"base\";\n\n  async report(_report: Report): Promise<void> {\n    throw new Error(\"Not implemented\");\n  }\n\n  protected formatReport(\n    report: Report,\n  ): { header: string; key: string; width: number; value: string | number }[][] {\n    return report.results.map((r) => {\n      return [\n        { header: \"ID\", key: \"ID\", width: 10, value: r.id },\n        { header: \"Input\", key: \"Input\", width: 40, value: JSON.stringify(r.input) },\n        {\n          header: \"Output\",\n          key: \"Output\",\n          width: 40,\n          value: r.output ? JSON.stringify(r.output) : \"-\",\n        },\n        {\n          header: \"Expected\",\n          key: \"Expected\",\n          width: 40,\n          value: r.expected ? JSON.stringify(r.expected) : \"-\",\n        },\n        {\n          header: \"Evaluations\",\n          key: \"Evaluations\",\n          width: 30,\n          value: r.evaluations.map((e) => `${e.name}:${e.score}`).join(\", \"),\n        },\n        {\n          header: \"Rating\",\n          key: \"Rating\",\n          width: 20,\n          value: r.evaluations.map((e) => `${e.rating}`).join(\", \"),\n        },\n        { header: \"Error\", key: \"Error\", width: 20, value: r.error ?? \"-\" },\n        {\n          header: \"Reason\",\n          key: \"Reason\",\n          width: 50,\n          value: r.evaluations\n            .map((e) => e.reason ?? \"\")\n            .filter(Boolean)\n            .join(\" | \"),\n        },\n        {\n          header: \"Latency\",\n          key: \"Latency\",\n          width: 15,\n          value: r.latency ? `${r.latency.toFixed(2)}s` : \"-\",\n        },\n        {\n          header: \"Tokens\",\n          key: \"Tokens\",\n          width: 40,\n          value: r.usage\n            ? `${(r.usage.inputTokens || 0) + (r.usage.outputTokens || 0)} (input:${r.usage.inputTokens || 0}, output:${r.usage.outputTokens || 0})`\n            : \"-\",\n        },\n      ];\n    });\n  }\n\n  protected formatSummary(\n    summary: Report[\"summary\"],\n  ): { header: string; key: string; width: number; value: string | number }[] {\n    return [\n      {\n        header: \"Total\",\n        key: \"Total\",\n        width: 10,\n        value: summary.total,\n      },\n      {\n        header: \"Success Rate\",\n        key: \"SuccessRate\",\n        width: 15,\n        value: summary.successRate,\n      },\n      {\n        header: \"Total Duration\",\n        key: \"Duration\",\n        width: 15,\n        value: summary.duration ? `${summary.duration.toFixed(3)}s` : \"-\",\n      },\n      {\n        header: \"Avg Latency\",\n        key: \"AvgLatency\",\n        width: 15,\n        value: summary.avgLatency ? `${summary.avgLatency.toFixed(3)}s` : \"-\",\n      },\n      {\n        header: \"Total Tokens\",\n        key: \"TotalTokens\",\n        width: 15,\n        value: summary.totalTokens ?? \"-\",\n      },\n      {\n        header: \"Errors\",\n        key: \"Errors\",\n        width: 8,\n        value: summary.errorCount ?? 0,\n      },\n    ];\n  }\n}\n\nexport class ConsoleReporter extends BaseReporter {\n  override name = \"console\";\n\n  override async report(report: Report): Promise<void> {\n    const summary = report.summary;\n\n    console.log(\"\\n=== 📊 Evaluation Summary ===\");\n    const summaryList = this.formatSummary(summary);\n    const summaryTable = new Table({\n      head: summaryList.map((h) => h.header),\n      colWidths: summaryList.map((h) => h.width),\n      chars,\n    });\n\n    summaryTable.push(summaryList.map((h) => h.value));\n    console.log(summaryTable.toString());\n\n    const list = this.formatReport(report);\n    if (!list.length) return;\n\n    console.log(\"\\n=== 📋 Detailed Results ===\");\n    const pages = renderPagedTable(list, { maxCols: 5, chars });\n    for (const page of pages) {\n      console.log(page);\n    }\n\n    const failed = report.results.filter((r) => r.error);\n    if (failed.length) {\n      console.log(chalk.red(\"\\n=== ❌ Failed Cases ===\"));\n      for (const f of failed) {\n        console.log(\n          `#${f.id} Input: ${JSON.stringify(f.input)}\\n  Expected: ${\n            f.expected ? JSON.stringify(f.expected) : \"-\"\n          }\\n  Output: ${f.output ? JSON.stringify(f.output) : \"-\"}\\n  Error: ${f.error ?? \"-\"}\\n`,\n        );\n      }\n    }\n  }\n}\n\nexport class CsvReporter extends BaseReporter {\n  override name = \"csv\";\n\n  constructor(private filePath: string) {\n    super();\n  }\n\n  private async writeCsv(filePath: string, data: Record<string, any>[], headers: string[]) {\n    fs.mkdirSync(path.dirname(filePath), { recursive: true });\n\n    const stream = format({ headers });\n    const writeStream = fs.createWriteStream(filePath);\n    stream.pipe(writeStream);\n\n    for (const row of data) {\n      stream.write(row);\n    }\n\n    stream.end();\n\n    await new Promise<void>((resolve, reject) => {\n      writeStream.on(\"finish\", resolve);\n      writeStream.on(\"error\", reject);\n    });\n  }\n\n  override async report(report: Report): Promise<void> {\n    const list = this.formatReport(report);\n    if (list.length > 0) {\n      const resultsHeaders = list[0]?.map((h) => h.header) ?? [];\n      const resultsRows = list.map((row) => {\n        const record: Record<string, string | number> = {};\n        for (const item of row) {\n          record[item.header] = item.value;\n        }\n        return record;\n      });\n\n      const ext = path.extname(this.filePath).toLowerCase();\n      const outputFile = ext ? this.filePath : `${this.filePath}.csv`;\n\n      await this.writeCsv(outputFile, resultsRows, resultsHeaders);\n\n      console.log(`✅ Results CSV saved to ${outputFile}`);\n    }\n  }\n}\n"],"mappings":";;;;;;;AAOA,MAAM,cAAc,MAAM;AAC1B,MAAM,QAAQ;CACZ,KAAK,YAAY,IAAI;CACrB,WAAW,YAAY,IAAI;CAC3B,YAAY,YAAY,IAAI;CAC5B,aAAa,YAAY,IAAI;CAC7B,QAAQ,YAAY,IAAI;CACxB,cAAc,YAAY,IAAI;CAC9B,eAAe,YAAY,IAAI;CAC/B,gBAAgB,YAAY,IAAI;CAChC,MAAM,YAAY,IAAI;CACtB,YAAY,YAAY,IAAI;CAC5B,KAAK,YAAY,IAAI;CACrB,WAAW,YAAY,IAAI;CAC3B,OAAO,YAAY,IAAI;CACvB,aAAa,YAAY,IAAI;CAC7B,QAAQ,YAAY,IAAI;CACzB;AAED,MAAM,kBAAkB;AAExB,SAAS,iBACP,MACA,EAAE,UAAU,GAAG,SACf;AACA,KAAI,CAAC,KAAK,OAAQ,QAAO,EAAE;CAE3B,MAAM,SAAmB,EAAE;CAC3B,MAAM,OAAO,KAAK,IAAI,KAAK,MAAM,EAAE,OAAO,IAAI,EAAE;CAChD,MAAM,SAAS,KAAK,IAAI,KAAK,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE;AAEvD,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,SAAS;EAI7C,MAAM,QAAQ,IAAI,MAAM;GACtB,MAJc,KAAK,MAAM,GAAG,IAAI,QAAQ;GAKxC,WAJgB,OAAO,MAAM,GAAG,IAAI,QAAQ;GAK5C,UAAU;GACV;GACD,CAAC;AAEF,OAAK,MAAM,OAAO,KAChB,OAAM,KACJ,IACG,MAAM,GAAG,IAAI,QAAQ,CACrB,KAAK,MACJ,OAAO,EAAE,MAAM,CAAC,SAAS,kBACrB,GAAG,OAAO,EAAE,MAAM,CAAC,MAAM,GAAG,gBAAgB,CAAC,OAC7C,EAAE,MACP,CACJ;AAGH,SAAO,KAAK,MAAM,UAAU,CAAC;;AAG/B,QAAO;;AAGT,IAAa,eAAb,MAA8C;CAC5C,OAAO;CAEP,MAAM,OAAO,SAAgC;AAC3C,QAAM,IAAI,MAAM,kBAAkB;;CAGpC,AAAU,aACR,QAC4E;AAC5E,SAAO,OAAO,QAAQ,KAAK,MAAM;AAC/B,UAAO;IACL;KAAE,QAAQ;KAAM,KAAK;KAAM,OAAO;KAAI,OAAO,EAAE;KAAI;IACnD;KAAE,QAAQ;KAAS,KAAK;KAAS,OAAO;KAAI,OAAO,KAAK,UAAU,EAAE,MAAM;KAAE;IAC5E;KACE,QAAQ;KACR,KAAK;KACL,OAAO;KACP,OAAO,EAAE,SAAS,KAAK,UAAU,EAAE,OAAO,GAAG;KAC9C;IACD;KACE,QAAQ;KACR,KAAK;KACL,OAAO;KACP,OAAO,EAAE,WAAW,KAAK,UAAU,EAAE,SAAS,GAAG;KAClD;IACD;KACE,QAAQ;KACR,KAAK;KACL,OAAO;KACP,OAAO,EAAE,YAAY,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,EAAE,QAAQ,CAAC,KAAK,KAAK;KACnE;IACD;KACE,QAAQ;KACR,KAAK;KACL,OAAO;KACP,OAAO,EAAE,YAAY,KAAK,MAAM,GAAG,EAAE,SAAS,CAAC,KAAK,KAAK;KAC1D;IACD;KAAE,QAAQ;KAAS,KAAK;KAAS,OAAO;KAAI,OAAO,EAAE,SAAS;KAAK;IACnE;KACE,QAAQ;KACR,KAAK;KACL,OAAO;KACP,OAAO,EAAE,YACN,KAAK,MAAM,EAAE,UAAU,GAAG,CAC1B,OAAO,QAAQ,CACf,KAAK,MAAM;KACf;IACD;KACE,QAAQ;KACR,KAAK;KACL,OAAO;KACP,OAAO,EAAE,UAAU,GAAG,EAAE,QAAQ,QAAQ,EAAE,CAAC,KAAK;KACjD;IACD;KACE,QAAQ;KACR,KAAK;KACL,OAAO;KACP,OAAO,EAAE,QACL,IAAI,EAAE,MAAM,eAAe,MAAM,EAAE,MAAM,gBAAgB,GAAG,UAAU,EAAE,MAAM,eAAe,EAAE,WAAW,EAAE,MAAM,gBAAgB,EAAE,KACpI;KACL;IACF;IACD;;CAGJ,AAAU,cACR,SAC0E;AAC1E,SAAO;GACL;IACE,QAAQ;IACR,KAAK;IACL,OAAO;IACP,OAAO,QAAQ;IAChB;GACD;IACE,QAAQ;IACR,KAAK;IACL,OAAO;IACP,OAAO,QAAQ;IAChB;GACD;IACE,QAAQ;IACR,KAAK;IACL,OAAO;IACP,OAAO,QAAQ,WAAW,GAAG,QAAQ,SAAS,QAAQ,EAAE,CAAC,KAAK;IAC/D;GACD;IACE,QAAQ;IACR,KAAK;IACL,OAAO;IACP,OAAO,QAAQ,aAAa,GAAG,QAAQ,WAAW,QAAQ,EAAE,CAAC,KAAK;IACnE;GACD;IACE,QAAQ;IACR,KAAK;IACL,OAAO;IACP,OAAO,QAAQ,eAAe;IAC/B;GACD;IACE,QAAQ;IACR,KAAK;IACL,OAAO;IACP,OAAO,QAAQ,cAAc;IAC9B;GACF;;;AAIL,IAAa,kBAAb,cAAqC,aAAa;CAChD,AAAS,OAAO;CAEhB,MAAe,OAAO,QAA+B;EACnD,MAAM,UAAU,OAAO;AAEvB,UAAQ,IAAI,kCAAkC;EAC9C,MAAM,cAAc,KAAK,cAAc,QAAQ;EAC/C,MAAM,eAAe,IAAI,MAAM;GAC7B,MAAM,YAAY,KAAK,MAAM,EAAE,OAAO;GACtC,WAAW,YAAY,KAAK,MAAM,EAAE,MAAM;GAC1C;GACD,CAAC;AAEF,eAAa,KAAK,YAAY,KAAK,MAAM,EAAE,MAAM,CAAC;AAClD,UAAQ,IAAI,aAAa,UAAU,CAAC;EAEpC,MAAM,OAAO,KAAK,aAAa,OAAO;AACtC,MAAI,CAAC,KAAK,OAAQ;AAElB,UAAQ,IAAI,gCAAgC;EAC5C,MAAM,QAAQ,iBAAiB,MAAM;GAAE,SAAS;GAAG;GAAO,CAAC;AAC3D,OAAK,MAAM,QAAQ,MACjB,SAAQ,IAAI,KAAK;EAGnB,MAAM,SAAS,OAAO,QAAQ,QAAQ,MAAM,EAAE,MAAM;AACpD,MAAI,OAAO,QAAQ;AACjB,WAAQ,IAAI,MAAM,IAAI,2BAA2B,CAAC;AAClD,QAAK,MAAM,KAAK,OACd,SAAQ,IACN,IAAI,EAAE,GAAG,UAAU,KAAK,UAAU,EAAE,MAAM,CAAC,gBACzC,EAAE,WAAW,KAAK,UAAU,EAAE,SAAS,GAAG,IAC3C,cAAc,EAAE,SAAS,KAAK,UAAU,EAAE,OAAO,GAAG,IAAI,aAAa,EAAE,SAAS,IAAI,IACtF;;;;AAMT,IAAa,cAAb,cAAiC,aAAa;CAC5C,AAAS,OAAO;CAEhB,YAAY,AAAQ,UAAkB;AACpC,SAAO;EADW;;CAIpB,MAAc,SAAS,UAAkB,MAA6B,SAAmB;AACvF,KAAG,UAAU,KAAK,QAAQ,SAAS,EAAE,EAAE,WAAW,MAAM,CAAC;EAEzD,MAAM,SAAS,OAAO,EAAE,SAAS,CAAC;EAClC,MAAM,cAAc,GAAG,kBAAkB,SAAS;AAClD,SAAO,KAAK,YAAY;AAExB,OAAK,MAAM,OAAO,KAChB,QAAO,MAAM,IAAI;AAGnB,SAAO,KAAK;AAEZ,QAAM,IAAI,SAAe,WAAS,WAAW;AAC3C,eAAY,GAAG,UAAUA,UAAQ;AACjC,eAAY,GAAG,SAAS,OAAO;IAC/B;;CAGJ,MAAe,OAAO,QAA+B;EACnD,MAAM,OAAO,KAAK,aAAa,OAAO;AACtC,MAAI,KAAK,SAAS,GAAG;GACnB,MAAM,iBAAiB,KAAK,IAAI,KAAK,MAAM,EAAE,OAAO,IAAI,EAAE;GAC1D,MAAM,cAAc,KAAK,KAAK,QAAQ;IACpC,MAAM,SAA0C,EAAE;AAClD,SAAK,MAAM,QAAQ,IACjB,QAAO,KAAK,UAAU,KAAK;AAE7B,WAAO;KACP;GAGF,MAAM,aADM,KAAK,QAAQ,KAAK,SAAS,CAAC,aAAa,GAC5B,KAAK,WAAW,GAAG,KAAK,SAAS;AAE1D,SAAM,KAAK,SAAS,YAAY,aAAa,eAAe;AAE5D,WAAQ,IAAI,0BAA0B,aAAa"}

package/dist/utils/evaluation/runner.cjs

@@ -0,0 +1,85 @@
+const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
+let _aigne_core = require("@aigne/core");
+
+//#region src/utils/evaluation/runner.ts
+function withTimeout(promise, ms, id) {
+	return new Promise((resolve, reject) => {
+		const timer = setTimeout(() => {
+			reject(/* @__PURE__ */ new Error(`Task ${id} timed out after ${ms}ms`));
+		}, ms);
+		promise.then((res) => {
+			clearTimeout(timer);
+			resolve(res);
+		}).catch((err) => {
+			clearTimeout(timer);
+			reject(err);
+		});
+	});
+}
+var DefaultRunnerWithConcurrency = class {
+	name = "default-runner-with-concurrency";
+	constructor(agent, aigne = new _aigne_core.AIGNE()) {
+		this.agent = agent;
+		this.aigne = aigne;
+	}
+	async *run(dataset, options) {
+		const concurrency = options?.concurrency ?? 1;
+		const timeoutMs = options?.timeoutMs ?? 0;
+		let index = 0;
+		const yieldQueue = [];
+		let waitingResolve = null;
+		let activeWorkers = 0;
+		const runTask = async (item) => {
+			const start = Date.now();
+			options?.hooks?.onBeforeRun?.(item);
+			try {
+				const execPromise = this.aigne.invoke(this.agent, item.input, { returnMetadata: true });
+				const result = timeoutMs > 0 ? await withTimeout(execPromise, timeoutMs, item.id) : await execPromise;
+				const { $meta, ...output } = result;
+				options?.hooks?.onAfterRun?.(result);
+				return {
+					...item,
+					output,
+					latency: (Date.now() - start) / 1e3,
+					usage: $meta?.usage || {}
+				};
+			} catch (err) {
+				options?.hooks?.onError?.(err);
+				return {
+					...item,
+					error: err.message
+				};
+			}
+		};
+		const worker = async () => {
+			activeWorkers++;
+			try {
+				while (true) {
+					const currentIndex = index++;
+					if (currentIndex >= dataset.length) break;
+					const item = dataset[currentIndex];
+					if (!item) continue;
+					const res = await runTask(item);
+					yieldQueue.push(res);
+					waitingResolve?.();
+				}
+			} finally {
+				activeWorkers--;
+				waitingResolve?.();
+			}
+		};
+		Array.from({ length: Math.min(concurrency, dataset.length) }, () => worker());
+		while (yieldQueue.length > 0 || activeWorkers > 0) if (yieldQueue.length > 0) {
+			const result = yieldQueue.shift();
+			if (result) yield result;
+		} else {
+			await new Promise((resolve) => {
+				waitingResolve = resolve;
+			});
+			waitingResolve = null;
+		}
+	}
+};
+
+//#endregion
+exports.DefaultRunnerWithConcurrency = DefaultRunnerWithConcurrency;

package/dist/utils/evaluation/runner.mjs

@@ -0,0 +1,85 @@
+import { AIGNE } from "@aigne/core";
+
+//#region src/utils/evaluation/runner.ts
+function withTimeout(promise, ms, id) {
+	return new Promise((resolve, reject) => {
+		const timer = setTimeout(() => {
+			reject(/* @__PURE__ */ new Error(`Task ${id} timed out after ${ms}ms`));
+		}, ms);
+		promise.then((res) => {
+			clearTimeout(timer);
+			resolve(res);
+		}).catch((err) => {
+			clearTimeout(timer);
+			reject(err);
+		});
+	});
+}
+var DefaultRunnerWithConcurrency = class {
+	name = "default-runner-with-concurrency";
+	constructor(agent, aigne = new AIGNE()) {
+		this.agent = agent;
+		this.aigne = aigne;
+	}
+	async *run(dataset, options) {
+		const concurrency = options?.concurrency ?? 1;
+		const timeoutMs = options?.timeoutMs ?? 0;
+		let index = 0;
+		const yieldQueue = [];
+		let waitingResolve = null;
+		let activeWorkers = 0;
+		const runTask = async (item) => {
+			const start = Date.now();
+			options?.hooks?.onBeforeRun?.(item);
+			try {
+				const execPromise = this.aigne.invoke(this.agent, item.input, { returnMetadata: true });
+				const result = timeoutMs > 0 ? await withTimeout(execPromise, timeoutMs, item.id) : await execPromise;
+				const { $meta, ...output } = result;
+				options?.hooks?.onAfterRun?.(result);
+				return {
+					...item,
+					output,
+					latency: (Date.now() - start) / 1e3,
+					usage: $meta?.usage || {}
+				};
+			} catch (err) {
+				options?.hooks?.onError?.(err);
+				return {
+					...item,
+					error: err.message
+				};
+			}
+		};
+		const worker = async () => {
+			activeWorkers++;
+			try {
+				while (true) {
+					const currentIndex = index++;
+					if (currentIndex >= dataset.length) break;
+					const item = dataset[currentIndex];
+					if (!item) continue;
+					const res = await runTask(item);
+					yieldQueue.push(res);
+					waitingResolve?.();
+				}
+			} finally {
+				activeWorkers--;
+				waitingResolve?.();
+			}
+		};
+		Array.from({ length: Math.min(concurrency, dataset.length) }, () => worker());
+		while (yieldQueue.length > 0 || activeWorkers > 0) if (yieldQueue.length > 0) {
+			const result = yieldQueue.shift();
+			if (result) yield result;
+		} else {
+			await new Promise((resolve) => {
+				waitingResolve = resolve;
+			});
+			waitingResolve = null;
+		}
+	}
+};
+
+//#endregion
+export { DefaultRunnerWithConcurrency };
+//# sourceMappingURL=runner.mjs.map

package/dist/utils/evaluation/runner.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.mjs","names":[],"sources":["../../../src/utils/evaluation/runner.ts"],"sourcesContent":["import { type Agent, AIGNE } from \"@aigne/core\";\nimport type { DatasetItem, Runner, RunOptions, RunResult } from \"./type.js\";\n\nexport class DefaultRunner implements Runner {\n  name = \"default-runner\";\n\n  constructor(\n    private agent: Agent,\n    private aigne: AIGNE = new AIGNE(),\n  ) {}\n\n  async *run(dataset: DatasetItem[], options?: RunOptions): AsyncGenerator<RunResult> {\n    const timeoutMs = options?.timeoutMs ?? 0;\n\n    const runTask = async (item: DatasetItem): Promise<RunResult> => {\n      const start = Date.now();\n      options?.hooks?.onBeforeRun?.(item);\n\n      try {\n        const execPromise = this.aigne.invoke(this.agent, item.input, { returnMetadata: true });\n        const result =\n          timeoutMs > 0 ? await withTimeout(execPromise, timeoutMs, item.id) : await execPromise;\n        const { $meta, ...output } = result;\n\n        options?.hooks?.onAfterRun?.(result);\n\n        return {\n          ...item,\n          output,\n          latency: (Date.now() - start) / 1000,\n          usage: $meta?.usage || {},\n        };\n      } catch (err: any) {\n        options?.hooks?.onError?.(err);\n\n        return {\n          ...item,\n          error: err.message,\n        };\n      }\n    };\n\n    for (const item of dataset) {\n      yield await runTask(item);\n    }\n  }\n}\n\nfunction withTimeout<T>(promise: Promise<T>, ms: number, id: string | number): Promise<T> {\n  return new Promise<T>((resolve, reject) => {\n    const timer = setTimeout(() => {\n      reject(new Error(`Task ${id} timed out after ${ms}ms`));\n    }, ms);\n\n    promise\n      .then((res) => {\n        clearTimeout(timer);\n        resolve(res);\n      })\n      .catch((err) => {\n        clearTimeout(timer);\n        reject(err);\n      });\n  });\n}\n\nexport class DefaultRunnerWithConcurrency implements Runner {\n  name = \"default-runner-with-concurrency\";\n\n  constructor(\n    private agent: Agent,\n    private aigne: AIGNE = new AIGNE(),\n  ) {}\n\n  async *run(dataset: DatasetItem[], options?: RunOptions): AsyncGenerator<RunResult> {\n    const concurrency = options?.concurrency ?? 1;\n    const timeoutMs = options?.timeoutMs ?? 0;\n\n    let index = 0;\n    const yieldQueue: RunResult[] = [];\n    let waitingResolve: (() => void) | null = null;\n    let activeWorkers = 0;\n\n    const runTask = async (item: DatasetItem): Promise<RunResult> => {\n      const start = Date.now();\n      options?.hooks?.onBeforeRun?.(item);\n\n      try {\n        const execPromise = this.aigne.invoke(this.agent, item.input, { returnMetadata: true });\n        const result =\n          timeoutMs > 0 ? await withTimeout(execPromise, timeoutMs, item.id) : await execPromise;\n        const { $meta, ...output } = result;\n\n        options?.hooks?.onAfterRun?.(result);\n\n        return {\n          ...item,\n          output,\n          latency: (Date.now() - start) / 1000,\n          usage: $meta?.usage || {},\n        };\n      } catch (err: any) {\n        options?.hooks?.onError?.(err);\n\n        return {\n          ...item,\n          error: err.message,\n        };\n      }\n    };\n\n    const worker = async () => {\n      activeWorkers++;\n      try {\n        while (true) {\n          const currentIndex = index++;\n          if (currentIndex >= dataset.length) break;\n\n          const item = dataset[currentIndex];\n          if (!item) continue;\n          const res = await runTask(item);\n\n          yieldQueue.push(res);\n          waitingResolve?.();\n        }\n      } finally {\n        activeWorkers--;\n        waitingResolve?.();\n      }\n    };\n\n    Array.from({ length: Math.min(concurrency, dataset.length) }, () => worker());\n\n    while (yieldQueue.length > 0 || activeWorkers > 0) {\n      if (yieldQueue.length > 0) {\n        const result = yieldQueue.shift();\n        if (result) yield result;\n      } else {\n        await new Promise<void>((resolve) => {\n          waitingResolve = resolve;\n        });\n        waitingResolve = null;\n      }\n    }\n  }\n}\n"],"mappings":";;;AAgDA,SAAS,YAAe,SAAqB,IAAY,IAAiC;AACxF,QAAO,IAAI,SAAY,SAAS,WAAW;EACzC,MAAM,QAAQ,iBAAiB;AAC7B,0BAAO,IAAI,MAAM,QAAQ,GAAG,mBAAmB,GAAG,IAAI,CAAC;KACtD,GAAG;AAEN,UACG,MAAM,QAAQ;AACb,gBAAa,MAAM;AACnB,WAAQ,IAAI;IACZ,CACD,OAAO,QAAQ;AACd,gBAAa,MAAM;AACnB,UAAO,IAAI;IACX;GACJ;;AAGJ,IAAa,+BAAb,MAA4D;CAC1D,OAAO;CAEP,YACE,AAAQ,OACR,AAAQ,QAAe,IAAI,OAAO,EAClC;EAFQ;EACA;;CAGV,OAAO,IAAI,SAAwB,SAAiD;EAClF,MAAM,cAAc,SAAS,eAAe;EAC5C,MAAM,YAAY,SAAS,aAAa;EAExC,IAAI,QAAQ;EACZ,MAAM,aAA0B,EAAE;EAClC,IAAI,iBAAsC;EAC1C,IAAI,gBAAgB;EAEpB,MAAM,UAAU,OAAO,SAA0C;GAC/D,MAAM,QAAQ,KAAK,KAAK;AACxB,YAAS,OAAO,cAAc,KAAK;AAEnC,OAAI;IACF,MAAM,cAAc,KAAK,MAAM,OAAO,KAAK,OAAO,KAAK,OAAO,EAAE,gBAAgB,MAAM,CAAC;IACvF,MAAM,SACJ,YAAY,IAAI,MAAM,YAAY,aAAa,WAAW,KAAK,GAAG,GAAG,MAAM;IAC7E,MAAM,EAAE,OAAO,GAAG,WAAW;AAE7B,aAAS,OAAO,aAAa,OAAO;AAEpC,WAAO;KACL,GAAG;KACH;KACA,UAAU,KAAK,KAAK,GAAG,SAAS;KAChC,OAAO,OAAO,SAAS,EAAE;KAC1B;YACM,KAAU;AACjB,aAAS,OAAO,UAAU,IAAI;AAE9B,WAAO;KACL,GAAG;KACH,OAAO,IAAI;KACZ;;;EAIL,MAAM,SAAS,YAAY;AACzB;AACA,OAAI;AACF,WAAO,MAAM;KACX,MAAM,eAAe;AACrB,SAAI,gBAAgB,QAAQ,OAAQ;KAEpC,MAAM,OAAO,QAAQ;AACrB,SAAI,CAAC,KAAM;KACX,MAAM,MAAM,MAAM,QAAQ,KAAK;AAE/B,gBAAW,KAAK,IAAI;AACpB,uBAAkB;;aAEZ;AACR;AACA,sBAAkB;;;AAItB,QAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,QAAQ,OAAO,EAAE,QAAQ,QAAQ,CAAC;AAE7E,SAAO,WAAW,SAAS,KAAK,gBAAgB,EAC9C,KAAI,WAAW,SAAS,GAAG;GACzB,MAAM,SAAS,WAAW,OAAO;AACjC,OAAI,OAAQ,OAAM;SACb;AACL,SAAM,IAAI,SAAe,YAAY;AACnC,qBAAiB;KACjB;AACF,oBAAiB"}

package/dist/utils/get-url-origin.cjs

@@ -0,0 +1,12 @@
+
+//#region src/utils/get-url-origin.ts
+const getUrlOrigin = (url) => {
+	try {
+		return new URL(url).origin;
+	} catch {
+		return url;
+	}
+};
+
+//#endregion
+exports.getUrlOrigin = getUrlOrigin;

package/dist/utils/get-url-origin.d.cts

@@ -0,0 +1,5 @@
+//#region src/utils/get-url-origin.d.ts
+declare const getUrlOrigin: (url: string) => string;
+//#endregion
+export { getUrlOrigin };
+//# sourceMappingURL=get-url-origin.d.cts.map

package/dist/utils/get-url-origin.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-url-origin.d.cts","names":[],"sources":["../../src/utils/get-url-origin.ts"],"mappings":";cAAa,YAAA,GAAA,GAAA"}

package/dist/utils/get-url-origin.d.mts

@@ -0,0 +1,5 @@
+//#region src/utils/get-url-origin.d.ts
+declare const getUrlOrigin: (url: string) => string;
+//#endregion
+export { getUrlOrigin };
+//# sourceMappingURL=get-url-origin.d.mts.map

package/dist/utils/get-url-origin.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-url-origin.d.mts","names":[],"sources":["../../src/utils/get-url-origin.ts"],"mappings":";cAAa,YAAA,GAAA,GAAA"}