@aigne/cli 1.60.0-beta → 1.74.0-beta

package/dist/utils/download.d.mts

@@ -0,0 +1,7 @@
+//#region src/utils/download.d.ts
+declare function downloadAndExtract(url: string, dir: string, options?: {
+  strip?: number;
+}): Promise<void>;
+//#endregion
+export { downloadAndExtract };
+//# sourceMappingURL=download.d.mts.map

package/dist/utils/download.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"download.d.mts","names":[],"sources":["../../src/utils/download.ts"],"mappings":";iBAOsB,kBAAA,CAAA,GAAA,UAAA,GAAA,UAAA,OAAA;EAAA,KAAA;AAAA,IAGY,OAAA"}

package/dist/utils/download.mjs

@@ -0,0 +1,25 @@
+import { mkdir } from "node:fs/promises";
+import { fetch } from "@aigne/core/utils/fetch";
+import { Readable } from "node:stream";
+import { finished } from "node:stream/promises";
+import { x } from "tar";
+
+//#region src/utils/download.ts
+async function downloadAndExtract(url, dir, options = {}) {
+	const response = await fetch(url);
+	if (!response.body) throw new Error(`Failed to download package from ${url}: Unexpected to get empty response`);
+	try {
+		await mkdir(dir, { recursive: true });
+		await finished(Readable.fromWeb(response.body).pipe(x({
+			C: dir,
+			...options
+		})));
+	} catch (error) {
+		error.message = `Failed to extract package from ${url}: ${error.message}`;
+		throw error;
+	}
+}
+
+//#endregion
+export { downloadAndExtract };
+//# sourceMappingURL=download.mjs.map

package/dist/utils/download.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"download.mjs","names":[],"sources":["../../src/utils/download.ts"],"sourcesContent":["import { mkdir } from \"node:fs/promises\";\nimport { Readable } from \"node:stream\";\nimport { finished } from \"node:stream/promises\";\nimport type { ReadableStream } from \"node:stream/web\";\nimport { fetch } from \"@aigne/core/utils/fetch\";\nimport { x } from \"tar\";\n\nexport async function downloadAndExtract(\n  url: string,\n  dir: string,\n  options: { strip?: number } = {},\n) {\n  const response = await fetch(url);\n\n  if (!response.body) {\n    throw new Error(`Failed to download package from ${url}: Unexpected to get empty response`);\n  }\n\n  try {\n    await mkdir(dir, { recursive: true });\n\n    await finished(\n      Readable.fromWeb(response.body as unknown as ReadableStream).pipe(x({ C: dir, ...options })),\n    );\n  } catch (error) {\n    error.message = `Failed to extract package from ${url}: ${error.message}`;\n    throw error;\n  }\n}\n"],"mappings":";;;;;;;AAOA,eAAsB,mBACpB,KACA,KACA,UAA8B,EAAE,EAChC;CACA,MAAM,WAAW,MAAM,MAAM,IAAI;AAEjC,KAAI,CAAC,SAAS,KACZ,OAAM,IAAI,MAAM,mCAAmC,IAAI,oCAAoC;AAG7F,KAAI;AACF,QAAM,MAAM,KAAK,EAAE,WAAW,MAAM,CAAC;AAErC,QAAM,SACJ,SAAS,QAAQ,SAAS,KAAkC,CAAC,KAAK,EAAE;GAAE,GAAG;GAAK,GAAG;GAAS,CAAC,CAAC,CAC7F;UACM,OAAO;AACd,QAAM,UAAU,kCAAkC,IAAI,IAAI,MAAM;AAChE,QAAM"}

package/dist/utils/evaluation/core.cjs

@@ -0,0 +1,84 @@
+const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
+const require_reporter = require('./reporter.cjs');
+let _aigne_listr2 = require("@aigne/listr2");
+
+//#region src/utils/evaluation/core.ts
+function aggregateSummary(results, duration) {
+	const total = results.length;
+	const scores = results.flatMap((r) => r.evaluations.map((e) => e.score));
+	const successRate = Number((scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0).toFixed(2));
+	const latencies = results.map((r) => r.latency || 0);
+	const totalTokens = results.reduce((a, r) => a + (r.usage?.inputTokens || 0) + (r.usage?.outputTokens || 0), 0);
+	const errors = results.filter((r) => r.error).length;
+	return {
+		total,
+		successRate,
+		duration: Number(duration.toFixed(3)),
+		avgLatency: latencies.reduce((a, b) => a + b, 0) / (latencies.length || 1),
+		maxLatency: Math.max(...latencies, 0),
+		minLatency: Math.min(...latencies, 0),
+		totalTokens,
+		errorCount: errors,
+		scoreDistribution: {
+			min: Math.min(...scores, 0),
+			max: Math.max(...scores, 0),
+			mean: successRate,
+			median: scores.length ? scores.sort((a, b) => a - b)[Math.floor(scores.length / 2)] : 0,
+			variance: scores.length > 1 ? scores.reduce((a, b) => a + (b - successRate) ** 2, 0) / scores.length : 0
+		}
+	};
+}
+async function runEvaluationPipeline(params) {
+	const now = Date.now();
+	const { dataset, runner, evaluators, reporters = [new require_reporter.ConsoleReporter()], options } = params;
+	const results = [];
+	const { items } = await new _aigne_listr2.Listr([{
+		title: "Load dataset",
+		task: async (ctx, _task) => {
+			ctx.items = await dataset.loadWithOptions();
+		}
+	}], { registerSignalListeners: false }).run();
+	await new _aigne_listr2.Listr(items.map((item) => {
+		const input = JSON.stringify(item.input);
+		return {
+			title: `Run evaluations with input: ${input.length > 100 ? `${input.slice(0, 100)}...` : input}`,
+			task: async (ctx, task) => {
+				task.output = `Start running agent with input: ${JSON.stringify(item.input, null, 2)}`;
+				const runnerResults = await runner.run([item], options);
+				for await (const result of runnerResults) {
+					task.output = `Start running evaluation with: ${JSON.stringify({
+						input: result.input,
+						output: result.output,
+						expected: result.expected
+					}, null, 2)}`;
+					const evaluations = [];
+					for (const evaluator of evaluators) {
+						const evals = await evaluator.evaluate(result);
+						evaluations.push(...evals);
+					}
+					results.push({
+						...result,
+						evaluations
+					});
+					task.output = `Finish running evaluation`;
+				}
+				ctx.results = results;
+			}
+		};
+	}), {
+		concurrent: options?.concurrency ? Math.min(items.length, options?.concurrency) : false,
+		exitOnError: true,
+		rendererOptions: { collapseSubtasks: false },
+		registerSignalListeners: false
+	}).run();
+	const summary = aggregateSummary(results, (Date.now() - now) / 1e3);
+	const report = {
+		dataset: dataset.name,
+		results,
+		summary
+	};
+	for (const reporter of reporters) await reporter.report(report);
+}
+
+//#endregion
+exports.runEvaluationPipeline = runEvaluationPipeline;

package/dist/utils/evaluation/core.mjs

@@ -0,0 +1,84 @@
+import { ConsoleReporter } from "./reporter.mjs";
+import { Listr } from "@aigne/listr2";
+
+//#region src/utils/evaluation/core.ts
+function aggregateSummary(results, duration) {
+	const total = results.length;
+	const scores = results.flatMap((r) => r.evaluations.map((e) => e.score));
+	const successRate = Number((scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0).toFixed(2));
+	const latencies = results.map((r) => r.latency || 0);
+	const totalTokens = results.reduce((a, r) => a + (r.usage?.inputTokens || 0) + (r.usage?.outputTokens || 0), 0);
+	const errors = results.filter((r) => r.error).length;
+	return {
+		total,
+		successRate,
+		duration: Number(duration.toFixed(3)),
+		avgLatency: latencies.reduce((a, b) => a + b, 0) / (latencies.length || 1),
+		maxLatency: Math.max(...latencies, 0),
+		minLatency: Math.min(...latencies, 0),
+		totalTokens,
+		errorCount: errors,
+		scoreDistribution: {
+			min: Math.min(...scores, 0),
+			max: Math.max(...scores, 0),
+			mean: successRate,
+			median: scores.length ? scores.sort((a, b) => a - b)[Math.floor(scores.length / 2)] : 0,
+			variance: scores.length > 1 ? scores.reduce((a, b) => a + (b - successRate) ** 2, 0) / scores.length : 0
+		}
+	};
+}
+async function runEvaluationPipeline(params) {
+	const now = Date.now();
+	const { dataset, runner, evaluators, reporters = [new ConsoleReporter()], options } = params;
+	const results = [];
+	const { items } = await new Listr([{
+		title: "Load dataset",
+		task: async (ctx, _task) => {
+			ctx.items = await dataset.loadWithOptions();
+		}
+	}], { registerSignalListeners: false }).run();
+	await new Listr(items.map((item) => {
+		const input = JSON.stringify(item.input);
+		return {
+			title: `Run evaluations with input: ${input.length > 100 ? `${input.slice(0, 100)}...` : input}`,
+			task: async (ctx, task) => {
+				task.output = `Start running agent with input: ${JSON.stringify(item.input, null, 2)}`;
+				const runnerResults = await runner.run([item], options);
+				for await (const result of runnerResults) {
+					task.output = `Start running evaluation with: ${JSON.stringify({
+						input: result.input,
+						output: result.output,
+						expected: result.expected
+					}, null, 2)}`;
+					const evaluations = [];
+					for (const evaluator of evaluators) {
+						const evals = await evaluator.evaluate(result);
+						evaluations.push(...evals);
+					}
+					results.push({
+						...result,
+						evaluations
+					});
+					task.output = `Finish running evaluation`;
+				}
+				ctx.results = results;
+			}
+		};
+	}), {
+		concurrent: options?.concurrency ? Math.min(items.length, options?.concurrency) : false,
+		exitOnError: true,
+		rendererOptions: { collapseSubtasks: false },
+		registerSignalListeners: false
+	}).run();
+	const summary = aggregateSummary(results, (Date.now() - now) / 1e3);
+	const report = {
+		dataset: dataset.name,
+		results,
+		summary
+	};
+	for (const reporter of reporters) await reporter.report(report);
+}
+
+//#endregion
+export { runEvaluationPipeline };
+//# sourceMappingURL=core.mjs.map

package/dist/utils/evaluation/core.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"core.mjs","names":[],"sources":["../../../src/utils/evaluation/core.ts"],"sourcesContent":["import { Listr } from \"@aigne/listr2\";\nimport { ConsoleReporter } from \"./reporter.js\";\nimport type {\n  Dataset,\n  DatasetItem,\n  Evaluation,\n  EvaluationResult,\n  EvaluationSummary,\n  Evaluator,\n  Report,\n  Reporter,\n  Runner,\n  RunOptions,\n} from \"./type.js\";\n\nfunction aggregateSummary(results: EvaluationResult[], duration: number): EvaluationSummary {\n  const total = results.length;\n  const scores = results.flatMap((r) => r.evaluations.map((e) => e.score));\n  const successRate = Number(\n    (scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0).toFixed(2),\n  );\n\n  const latencies = results.map((r) => r.latency || 0);\n  const totalTokens = results.reduce(\n    (a, r) => a + (r.usage?.inputTokens || 0) + (r.usage?.outputTokens || 0),\n    0,\n  );\n\n  const errors = results.filter((r) => r.error).length;\n\n  return {\n    total,\n    successRate,\n    duration: Number(duration.toFixed(3)),\n    avgLatency: latencies.reduce((a, b) => a + b, 0) / (latencies.length || 1),\n    maxLatency: Math.max(...latencies, 0),\n    minLatency: Math.min(...latencies, 0),\n    totalTokens,\n    errorCount: errors,\n    scoreDistribution: {\n      min: Math.min(...scores, 0),\n      max: Math.max(...scores, 0),\n      mean: successRate,\n      median: scores.length ? scores.sort((a, b) => a - b)[Math.floor(scores.length / 2)] : 0,\n      variance:\n        scores.length > 1\n          ? scores.reduce((a, b) => a + (b - successRate) ** 2, 0) / scores.length\n          : 0,\n    },\n  };\n}\n\ntype EvaluationPipelineContext = {\n  items: DatasetItem[];\n  results: EvaluationResult[];\n};\n\nexport async function runEvaluationPipeline(params: {\n  dataset: Dataset;\n  runner: Runner;\n  evaluators: Evaluator[];\n  reporters?: Reporter[];\n  options?: RunOptions;\n}) {\n  const now = Date.now();\n  const { dataset, runner, evaluators, reporters = [new ConsoleReporter()], options } = params;\n\n  const results: EvaluationPipelineContext[\"results\"] = [];\n\n  const task1 = new Listr<{ items: DatasetItem[] }>(\n    [\n      {\n        title: \"Load dataset\",\n        task: async (ctx, _task) => {\n          ctx.items = await dataset.loadWithOptions();\n        },\n      },\n    ],\n    {\n      registerSignalListeners: false,\n    },\n  );\n\n  const { items } = await task1.run();\n\n  const task2 = new Listr<EvaluationPipelineContext>(\n    items.map((item) => {\n      const input = JSON.stringify(item.input);\n      return {\n        title: `Run evaluations with input: ${input.length > 100 ? `${input.slice(0, 100)}...` : input}`,\n        task: async (ctx, task) => {\n          task.output = `Start running agent with input: ${JSON.stringify(item.input, null, 2)}`;\n\n          const runnerResults = await runner.run([item], options);\n\n          for await (const result of runnerResults) {\n            task.output = `Start running evaluation with: ${JSON.stringify(\n              {\n                input: result.input,\n                output: result.output,\n                expected: result.expected,\n              },\n              null,\n              2,\n            )}`;\n\n            const evaluations: Evaluation[] = [];\n            for (const evaluator of evaluators) {\n              const evals = await evaluator.evaluate(result);\n              evaluations.push(...evals);\n            }\n\n            results.push({ ...result, evaluations });\n\n            task.output = `Finish running evaluation`;\n          }\n\n          ctx.results = results;\n        },\n      };\n    }),\n    {\n      concurrent: options?.concurrency ? Math.min(items.length, options?.concurrency) : false,\n      exitOnError: true,\n      rendererOptions: {\n        collapseSubtasks: false,\n      },\n      registerSignalListeners: false,\n    },\n  );\n\n  await task2.run();\n\n  const summary: EvaluationSummary = aggregateSummary(results, (Date.now() - now) / 1000);\n  const report: Report = { dataset: dataset.name, results, summary };\n\n  for (const reporter of reporters) {\n    await reporter.report(report);\n  }\n}\n"],"mappings":";;;;AAeA,SAAS,iBAAiB,SAA6B,UAAqC;CAC1F,MAAM,QAAQ,QAAQ;CACtB,MAAM,SAAS,QAAQ,SAAS,MAAM,EAAE,YAAY,KAAK,MAAM,EAAE,MAAM,CAAC;CACxE,MAAM,cAAc,QACjB,OAAO,SAAS,IAAI,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,EAAE,GAAG,OAAO,SAAS,GAAG,QAAQ,EAAE,CACvF;CAED,MAAM,YAAY,QAAQ,KAAK,MAAM,EAAE,WAAW,EAAE;CACpD,MAAM,cAAc,QAAQ,QACzB,GAAG,MAAM,KAAK,EAAE,OAAO,eAAe,MAAM,EAAE,OAAO,gBAAgB,IACtE,EACD;CAED,MAAM,SAAS,QAAQ,QAAQ,MAAM,EAAE,MAAM,CAAC;AAE9C,QAAO;EACL;EACA;EACA,UAAU,OAAO,SAAS,QAAQ,EAAE,CAAC;EACrC,YAAY,UAAU,QAAQ,GAAG,MAAM,IAAI,GAAG,EAAE,IAAI,UAAU,UAAU;EACxE,YAAY,KAAK,IAAI,GAAG,WAAW,EAAE;EACrC,YAAY,KAAK,IAAI,GAAG,WAAW,EAAE;EACrC;EACA,YAAY;EACZ,mBAAmB;GACjB,KAAK,KAAK,IAAI,GAAG,QAAQ,EAAE;GAC3B,KAAK,KAAK,IAAI,GAAG,QAAQ,EAAE;GAC3B,MAAM;GACN,QAAQ,OAAO,SAAS,OAAO,MAAM,GAAG,MAAM,IAAI,EAAE,CAAC,KAAK,MAAM,OAAO,SAAS,EAAE,IAAI;GACtF,UACE,OAAO,SAAS,IACZ,OAAO,QAAQ,GAAG,MAAM,KAAK,IAAI,gBAAgB,GAAG,EAAE,GAAG,OAAO,SAChE;GACP;EACF;;AAQH,eAAsB,sBAAsB,QAMzC;CACD,MAAM,MAAM,KAAK,KAAK;CACtB,MAAM,EAAE,SAAS,QAAQ,YAAY,YAAY,CAAC,IAAI,iBAAiB,CAAC,EAAE,YAAY;CAEtF,MAAM,UAAgD,EAAE;CAgBxD,MAAM,EAAE,UAAU,MAdJ,IAAI,MAChB,CACE;EACE,OAAO;EACP,MAAM,OAAO,KAAK,UAAU;AAC1B,OAAI,QAAQ,MAAM,QAAQ,iBAAiB;;EAE9C,CACF,EACD,EACE,yBAAyB,OAC1B,CACF,CAE6B,KAAK;AAgDnC,OA9Cc,IAAI,MAChB,MAAM,KAAK,SAAS;EAClB,MAAM,QAAQ,KAAK,UAAU,KAAK,MAAM;AACxC,SAAO;GACL,OAAO,+BAA+B,MAAM,SAAS,MAAM,GAAG,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO;GACzF,MAAM,OAAO,KAAK,SAAS;AACzB,SAAK,SAAS,mCAAmC,KAAK,UAAU,KAAK,OAAO,MAAM,EAAE;IAEpF,MAAM,gBAAgB,MAAM,OAAO,IAAI,CAAC,KAAK,EAAE,QAAQ;AAEvD,eAAW,MAAM,UAAU,eAAe;AACxC,UAAK,SAAS,kCAAkC,KAAK,UACnD;MACE,OAAO,OAAO;MACd,QAAQ,OAAO;MACf,UAAU,OAAO;MAClB,EACD,MACA,EACD;KAED,MAAM,cAA4B,EAAE;AACpC,UAAK,MAAM,aAAa,YAAY;MAClC,MAAM,QAAQ,MAAM,UAAU,SAAS,OAAO;AAC9C,kBAAY,KAAK,GAAG,MAAM;;AAG5B,aAAQ,KAAK;MAAE,GAAG;MAAQ;MAAa,CAAC;AAExC,UAAK,SAAS;;AAGhB,QAAI,UAAU;;GAEjB;GACD,EACF;EACE,YAAY,SAAS,cAAc,KAAK,IAAI,MAAM,QAAQ,SAAS,YAAY,GAAG;EAClF,aAAa;EACb,iBAAiB,EACf,kBAAkB,OACnB;EACD,yBAAyB;EAC1B,CACF,CAEW,KAAK;CAEjB,MAAM,UAA6B,iBAAiB,UAAU,KAAK,KAAK,GAAG,OAAO,IAAK;CACvF,MAAM,SAAiB;EAAE,SAAS,QAAQ;EAAM;EAAS;EAAS;AAElE,MAAK,MAAM,YAAY,UACrB,OAAM,SAAS,OAAO,OAAO"}

package/dist/utils/evaluation/dataset.cjs

@@ -0,0 +1,47 @@
+const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
+let node_fs_promises = require("node:fs/promises");
+node_fs_promises = require_rolldown_runtime.__toESM(node_fs_promises);
+let zod = require("zod");
+
+//#region src/utils/evaluation/dataset.ts
+const recordSchema = zod.z.record(zod.z.any());
+const datasetItemSchema = zod.z.object({
+	id: zod.z.union([zod.z.string(), zod.z.number()]),
+	input: recordSchema,
+	output: recordSchema.optional(),
+	expected: recordSchema.optional(),
+	metadata: recordSchema.optional(),
+	tags: zod.z.array(zod.z.string()).optional(),
+	selected: zod.z.boolean().optional()
+});
+const datasetSchema = zod.z.array(datasetItemSchema);
+var FileDataset = class {
+	name = "file-dataset";
+	filePath;
+	constructor(filePath) {
+		this.filePath = filePath;
+	}
+	async load() {
+		let list;
+		try {
+			list = await node_fs_promises.default.readFile(this.filePath, "utf-8");
+		} catch (err) {
+			throw new Error(`Failed to read dataset file: ${err.message}`);
+		}
+		let parsed;
+		try {
+			parsed = JSON.parse(list);
+		} catch (err) {
+			throw new Error(`Invalid JSON in dataset file: ${err.message}`);
+		}
+		const result = await datasetSchema.safeParseAsync(parsed);
+		if (!result.success) throw new Error(`Invalid dataset file: ${JSON.stringify(result.error.format())}`);
+		return result.data;
+	}
+	async loadWithOptions() {
+		return this.load();
+	}
+};
+
+//#endregion
+exports.FileDataset = FileDataset;

package/dist/utils/evaluation/dataset.mjs

@@ -0,0 +1,46 @@
+import fs from "node:fs/promises";
+import { z as z$1 } from "zod";
+
+//#region src/utils/evaluation/dataset.ts
+const recordSchema = z$1.record(z$1.any());
+const datasetItemSchema = z$1.object({
+	id: z$1.union([z$1.string(), z$1.number()]),
+	input: recordSchema,
+	output: recordSchema.optional(),
+	expected: recordSchema.optional(),
+	metadata: recordSchema.optional(),
+	tags: z$1.array(z$1.string()).optional(),
+	selected: z$1.boolean().optional()
+});
+const datasetSchema = z$1.array(datasetItemSchema);
+var FileDataset = class {
+	name = "file-dataset";
+	filePath;
+	constructor(filePath) {
+		this.filePath = filePath;
+	}
+	async load() {
+		let list;
+		try {
+			list = await fs.readFile(this.filePath, "utf-8");
+		} catch (err) {
+			throw new Error(`Failed to read dataset file: ${err.message}`);
+		}
+		let parsed;
+		try {
+			parsed = JSON.parse(list);
+		} catch (err) {
+			throw new Error(`Invalid JSON in dataset file: ${err.message}`);
+		}
+		const result = await datasetSchema.safeParseAsync(parsed);
+		if (!result.success) throw new Error(`Invalid dataset file: ${JSON.stringify(result.error.format())}`);
+		return result.data;
+	}
+	async loadWithOptions() {
+		return this.load();
+	}
+};
+
+//#endregion
+export { FileDataset };
+//# sourceMappingURL=dataset.mjs.map

package/dist/utils/evaluation/dataset.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.mjs","names":["z"],"sources":["../../../src/utils/evaluation/dataset.ts"],"sourcesContent":["import fs from \"node:fs/promises\";\nimport { z } from \"zod\";\nimport type { Dataset, DatasetItem } from \"./type.js\";\n\nconst recordSchema = z.record(z.any());\n\nconst datasetItemSchema = z.object({\n  id: z.union([z.string(), z.number()]),\n  input: recordSchema,\n  output: recordSchema.optional(),\n  expected: recordSchema.optional(),\n  metadata: recordSchema.optional(),\n  tags: z.array(z.string()).optional(),\n  selected: z.boolean().optional(),\n});\n\nconst datasetSchema = z.array(datasetItemSchema);\n\nexport class FileDataset implements Dataset {\n  name = \"file-dataset\";\n  private filePath: string;\n\n  constructor(filePath: string) {\n    this.filePath = filePath;\n  }\n\n  async load(): Promise<DatasetItem[]> {\n    let list: string;\n    try {\n      list = await fs.readFile(this.filePath, \"utf-8\");\n    } catch (err) {\n      throw new Error(`Failed to read dataset file: ${err.message}`);\n    }\n\n    let parsed: DatasetItem[];\n    try {\n      parsed = JSON.parse(list);\n    } catch (err) {\n      throw new Error(`Invalid JSON in dataset file: ${err.message}`);\n    }\n\n    const result = await datasetSchema.safeParseAsync(parsed);\n    if (!result.success) {\n      throw new Error(`Invalid dataset file: ${JSON.stringify(result.error.format())}`);\n    }\n\n    return result.data;\n  }\n\n  async loadWithOptions(): Promise<DatasetItem[]> {\n    return this.load();\n  }\n}\n\nexport class JsonDataset implements Dataset {\n  name = \"json-dataset\";\n  private data: DatasetItem[];\n\n  constructor(data: DatasetItem[]) {\n    this.data = data;\n  }\n\n  async load(): Promise<DatasetItem[]> {\n    const result = await datasetSchema.safeParseAsync(this.data);\n\n    if (!result.success) {\n      throw new Error(`Invalid dataset file: ${JSON.stringify(result.error.format())}`);\n    }\n\n    return result.data;\n  }\n\n  async loadWithOptions(): Promise<DatasetItem[]> {\n    return this.load();\n  }\n}\n"],"mappings":";;;;AAIA,MAAM,eAAeA,IAAE,OAAOA,IAAE,KAAK,CAAC;AAEtC,MAAM,oBAAoBA,IAAE,OAAO;CACjC,IAAIA,IAAE,MAAM,CAACA,IAAE,QAAQ,EAAEA,IAAE,QAAQ,CAAC,CAAC;CACrC,OAAO;CACP,QAAQ,aAAa,UAAU;CAC/B,UAAU,aAAa,UAAU;CACjC,UAAU,aAAa,UAAU;CACjC,MAAMA,IAAE,MAAMA,IAAE,QAAQ,CAAC,CAAC,UAAU;CACpC,UAAUA,IAAE,SAAS,CAAC,UAAU;CACjC,CAAC;AAEF,MAAM,gBAAgBA,IAAE,MAAM,kBAAkB;AAEhD,IAAa,cAAb,MAA4C;CAC1C,OAAO;CACP,AAAQ;CAER,YAAY,UAAkB;AAC5B,OAAK,WAAW;;CAGlB,MAAM,OAA+B;EACnC,IAAI;AACJ,MAAI;AACF,UAAO,MAAM,GAAG,SAAS,KAAK,UAAU,QAAQ;WACzC,KAAK;AACZ,SAAM,IAAI,MAAM,gCAAgC,IAAI,UAAU;;EAGhE,IAAI;AACJ,MAAI;AACF,YAAS,KAAK,MAAM,KAAK;WAClB,KAAK;AACZ,SAAM,IAAI,MAAM,iCAAiC,IAAI,UAAU;;EAGjE,MAAM,SAAS,MAAM,cAAc,eAAe,OAAO;AACzD,MAAI,CAAC,OAAO,QACV,OAAM,IAAI,MAAM,yBAAyB,KAAK,UAAU,OAAO,MAAM,QAAQ,CAAC,GAAG;AAGnF,SAAO,OAAO;;CAGhB,MAAM,kBAA0C;AAC9C,SAAO,KAAK,MAAM"}

package/dist/utils/evaluation/evaluator.cjs

@@ -0,0 +1,109 @@
+const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
+let _aigne_core = require("@aigne/core");
+let zod = require("zod");
+
+//#region src/utils/evaluation/evaluator.ts
+const defaultAgent = _aigne_core.AIAgent.from({
+	name: "LLMEvaluator",
+	instructions: `
+# Instructions
+You are an expert evaluator. Your task is to evaluate the quality of AI-generated responses.
+You will be given:
+1. User Input (Prompt)
+2. AI-generated Output
+3. Expected Output
+
+## Evaluation Methods
+Follow these three correlation checks before assigning a score:
+1. **AI Output vs User Input**: Check if the AI response is relevant to the user input.
+2. **Expected Output vs User Input**: Check if the expected output is relevant to the user input.
+3. **AI Output vs Expected Output**: Check the similarity and alignment between the AI output and the expected output.
+
+Then assign a rating and a score based on the overall quality.
+
+## Criteria
+- **Instruction following**: Does the AI response follow the prompt’s requirements?
+- **Groundedness**: Is the AI response consistent with the expected output and free from irrelevant information?
+- **Completeness**: Does the AI response fully address the task?
+- **Accuracy/Correctness**: Is the AI response factually correct and logically consistent?
+- **Fluency**: Is the AI response clear, structured, and easy to read?
+
+## Rating Rubric (1–5)
+- **5 - Very Good**: Highly relevant, closely aligned with the expected output, accurate, complete, and fluent.
+- **4 - Good**: Relevant, mostly aligned with the expected output, generally accurate and complete, only minor issues.
+- **3 - Ok**: Somewhat relevant, partially aligned, or missing important details.
+- **2 - Bad**: Weak relevance, low similarity with expected output, contains significant errors or omissions.
+- **1 - Very Bad**: Irrelevant, fails to align with expected output, or completely incorrect.
+
+## Evaluation Steps
+1. Compare the **semantic content** of AI Output vs Expected Output.
+  - Ignore JSON keys, object structure, formatting, whitespace, capitalization, and minor punctuation differences.
+  - If meaning is the same but phrasing differs slightly, assign a higher score (4–5).
+  - If AI output deviates significantly, assign a lower score (1–2).
+  - If AI output is empty, assign a lower score (1–2).
+2. Assess against criteria: instruction following, groundedness, completeness, correctness, fluency.
+3. Assign a 1–5 integer score.
+4. Provide reasoning, and explicitly justify why this result is **not** a 1/2/3 case (why it avoids being a negative example).
+
+# Response Output Format
+Your output must strictly follow this three-line format:
+- First line: rating (Very Good, Good, Ok, Bad, Very Bad)
+- Second line: reasoning (must include justification why it is not a 1, 2, or 3 if scored higher)
+- Third line: SCORE: [1-5]
+
+Example:
+Good
+The response follows most instructions and is largely consistent with the expected output, but it omits one detail. This prevents it from being 5. However, it is more accurate and complete than an "Ok" response, so it deserves 4.
+SCORE: 4
+
+# User Inputs and AI-generated Response
+### Input
+{{input}}
+
+### AI-generated Output
+{{output}}
+
+### Expected Output
+{{expectedOutput}}
+`,
+	inputSchema: zod.z.object({
+		input: zod.z.string().describe("The input content to analyze"),
+		output: zod.z.string().describe("The output content to analyze"),
+		expectedOutput: zod.z.string().describe("The expected output content to analyze")
+	}),
+	outputSchema: zod.z.object({
+		rating: zod.z.enum([
+			"Very Good",
+			"Good",
+			"Ok",
+			"Bad",
+			"Very Bad"
+		]).describe("The rating of the output"),
+		reasoning: zod.z.string().describe("The reasoning of the rating, including justification"),
+		score: zod.z.number().int().min(1).max(5).describe("The score of the output, 1–5, 5 is the best")
+	})
+});
+var LLMEvaluator = class {
+	name = "llm-as-judge";
+	constructor(aigne = new _aigne_core.AIGNE(), agent = defaultAgent) {
+		this.aigne = aigne;
+		this.agent = agent;
+	}
+	async evaluate(dataset) {
+		const result = await this.aigne.invoke(this.agent, {
+			input: typeof dataset.input === "string" ? dataset.input : JSON.stringify(dataset.input, null, 2),
+			output: dataset.output ? JSON.stringify(dataset.output, null, 2) : "",
+			expectedOutput: JSON.stringify(dataset.expected, null, 2)
+		}, { returnMetadata: true });
+		return [{
+			name: this.name,
+			rating: result.rating,
+			score: result.score,
+			reason: result.reasoning,
+			usage: result?.$meta?.usage ?? {}
+		}];
+	}
+};
+
+//#endregion
+exports.LLMEvaluator = LLMEvaluator;

package/dist/utils/evaluation/{evaluator.js → evaluator.mjs}

@@ -1,6 +1,10 @@
 import { AIAgent, AIGNE } from "@aigne/core";
-import { z } from "zod";
-const EVALUATOR_PROMPT = `
+import { z as z$1 } from "zod";
+
+//#region src/utils/evaluation/evaluator.ts
+const defaultAgent = AIAgent.from({
+	name: "LLMEvaluator",
+	instructions: `
 # Instructions
 You are an expert evaluator. Your task is to evaluate the quality of AI-generated responses.
 You will be given:
@@ -60,47 +64,46 @@ SCORE: 4
 
 ### Expected Output
 {{expectedOutput}}
-`;
-const defaultAgent = AIAgent.from({
-    name: "LLMEvaluator",
-    instructions: EVALUATOR_PROMPT,
-    inputSchema: z.object({
-        input: z.string().describe("The input content to analyze"),
-        output: z.string().describe("The output content to analyze"),
-        expectedOutput: z.string().describe("The expected output content to analyze"),
-    }),
-    outputSchema: z.object({
-        rating: z
-            .enum(["Very Good", "Good", "Ok", "Bad", "Very Bad"])
-            .describe("The rating of the output"),
-        reasoning: z.string().describe("The reasoning of the rating, including justification"),
-        score: z.number().int().min(1).max(5).describe("The score of the output, 1–5, 5 is the best"),
-    }),
+`,
+	inputSchema: z$1.object({
+		input: z$1.string().describe("The input content to analyze"),
+		output: z$1.string().describe("The output content to analyze"),
+		expectedOutput: z$1.string().describe("The expected output content to analyze")
+	}),
+	outputSchema: z$1.object({
+		rating: z$1.enum([
+			"Very Good",
+			"Good",
+			"Ok",
+			"Bad",
+			"Very Bad"
+		]).describe("The rating of the output"),
+		reasoning: z$1.string().describe("The reasoning of the rating, including justification"),
+		score: z$1.number().int().min(1).max(5).describe("The score of the output, 1–5, 5 is the best")
+	})
 });
-export class LLMEvaluator {
-    aigne;
-    agent;
-    name = "llm-as-judge";
-    constructor(aigne = new AIGNE(), agent = defaultAgent) {
-        this.aigne = aigne;
-        this.agent = agent;
-    }
-    async evaluate(dataset) {
-        const result = await this.aigne.invoke(this.agent, {
-            input: typeof dataset.input === "string"
-                ? dataset.input
-                : JSON.stringify(dataset.input, null, 2),
-            output: dataset.output ? JSON.stringify(dataset.output, null, 2) : "",
-            expectedOutput: JSON.stringify(dataset.expected, null, 2),
-        }, { returnMetadata: true });
-        return [
-            {
-                name: this.name,
-                rating: result.rating,
-                score: result.score,
-                reason: result.reasoning,
-                usage: result?.$meta?.usage ?? {},
-            },
-        ];
-    }
-}
+var LLMEvaluator = class {
+	name = "llm-as-judge";
+	constructor(aigne = new AIGNE(), agent = defaultAgent) {
+		this.aigne = aigne;
+		this.agent = agent;
+	}
+	async evaluate(dataset) {
+		const result = await this.aigne.invoke(this.agent, {
+			input: typeof dataset.input === "string" ? dataset.input : JSON.stringify(dataset.input, null, 2),
+			output: dataset.output ? JSON.stringify(dataset.output, null, 2) : "",
+			expectedOutput: JSON.stringify(dataset.expected, null, 2)
+		}, { returnMetadata: true });
+		return [{
+			name: this.name,
+			rating: result.rating,
+			score: result.score,
+			reason: result.reasoning,
+			usage: result?.$meta?.usage ?? {}
+		}];
+	}
+};
+
+//#endregion
+export { LLMEvaluator };
+//# sourceMappingURL=evaluator.mjs.map

package/dist/utils/evaluation/evaluator.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.mjs","names":["z"],"sources":["../../../src/utils/evaluation/evaluator.ts"],"sourcesContent":["import { type Agent, AIAgent, AIGNE } from \"@aigne/core\";\nimport { z } from \"zod\";\nimport type { DatasetItem, Evaluation, Evaluator } from \"./type.js\";\n\nconst EVALUATOR_PROMPT = `\n# Instructions\nYou are an expert evaluator. Your task is to evaluate the quality of AI-generated responses.\nYou will be given:\n1. User Input (Prompt)\n2. AI-generated Output\n3. Expected Output\n\n## Evaluation Methods\nFollow these three correlation checks before assigning a score:\n1. **AI Output vs User Input**: Check if the AI response is relevant to the user input.\n2. **Expected Output vs User Input**: Check if the expected output is relevant to the user input.\n3. **AI Output vs Expected Output**: Check the similarity and alignment between the AI output and the expected output.\n\nThen assign a rating and a score based on the overall quality.\n\n## Criteria\n- **Instruction following**: Does the AI response follow the prompt’s requirements?\n- **Groundedness**: Is the AI response consistent with the expected output and free from irrelevant information?\n- **Completeness**: Does the AI response fully address the task?\n- **Accuracy/Correctness**: Is the AI response factually correct and logically consistent?\n- **Fluency**: Is the AI response clear, structured, and easy to read?\n\n## Rating Rubric (1–5)\n- **5 - Very Good**: Highly relevant, closely aligned with the expected output, accurate, complete, and fluent.\n- **4 - Good**: Relevant, mostly aligned with the expected output, generally accurate and complete, only minor issues.\n- **3 - Ok**: Somewhat relevant, partially aligned, or missing important details.\n- **2 - Bad**: Weak relevance, low similarity with expected output, contains significant errors or omissions.\n- **1 - Very Bad**: Irrelevant, fails to align with expected output, or completely incorrect.\n\n## Evaluation Steps\n1. Compare the **semantic content** of AI Output vs Expected Output.\n  - Ignore JSON keys, object structure, formatting, whitespace, capitalization, and minor punctuation differences.\n  - If meaning is the same but phrasing differs slightly, assign a higher score (4–5).\n  - If AI output deviates significantly, assign a lower score (1–2).\n  - If AI output is empty, assign a lower score (1–2).\n2. Assess against criteria: instruction following, groundedness, completeness, correctness, fluency.\n3. Assign a 1–5 integer score.\n4. Provide reasoning, and explicitly justify why this result is **not** a 1/2/3 case (why it avoids being a negative example).\n\n# Response Output Format\nYour output must strictly follow this three-line format:\n- First line: rating (Very Good, Good, Ok, Bad, Very Bad)\n- Second line: reasoning (must include justification why it is not a 1, 2, or 3 if scored higher)\n- Third line: SCORE: [1-5]\n\nExample:\nGood\nThe response follows most instructions and is largely consistent with the expected output, but it omits one detail. This prevents it from being 5. However, it is more accurate and complete than an \"Ok\" response, so it deserves 4.\nSCORE: 4\n\n# User Inputs and AI-generated Response\n### Input\n{{input}}\n\n### AI-generated Output\n{{output}}\n\n### Expected Output\n{{expectedOutput}}\n`;\n\nconst defaultAgent = AIAgent.from({\n  name: \"LLMEvaluator\",\n  instructions: EVALUATOR_PROMPT,\n  inputSchema: z.object({\n    input: z.string().describe(\"The input content to analyze\"),\n    output: z.string().describe(\"The output content to analyze\"),\n    expectedOutput: z.string().describe(\"The expected output content to analyze\"),\n  }),\n  outputSchema: z.object({\n    rating: z\n      .enum([\"Very Good\", \"Good\", \"Ok\", \"Bad\", \"Very Bad\"])\n      .describe(\"The rating of the output\"),\n    reasoning: z.string().describe(\"The reasoning of the rating, including justification\"),\n    score: z.number().int().min(1).max(5).describe(\"The score of the output, 1–5, 5 is the best\"),\n  }),\n});\n\nexport class LLMEvaluator implements Evaluator {\n  name = \"llm-as-judge\";\n\n  constructor(\n    private readonly aigne: AIGNE = new AIGNE(),\n    private readonly agent: Agent = defaultAgent,\n  ) {}\n\n  async evaluate(dataset: DatasetItem): Promise<Evaluation[]> {\n    const result = await this.aigne.invoke(\n      this.agent,\n      {\n        input:\n          typeof dataset.input === \"string\"\n            ? dataset.input\n            : JSON.stringify(dataset.input, null, 2),\n        output: dataset.output ? JSON.stringify(dataset.output, null, 2) : \"\",\n        expectedOutput: JSON.stringify(dataset.expected, null, 2),\n      },\n      { returnMetadata: true },\n    );\n\n    return [\n      {\n        name: this.name,\n        rating: result.rating,\n        score: result.score,\n        reason: result.reasoning,\n        usage: result?.$meta?.usage ?? {},\n      },\n    ];\n  }\n}\n"],"mappings":";;;;AAkEA,MAAM,eAAe,QAAQ,KAAK;CAChC,MAAM;CACN,cAhEuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiEvB,aAAaA,IAAE,OAAO;EACpB,OAAOA,IAAE,QAAQ,CAAC,SAAS,+BAA+B;EAC1D,QAAQA,IAAE,QAAQ,CAAC,SAAS,gCAAgC;EAC5D,gBAAgBA,IAAE,QAAQ,CAAC,SAAS,yCAAyC;EAC9E,CAAC;CACF,cAAcA,IAAE,OAAO;EACrB,QAAQA,IACL,KAAK;GAAC;GAAa;GAAQ;GAAM;GAAO;GAAW,CAAC,CACpD,SAAS,2BAA2B;EACvC,WAAWA,IAAE,QAAQ,CAAC,SAAS,uDAAuD;EACtF,OAAOA,IAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,8CAA8C;EAC9F,CAAC;CACH,CAAC;AAEF,IAAa,eAAb,MAA+C;CAC7C,OAAO;CAEP,YACE,AAAiB,QAAe,IAAI,OAAO,EAC3C,AAAiB,QAAe,cAChC;EAFiB;EACA;;CAGnB,MAAM,SAAS,SAA6C;EAC1D,MAAM,SAAS,MAAM,KAAK,MAAM,OAC9B,KAAK,OACL;GACE,OACE,OAAO,QAAQ,UAAU,WACrB,QAAQ,QACR,KAAK,UAAU,QAAQ,OAAO,MAAM,EAAE;GAC5C,QAAQ,QAAQ,SAAS,KAAK,UAAU,QAAQ,QAAQ,MAAM,EAAE,GAAG;GACnE,gBAAgB,KAAK,UAAU,QAAQ,UAAU,MAAM,EAAE;GAC1D,EACD,EAAE,gBAAgB,MAAM,CACzB;AAED,SAAO,CACL;GACE,MAAM,KAAK;GACX,QAAQ,OAAO;GACf,OAAO,OAAO;GACd,QAAQ,OAAO;GACf,OAAO,QAAQ,OAAO,SAAS,EAAE;GAClC,CACF"}