@agentica/benchmark 0.43.3 → 0.44.0-dev.20260313

package/src/internal/AgenticaCallBenchmarkReporter.ts

@@ -1,192 +1,192 @@
-/**
- * @module
- * This file contains functions to work with AgenticaCallBenchmarkReporter.
- *
- * @author Wrtn Technologies
- */
-import type { AgenticaTokenUsage } from "@agentica/core";
-
-import type { IAgenticaCallBenchmarkEvent } from "../structures/IAgenticaCallBenchmarkEvent";
-import type { IAgenticaCallBenchmarkResult } from "../structures/IAgenticaCallBenchmarkResult";
-
-import { MathUtil } from "../utils/MathUtil";
-
-import { AgenticaBenchmarkUtil } from "./AgenticaBenchmarkUtil";
-import { AgenticaPromptReporter } from "./AgenticaPromptReporter";
-
-export const AgenticaCallBenchmarkReporter = {
-  markdown,
-};
-
-export function markdown(result: IAgenticaCallBenchmarkResult): Record<string, string> {
-  return Object.fromEntries([
-    ["./README.md", writeIndex(result)],
-    ...result.experiments
-      .map<[string, string][]>(exp => [
-        [`./${exp.scenario.name}/README.md`, writeExperimentIndex(exp)],
-        ...exp.events.map<[string, string]>((event, i) => [
-          `./${exp.scenario.name}/${i + 1}.${event.type}.md`,
-          writeExperimentEvent(event, i),
-        ]),
-      ])
-      .flat(),
-  ]);
-}
-
-function writeIndex(result: IAgenticaCallBenchmarkResult): string {
-  const events: IAgenticaCallBenchmarkEvent[] = result.experiments
-    .map(r => r.events)
-    .flat();
-  const average: number
-    = events
-      .map(e => e.completed_at.getTime() - e.started_at.getTime())
-      .reduce((a, b) => a + b, 0) / events.length;
-  const aggregate: AgenticaTokenUsage.IComponent = result.usage.aggregate;
-  return [
-    "# LLM Function Call Benchmark",
-    "## Summary",
-    `  - Aggregation:`,
-    `    - Scenarios: #${result.experiments.length.toLocaleString()}`,
-    `    - Trial: ${events.length}`,
-    `    - Success: ${events.filter(e => e.type === "success").length}`,
-    `    - Failure: ${events.filter(e => e.type === "failure").length}`,
-    `    - Average Time: ${MathUtil.round(average).toLocaleString()} ms`,
-    `  - Token Usage`,
-    `    - Total: ${aggregate.total.toLocaleString()}`,
-    `    - Input`,
-    `      - Total: ${aggregate.input.total.toLocaleString()}`,
-    `      - Cached: ${aggregate.input.cached.toLocaleString()}`,
-    `    - Output:`,
-    `      - Total: ${aggregate.output.total.toLocaleString()}`,
-    `      - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`,
-    `      - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`,
-    `      - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`,
-    "",
-    "## Experiments",
-    " Name | Select | Call | Time/Avg ",
-    ":-----|:-------|:-----|----------:",
-    ...result.experiments.map(exp =>
-      [
-        `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`,
-        drawStatus(
-          exp.events,
-          e => e.type !== "error" && e.select === true,
-        ),
-        drawStatus(exp.events, e => e.type !== "error" && e.call === true),
-        `${MathUtil.round(
-          exp.events
-            .map(e => e.completed_at.getTime() - e.started_at.getTime())
-            .reduce((a, b) => a + b, 0) / exp.events.length,
-        ).toLocaleString()} ms`,
-      ].join(" | "),
-    ),
-  ].join("\n");
-}
-
-function writeExperimentIndex(exp: IAgenticaCallBenchmarkResult.IExperiment): string {
-  return [
-    `# ${exp.scenario.name}`,
-    "## Summary",
-    `  - Scenarios: #${exp.events.length.toLocaleString()}`,
-    `  - Success: ${exp.events.filter(e => e.type === "success").length}`,
-    `  - Failure: ${exp.events.filter(e => e.type === "failure").length}`,
-    `  - Average Time: ${MathUtil.round(
-      exp.events
-        .map(e => e.completed_at.getTime() - e.started_at.getTime())
-        .reduce((a, b) => a + b, 0) / exp.events.length,
-    ).toLocaleString()} ms`,
-    "",
-    "## Events",
-    " Name | Type | Time",
-    ":-----|:-----|----:",
-    ...exp.events.map((e, i) =>
-      [
-        `[${i + 1}.](./${i + 1}.${e.type}.md)`,
-        e.type,
-        `${MathUtil.round(e.completed_at.getTime() - e.started_at.getTime())} ms`,
-      ].join(" | "),
-    ),
-    "",
-    "## Scenario",
-    "### User Prompt",
-    exp.scenario.text,
-    "",
-    "### Expected",
-    "```json",
-    JSON.stringify(
-      AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected),
-      null,
-      2,
-    ),
-    "```",
-  ].join("\n");
-}
-
-function writeExperimentEvent(event: IAgenticaCallBenchmarkEvent, index: number): string {
-  return [
-    `# ${index + 1}. ${event.type}`,
-    "## Summary",
-    `  - Name: ${event.scenario.name}`,
-    `  - Type: ${event.type}`,
-    `  - Time: ${MathUtil.round(
-      event.completed_at.getTime() - event.started_at.getTime(),
-    ).toLocaleString()} ms`,
-    ...(event.type !== "error"
-      ? [
-          `  - Select: ${event.select ? "✅" : "❌"}`,
-          `  - Call: ${event.call ? "✅" : "❌"}`,
-        ]
-      : []),
-    `  - Token Usage:`,
-    `    - Total: ${JSON.stringify(event.usage.aggregate.total)}`,
-    `    - Input`,
-    `      - Total: ${event.usage.aggregate.input.total}`,
-    `      - Cached: ${event.usage.aggregate.input.cached}`,
-    `    - Output:`,
-    `      - Total: ${event.usage.aggregate.output.total}`,
-    `      - Accepted Prediction: ${event.usage.aggregate.output.accepted_prediction}`,
-    `      - Reasoning: ${event.usage.aggregate.output.reasoning}`,
-    `      - Rejected Prediction: ${event.usage.aggregate.output.rejected_prediction}`,
-
-    "",
-    "## Scenario",
-    "### User Prompt",
-    event.scenario.text,
-    "",
-    "### Expected",
-    "```json",
-    JSON.stringify(
-      AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected),
-      null,
-      2,
-    ),
-    "```",
-    "",
-    "## Prompt Histories",
-    ...event.prompts.map(AgenticaPromptReporter.markdown),
-    "",
-    ...(event.type === "error"
-      ? [
-          "## Error",
-          "```json",
-          JSON.stringify(
-            AgenticaBenchmarkUtil.errorToJson(event.error),
-            null,
-            2,
-          ),
-          "```",
-        ]
-      : []),
-  ].join("\n");
-}
-
-function drawStatus(events: IAgenticaCallBenchmarkEvent[], success: (e: IAgenticaCallBenchmarkEvent) => boolean): string {
-  const count: number = Math.floor(
-    (events.filter(success).length / events.length) * 10,
-  );
-  // @TODO use String.prototype.padStart, padEnd or String.prototype.repeat
-  return (
-    Array.from({ length: count }).fill("■").join("")
-    + Array.from({ length: 10 - count }).fill("□").join("")
-  );
-}
+/**
+ * @module
+ * This file contains functions to work with AgenticaCallBenchmarkReporter.
+ *
+ * @author Wrtn Technologies
+ */
+import type { AgenticaTokenUsage } from "@agentica/core";
+
+import type { IAgenticaCallBenchmarkEvent } from "../structures/IAgenticaCallBenchmarkEvent";
+import type { IAgenticaCallBenchmarkResult } from "../structures/IAgenticaCallBenchmarkResult";
+
+import { MathUtil } from "../utils/MathUtil";
+
+import { AgenticaBenchmarkUtil } from "./AgenticaBenchmarkUtil";
+import { AgenticaPromptReporter } from "./AgenticaPromptReporter";
+
+export const AgenticaCallBenchmarkReporter = {
+  markdown,
+};
+
+export function markdown(result: IAgenticaCallBenchmarkResult): Record<string, string> {
+  return Object.fromEntries([
+    ["./README.md", writeIndex(result)],
+    ...result.experiments
+      .map<[string, string][]>(exp => [
+        [`./${exp.scenario.name}/README.md`, writeExperimentIndex(exp)],
+        ...exp.events.map<[string, string]>((event, i) => [
+          `./${exp.scenario.name}/${i + 1}.${event.type}.md`,
+          writeExperimentEvent(event, i),
+        ]),
+      ])
+      .flat(),
+  ]);
+}
+
+function writeIndex(result: IAgenticaCallBenchmarkResult): string {
+  const events: IAgenticaCallBenchmarkEvent[] = result.experiments
+    .map(r => r.events)
+    .flat();
+  const average: number
+    = events
+      .map(e => e.completed_at.getTime() - e.started_at.getTime())
+      .reduce((a, b) => a + b, 0) / events.length;
+  const aggregate: AgenticaTokenUsage.IComponent = result.usage.aggregate;
+  return [
+    "# LLM Function Call Benchmark",
+    "## Summary",
+    `  - Aggregation:`,
+    `    - Scenarios: #${result.experiments.length.toLocaleString()}`,
+    `    - Trial: ${events.length}`,
+    `    - Success: ${events.filter(e => e.type === "success").length}`,
+    `    - Failure: ${events.filter(e => e.type === "failure").length}`,
+    `    - Average Time: ${MathUtil.round(average).toLocaleString()} ms`,
+    `  - Token Usage`,
+    `    - Total: ${aggregate.total.toLocaleString()}`,
+    `    - Input`,
+    `      - Total: ${aggregate.input.total.toLocaleString()}`,
+    `      - Cached: ${aggregate.input.cached.toLocaleString()}`,
+    `    - Output:`,
+    `      - Total: ${aggregate.output.total.toLocaleString()}`,
+    `      - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`,
+    `      - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`,
+    `      - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`,
+    "",
+    "## Experiments",
+    " Name | Select | Call | Time/Avg ",
+    ":-----|:-------|:-----|----------:",
+    ...result.experiments.map(exp =>
+      [
+        `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`,
+        drawStatus(
+          exp.events,
+          e => e.type !== "error" && e.select === true,
+        ),
+        drawStatus(exp.events, e => e.type !== "error" && e.call === true),
+        `${MathUtil.round(
+          exp.events
+            .map(e => e.completed_at.getTime() - e.started_at.getTime())
+            .reduce((a, b) => a + b, 0) / exp.events.length,
+        ).toLocaleString()} ms`,
+      ].join(" | "),
+    ),
+  ].join("\n");
+}
+
+function writeExperimentIndex(exp: IAgenticaCallBenchmarkResult.IExperiment): string {
+  return [
+    `# ${exp.scenario.name}`,
+    "## Summary",
+    `  - Scenarios: #${exp.events.length.toLocaleString()}`,
+    `  - Success: ${exp.events.filter(e => e.type === "success").length}`,
+    `  - Failure: ${exp.events.filter(e => e.type === "failure").length}`,
+    `  - Average Time: ${MathUtil.round(
+      exp.events
+        .map(e => e.completed_at.getTime() - e.started_at.getTime())
+        .reduce((a, b) => a + b, 0) / exp.events.length,
+    ).toLocaleString()} ms`,
+    "",
+    "## Events",
+    " Name | Type | Time",
+    ":-----|:-----|----:",
+    ...exp.events.map((e, i) =>
+      [
+        `[${i + 1}.](./${i + 1}.${e.type}.md)`,
+        e.type,
+        `${MathUtil.round(e.completed_at.getTime() - e.started_at.getTime())} ms`,
+      ].join(" | "),
+    ),
+    "",
+    "## Scenario",
+    "### User Prompt",
+    exp.scenario.text,
+    "",
+    "### Expected",
+    "```json",
+    JSON.stringify(
+      AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected),
+      null,
+      2,
+    ),
+    "```",
+  ].join("\n");
+}
+
+function writeExperimentEvent(event: IAgenticaCallBenchmarkEvent, index: number): string {
+  return [
+    `# ${index + 1}. ${event.type}`,
+    "## Summary",
+    `  - Name: ${event.scenario.name}`,
+    `  - Type: ${event.type}`,
+    `  - Time: ${MathUtil.round(
+      event.completed_at.getTime() - event.started_at.getTime(),
+    ).toLocaleString()} ms`,
+    ...(event.type !== "error"
+      ? [
+          `  - Select: ${event.select ? "✅" : "❌"}`,
+          `  - Call: ${event.call ? "✅" : "❌"}`,
+        ]
+      : []),
+    `  - Token Usage:`,
+    `    - Total: ${JSON.stringify(event.usage.aggregate.total)}`,
+    `    - Input`,
+    `      - Total: ${event.usage.aggregate.input.total}`,
+    `      - Cached: ${event.usage.aggregate.input.cached}`,
+    `    - Output:`,
+    `      - Total: ${event.usage.aggregate.output.total}`,
+    `      - Accepted Prediction: ${event.usage.aggregate.output.accepted_prediction}`,
+    `      - Reasoning: ${event.usage.aggregate.output.reasoning}`,
+    `      - Rejected Prediction: ${event.usage.aggregate.output.rejected_prediction}`,
+
+    "",
+    "## Scenario",
+    "### User Prompt",
+    event.scenario.text,
+    "",
+    "### Expected",
+    "```json",
+    JSON.stringify(
+      AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected),
+      null,
+      2,
+    ),
+    "```",
+    "",
+    "## Prompt Histories",
+    ...event.prompts.map(AgenticaPromptReporter.markdown),
+    "",
+    ...(event.type === "error"
+      ? [
+          "## Error",
+          "```json",
+          JSON.stringify(
+            AgenticaBenchmarkUtil.errorToJson(event.error),
+            null,
+            2,
+          ),
+          "```",
+        ]
+      : []),
+  ].join("\n");
+}
+
+function drawStatus(events: IAgenticaCallBenchmarkEvent[], success: (e: IAgenticaCallBenchmarkEvent) => boolean): string {
+  const count: number = Math.floor(
+    (events.filter(success).length / events.length) * 10,
+  );
+  // @TODO use String.prototype.padStart, padEnd or String.prototype.repeat
+  return (
+    Array.from({ length: count }).fill("■").join("")
+    + Array.from({ length: 10 - count }).fill("□").join("")
+  );
+}

package/src/internal/AgenticaPromptReporter.ts

@@ -1,63 +1,63 @@
-/**
- * @module
- * This file contains functions to work with AgenticaPromptReporter.
- *
- * @author Wrtn Technologies
- */
-import type { AgenticaHistory } from "@agentica/core";
-
-export const AgenticaPromptReporter = {
-  markdown,
-};
-
-function markdown(p: AgenticaHistory): string {
-  if (p.type === "userMessage") {
-    return [`### User Input`, p.contents, ""].join("\n");
-  }
-  if (p.type === "assistantMessage") {
-    return [`### Assistant`, p.text, ""].join("\n");
-  }
-  if (p.type === "systemMessage") {
-    return [`### System`, p.text, ""].join("\n");
-  }
-
-  if (p.type === "select" || p.type === "cancel") {
-    return [
-      `### ${p.type === "select" ? "Select" : "Cancel"}`,
-      `  - controller: ${p.selection.operation.controller.name}`,
-      `  - function: ${p.selection.operation.function.name}`,
-      `  - reason: ${p.selection.reason}`,
-      "",
-      ...((p.selection.operation.function.description?.length ?? 0) !== 0
-        ? [p.selection.operation.function.description, ""]
-        : []),
-    ].join("\n");
-  }
-
-  if (p.type === "describe") {
-    return [
-      "### Describe",
-      ...p.executes.map(e => `  - ${e.operation.name}`),
-      "",
-      ...p.text.split("\n").map(s => `> ${s}`),
-      "",
-    ].join("\n");
-  }
-
-  if (p.type === "execute") {
-    return [
-      "### Execute",
-      `  - name: ${p.operation.name}`,
-      `  - controller: ${p.operation.controller.name}`,
-      `  - function: ${p.operation.function.name}`,
-      "",
-      "```json",
-      JSON.stringify(p.arguments, null, 2),
-      "```",
-      "",
-    ].join("\n");
-  }
-
-  p satisfies never;
-  throw new Error("Invalid history type");
-}
+/**
+ * @module
+ * This file contains functions to work with AgenticaPromptReporter.
+ *
+ * @author Wrtn Technologies
+ */
+import type { AgenticaHistory } from "@agentica/core";
+
+export const AgenticaPromptReporter = {
+  markdown,
+};
+
+function markdown(p: AgenticaHistory): string {
+  if (p.type === "userMessage") {
+    return [`### User Input`, p.contents, ""].join("\n");
+  }
+  if (p.type === "assistantMessage") {
+    return [`### Assistant`, p.text, ""].join("\n");
+  }
+  if (p.type === "systemMessage") {
+    return [`### System`, p.text, ""].join("\n");
+  }
+
+  if (p.type === "select" || p.type === "cancel") {
+    return [
+      `### ${p.type === "select" ? "Select" : "Cancel"}`,
+      `  - controller: ${p.selection.operation.controller.name}`,
+      `  - function: ${p.selection.operation.function.name}`,
+      `  - reason: ${p.selection.reason}`,
+      "",
+      ...((p.selection.operation.function.description?.length ?? 0) !== 0
+        ? [p.selection.operation.function.description, ""]
+        : []),
+    ].join("\n");
+  }
+
+  if (p.type === "describe") {
+    return [
+      "### Describe",
+      ...p.executes.map(e => `  - ${e.operation.name}`),
+      "",
+      ...p.text.split("\n").map(s => `> ${s}`),
+      "",
+    ].join("\n");
+  }
+
+  if (p.type === "execute") {
+    return [
+      "### Execute",
+      `  - name: ${p.operation.name}`,
+      `  - controller: ${p.operation.controller.name}`,
+      `  - function: ${p.operation.function.name}`,
+      "",
+      "```json",
+      JSON.stringify(p.arguments, null, 2),
+      "```",
+      "",
+    ].join("\n");
+  }
+
+  p satisfies never;
+  throw new Error("Invalid history type");
+}