@agentica/benchmark 0.12.21 → 0.13.0

package/src/internal/AgenticaSelectBenchmarkReporter.ts

@@ -1,215 +1,217 @@
-import { AgenticaTokenUsage } from "@agentica/core";
-import { ILlmSchema } from "@samchon/openapi";
+/**
+ * @module
+ * This file contains functions to work with AgenticaSelectBenchmarkReporter.
+ *
+ * @author Wrtn Technologies
+ */
+import type { AgenticaTokenUsage } from "@agentica/core";
+import type { ILlmSchema } from "@samchon/openapi";
 
-import { IAgenticaSelectBenchmarkEvent } from "../structures/IAgenticaSelectBenchmarkEvent";
-import { IAgenticaSelectBenchmarkResult } from "../structures/IAgenticaSelectBenchmarkResult";
+import type { IAgenticaSelectBenchmarkEvent } from "../structures/IAgenticaSelectBenchmarkEvent";
+import type { IAgenticaSelectBenchmarkResult } from "../structures/IAgenticaSelectBenchmarkResult";
 import { MathUtil } from "../utils/MathUtil";
 import { AgenticaBenchmarkUtil } from "./AgenticaBenchmarkUtil";
 
 /**
  * @internal
  */
-export namespace AgenticaSelectBenchmarkReporter {
-  export const markdown = <Model extends ILlmSchema.Model>(
-    result: IAgenticaSelectBenchmarkResult<Model>,
-  ): Record<string, string> =>
-    Object.fromEntries([
-      ["./README.md", writeIndex(result)],
-      ...result.experiments
-        .map((exp) => [
-          [`./${exp.scenario.name}/README.md`, writeExperimentIndex(exp)],
-          ...exp.events.map((event, i) => [
-            `./${exp.scenario.name}/${i + 1}.${event.type}.md`,
-            writeExperimentEvent(event, i),
-          ]),
-        ])
-        .flat(),
-    ]);
+export const AgenticaSelectBenchmarkReporter = {
+  markdown,
+};
 
-  const writeIndex = <Model extends ILlmSchema.Model>(
-    result: IAgenticaSelectBenchmarkResult<Model>,
-  ): string => {
-    const events: IAgenticaSelectBenchmarkEvent<Model>[] = result.experiments
-      .map((r) => r.events)
-      .flat();
-    const average: number =
-      events
-        .map((e) => e.completed_at.getTime() - e.started_at.getTime())
+export function markdown<Model extends ILlmSchema.Model>(result: IAgenticaSelectBenchmarkResult<Model>): Record<string, string> {
+  const iterator = [
+    ["./README.md", writeIndex(result)],
+    ...result.experiments
+      .map<[string, string][]>(exp => [
+        [`./${exp.scenario.name}/README.md`, writeExperimentIndex(exp)],
+        ...exp.events.map<[string, string]>((event, i) => [
+          `./${exp.scenario.name}/${i + 1}.${event.type}.md`,
+          writeExperimentEvent(event, i),
+        ]),
+      ])
+      .flat(),
+  ] satisfies [string, string][];
+
+  return Object.fromEntries(iterator);
+}
+
+function writeIndex<Model extends ILlmSchema.Model>(result: IAgenticaSelectBenchmarkResult<Model>): string {
+  const events: IAgenticaSelectBenchmarkEvent<Model>[] = result.experiments
+    .map(r => r.events)
+    .flat();
+  const average: number
+      = events
+        .map(e => e.completed_at.getTime() - e.started_at.getTime())
         .reduce((a, b) => a + b, 0) / events.length;
-    const aggregate: AgenticaTokenUsage.IComponent = result.usage.aggregate;
-    return [
-      "# LLM Function Selection Benchmark",
-      "## Summary",
-      `  - Aggregation:`,
-      `    - Scenarios: #${result.experiments.length.toLocaleString()}`,
-      `    - Trial: ${events.length}`,
-      `    - Success: ${events.filter((e) => e.type === "success").length}`,
-      `    - Failure: ${events.filter((e) => e.type === "failure").length}`,
-      `    - Average Time: ${MathUtil.round(average).toLocaleString()} ms`,
-      `  - Token Usage`,
-      `    - Total: ${aggregate.total.toLocaleString()}`,
-      `    - Input`,
-      `      - Total: ${aggregate.input.total.toLocaleString()}`,
-      `      - Cached: ${aggregate.input.cached.toLocaleString()}`,
-      `    - Output:`,
-      `      - Total: ${aggregate.output.total.toLocaleString()}`,
-      `      - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`,
-      `      - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`,
-      `      - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`,
-      "",
-      "## Experiments",
-      " Name | Status | Time/Avg  ",
-      ":-----|:-------|----------:",
-      ...result.experiments.map((exp) =>
-        [
-          `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`,
-          (() => {
-            const success: number = Math.floor(
-              (exp.events.filter((e) => e.type === "success").length /
-                exp.events.length) *
-                10,
-            );
-            return (
-              new Array(success).fill("■").join("") +
-              new Array(10 - success).fill("□").join("")
-            );
-          })(),
-          MathUtil.round(
-            exp.events
-              .map(
-                (event) =>
-                  event.completed_at.getTime() - event.started_at.getTime(),
-              )
-              .reduce((a, b) => a + b, 0) / exp.events.length,
-          ).toLocaleString() + " ms",
-        ].join(" | "),
-      ),
-    ].join("\n");
-  };
+  const aggregate: AgenticaTokenUsage.IComponent = result.usage.aggregate;
+  return [
+    "# LLM Function Selection Benchmark",
+    "## Summary",
+    `  - Aggregation:`,
+    `    - Scenarios: #${result.experiments.length.toLocaleString()}`,
+    `    - Trial: ${events.length}`,
+    `    - Success: ${events.filter(e => e.type === "success").length}`,
+    `    - Failure: ${events.filter(e => e.type === "failure").length}`,
+    `    - Average Time: ${MathUtil.round(average).toLocaleString()} ms`,
+    `  - Token Usage`,
+    `    - Total: ${aggregate.total.toLocaleString()}`,
+    `    - Input`,
+    `      - Total: ${aggregate.input.total.toLocaleString()}`,
+    `      - Cached: ${aggregate.input.cached.toLocaleString()}`,
+    `    - Output:`,
+    `      - Total: ${aggregate.output.total.toLocaleString()}`,
+    `      - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`,
+    `      - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`,
+    `      - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`,
+    "",
+    "## Experiments",
+    " Name | Status | Time/Avg  ",
+    ":-----|:-------|----------:",
+    ...result.experiments.map(exp =>
+      [
+        `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`,
+        (() => {
+          const success: number = Math.floor(
+            (exp.events.filter(e => e.type === "success").length
+              / exp.events.length)
+            * 10,
+          );
+          return (
+            Array.from({ length: success }).fill("■").join("")
+            + Array.from({ length: 10 - success }).fill("□").join("")
+          );
+        })(),
+        `${MathUtil.round(
+          exp.events
+            .map(
+              event =>
+                event.completed_at.getTime() - event.started_at.getTime(),
+            )
+            .reduce((a, b) => a + b, 0) / exp.events.length,
+        ).toLocaleString()} ms`,
+      ].join(" | "),
+    ),
+  ].join("\n");
+}
 
-  const writeExperimentIndex = <Model extends ILlmSchema.Model>(
-    exp: IAgenticaSelectBenchmarkResult.IExperiment<Model>,
-  ): string => {
-    const aggregate: AgenticaTokenUsage.IComponent = exp.usage.aggregate;
-    return [
-      `# ${exp.scenario.name}`,
-      "## Summary",
-      "  - Aggregation:",
-      `    - Trial: ${exp.events.length}`,
-      `    - Success: ${exp.events.filter((e) => e.type === "success").length}`,
-      `    - Failure: ${exp.events.filter((e) => e.type === "failure").length}`,
-      `    - Average Time: ${MathUtil.round(
-        exp.events
-          .map(
-            (event) =>
-              event.completed_at.getTime() - event.started_at.getTime(),
-          )
-          .reduce((a, b) => a + b, 0) / exp.events.length,
-      ).toLocaleString()} ms`,
-      `  - Token Usage`,
-      `    - Total: ${aggregate.total.toLocaleString()}`,
-      `    - Input`,
-      `      - Total: ${aggregate.input.total.toLocaleString()}`,
-      `      - Cached: ${aggregate.input.cached.toLocaleString()}`,
-      `    - Output:`,
-      `      - Total: ${aggregate.output.total.toLocaleString()}`,
-      `      - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`,
-      `      - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`,
-      `      - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`,
-      "",
-      "## Events",
-      " No | Type | Time",
-      "---:|:-----|----:",
-      ...exp.events.map((e, i) =>
-        [
-          `[${i + 1}.](./${i + 1}.${e.type}.md)`,
-          e.type,
-          MathUtil.round(e.completed_at.getTime() - e.started_at.getTime()) +
-            " ms",
-        ].join(" | "),
-      ),
-      "",
-      "## Scenario",
-      "### User Prompt",
-      exp.scenario.text,
-      "",
-      "### Expected",
-      "```json",
-      JSON.stringify(
-        AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected),
-        null,
-        2,
-      ),
-      "```",
-    ].join("\n");
-  };
+function writeExperimentIndex<Model extends ILlmSchema.Model>(exp: IAgenticaSelectBenchmarkResult.IExperiment<Model>): string {
+  const aggregate: AgenticaTokenUsage.IComponent = exp.usage.aggregate;
+  return [
+    `# ${exp.scenario.name}`,
+    "## Summary",
+    "  - Aggregation:",
+    `    - Trial: ${exp.events.length}`,
+    `    - Success: ${exp.events.filter(e => e.type === "success").length}`,
+    `    - Failure: ${exp.events.filter(e => e.type === "failure").length}`,
+    `    - Average Time: ${MathUtil.round(
+      exp.events
+        .map(
+          event =>
+            event.completed_at.getTime() - event.started_at.getTime(),
+        )
+        .reduce((a, b) => a + b, 0) / exp.events.length,
+    ).toLocaleString()} ms`,
+    `  - Token Usage`,
+    `    - Total: ${aggregate.total.toLocaleString()}`,
+    `    - Input`,
+    `      - Total: ${aggregate.input.total.toLocaleString()}`,
+    `      - Cached: ${aggregate.input.cached.toLocaleString()}`,
+    `    - Output:`,
+    `      - Total: ${aggregate.output.total.toLocaleString()}`,
+    `      - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`,
+    `      - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`,
+    `      - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`,
+    "",
+    "## Events",
+    " No | Type | Time",
+    "---:|:-----|----:",
+    ...exp.events.map((e, i) =>
+      [
+        `[${i + 1}.](./${i + 1}.${e.type}.md)`,
+        e.type,
+        `${MathUtil.round(e.completed_at.getTime() - e.started_at.getTime())
+        } ms`,
+      ].join(" | "),
+    ),
+    "",
+    "## Scenario",
+    "### User Prompt",
+    exp.scenario.text,
+    "",
+    "### Expected",
+    "```json",
+    JSON.stringify(
+      AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected),
+      null,
+      2,
+    ),
+    "```",
+  ].join("\n");
+}
 
-  const writeExperimentEvent = <Model extends ILlmSchema.Model>(
-    event: IAgenticaSelectBenchmarkEvent<Model>,
-    index: number,
-  ): string => {
-    return [
-      `# ${index + 1}. ${event.type}`,
-      `## Summary`,
-      `  - Name: ${event.scenario.name}`,
-      `  - Type: ${event.type}`,
-      `  - Time: ${(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`,
-      ...(event.type !== "error"
-        ? [
-            "  - Token Usage",
-            `    - Total: ${event.usage.aggregate.toLocaleString()}`,
-            `    - Prompt`,
-            `      - Total: ${event.usage.aggregate.input.total.toLocaleString()}`,
-            `      - Cached: ${event.usage.aggregate.input.cached.toLocaleString()}`,
-            `    - Completion:`,
-            `      - Total: ${event.usage.aggregate.output.total.toLocaleString()}`,
-            `      - Reasoning: ${event.usage.aggregate.output.reasoning.toLocaleString()}`,
-            `      - Accepted Prediction: ${event.usage.aggregate.output.accepted_prediction.toLocaleString()}`,
-            `      - Rejected Prediction: ${event.usage.aggregate.output.rejected_prediction.toLocaleString()}`,
-          ]
-        : []),
-      "",
-      "## Scenario",
-      "### User Prompt",
-      event.scenario.text,
-      "",
-      "### Expected",
-      "```json",
-      JSON.stringify(
-        AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected),
-        null,
-        2,
-      ),
-      "```",
-      "",
-      ...(event.type === "success" || event.type === "failure"
-        ? [
-            "## Result",
-            ...event.selected.map((s) =>
-              [
-                `### ${s.operation.name}`,
-                `  - Controller: \`${s.operation.controller.name}\``,
-                `  - Function: \`${s.operation.function.name}\``,
-                `  - Reason: ${s.reason}`,
-                "",
-                ...(s.operation.function.description
-                  ? [s.operation.function.description, ""]
-                  : []),
-              ].join("\n"),
-            ),
-          ]
-        : []),
-      ...(event.type === "error"
-        ? [
-            "## Error",
-            "```json",
-            AgenticaBenchmarkUtil.errorToJson(
-              JSON.stringify(event.error, null, 2),
-            ),
-            "```",
-            "",
-          ]
-        : []),
-    ].join("\n");
-  };
+function writeExperimentEvent<Model extends ILlmSchema.Model>(event: IAgenticaSelectBenchmarkEvent<Model>, index: number): string {
+  return [
+    `# ${index + 1}. ${event.type}`,
+    `## Summary`,
+    `  - Name: ${event.scenario.name}`,
+    `  - Type: ${event.type}`,
+    `  - Time: ${(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`,
+    ...(event.type !== "error"
+      ? [
+          "  - Token Usage",
+          `    - Total: ${event.usage.aggregate.toLocaleString()}`,
+          `    - Prompt`,
+          `      - Total: ${event.usage.aggregate.input.total.toLocaleString()}`,
+          `      - Cached: ${event.usage.aggregate.input.cached.toLocaleString()}`,
+          `    - Completion:`,
+          `      - Total: ${event.usage.aggregate.output.total.toLocaleString()}`,
+          `      - Reasoning: ${event.usage.aggregate.output.reasoning.toLocaleString()}`,
+          `      - Accepted Prediction: ${event.usage.aggregate.output.accepted_prediction.toLocaleString()}`,
+          `      - Rejected Prediction: ${event.usage.aggregate.output.rejected_prediction.toLocaleString()}`,
+        ]
+      : []),
+    "",
+    "## Scenario",
+    "### User Prompt",
+    event.scenario.text,
+    "",
+    "### Expected",
+    "```json",
+    JSON.stringify(
+      AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected),
+      null,
+      2,
+    ),
+    "```",
+    "",
+    ...(event.type === "success" || event.type === "failure"
+      ? [
+          "## Result",
+          ...event.selected.map(s =>
+            [
+              `### ${s.operation.name}`,
+              `  - Controller: \`${s.operation.controller.name}\``,
+              `  - Function: \`${s.operation.function.name}\``,
+              `  - Reason: ${s.reason}`,
+              "",
+              ...(s.operation.function.description !== undefined && s.operation.function.description !== ""
+                ? [s.operation.function.description, ""]
+                : []),
+            ].join("\n"),
+          ),
+        ]
+      : []),
+    ...(event.type === "error"
+      ? [
+          "## Error",
+          "```json",
+          AgenticaBenchmarkUtil.errorToJson(
+            JSON.stringify(event.error, null, 2),
+          ),
+          "```",
+          "",
+        ]
+      : []),
+  ].join("\n");
 }

package/src/structures/IAgenticaBenchmarkExpected.ts

@@ -1,5 +1,11 @@
-import { AgenticaOperation } from "@agentica/core";
-import { ILlmSchema } from "@samchon/openapi";
+/**
+ * @module
+ * This file contains the implementation of the IAgenticaBenchmarkExpected class.
+ *
+ * @author Wrtn Technologies
+ */
+import type { AgenticaOperation } from "@agentica/core";
+import type { ILlmSchema } from "@samchon/openapi";
 
 /**
  * Expected operation determinant.
@@ -18,6 +24,7 @@ export type IAgenticaBenchmarkExpected<Model extends ILlmSchema.Model> =
   | IAgenticaBenchmarkExpected.IAnyOf<Model>
   | IAgenticaBenchmarkExpected.IArray<Model>
   | IAgenticaBenchmarkExpected.IStandalone<Model>;
+
 export namespace IAgenticaBenchmarkExpected {
   /**
    * All of them must meet the condition, but sequence is not important.

package/src/structures/IAgenticaCallBenchmarkEvent.ts

@@ -1,7 +1,13 @@
-import { AgenticaPrompt, AgenticaTokenUsage } from "@agentica/core";
-import { ILlmSchema } from "@samchon/openapi";
+/**
+ * @module
+ * This file contains the implementation of the IAgenticaCallBenchmarkEvent class.
+ *
+ * @author Wrtn Technologies
+ */
+import type { AgenticaPrompt, AgenticaTokenUsage } from "@agentica/core";
+import type { ILlmSchema } from "@samchon/openapi";
 
-import { IAgenticaCallBenchmarkScenario } from "./IAgenticaCallBenchmarkScenario";
+import type { IAgenticaCallBenchmarkScenario } from "./IAgenticaCallBenchmarkScenario";
 
 /**
  * Event of LLM function selection benchmark.

package/src/structures/IAgenticaCallBenchmarkResult.ts

@@ -1,8 +1,14 @@
-import { AgenticaTokenUsage } from "@agentica/core";
-import { ILlmSchema } from "@samchon/openapi";
+/**
+ * @module
+ * This file contains the implementation of the IAgenticaCallBenchmarkResult class.
+ *
+ * @author Wrtn Technologies
+ */
+import type { AgenticaTokenUsage } from "@agentica/core";
+import type { ILlmSchema } from "@samchon/openapi";
 
-import { IAgenticaCallBenchmarkEvent } from "./IAgenticaCallBenchmarkEvent";
-import { IAgenticaCallBenchmarkScenario } from "./IAgenticaCallBenchmarkScenario";
+import type { IAgenticaCallBenchmarkEvent } from "./IAgenticaCallBenchmarkEvent";
+import type { IAgenticaCallBenchmarkScenario } from "./IAgenticaCallBenchmarkScenario";
 
 /**
  * Result of the LLM function calling benchmark.

package/src/structures/IAgenticaCallBenchmarkScenario.ts

@@ -1,6 +1,12 @@
-import { ILlmSchema } from "@samchon/openapi";
+/**
+ * @module
+ * This file contains the implementation of the IAgenticaCallBenchmarkScenario class.
+ *
+ * @author Wrtn Technologies
+ */
+import type { ILlmSchema } from "@samchon/openapi";
 
-import { IAgenticaBenchmarkExpected } from "./IAgenticaBenchmarkExpected";
+import type { IAgenticaBenchmarkExpected } from "./IAgenticaBenchmarkExpected";
 
 /**
  * Scenario of function calling.

package/src/structures/IAgenticaSelectBenchmarkEvent.ts

@@ -1,11 +1,17 @@
-import {
+/**
+ * @module
+ * This file contains the implementation of the IAgenticaSelectBenchmarkEvent class.
+ *
+ * @author Wrtn Technologies
+ */
+import type {
   AgenticaOperationSelection,
   AgenticaTextPrompt,
   AgenticaTokenUsage,
 } from "@agentica/core";
-import { ILlmSchema } from "@samchon/openapi";
+import type { ILlmSchema } from "@samchon/openapi";
 
-import { IAgenticaSelectBenchmarkScenario } from "./IAgenticaSelectBenchmarkScenario";
+import type { IAgenticaSelectBenchmarkScenario } from "./IAgenticaSelectBenchmarkScenario";
 
 /**
  * Event of LLM function selection benchmark.

package/src/structures/IAgenticaSelectBenchmarkResult.ts

@@ -1,8 +1,14 @@
-import { AgenticaTokenUsage } from "@agentica/core";
-import { ILlmSchema } from "@samchon/openapi";
+/**
+ * @module
+ * This file contains the implementation of the IAgenticaSelectBenchmarkResult class.
+ *
+ * @author Wrtn Technologies
+ */
+import type { AgenticaTokenUsage } from "@agentica/core";
+import type { ILlmSchema } from "@samchon/openapi";
 
-import { IAgenticaSelectBenchmarkEvent } from "./IAgenticaSelectBenchmarkEvent";
-import { IAgenticaSelectBenchmarkScenario } from "./IAgenticaSelectBenchmarkScenario";
+import type { IAgenticaSelectBenchmarkEvent } from "./IAgenticaSelectBenchmarkEvent";
+import type { IAgenticaSelectBenchmarkScenario } from "./IAgenticaSelectBenchmarkScenario";
 
 /**
  * Result of the LLM function selection benchmark.

package/src/structures/IAgenticaSelectBenchmarkScenario.ts

@@ -1,6 +1,12 @@
-import { ILlmSchema } from "@samchon/openapi";
+/**
+ * @module
+ * This file contains the implementation of the IAgenticaSelectBenchmarkScenario class.
+ *
+ * @author Wrtn Technologies
+ */
+import type { ILlmSchema } from "@samchon/openapi";
 
-import { IAgenticaBenchmarkExpected } from "./IAgenticaBenchmarkExpected";
+import type { IAgenticaBenchmarkExpected } from "./IAgenticaBenchmarkExpected";
 
 /**
  * Scenario of function selection.

package/src/utils/MathUtil.ts

@@ -1,3 +1,16 @@
-export namespace MathUtil {
-  export const round = (value: number): number => Math.floor(value * 100) / 100;
-}
+/**
+ * @module
+ * This file contains functions to work with MathUtil.
+ *
+ * @author Wrtn Technologies
+ */
+
+export const MathUtil = {
+  /**
+   * Round a number to 2 decimal places.
+   *
+   * @param value - The number to round.
+   * @returns The rounded number.
+   */
+  round: (value: number): number => Math.floor(value * 100) / 100,
+};