@agentica/benchmark 0.43.3 → 0.44.0-dev.20260313

package/src/MicroAgenticaCallBenchmark.ts

@@ -1,281 +1,281 @@
-import type { MicroAgentica } from "@agentica/core";
-import type { tags } from "typia";
-
-/**
- * @module
- * This file contains the implementation of the AgenticaCallBenchmark class.
- *
- * @author Wrtn Technologies
- */
-import { AgenticaTokenUsage } from "@agentica/core";
-import { Semaphore } from "tstl";
-
-import type { IAgenticaCallBenchmarkEvent } from "./structures/IAgenticaCallBenchmarkEvent";
-import type { IAgenticaCallBenchmarkResult } from "./structures/IAgenticaCallBenchmarkResult";
-import type { IAgenticaCallBenchmarkScenario } from "./structures/IAgenticaCallBenchmarkScenario";
-
-import { AgenticaBenchmarkPredicator } from "./internal/AgenticaBenchmarkPredicator";
-import { AgenticaCallBenchmarkReporter } from "./internal/AgenticaCallBenchmarkReporter";
-
-/**
- * LLM function calling selection benchmark.
- *
- * `AgenticaCallBenchmark` is a class for the benchmark of the
- * LLM (Large Model Language) function calling part. It utilizes both
- * `selector` and `caller` agents and tests whether the expected
- * {@link IAgenticaOperation operations} are properly selected and
- * called from the given
- * {@link IAgenticaCallBenchmarkScenario scenarios}.
- *
- * Note that, this `MicroAgenticaCallBenchmark` consumes a lot of time and
- * LLM token costs because it needs the whole process of the
- * {@link MicroAgentica} class with a lot of repetitions. If you don't want
- * such a heavy benchmark, consider to using
- * {@link AgenticaSelectBenchmark} instead. In my experience,
- * {@link MicroAgentica} does not fail to function calling, so the function
- * selection benchmark is much economical.
- *
- * @author Samchon
- */
-export class MicroAgenticaCallBenchmark {
-  private agent_: MicroAgentica;
-  private scenarios_: IAgenticaCallBenchmarkScenario[];
-  private config_: MicroAgenticaCallBenchmark.IConfig;
-  private result_: IAgenticaCallBenchmarkResult | null;
-
-  /**
-   * Initializer Constructor.
-   *
-   * @param props Properties of the selection benchmark
-   */
-  public constructor(props: MicroAgenticaCallBenchmark.IProps) {
-    this.agent_ = props.agent;
-    this.scenarios_ = props.scenarios.slice();
-    this.config_ = {
-      repeat: props.config?.repeat ?? 10,
-      simultaneous: props.config?.simultaneous ?? 10,
-      consent: props.config?.consent ?? 3,
-    };
-    this.result_ = null;
-  }
-
-  /**
-   * Execute the benchmark.
-   *
-   * Execute the benchmark of the LLM function calling, and returns
-   * the result of the benchmark.
-   *
-   * If you wanna see progress of the benchmark, you can pass a callback
-   * function as the argument of the `listener`. The callback function
-   * would be called whenever a benchmark event is occurred.
-   *
-   * Also, you can publish a markdown format report by calling
-   * the {@link report} function after the benchmark execution.
-   *
-   * @param listener Callback function listening the benchmark events
-   * @returns Results of the function calling benchmark
-   */
-  public async execute(
-    listener?: (event: IAgenticaCallBenchmarkEvent) => void,
-  ): Promise<IAgenticaCallBenchmarkResult> {
-    const started_at: Date = new Date();
-    const semaphore: Semaphore = new Semaphore(this.config_.simultaneous);
-    const task = this.scenarios_.map(async (scenario) => {
-      const events: IAgenticaCallBenchmarkEvent[]
-        = await Promise.all(
-          Array.from({ length: this.config_.repeat }).map(async () => {
-            await semaphore.acquire();
-            const e: IAgenticaCallBenchmarkEvent
-              = await this.step(scenario);
-            await semaphore.release();
-
-            if (listener !== undefined) {
-              listener(e);
-            }
-
-            return e;
-          }),
-        );
-      return {
-        scenario,
-        events,
-        usage: events
-          .filter(e => e.type !== "error")
-          .map(e => e.usage)
-          .reduce((acc, cur) => AgenticaTokenUsage.plus(acc, cur), AgenticaTokenUsage.zero()),
-      };
-    });
-    const experiments: IAgenticaCallBenchmarkResult.IExperiment[]
-      = await Promise.all(task);
-    return (this.result_ = {
-      experiments,
-      started_at,
-      completed_at: new Date(),
-      usage: experiments
-        .map(p => p.usage)
-        .reduce((acc, cur) => AgenticaTokenUsage.plus(acc, cur), AgenticaTokenUsage.zero()),
-    });
-  }
-
-  /**
-   * Report the benchmark result as markdown files.
-   *
-   * Report the benchmark result {@link execute}d by
-   * `AgenticaCallBenchmark` as markdown files, and returns a dictionary
-   * object of the markdown reporting files. The key of the dictionary
-   * would be file name, and the value would be the markdown content.
-   *
-   * For reference, the markdown files are composed like below:
-   *
-   * - `./README.md`
-   * - `./scenario-1/README.md`
-   * - `./scenario-1/1.success.md`
-   * - `./scenario-1/2.failure.md`
-   * - `./scenario-1/3.error.md`
-   *
-   * @returns Dictionary of markdown files.
-   */
-  public report(): Record<string, string> {
-    if (this.result_ === null) {
-      throw new Error("Benchmark is not executed yet.");
-    }
-    return AgenticaCallBenchmarkReporter.markdown(this.result_);
-  }
-
-  private async step(
-    scenario: IAgenticaCallBenchmarkScenario,
-  ): Promise<IAgenticaCallBenchmarkEvent> {
-    const agent: MicroAgentica = this.agent_.clone();
-    const started_at: Date = new Date();
-    const success = () =>
-      AgenticaBenchmarkPredicator.success({
-        expected: scenario.expected,
-        operations: agent
-          .getHistories()
-          .filter(p => p.type === "execute")
-          .map(p => p.operation),
-        strict: false,
-      });
-    const out = (): IAgenticaCallBenchmarkEvent => {
-      const select = AgenticaBenchmarkPredicator.success({
-        expected: scenario.expected,
-        operations: agent
-          .getHistories()
-          .filter(p => p.type === "execute")
-          .map(p => p.operation),
-        strict: false,
-      });
-      const call = success();
-      return {
-        type: (call ? "success" : "failure") as "failure",
-        scenario,
-        select,
-        call,
-        prompts: agent.getHistories(),
-        usage: agent.getTokenUsage(),
-        started_at,
-        completed_at: new Date(),
-      } satisfies IAgenticaCallBenchmarkEvent.IFailure;
-    };
-
-    try {
-      await agent.conversate(scenario.text);
-      if (success()) {
-        return out();
-      }
-
-      for (let i: number = 0; i < this.config_.consent; ++i) {
-        const next: string | null
-          = await AgenticaBenchmarkPredicator.isNext(agent);
-        if (next === null) {
-          break;
-        }
-
-        await agent.conversate(next);
-        if (success()) {
-          return out();
-        }
-      }
-      return out();
-    }
-    catch (error) {
-      return {
-        type: "error",
-        scenario,
-        prompts: agent.getHistories(),
-        usage: agent.getTokenUsage(),
-        error,
-        started_at,
-        completed_at: new Date(),
-      };
-    }
-  }
-}
-export namespace MicroAgenticaCallBenchmark {
-  /**
-   * Properties of the {@link MicroAgenticaCallBenchmark} constructor.
-   */
-  export interface IProps {
-    /**
-     * AI agent instance.
-     */
-    agent: MicroAgentica;
-
-    /**
-     * List of scenarios what you expect.
-     */
-    scenarios: IAgenticaCallBenchmarkScenario[];
-
-    /**
-     * Configuration for the benchmark.
-     */
-    config?: Partial<IConfig>;
-  }
-
-  /**
-   * Configuration for the benchmark.
-   *
-   * `AgenticaSelectBenchmark.IConfig` is a data structure which
-   * represents a configuration for the benchmark, especially the
-   * capacity information of the benchmark execution.
-   */
-  export interface IConfig {
-    /**
-     * Repeat count.
-     *
-     * The number of repeating count for the benchmark execution
-     * for each scenario.
-     *
-     * @default 10
-     */
-    repeat: number & tags.Type<"uint32"> & tags.Minimum<1>;
-
-    /**
-     * Simultaneous count.
-     *
-     * The number of simultaneous count for the parallel benchmark
-     * execution.
-     *
-     * If you configure this property greater than `1`, the benchmark
-     * for each scenario would be executed in parallel in the given
-     * count.
-     *
-     * @default 10
-     */
-    simultaneous: number & tags.Type<"uint32"> & tags.Minimum<1>;
-
-    /**
-     * Number of consents.
-     *
-     * AI agent sometimes asks user to consent to the function
-     * calling, and perform it at the next step.
-     *
-     * This property represents the number of consents to allow.
-     * If the number of consents from the AI agent exceeds the
-     * configured value, the benchmark will be failed.
-     *
-     * @default 3
-     */
-    consent: number;
-  }
-}
+import type { MicroAgentica } from "@agentica/core";
+import type { tags } from "typia";
+
+/**
+ * @module
+ * This file contains the implementation of the AgenticaCallBenchmark class.
+ *
+ * @author Wrtn Technologies
+ */
+import { AgenticaTokenUsage } from "@agentica/core";
+import { Semaphore } from "tstl";
+
+import type { IAgenticaCallBenchmarkEvent } from "./structures/IAgenticaCallBenchmarkEvent";
+import type { IAgenticaCallBenchmarkResult } from "./structures/IAgenticaCallBenchmarkResult";
+import type { IAgenticaCallBenchmarkScenario } from "./structures/IAgenticaCallBenchmarkScenario";
+
+import { AgenticaBenchmarkPredicator } from "./internal/AgenticaBenchmarkPredicator";
+import { AgenticaCallBenchmarkReporter } from "./internal/AgenticaCallBenchmarkReporter";
+
+/**
+ * LLM function calling selection benchmark.
+ *
+ * `AgenticaCallBenchmark` is a class for the benchmark of the
+ * LLM (Large Model Language) function calling part. It utilizes both
+ * `selector` and `caller` agents and tests whether the expected
+ * {@link IAgenticaOperation operations} are properly selected and
+ * called from the given
+ * {@link IAgenticaCallBenchmarkScenario scenarios}.
+ *
+ * Note that, this `MicroAgenticaCallBenchmark` consumes a lot of time and
+ * LLM token costs because it needs the whole process of the
+ * {@link MicroAgentica} class with a lot of repetitions. If you don't want
+ * such a heavy benchmark, consider to using
+ * {@link AgenticaSelectBenchmark} instead. In my experience,
+ * {@link MicroAgentica} does not fail to function calling, so the function
+ * selection benchmark is much economical.
+ *
+ * @author Samchon
+ */
+export class MicroAgenticaCallBenchmark {
+  private agent_: MicroAgentica;
+  private scenarios_: IAgenticaCallBenchmarkScenario[];
+  private config_: MicroAgenticaCallBenchmark.IConfig;
+  private result_: IAgenticaCallBenchmarkResult | null;
+
+  /**
+   * Initializer Constructor.
+   *
+   * @param props Properties of the selection benchmark
+   */
+  public constructor(props: MicroAgenticaCallBenchmark.IProps) {
+    this.agent_ = props.agent;
+    this.scenarios_ = props.scenarios.slice();
+    this.config_ = {
+      repeat: props.config?.repeat ?? 10,
+      simultaneous: props.config?.simultaneous ?? 10,
+      consent: props.config?.consent ?? 3,
+    };
+    this.result_ = null;
+  }
+
+  /**
+   * Execute the benchmark.
+   *
+   * Execute the benchmark of the LLM function calling, and returns
+   * the result of the benchmark.
+   *
+   * If you wanna see progress of the benchmark, you can pass a callback
+   * function as the argument of the `listener`. The callback function
+   * would be called whenever a benchmark event is occurred.
+   *
+   * Also, you can publish a markdown format report by calling
+   * the {@link report} function after the benchmark execution.
+   *
+   * @param listener Callback function listening the benchmark events
+   * @returns Results of the function calling benchmark
+   */
+  public async execute(
+    listener?: (event: IAgenticaCallBenchmarkEvent) => void,
+  ): Promise<IAgenticaCallBenchmarkResult> {
+    const started_at: Date = new Date();
+    const semaphore: Semaphore = new Semaphore(this.config_.simultaneous);
+    const task = this.scenarios_.map(async (scenario) => {
+      const events: IAgenticaCallBenchmarkEvent[]
+        = await Promise.all(
+          Array.from({ length: this.config_.repeat }).map(async () => {
+            await semaphore.acquire();
+            const e: IAgenticaCallBenchmarkEvent
+              = await this.step(scenario);
+            await semaphore.release();
+
+            if (listener !== undefined) {
+              listener(e);
+            }
+
+            return e;
+          }),
+        );
+      return {
+        scenario,
+        events,
+        usage: events
+          .filter(e => e.type !== "error")
+          .map(e => e.usage)
+          .reduce((acc, cur) => AgenticaTokenUsage.plus(acc, cur), AgenticaTokenUsage.zero()),
+      };
+    });
+    const experiments: IAgenticaCallBenchmarkResult.IExperiment[]
+      = await Promise.all(task);
+    return (this.result_ = {
+      experiments,
+      started_at,
+      completed_at: new Date(),
+      usage: experiments
+        .map(p => p.usage)
+        .reduce((acc, cur) => AgenticaTokenUsage.plus(acc, cur), AgenticaTokenUsage.zero()),
+    });
+  }
+
+  /**
+   * Report the benchmark result as markdown files.
+   *
+   * Report the benchmark result {@link execute}d by
+   * `AgenticaCallBenchmark` as markdown files, and returns a dictionary
+   * object of the markdown reporting files. The key of the dictionary
+   * would be file name, and the value would be the markdown content.
+   *
+   * For reference, the markdown files are composed like below:
+   *
+   * - `./README.md`
+   * - `./scenario-1/README.md`
+   * - `./scenario-1/1.success.md`
+   * - `./scenario-1/2.failure.md`
+   * - `./scenario-1/3.error.md`
+   *
+   * @returns Dictionary of markdown files.
+   */
+  public report(): Record<string, string> {
+    if (this.result_ === null) {
+      throw new Error("Benchmark is not executed yet.");
+    }
+    return AgenticaCallBenchmarkReporter.markdown(this.result_);
+  }
+
+  private async step(
+    scenario: IAgenticaCallBenchmarkScenario,
+  ): Promise<IAgenticaCallBenchmarkEvent> {
+    const agent: MicroAgentica = this.agent_.clone();
+    const started_at: Date = new Date();
+    const success = () =>
+      AgenticaBenchmarkPredicator.success({
+        expected: scenario.expected,
+        operations: agent
+          .getHistories()
+          .filter(p => p.type === "execute")
+          .map(p => p.operation),
+        strict: false,
+      });
+    const out = (): IAgenticaCallBenchmarkEvent => {
+      const select = AgenticaBenchmarkPredicator.success({
+        expected: scenario.expected,
+        operations: agent
+          .getHistories()
+          .filter(p => p.type === "execute")
+          .map(p => p.operation),
+        strict: false,
+      });
+      const call = success();
+      return {
+        type: (call ? "success" : "failure") as "failure",
+        scenario,
+        select,
+        call,
+        prompts: agent.getHistories(),
+        usage: agent.getTokenUsage(),
+        started_at,
+        completed_at: new Date(),
+      } satisfies IAgenticaCallBenchmarkEvent.IFailure;
+    };
+
+    try {
+      await agent.conversate(scenario.text);
+      if (success()) {
+        return out();
+      }
+
+      for (let i: number = 0; i < this.config_.consent; ++i) {
+        const next: string | null
+          = await AgenticaBenchmarkPredicator.isNext(agent);
+        if (next === null) {
+          break;
+        }
+
+        await agent.conversate(next);
+        if (success()) {
+          return out();
+        }
+      }
+      return out();
+    }
+    catch (error) {
+      return {
+        type: "error",
+        scenario,
+        prompts: agent.getHistories(),
+        usage: agent.getTokenUsage(),
+        error,
+        started_at,
+        completed_at: new Date(),
+      };
+    }
+  }
+}
+export namespace MicroAgenticaCallBenchmark {
+  /**
+   * Properties of the {@link MicroAgenticaCallBenchmark} constructor.
+   */
+  export interface IProps {
+    /**
+     * AI agent instance.
+     */
+    agent: MicroAgentica;
+
+    /**
+     * List of scenarios what you expect.
+     */
+    scenarios: IAgenticaCallBenchmarkScenario[];
+
+    /**
+     * Configuration for the benchmark.
+     */
+    config?: Partial<IConfig>;
+  }
+
+  /**
+   * Configuration for the benchmark.
+   *
+   * `AgenticaSelectBenchmark.IConfig` is a data structure which
+   * represents a configuration for the benchmark, especially the
+   * capacity information of the benchmark execution.
+   */
+  export interface IConfig {
+    /**
+     * Repeat count.
+     *
+     * The number of repeating count for the benchmark execution
+     * for each scenario.
+     *
+     * @default 10
+     */
+    repeat: number & tags.Type<"uint32"> & tags.Minimum<1>;
+
+    /**
+     * Simultaneous count.
+     *
+     * The number of simultaneous count for the parallel benchmark
+     * execution.
+     *
+     * If you configure this property greater than `1`, the benchmark
+     * for each scenario would be executed in parallel in the given
+     * count.
+     *
+     * @default 10
+     */
+    simultaneous: number & tags.Type<"uint32"> & tags.Minimum<1>;
+
+    /**
+     * Number of consents.
+     *
+     * AI agent sometimes asks user to consent to the function
+     * calling, and perform it at the next step.
+     *
+     * This property represents the number of consents to allow.
+     * If the number of consents from the AI agent exceeds the
+     * configured value, the benchmark will be failed.
+     *
+     * @default 3
+     */
+    consent: number;
+  }
+}

package/src/index.ts

@@ -1,3 +1,3 @@
-export * from "./AgenticaCallBenchmark";
-export * from "./AgenticaSelectBenchmark";
-export * from "./MicroAgenticaCallBenchmark";
+export * from "./AgenticaCallBenchmark";
+export * from "./AgenticaSelectBenchmark";
+export * from "./MicroAgenticaCallBenchmark";