@agentica/benchmark 0.7.0-dev.20250224-2

package/src/structures/IAgenticaBenchmarkExpected.ts

@@ -0,0 +1,58 @@
+import { IAgenticaOperation } from "@agentica/core";
+
+/**
+ * Expected operation determinant.
+ *
+ * `IAgenticaBenchmarkExpected` is a type for determining what
+ * operation is expected in the benchmarking.
+ *
+ * And `IAgenticaBenchmarkExpected` is an union type of 4 types,
+ * especially designed for the detailed determination of the expected
+ * operations.
+ *
+ * @author Samchon
+ */
+export type IAgenticaBenchmarkExpected =
+  | IAgenticaBenchmarkExpected.IAllOf
+  | IAgenticaBenchmarkExpected.IAnyOf
+  | IAgenticaBenchmarkExpected.IArray
+  | IAgenticaBenchmarkExpected.IStandalone;
+export namespace IAgenticaBenchmarkExpected {
+  /**
+   * All of them must meet the condition, but sequence is not important.
+   */
+  export interface IAllOf {
+    type: "allOf";
+    allOf: Array<
+      Exclude<IAgenticaBenchmarkExpected, IAgenticaBenchmarkExpected.IAllOf>
+    >;
+  }
+
+  /**
+   * At least one of them must meet the condition.
+   */
+  export interface IAnyOf {
+    type: "anyOf";
+    anyOf: Array<
+      Exclude<IAgenticaBenchmarkExpected, IAgenticaBenchmarkExpected.IAnyOf>
+    >;
+  }
+
+  /**
+   * All of them must meet the condition, and sequence is important.
+   */
+  export interface IArray {
+    type: "array";
+    items: Array<
+      Exclude<IAgenticaBenchmarkExpected, IAgenticaBenchmarkExpected.IArray>
+    >;
+  }
+
+  /**
+   * Standalone operation.
+   */
+  export interface IStandalone {
+    type: "standalone";
+    operation: IAgenticaOperation;
+  }
+}

package/src/structures/IAgenticaCallBenchmarkEvent.ts

@@ -0,0 +1,109 @@
+import { IAgenticaPrompt, IAgenticaTokenUsage } from "@agentica/core";
+
+import { IAgenticaCallBenchmarkScenario } from "./IAgenticaCallBenchmarkScenario";
+
+/**
+ * Event of LLM function selection benchmark.
+ *
+ * `IAgenticaCallBenchmarkEvent` is an union type of the events occurred
+ * during the LLM function calling benchmark, representing one phase of
+ * the benchmark testing about a scenario.
+ *
+ * In other words, when {@link AgenticaCallBenchmark} executes the
+ * benchmark, it will run the benchmark will test a scenario repeately with
+ * the given configuration {@link AgenticaCallBenchmark.IConfig.repeat}.
+ * And in the repeated benchmark about a scenario,
+ * `IAgenticaCallBenchmarkEvent` is one of the repeated testing.
+ *
+ * For reference, `IAgenticaCallBenchmarkEvent` is categorized into three
+ * types: `success`, `failure`, and `error`. The `success` means the
+ * benchmark testing is fully meet the expected scenario, and `failure`
+ * means that the `selector` or `caller` agents had not selected or
+ * called the expected operations. The last type, `error`, means that
+ * an error had been occurred during the benchmark testing.
+ *
+ * @author Samchon
+ */
+export type IAgenticaCallBenchmarkEvent =
+  | IAgenticaCallBenchmarkEvent.ISuccess
+  | IAgenticaCallBenchmarkEvent.IFailure
+  | IAgenticaCallBenchmarkEvent.IError;
+export namespace IAgenticaCallBenchmarkEvent {
+  /**
+   * Success event type.
+   *
+   * The `success` event type represents that the benchmark
+   * testing is fully meet the expected scenario.
+   */
+  export interface ISuccess extends IEventBase<"success"> {
+    /**
+     * Whether succeeded to function selection.
+     */
+    select: true;
+
+    /**
+     * Whether succeeded to function call.
+     */
+    call: true;
+  }
+
+  /**
+   * Failure event type.
+   *
+   * The `failure` event type represents that the `selector`
+   * or `caller` agents have not selected or called following the
+   * expected scenario in the benchmark testing.
+   */
+  export interface IFailure extends IEventBase<"failure"> {
+    /**
+     * Whether succeeded to function selection.
+     */
+    select: boolean;
+
+    /**
+     * Whether succeeded to function call.
+     */
+    call: boolean;
+  }
+
+  export interface IError extends IEventBase<"error"> {
+    /**
+     * Error occurred during the benchmark.
+     */
+    error: unknown;
+  }
+
+  interface IEventBase<Type extends string> {
+    /**
+     * Discriminant type.
+     */
+    type: Type;
+
+    /**
+     * Expected scenario.
+     */
+    scenario: IAgenticaCallBenchmarkScenario;
+
+    /**
+     * Prompt histories.
+     *
+     * List of prompts occurred during the benchmark testing.
+     */
+    prompts: IAgenticaPrompt[];
+
+    /**
+     * Usage of the token during the benchmark.
+     */
+    usage: IAgenticaTokenUsage;
+
+    /**
+     * When the benchmark testing started.
+     */
+    started_at: Date;
+
+    /**
+     * When the benchmark testing completed.
+     */
+    completed_at: Date;
+  }
+}

package/src/structures/IAgenticaCallBenchmarkResult.ts

@@ -0,0 +1,69 @@
+import { IAgenticaTokenUsage } from "@agentica/core";
+
+import { IAgenticaCallBenchmarkEvent } from "./IAgenticaCallBenchmarkEvent";
+import { IAgenticaCallBenchmarkScenario } from "./IAgenticaCallBenchmarkScenario";
+
+/**
+ * Result of the LLM function calling benchmark.
+ *
+ * `IAgenticaCallBenchmarkResult` is a structure representing the result
+ * of the LLM function calling benchmark executed by the
+ * {@link AgenticaCallBenchmark.execute execute} function.
+ *
+ * It contains every experiment results for each scenario, and aggregated
+ * LLM token cost in the benchmark process.
+ *
+ * In each scenario, as the benchmark program experiments multiple times
+ * about a scenario, it will contain multiple events. Also, because of the
+ * characteristics of the LLM which is not predictable, the result can be
+ * different in each event.
+ *
+ * @author Samchon
+ */
+export interface IAgenticaCallBenchmarkResult {
+  /**
+   * Experiments for each scenario.
+   */
+  experiments: IAgenticaCallBenchmarkResult.IExperiment[];
+
+  /**
+   * Aggregated token usage information.
+   */
+  usage: IAgenticaTokenUsage;
+
+  /**
+   * Start time of the benchmark.
+   */
+  started_at: Date;
+
+  /**
+   * End time of the benchmark.
+   */
+  completed_at: Date;
+}
+export namespace IAgenticaCallBenchmarkResult {
+  /**
+   * Experiment result about a scenario.
+   */
+  export interface IExperiment {
+    /**
+     * Scenario of the experiment.
+     */
+    scenario: IAgenticaCallBenchmarkScenario;
+
+    /**
+     * Events occurred during the benchmark in the scenario.
+     *
+     * When benchmarking a scenario, {@link AgenticaCallBenchmark} will
+     * test a scenario multiple times with the given
+     * {@link AgenticaCallBenchmark.IConfig.repeat repeat} count.
+     * And the event is one of the repeated benchmark results.
+     */
+    events: IAgenticaCallBenchmarkEvent[];
+
+    /**
+     * LLM token usage information.
+     */
+    usage: IAgenticaTokenUsage;
+  }
+}

package/src/structures/IAgenticaCallBenchmarkScenario.ts

@@ -0,0 +1,39 @@
+import { IAgenticaBenchmarkExpected } from "./IAgenticaBenchmarkExpected";
+
+/**
+ * Scenario of function calling.
+ *
+ * `IAgenticaCallBenchmarkScenario` is a data structure which
+ * represents a function calling benchmark scenario. It contains two
+ * properties; {@linkk text} and {@link operations}.
+ *
+ * The {@link text} means the conversation text from the user, and
+ * the other {@link operations} are the expected operations that
+ * should be selected by the `caller` agent through the {@link text}
+ * conversation.
+ *
+ * @author Samchon
+ */
+export interface IAgenticaCallBenchmarkScenario {
+  /**
+   * Name of the scenario.
+   *
+   * It must be unique within the benchmark scenarios.
+   */
+  name: string;
+
+  /**
+   * The prompt text from user.
+   */
+  text: string;
+
+  /**
+   * Expected function calling sequence.
+   *
+   * Sequence of operations (API operation or class function) that
+   * should be called by both `selector` and `caller` agents from
+   * the user's {@link text} conversation for the LLM
+   * (Large Language Model) function calling.
+   */
+  expected: IAgenticaBenchmarkExpected;
+}

package/src/structures/IAgenticaSelectBenchmarkEvent.ts

@@ -0,0 +1,110 @@
+import {
+  IAgenticaOperationSelection,
+  IAgenticaPrompt,
+  IAgenticaTokenUsage,
+} from "@agentica/core";
+
+import { IAgenticaSelectBenchmarkScenario } from "./IAgenticaSelectBenchmarkScenario";
+
+/**
+ * Event of LLM function selection benchmark.
+ *
+ * `IAgenticaSelectBenchmarkEvent` is an union type of the events occurred
+ * during the LLM function selection benchmark, representing one phase of
+ * the benchmark testing about a scenario.
+ *
+ * In other words, when {@link AgenticaSelectBenchmark} executes the
+ * benchmark, it will run the benchmark will test a scenario repeately with
+ * the given configuration {@link AgenticaSelectBenchmark.IConfig.repeat}.
+ * And in the repeated benchmark about a scenario,
+ * `IAgenticaSelectBenchmarkEvent` is one of the repeated testing.
+ *
+ * For reference, `IAgenticaSelectBenchmarkEvent` is categorized into three
+ * types: `success`, `failure`, and `error`. The `success` means the
+ * benchmark testing is fully meet the expected scenario, and `failure`
+ * means that the `selector` had not selected the expected operations. The
+ * last type, `error`, means that an error had been occurred during the
+ * benchmark testing.
+ *
+ * @author Samchon
+ */
+export type IAgenticaSelectBenchmarkEvent =
+  | IAgenticaSelectBenchmarkEvent.ISuccess
+  | IAgenticaSelectBenchmarkEvent.IFailure
+  | IAgenticaSelectBenchmarkEvent.IError;
+export namespace IAgenticaSelectBenchmarkEvent {
+  /**
+   * Success event type.
+   *
+   * The `success` event type represents that the benchmark testing is
+   * fully meet the expected scenario.
+   */
+  export interface ISuccess extends IEventBase<"success"> {
+    /**
+     * Usage of the token during the benchmark.
+     */
+    usage: IAgenticaTokenUsage;
+
+    /**
+     * Selected operations in the benchmark.
+     */
+    selected: IAgenticaOperationSelection[];
+
+    /**
+     * Prompt messages from the assistant.
+     */
+    assistantPrompts: IAgenticaPrompt.IText<"assistant">[];
+  }
+
+  /**
+   * Failure event type.
+   *
+   * The `failure` event type represents that the `selector` had not
+   * selected the expected scenario in the benchmark testing.
+   */
+  export interface IFailure extends IEventBase<"failure"> {
+    /**
+     * Usage of the token during the benchmark.
+     */
+    usage: IAgenticaTokenUsage;
+
+    /**
+     * Selected operations in the benchmark.
+     */
+    selected: IAgenticaOperationSelection[];
+
+    /**
+     * Prompt messages from the assistant.
+     */
+    assistantPrompts: IAgenticaPrompt.IText<"assistant">[];
+  }
+
+  export interface IError extends IEventBase<"error"> {
+    /**
+     * Error occurred during the benchmark.
+     */
+    error: unknown;
+  }
+
+  interface IEventBase<Type extends string> {
+    /**
+     * Discriminant type.
+     */
+    type: Type;
+
+    /**
+     * Expected scenario.
+     */
+    scenario: IAgenticaSelectBenchmarkScenario;
+
+    /**
+     * When the benchmark testing started.
+     */
+    started_at: Date;
+
+    /**
+     * When the benchmark testing completed.
+     */
+    completed_at: Date;
+  }
+}

package/src/structures/IAgenticaSelectBenchmarkResult.ts

@@ -0,0 +1,69 @@
+import { IAgenticaTokenUsage } from "@agentica/core";
+
+import { IAgenticaSelectBenchmarkEvent } from "./IAgenticaSelectBenchmarkEvent";
+import { IAgenticaSelectBenchmarkScenario } from "./IAgenticaSelectBenchmarkScenario";
+
+/**
+ * Result of the LLM function selection benchmark.
+ *
+ * `IAgenticaSelectBenchmarkResult` is a structure representing the result
+ * of the LLM function selection benchmark executed by the
+ * {@link AgenticaSelectBenchmark.execute execute} function.
+ *
+ * It contains every experiment results for each scenario, and aggregated
+ * LLM token cost in the benchmark process.
+ *
+ * In each scenario, as the benchmark program experiments multiple times
+ * about a scenario, it will contain multiple events. Also, because of the
+ * characteristics of the LLM which is not predictable, the result can be
+ * different in each event.
+ *
+ * @author Samchon
+ */
+export interface IAgenticaSelectBenchmarkResult {
+  /**
+   * Experiments for each scenario.
+   */
+  experiments: IAgenticaSelectBenchmarkResult.IExperiment[];
+
+  /**
+   * Aggregated token usage information.
+   */
+  usage: IAgenticaTokenUsage;
+
+  /**
+   * Start time of the benchmark.
+   */
+  started_at: Date;
+
+  /**
+   * End time of the benchmark.
+   */
+  completed_at: Date;
+}
+export namespace IAgenticaSelectBenchmarkResult {
+  /**
+   * Experiment result about a scenario.
+   */
+  export interface IExperiment {
+    /**
+     * Expected scenario.
+     */
+    scenario: IAgenticaSelectBenchmarkScenario;
+
+    /**
+     * Events occurred during the benchmark in the scenario.
+     *
+     * When benchmarking a scenario, {@link AgenticaSelectBenchmark} will
+     * test a scenario multiple times with the given
+     * {@link AgenticaSelectBenchmark.IConfig.repeat repeat} count.
+     * And the event is one of the repeated benchmark results.
+     */
+    events: IAgenticaSelectBenchmarkEvent[];
+
+    /**
+     * LLM token usage information.
+     */
+    usage: IAgenticaTokenUsage;
+  }
+}

package/src/structures/IAgenticaSelectBenchmarkScenario.ts

@@ -0,0 +1,39 @@
+import { IAgenticaBenchmarkExpected } from "./IAgenticaBenchmarkExpected";
+
+/**
+ * Scenario of function selection.
+ *
+ * `IAgenticaSelectBenchmarkScenario` is a data structure which
+ * represents a function selection benchmark scenario. It contains two
+ * properties; {@linkk text} and {@link operations}.
+ *
+ * The {@link text} means the conversation text from the user, and
+ * the other {@link operations} are the expected operations that
+ * should be selected by the `selector` agent through the {@link text}
+ * conversation.
+ *
+ * @author Samchon
+ */
+export interface IAgenticaSelectBenchmarkScenario {
+  /**
+   * Name of the scenario.
+   *
+   * It must be unique within the benchmark scenarios.
+   */
+  name: string;
+
+  /**
+   * The prompt text from user.
+   */
+  text: string;
+
+  /**
+   * Expected function selection sequence.
+   *
+   * Sequence of operations (API operation or class function) that
+   * should be selected by the `selector` agent from the user's
+   * {@link text} conversation for the LLM (Large Language Model)
+   * function selection.
+   */
+  expected: IAgenticaBenchmarkExpected;
+}

package/src/utils/MathUtil.ts

@@ -0,0 +1,3 @@
+export namespace MathUtil {
+  export const round = (value: number): number => Math.floor(value * 100) / 100;
+}

package/src/utils/TokenUsageComputer.ts

@@ -0,0 +1,40 @@
+import { IAgenticaTokenUsage } from "@agentica/core/src/structures/IAgenticaTokenUsage";
+
+export namespace TokenUsageComputer {
+  export const zero = (): IAgenticaTokenUsage => ({
+    total: 0,
+    prompt: {
+      total: 0,
+      audio: 0,
+      cached: 0,
+    },
+    completion: {
+      total: 0,
+      accepted_prediction: 0,
+      audio: 0,
+      reasoning: 0,
+      rejected_prediction: 0,
+    },
+  });
+
+  export const plus = (
+    a: IAgenticaTokenUsage,
+    b: IAgenticaTokenUsage,
+  ): IAgenticaTokenUsage => ({
+    total: a.total + b.total,
+    prompt: {
+      total: a.prompt.total + b.prompt.total,
+      audio: a.prompt.audio + b.prompt.audio,
+      cached: a.prompt.cached + b.prompt.cached,
+    },
+    completion: {
+      total: a.completion.total + b.completion.total,
+      accepted_prediction:
+        a.completion.accepted_prediction + b.completion.accepted_prediction,
+      audio: a.completion.audio + b.completion.audio,
+      reasoning: a.completion.reasoning + b.completion.reasoning,
+      rejected_prediction:
+        a.completion.rejected_prediction + b.completion.rejected_prediction,
+    },
+  });
+}