@agentica/benchmark 0.7.0-dev.20250224-2

package/src/internal/AgenticaBenchmarkPredicator.ts

@@ -0,0 +1,216 @@
+import { Agentica, IAgenticaOperation, IAgenticaPrompt } from "@agentica/core";
+import { ILlmFunction } from "@samchon/openapi";
+import OpenAI from "openai";
+import typia from "typia";
+
+import { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
+
+export namespace AgenticaBenchmarkPredicator {
+  export const isNext = async (agent: Agentica): Promise<string | null> => {
+    const last: IAgenticaPrompt | undefined = agent.getPromptHistories().at(-1);
+    if (last?.type !== "text" || last.role !== "assistant") return null;
+
+    const consent: ILlmFunction<"chatgpt"> = typia.llm.application<
+      IPredicatorApplication,
+      "chatgpt"
+    >().functions[0]!;
+    const result: OpenAI.ChatCompletion = await agent[
+      "props"
+    ].provider.api.chat.completions.create(
+      {
+        model: agent["props"].provider.model,
+        messages: [
+          {
+            role: "system",
+            content: [
+              "You are an helpful assistant.",
+              "",
+              "If what the assistant said seems like to asking for",
+              "user's consent about some function calling at the next step,",
+              "use the tools appropriately to step to the next.",
+            ].join("\n"),
+          },
+          {
+            role: "assistant",
+            content: last.text,
+          },
+        ],
+        tools: [
+          {
+            type: "function",
+            function: {
+              name: consent.name,
+              description: consent.description,
+              parameters: consent.parameters as Record<string, any>,
+            },
+          },
+        ],
+        tool_choice: "required",
+        parallel_tool_calls: false,
+      },
+      agent["props"].provider.options,
+    );
+    const toolCall: OpenAI.ChatCompletionMessageToolCall | undefined = (
+      result.choices[0]?.message.tool_calls ?? []
+    ).filter(
+      (tc) => tc.type === "function" && tc.function.name === consent.name,
+    )?.[0];
+    if (toolCall === undefined) return null;
+    const input: IConsentProps = JSON.parse(toolCall.function.arguments);
+    return typia.is(input) ? input.reply : null;
+  };
+
+  /**
+   * Check if the called operations match the expected operations.
+   *
+   * @param props Properties for checking the match of the called operations
+   * and the expected operations
+   *
+   * @returns `true` if the called operations match the expected operations,
+   * otherwise `false`.
+   */
+  export const success = (props: {
+    /**
+     * Expected operations to be called.
+     *
+     * For 'allOf' within an 'array', the next expected element starts checking from the element that follows the last called element in 'allOf'.
+     */
+    expected: IAgenticaBenchmarkExpected;
+
+    /**
+     * Specified operations.
+     */
+    operations: Array<IAgenticaOperation | IAgenticaPrompt.IExecute>;
+
+    /**
+     * If it's `false`, check the array and let it go even if there's something wrong between them.
+     *
+     * @default `false`
+     */
+    strict?: boolean;
+  }): boolean => successInner(props).result;
+
+  const successInner = (
+    props: Parameters<typeof success>[0],
+  ):
+    | {
+        result: true;
+        take: number;
+      }
+    | {
+        result: false;
+      } => {
+    const call = (
+      expected: IAgenticaBenchmarkExpected,
+      overrideOperations?: Array<IAgenticaOperation | IAgenticaPrompt.IExecute>,
+    ) =>
+      successInner({
+        expected,
+        operations: overrideOperations ?? props.operations,
+        strict: props.strict,
+      });
+
+    switch (props.expected.type) {
+      case "array": {
+        let take = 0;
+        const targetIterator = props.expected.items[Symbol.iterator]();
+        let targeted = targetIterator.next();
+
+        while (true) {
+          if (targeted.done) {
+            return {
+              result: true,
+              take,
+            };
+          }
+          if (take >= props.operations.length) {
+            return { result: false };
+          }
+
+          const result = call(targeted.value, props.operations.slice(take));
+          if (!result.result) {
+            if (!props.strict) {
+              take += 1;
+              continue;
+            }
+            return { result: false };
+          }
+
+          take += result.take;
+          targeted = targetIterator.next();
+        }
+      }
+      case "standalone": {
+        const target = props.expected.operation;
+        const result = props.operations.some((op) => op.name === target.name);
+        if (result) {
+          return { result, take: 1 };
+        }
+        return {
+          result,
+        };
+      }
+      case "anyOf":
+        for (const expected of props.expected.anyOf) {
+          const callResult = call(expected);
+          if (callResult.result) {
+            return callResult;
+          }
+        }
+
+        return { result: false };
+      case "allOf": {
+        /**
+         * @example
+         * expected = [4, 2];
+         * called = [1, 2, 3, 4, 5];
+         *
+         * { result: true, take: 3 };
+         */
+        const result = props.expected.allOf.map((expected) => call(expected));
+        if (result.every((r) => r.result)) {
+          return {
+            result: true,
+            take: result.reduce((acc, r) => Math.max(acc, r.take), 0),
+          };
+        }
+
+        return {
+          result: false,
+        };
+      }
+    }
+  };
+}
+
+interface IPredicatorApplication {
+  /**
+   * Ask user to consent for what the AI agent wants to do next.
+   *
+   * If AI agent wants to do some function calling at next,
+   * but it needs the user's consent about the function calling to do,
+   * then call this tool function.
+   *
+   * @param props Properties for asking the user's consent
+   */
+  consent(props: IConsentProps): void;
+}
+
+/**
+ * Properties for asking the user's consent
+ */
+interface IConsentProps {
+  /**
+   * Reason of the message implying what the AI agent wants
+   * to do at the next step after the user's consent.
+   */
+  content: string;
+
+  /**
+   * Recommended reply message for the user.
+   *
+   * The message what AI agent wants the user to reply
+   * accepting the AI agent's next job suggestion.
+   */
+  reply: string;
+}

package/src/internal/AgenticaBenchmarkUtil.ts

@@ -0,0 +1,40 @@
+import { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
+
+export namespace AgenticaBenchmarkUtil {
+  export const errorToJson = (error: any): any => {
+    if (error instanceof Error)
+      return {
+        ...error,
+        name: error.name,
+        message: error.message,
+        stack: error.stack,
+      };
+    return error;
+  };
+
+  export const expectedToJson = (expected: IAgenticaBenchmarkExpected): any => {
+    if (expected.type === "standalone")
+      return {
+        type: expected.type,
+        operation: {
+          name: expected.operation.name,
+          description: expected.operation.function.description,
+        },
+      };
+    else if (expected.type === "array")
+      return {
+        type: expected.type,
+        items: expected.items.map(expectedToJson),
+      };
+    else if (expected.type === "allOf")
+      return {
+        type: expected.type,
+        allOf: expected.allOf.map(expectedToJson),
+      };
+    else
+      return {
+        type: expected.type,
+        anyOf: expected.anyOf.map(expectedToJson),
+      };
+  };
+}

package/src/internal/AgenticaCallBenchmarkReporter.ts

@@ -0,0 +1,177 @@
+import { IAgenticaCallBenchmarkEvent } from "../structures/IAgenticaCallBenchmarkEvent";
+import { IAgenticaCallBenchmarkResult } from "../structures/IAgenticaCallBenchmarkResult";
+import { MathUtil } from "../utils/MathUtil";
+import { AgenticaBenchmarkUtil } from "./AgenticaBenchmarkUtil";
+import { AgenticaPromptReporter } from "./AgenticaPromptReporter";
+
+export namespace AgenticaCallBenchmarkReporter {
+  export const markdown = (
+    result: IAgenticaCallBenchmarkResult,
+  ): Record<string, string> =>
+    Object.fromEntries([
+      ["./README.md", writeIndex(result)],
+      ...result.experiments
+        .map((exp) => [
+          [`./${exp.scenario.name}/README.md`, writeExperimentIndex(exp)],
+          ...exp.events.map((event, i) => [
+            `./${exp.scenario.name}/${i + 1}.${event.type}.md`,
+            writeExperimentEvent(event, i),
+          ]),
+        ])
+        .flat(),
+    ]);
+
+  const writeIndex = (result: IAgenticaCallBenchmarkResult): string => {
+    const events: IAgenticaCallBenchmarkEvent[] = result.experiments
+      .map((r) => r.events)
+      .flat();
+    const average: number =
+      events
+        .map((e) => e.completed_at.getTime() - e.started_at.getTime())
+        .reduce((a, b) => a + b, 0) / events.length;
+    return [
+      "# LLM Function Call Benchmark",
+      "## Summary",
+      `  - Aggregation:`,
+      `    - Scenarios: #${result.experiments.length.toLocaleString()}`,
+      `    - Trial: ${events.length}`,
+      `    - Success: ${events.filter((e) => e.type === "success").length}`,
+      `    - Failure: ${events.filter((e) => e.type === "failure").length}`,
+      `    - Average Time: ${MathUtil.round(average).toLocaleString()} ms`,
+      `  - Token Usage`,
+      `    - Total: ${result.usage.total.toLocaleString()}`,
+      `    - Prompt`,
+      `      - Total: ${result.usage.prompt.total.toLocaleString()}`,
+      `      - Audio: ${result.usage.prompt.audio.toLocaleString()}`,
+      `      - Cached: ${result.usage.prompt.cached.toLocaleString()}`,
+      `    - Completion:`,
+      `      - Total: ${result.usage.completion.total.toLocaleString()}`,
+      `      - Accepted Prediction: ${result.usage.completion.accepted_prediction.toLocaleString()}`,
+      `      - Audio: ${result.usage.completion.audio.toLocaleString()}`,
+      `      - Reasoning: ${result.usage.completion.reasoning.toLocaleString()}`,
+      `      - Rejected Prediction: ${result.usage.completion.rejected_prediction.toLocaleString()}`,
+      "",
+      "## Experiments",
+      " Name | Select | Call | Time/Avg ",
+      ":-----|:-------|:-----|----------:",
+      ...result.experiments.map((exp) =>
+        [
+          `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`,
+          drawStatus(
+            exp.events,
+            (e) => e.type !== "error" && e.select === true,
+          ),
+          drawStatus(exp.events, (e) => e.type !== "error" && e.call === true),
+          `${MathUtil.round(
+            exp.events
+              .map((e) => e.completed_at.getTime() - e.started_at.getTime())
+              .reduce((a, b) => a + b, 0) / exp.events.length,
+          ).toLocaleString()} ms`,
+        ].join(" | "),
+      ),
+    ].join("\n");
+  };
+
+  const writeExperimentIndex = (
+    exp: IAgenticaCallBenchmarkResult.IExperiment,
+  ): string => {
+    return [
+      `# ${exp.scenario.name}`,
+      "## Summary",
+      `  - Scenarios: #${exp.events.length.toLocaleString()}`,
+      `  - Success: ${exp.events.filter((e) => e.type === "success").length}`,
+      `  - Failure: ${exp.events.filter((e) => e.type === "failure").length}`,
+      `  - Average Time: ${MathUtil.round(
+        exp.events
+          .map((e) => e.completed_at.getTime() - e.started_at.getTime())
+          .reduce((a, b) => a + b, 0) / exp.events.length,
+      ).toLocaleString()} ms`,
+      "",
+      "## Events",
+      " Name | Type | Time",
+      ":-----|:-----|----:",
+      ...exp.events.map((e, i) =>
+        [
+          `[${i + 1}.](./${i + 1}.${e.type}.md)`,
+          e.type,
+          `${MathUtil.round(e.completed_at.getTime() - e.started_at.getTime())} ms`,
+        ].join(" | "),
+      ),
+      "",
+      "## Scenario",
+      "### User Prompt",
+      exp.scenario.text,
+      "",
+      "### Expected",
+      "```json",
+      JSON.stringify(
+        AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected),
+        null,
+        2,
+      ),
+      "```",
+    ].join("\n");
+  };
+
+  const writeExperimentEvent = (
+    event: IAgenticaCallBenchmarkEvent,
+    index: number,
+  ): string => {
+    return [
+      `# ${index}. ${event.type}`,
+      "## Summary",
+      `  - Name: ${event.scenario.name}`,
+      `  - Type: ${event.type}`,
+      `  - Time: ${MathUtil.round(
+        event.completed_at.getTime() - event.started_at.getTime(),
+      ).toLocaleString()} ms`,
+      ...(event.type !== "error"
+        ? [
+            `  - Select: ${event.select ? "✅" : "❌"}`,
+            `  - Call: ${event.call ? "✅" : "❌"}`,
+          ]
+        : []),
+      `  - Token Usage: ${event.usage.toLocaleString()}`,
+      "",
+      "## Scenario",
+      "### User Prompt",
+      event.scenario.text,
+      "",
+      "### Expected",
+      "```json",
+      JSON.stringify(
+        AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected),
+        null,
+        2,
+      ),
+      "```",
+      "",
+      "## Prompt Histories",
+      ...event.prompts.map(AgenticaPromptReporter.markdown),
+      "",
+      ...(event.type === "error"
+        ? [
+            "## Error",
+            "```json",
+            JSON.stringify(
+              AgenticaBenchmarkUtil.errorToJson(event.error),
+              null,
+              2,
+            ),
+            "```",
+          ]
+        : []),
+    ].join("\n");
+  };
+
+  const drawStatus = (
+    events: IAgenticaCallBenchmarkEvent[],
+    success: (e: IAgenticaCallBenchmarkEvent) => boolean,
+  ): string => {
+    const count: number = events.filter(success).length;
+    return (
+      new Array(count).fill("■").join("") +
+      new Array(10 - count).fill("□").join("")
+    );
+  };
+}

package/src/internal/AgenticaPromptReporter.ts

@@ -0,0 +1,43 @@
+import { IAgenticaPrompt } from "@agentica/core";
+
+export namespace AgenticaPromptReporter {
+  export const markdown = (p: IAgenticaPrompt): string => {
+    if (p.type === "text")
+      return [`### Text (${p.role})`, p.text, ""].join("\n");
+    else if (p.type === "select" || p.type === "cancel")
+      return [
+        `### ${p.type === "select" ? "Select" : "Cancel"}`,
+        ...p.operations
+          .map((op) => [
+            `#### ${op.name}`,
+            `  - controller: ${op.controller.name}`,
+            `  - function: ${op.function.name}`,
+            `  - reason: ${op.reason}`,
+            "",
+            ...(!!op.function.description?.length
+              ? [op.function.description, ""]
+              : []),
+          ])
+          .flat(),
+      ].join("\n");
+    else if (p.type === "describe")
+      return [
+        "### Describe",
+        ...p.executions.map((e) => `  - ${e.name}`),
+        "",
+        ...p.text.split("\n").map((s) => `> ${s}`),
+        "",
+      ].join("\n");
+    return [
+      "### Execute",
+      `  - name: ${p.name}`,
+      `  - controller: ${p.controller.name}`,
+      `  - function: ${p.function.name}`,
+      "",
+      "```json",
+      JSON.stringify(p.arguments, null, 2),
+      "```",
+      "",
+    ].join("\n");
+  };
+}

package/src/internal/AgenticaSelectBenchmarkReporter.ts

@@ -0,0 +1,212 @@
+import { IAgenticaSelectBenchmarkEvent } from "../structures/IAgenticaSelectBenchmarkEvent";
+import { IAgenticaSelectBenchmarkResult } from "../structures/IAgenticaSelectBenchmarkResult";
+import { MathUtil } from "../utils/MathUtil";
+import { AgenticaBenchmarkUtil } from "./AgenticaBenchmarkUtil";
+
+/**
+ * @internal
+ */
+export namespace AgenticaSelectBenchmarkReporter {
+  export const markdown = (
+    result: IAgenticaSelectBenchmarkResult,
+  ): Record<string, string> =>
+    Object.fromEntries([
+      ["./README.md", writeIndex(result)],
+      ...result.experiments
+        .map((exp) => [
+          [`./${exp.scenario.name}/README.md`, writeExperimentIndex(exp)],
+          ...exp.events.map((event, i) => [
+            `./${exp.scenario.name}/${i + 1}.${event.type}.md`,
+            writeExperimentEvent(event, i),
+          ]),
+        ])
+        .flat(),
+    ]);
+
+  const writeIndex = (result: IAgenticaSelectBenchmarkResult): string => {
+    const events: IAgenticaSelectBenchmarkEvent[] = result.experiments
+      .map((r) => r.events)
+      .flat();
+    const average: number =
+      events
+        .map((e) => e.completed_at.getTime() - e.started_at.getTime())
+        .reduce((a, b) => a + b, 0) / events.length;
+    return [
+      "# LLM Function Selection Benchmark",
+      "## Summary",
+      `  - Aggregation:`,
+      `    - Scenarios: #${result.experiments.length.toLocaleString()}`,
+      `    - Trial: ${events.length}`,
+      `    - Success: ${events.filter((e) => e.type === "success").length}`,
+      `    - Failure: ${events.filter((e) => e.type === "failure").length}`,
+      `    - Average Time: ${MathUtil.round(average).toLocaleString()} ms`,
+      `  - Token Usage`,
+      `    - Total: ${result.usage.total.toLocaleString()}`,
+      `    - Prompt`,
+      `      - Total: ${result.usage.prompt.total.toLocaleString()}`,
+      `      - Audio: ${result.usage.prompt.audio.toLocaleString()}`,
+      `      - Cached: ${result.usage.prompt.cached.toLocaleString()}`,
+      `    - Completion:`,
+      `      - Total: ${result.usage.completion.total.toLocaleString()}`,
+      `      - Accepted Prediction: ${result.usage.completion.accepted_prediction.toLocaleString()}`,
+      `      - Audio: ${result.usage.completion.audio.toLocaleString()}`,
+      `      - Reasoning: ${result.usage.completion.reasoning.toLocaleString()}`,
+      `      - Rejected Prediction: ${result.usage.completion.rejected_prediction.toLocaleString()}`,
+      "",
+      "## Experiments",
+      " Name | Status | Time/Avg  ",
+      ":-----|:-------|----------:",
+      ...result.experiments.map((exp) =>
+        [
+          `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`,
+          (() => {
+            const success: number = Math.floor(
+              (exp.events.filter((e) => e.type === "success").length /
+                exp.events.length) *
+                10,
+            );
+            return (
+              new Array(success).fill("■").join("") +
+              new Array(10 - success).fill("□").join("")
+            );
+          })(),
+          MathUtil.round(
+            exp.events
+              .map(
+                (event) =>
+                  event.completed_at.getTime() - event.started_at.getTime(),
+              )
+              .reduce((a, b) => a + b, 0) / exp.events.length,
+          ).toLocaleString() + " ms",
+        ].join(" | "),
+      ),
+    ].join("\n");
+  };
+
+  const writeExperimentIndex = (
+    exp: IAgenticaSelectBenchmarkResult.IExperiment,
+  ): string => {
+    return [
+      `# ${exp.scenario.name}`,
+      "## Summary",
+      "  - Aggregation:",
+      `    - Trial: ${exp.events.length}`,
+      `    - Success: ${exp.events.filter((e) => e.type === "success").length}`,
+      `    - Failure: ${exp.events.filter((e) => e.type === "failure").length}`,
+      `    - Average Time: ${MathUtil.round(
+        exp.events
+          .map(
+            (event) =>
+              event.completed_at.getTime() - event.started_at.getTime(),
+          )
+          .reduce((a, b) => a + b, 0) / exp.events.length,
+      ).toLocaleString()} ms`,
+      "  - Token Usage",
+      `    - Total: ${exp.usage.total.toLocaleString()}`,
+      `    - Prompt`,
+      `      - Total: ${exp.usage.prompt.total.toLocaleString()}`,
+      `      - Audio: ${exp.usage.prompt.audio.toLocaleString()}`,
+      `      - Cached: ${exp.usage.prompt.cached.toLocaleString()}`,
+      `    - Completion:`,
+      `      - Total: ${exp.usage.completion.total.toLocaleString()}`,
+      `      - Accepted Prediction: ${exp.usage.completion.accepted_prediction.toLocaleString()}`,
+      `      - Audio: ${exp.usage.completion.audio.toLocaleString()}`,
+      `      - Reasoning: ${exp.usage.completion.reasoning.toLocaleString()}`,
+      `      - Rejected Prediction: ${exp.usage.completion.rejected_prediction.toLocaleString()}`,
+      "",
+      "## Events",
+      " No | Type | Time",
+      "---:|:-----|----:",
+      ...exp.events.map((e, i) =>
+        [
+          `[${i + 1}.](./${i + 1}.${e.type}.md)`,
+          e.type,
+          MathUtil.round(e.completed_at.getTime() - e.started_at.getTime()) +
+            " ms",
+        ].join(" | "),
+      ),
+      "",
+      "## Scenario",
+      "### User Prompt",
+      exp.scenario.text,
+      "",
+      "### Expected",
+      "```json",
+      JSON.stringify(
+        AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected),
+        null,
+        2,
+      ),
+      "```",
+    ].join("\n");
+  };
+
+  const writeExperimentEvent = (
+    event: IAgenticaSelectBenchmarkEvent,
+    index: number,
+  ): string => {
+    return [
+      `# ${index}. ${event.type}`,
+      `## Summary`,
+      `  - Name: ${event.scenario.name}`,
+      `  - Type: ${event.type}`,
+      `  - Time: ${(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`,
+      ...(event.type !== "error"
+        ? [
+            "  - Token Usage",
+            `    - Total: ${event.usage.total.toLocaleString()}`,
+            `    - Prompt`,
+            `      - Total: ${event.usage.prompt.total.toLocaleString()}`,
+            `      - Audio: ${event.usage.prompt.audio.toLocaleString()}`,
+            `      - Cached: ${event.usage.prompt.cached.toLocaleString()}`,
+            `    - Completion:`,
+            `      - Total: ${event.usage.completion.total.toLocaleString()}`,
+            `      - Accepted Prediction: ${event.usage.completion.accepted_prediction.toLocaleString()}`,
+            `      - Audio: ${event.usage.completion.audio.toLocaleString()}`,
+            `      - Reasoning: ${event.usage.completion.reasoning.toLocaleString()}`,
+            `      - Rejected Prediction: ${event.usage.completion.rejected_prediction.toLocaleString()}`,
+          ]
+        : []),
+      "",
+      "## Scenario",
+      "### User Prompt",
+      event.scenario.text,
+      "",
+      "### Expected",
+      "```json",
+      JSON.stringify(
+        AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected),
+        null,
+        2,
+      ),
+      "```",
+      "",
+      ...(event.type === "success" || event.type === "failure"
+        ? [
+            "## Result",
+            ...event.selected.map((s) =>
+              [
+                `### ${s.name}`,
+                `  - Controller: \`${s.controller.name}\``,
+                `  - Function: \`${s.function.name}\``,
+                `  - Reason: ${s.reason}`,
+                "",
+                ...(s.function.description ? [s.function.description, ""] : []),
+              ].join("\n"),
+            ),
+          ]
+        : []),
+      ...(event.type === "error"
+        ? [
+            "## Error",
+            "```json",
+            AgenticaBenchmarkUtil.errorToJson(
+              JSON.stringify(event.error, null, 2),
+            ),
+            "```",
+            "",
+          ]
+        : []),
+    ].join("\n");
+  };
+}