@agentica/benchmark 0.43.2 → 0.44.0-dev.20260313

package/src/internal/AgenticaBenchmarkPredicator.ts

@@ -1,241 +1,240 @@
-/**
- * @module
- * This file contains functions to work with AgenticaBenchmarkPredicator.
- *
- * @author Wrtn Technologies
- */
-
-import type { Agentica, AgenticaHistory, AgenticaOperation, MicroAgentica } from "@agentica/core";
-import type { ILlmFunction } from "@samchon/openapi";
-import type OpenAI from "openai";
-
-import typia from "typia";
-
-import type { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
-
-export const AgenticaBenchmarkPredicator = {
-  isNext,
-  success,
-};
-
-interface IPredicatorApplication {
-  /**
-   * Ask user to consent for what the AI agent wants to do next.
-   *
-   * If AI agent wants to do some function calling at next,
-   * but it needs the user's consent about the function calling to do,
-   * then call this tool function.
-   *
-   * @param props Properties for asking the user's consent
-   */
-  consent: (props: IConsentProps) => void;
-}
-
-/**
- * Properties for asking the user's consent
- */
-interface IConsentProps {
-  /**
-   * Reason of the message implying what the AI agent wants
-   * to do at the next step after the user's consent.
-   */
-  content: string;
-
-  /**
-   * Recommended reply message for the user.
-   *
-   * The message what AI agent wants the user to reply
-   * accepting the AI agent's next job suggestion.
-   */
-  reply: string;
-}
-
-async function isNext(agent: Agentica | MicroAgentica): Promise<string | null> {
-  const last: AgenticaHistory | undefined = agent
-    .getHistories()
-    .at(-1);
-
-  /**
-   * Agentica Props is private, we can't access it
-   * The provided code follows the original source prior to modification.
-   * However, due to compilation errors, a workaround was implemented.
-   * Please apply any available patches to resolve this issue.
-   */
-  const llmVendor = agent.getVendor();
-  const isAssistantHistory = last?.type === "assistantMessage";
-  if (!isAssistantHistory) {
-    return null;
-  }
-
-  const consent: ILlmFunction = typia.llm.application<
-    IPredicatorApplication
-  >().functions[0]!;
-  const result: OpenAI.ChatCompletion = await llmVendor.api.chat.completions.create(
-    {
-      model: llmVendor.model,
-      messages: [
-        {
-          role: "system",
-          content: [
-            "You are an helpful assistant.",
-            "",
-            "If what the assistant said seems like to asking for",
-            "user's consent about some function calling at the next step,",
-            "use the tools appropriately to step to the next.",
-          ].join("\n"),
-        },
-        {
-          role: "assistant",
-          content: last.text,
-        },
-      ],
-      tools: [
-        {
-          type: "function",
-          function: {
-            name: consent.name,
-            description: consent.description,
-            parameters: consent.parameters as Record<string, any>,
-          },
-        },
-      ],
-      tool_choice: "required",
-      // parallel_tool_calls: false,
-    },
-    llmVendor.options,
-  );
-
-  const toolCall: OpenAI.ChatCompletionMessageToolCall | undefined = (
-    result.choices[0]?.message.tool_calls ?? []
-  ).filter(
-    tc => tc.type === "function" && tc.function.name === consent.name,
-  )?.[0];
-
-  if (toolCall === undefined || toolCall.type !== "function") {
-    return null;
-  }
-
-  const input = typia.json.isParse<IConsentProps>(toolCall.function.arguments);
-  return input !== null ? input.reply : null;
-}
-
-/**
- * Check if the called operations match the expected operations.
- *
- * @param props Properties for checking the match of the called operations
- *              and the expected operations
- * @returns `true` if the called operations match the expected operations,
- * otherwise `false`.
- */
-export function success(props: {
-  /**
-   * Expected operations to be called.
-   *
-   * For 'allOf' within an 'array', the next expected element starts checking from the element that follows the last called element in 'allOf'.
-   */
-  expected: IAgenticaBenchmarkExpected;
-
-  /**
-   * Specified operations.
-   */
-  operations: Array<AgenticaOperation>;
-
-  /**
-   * If it's `false`, check the array and let it go even if there's something wrong between them.
-   *
-   * @default `false`
-   */
-  strict?: boolean;
-}): boolean {
-  return successInner(props).result;
-}
-
-function successInner(props: Parameters<typeof success>[0]):
-  | {
-    result: true;
-    take: number;
-  }
-  | {
-    result: false;
-  } {
-  const call = (
-    expected: IAgenticaBenchmarkExpected,
-    overrideOperations?: Array<AgenticaOperation>,
-  ) =>
-    successInner({
-      expected,
-      operations: overrideOperations ?? props.operations,
-      strict: props.strict,
-    });
-
-  switch (props.expected.type) {
-    case "array": {
-      let take = 0;
-      const targetIterator = props.expected.items[Symbol.iterator]();
-      let targeted = targetIterator.next();
-
-      while (true) {
-        if (targeted.done === true) {
-          return {
-            result: true,
-            take,
-          };
-        }
-        if (take >= props.operations.length) {
-          return { result: false };
-        }
-
-        const result = call(targeted.value, props.operations.slice(take));
-        if (!result.result) {
-          if (props.strict === true) {
-            return { result: false };
-          }
-          take += 1;
-          continue;
-        }
-
-        take += result.take;
-        targeted = targetIterator.next();
-      }
-    }
-    case "standalone": {
-      const target = props.expected.operation;
-      const result = props.operations.some(op => op.name === target.name);
-      if (result) {
-        return { result, take: 1 };
-      }
-      return {
-        result,
-      };
-    }
-    case "anyOf":
-      for (const expected of props.expected.anyOf) {
-        const callResult = call(expected);
-        if (callResult.result) {
-          return callResult;
-        }
-      }
-
-      return { result: false };
-    case "allOf": {
-      /**
-       * @example
-       * expected = [4, 2];
-       * called = [1, 2, 3, 4, 5];
-       *
-       * { result: true, take: 3 };
-       */
-      const result = props.expected.allOf.map(expected => call(expected));
-      if (result.every(r => r.result)) {
-        return {
-          result: true,
-          take: result.reduce((acc, r) => Math.max(acc, r.take), 0),
-        };
-      }
-
-      return {
-        result: false,
-      };
-    }
-  }
-}
+/**
+ * @module
+ * This file contains functions to work with AgenticaBenchmarkPredicator.
+ *
+ * @author Wrtn Technologies
+ */
+
+import type { Agentica, AgenticaHistory, AgenticaOperation, MicroAgentica } from "@agentica/core";
+import type OpenAI from "openai";
+import type { ILlmFunction } from "typia";
+
+import typia from "typia";
+
+import type { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
+
+export const AgenticaBenchmarkPredicator = {
+  isNext,
+  success,
+};
+
+interface IPredicatorApplication {
+  /**
+   * Ask user to consent for what the AI agent wants to do next.
+   *
+   * If AI agent wants to do some function calling at next,
+   * but it needs the user's consent about the function calling to do,
+   * then call this tool function.
+   *
+   * @param props Properties for asking the user's consent
+   */
+  consent: (props: IConsentProps) => void;
+}
+
+/**
+ * Properties for asking the user's consent
+ */
+interface IConsentProps {
+  /**
+   * Reason of the message implying what the AI agent wants
+   * to do at the next step after the user's consent.
+   */
+  content: string;
+
+  /**
+   * Recommended reply message for the user.
+   *
+   * The message what AI agent wants the user to reply
+   * accepting the AI agent's next job suggestion.
+   */
+  reply: string;
+}
+
+async function isNext(agent: Agentica | MicroAgentica): Promise<string | null> {
+  const histories: AgenticaHistory[] = agent.getHistories();
+  const last: AgenticaHistory | undefined = histories[histories.length - 1];
+
+  /**
+   * Agentica Props is private, we can't access it
+   * The provided code follows the original source prior to modification.
+   * However, due to compilation errors, a workaround was implemented.
+   * Please apply any available patches to resolve this issue.
+   */
+  const llmVendor = agent.getVendor();
+  const isAssistantHistory = last?.type === "assistantMessage";
+  if (!isAssistantHistory) {
+    return null;
+  }
+
+  const consent: ILlmFunction = typia.llm.application<
+    IPredicatorApplication
+  >().functions[0]!;
+  const result: OpenAI.ChatCompletion = await llmVendor.api.chat.completions.create(
+    {
+      model: llmVendor.model,
+      messages: [
+        {
+          role: "system",
+          content: [
+            "You are an helpful assistant.",
+            "",
+            "If what the assistant said seems like to asking for",
+            "user's consent about some function calling at the next step,",
+            "use the tools appropriately to step to the next.",
+          ].join("\n"),
+        },
+        {
+          role: "assistant",
+          content: last.text,
+        },
+      ],
+      tools: [
+        {
+          type: "function",
+          function: {
+            name: consent.name,
+            description: consent.description,
+            parameters: consent.parameters as Record<string, any>,
+          },
+        },
+      ],
+      tool_choice: "required",
+      // parallel_tool_calls: false,
+    },
+    llmVendor.options,
+  );
+
+  const toolCall: OpenAI.ChatCompletionMessageToolCall | undefined = (
+    result.choices[0]?.message.tool_calls ?? []
+  ).filter(
+    tc => tc.type === "function" && tc.function.name === consent.name,
+  )?.[0];
+
+  if (toolCall === undefined || toolCall.type !== "function") {
+    return null;
+  }
+
+  const input = typia.json.isParse<IConsentProps>(toolCall.function.arguments);
+  return input !== null ? input.reply : null;
+}
+
+/**
+ * Check if the called operations match the expected operations.
+ *
+ * @param props Properties for checking the match of the called operations
+ *              and the expected operations
+ * @returns `true` if the called operations match the expected operations,
+ * otherwise `false`.
+ */
+export function success(props: {
+  /**
+   * Expected operations to be called.
+   *
+   * For 'allOf' within an 'array', the next expected element starts checking from the element that follows the last called element in 'allOf'.
+   */
+  expected: IAgenticaBenchmarkExpected;
+
+  /**
+   * Specified operations.
+   */
+  operations: Array<AgenticaOperation>;
+
+  /**
+   * If it's `false`, check the array and let it go even if there's something wrong between them.
+   *
+   * @default `false`
+   */
+  strict?: boolean;
+}): boolean {
+  return successInner(props).result;
+}
+
+function successInner(props: Parameters<typeof success>[0]):
+  | {
+    result: true;
+    take: number;
+  }
+  | {
+    result: false;
+  } {
+  const call = (
+    expected: IAgenticaBenchmarkExpected,
+    overrideOperations?: Array<AgenticaOperation>,
+  ) =>
+    successInner({
+      expected,
+      operations: overrideOperations ?? props.operations,
+      strict: props.strict,
+    });
+
+  switch (props.expected.type) {
+    case "array": {
+      let take = 0;
+      const targetIterator = props.expected.items[Symbol.iterator]();
+      let targeted = targetIterator.next();
+
+      while (true) {
+        if (targeted.done === true) {
+          return {
+            result: true,
+            take,
+          };
+        }
+        if (take >= props.operations.length) {
+          return { result: false };
+        }
+
+        const result = call(targeted.value, props.operations.slice(take));
+        if (!result.result) {
+          if (props.strict === true) {
+            return { result: false };
+          }
+          take += 1;
+          continue;
+        }
+
+        take += result.take;
+        targeted = targetIterator.next();
+      }
+    }
+    case "standalone": {
+      const target = props.expected.operation;
+      const result = props.operations.some(op => op.name === target.name);
+      if (result) {
+        return { result, take: 1 };
+      }
+      return {
+        result,
+      };
+    }
+    case "anyOf":
+      for (const expected of props.expected.anyOf) {
+        const callResult = call(expected);
+        if (callResult.result) {
+          return callResult;
+        }
+      }
+
+      return { result: false };
+    case "allOf": {
+      /**
+       * @example
+       * expected = [4, 2];
+       * called = [1, 2, 3, 4, 5];
+       *
+       * { result: true, take: 3 };
+       */
+      const result = props.expected.allOf.map(expected => call(expected));
+      if (result.every(r => r.result)) {
+        return {
+          result: true,
+          take: result.reduce((acc, r) => Math.max(acc, r.take), 0),
+        };
+      }
+
+      return {
+        result: false,
+      };
+    }
+  }
+}

package/src/internal/AgenticaBenchmarkUtil.ts

@@ -1,60 +1,60 @@
-/**
- * @module
- * This file contains functions to work with AgenticaBenchmarkUtil.
- *
- * @author Wrtn Technologies
- */
-import type { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
-
-export const AgenticaBenchmarkUtil = {
-  errorToJson,
-  expectedToJson,
-};
-
-function errorToJson<T>(error: T): T | ({
-  [k in keyof T]: T[k]
-} & {
-  name: string;
-  message: string;
-  stack: string;
-}) {
-  if (error instanceof Error) {
-    return {
-      ...error,
-      name: error.name,
-      message: error.message,
-      stack: error.stack,
-    };
-  }
-  return error;
-}
-
-function expectedToJson(expected: IAgenticaBenchmarkExpected): any {
-  if (expected.type === "standalone") {
-    return {
-      type: expected.type,
-      operation: {
-        name: expected.operation.name,
-        description: expected.operation.function.description,
-      },
-    };
-  }
-  else if (expected.type === "array") {
-    return {
-      type: expected.type,
-      items: expected.items.map(expectedToJson),
-    };
-  }
-  else if (expected.type === "allOf") {
-    return {
-      type: expected.type,
-      allOf: expected.allOf.map(expectedToJson),
-    };
-  }
-  else {
-    return {
-      type: expected.type,
-      anyOf: expected.anyOf.map(expectedToJson),
-    };
-  }
-}
+/**
+ * @module
+ * This file contains functions to work with AgenticaBenchmarkUtil.
+ *
+ * @author Wrtn Technologies
+ */
+import type { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
+
+export const AgenticaBenchmarkUtil = {
+  errorToJson,
+  expectedToJson,
+};
+
+function errorToJson<T>(error: T): T | ({
+  [k in keyof T]: T[k]
+} & {
+  name: string;
+  message: string;
+  stack: string;
+}) {
+  if (error instanceof Error) {
+    return {
+      ...error,
+      name: error.name,
+      message: error.message,
+      stack: error.stack,
+    };
+  }
+  return error;
+}
+
+function expectedToJson(expected: IAgenticaBenchmarkExpected): any {
+  if (expected.type === "standalone") {
+    return {
+      type: expected.type,
+      operation: {
+        name: expected.operation.name,
+        description: expected.operation.function.description,
+      },
+    };
+  }
+  else if (expected.type === "array") {
+    return {
+      type: expected.type,
+      items: expected.items.map(expectedToJson),
+    };
+  }
+  else if (expected.type === "allOf") {
+    return {
+      type: expected.type,
+      allOf: expected.allOf.map(expectedToJson),
+    };
+  }
+  else {
+    return {
+      type: expected.type,
+      anyOf: expected.anyOf.map(expectedToJson),
+    };
+  }
+}