@autobe/agent 0.17.1 → 0.19.1

package/src/orchestrate/test/{transformTestCorrectHistories.ts → histories/transformTestCorrectHistories.ts}

@@ -2,8 +2,8 @@ import { IAgenticaHistoryJson } from "@agentica/core";
 import { IAutoBeTypeScriptCompileResult } from "@autobe/interface";
 import { v4 } from "uuid";
 
-import { AutoBeSystemPromptConstant } from "../../constants/AutoBeSystemPromptConstant";
-import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
+import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { IAutoBeTestFunction } from "../structures/IAutoBeTestFunction";
 import { transformTestWriteHistories } from "./transformTestWriteHistories";
 
 export const transformTestCorrectHistories = (

package/src/orchestrate/test/{transformTestHistories.ts → histories/transformTestHistories.ts}

@@ -1,7 +1,7 @@
 import { IAgenticaHistoryJson } from "@agentica/core";
 import { v4 } from "uuid";
 
-import { AutoBeState } from "../../context/AutoBeState";
+import { AutoBeState } from "../../../context/AutoBeState";
 
 export const transformTestHistories = (
   state: AutoBeState,

package/src/orchestrate/test/histories/transformTestScenarioHistories.ts

@@ -0,0 +1,121 @@
+import { IAgenticaHistoryJson } from "@agentica/core";
+import { AutoBeInterfaceAuthorization, AutoBeOpenApi } from "@autobe/interface";
+import { MapUtil } from "@autobe/utils";
+import { v4 } from "uuid";
+
+import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { AutoBeState } from "../../../context/AutoBeState";
+import { IAutoBeTestScenarioAuthorizationRole } from "../structures/IAutoBeTestScenarioAuthorizationRole";
+
+export const transformTestScenarioHistories = (
+  state: AutoBeState,
+  entire: AutoBeOpenApi.IOperation[],
+  include: AutoBeOpenApi.IOperation[],
+  exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+): Array<
+  IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
+> => {
+  const authorizations: AutoBeInterfaceAuthorization[] =
+    state.interface?.authorizations ?? [];
+
+  const authorizationRoles: Map<string, IAutoBeTestScenarioAuthorizationRole> =
+    new Map();
+
+  for (const authorization of authorizations) {
+    for (const op of authorization.operations) {
+      if (op.authorizationType === null) continue;
+      const value: IAutoBeTestScenarioAuthorizationRole = MapUtil.take(
+        authorizationRoles,
+        authorization.role,
+        () => ({
+          name: authorization.role,
+          join: null,
+          login: null,
+        }),
+      );
+      if (op.authorizationType === "join") value.join = op;
+      else if (op.authorizationType === "login") value.login = op;
+    }
+  }
+
+  return [
+    {
+      id: v4(),
+      created_at: new Date().toISOString(),
+      type: "systemMessage",
+      text: AutoBeSystemPromptConstant.TEST_SCENARIO,
+    } satisfies IAgenticaHistoryJson.ISystemMessage,
+    {
+      id: v4(),
+      created_at: new Date().toISOString(),
+      type: "systemMessage",
+      text: [
+        "# Operations",
+        "",
+        "Below are the full operations. Please refer to this.",
+        "Your role is to draft all test cases for each given Operation.",
+        "It is also permissible to write multiple test codes on a single endpoint.",
+        "However, rather than meaningless tests, business logic tests should be written and an E2E test situation should be assumed.",
+        "",
+        "Please carefully analyze each operation to identify all dependencies required for testing.",
+        "For example, if you want to test liking and then deleting a post,",
+        "you might think to test post creation, liking, and unlike operations.",
+        "However, even if not explicitly mentioned, user registration or login are essential prerequisites.",
+        "Pay close attention to IDs and related values in the API,",
+        "and ensure you identify all dependencies between endpoints.",
+        "",
+        "```json",
+        JSON.stringify(
+          entire.map((el) => ({
+            ...el,
+            specification: undefined,
+          })),
+        ),
+        "```",
+      ].join("\n"),
+    } satisfies IAgenticaHistoryJson.ISystemMessage,
+    {
+      id: v4(),
+      created_at: new Date().toISOString(),
+      type: "systemMessage",
+      text: [
+        "# Included in Test Plan",
+        "",
+        "Below are the endpoints that have been included in the test plan.",
+        "Each endpoint shows its authentication requirements and related authentication APIs.",
+        "When testing endpoints that require authentication, ensure you include the corresponding join/login operations in your test scenario to establish proper authentication context.",
+        "",
+        include
+          .map((el, i) => {
+            const roles = Array.from(authorizationRoles.values()).filter(
+              (role) => role.name === el.authorizationRole,
+            );
+            return [
+              `## ${i + 1}. ${el.method.toUpperCase()} ${el.path}`,
+              "",
+              "Related Authentication APIs:",
+              "",
+              roles.length > 0
+                ? roles.map((role) => {
+                    return [
+                      `- ${role.join?.method.toUpperCase()}: ${role.join?.path}`,
+                      `- ${role.login?.method.toUpperCase()}: ${role.login?.path}`,
+                    ].join("\n");
+                  })
+                : "- None",
+            ];
+          })
+          .join("\n"),
+        "",
+        "# Excluded from Test Plan",
+        "",
+        "These are the endpoints that have already been used in test codes generated as part of a plan group.",
+        "These endpoints do not need to be tested again.",
+        "However, it is allowed to reference or depend on these endpoints when writing test codes for other purposes.",
+        exclude
+          .map((el) => `- ${el.method.toUpperCase()}: ${el.path}`)
+          .join("\n"),
+      ].join("\n"),
+    } satisfies IAgenticaHistoryJson.ISystemMessage,
+  ];
+};

package/src/orchestrate/test/{transformTestWriteHistories.ts → histories/transformTestWriteHistories.ts}

@@ -9,8 +9,8 @@ import {
 import typia from "typia";
 import { v4 } from "uuid";
 
-import { AutoBeSystemPromptConstant } from "../../constants/AutoBeSystemPromptConstant";
-import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
+import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { IAutoBeTestScenarioArtifacts } from "../structures/IAutoBeTestScenarioArtifacts";
 
 export function transformTestWriteHistories(
   scenario: AutoBeTestScenario,
@@ -24,7 +24,7 @@ export function transformTestWriteHistories(
       text: AutoBeSystemPromptConstant.TEST_WRITE.replace(
         "{{AutoBeTestScenario}}",
         JSON.stringify(typia.llm.parameters<AutoBeTestScenario, "llama">()),
-      ),
+      ).replaceAll("{{FUNCTION_NAME}}", scenario.functionName),
     },
     {
       id: v4(),

package/src/orchestrate/test/orchestrateTest.ts

@@ -2,7 +2,7 @@ import {
   AutoBeAssistantMessageHistory,
   AutoBeOpenApi,
   AutoBeTestHistory,
-  AutoBeTestScenarioEvent,
+  AutoBeTestScenario,
   AutoBeTestValidateEvent,
   IAutoBeCompiler,
   IAutoBeTypeScriptCompileResult,
@@ -12,6 +12,7 @@ import { v4 } from "uuid";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { IAutoBeApplicationProps } from "../../context/IAutoBeApplicationProps";
+import { predicateStateMessage } from "../../utils/predicateStateMessage";
 import { orchestrateTestCorrect } from "./orchestrateTestCorrect";
 import { orchestrateTestScenario } from "./orchestrateTestScenario";
 import { orchestrateTestWrite } from "./orchestrateTestWrite";
@@ -22,7 +23,17 @@ export const orchestrateTest =
   async (
     props: IAutoBeApplicationProps,
   ): Promise<AutoBeAssistantMessageHistory | AutoBeTestHistory> => {
+    // PREDICATION
     const start: Date = new Date();
+    const predicate: string | null = predicateStateMessage(ctx.state(), "test");
+    if (predicate !== null)
+      return ctx.assistantMessage({
+        type: "assistantMessage",
+        id: v4(),
+        created_at: start.toISOString(),
+        text: predicate,
+        completed_at: new Date().toISOString(),
+      });
     ctx.dispatch({
       type: "testStart",
       created_at: start.toISOString(),
@@ -30,6 +41,7 @@ export const orchestrateTest =
       step: ctx.state().analyze?.step ?? 0,
     });
 
+    // CHECK OPERATIONS
     const operations: AutoBeOpenApi.IOperation[] =
       ctx.state().interface?.document.operations ?? [];
     if (operations.length === 0)
@@ -44,14 +56,12 @@ export const orchestrateTest =
       });
 
     // PLAN
-    const scenarioEvent: AutoBeTestScenarioEvent =
-      await orchestrateTestScenario(ctx);
-    ctx.dispatch(scenarioEvent);
+    const scenarios: AutoBeTestScenario[] = await orchestrateTestScenario(ctx);
 
     // TEST CODE
     const written: IAutoBeTestWriteResult[] = await orchestrateTestWrite(
       ctx,
-      scenarioEvent.scenarios,
+      scenarios,
     );
 
     const corrects: AutoBeTestValidateEvent[] = await orchestrateTestCorrect(
@@ -77,9 +87,10 @@ export const orchestrateTest =
       });
     return ctx.dispatch({
       type: "testComplete",
-      created_at: start.toISOString(),
+      created_at: new Date().toISOString(),
       files: success.map((s) => s.file),
       compiled,
       step: ctx.state().interface?.step ?? 0,
+      elapsed: new Date().getTime() - start.getTime(),
     });
   };

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -1,4 +1,4 @@
-import { IAgenticaController, MicroAgentica } from "@agentica/core";
+import { IAgenticaController } from "@agentica/core";
 import {
   AutoBeTestValidateEvent,
   IAutoBeCompiler,
@@ -10,14 +10,12 @@ import typia from "typia";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
-import { enforceToolCall } from "../../utils/enforceToolCall";
-import { forceRetry } from "../../utils/forceRetry";
 import { completeTestCode } from "./compile/completeTestCode";
+import { transformTestCorrectHistories } from "./histories/transformTestCorrectHistories";
 import { IAutoBeTestCorrectApplication } from "./structures/IAutoBeTestCorrectApplication";
 import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
 import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
 import { IAutoBeTestWriteResult } from "./structures/IAutoBeTestWriteResult";
-import { transformTestCorrectHistories } from "./transformTestCorrectHistories";
 
 export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
@@ -27,25 +25,23 @@ export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   const result: Array<AutoBeTestValidateEvent | null> = await Promise.all(
     writeResult.map(async (w) => {
       try {
-        return await forceRetry(async () => {
-          const event: AutoBeTestValidateEvent = await compile(ctx, {
+        const event: AutoBeTestValidateEvent = await compile(ctx, {
+          artifacts: w.artifacts,
+          scenario: w.scenario,
+          location: w.event.location,
+          script: w.event.final,
+        });
+        return predicate(
+          ctx,
+          {
             artifacts: w.artifacts,
             scenario: w.scenario,
             location: w.event.location,
             script: w.event.final,
-          });
-          return predicate(
-            ctx,
-            {
-              artifacts: w.artifacts,
-              scenario: w.scenario,
-              location: w.event.location,
-              script: w.event.final,
-            },
-            event,
-            life,
-          );
-        });
+          },
+          event,
+          life,
+        );
       } catch {
         return null;
       }
@@ -103,37 +99,20 @@ const correct = async <Model extends ILlmSchema.Model>(
   const pointer: IPointer<IAutoBeTestCorrectApplication.IProps | null> = {
     value: null,
   };
-  const agentica = new MicroAgentica({
-    model: ctx.model,
-    vendor: ctx.vendor,
-    config: {
-      ...(ctx.config ?? {}),
-      executor: {
-        describe: null,
-      },
-      retry: 4,
-    },
+  const { tokenUsage } = await ctx.conversate({
+    source: "testCorrect",
     histories: transformTestCorrectHistories(content, validate.result),
-    controllers: [
-      createApplication({
-        model: ctx.model,
-        artifacts: content.artifacts,
-        build: (next) => {
-          pointer.value = next;
-        },
-      }),
-    ],
-  });
-  enforceToolCall(agentica);
-
-  await agentica
-    .conversate(
+    controller: createController({
+      model: ctx.model,
+      artifacts: content.artifacts,
+      build: (next) => {
+        pointer.value = next;
+      },
+    }),
+    enforceFunctionCall: true,
+    message:
       "Fix the `AutoBeTest.IFunction` data to resolve the compilation error.",
-    )
-    .finally(() => {
-      const tokenUsage = agentica.getTokenUsage().aggregate;
-      ctx.usage().record(tokenUsage, ["test"]);
-    });
+  });
   if (pointer.value === null) throw new Error("Failed to modify test code.");
 
   const compiler: IAutoBeCompiler = await ctx.compiler();
@@ -144,6 +123,7 @@ const correct = async <Model extends ILlmSchema.Model>(
     created_at: new Date().toISOString(),
     file: validate.file,
     result: validate.result,
+    tokenUsage,
     step: ctx.state().analyze?.step ?? 0,
     ...pointer.value,
   });
@@ -155,7 +135,7 @@ const correct = async <Model extends ILlmSchema.Model>(
   return predicate(ctx, newContent, newValidate, life);
 };
 
-const createApplication = <Model extends ILlmSchema.Model>(props: {
+const createController = <Model extends ILlmSchema.Model>(props: {
   model: Model;
   artifacts: IAutoBeTestScenarioArtifacts;
   build: (next: IAutoBeTestCorrectApplication.IProps) => void;
@@ -164,7 +144,7 @@ const createApplication = <Model extends ILlmSchema.Model>(props: {
 
   const application: ILlmApplication<Model> = collection[
     props.model
-  ] as unknown as ILlmApplication<Model>;
+  ] satisfies ILlmApplication<any> as unknown as ILlmApplication<Model>;
   return {
     protocol: "class",
     name: "Modify Test Code",
@@ -179,19 +159,9 @@ const createApplication = <Model extends ILlmSchema.Model>(props: {
   };
 };
 
-const claude = typia.llm.application<
-  IAutoBeTestCorrectApplication,
-  "claude",
-  {
-    reference: true;
-  }
->();
+const claude = typia.llm.application<IAutoBeTestCorrectApplication, "claude">();
 const collection = {
-  chatgpt: typia.llm.application<
-    IAutoBeTestCorrectApplication,
-    "chatgpt",
-    { reference: true }
-  >(),
+  chatgpt: typia.llm.application<IAutoBeTestCorrectApplication, "chatgpt">(),
   claude,
   llama: claude,
   deepseek: claude,