@autobe/agent 0.16.1 → 0.18.0

package/src/orchestrate/test/experimental/orchestrateTestWrite.ast

@@ -145,7 +145,7 @@ async function process<Model extends ILlmSchema.Model>(
       failure,
     }),
     controllers: [
-      createApplication({
+      createController({
         model: ctx.model,
         document: ctx.state().interface!.document,
         build: (next) => {
@@ -203,7 +203,7 @@ function getUnionTypeName(unit: IJsonSchemaUnit): string {
     .join(" | ");
 }
 
-function createApplication<Model extends ILlmSchema.Model>(props: {
+function createController<Model extends ILlmSchema.Model>(props: {
   model: Model;
   document: AutoBeOpenApi.IDocument;
   build: (next: ICreateTestCodeProps) => void;
@@ -225,12 +225,8 @@ function createApplication<Model extends ILlmSchema.Model>(props: {
       AutoBeEndpointComparator.hashCode,
       AutoBeEndpointComparator.equals,
     );
-  const application: ILlmApplication<Model> = collection[
-    props.model
-  ] as unknown as ILlmApplication<Model>;
-  application.functions[0].validate = (
-    input: unknown,
-  ): IValidation<unknown> => {
+
+  const validate = (input: unknown): IValidation<ICreateTestCodeProps> => {
     const result: IValidation<ICreateTestCodeProps> =
       typia.validate<ICreateTestCodeProps>(input);
     if (result.success === false) return result;
@@ -250,6 +246,11 @@ function createApplication<Model extends ILlmSchema.Model>(props: {
           errors: context.errors,
         };
   };
+  const application: ILlmApplication<Model> = collection[
+    props.model === "chatgpt" ? "chatgpt" : "claude"
+  ](
+    validate,
+  ) satisfies ILlmApplication<any> as unknown as ILlmApplication<Model>;
 
   return {
     protocol: "class",
@@ -263,23 +264,23 @@ function createApplication<Model extends ILlmSchema.Model>(props: {
   };
 }
 
-const claude = typia.llm.application<
-  IApplication,
-  "claude",
-  { reference: true }
->();
 const collection = {
-  chatgpt: typia.llm.application<
-    IApplication,
-    "chatgpt",
-    { reference: true }
-  >(),
-  claude,
-  llama: claude,
-  deepseek: claude,
-  "3.1": claude,
+  chatgpt: (validate: Validator) =>
+    typia.llm.application<IApplication, "chatgpt">({
+      validate: {
+        createTestCode: validate,
+      },
+    }),
+  claude: (validate: Validator) =>
+    typia.llm.application<IApplication, "claude">({
+      validate: {
+        createTestCode: validate,
+      },
+    }),
 };
 
+type Validator = (input: unknown) => IValidation<ICreateTestCodeProps>;
+
 interface IApplication {
   createTestCode(props: ICreateTestCodeProps): void;
 }

package/src/orchestrate/test/{transformTestCorrectHistories.ts → histories/transformTestCorrectHistories.ts}

@@ -2,8 +2,8 @@ import { IAgenticaHistoryJson } from "@agentica/core";
 import { IAutoBeTypeScriptCompileResult } from "@autobe/interface";
 import { v4 } from "uuid";
 
-import { AutoBeSystemPromptConstant } from "../../constants/AutoBeSystemPromptConstant";
-import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
+import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { IAutoBeTestFunction } from "../structures/IAutoBeTestFunction";
 import { transformTestWriteHistories } from "./transformTestWriteHistories";
 
 export const transformTestCorrectHistories = (

package/src/orchestrate/test/{transformTestHistories.ts → histories/transformTestHistories.ts}

@@ -1,7 +1,7 @@
 import { IAgenticaHistoryJson } from "@agentica/core";
 import { v4 } from "uuid";
 
-import { AutoBeState } from "../../context/AutoBeState";
+import { AutoBeState } from "../../../context/AutoBeState";
 
 export const transformTestHistories = (
   state: AutoBeState,

package/src/orchestrate/test/histories/transformTestScenarioHistories.ts

@@ -0,0 +1,70 @@
+import { IAgenticaHistoryJson } from "@agentica/core";
+import { AutoBeOpenApi } from "@autobe/interface";
+import { v4 } from "uuid";
+
+import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+
+export const transformTestScenarioHistories = (
+  entire: AutoBeOpenApi.IOperation[],
+  include: AutoBeOpenApi.IOperation[],
+  exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+): Array<
+  IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
+> => [
+  {
+    id: v4(),
+    created_at: new Date().toISOString(),
+    type: "systemMessage",
+    text: AutoBeSystemPromptConstant.TEST_SCENARIO,
+  } satisfies IAgenticaHistoryJson.ISystemMessage,
+  {
+    id: v4(),
+    created_at: new Date().toISOString(),
+    type: "systemMessage",
+    text: [
+      "# Operations",
+      "Below are the full operations. Please refer to this.",
+      "Your role is to draft all test cases for each given Operation.",
+      "It is also permissible to write multiple test codes on a single endpoint.",
+      "However, rather than meaningless tests, business logic tests should be written and an E2E test situation should be assumed.",
+      "",
+      "Please carefully analyze each operation to identify all dependencies required for testing.",
+      "For example, if you want to test liking and then deleting a post,",
+      "you might think to test post creation, liking, and unlike operations.",
+      "However, even if not explicitly mentioned, user registration and login are essential prerequisites.",
+      "Pay close attention to IDs and related values in the API,",
+      "and ensure you identify all dependencies between endpoints.",
+      "",
+      "```json",
+      JSON.stringify(
+        entire.map((el) => ({
+          ...el,
+          specification: undefined,
+        })),
+      ),
+      "```",
+    ].join("\n"),
+  } satisfies IAgenticaHistoryJson.ISystemMessage,
+  {
+    id: v4(),
+    created_at: new Date().toISOString(),
+    type: "systemMessage",
+    text: [
+      "# Included in Test Plan",
+      include
+        .map(
+          (el) =>
+            `- ${el.method.toUpperCase()}: ${el.path} ${el.authorizationRole ? `(Role: ${el.authorizationRole})` : ""}`,
+        )
+        .join("\n"),
+      "",
+      "# Excluded from Test Plan",
+      "These are the endpoints that have already been used in test codes generated as part of a plan group.",
+      "These endpoints do not need to be tested again.",
+      "However, it is allowed to reference or depend on these endpoints when writing test codes for other purposes.",
+      exclude
+        .map((el) => `- ${el.method.toUpperCase()}: ${el.path}`)
+        .join("\n"),
+    ].join("\n"),
+  } satisfies IAgenticaHistoryJson.ISystemMessage,
+];

package/src/orchestrate/test/{transformTestWriteHistories.ts → histories/transformTestWriteHistories.ts}

@@ -9,8 +9,8 @@ import {
 import typia from "typia";
 import { v4 } from "uuid";
 
-import { AutoBeSystemPromptConstant } from "../../constants/AutoBeSystemPromptConstant";
-import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
+import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { IAutoBeTestScenarioArtifacts } from "../structures/IAutoBeTestScenarioArtifacts";
 
 export function transformTestWriteHistories(
   scenario: AutoBeTestScenario,
@@ -24,7 +24,7 @@ export function transformTestWriteHistories(
       text: AutoBeSystemPromptConstant.TEST_WRITE.replace(
         "{{AutoBeTestScenario}}",
         JSON.stringify(typia.llm.parameters<AutoBeTestScenario, "llama">()),
-      ),
+      ).replaceAll("{{FUNCTION_NAME}}", scenario.functionName),
     },
     {
       id: v4(),

package/src/orchestrate/test/orchestrateTest.ts

@@ -1,9 +1,8 @@
 import {
   AutoBeAssistantMessageHistory,
   AutoBeOpenApi,
-  AutoBeTestFile,
   AutoBeTestHistory,
-  AutoBeTestScenarioEvent,
+  AutoBeTestScenario,
   AutoBeTestValidateEvent,
   IAutoBeCompiler,
   IAutoBeTypeScriptCompileResult,
@@ -33,8 +32,8 @@ export const orchestrateTest =
 
     const operations: AutoBeOpenApi.IOperation[] =
       ctx.state().interface?.document.operations ?? [];
-    if (operations.length === 0) {
-      const history: AutoBeAssistantMessageHistory = {
+    if (operations.length === 0)
+      return ctx.assistantMessage({
         id: v4(),
         type: "assistantMessage",
         created_at: start.toISOString(),
@@ -42,21 +41,15 @@ export const orchestrateTest =
         text:
           "Unable to write test code because there are no Operations, " +
           "please check if the Interface agent is called.",
-      };
-      ctx.histories().push(history);
-      ctx.dispatch(history);
-      return history;
-    }
+      });
 
     // PLAN
-    const scenarioEvent: AutoBeTestScenarioEvent =
-      await orchestrateTestScenario(ctx);
-    ctx.dispatch(scenarioEvent);
+    const scenarios: AutoBeTestScenario[] = await orchestrateTestScenario(ctx);
 
     // TEST CODE
     const written: IAutoBeTestWriteResult[] = await orchestrateTestWrite(
       ctx,
-      scenarioEvent.scenarios,
+      scenarios,
     );
 
     const corrects: AutoBeTestValidateEvent[] = await orchestrateTestCorrect(
@@ -69,7 +62,6 @@ export const orchestrateTest =
 
     // DO COMPILE
     const compiler: IAutoBeCompiler = await ctx.compiler();
-    const result: AutoBeTestFile[] = success.map((c) => c.file);
     const compiled: IAutoBeTypeScriptCompileResult =
       await compiler.typescript.compile({
         files: Object.fromEntries([
@@ -78,28 +70,15 @@ export const orchestrateTest =
               dbms: "sqlite",
             }),
           ).filter(([key]) => key.endsWith(".ts")),
-          ...result.map((f) => [f.location, f.content]),
+          ...success.map((s) => [s.file.location, s.file.content]),
         ]),
       });
-
-    const history: AutoBeTestHistory = {
-      type: "test",
-      id: v4(),
-      completed_at: new Date().toISOString(),
-      created_at: start.toISOString(),
-      files: result,
-      compiled,
-      reason: "Step to the test generation referencing the interface",
-      step: ctx.state().interface?.step ?? 0,
-    };
-    ctx.dispatch({
+    return ctx.dispatch({
       type: "testComplete",
-      created_at: start.toISOString(),
-      files: result,
+      created_at: new Date().toISOString(),
+      files: success.map((s) => s.file),
       compiled,
       step: ctx.state().interface?.step ?? 0,
+      elapsed: new Date().getTime() - start.getTime(),
     });
-    ctx.state().test = history;
-    ctx.histories().push(history);
-    return history;
   };

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -1,4 +1,4 @@
-import { IAgenticaController, MicroAgentica } from "@agentica/core";
+import { IAgenticaController } from "@agentica/core";
 import {
   AutoBeTestValidateEvent,
   IAutoBeCompiler,
@@ -10,14 +10,12 @@ import typia from "typia";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
-import { enforceToolCall } from "../../utils/enforceToolCall";
-import { forceRetry } from "../../utils/forceRetry";
 import { completeTestCode } from "./compile/completeTestCode";
+import { transformTestCorrectHistories } from "./histories/transformTestCorrectHistories";
 import { IAutoBeTestCorrectApplication } from "./structures/IAutoBeTestCorrectApplication";
 import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
 import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
 import { IAutoBeTestWriteResult } from "./structures/IAutoBeTestWriteResult";
-import { transformTestCorrectHistories } from "./transformTestCorrectHistories";
 
 export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
@@ -27,25 +25,23 @@ export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   const result: Array<AutoBeTestValidateEvent | null> = await Promise.all(
     writeResult.map(async (w) => {
       try {
-        return await forceRetry(async () => {
-          const event: AutoBeTestValidateEvent = await compile(ctx, {
+        const event: AutoBeTestValidateEvent = await compile(ctx, {
+          artifacts: w.artifacts,
+          scenario: w.scenario,
+          location: w.event.location,
+          script: w.event.final,
+        });
+        return predicate(
+          ctx,
+          {
             artifacts: w.artifacts,
             scenario: w.scenario,
             location: w.event.location,
             script: w.event.final,
-          });
-          return predicate(
-            ctx,
-            {
-              artifacts: w.artifacts,
-              scenario: w.scenario,
-              location: w.event.location,
-              script: w.event.final,
-            },
-            event,
-            life,
-          );
-        });
+          },
+          event,
+          life,
+        );
       } catch {
         return null;
       }
@@ -103,37 +99,20 @@ const correct = async <Model extends ILlmSchema.Model>(
   const pointer: IPointer<IAutoBeTestCorrectApplication.IProps | null> = {
     value: null,
   };
-  const agentica = new MicroAgentica({
-    model: ctx.model,
-    vendor: ctx.vendor,
-    config: {
-      ...(ctx.config ?? {}),
-      executor: {
-        describe: null,
-      },
-      retry: 4,
-    },
+  const { tokenUsage } = await ctx.conversate({
+    source: "testCorrect",
     histories: transformTestCorrectHistories(content, validate.result),
-    controllers: [
-      createApplication({
-        model: ctx.model,
-        artifacts: content.artifacts,
-        build: (next) => {
-          pointer.value = next;
-        },
-      }),
-    ],
-  });
-  enforceToolCall(agentica);
-
-  await agentica
-    .conversate(
+    controller: createController({
+      model: ctx.model,
+      artifacts: content.artifacts,
+      build: (next) => {
+        pointer.value = next;
+      },
+    }),
+    enforceFunctionCall: true,
+    message:
       "Fix the `AutoBeTest.IFunction` data to resolve the compilation error.",
-    )
-    .finally(() => {
-      const tokenUsage = agentica.getTokenUsage();
-      ctx.usage().record(tokenUsage, ["test"]);
-    });
+  });
   if (pointer.value === null) throw new Error("Failed to modify test code.");
 
   const compiler: IAutoBeCompiler = await ctx.compiler();
@@ -144,6 +123,7 @@ const correct = async <Model extends ILlmSchema.Model>(
     created_at: new Date().toISOString(),
     file: validate.file,
     result: validate.result,
+    tokenUsage,
     step: ctx.state().analyze?.step ?? 0,
     ...pointer.value,
   });
@@ -155,7 +135,7 @@ const correct = async <Model extends ILlmSchema.Model>(
   return predicate(ctx, newContent, newValidate, life);
 };
 
-const createApplication = <Model extends ILlmSchema.Model>(props: {
+const createController = <Model extends ILlmSchema.Model>(props: {
   model: Model;
   artifacts: IAutoBeTestScenarioArtifacts;
   build: (next: IAutoBeTestCorrectApplication.IProps) => void;
@@ -164,7 +144,7 @@ const createApplication = <Model extends ILlmSchema.Model>(props: {
 
   const application: ILlmApplication<Model> = collection[
     props.model
-  ] as unknown as ILlmApplication<Model>;
+  ] satisfies ILlmApplication<any> as unknown as ILlmApplication<Model>;
   return {
     protocol: "class",
     name: "Modify Test Code",
@@ -179,19 +159,9 @@ const createApplication = <Model extends ILlmSchema.Model>(props: {
   };
 };
 
-const claude = typia.llm.application<
-  IAutoBeTestCorrectApplication,
-  "claude",
-  {
-    reference: true;
-  }
->();
+const claude = typia.llm.application<IAutoBeTestCorrectApplication, "claude">();
 const collection = {
-  chatgpt: typia.llm.application<
-    IAutoBeTestCorrectApplication,
-    "chatgpt",
-    { reference: true }
-  >(),
+  chatgpt: typia.llm.application<IAutoBeTestCorrectApplication, "chatgpt">(),
   claude,
   llama: claude,
   deepseek: claude,