@autobe/agent 0.19.1 → 0.21.0

package/src/orchestrate/test/histories/transformTestCorrectHistories.ts

@@ -1,46 +1,59 @@
 import { IAgenticaHistoryJson } from "@agentica/core";
 import { IAutoBeTypeScriptCompileResult } from "@autobe/interface";
-import { v4 } from "uuid";
+import { StringUtil } from "@autobe/utils";
+import { ILlmSchema } from "@samchon/openapi";
+import { v7 } from "uuid";
 
 import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { AutoBeContext } from "../../../context/AutoBeContext";
 import { IAutoBeTestFunction } from "../structures/IAutoBeTestFunction";
 import { transformTestWriteHistories } from "./transformTestWriteHistories";
 
-export const transformTestCorrectHistories = (
+export const transformTestCorrectHistories = async <
+  Model extends ILlmSchema.Model,
+>(
+  ctx: AutoBeContext<Model>,
   func: IAutoBeTestFunction,
   failure: IAutoBeTypeScriptCompileResult.IFailure,
-): Array<
-  IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
-> => [
-  ...transformTestWriteHistories(func.scenario, func.artifacts),
-  {
-    id: v4(),
-    created_at: new Date().toISOString(),
-    type: "assistantMessage",
-    text: [
-      "## Generated TypeScript Code",
-      "```typescript",
-      func.script,
-      "```",
-      "",
-      "## Compile Errors",
-      "Fix the compilation error in the provided code.",
-      "",
-      "```json",
-      JSON.stringify(failure.diagnostics),
-      "```",
-    ].join("\n"),
-  },
-  {
-    id: v4(),
-    created_at: new Date().toISOString(),
-    type: "systemMessage",
-    text: AutoBeSystemPromptConstant.TEST_CORRECT.replace(
-      "{{API_DTO_SCHEMAS}}",
-      transformTestWriteHistories.structures(func.artifacts),
-    ).replace(
-      "{{API_SDK_FUNCTIONS}}",
-      transformTestWriteHistories.functional(func.artifacts),
-    ),
-  },
-];
+): Promise<
+  Array<
+    IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
+  >
+> => {
+  const previous = await transformTestWriteHistories(
+    ctx,
+    func.scenario,
+    func.artifacts,
+  );
+  return [
+    ...previous.slice(0, -1),
+    {
+      id: v7(),
+      created_at: new Date().toISOString(),
+      type: "systemMessage",
+      text: AutoBeSystemPromptConstant.TEST_CORRECT,
+    },
+    {
+      id: v7(),
+      created_at: new Date().toISOString(),
+      type: "assistantMessage",
+      text: StringUtil.trim`
+        ${previous.at(-1)!.text}
+
+        ## Generated TypeScript Code
+
+        \`\`\`typescript
+        ${func.script}
+        \`\`\`
+
+        ## Compile Errors
+        
+        Fix the compilation error in the provided code.
+
+        \`\`\`json
+        ${JSON.stringify(failure.diagnostics)}
+        \`\`\`
+      `,
+    },
+  ];
+};

package/src/orchestrate/test/histories/transformTestScenarioHistories.ts

@@ -1,7 +1,7 @@
 import { IAgenticaHistoryJson } from "@agentica/core";
 import { AutoBeInterfaceAuthorization, AutoBeOpenApi } from "@autobe/interface";
-import { MapUtil } from "@autobe/utils";
-import { v4 } from "uuid";
+import { MapUtil, StringUtil } from "@autobe/utils";
+import { v7 } from "uuid";
 
 import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
 import { AutoBeState } from "../../../context/AutoBeState";
@@ -40,82 +40,87 @@ export const transformTestScenarioHistories = (
 
   return [
     {
-      id: v4(),
+      id: v7(),
       created_at: new Date().toISOString(),
       type: "systemMessage",
       text: AutoBeSystemPromptConstant.TEST_SCENARIO,
     } satisfies IAgenticaHistoryJson.ISystemMessage,
     {
-      id: v4(),
+      id: v7(),
       created_at: new Date().toISOString(),
       type: "systemMessage",
-      text: [
-        "# Operations",
-        "",
-        "Below are the full operations. Please refer to this.",
-        "Your role is to draft all test cases for each given Operation.",
-        "It is also permissible to write multiple test codes on a single endpoint.",
-        "However, rather than meaningless tests, business logic tests should be written and an E2E test situation should be assumed.",
-        "",
-        "Please carefully analyze each operation to identify all dependencies required for testing.",
-        "For example, if you want to test liking and then deleting a post,",
-        "you might think to test post creation, liking, and unlike operations.",
-        "However, even if not explicitly mentioned, user registration or login are essential prerequisites.",
-        "Pay close attention to IDs and related values in the API,",
-        "and ensure you identify all dependencies between endpoints.",
-        "",
-        "```json",
-        JSON.stringify(
+      text: StringUtil.trim`
+        # Operations
+
+        Below are the full operations. Please refer to this.
+        Your role is to draft all test cases for each given Operation.
+        It is also permissible to write multiple test codes on a single endpoint.
+        However, rather than meaningless tests, business logic tests should be written and an E2E test situation should be assumed.
+
+        Please carefully analyze each operation to identify all dependencies required for testing.
+        For example, if you want to test liking and then deleting a post,
+        you might think to test post creation, liking, and unlike operations.
+        However, even if not explicitly mentioned, user registration or login are essential prerequisites.
+        Pay close attention to IDs and related values in the API,
+        and ensure you identify all dependencies between endpoints.
+
+        \`\`\`json
+        ${JSON.stringify(
           entire.map((el) => ({
             ...el,
             specification: undefined,
           })),
-        ),
-        "```",
-      ].join("\n"),
+        )}
+        \`\`\`
+      `,
     } satisfies IAgenticaHistoryJson.ISystemMessage,
     {
-      id: v4(),
+      id: v7(),
       created_at: new Date().toISOString(),
       type: "systemMessage",
-      text: [
-        "# Included in Test Plan",
-        "",
-        "Below are the endpoints that have been included in the test plan.",
-        "Each endpoint shows its authentication requirements and related authentication APIs.",
-        "When testing endpoints that require authentication, ensure you include the corresponding join/login operations in your test scenario to establish proper authentication context.",
-        "",
-        include
+      text: StringUtil.trim`
+        # Included in Test Plan
+
+        Below are the endpoints that have been included in the test plan.
+        Each endpoint shows its authentication requirements and related authentication APIs.
+        When testing endpoints that require authentication, ensure you include the corresponding join/login operations in your test scenario to establish proper authentication context.
+
+        ${include
           .map((el, i) => {
             const roles = Array.from(authorizationRoles.values()).filter(
               (role) => role.name === el.authorizationRole,
             );
-            return [
-              `## ${i + 1}. ${el.method.toUpperCase()} ${el.path}`,
-              "",
-              "Related Authentication APIs:",
-              "",
-              roles.length > 0
-                ? roles.map((role) => {
-                    return [
-                      `- ${role.join?.method.toUpperCase()}: ${role.join?.path}`,
-                      `- ${role.login?.method.toUpperCase()}: ${role.login?.path}`,
-                    ].join("\n");
-                  })
-                : "- None",
-            ];
+            return StringUtil.trim`
+              ## ${i + 1}. ${el.method.toUpperCase()} ${el.path}
+
+              Related Authentication APIs:
+
+              ${
+                roles.length > 0
+                  ? roles
+                      .map((role) => {
+                        return StringUtil.trim`
+                          - ${role.join?.method.toUpperCase()}: ${role.join?.path}
+                          - ${role.login?.method.toUpperCase()}: ${role.login?.path}
+                        `;
+                      })
+                      .join("\n")
+                  : "- None"
+              }
+            `;
           })
-          .join("\n"),
-        "",
-        "# Excluded from Test Plan",
-        "",
-        "These are the endpoints that have already been used in test codes generated as part of a plan group.",
-        "These endpoints do not need to be tested again.",
-        "However, it is allowed to reference or depend on these endpoints when writing test codes for other purposes.",
-        exclude
+          .join("\n")}
+
+        # Excluded from Test Plan
+
+        These are the endpoints that have already been used in test codes generated as part of a plan group.
+        These endpoints do not need to be tested again.
+        However, it is allowed to reference or depend on these endpoints when writing test codes for other purposes.
+
+        ${exclude
           .map((el) => `- ${el.method.toUpperCase()}: ${el.path}`)
-          .join("\n"),
-      ].join("\n"),
+          .join("\n")}
+      `,
     } satisfies IAgenticaHistoryJson.ISystemMessage,
   ];
 };

package/src/orchestrate/test/histories/transformTestWriteHistories.ts

@@ -4,37 +4,51 @@ import { StringUtil, transformOpenApiDocument } from "@autobe/utils";
 import {
   HttpMigration,
   IHttpMigrateApplication,
+  ILlmSchema,
   OpenApi,
 } from "@samchon/openapi";
 import typia from "typia";
-import { v4 } from "uuid";
+import { v7 } from "uuid";
 
 import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { AutoBeContext } from "../../../context/AutoBeContext";
+import { getTestExternalDeclarations } from "../compile/getTestExternalDeclarations";
 import { IAutoBeTestScenarioArtifacts } from "../structures/IAutoBeTestScenarioArtifacts";
 
-export function transformTestWriteHistories(
+export async function transformTestWriteHistories<
+  Model extends ILlmSchema.Model,
+>(
+  ctx: AutoBeContext<Model>,
   scenario: AutoBeTestScenario,
   artifacts: IAutoBeTestScenarioArtifacts,
-): Array<IAgenticaHistoryJson.ISystemMessage> {
+): Promise<
+  Array<
+    IAgenticaHistoryJson.ISystemMessage | IAgenticaHistoryJson.IAssistantMessage
+  >
+> {
   return [
     {
-      id: v4(),
+      id: v7(),
       created_at: new Date().toISOString(),
       type: "systemMessage",
       text: AutoBeSystemPromptConstant.TEST_WRITE.replace(
         "{{AutoBeTestScenario}}",
         JSON.stringify(typia.llm.parameters<AutoBeTestScenario, "llama">()),
-      ).replaceAll("{{FUNCTION_NAME}}", scenario.functionName),
+      ),
     },
     {
-      id: v4(),
+      id: v7(),
       created_at: new Date().toISOString(),
-      type: "systemMessage",
+      type: "assistantMessage",
       text: StringUtil.trim`
         Here is the list of input material composition.
 
         Make e2e test functions based on the following information.
 
+        ## Function Name
+
+        The e2e test function name must be ${JSON.stringify(scenario.functionName)}.
+
         ## Scenario Plan
 
         Here is the scenario plan what you have to implement.
@@ -66,6 +80,12 @@ export function transformTestWriteHistories(
         \`\`\`json
         ${JSON.stringify(artifacts.e2e)}
         \`\`\`
+
+        ## External Definitions
+
+        \`\`\`json
+        ${JSON.stringify(await getTestExternalDeclarations(ctx))}
+        \`\`\`
       `,
     },
   ];

package/src/orchestrate/test/orchestrateTest.ts

@@ -8,7 +8,7 @@ import {
   IAutoBeTypeScriptCompileResult,
 } from "@autobe/interface";
 import { ILlmSchema } from "@samchon/openapi";
-import { v4 } from "uuid";
+import { v7 } from "uuid";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { IAutoBeApplicationProps } from "../../context/IAutoBeApplicationProps";
@@ -29,13 +29,14 @@ export const orchestrateTest =
     if (predicate !== null)
       return ctx.assistantMessage({
         type: "assistantMessage",
-        id: v4(),
+        id: v7(),
         created_at: start.toISOString(),
         text: predicate,
         completed_at: new Date().toISOString(),
       });
     ctx.dispatch({
       type: "testStart",
+      id: v7(),
       created_at: start.toISOString(),
       reason: props.reason,
       step: ctx.state().analyze?.step ?? 0,
@@ -46,7 +47,7 @@ export const orchestrateTest =
       ctx.state().interface?.document.operations ?? [];
     if (operations.length === 0)
       return ctx.assistantMessage({
-        id: v4(),
+        id: v7(),
         type: "assistantMessage",
         created_at: start.toISOString(),
         completed_at: new Date().toISOString(),
@@ -63,14 +64,10 @@ export const orchestrateTest =
       ctx,
       scenarios,
     );
-
     const corrects: AutoBeTestValidateEvent[] = await orchestrateTestCorrect(
       ctx,
       written,
     );
-    const success: AutoBeTestValidateEvent[] = corrects.filter(
-      (c) => c.result.type === "success",
-    );
 
     // DO COMPILE
     const compiler: IAutoBeCompiler = await ctx.compiler();
@@ -82,13 +79,14 @@ export const orchestrateTest =
               dbms: "sqlite",
             }),
           ).filter(([key]) => key.endsWith(".ts")),
-          ...success.map((s) => [s.file.location, s.file.content]),
+          ...corrects.map((s) => [s.file.location, s.file.content]),
         ]),
       });
     return ctx.dispatch({
       type: "testComplete",
+      id: v7(),
       created_at: new Date().toISOString(),
-      files: success.map((s) => s.file),
+      files: corrects.map((s) => s.file),
       compiled,
       step: ctx.state().interface?.step ?? 0,
       elapsed: new Date().getTime() - start.getTime(),

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -4,12 +4,15 @@ import {
   IAutoBeCompiler,
   IAutoBeTypeScriptCompileResult,
 } from "@autobe/interface";
+import { StringUtil } from "@autobe/utils";
 import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
+import { v7 } from "uuid";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
+import { executeCachedBatch } from "../../utils/executeCachedBatch";
 import { completeTestCode } from "./compile/completeTestCode";
 import { transformTestCorrectHistories } from "./histories/transformTestCorrectHistories";
 import { IAutoBeTestCorrectApplication } from "./structures/IAutoBeTestCorrectApplication";
@@ -22,31 +25,32 @@ export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   writeResult: IAutoBeTestWriteResult[],
   life: number = 4,
 ): Promise<AutoBeTestValidateEvent[]> => {
-  const result: Array<AutoBeTestValidateEvent | null> = await Promise.all(
-    writeResult.map(async (w) => {
-      try {
-        const event: AutoBeTestValidateEvent = await compile(ctx, {
-          artifacts: w.artifacts,
-          scenario: w.scenario,
-          location: w.event.location,
-          script: w.event.final,
-        });
-        return predicate(
-          ctx,
-          {
+  const result: Array<AutoBeTestValidateEvent | null> =
+    await executeCachedBatch(
+      writeResult.map((w) => async () => {
+        try {
+          const event: AutoBeTestValidateEvent = await compile(ctx, {
             artifacts: w.artifacts,
             scenario: w.scenario,
             location: w.event.location,
             script: w.event.final,
-          },
-          event,
-          life,
-        );
-      } catch {
-        return null;
-      }
-    }),
-  );
+          });
+          return await predicate(
+            ctx,
+            {
+              artifacts: w.artifacts,
+              scenario: w.scenario,
+              location: w.event.location,
+              script: w.event.final,
+            },
+            event,
+            life,
+          );
+        } catch {
+          return null;
+        }
+      }),
+    );
   return result.filter((r) => r !== null);
 };
 
@@ -64,6 +68,7 @@ const compile = async <Model extends ILlmSchema.Model>(
   });
   return {
     type: "testValidate",
+    id: v7(),
     file: {
       scenario: func.scenario,
       location: func.location,
@@ -83,7 +88,7 @@ const predicate = async <Model extends ILlmSchema.Model>(
 ): Promise<AutoBeTestValidateEvent> => {
   if (event.result.type === "failure") ctx.dispatch(event);
   return event.result.type === "failure"
-    ? correct(ctx, content, event, life - 1)
+    ? await correct(ctx, content, event, life - 1)
     : event;
 };
 
@@ -101,7 +106,11 @@ const correct = async <Model extends ILlmSchema.Model>(
   };
   const { tokenUsage } = await ctx.conversate({
     source: "testCorrect",
-    histories: transformTestCorrectHistories(content, validate.result),
+    histories: await transformTestCorrectHistories(
+      ctx,
+      content,
+      validate.result,
+    ),
     controller: createController({
       model: ctx.model,
       artifacts: content.artifacts,
@@ -110,8 +119,12 @@ const correct = async <Model extends ILlmSchema.Model>(
       },
     }),
     enforceFunctionCall: true,
-    message:
-      "Fix the `AutoBeTest.IFunction` data to resolve the compilation error.",
+    message: StringUtil.trim`
+      Fix the AutoBeTest.IFunction data to resolve the compilation error.
+
+      You don't need to explain me anything, but just fix it immediately
+      without any hesitation, explanation, and questions.
+    `,
   });
   if (pointer.value === null) throw new Error("Failed to modify test code.");
 
@@ -120,6 +133,7 @@ const correct = async <Model extends ILlmSchema.Model>(
 
   ctx.dispatch({
     type: "testCorrect",
+    id: v7(),
     created_at: new Date().toISOString(),
     file: validate.file,
     result: validate.result,

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -9,10 +9,12 @@ import { AutoBeEndpointComparator, MapUtil, StringUtil } from "@autobe/utils";
 import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
 import { HashMap, IPointer, Pair } from "tstl";
 import typia from "typia";
+import { v7 } from "uuid";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { divideArray } from "../../utils/divideArray";
+import { executeCachedBatch } from "../../utils/executeCachedBatch";
 import { transformTestScenarioHistories } from "./histories/transformTestScenarioHistories";
 import { IAutoBeTestScenarioApplication } from "./structures/IAutoBeTestScenarioApplication";
 import { IAutoBeTestScenarioAuthorizationRole } from "./structures/IAutoBeTestScenarioAuthorizationRole";
@@ -64,18 +66,18 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
       array: include,
       capacity: 5,
     });
-    await Promise.all(
-      matrix.map(async (include) => {
+    await executeCachedBatch(
+      matrix.map((include) => async (promptCacheKey) => {
         exclude.push(
-          ...(await divideAndConquer(
-            ctx,
+          ...(await divideAndConquer(ctx, {
             dict,
             endpointNotFound,
-            operations,
+            entire: operations,
             include,
-            exclude.map((x) => x.endpoint),
+            exclude: exclude.map((x) => x.endpoint),
             progress,
-          )),
+            promptCacheKey,
+          })),
         );
       }),
     );
@@ -106,13 +108,16 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
 
 const divideAndConquer = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>,
-  endpointNotFound: string,
-  entire: AutoBeOpenApi.IOperation[],
-  include: AutoBeOpenApi.IOperation[],
-  exclude: AutoBeOpenApi.IEndpoint[],
-  progress: AutoBeProgressEventBase,
-) => {
+  props: {
+    dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>;
+    endpointNotFound: string;
+    entire: AutoBeOpenApi.IOperation[];
+    include: AutoBeOpenApi.IOperation[];
+    exclude: AutoBeOpenApi.IEndpoint[];
+    progress: AutoBeProgressEventBase;
+    promptCacheKey: string;
+  },
+): Promise<IAutoBeTestScenarioApplication.IScenarioGroup[]> => {
   const pointer: IPointer<IAutoBeTestScenarioApplication.IScenarioGroup[]> = {
     value: [],
   };
@@ -124,14 +129,14 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
     source: "testScenarios",
     histories: transformTestScenarioHistories(
       ctx.state(),
-      entire,
-      include,
-      exclude,
+      props.entire,
+      props.include,
+      props.exclude,
     ),
     controller: createController({
       model: ctx.model,
-      endpointNotFound,
-      dict,
+      endpointNotFound: props.endpointNotFound,
+      dict: props.dict,
       authorizations,
       build: (next) => {
         pointer.value ??= [];
@@ -139,11 +144,13 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
       },
     }),
     enforceFunctionCall: true,
+    promptCacheKey: props.promptCacheKey,
     message: `Create e2e test scenarios.`,
   });
   if (pointer.value.length === 0) return [];
   ctx.dispatch({
     type: "testScenarios",
+    id: v7(),
     tokenUsage,
     scenarios: pointer.value
       .map((v) =>
@@ -158,8 +165,8 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
         ),
       )
       .flat(),
-    completed: ++progress.completed,
-    total: progress.total,
+    completed: ++props.progress.completed,
+    total: props.progress.total,
     step: ctx.state().interface?.step ?? 0,
     created_at: new Date().toISOString(),
   });