@autobe/agent 0.20.0 → 0.22.0

package/src/orchestrate/test/compile/completeTestCode.ts

@@ -1,39 +1,28 @@
 import { StringUtil } from "@autobe/utils";
 
 import { IAutoBeTestScenarioArtifacts } from "../structures/IAutoBeTestScenarioArtifacts";
+import { getTestImportStatements } from "./getTestImportStatements";
 
 export function completeTestCode(
   artifacts: IAutoBeTestScenarioArtifacts,
   code: string,
 ): string {
-  const typeReferences: string[] = Array.from(
-    new Set(
-      Object.keys(artifacts.document.components.schemas).map(
-        (key) => key.split(".")[0]!,
-      ),
-    ),
-  );
+  code = code
+    .split("\r\n")
+    .join("\n")
+    .split("\n")
+    .filter((str) => str.trim().startsWith("import") === false)
+    .join("\n");
 
-  code = code.replace(/^[ \t]*import\b[\s\S]*?;[ \t]*$/gm, "").trim();
-  code = code.replace(/^[ \t]*import\b[\s\S]*?;[ \t]*$/gm, "").trim();
-  code = code.replaceAll(
-    'string & Format<"uuid">',
-    'string & tags.Format<"uuid">',
-  );
-  code = StringUtil.trim`
-    import { ArrayUtil, RandomGenerator, TestValidator } from "@nestia/e2e";
-    import { IConnection } from "@nestia/fetcher";
-    import typia, { tags } from "typia";
-    
-    import api from "@ORGANIZATION/PROJECT-api";
-    ${typeReferences
-      .map(
-        (ref) =>
-          `import type { ${ref} } from "@ORGANIZATION/PROJECT-api/lib/structures/${ref}";`,
-      )
-      .join("\n")}
+  // code = code.replace(/^[ \t]*import\b[\s\S]*?;[ \t]*$/gm, "").trim();
+  // code = code.replace(/^[ \t]*import\b[\s\S]*?;[ \t]*$/gm, "").trim();
+  // code = code.replaceAll(
+  //   'string & Format<"uuid">',
+  //   'string & tags.Format<"uuid">',
+  // );
+  return StringUtil.trim`
+    ${getTestImportStatements(artifacts.document)}
     
     ${code}
   `;
-  return code;
 }

package/src/orchestrate/test/compile/getTestExternalDeclarations.ts

@@ -19,7 +19,10 @@ const singleton = new Singleton(async (compiler: IAutoBeTestCompiler) => {
     return { [location]: content };
   };
   return {
+    ...external("node_modules/@nestia/e2e/lib/ArrayUtil.d.ts"),
+    ...external("node_modules/@nestia/e2e/lib/RandomGenerator.d.ts"),
     ...external("node_modules/@nestia/e2e/lib/TestValidator.d.ts"),
     ...external("node_modules/@nestia/fetcher/lib/IConnection.d.ts"),
+    ...external("node_modules/typia/lib/module.d.ts"),
   } satisfies Record<string, string>;
 });

package/src/orchestrate/test/compile/getTestImportStatements.ts

@@ -0,0 +1,25 @@
+import { AutoBeOpenApi } from "@autobe/interface";
+import { StringUtil } from "@autobe/utils";
+
+export const getTestImportStatements = (
+  document: AutoBeOpenApi.IDocument,
+): string => {
+  const typeReferences: string[] = Array.from(
+    new Set(
+      Object.keys(document.components.schemas).map((key) => key.split(".")[0]!),
+    ),
+  ).sort();
+  return StringUtil.trim`
+    import { ArrayUtil, RandomGenerator, TestValidator } from "@nestia/e2e";
+    import { IConnection } from "@nestia/fetcher";
+    import typia, { tags } from "typia";
+    
+    import api from "@ORGANIZATION/PROJECT-api";
+    ${typeReferences
+      .map(
+        (ref) =>
+          `import type { ${ref} } from "@ORGANIZATION/PROJECT-api/lib/structures/${ref}";`,
+      )
+      .join("\n")}
+  `;
+};

package/src/orchestrate/test/compile/getTestScenarioArtifacts.ts

@@ -7,10 +7,14 @@ import { ILlmSchema, OpenApiTypeChecker } from "@samchon/openapi";
 
 import { AutoBeContext } from "../../../context/AutoBeContext";
 import { IAutoBeTestScenarioArtifacts } from "../structures/IAutoBeTestScenarioArtifacts";
+import { getTestTemplateCode } from "./getTestTemplateCode";
 
 export async function getTestScenarioArtifacts<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  scenario: Pick<AutoBeTestScenario, "endpoint" | "dependencies">,
+  scenario: Pick<
+    AutoBeTestScenario,
+    "endpoint" | "dependencies" | "functionName"
+  >,
 ): Promise<IAutoBeTestScenarioArtifacts> {
   const compiler: IAutoBeCompiler = await ctx.compiler();
   const document: AutoBeOpenApi.IDocument = filterDocument(
@@ -35,6 +39,7 @@ export async function getTestScenarioArtifacts<Model extends ILlmSchema.Model>(
     sdk: filter("src/api", "src/api/structures"),
     dto: filter("src/api/structures"),
     e2e: filter("test/features"),
+    template: getTestTemplateCode(scenario, document),
   };
 }
 

package/src/orchestrate/test/compile/getTestTemplateCode.ts

@@ -0,0 +1,25 @@
+import { AutoBeOpenApi, AutoBeTestScenario } from "@autobe/interface";
+import { StringUtil } from "@autobe/utils";
+
+import { getTestImportStatements } from "./getTestImportStatements";
+
+export const getTestTemplateCode = (
+  scenario: Pick<
+    AutoBeTestScenario,
+    "endpoint" | "dependencies" | "functionName"
+  >,
+  document: AutoBeOpenApi.IDocument,
+): string => {
+  return StringUtil.trim`
+    ${getTestImportStatements(document)}
+
+    /**
+     * <SCENARIO DESCRIPTION HERE>
+     */
+    export async function ${scenario.functionName}(
+      connection: api.IConnection,
+    ) {
+      // <E2E TEST CODE HERE>
+    }
+  `;
+};

package/src/orchestrate/test/histories/transformTestCorrectHistories.ts

@@ -1,47 +1,59 @@
 import { IAgenticaHistoryJson } from "@agentica/core";
-import { IAutoBeTypeScriptCompileResult } from "@autobe/interface";
 import { StringUtil } from "@autobe/utils";
-import { v7 } from "uuid";
+import { ILlmSchema } from "@samchon/openapi";
+import { v4, v7 } from "uuid";
 
 import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { AutoBeContext } from "../../../context/AutoBeContext";
 import { IAutoBeTestFunction } from "../structures/IAutoBeTestFunction";
+import { IAutoBeTestFunctionFailure } from "../structures/IAutoBeTestFunctionFailure";
 import { transformTestWriteHistories } from "./transformTestWriteHistories";
 
-export const transformTestCorrectHistories = (
+export const transformTestCorrectHistories = async <
+  Model extends ILlmSchema.Model,
+>(
+  ctx: AutoBeContext<Model>,
   func: IAutoBeTestFunction,
-  failure: IAutoBeTypeScriptCompileResult.IFailure,
-): Array<
-  IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
-> => [
-  ...transformTestWriteHistories(func.scenario, func.artifacts),
-  {
-    id: v7(),
-    created_at: new Date().toISOString(),
-    type: "assistantMessage",
-    text: StringUtil.trim`
-      ## Generated TypeScript Code
-      \`\`\`typescript
-      ${func.script}
-      \`\`\`
+  failures: IAutoBeTestFunctionFailure[],
+): Promise<
+  Array<
+    IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
+  >
+> => {
+  const previous: Array<
+    IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
+  > = await transformTestWriteHistories(ctx, func.scenario, func.artifacts);
+  return [
+    ...previous.slice(0, -1),
+    {
+      id: v7(),
+      created_at: new Date().toISOString(),
+      type: "systemMessage",
+      text: AutoBeSystemPromptConstant.TEST_CORRECT,
+    },
+    previous.at(-1)!,
+    ...failures.map(
+      (f) =>
+        ({
+          id: v4(),
+          created_at: new Date().toISOString(),
+          type: "assistantMessage",
+          text: StringUtil.trim`
+            ## Generated TypeScript Code
 
-      ## Compile Errors
-      Fix the compilation error in the provided code.
+            \`\`\`typescript
+            ${f.function.script}
+            \`\`\`
 
-      \`\`\`json
-      ${JSON.stringify(failure.diagnostics)}
-      \`\`\`
-    `,
-  },
-  {
-    id: v7(),
-    created_at: new Date().toISOString(),
-    type: "systemMessage",
-    text: AutoBeSystemPromptConstant.TEST_CORRECT.replace(
-      "{{API_DTO_SCHEMAS}}",
-      transformTestWriteHistories.structures(func.artifacts),
-    ).replace(
-      "{{API_SDK_FUNCTIONS}}",
-      transformTestWriteHistories.functional(func.artifacts),
+            ## Compile Errors
+
+            Fix the compilation error in the provided code.
+
+            \`\`\`json
+            ${JSON.stringify(f.failure.diagnostics)}
+            \`\`\`
+          `,
+        }) satisfies IAgenticaHistoryJson.IAssistantMessage,
     ),
-  },
-];
+  ];
+};

package/src/orchestrate/test/histories/transformTestScenarioHistories.ts

@@ -48,7 +48,7 @@ export const transformTestScenarioHistories = (
     {
       id: v7(),
       created_at: new Date().toISOString(),
-      type: "systemMessage",
+      type: "assistantMessage",
       text: StringUtil.trim`
         # Operations
 
@@ -73,17 +73,18 @@ export const transformTestScenarioHistories = (
         )}
         \`\`\`
       `,
-    } satisfies IAgenticaHistoryJson.ISystemMessage,
+    } satisfies IAgenticaHistoryJson.IAssistantMessage,
     {
       id: v7(),
       created_at: new Date().toISOString(),
-      type: "systemMessage",
+      type: "assistantMessage",
       text: StringUtil.trim`
         # Included in Test Plan
 
         Below are the endpoints that have been included in the test plan.
         Each endpoint shows its authentication requirements and related authentication APIs.
-        When testing endpoints that require authentication, ensure you include the corresponding join/login operations in your test scenario to establish proper authentication context.
+        When testing endpoints that require authentication, ensure you include the corresponding 
+        join/login operations in your test scenario to establish proper authentication context.
 
         ${include
           .map((el, i) => {
@@ -121,6 +122,6 @@ export const transformTestScenarioHistories = (
           .map((el) => `- ${el.method.toUpperCase()}: ${el.path}`)
           .join("\n")}
       `,
-    } satisfies IAgenticaHistoryJson.ISystemMessage,
+    } satisfies IAgenticaHistoryJson.IAssistantMessage,
   ];
 };

package/src/orchestrate/test/histories/transformTestWriteHistories.ts

@@ -4,18 +4,28 @@ import { StringUtil, transformOpenApiDocument } from "@autobe/utils";
 import {
   HttpMigration,
   IHttpMigrateApplication,
+  ILlmSchema,
   OpenApi,
 } from "@samchon/openapi";
 import typia from "typia";
 import { v7 } from "uuid";
 
 import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { AutoBeContext } from "../../../context/AutoBeContext";
+import { getTestExternalDeclarations } from "../compile/getTestExternalDeclarations";
 import { IAutoBeTestScenarioArtifacts } from "../structures/IAutoBeTestScenarioArtifacts";
 
-export function transformTestWriteHistories(
+export async function transformTestWriteHistories<
+  Model extends ILlmSchema.Model,
+>(
+  ctx: AutoBeContext<Model>,
   scenario: AutoBeTestScenario,
   artifacts: IAutoBeTestScenarioArtifacts,
-): Array<IAgenticaHistoryJson.ISystemMessage> {
+): Promise<
+  Array<
+    IAgenticaHistoryJson.ISystemMessage | IAgenticaHistoryJson.IAssistantMessage
+  >
+> {
   return [
     {
       id: v7(),
@@ -24,17 +34,21 @@ export function transformTestWriteHistories(
       text: AutoBeSystemPromptConstant.TEST_WRITE.replace(
         "{{AutoBeTestScenario}}",
         JSON.stringify(typia.llm.parameters<AutoBeTestScenario, "llama">()),
-      ).replaceAll("{{FUNCTION_NAME}}", scenario.functionName),
+      ),
     },
     {
       id: v7(),
       created_at: new Date().toISOString(),
-      type: "systemMessage",
+      type: "assistantMessage",
       text: StringUtil.trim`
         Here is the list of input material composition.
 
         Make e2e test functions based on the following information.
 
+        ## Function Name
+
+        The e2e test function name must be ${JSON.stringify(scenario.functionName)}.
+
         ## Scenario Plan
 
         Here is the scenario plan what you have to implement.
@@ -66,6 +80,31 @@ export function transformTestWriteHistories(
         \`\`\`json
         ${JSON.stringify(artifacts.e2e)}
         \`\`\`
+
+        ## External Definitions
+
+        Here is the external declaration files (d.ts) you can reference.
+
+        \`\`\`json
+        ${JSON.stringify(await getTestExternalDeclarations(ctx))}
+        \`\`\`
+
+        ## Template Code
+
+        Here is the template e2e test code what you must follow.
+
+        You're only allowed to modify the "<SCENARIO DESCRIPTION HERE>" and
+        code inside the function block marked as "// <E2E TEST CODE HERE>". 
+        Change the template code by writing your scenario description to the 
+        comment, and filling your implementation logic into the function.
+
+        Note that, you don't need to add any "import" statement more than
+        this template code. Everything you need is already imported, so
+        make your implementation code in the import scope.
+
+        \`\`\`typescript
+        ${artifacts.template}
+        \`\`\`
       `,
     },
   ];

package/src/orchestrate/test/orchestrateTest.ts

@@ -68,6 +68,12 @@ export const orchestrateTest =
       ctx,
       written,
     );
+    // for (const c of corrects)
+    //   if (c.result.type !== "success")
+    //     c.file.content = c.file.content
+    //       .split("\n")
+    //       .map((s) => `// ${s}`)
+    //       .join("\n");
 
     // DO COMPILE
     const compiler: IAutoBeCompiler = await ctx.compiler();

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -17,13 +17,13 @@ import { completeTestCode } from "./compile/completeTestCode";
 import { transformTestCorrectHistories } from "./histories/transformTestCorrectHistories";
 import { IAutoBeTestCorrectApplication } from "./structures/IAutoBeTestCorrectApplication";
 import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
+import { IAutoBeTestFunctionFailure } from "./structures/IAutoBeTestFunctionFailure";
 import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
 import { IAutoBeTestWriteResult } from "./structures/IAutoBeTestWriteResult";
 
 export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   writeResult: IAutoBeTestWriteResult[],
-  life: number = 4,
 ): Promise<AutoBeTestValidateEvent[]> => {
   const result: Array<AutoBeTestValidateEvent | null> =
     await executeCachedBatch(
@@ -33,7 +33,7 @@ export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
             artifacts: w.artifacts,
             scenario: w.scenario,
             location: w.event.location,
-            script: w.event.final,
+            script: w.event.final ?? w.event.draft,
           });
           return await predicate(
             ctx,
@@ -41,10 +41,11 @@ export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
               artifacts: w.artifacts,
               scenario: w.scenario,
               location: w.event.location,
-              script: w.event.final,
+              script: w.event.final ?? w.event.draft,
             },
+            [],
             event,
-            life,
+            ctx.retry,
           );
         } catch {
           return null;
@@ -83,18 +84,20 @@ const compile = async <Model extends ILlmSchema.Model>(
 const predicate = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   content: IAutoBeTestFunction,
+  failures: IAutoBeTestFunctionFailure[],
   event: AutoBeTestValidateEvent,
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
   if (event.result.type === "failure") ctx.dispatch(event);
   return event.result.type === "failure"
-    ? await correct(ctx, content, event, life - 1)
+    ? await correct(ctx, content, failures, event, life - 1)
     : event;
 };
 
 const correct = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   content: IAutoBeTestFunction,
+  failures: IAutoBeTestFunctionFailure[],
   validate: AutoBeTestValidateEvent,
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
@@ -106,7 +109,13 @@ const correct = async <Model extends ILlmSchema.Model>(
   };
   const { tokenUsage } = await ctx.conversate({
     source: "testCorrect",
-    histories: transformTestCorrectHistories(content, validate.result),
+    histories: await transformTestCorrectHistories(ctx, content, [
+      ...failures,
+      {
+        function: content,
+        failure: validate.result,
+      },
+    ]),
     controller: createController({
       model: ctx.model,
       artifacts: content.artifacts,
@@ -125,7 +134,14 @@ const correct = async <Model extends ILlmSchema.Model>(
   if (pointer.value === null) throw new Error("Failed to modify test code.");
 
   const compiler: IAutoBeCompiler = await ctx.compiler();
-  pointer.value.final = await compiler.typescript.beautify(pointer.value.final);
+  if (pointer.value.revise)
+    pointer.value.revise.final = await compiler.typescript.beautify(
+      pointer.value.revise.final,
+    );
+  else
+    pointer.value.draft = await compiler.typescript.beautify(
+      pointer.value.draft,
+    );
 
   ctx.dispatch({
     type: "testCorrect",
@@ -135,14 +151,29 @@ const correct = async <Model extends ILlmSchema.Model>(
     result: validate.result,
     tokenUsage,
     step: ctx.state().analyze?.step ?? 0,
-    ...pointer.value,
+    think: pointer.value.think,
+    draft: pointer.value.draft,
+    review: pointer.value.revise?.review,
+    final: pointer.value.revise?.final,
   });
   const newContent: IAutoBeTestFunction = {
     ...content,
-    script: pointer.value.final,
+    script: pointer.value.revise?.final ?? pointer.value.draft,
   };
   const newValidate: AutoBeTestValidateEvent = await compile(ctx, newContent);
-  return predicate(ctx, newContent, newValidate, life);
+  return predicate(
+    ctx,
+    newContent,
+    [
+      ...failures,
+      {
+        function: content,
+        failure: validate.result,
+      },
+    ],
+    newValidate,
+    life,
+  );
 };
 
 const createController = <Model extends ILlmSchema.Model>(props: {
@@ -162,7 +193,11 @@ const createController = <Model extends ILlmSchema.Model>(props: {
     execute: {
       rewrite: (next) => {
         next.draft = completeTestCode(props.artifacts, next.draft);
-        next.final = completeTestCode(props.artifacts, next.final);
+        if (next.revise)
+          next.revise.final = completeTestCode(
+            props.artifacts,
+            next.revise.final,
+          );
         props.build(next);
       },
     } satisfies IAutoBeTestCorrectApplication,

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -64,7 +64,7 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   do {
     const matrix: AutoBeOpenApi.IOperation[][] = divideArray({
       array: include,
-      capacity: 5,
+      capacity: 4,
     });
     await executeCachedBatch(
       matrix.map((include) => async (promptCacheKey) => {
@@ -121,56 +121,60 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
   const pointer: IPointer<IAutoBeTestScenarioApplication.IScenarioGroup[]> = {
     value: [],
   };
-
   const authorizations: AutoBeInterfaceAuthorization[] =
     ctx.state().interface?.authorizations ?? [];
 
-  const { tokenUsage } = await ctx.conversate({
-    source: "testScenarios",
-    histories: transformTestScenarioHistories(
-      ctx.state(),
-      props.entire,
-      props.include,
-      props.exclude,
-    ),
-    controller: createController({
-      model: ctx.model,
-      endpointNotFound: props.endpointNotFound,
-      dict: props.dict,
-      authorizations,
-      build: (next) => {
-        pointer.value ??= [];
-        pointer.value.push(...next.scenarioGroups);
-      },
-    }),
-    enforceFunctionCall: true,
-    promptCacheKey: props.promptCacheKey,
-    message: `Create e2e test scenarios.`,
-  });
-  if (pointer.value.length === 0) return [];
-  ctx.dispatch({
-    type: "testScenarios",
-    id: v7(),
-    tokenUsage,
-    scenarios: pointer.value
-      .map((v) =>
-        v.scenarios.map(
-          (s) =>
-            ({
-              endpoint: v.endpoint,
-              draft: s.draft,
-              functionName: s.functionName,
-              dependencies: s.dependencies,
-            }) satisfies AutoBeTestScenario,
-        ),
-      )
-      .flat(),
-    completed: ++props.progress.completed,
-    total: props.progress.total,
-    step: ctx.state().interface?.step ?? 0,
-    created_at: new Date().toISOString(),
-  });
-  return pointer.value;
+  try {
+    const { tokenUsage } = await ctx.conversate({
+      source: "testScenarios",
+      histories: transformTestScenarioHistories(
+        ctx.state(),
+        props.entire,
+        props.include,
+        props.exclude,
+      ),
+      controller: createController({
+        model: ctx.model,
+        endpointNotFound: props.endpointNotFound,
+        dict: props.dict,
+        authorizations,
+        build: (next) => {
+          pointer.value ??= [];
+          pointer.value.push(...next.scenarioGroups);
+        },
+      }),
+      enforceFunctionCall: true,
+      promptCacheKey: props.promptCacheKey,
+      message: `Create e2e test scenarios.`,
+    });
+    if (pointer.value.length === 0) return [];
+    ctx.dispatch({
+      type: "testScenarios",
+      id: v7(),
+      tokenUsage,
+      scenarios: pointer.value
+        .map((v) =>
+          v.scenarios.map(
+            (s) =>
+              ({
+                endpoint: v.endpoint,
+                draft: s.draft,
+                functionName: s.functionName,
+                dependencies: s.dependencies,
+              }) satisfies AutoBeTestScenario,
+          ),
+        )
+        .flat(),
+      completed: (props.progress.completed += pointer.value.length),
+      total: props.progress.total,
+      step: ctx.state().interface?.step ?? 0,
+      created_at: new Date().toISOString(),
+    });
+    return pointer.value;
+  } catch {
+    console.log("test scenario, failed to function call", props.include);
+    return [];
+  }
 };
 
 function createController<Model extends ILlmSchema.Model>(props: {