@autobe/agent 0.8.0 → 0.9.0

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -1,7 +1,9 @@
 import { IAgenticaController, MicroAgentica } from "@agentica/core";
 import {
-  AutoBeTestProgressEvent,
+  AutoBeOpenApi,
+  AutoBeTestScenarioEvent,
   AutoBeTestValidateEvent,
+  AutoBeTestWriteEvent,
   IAutoBeTypeScriptCompilerResult,
 } from "@autobe/interface";
 import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
@@ -10,32 +12,49 @@ import typia from "typia";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
+import { randomBackoffRetry } from "../../utils/backoffRetry";
 import { enforceToolCall } from "../../utils/enforceToolCall";
+import { filterDocument } from "./orchestrateTestProgress";
 import { transformTestCorrectHistories } from "./transformTestCorrectHistories";
 
 export async function orchestrateTestCorrect<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  codes: AutoBeTestProgressEvent[],
+  codes: AutoBeTestWriteEvent[],
+  scenarios: AutoBeTestScenarioEvent.IScenario[],
   life: number = 4,
 ): Promise<AutoBeTestValidateEvent> {
+  const scenarioMap: Map<string, AutoBeTestScenarioEvent.IScenario> = new Map();
+  codes.forEach(({ filename }, index) => {
+    scenarioMap.set(filename, scenarios[index]);
+  });
+
   // 1) Build map of new test files from progress events
-  const testFiles = Object.fromEntries(
-    codes.map(({ filename, content }) => [
-      `test/features/api/${filename}`,
-      content,
-    ]),
-  );
+  const testFiles: Record<string, string> = codes
+    .map(({ filename, content }) => {
+      return {
+        [`test/features/api/${filename}`]: content,
+      };
+    })
+    .reduce<Record<string, string>>((acc, cur) => Object.assign(acc, cur), {});
 
   // 2) Keep only files outside the test directory from current state
-  const retainedFiles = Object.fromEntries(
-    Object.entries(ctx.state().interface?.files ?? {}).filter(
-      ([filename]) => !filename.startsWith("test/features/api"),
-    ),
-  );
+  const retainedFiles: Record<string, string> = Object.entries(
+    ctx.state().interface?.files ?? {},
+  )
+    .filter(([filename]) => {
+      return !filename.startsWith("test/features/api");
+    })
+    .map(([filename, content]) => {
+      return { [filename]: content };
+    })
+    .reduce<Record<string, string>>((acc, cur) => Object.assign(acc, cur), {});
 
   // 3) Merge and filter: keep .ts/.json, drop anything under "benchmark"
-  const mergedFiles = { ...retainedFiles, ...testFiles };
-  const files = Object.fromEntries(
+  const mergedFiles: Record<string, string> = {
+    ...retainedFiles,
+    ...testFiles,
+  };
+  const files: Record<string, string> = Object.fromEntries(
     Object.entries(mergedFiles).filter(
       ([filename]) =>
         (filename.endsWith(".ts") && !filename.startsWith("test/benchmark/")) ||
@@ -44,7 +63,12 @@ export async function orchestrateTestCorrect<Model extends ILlmSchema.Model>(
   );
 
   // 4) Ask the LLM to correct the filtered file set
-  const response = await step(ctx, files, life);
+  const response: AutoBeTestValidateEvent = await step(
+    ctx,
+    files,
+    scenarioMap,
+    life,
+  );
 
   // 5) Combine original + corrected files and dispatch event
   const event: AutoBeTestValidateEvent = {
@@ -52,7 +76,6 @@ export async function orchestrateTestCorrect<Model extends ILlmSchema.Model>(
     type: "testValidate",
     files: { ...mergedFiles, ...response.files },
   };
-
   return event;
 }
 
@@ -74,13 +97,15 @@ export async function orchestrateTestCorrect<Model extends ILlmSchema.Model>(
 async function step<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   files: Record<string, string>,
+  scenarioMap: Map<string, AutoBeTestScenarioEvent.IScenario>,
   life: number,
 ): Promise<AutoBeTestValidateEvent> {
   // COMPILE TEST CODE
-  const result = await ctx.compiler.typescript({
-    files,
-  });
 
+  const result: IAutoBeTypeScriptCompilerResult =
+    await ctx.compiler.typescript.compile({
+      files,
+    });
   if (result.type === "success") {
     // SUCCESS
     return {
@@ -101,7 +126,6 @@ async function step<Model extends ILlmSchema.Model>(
       result,
       step: ctx.state().interface?.step ?? 0,
     });
-
     throw new Error(JSON.stringify(result.error, null, 2));
   }
 
@@ -155,11 +179,17 @@ async function step<Model extends ILlmSchema.Model>(
     };
 
   // VALIDATION FAILED
-  const validate = await Promise.all(
+  const validate: [string, string][] = await Promise.all(
     Object.entries(diagnostics).map(async ([filename, d]) => {
-      const code = files[filename];
-      const response = await process(ctx, d, code);
-
+      const scenario: AutoBeTestScenarioEvent.IScenario =
+        scenarioMap.get(filename)!;
+      const code: string = files[filename];
+      const response: ICorrectTestFunctionProps = await process(
+        ctx,
+        d,
+        code,
+        scenario,
+      );
       ctx.dispatch({
         type: "testCorrect",
         created_at: new Date().toISOString(),
@@ -176,9 +206,11 @@ async function step<Model extends ILlmSchema.Model>(
     }),
   );
 
-  const newFiles = { ...files, ...Object.fromEntries(validate) };
-
-  return step(ctx, newFiles, life - 1);
+  const newFiles: Record<string, string> = {
+    ...files,
+    ...Object.fromEntries(validate),
+  };
+  return step(ctx, newFiles, scenarioMap, life - 1);
 }
 
 /**
@@ -195,26 +227,32 @@ async function process<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   diagnostics: IAutoBeTypeScriptCompilerResult.IDiagnostic[],
   code: string,
+  scenario: AutoBeTestScenarioEvent.IScenario,
 ): Promise<ICorrectTestFunctionProps> {
   const pointer: IPointer<ICorrectTestFunctionProps | null> = {
     value: null,
   };
 
-  const apiFiles = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("src/api/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
+  let document: AutoBeOpenApi.IDocument | null = null;
+  if (scenario) {
+    document = filterDocument(scenario, ctx.state().interface!.document);
+  }
 
-  const dtoFiles = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("src/api/structures/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
+  // const apiFiles = Object.entries(ctx.state().interface?.files ?? {})
+  //   .filter(([filename]) => {
+  //     return filename.startsWith("src/api/");
+  //   })
+  //   .reduce<Record<string, string>>((acc, [filename, content]) => {
+  //     return Object.assign(acc, { [filename]: content });
+  //   }, {});
+
+  // const dtoFiles = Object.entries(ctx.state().interface?.files ?? {})
+  //   .filter(([filename]) => {
+  //     return filename.startsWith("src/api/structures/");
+  //   })
+  //   .reduce<Record<string, string>>((acc, [filename, content]) => {
+  //     return Object.assign(acc, { [filename]: content });
+  //   }, {});
 
   const agentica = new MicroAgentica({
     model: ctx.model,
@@ -222,7 +260,7 @@ async function process<Model extends ILlmSchema.Model>(
     config: {
       ...(ctx.config ?? {}),
     },
-    histories: transformTestCorrectHistories(apiFiles, dtoFiles),
+    histories: transformTestCorrectHistories(document),
     controllers: [
       createApplication({
         model: ctx.model,
@@ -231,38 +269,53 @@ async function process<Model extends ILlmSchema.Model>(
         },
       }),
     ],
+    tokenUsage: ctx.usage(),
   });
   enforceToolCall(agentica);
 
-  await agentica.conversate(
-    [
-      "Fix the compilation error in the provided code.",
-      "",
-      "## Original Code",
-      "```typescript",
-      code,
-      "```",
-      "",
-      diagnostics.map((diagnostic) => {
-        if (diagnostic.start === undefined || diagnostic.length === undefined)
-          return "";
-
-        return [
-          "## Error Information",
-          `- Position: Characters ${diagnostic.start} to ${diagnostic.start + diagnostic.length}`,
-          `- Error Message: ${diagnostic.messageText}`,
-          `- Problematic Code: \`${code.substring(diagnostic.start, diagnostic.start + diagnostic.length)}\``,
-          "",
-        ].join("\n");
-      }),
-      "## Instructions",
-      "1. Focus on the specific error location and message",
-      "2. Provide the corrected TypeScript code",
-      "3. Ensure the fix resolves the compilation error",
-      "",
-      "Return only the fixed code without explanations.",
-    ].join("\n"),
-  );
+  await randomBackoffRetry(async () => {
+    await agentica.conversate(
+      [
+        "Fix the compilation error in the provided code.",
+        "",
+        "## Original Code",
+        "```typescript",
+        code,
+        "```",
+        "",
+        diagnostics.map((diagnostic) => {
+          if (diagnostic.start === undefined || diagnostic.length === undefined)
+            return "";
+
+          const checkDtoRegexp = `Cannot find module '@ORGANIZATION/template-api/lib/structures/IBbsArticleComment' or its corresponding type declarations.`;
+          const [group] = [
+            ...checkDtoRegexp.matchAll(
+              /Cannot find module '(.*lib\/structures\/.*)'/g,
+            ),
+          ];
+
+          const [_, filename] = group ?? [];
+
+          return [
+            "## Error Information",
+            `- Position: Characters ${diagnostic.start} to ${diagnostic.start + diagnostic.length}`,
+            `- Error Message: ${diagnostic.messageText}`,
+            `- Problematic Code: \`${code.substring(diagnostic.start, diagnostic.start + diagnostic.length)}\``,
+            filename
+              ? `The type files located under **/lib/structures are declared in '@ORGANIZATION/PROJECT-api/lib/structures'.\n` +
+                `Note: '@ORGANIZATION/PROJECT-api' must be written exactly as is and should not be replaced.\n`
+              : "",
+          ].join("\n");
+        }),
+        "## Instructions",
+        "1. Focus on the specific error location and message",
+        "2. Provide the corrected TypeScript code",
+        "3. Ensure the fix resolves the compilation error",
+        "",
+        "Return only the fixed code without explanations.",
+      ].join("\n"),
+    );
+  });
   if (pointer.value === null) throw new Error("Failed to modify test code.");
   return pointer.value;
 }

package/src/orchestrate/test/orchestrateTestProgress.ts

@@ -1,6 +1,14 @@
 import { IAgenticaController, MicroAgentica } from "@agentica/core";
-import { AutoBeTest, AutoBeTestProgressEvent } from "@autobe/interface";
-import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
+import {
+  AutoBeOpenApi,
+  AutoBeTestScenarioEvent,
+  AutoBeTestWriteEvent,
+} from "@autobe/interface";
+import {
+  ILlmApplication,
+  ILlmSchema,
+  OpenApiTypeChecker,
+} from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
 
@@ -11,22 +19,21 @@ import { transformTestProgressHistories } from "./transformTestProgressHistories
 
 export async function orchestrateTestProgress<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  scenarios: AutoBeTest.Scenario[],
-): Promise<AutoBeTestProgressEvent[]> {
+  scenarios: AutoBeTestScenarioEvent.IScenario[],
+): Promise<AutoBeTestWriteEvent[]> {
   const start: Date = new Date();
   let complete: number = 0;
 
-  const events: AutoBeTestProgressEvent[] = await Promise.all(
+  const events: AutoBeTestWriteEvent[] = await Promise.all(
     /**
-     * Generate test code for each scenario. Maps through scenarios array to
-     * create individual test code implementations. Each scenario is processed
-     * to generate corresponding test code and progress events.
+     * Generate test code for each scenario. Maps through plans array to create
+     * individual test code implementations. Each scenario is processed to
+     * generate corresponding test code and progress events.
      */
     scenarios.map(async (scenario) => {
       const code: ICreateTestCodeProps = await process(ctx, scenario);
-
-      const event: AutoBeTestProgressEvent = {
-        type: "testProgress",
+      const event: AutoBeTestWriteEvent = {
+        type: "testWrite",
         created_at: start.toISOString(),
         filename: `${code.domain}/${scenario.functionName}.ts`,
         content: code.content,
@@ -54,27 +61,20 @@ export async function orchestrateTestProgress<Model extends ILlmSchema.Model>(
  */
 async function process<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  scenario: AutoBeTest.Scenario,
+  scenario: AutoBeTestScenarioEvent.IScenario,
 ): Promise<ICreateTestCodeProps> {
   const pointer: IPointer<ICreateTestCodeProps | null> = {
     value: null,
   };
-
-  const apiFiles = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("src/api/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
-
-  const dtoFiles = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("src/api/structures/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
+  const document: AutoBeOpenApi.IDocument = filterDocument(
+    scenario,
+    ctx.state().interface!.document,
+  );
+  const files: [string, string][] = Object.entries(
+    await ctx.compiler.interface.compile(document),
+  );
+  const filter = (prefix: string) =>
+    Object.fromEntries(files.filter(([key]) => key.startsWith(prefix)));
 
   const agentica = new MicroAgentica({
     model: ctx.model,
@@ -82,7 +82,12 @@ async function process<Model extends ILlmSchema.Model>(
     config: {
       ...(ctx.config ?? {}),
     },
-    histories: transformTestProgressHistories(apiFiles, dtoFiles),
+    histories: transformTestProgressHistories({
+      scenario: scenario,
+      dto: filter("src/api/structures"),
+      sdk: filter("src/api/functional"),
+      e2e: filter("test/features"),
+    }),
     controllers: [
       createApplication({
         model: ctx.model,
@@ -91,22 +96,61 @@ async function process<Model extends ILlmSchema.Model>(
         },
       }),
     ],
+    tokenUsage: ctx.usage(),
   });
   enforceToolCall(agentica);
 
-  await agentica.conversate(
-    [
-      "Create test code for below scenario:",
-      "",
-      "```json",
-      JSON.stringify(scenario, null, 2),
-      "```",
-    ].join("\n"),
-  );
+  await agentica.conversate("Create e2e test functions.");
   if (pointer.value === null) throw new Error("Failed to create test code.");
   return pointer.value;
 }
 
+export function filterDocument(
+  scenario: AutoBeTestScenarioEvent.IScenario,
+  document: AutoBeOpenApi.IDocument,
+): AutoBeOpenApi.IDocument {
+  const operations: AutoBeOpenApi.IOperation[] = document.operations.filter(
+    (op) => {
+      if (
+        scenario.endpoint.method === op.method &&
+        scenario.endpoint.path === op.path
+      ) {
+        return true;
+      } else if (
+        scenario.dependencies.some(
+          (dp) =>
+            dp.endpoint.method === op.method && dp.endpoint.path === op.path,
+        )
+      ) {
+        return true;
+      }
+    },
+  );
+  const components: AutoBeOpenApi.IComponents = {
+    schemas: {},
+  };
+  const visit = (typeName: string) => {
+    OpenApiTypeChecker.visit({
+      components: document.components,
+      schema: { $ref: `#/components/schemas/${typeName}` },
+      closure: (s) => {
+        if (OpenApiTypeChecker.isReference(s)) {
+          const key: string = s.$ref.split("/").pop()!;
+          components.schemas[key] = document.components.schemas[key];
+        }
+      },
+    });
+  };
+  for (const op of operations) {
+    if (op.requestBody) visit(op.requestBody.typeName);
+    if (op.responseBody) visit(op.responseBody.typeName);
+  }
+  return {
+    operations,
+    components,
+  };
+}
+
 function createApplication<Model extends ILlmSchema.Model>(props: {
   model: Model;
   build: (next: ICreateTestCodeProps) => void;
@@ -178,7 +222,7 @@ interface ICreateTestCodeProps {
    * #### Execution Strategy
    *
    * - Outline step-by-step test execution flow
-   * - Plan error handling and exception scenarios
+   * - Plan error handling and exception plans
    * - Define cleanup and teardown procedures
    * - Identify dependencies and prerequisites
    *
@@ -188,12 +232,12 @@ interface ICreateTestCodeProps {
    *     1. Prepare valid article data with required fields
    *     2. Execute POST request to create article
    *     3. Validate response structure and data integrity
-   *     4. Test error scenarios (missing fields, invalid data)
+   *     4. Test error plans (missing fields, invalid data)
    *     5. Verify database state changes
-   *     6. Reconsider the plan if it doesn't follow the Test Generation
+   *     6. Reconsider the scenario if it doesn't follow the Test Generation
    *        Guildelines.
    */
-  plan: string;
+  scenario: string;
 
   /**
    * Functional domain classification for test organization.