@autobe/agent 0.9.0 → 0.9.2

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -1,7 +1,7 @@
 import { IAgenticaController, MicroAgentica } from "@agentica/core";
 import {
-  AutoBeOpenApi,
-  AutoBeTestScenarioEvent,
+  AutoBeTestFile,
+  AutoBeTestScenario,
   AutoBeTestValidateEvent,
   AutoBeTestWriteEvent,
   IAutoBeTypeScriptCompilerResult,
@@ -14,59 +14,57 @@ import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { randomBackoffRetry } from "../../utils/backoffRetry";
 import { enforceToolCall } from "../../utils/enforceToolCall";
-import { filterDocument } from "./orchestrateTestProgress";
+import { compileTestScenario } from "./compileTestScenario";
+import { filterTestFileName } from "./filterTestFileName";
+import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
 import { transformTestCorrectHistories } from "./transformTestCorrectHistories";
 
 export async function orchestrateTestCorrect<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   codes: AutoBeTestWriteEvent[],
-  scenarios: AutoBeTestScenarioEvent.IScenario[],
+  scenarios: AutoBeTestScenario[],
   life: number = 4,
 ): Promise<AutoBeTestValidateEvent> {
-  const scenarioMap: Map<string, AutoBeTestScenarioEvent.IScenario> = new Map();
-  codes.forEach(({ filename }, index) => {
-    scenarioMap.set(filename, scenarios[index]);
-  });
+  const files: AutoBeTestFile[] = codes.map(
+    ({ filename, content }, index): AutoBeTestFile => {
+      const scenario: AutoBeTestScenario = scenarios[index];
+      return { location: filename, content, scenario };
+    },
+  );
 
   // 1) Build map of new test files from progress events
-  const testFiles: Record<string, string> = codes
-    .map(({ filename, content }) => {
-      return {
-        [`test/features/api/${filename}`]: content,
-      };
-    })
-    .reduce<Record<string, string>>((acc, cur) => Object.assign(acc, cur), {});
+  const testFiles: Record<string, string> = Object.fromEntries(
+    codes.map((c) => [c.filename, c.content]),
+  );
 
   // 2) Keep only files outside the test directory from current state
-  const retainedFiles: Record<string, string> = Object.entries(
-    ctx.state().interface?.files ?? {},
-  )
-    .filter(([filename]) => {
-      return !filename.startsWith("test/features/api");
-    })
-    .map(([filename, content]) => {
-      return { [filename]: content };
-    })
-    .reduce<Record<string, string>>((acc, cur) => Object.assign(acc, cur), {});
+  const retainedFiles: Record<string, string> = Object.fromEntries(
+    Object.entries(ctx.state().interface?.files ?? {}).filter(([key]) =>
+      filterTestFileName(key),
+    ),
+  );
 
   // 3) Merge and filter: keep .ts/.json, drop anything under "benchmark"
+  const external = async (
+    location: string,
+  ): Promise<Record<string, string>> => {
+    const content: string | undefined =
+      await ctx.compiler.typescript.getExternal(location);
+    if (content === undefined) throw new Error(`File not found: ${location}`);
+    return { [location]: content };
+  };
   const mergedFiles: Record<string, string> = {
     ...retainedFiles,
     ...testFiles,
+    ...(await external("node_modules/@nestia/e2e/lib/TestValidator.d.ts")),
+    ...(await external("node_modules/@nestia/fetcher/lib/IConnection.d.ts")),
   };
-  const files: Record<string, string> = Object.fromEntries(
-    Object.entries(mergedFiles).filter(
-      ([filename]) =>
-        (filename.endsWith(".ts") && !filename.startsWith("test/benchmark/")) ||
-        filename.endsWith(".json"),
-    ),
-  );
 
   // 4) Ask the LLM to correct the filtered file set
   const response: AutoBeTestValidateEvent = await step(
     ctx,
+    mergedFiles,
     files,
-    scenarioMap,
     life,
   );
 
@@ -74,7 +72,17 @@ export async function orchestrateTestCorrect<Model extends ILlmSchema.Model>(
   const event: AutoBeTestValidateEvent = {
     ...response,
     type: "testValidate",
-    files: { ...mergedFiles, ...response.files },
+    files: [
+      ...Object.entries(mergedFiles).map(
+        ([filename, content]): AutoBeTestFile => {
+          return {
+            location: filename,
+            content,
+          };
+        },
+      ),
+      ...response.files,
+    ],
   };
   return event;
 }
@@ -89,29 +97,35 @@ export async function orchestrateTestCorrect<Model extends ILlmSchema.Model>(
  * all generated test files are syntactically correct and compilable.
  *
  * @param ctx AutoBe context object
- * @param files Map of files to compile (filename: content)
+ * @param entireFiles Map of all files to compile (filename: content)
+ * @param testFiles Map of files to compile (filename: content)
  * @param life Number of remaining retry attempts
  * @returns Event object containing successful compilation result and modified
  *   files
  */
 async function step<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  files: Record<string, string>,
-  scenarioMap: Map<string, AutoBeTestScenarioEvent.IScenario>,
+  entireFiles: Record<string, string>,
+  testFiles: AutoBeTestFile[],
   life: number,
 ): Promise<AutoBeTestValidateEvent> {
   // COMPILE TEST CODE
-
   const result: IAutoBeTypeScriptCompilerResult =
     await ctx.compiler.typescript.compile({
-      files,
+      files: {
+        ...entireFiles,
+        ...testFiles
+          .map((el) => ({ [el.location]: el.content }))
+          .reduce((acc, cur) => Object.assign(acc, cur), {}),
+      },
     });
+
   if (result.type === "success") {
     // SUCCESS
     return {
       type: "testValidate",
       created_at: new Date().toISOString(),
-      files,
+      files: testFiles,
       result,
       step: ctx.state().interface?.step ?? 0,
     };
@@ -122,7 +136,7 @@ async function step<Model extends ILlmSchema.Model>(
     ctx.dispatch({
       type: "testValidate",
       created_at: new Date().toISOString(),
-      files,
+      files: testFiles,
       result,
       step: ctx.state().interface?.step ?? 0,
     });
@@ -151,7 +165,7 @@ async function step<Model extends ILlmSchema.Model>(
     return {
       type: "testValidate",
       created_at: new Date().toISOString(),
-      files,
+      files: testFiles,
       result: {
         ...result,
         type: "success",
@@ -164,7 +178,7 @@ async function step<Model extends ILlmSchema.Model>(
   ctx.dispatch({
     type: "testValidate",
     created_at: new Date().toISOString(),
-    files,
+    files: testFiles,
     result,
     step: ctx.state().interface?.step ?? 0,
   });
@@ -173,44 +187,66 @@ async function step<Model extends ILlmSchema.Model>(
     return {
       type: "testValidate",
       created_at: new Date().toISOString(),
-      files,
+      files: testFiles,
       result,
       step: ctx.state().interface?.step ?? 0,
     };
 
   // VALIDATION FAILED
-  const validate: [string, string][] = await Promise.all(
-    Object.entries(diagnostics).map(async ([filename, d]) => {
-      const scenario: AutoBeTestScenarioEvent.IScenario =
-        scenarioMap.get(filename)!;
-      const code: string = files[filename];
-      const response: ICorrectTestFunctionProps = await process(
-        ctx,
-        d,
-        code,
-        scenario,
-      );
-      ctx.dispatch({
-        type: "testCorrect",
-        created_at: new Date().toISOString(),
-        files: { ...files, [filename]: response.content },
-        result,
-        solution: response.solution,
-        think_without_compile_error: response.think_without_compile_error,
-        think_again_with_compile_error: response.think_again_with_compile_error,
-        step: ctx.state().interface?.step ?? 0,
-      });
-
-      // Return [filename, modified code]
-      return [filename, response.content];
-    }),
+  const validatedFiles: AutoBeTestFile[] = await Promise.all(
+    Object.entries(diagnostics).map(
+      async ([filename, d]): Promise<AutoBeTestFile> => {
+        const file = testFiles.find((f) => f.location === filename);
+        const code: string = file?.content!;
+        const scenario = file?.scenario!;
+
+        const response: ICorrectTestFunctionProps = await process(
+          ctx,
+          d,
+          code,
+          scenario,
+        );
+        ctx.dispatch({
+          type: "testCorrect",
+          created_at: new Date().toISOString(),
+          files: { ...testFiles, [filename]: response.content },
+          result,
+          solution: response.solution,
+          think_without_compile_error: response.think_without_compile_error,
+          think_again_with_compile_error:
+            response.think_again_with_compile_error,
+          step: ctx.state().interface?.step ?? 0,
+        });
+
+        return {
+          location: filename,
+          content: response.content,
+          scenario: scenario,
+        };
+      },
+    ),
   );
 
-  const newFiles: Record<string, string> = {
-    ...files,
-    ...Object.fromEntries(validate),
-  };
-  return step(ctx, newFiles, scenarioMap, life - 1);
+  return step(
+    ctx,
+    Object.entries(entireFiles)
+      .map(([filename, content]) => {
+        const overwritten = validatedFiles.find(
+          (el) => el.location === filename,
+        );
+        return overwritten
+          ? { [overwritten.location]: overwritten.content }
+          : {
+              [filename]: content,
+            };
+      })
+      .reduce((acc, cur) => Object.assign(acc, cur), {}),
+    testFiles.map((f) => {
+      const validated = validatedFiles.find((v) => v.location === f.location);
+      return validated ? validated : f;
+    }),
+    life - 1,
+  );
 }
 
 /**
@@ -227,32 +263,28 @@ async function process<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   diagnostics: IAutoBeTypeScriptCompilerResult.IDiagnostic[],
   code: string,
-  scenario: AutoBeTestScenarioEvent.IScenario,
+  scenario: AutoBeTestScenario,
 ): Promise<ICorrectTestFunctionProps> {
   const pointer: IPointer<ICorrectTestFunctionProps | null> = {
     value: null,
   };
+  const artifacts: IAutoBeTestScenarioArtifacts = await compileTestScenario(
+    ctx,
+    scenario,
+  );
 
-  let document: AutoBeOpenApi.IDocument | null = null;
-  if (scenario) {
-    document = filterDocument(scenario, ctx.state().interface!.document);
-  }
-
-  // const apiFiles = Object.entries(ctx.state().interface?.files ?? {})
-  //   .filter(([filename]) => {
-  //     return filename.startsWith("src/api/");
-  //   })
-  //   .reduce<Record<string, string>>((acc, [filename, content]) => {
-  //     return Object.assign(acc, { [filename]: content });
-  //   }, {});
-
-  // const dtoFiles = Object.entries(ctx.state().interface?.files ?? {})
-  //   .filter(([filename]) => {
-  //     return filename.startsWith("src/api/structures/");
-  //   })
-  //   .reduce<Record<string, string>>((acc, [filename, content]) => {
-  //     return Object.assign(acc, { [filename]: content });
-  //   }, {});
+  const lines = code.split("\n").map((line, num, arr) => {
+    const start = arr
+      .slice(0, num)
+      .map((el) => el.length + 1)
+      .reduce((acc, cur) => acc + cur, 0);
+    return {
+      line: num + 1,
+      text: line,
+      start: start,
+      end: start + line.length + 1, // exclusive
+    };
+  });
 
   const agentica = new MicroAgentica({
     model: ctx.model,
@@ -260,7 +292,15 @@ async function process<Model extends ILlmSchema.Model>(
     config: {
       ...(ctx.config ?? {}),
     },
-    histories: transformTestCorrectHistories(document),
+    histories: transformTestCorrectHistories(
+      code,
+      artifacts,
+      diagnostics.map((diagnostic) =>
+        diagnostic.start === undefined || diagnostic.length === undefined
+          ? ""
+          : formatDiagnostic(code, lines, diagnostic),
+      ),
+    ),
     controllers: [
       createApplication({
         model: ctx.model,
@@ -276,38 +316,7 @@ async function process<Model extends ILlmSchema.Model>(
   await randomBackoffRetry(async () => {
     await agentica.conversate(
       [
-        "Fix the compilation error in the provided code.",
-        "",
-        "## Original Code",
-        "```typescript",
-        code,
-        "```",
-        "",
-        diagnostics.map((diagnostic) => {
-          if (diagnostic.start === undefined || diagnostic.length === undefined)
-            return "";
-
-          const checkDtoRegexp = `Cannot find module '@ORGANIZATION/template-api/lib/structures/IBbsArticleComment' or its corresponding type declarations.`;
-          const [group] = [
-            ...checkDtoRegexp.matchAll(
-              /Cannot find module '(.*lib\/structures\/.*)'/g,
-            ),
-          ];
-
-          const [_, filename] = group ?? [];
-
-          return [
-            "## Error Information",
-            `- Position: Characters ${diagnostic.start} to ${diagnostic.start + diagnostic.length}`,
-            `- Error Message: ${diagnostic.messageText}`,
-            `- Problematic Code: \`${code.substring(diagnostic.start, diagnostic.start + diagnostic.length)}\``,
-            filename
-              ? `The type files located under **/lib/structures are declared in '@ORGANIZATION/PROJECT-api/lib/structures'.\n` +
-                `Note: '@ORGANIZATION/PROJECT-api' must be written exactly as is and should not be replaced.\n`
-              : "",
-          ].join("\n");
-        }),
-        "## Instructions",
+        "# Instructions",
         "1. Focus on the specific error location and message",
         "2. Provide the corrected TypeScript code",
         "3. Ensure the fix resolves the compilation error",
@@ -317,9 +326,160 @@ async function process<Model extends ILlmSchema.Model>(
     );
   });
   if (pointer.value === null) throw new Error("Failed to modify test code.");
+
+  const typeReferences: string[] = Array.from(
+    new Set(
+      Object.keys(artifacts.document.components.schemas).map(
+        (key) => key.split(".")[0]!,
+      ),
+    ),
+  );
+
+  pointer.value.content = pointer.value.content
+    .replace(/^[ \t]*import\b[\s\S]*?;[ \t]*$/gm, "")
+    .trim();
+  pointer.value.content = [
+    `import { TestValidator } from "@nestia/e2e";`,
+    `import typia, { tags } from "typia";`,
+    "",
+    `import api from "@ORGANIZATION/PROJECT-api";`,
+    ...typeReferences.map(
+      (ref) =>
+        `import type { ${ref} } from "@ORGANIZATION/PROJECT-api/lib/structures/${ref}";`,
+    ),
+    "",
+    pointer.value.content,
+  ].join("\n");
+
+  pointer.value.content = pointer.value.content.replaceAll(
+    'string & Format<"uuid">',
+    'string & tags.Format<"uuid">',
+  );
+
   return pointer.value;
 }
 
+function formatDiagnostic(
+  code: string,
+  lines: {
+    line: number; // line number
+    text: string;
+    start: number;
+    end: number; // exclusive
+  }[],
+  diagnostic: IAutoBeTypeScriptCompilerResult.IDiagnostic,
+): string {
+  const { start, length, messageText } = diagnostic;
+  const message = messageText;
+  if (typeof start === "number" && typeof length === "number") {
+    const end = start + length;
+    const problematicCode = code.substring(start, end);
+    const errorLine = lines.find((line) => line.end > start) ?? null;
+    const lineText = errorLine?.text ?? "";
+
+    const hints = getHints(message, lineText);
+
+    function createAdjustedArray(n: number): number[] {
+      let start = n - 2;
+
+      // 시작 값이 음수라면, 0부터 시작해서 5개의 숫자 생성
+      if (start < 0) {
+        start = 0;
+      }
+
+      return Array.from({ length: 5 }, (_, i) => start + i);
+    }
+
+    const errorLines = createAdjustedArray(errorLine?.line ?? 0);
+
+    const context = errorLines
+      .map((num) => {
+        if (num === errorLine?.line) {
+          if (lines[num - 1]) {
+            return `${lines[num - 1]?.text} // <- ERROR LINE (line:${num})`;
+          }
+        }
+        if (lines[num - 1]) {
+          return lines[num - 1]?.text;
+        }
+
+        return null;
+      })
+      .filter((el) => el !== null);
+
+    return [
+      "## Error Information",
+      `- Position: Characters ${start} to ${end}`,
+      `- Error Message: ${message}`,
+      `- Error Lines: \n${
+        context.length
+          ? [
+              "\t```ts", //
+              ...context.map((el) => `\t${el}`),
+              "\t```",
+            ].join("\n")
+          : "(none)"
+      }`,
+      `- Problematic Code: ${problematicCode.length > 0 ? `\`${problematicCode}\`` : "(none)"}`,
+      ...hints.map((hint) => `- Hint: ${hint}`),
+    ].join("\n");
+  }
+  return ["## Error Information", `- Error Message: ${message}`].join("\n");
+}
+
+function getHints(message: string, lineText: string): string[] {
+  const isTestValidator = lineText.includes("TestValidator");
+  const isTypia =
+    message === "Cannot find name 'Format'. Did you mean 'FormData'?";
+  const isJest = message === "Cannot find name 'expect'.";
+  const isCurrying =
+    isTestValidator && message === "Expected 1 arguments, but got 2";
+  const isAssignability =
+    /Argument of type '([^']+)' is not assignable to parameter of type '([^']+)'/.test(
+      message,
+    );
+
+  const hints: string[] = [];
+
+  if (isTypia) {
+    hints.push(
+      "If you want to use typia tags, use `tags.Format` instead of `Format`.",
+    );
+  }
+
+  if (isJest) {
+    hints.push(
+      'Detected invalid `expect` usage. Use `TestValidator.equals("description")(expected)(actual)`.',
+    );
+  }
+
+  if (isCurrying) {
+    hints.push(
+      "`TestValidator.equals` is a curried function and must be called in **three steps**: `title → expected → actual`.",
+    );
+  } else if (isTestValidator) {
+    hints.push(
+      "The second argument `expected` must be assignable from the type of `actual`. Consider swapping the order if you get a type error.",
+    );
+  }
+
+  if (isAssignability && isTestValidator) {
+    const match = lineText
+      .trim()
+      .match(/TestValidator\.equals\("([^"]+)"\)\(([^)]+)\)\(([^)]+)\)/);
+    if (match) {
+      const [, title, expected, actual] = match;
+      if (actual.includes(expected)) {
+        hints.push(
+          `You can try rearranging the order like this: TestValidator.equals("${title}")(${actual})(${expected})`,
+        );
+      }
+    }
+  }
+
+  return hints;
+}
+
 function createApplication<Model extends ILlmSchema.Model>(props: {
   model: Model;
   build: (next: ICorrectTestFunctionProps) => void;

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -3,8 +3,11 @@ import {
   IAgenticaHistoryJson,
   MicroAgentica,
 } from "@agentica/core";
-import { AutoBeOpenApi } from "@autobe/interface";
-import { AutoBeTestScenarioEvent } from "@autobe/interface";
+import {
+  AutoBeOpenApi,
+  AutoBeTestScenario,
+  AutoBeTestScenarioEvent,
+} from "@autobe/interface";
 import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
@@ -31,7 +34,7 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   let include: AutoBeOpenApi.IOperation[] = Array.from(operations);
 
   do {
-    const matrix = divideArray({ array: include, capacity: 30 });
+    const matrix = divideArray({ array: include, capacity: 5 });
 
     await Promise.all(
       matrix.map(async (_include) => {
@@ -68,8 +71,8 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
           endpoint: pg.endpoint,
           draft: plan.draft,
           functionName: plan.functionName,
-          dependencies: plan.dependsOn,
-        } satisfies AutoBeTestScenarioEvent.IScenario;
+          dependencies: plan.dependencies,
+        } satisfies AutoBeTestScenario;
       });
     }),
     created_at: new Date().toISOString(),
@@ -120,7 +123,9 @@ const createHistoryProperties = (
   operations: AutoBeOpenApi.IOperation[],
   include: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
   exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
-) => [
+): Array<
+  IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
+> => [
   {
     id: v4(),
     created_at: new Date().toISOString(),
@@ -132,17 +137,29 @@ const createHistoryProperties = (
     created_at: new Date().toISOString(),
     type: "systemMessage",
     text: [
+      "# Operations",
       "Below are the full operations. Please refer to this.",
       "Your role is to draft all test cases for each given Operation.",
       "It is also permissible to write multiple test codes on a single endpoint.",
       "However, rather than meaningless tests, business logic tests should be written and an E2E test situation should be assumed.",
       "",
+      "Please carefully analyze each operation to identify all dependencies required for testing.",
+      "For example, if you want to test liking and then deleting a post,",
+      "you might think to test post creation, liking, and unlike operations.",
+      "However, even if not explicitly mentioned, user registration and login are essential prerequisites.",
+      "Pay close attention to IDs and related values in the API,",
+      "and ensure you identify all dependencies between endpoints.",
+      "",
       "```json",
       JSON.stringify(
         operations.map((el) => ({
           path: el.path,
           method: el.method,
           summary: el.summary,
+          description: el.description,
+          parameters: el.parameters,
+          requestBody: el.requestBody,
+          responseBody: el.responseBody,
         })),
       ),
       "```",