@autobe/agent 0.9.2 → 0.10.0

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -8,8 +8,9 @@ import {
   AutoBeTestScenario,
   AutoBeTestScenarioEvent,
 } from "@autobe/interface";
+import { AutoBeEndpointComparator } from "@autobe/utils";
 import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
-import { IPointer } from "tstl";
+import { HashMap, IPointer, Pair } from "tstl";
 import typia from "typia";
 import { v4 } from "uuid";
 
@@ -18,37 +19,67 @@ import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { divideArray } from "../../utils/divideArray";
 import { enforceToolCall } from "../../utils/enforceToolCall";
+import { forceRetry } from "../../utils/forceRetry";
 import { IAutoBeTestScenarioApplication } from "./structures/IAutoBeTestScenarioApplication";
 
 export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
 ): Promise<AutoBeTestScenarioEvent> {
-  const operations = ctx.state().interface?.document.operations ?? [];
+  const operations: AutoBeOpenApi.IOperation[] =
+    ctx.state().interface?.document.operations ?? [];
   if (operations.length === 0) {
     throw new Error(
       "Cannot write test scenarios because these are no operations.",
     );
   }
 
+  const dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation> =
+    new HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>(
+      operations.map(
+        (op) =>
+          new Pair(
+            {
+              path: op.path,
+              method: op.method,
+            },
+            op,
+          ),
+      ),
+      AutoBeEndpointComparator.hashCode,
+      AutoBeEndpointComparator.equals,
+    );
+  const endpointNotFound: string = [
+    `You have to select one of the endpoints below`,
+    "",
+    " method | path ",
+    "--------|------",
+    ...operations.map((op) => `\`${op.method}\` | \`${op.path}\``).join("\n"),
+  ].join("\n");
+
   const exclude: IAutoBeTestScenarioApplication.IScenarioGroup[] = [];
   let include: AutoBeOpenApi.IOperation[] = Array.from(operations);
 
   do {
-    const matrix = divideArray({ array: include, capacity: 5 });
-
+    const matrix: AutoBeOpenApi.IOperation[][] = divideArray({
+      array: include,
+      capacity: 5,
+    });
     await Promise.all(
-      matrix.map(async (_include) => {
+      matrix.map(async (include) => {
         exclude.push(
-          ...(await execute(
-            ctx,
-            operations,
-            _include,
-            exclude.map((x) => x.endpoint),
+          ...(await forceRetry(() =>
+            execute(
+              ctx,
+              dict,
+              endpointNotFound,
+              operations,
+              include,
+              exclude.map((x) => x.endpoint),
+            ),
           )),
         );
       }),
     );
-
     include = include.filter((op) => {
       if (
         exclude.some(
@@ -81,9 +112,11 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
 
 const execute = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  ops: AutoBeOpenApi.IOperation[],
-  include: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
-  exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+  dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>,
+  endpointNotFound: string,
+  entire: AutoBeOpenApi.IOperation[],
+  include: AutoBeOpenApi.IEndpoint[],
+  exclude: AutoBeOpenApi.IEndpoint[],
 ) => {
   const pointer: IPointer<IAutoBeTestScenarioApplication.IScenarioGroup[]> = {
     value: [],
@@ -97,11 +130,12 @@ const execute = async <Model extends ILlmSchema.Model>(
         describe: null,
       },
     },
-    tokenUsage: ctx.usage(),
-    histories: createHistoryProperties(ops, include, exclude),
+    histories: createHistoryProperties(entire, include, exclude),
     controllers: [
       createApplication({
         model: ctx.model,
+        endpointNotFound,
+        dict,
         build: (next) => {
           pointer.value ??= [];
           pointer.value.push(...next.scenarioGroups);
@@ -111,7 +145,10 @@ const execute = async <Model extends ILlmSchema.Model>(
   });
   enforceToolCall(agentica);
 
-  await agentica.conversate(`create test scenarios.`);
+  await agentica.conversate(`create test scenarios.`).finally(() => {
+    const tokenUsage = agentica.getTokenUsage();
+    ctx.usage().record(tokenUsage, ["test"]);
+  });
   if (pointer.value.length === 0) {
     throw new Error("Failed to create test plans.");
   }
@@ -120,7 +157,7 @@ const execute = async <Model extends ILlmSchema.Model>(
 };
 
 const createHistoryProperties = (
-  operations: AutoBeOpenApi.IOperation[],
+  entire: AutoBeOpenApi.IOperation[],
   include: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
   exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
 ): Array<
@@ -152,14 +189,9 @@ const createHistoryProperties = (
       "",
       "```json",
       JSON.stringify(
-        operations.map((el) => ({
-          path: el.path,
-          method: el.method,
-          summary: el.summary,
-          description: el.description,
-          parameters: el.parameters,
-          requestBody: el.requestBody,
-          responseBody: el.responseBody,
+        entire.map((el) => ({
+          ...el,
+          specification: undefined,
         })),
       ),
       "```",
@@ -188,6 +220,8 @@ const createHistoryProperties = (
 
 function createApplication<Model extends ILlmSchema.Model>(props: {
   model: Model;
+  endpointNotFound: string;
+  dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>;
   build: (next: IAutoBeTestScenarioApplication.IProps) => void;
 }): IAgenticaController.IClass<Model> {
   assertSchemaModel(props.model);
@@ -201,55 +235,53 @@ function createApplication<Model extends ILlmSchema.Model>(props: {
       typia.validate<IAutoBeTestScenarioApplication.IProps>(next);
     if (result.success === false) return result;
 
-    const errors: IValidation.IError[] = [];
-    result.data.scenarioGroups.forEach((pg, i, arr) => {
-      arr.forEach((target, j) => {
-        if (
-          i !== j &&
-          target.endpoint.method === pg.endpoint.method &&
-          target.endpoint.path === pg.endpoint.path
-        ) {
-          if (
-            !errors.some(
-              (el) =>
-                el.path !== `planGroups[${j}].path` &&
-                el.value !== target.endpoint.path,
-            )
-          ) {
-            errors.push({
-              path: `planGroups[${j}].path`,
-              expected: `planGroup's {method + path} cannot duplicated.`,
-              value: target.endpoint.path,
-            });
-          }
+    // merge to unique scenario groups
+    const scenarioGroups: IAutoBeTestScenarioApplication.IScenarioGroup[] = [];
+    result.data.scenarioGroups.forEach((sg) => {
+      const created = scenarioGroups.find(
+        (el) =>
+          el.endpoint.method === sg.endpoint.method &&
+          el.endpoint.path === sg.endpoint.path,
+      );
+      if (created) {
+        created.scenarios.push(...sg.scenarios);
+      } else {
+        scenarioGroups.push(sg);
+      }
+    });
 
-          if (
-            !errors.some(
-              (el) =>
-                el.path !== `planGroups[${j}].method` &&
-                el.value !== target.endpoint.method,
-            )
-          ) {
+    // validate endpoints
+    const errors: IValidation.IError[] = [];
+    scenarioGroups.forEach((group, i) => {
+      if (props.dict.has(group.endpoint) === false)
+        errors.push({
+          value: group.endpoint,
+          path: `$input.scenarioGroups[${i}].endpoint`,
+          expected: "AutoBeOpenApi.IEndpoint",
+          description: props.endpointNotFound,
+        });
+      group.scenarios.forEach((s, j) => {
+        s.dependencies.forEach((dep, k) => {
+          if (props.dict.has(dep.endpoint) === false)
             errors.push({
-              path: `planGroups[${j}].method`,
-              expected: `planGroup's {method + path} cannot duplicated.`,
-              value: target.endpoint.method,
+              value: dep.endpoint,
+              path: `$input.scenarioGroups[${i}].scenarios[${j}].dependencies[${k}].endpoint`,
+              expected: "AutoBeOpenApi.IEndpoint",
+              description: props.endpointNotFound,
             });
-          }
-        }
+        });
       });
     });
-
-    if (errors.length !== 0) {
-      console.log(JSON.stringify(errors, null, 2), "errors");
-      return {
-        success: false,
-        errors,
-        data: next,
-      };
-    }
-
-    return result;
+    return errors.length === 0
+      ? {
+          success: true,
+          data: scenarioGroups,
+        }
+      : {
+          success: false,
+          data: scenarioGroups,
+          errors,
+        };
   };
   return {
     protocol: "class",
@@ -280,5 +312,4 @@ const collection = {
   llama: claude,
   deepseek: claude,
   "3.1": claude,
-  "3.0": typia.llm.application<IAutoBeTestScenarioApplication, "3.0">(),
 };

package/src/orchestrate/test/orchestrateTestWrite.ts

@@ -7,40 +7,54 @@ import typia from "typia";
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { enforceToolCall } from "../../utils/enforceToolCall";
-import { compileTestScenario } from "./compileTestScenario";
+import { forceRetry } from "../../utils/forceRetry";
+import { completeTestCode } from "./compile/completeTestCode";
+import { getTestScenarioArtifacts } from "./compile/getTestScenarioArtifacts";
 import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
+import { IAutoBeTestWriteApplication } from "./structures/IAutoBeTestWriteApplication";
+import { IAutoBeTestWriteResult } from "./structures/IAutoBeTestWriteResult";
 import { transformTestWriteHistories } from "./transformTestWriteHistories";
 
 export async function orchestrateTestWrite<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   scenarios: AutoBeTestScenario[],
-): Promise<AutoBeTestWriteEvent[]> {
+): Promise<IAutoBeTestWriteResult[]> {
   const start: Date = new Date();
   let complete: number = 0;
 
-  const events: AutoBeTestWriteEvent[] = await Promise.all(
+  return Promise.all(
     /**
      * Generate test code for each scenario. Maps through plans array to create
      * individual test code implementations. Each scenario is processed to
      * generate corresponding test code and progress events.
      */
-    scenarios.map(async (scenario) => {
-      const code: ICreateTestCodeProps = await process(ctx, scenario);
-      const event: AutoBeTestWriteEvent = {
-        type: "testWrite",
-        created_at: start.toISOString(),
-        filename: `test/features/api/${code.domain}/${scenario.functionName}.ts`,
-        content: code.content,
-        completed: ++complete,
-        total: scenarios.length,
-        step: ctx.state().interface?.step ?? 0,
-      };
-      ctx.dispatch(event);
-      return event;
-    }),
+    scenarios.map((scenario) =>
+      forceRetry(async () => {
+        const artifacts: IAutoBeTestScenarioArtifacts =
+          await getTestScenarioArtifacts(ctx, scenario);
+        const result: IAutoBeTestWriteApplication.IProps = await process(
+          ctx,
+          scenario,
+          artifacts,
+        );
+        const event: AutoBeTestWriteEvent = {
+          type: "testWrite",
+          created_at: start.toISOString(),
+          location: `test/features/api/${result.domain}/${scenario.functionName}.ts`,
+          ...result,
+          completed: ++complete,
+          total: scenarios.length,
+          step: ctx.state().interface?.step ?? 0,
+        };
+        ctx.dispatch(event);
+        return {
+          scenario,
+          artifacts,
+          event,
+        };
+      }),
+    ),
   );
-
-  return events;
 }
 
 /**
@@ -50,80 +64,49 @@ export async function orchestrateTestWrite<Model extends ILlmSchema.Model>(
  *
  * @param ctx - The AutoBeContext containing model, vendor and configuration
  * @param scenario - The test scenario information to generate code for
- * @returns Promise resolving to ICreateTestCodeProps containing the generated
- *   test code
+ * @param artifacts - The artifacts containing the reference files and schemas
+ * @returns Promise resolving to IAutoBeTestWriteApplication.IProps containing
+ *   the generated test code
  */
 async function process<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   scenario: AutoBeTestScenario,
-): Promise<ICreateTestCodeProps> {
-  const pointer: IPointer<ICreateTestCodeProps | null> = {
+  artifacts: IAutoBeTestScenarioArtifacts,
+): Promise<IAutoBeTestWriteApplication.IProps> {
+  const pointer: IPointer<IAutoBeTestWriteApplication.IProps | null> = {
     value: null,
   };
-  const artifacts: IAutoBeTestScenarioArtifacts = await compileTestScenario(
-    ctx,
-    scenario,
-  );
-
-  const agentica = new MicroAgentica({
+  const agentica: MicroAgentica<Model> = new MicroAgentica({
     model: ctx.model,
     vendor: ctx.vendor,
     config: {
       ...(ctx.config ?? {}),
     },
-    histories: transformTestWriteHistories({
-      scenario,
-      artifacts,
-    }),
+    histories: transformTestWriteHistories(scenario, artifacts),
     controllers: [
       createApplication({
         model: ctx.model,
+        artifacts,
         build: (next) => {
           pointer.value = next;
         },
       }),
     ],
-    tokenUsage: ctx.usage(),
   });
   enforceToolCall(agentica);
 
-  await agentica.conversate("Create e2e test functions.");
+  await agentica.conversate("Create e2e test functions.").finally(() => {
+    const tokenUsage = agentica.getTokenUsage();
+    ctx.usage().record(tokenUsage, ["test"]);
+  });
   if (pointer.value === null) throw new Error("Failed to create test code.");
-
-  const typeReferences: string[] = Array.from(
-    new Set(
-      Object.keys(artifacts.document.components.schemas).map(
-        (key) => key.split(".")[0]!,
-      ),
-    ),
-  );
-  pointer.value.content = pointer.value.content
-    .replace(/^[ \t]*import\b[\s\S]*?;[ \t]*$/gm, "")
-    .trim();
-  pointer.value.content = [
-    `import { TestValidator } from "@nestia/e2e";`,
-    `import typia, { tags } from "typia";`,
-    "",
-    `import api from "@ORGANIZATION/PROJECT-api";`,
-    ...typeReferences.map(
-      (ref) =>
-        `import type { ${ref} } from "@ORGANIZATION/PROJECT-api/lib/structures/${ref}";`,
-    ),
-    "",
-    pointer.value.content,
-  ].join("\n");
-
-  pointer.value.content = pointer.value.content.replaceAll(
-    'string & Format<"uuid">',
-    'string & tags.Format<"uuid">',
-  );
-
   return pointer.value;
 }
 
 function createApplication<Model extends ILlmSchema.Model>(props: {
   model: Model;
-  build: (next: ICreateTestCodeProps) => void;
+  artifacts: IAutoBeTestScenarioArtifacts;
+  build: (next: IAutoBeTestWriteApplication.IProps) => void;
 }): IAgenticaController.IClass<Model> {
   assertSchemaModel(props.model);
 
@@ -135,15 +118,17 @@ function createApplication<Model extends ILlmSchema.Model>(props: {
     name: "Create Test Code",
     application,
     execute: {
-      createTestCode: (next) => {
+      write: (next) => {
+        next.draft = completeTestCode(props.artifacts, next.draft);
+        next.final = completeTestCode(props.artifacts, next.final);
         props.build(next);
       },
-    } satisfies IApplication,
+    } satisfies IAutoBeTestWriteApplication,
   };
 }
 
 const claude = typia.llm.application<
-  IApplication,
+  IAutoBeTestWriteApplication,
   "claude",
   {
     reference: true;
@@ -151,7 +136,7 @@ const claude = typia.llm.application<
 >();
 const collection = {
   chatgpt: typia.llm.application<
-    IApplication,
+    IAutoBeTestWriteApplication,
     "chatgpt",
     { reference: true }
   >(),
@@ -159,116 +144,5 @@ const collection = {
   llama: claude,
   deepseek: claude,
   "3.1": claude,
-  "3.0": typia.llm.application<IApplication, "3.0">(),
+  "3.0": typia.llm.application<IAutoBeTestWriteApplication, "3.0">(),
 };
-
-interface IApplication {
-  createTestCode(props: ICreateTestCodeProps): void;
-}
-
-interface ICreateTestCodeProps {
-  /**
-   * Strategic approach for test implementation.
-   *
-   * Define the high-level strategy and logical flow for testing the given
-   * scenario. Focus on test methodology, data preparation, and assertion
-   * strategy.
-   *
-   * ### Critical Requirements
-   *
-   * - Must follow the Test Generation Guidelines.
-   * - Must Planning the test code Never occur the TypeScript compile error.
-   * - NEVER include import statements in planning or implementation.
-   *
-   * ### Planning Elements:
-   *
-   * #### Test Methodology
-   *
-   * - Identify test scenario type (CRUD operation, authentication flow,
-   *   validation test)
-   * - Define test data requirements and preparation strategy
-   * - Plan positive/negative test cases and edge cases
-   * - Design assertion logic and validation points
-   *
-   * #### Execution Strategy
-   *
-   * - Outline step-by-step test execution flow
-   * - Plan error handling and exception plans
-   * - Define cleanup and teardown procedures
-   * - Identify dependencies and prerequisites
-   *
-   * ### Example Plan:
-   *
-   *     Test Strategy: Article Creation Validation
-   *     1. Prepare valid article data with required fields
-   *     2. Execute POST request to create article
-   *     3. Validate response structure and data integrity
-   *     4. Test error plans (missing fields, invalid data)
-   *     5. Verify database state changes
-   *     6. Reconsider the scenario if it doesn't follow the Test Generation
-   *        Guidelines.
-   */
-  scenario: string;
-
-  /**
-   * Functional domain classification for test organization.
-   *
-   * Determines file structure and test categorization based on API
-   * functionality. Used for organizing tests into logical groups and directory
-   * hierarchies.
-   *
-   * ### Naming Rules:
-   *
-   * - Lowercase English words only
-   * - Singular nouns (e.g., "article", "user", "comment")
-   * - Kebab-case for compound words (e.g., "user-profile", "payment-method")
-   * - Match primary API resource being tested
-   * - Domain Name must be named only one word.
-   *
-   * ### Domain Examples:
-   *
-   * - `article` → Article management operations
-   * - `comment` → Comment-related functionality
-   * - `auth` → Authentication and authorization
-   * - `user` → User management operations
-   * - `payment` → Payment processing
-   * - `notification` → Notification system
-   */
-  domain: string;
-
-  /**
-   * Complete TypeScript E2E test implementation.
-   *
-   * Generate fully functional, compilation-error-free test code following
-   *
-   * @nestia/e2e framework conventions and TypeScript best practices.
-   *
-   * ### Technical Implementation Requirements:
-   *
-   * #### NO IMPORT DECLARATIONS
-   * - NEVER write any import statements
-   * - Start code directly with `export async function`
-   * - All dependencies assumed globally available:
-   *   - `api` for SDK functions
-   *   - `typia` for validation and random data
-   *   - All DTO types (ITargetType, etc.)
-   *   - `TestValidator` for assertions
-   *
-   * #### Code Quality Standards
-   * - Zero TypeScript compilation errors (mandatory)
-   * - Explicit type annotations for all variables
-   * - Proper async/await patterns throughout
-   * - Comprehensive error handling
-   * - Clean, readable code structure
-   * - Consistent formatting and naming conventions
-   *
-   * ### Critical Error Prevention
-   * - Verify all API function signatures and parameter types
-   * - Ensure type compatibility between variables and assignments
-   * - Include all required object properties and methods
-   * - Confirm proper generic type usage
-   * - Test async function declarations and Promise handling
-   * - NO IMPORT STATEMENTS ANYWHERE IN CODE
-   */
-  content: string;
-}