@autobe/agent 0.9.1 → 0.10.0

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -8,8 +8,9 @@ import {
   AutoBeTestScenario,
   AutoBeTestScenarioEvent,
 } from "@autobe/interface";
+import { AutoBeEndpointComparator } from "@autobe/utils";
 import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
-import { IPointer } from "tstl";
+import { HashMap, IPointer, Pair } from "tstl";
 import typia from "typia";
 import { v4 } from "uuid";
 
@@ -18,37 +19,67 @@ import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { divideArray } from "../../utils/divideArray";
 import { enforceToolCall } from "../../utils/enforceToolCall";
+import { forceRetry } from "../../utils/forceRetry";
 import { IAutoBeTestScenarioApplication } from "./structures/IAutoBeTestScenarioApplication";
 
 export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
 ): Promise<AutoBeTestScenarioEvent> {
-  const operations = ctx.state().interface?.document.operations ?? [];
+  const operations: AutoBeOpenApi.IOperation[] =
+    ctx.state().interface?.document.operations ?? [];
   if (operations.length === 0) {
     throw new Error(
       "Cannot write test scenarios because these are no operations.",
     );
   }
 
+  const dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation> =
+    new HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>(
+      operations.map(
+        (op) =>
+          new Pair(
+            {
+              path: op.path,
+              method: op.method,
+            },
+            op,
+          ),
+      ),
+      AutoBeEndpointComparator.hashCode,
+      AutoBeEndpointComparator.equals,
+    );
+  const endpointNotFound: string = [
+    `You have to select one of the endpoints below`,
+    "",
+    " method | path ",
+    "--------|------",
+    ...operations.map((op) => `\`${op.method}\` | \`${op.path}\``).join("\n"),
+  ].join("\n");
+
   const exclude: IAutoBeTestScenarioApplication.IScenarioGroup[] = [];
   let include: AutoBeOpenApi.IOperation[] = Array.from(operations);
 
   do {
-    const matrix = divideArray({ array: include, capacity: 30 });
-
+    const matrix: AutoBeOpenApi.IOperation[][] = divideArray({
+      array: include,
+      capacity: 5,
+    });
     await Promise.all(
-      matrix.map(async (_include) => {
+      matrix.map(async (include) => {
         exclude.push(
-          ...(await execute(
-            ctx,
-            operations,
-            _include,
-            exclude.map((x) => x.endpoint),
+          ...(await forceRetry(() =>
+            execute(
+              ctx,
+              dict,
+              endpointNotFound,
+              operations,
+              include,
+              exclude.map((x) => x.endpoint),
+            ),
           )),
         );
       }),
     );
-
     include = include.filter((op) => {
       if (
         exclude.some(
@@ -71,7 +102,7 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
           endpoint: pg.endpoint,
           draft: plan.draft,
           functionName: plan.functionName,
-          dependencies: plan.dependsOn,
+          dependencies: plan.dependencies,
         } satisfies AutoBeTestScenario;
       });
     }),
@@ -81,9 +112,11 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
 
 const execute = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  ops: AutoBeOpenApi.IOperation[],
-  include: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
-  exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+  dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>,
+  endpointNotFound: string,
+  entire: AutoBeOpenApi.IOperation[],
+  include: AutoBeOpenApi.IEndpoint[],
+  exclude: AutoBeOpenApi.IEndpoint[],
 ) => {
   const pointer: IPointer<IAutoBeTestScenarioApplication.IScenarioGroup[]> = {
     value: [],
@@ -97,11 +130,12 @@ const execute = async <Model extends ILlmSchema.Model>(
         describe: null,
       },
     },
-    tokenUsage: ctx.usage(),
-    histories: createHistoryProperties(ops, include, exclude),
+    histories: createHistoryProperties(entire, include, exclude),
     controllers: [
       createApplication({
         model: ctx.model,
+        endpointNotFound,
+        dict,
         build: (next) => {
           pointer.value ??= [];
           pointer.value.push(...next.scenarioGroups);
@@ -111,7 +145,10 @@ const execute = async <Model extends ILlmSchema.Model>(
   });
   enforceToolCall(agentica);
 
-  await agentica.conversate(`create test scenarios.`);
+  await agentica.conversate(`create test scenarios.`).finally(() => {
+    const tokenUsage = agentica.getTokenUsage();
+    ctx.usage().record(tokenUsage, ["test"]);
+  });
   if (pointer.value.length === 0) {
     throw new Error("Failed to create test plans.");
   }
@@ -120,10 +157,12 @@ const execute = async <Model extends ILlmSchema.Model>(
 };
 
 const createHistoryProperties = (
-  operations: AutoBeOpenApi.IOperation[],
+  entire: AutoBeOpenApi.IOperation[],
   include: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
   exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
-) => [
+): Array<
+  IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
+> => [
   {
     id: v4(),
     created_at: new Date().toISOString(),
@@ -135,17 +174,24 @@ const createHistoryProperties = (
     created_at: new Date().toISOString(),
     type: "systemMessage",
     text: [
+      "# Operations",
       "Below are the full operations. Please refer to this.",
       "Your role is to draft all test cases for each given Operation.",
       "It is also permissible to write multiple test codes on a single endpoint.",
       "However, rather than meaningless tests, business logic tests should be written and an E2E test situation should be assumed.",
       "",
+      "Please carefully analyze each operation to identify all dependencies required for testing.",
+      "For example, if you want to test liking and then deleting a post,",
+      "you might think to test post creation, liking, and unlike operations.",
+      "However, even if not explicitly mentioned, user registration and login are essential prerequisites.",
+      "Pay close attention to IDs and related values in the API,",
+      "and ensure you identify all dependencies between endpoints.",
+      "",
       "```json",
       JSON.stringify(
-        operations.map((el) => ({
-          path: el.path,
-          method: el.method,
-          summary: el.summary,
+        entire.map((el) => ({
+          ...el,
+          specification: undefined,
         })),
       ),
       "```",
@@ -174,6 +220,8 @@ const createHistoryProperties = (
 
 function createApplication<Model extends ILlmSchema.Model>(props: {
   model: Model;
+  endpointNotFound: string;
+  dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>;
   build: (next: IAutoBeTestScenarioApplication.IProps) => void;
 }): IAgenticaController.IClass<Model> {
   assertSchemaModel(props.model);
@@ -187,55 +235,53 @@ function createApplication<Model extends ILlmSchema.Model>(props: {
       typia.validate<IAutoBeTestScenarioApplication.IProps>(next);
     if (result.success === false) return result;
 
-    const errors: IValidation.IError[] = [];
-    result.data.scenarioGroups.forEach((pg, i, arr) => {
-      arr.forEach((target, j) => {
-        if (
-          i !== j &&
-          target.endpoint.method === pg.endpoint.method &&
-          target.endpoint.path === pg.endpoint.path
-        ) {
-          if (
-            !errors.some(
-              (el) =>
-                el.path !== `planGroups[${j}].path` &&
-                el.value !== target.endpoint.path,
-            )
-          ) {
-            errors.push({
-              path: `planGroups[${j}].path`,
-              expected: `planGroup's {method + path} cannot duplicated.`,
-              value: target.endpoint.path,
-            });
-          }
+    // merge to unique scenario groups
+    const scenarioGroups: IAutoBeTestScenarioApplication.IScenarioGroup[] = [];
+    result.data.scenarioGroups.forEach((sg) => {
+      const created = scenarioGroups.find(
+        (el) =>
+          el.endpoint.method === sg.endpoint.method &&
+          el.endpoint.path === sg.endpoint.path,
+      );
+      if (created) {
+        created.scenarios.push(...sg.scenarios);
+      } else {
+        scenarioGroups.push(sg);
+      }
+    });
 
-          if (
-            !errors.some(
-              (el) =>
-                el.path !== `planGroups[${j}].method` &&
-                el.value !== target.endpoint.method,
-            )
-          ) {
+    // validate endpoints
+    const errors: IValidation.IError[] = [];
+    scenarioGroups.forEach((group, i) => {
+      if (props.dict.has(group.endpoint) === false)
+        errors.push({
+          value: group.endpoint,
+          path: `$input.scenarioGroups[${i}].endpoint`,
+          expected: "AutoBeOpenApi.IEndpoint",
+          description: props.endpointNotFound,
+        });
+      group.scenarios.forEach((s, j) => {
+        s.dependencies.forEach((dep, k) => {
+          if (props.dict.has(dep.endpoint) === false)
             errors.push({
-              path: `planGroups[${j}].method`,
-              expected: `planGroup's {method + path} cannot duplicated.`,
-              value: target.endpoint.method,
+              value: dep.endpoint,
+              path: `$input.scenarioGroups[${i}].scenarios[${j}].dependencies[${k}].endpoint`,
+              expected: "AutoBeOpenApi.IEndpoint",
+              description: props.endpointNotFound,
             });
-          }
-        }
+        });
       });
     });
-
-    if (errors.length !== 0) {
-      console.log(JSON.stringify(errors, null, 2), "errors");
-      return {
-        success: false,
-        errors,
-        data: next,
-      };
-    }
-
-    return result;
+    return errors.length === 0
+      ? {
+          success: true,
+          data: scenarioGroups,
+        }
+      : {
+          success: false,
+          data: scenarioGroups,
+          errors,
+        };
   };
   return {
     protocol: "class",
@@ -266,5 +312,4 @@ const collection = {
   llama: claude,
   deepseek: claude,
   "3.1": claude,
-  "3.0": typia.llm.application<IAutoBeTestScenarioApplication, "3.0">(),
 };

package/src/orchestrate/test/orchestrateTestWrite.ts

@@ -7,40 +7,54 @@ import typia from "typia";
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { enforceToolCall } from "../../utils/enforceToolCall";
-import { compileTestScenario } from "./compileTestScenario";
+import { forceRetry } from "../../utils/forceRetry";
+import { completeTestCode } from "./compile/completeTestCode";
+import { getTestScenarioArtifacts } from "./compile/getTestScenarioArtifacts";
 import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
+import { IAutoBeTestWriteApplication } from "./structures/IAutoBeTestWriteApplication";
+import { IAutoBeTestWriteResult } from "./structures/IAutoBeTestWriteResult";
 import { transformTestWriteHistories } from "./transformTestWriteHistories";
 
 export async function orchestrateTestWrite<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   scenarios: AutoBeTestScenario[],
-): Promise<AutoBeTestWriteEvent[]> {
+): Promise<IAutoBeTestWriteResult[]> {
   const start: Date = new Date();
   let complete: number = 0;
 
-  const events: AutoBeTestWriteEvent[] = await Promise.all(
+  return Promise.all(
     /**
      * Generate test code for each scenario. Maps through plans array to create
      * individual test code implementations. Each scenario is processed to
      * generate corresponding test code and progress events.
      */
-    scenarios.map(async (scenario) => {
-      const code: ICreateTestCodeProps = await process(ctx, scenario);
-      const event: AutoBeTestWriteEvent = {
-        type: "testWrite",
-        created_at: start.toISOString(),
-        filename: `test/features/api/${code.domain}/${scenario.functionName}.ts`,
-        content: code.content,
-        completed: ++complete,
-        total: scenarios.length,
-        step: ctx.state().interface?.step ?? 0,
-      };
-      ctx.dispatch(event);
-      return event;
-    }),
+    scenarios.map((scenario) =>
+      forceRetry(async () => {
+        const artifacts: IAutoBeTestScenarioArtifacts =
+          await getTestScenarioArtifacts(ctx, scenario);
+        const result: IAutoBeTestWriteApplication.IProps = await process(
+          ctx,
+          scenario,
+          artifacts,
+        );
+        const event: AutoBeTestWriteEvent = {
+          type: "testWrite",
+          created_at: start.toISOString(),
+          location: `test/features/api/${result.domain}/${scenario.functionName}.ts`,
+          ...result,
+          completed: ++complete,
+          total: scenarios.length,
+          step: ctx.state().interface?.step ?? 0,
+        };
+        ctx.dispatch(event);
+        return {
+          scenario,
+          artifacts,
+          event,
+        };
+      }),
+    ),
   );
-
-  return events;
 }
 
 /**
@@ -50,51 +64,49 @@ export async function orchestrateTestWrite<Model extends ILlmSchema.Model>(
  *
  * @param ctx - The AutoBeContext containing model, vendor and configuration
  * @param scenario - The test scenario information to generate code for
- * @returns Promise resolving to ICreateTestCodeProps containing the generated
- *   test code
+ * @param artifacts - The artifacts containing the reference files and schemas
+ * @returns Promise resolving to IAutoBeTestWriteApplication.IProps containing
+ *   the generated test code
  */
 async function process<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   scenario: AutoBeTestScenario,
-): Promise<ICreateTestCodeProps> {
-  const pointer: IPointer<ICreateTestCodeProps | null> = {
+  artifacts: IAutoBeTestScenarioArtifacts,
+): Promise<IAutoBeTestWriteApplication.IProps> {
+  const pointer: IPointer<IAutoBeTestWriteApplication.IProps | null> = {
     value: null,
   };
-  const artifacts: IAutoBeTestScenarioArtifacts = await compileTestScenario(
-    ctx,
-    scenario,
-  );
-
-  const agentica = new MicroAgentica({
+  const agentica: MicroAgentica<Model> = new MicroAgentica({
     model: ctx.model,
     vendor: ctx.vendor,
     config: {
       ...(ctx.config ?? {}),
     },
-    histories: transformTestWriteHistories({
-      scenario,
-      artifacts,
-    }),
+    histories: transformTestWriteHistories(scenario, artifacts),
     controllers: [
       createApplication({
         model: ctx.model,
+        artifacts,
         build: (next) => {
           pointer.value = next;
         },
       }),
     ],
-    tokenUsage: ctx.usage(),
   });
   enforceToolCall(agentica);
 
-  await agentica.conversate("Create e2e test functions.");
+  await agentica.conversate("Create e2e test functions.").finally(() => {
+    const tokenUsage = agentica.getTokenUsage();
+    ctx.usage().record(tokenUsage, ["test"]);
+  });
   if (pointer.value === null) throw new Error("Failed to create test code.");
   return pointer.value;
 }
 
 function createApplication<Model extends ILlmSchema.Model>(props: {
   model: Model;
-  build: (next: ICreateTestCodeProps) => void;
+  artifacts: IAutoBeTestScenarioArtifacts;
+  build: (next: IAutoBeTestWriteApplication.IProps) => void;
 }): IAgenticaController.IClass<Model> {
   assertSchemaModel(props.model);
 
@@ -106,15 +118,17 @@ function createApplication<Model extends ILlmSchema.Model>(props: {
     name: "Create Test Code",
     application,
     execute: {
-      createTestCode: (next) => {
+      write: (next) => {
+        next.draft = completeTestCode(props.artifacts, next.draft);
+        next.final = completeTestCode(props.artifacts, next.final);
         props.build(next);
       },
-    } satisfies IApplication,
+    } satisfies IAutoBeTestWriteApplication,
   };
 }
 
 const claude = typia.llm.application<
-  IApplication,
+  IAutoBeTestWriteApplication,
   "claude",
   {
     reference: true;
@@ -122,7 +136,7 @@ const claude = typia.llm.application<
 >();
 const collection = {
   chatgpt: typia.llm.application<
-    IApplication,
+    IAutoBeTestWriteApplication,
     "chatgpt",
     { reference: true }
   >(),
@@ -130,117 +144,5 @@ const collection = {
   llama: claude,
   deepseek: claude,
   "3.1": claude,
-  "3.0": typia.llm.application<IApplication, "3.0">(),
+  "3.0": typia.llm.application<IAutoBeTestWriteApplication, "3.0">(),
 };
-
-interface IApplication {
-  createTestCode(props: ICreateTestCodeProps): void;
-}
-
-interface ICreateTestCodeProps {
-  /**
-   * Strategic approach for test implementation.
-   *
-   * Define the high-level strategy and logical flow for testing the given
-   * scenario. Focus on test methodology, data preparation, and assertion
-   * strategy.
-   *
-   * ### Critical Requirements
-   *
-   * - Must follow the Test Generation Guildelines.
-   * - Must Planning the test code Never occur the typescript compile error.
-   *
-   * ### Planning Elements:
-   *
-   * #### Test Methodology
-   *
-   * - Identify test scenario type (CRUD operation, authentication flow,
-   *   validation test)
-   * - Define test data requirements and preparation strategy
-   * - Plan positive/negative test cases and edge cases
-   * - Design assertion logic and validation points
-   *
-   * #### Execution Strategy
-   *
-   * - Outline step-by-step test execution flow
-   * - Plan error handling and exception plans
-   * - Define cleanup and teardown procedures
-   * - Identify dependencies and prerequisites
-   *
-   * ### Example Plan:
-   *
-   *     Test Strategy: Article Creation Validation
-   *     1. Prepare valid article data with required fields
-   *     2. Execute POST request to create article
-   *     3. Validate response structure and data integrity
-   *     4. Test error plans (missing fields, invalid data)
-   *     5. Verify database state changes
-   *     6. Reconsider the scenario if it doesn't follow the Test Generation
-   *        Guildelines.
-   */
-  scenario: string;
-
-  /**
-   * Functional domain classification for test organization.
-   *
-   * Determines file structure and test categorization based on API
-   * functionality. Used for organizing tests into logical groups and directory
-   * hierarchies.
-   *
-   * ### Naming Rules:
-   *
-   * - Lowercase English words only
-   * - Singular nouns (e.g., "article", "user", "comment")
-   * - Kebab-case for compound words (e.g., "user-profile", "payment-method")
-   * - Match primary API resource being tested
-   * - Domain Name must be named only one word.
-   *
-   * ### Domain Examples:
-   *
-   * - `article` → Article management operations
-   * - `comment` → Comment-related functionality
-   * - `auth` → Authentication and authorization
-   * - `user` → User management operations
-   * - `payment` → Payment processing
-   * - `notification` → Notification system
-   */
-  domain: string;
-
-  /**
-   * Complete TypeScript E2E test implementation.
-   *
-   * Generate fully functional, compilation-error-free test code following
-   *
-   * @nestia/e2e framework conventions and TypeScript best practices.
-   *
-   * ### Technical Implementation Requirements:
-   *
-   * #### Import Declarations
-   * ```typescript
-   * import api from "@ORGANIZATION/PROJECT-api";
-   * import { ITargetType } from "@ORGANIZATION/PROJECT-api/lib/structures/[path]";
-   * import { TestValidator } from "@nestia/e2e";
-   * import typia from "typia";
-   * ```
-   * - Must use exact `@ORGANIZATION/PROJECT-api` module path
-   * - Include `@ORGANIZATION` prefix in all API-related imports
-   * - Import specific DTO types from correct structure paths
-   *
-   * #### Code Quality Standards
-   * - Zero TypeScript compilation errors (mandatory)
-   * - Explicit type annotations for all variables
-   * - Proper async/await patterns throughout
-   * - Comprehensive error handling
-   * - Clean, readable code structure
-   * - Consistent formatting and naming conventions
-   *
-   * ### Critical Error Prevention
-   * - Verify all import paths are correct and accessible
-   * - Ensure type compatibility between variables and assignments
-   * - Include all required object properties and methods
-   * - Validate API function signatures and parameter types
-   * - Confirm proper generic type usage
-   * - Test async function declarations and Promise handling
-   */
-  content: string;
-}