@autobe/agent 0.8.0 → 0.9.1

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -1,198 +1,256 @@
-import { IAgenticaController, MicroAgentica } from "@agentica/core";
-import { AutoBeOpenApi, AutoBeTest } from "@autobe/interface";
-import { AutoBeTestScenarioEvent } from "@autobe/interface/src/events/AutoBeTestScenarioEvent";
-import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
-import { HashMap, HashSet, IPointer } from "tstl";
+import {
+  IAgenticaController,
+  IAgenticaHistoryJson,
+  MicroAgentica,
+} from "@agentica/core";
+import {
+  AutoBeOpenApi,
+  AutoBeTestScenario,
+  AutoBeTestScenarioEvent,
+} from "@autobe/interface";
+import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
+import { IPointer } from "tstl";
 import typia from "typia";
+import { v4 } from "uuid";
 
+import { AutoBeSystemPromptConstant } from "../../constants/AutoBeSystemPromptConstant";
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { divideArray } from "../../utils/divideArray";
 import { enforceToolCall } from "../../utils/enforceToolCall";
-import { OpenApiEndpointComparator } from "../interface/OpenApiEndpointComparator";
-import { transformTestScenarioHistories } from "./transformTestScenarioHistories";
+import { IAutoBeTestScenarioApplication } from "./structures/IAutoBeTestScenarioApplication";
 
 export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  capacity: number = 4,
 ): Promise<AutoBeTestScenarioEvent> {
-  const files = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("test/features/api/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
-
   const operations = ctx.state().interface?.document.operations ?? [];
-  const endpoints: Omit<AutoBeOpenApi.IOperation, "specification">[] =
-    operations.map((it) => {
-      return {
-        method: it.method,
-        path: it.path,
-        summary: it.summary,
-        description: it.description,
-        parameters: it.parameters,
-        requestBody: it.requestBody,
-        responseBody: it.responseBody,
-      };
-    });
+  if (operations.length === 0) {
+    throw new Error(
+      "Cannot write test scenarios because these are no operations.",
+    );
+  }
 
-  const matrix: AutoBeOpenApi.IEndpoint[][] = divideArray({
-    array: endpoints,
-    capacity,
-  });
-  const start: Date = new Date();
-
-  let completed: number = 0;
-
-  const scenarios: AutoBeTest.IScenario[][] = await Promise.all(
-    matrix.map(async (e) => {
-      const rows: AutoBeTest.IScenario[] = await divideAndConquer(
-        ctx,
-        e,
-        endpoints,
-        files,
-        3,
-        (count) => {
-          completed += count;
-        },
-      );
-      ctx.dispatch({
-        type: "testScenario",
-        scenarios: rows,
-        total: rows.flatMap((el) => el.scenarios).length,
-        step: ctx.state().test?.step ?? 0,
-        completed,
-        created_at: start.toISOString(),
-      });
-      return rows;
-    }),
-  );
+  const exclude: IAutoBeTestScenarioApplication.IScenarioGroup[] = [];
+  let include: AutoBeOpenApi.IOperation[] = Array.from(operations);
 
-  return {
-    type: "testScenario",
-    scenarios: scenarios.flat(),
-    total: scenarios.flat().flatMap((el) => el.scenarios).length,
-    step: ctx.state().test?.step ?? 0,
-    completed,
-    created_at: start.toISOString(),
-  };
-}
+  do {
+    const matrix = divideArray({ array: include, capacity: 30 });
 
-async function divideAndConquer<Model extends ILlmSchema.Model>(
-  ctx: AutoBeContext<Model>,
-  endpoints: AutoBeOpenApi.IEndpoint[],
-  allEndpoints: AutoBeOpenApi.IEndpoint[],
-  files: Record<string, string>,
-  retry: number,
-  progress: (completed: number) => void,
-): Promise<AutoBeTest.IScenario[]> {
-  const remained: HashSet<AutoBeOpenApi.IEndpoint> = new HashSet(
-    endpoints,
-    OpenApiEndpointComparator.hashCode,
-    OpenApiEndpointComparator.equals,
-  );
-  const scenarios: HashMap<AutoBeOpenApi.IEndpoint, AutoBeTest.Scenario[]> =
-    new HashMap(
-      OpenApiEndpointComparator.hashCode,
-      OpenApiEndpointComparator.equals,
-    );
-  for (let i: number = 0; i < retry; ++i) {
-    if (remained.empty() === true || scenarios.size() >= endpoints.length)
-      break;
-    const before: number = scenarios.size();
-    const newbie: AutoBeTest.IScenario[] = await process(
-      ctx,
-      Array.from(remained),
-      allEndpoints,
-      files,
+    await Promise.all(
+      matrix.map(async (_include) => {
+        exclude.push(
+          ...(await execute(
+            ctx,
+            operations,
+            _include,
+            exclude.map((x) => x.endpoint),
+          )),
+        );
+      }),
     );
-    for (const item of newbie) {
-      scenarios.set(item.endpoint, item.scenarios);
-      remained.erase(item.endpoint);
-    }
-    if (scenarios.size() - before !== 0) progress(scenarios.size() - before);
-  }
-  return Array.from(scenarios.toJSON()).map((it) => ({
-    endpoint: it.first,
-    scenarios: it.second,
-  }));
+
+    include = include.filter((op) => {
+      if (
+        exclude.some(
+          (pg) =>
+            pg.endpoint.method === op.method && pg.endpoint.path === op.path,
+        )
+      ) {
+        return false;
+      }
+      return true;
+    });
+  } while (include.length > 0);
+
+  return {
+    type: "testScenario",
+    step: ctx.state().analyze?.step ?? 0,
+    scenarios: exclude.flatMap((pg) => {
+      return pg.scenarios.map((plan) => {
+        return {
+          endpoint: pg.endpoint,
+          draft: plan.draft,
+          functionName: plan.functionName,
+          dependencies: plan.dependsOn,
+        } satisfies AutoBeTestScenario;
+      });
+    }),
+    created_at: new Date().toISOString(),
+  } as AutoBeTestScenarioEvent;
 }
 
-async function process<Model extends ILlmSchema.Model>(
+const execute = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  endpoints: AutoBeOpenApi.IEndpoint[],
-  allEndpoints: AutoBeOpenApi.IEndpoint[],
-  files: Record<string, string>,
-): Promise<AutoBeTest.IScenario[]> {
-  const pointer: IPointer<AutoBeTest.IScenario[] | null> = {
-    value: null,
+  ops: AutoBeOpenApi.IOperation[],
+  include: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+  exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+) => {
+  const pointer: IPointer<IAutoBeTestScenarioApplication.IScenarioGroup[]> = {
+    value: [],
   };
-
-  const agentica = new MicroAgentica({
+  const agentica: MicroAgentica<Model> = new MicroAgentica({
     model: ctx.model,
     vendor: ctx.vendor,
     config: {
-      ...(ctx.config ?? { locale: "en-US" }),
-      systemPrompt: {
-        describe: () => {
-          return "Answer only 'completion' or 'failure'.";
-        },
+      ...(ctx.config ?? {}),
+      executor: {
+        describe: null,
       },
     },
     tokenUsage: ctx.usage(),
-    histories: [
-      ...transformTestScenarioHistories(ctx.state(), allEndpoints, files),
-    ],
+    histories: createHistoryProperties(ops, include, exclude),
     controllers: [
       createApplication({
         model: ctx.model,
         build: (next) => {
           pointer.value ??= [];
-          pointer.value.push(...next.scenarios);
+          pointer.value.push(...next.scenarioGroups);
         },
       }),
     ],
   });
   enforceToolCall(agentica);
 
-  await agentica.conversate(
-    [
-      "Make User Scenarios for below endpoints:",
+  await agentica.conversate(`create test scenarios.`);
+  if (pointer.value.length === 0) {
+    throw new Error("Failed to create test plans.");
+  }
+
+  return pointer.value;
+};
+
+const createHistoryProperties = (
+  operations: AutoBeOpenApi.IOperation[],
+  include: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+  exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+) => [
+  {
+    id: v4(),
+    created_at: new Date().toISOString(),
+    type: "systemMessage",
+    text: AutoBeSystemPromptConstant.TEST_SCENARIO,
+  } satisfies IAgenticaHistoryJson.ISystemMessage,
+  {
+    id: v4(),
+    created_at: new Date().toISOString(),
+    type: "systemMessage",
+    text: [
+      "Below are the full operations. Please refer to this.",
+      "Your role is to draft all test cases for each given Operation.",
+      "It is also permissible to write multiple test codes on a single endpoint.",
+      "However, rather than meaningless tests, business logic tests should be written and an E2E test situation should be assumed.",
       "",
       "```json",
-      JSON.stringify(endpoints, null, 2),
+      JSON.stringify(
+        operations.map((el) => ({
+          path: el.path,
+          method: el.method,
+          summary: el.summary,
+        })),
+      ),
       "```",
     ].join("\n"),
-  );
-  if (pointer.value === null) throw new Error("Failed to make scenarios.");
-  return pointer.value;
-}
+  } satisfies IAgenticaHistoryJson.ISystemMessage,
+  {
+    id: v4(),
+    created_at: new Date().toISOString(),
+    type: "systemMessage",
+    text: [
+      "# Included in Test Plan",
+      include
+        .map((el) => `- ${el.method.toUpperCase()}: ${el.path}`)
+        .join("\n"),
+      "",
+      "# Excluded from Test Plan",
+      "These are the endpoints that have already been used in test codes generated as part of a plan group.",
+      "These endpoints do not need to be tested again.",
+      "However, it is allowed to reference or depend on these endpoints when writing test codes for other purposes.",
+      exclude
+        .map((el) => `- ${el.method.toUpperCase()}: ${el.path}`)
+        .join("\n"),
+    ].join("\n"),
+  } satisfies IAgenticaHistoryJson.ISystemMessage,
+];
 
 function createApplication<Model extends ILlmSchema.Model>(props: {
   model: Model;
-  build: (next: IMakeScenarioProps) => void;
+  build: (next: IAutoBeTestScenarioApplication.IProps) => void;
 }): IAgenticaController.IClass<Model> {
   assertSchemaModel(props.model);
 
   const application: ILlmApplication<Model> = collection[
     props.model
   ] as unknown as ILlmApplication<Model>;
+
+  application.functions[0].validate = (next: unknown): IValidation => {
+    const result: IValidation<IAutoBeTestScenarioApplication.IProps> =
+      typia.validate<IAutoBeTestScenarioApplication.IProps>(next);
+    if (result.success === false) return result;
+
+    const errors: IValidation.IError[] = [];
+    result.data.scenarioGroups.forEach((pg, i, arr) => {
+      arr.forEach((target, j) => {
+        if (
+          i !== j &&
+          target.endpoint.method === pg.endpoint.method &&
+          target.endpoint.path === pg.endpoint.path
+        ) {
+          if (
+            !errors.some(
+              (el) =>
+                el.path !== `planGroups[${j}].path` &&
+                el.value !== target.endpoint.path,
+            )
+          ) {
+            errors.push({
+              path: `planGroups[${j}].path`,
+              expected: `planGroup's {method + path} cannot duplicated.`,
+              value: target.endpoint.path,
+            });
+          }
+
+          if (
+            !errors.some(
+              (el) =>
+                el.path !== `planGroups[${j}].method` &&
+                el.value !== target.endpoint.method,
+            )
+          ) {
+            errors.push({
+              path: `planGroups[${j}].method`,
+              expected: `planGroup's {method + path} cannot duplicated.`,
+              value: target.endpoint.method,
+            });
+          }
+        }
+      });
+    });
+
+    if (errors.length !== 0) {
+      console.log(JSON.stringify(errors, null, 2), "errors");
+      return {
+        success: false,
+        errors,
+        data: next,
+      };
+    }
+
+    return result;
+  };
   return {
     protocol: "class",
-    name: "Make User Scenarios",
+    name: "Make test plans",
     application,
     execute: {
       makeScenario: (next) => {
         props.build(next);
       },
-    } satisfies IApplication,
+    } satisfies IAutoBeTestScenarioApplication,
   };
 }
 
 const claude = typia.llm.application<
-  IApplication,
+  IAutoBeTestScenarioApplication,
   "claude",
   {
     reference: true;
@@ -200,7 +258,7 @@ const claude = typia.llm.application<
 >();
 const collection = {
   chatgpt: typia.llm.application<
-    IApplication,
+    IAutoBeTestScenarioApplication,
     "chatgpt",
     { reference: true }
   >(),
@@ -208,19 +266,5 @@ const collection = {
   llama: claude,
   deepseek: claude,
   "3.1": claude,
-  "3.0": typia.llm.application<IApplication, "3.0">(),
+  "3.0": typia.llm.application<IAutoBeTestScenarioApplication, "3.0">(),
 };
-
-interface IApplication {
-  /**
-   * Make user scenarios for the given endpoints.
-   *
-   * @param props Properties containing the endpoints and user scenarios.
-   */
-  makeScenario(props: IMakeScenarioProps): void;
-}
-
-interface IMakeScenarioProps {
-  /** Array of user scenarios. */
-  scenarios: AutoBeTest.IScenario[];
-}

package/src/orchestrate/test/{orchestrateTestProgress.ts → orchestrateTestWrite.ts}

@@ -1,5 +1,5 @@
 import { IAgenticaController, MicroAgentica } from "@agentica/core";
-import { AutoBeTest, AutoBeTestProgressEvent } from "@autobe/interface";
+import { AutoBeTestScenario, AutoBeTestWriteEvent } from "@autobe/interface";
 import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
@@ -7,28 +7,29 @@ import typia from "typia";
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { enforceToolCall } from "../../utils/enforceToolCall";
-import { transformTestProgressHistories } from "./transformTestProgressHistories";
+import { compileTestScenario } from "./compileTestScenario";
+import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
+import { transformTestWriteHistories } from "./transformTestWriteHistories";
 
-export async function orchestrateTestProgress<Model extends ILlmSchema.Model>(
+export async function orchestrateTestWrite<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  scenarios: AutoBeTest.Scenario[],
-): Promise<AutoBeTestProgressEvent[]> {
+  scenarios: AutoBeTestScenario[],
+): Promise<AutoBeTestWriteEvent[]> {
   const start: Date = new Date();
   let complete: number = 0;
 
-  const events: AutoBeTestProgressEvent[] = await Promise.all(
+  const events: AutoBeTestWriteEvent[] = await Promise.all(
     /**
-     * Generate test code for each scenario. Maps through scenarios array to
-     * create individual test code implementations. Each scenario is processed
-     * to generate corresponding test code and progress events.
+     * Generate test code for each scenario. Maps through plans array to create
+     * individual test code implementations. Each scenario is processed to
+     * generate corresponding test code and progress events.
      */
     scenarios.map(async (scenario) => {
       const code: ICreateTestCodeProps = await process(ctx, scenario);
-
-      const event: AutoBeTestProgressEvent = {
-        type: "testProgress",
+      const event: AutoBeTestWriteEvent = {
+        type: "testWrite",
         created_at: start.toISOString(),
-        filename: `${code.domain}/${scenario.functionName}.ts`,
+        filename: `test/features/api/${code.domain}/${scenario.functionName}.ts`,
         content: code.content,
         completed: ++complete,
         total: scenarios.length,
@@ -54,27 +55,15 @@ export async function orchestrateTestProgress<Model extends ILlmSchema.Model>(
  */
 async function process<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  scenario: AutoBeTest.Scenario,
+  scenario: AutoBeTestScenario,
 ): Promise<ICreateTestCodeProps> {
   const pointer: IPointer<ICreateTestCodeProps | null> = {
     value: null,
   };
-
-  const apiFiles = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("src/api/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
-
-  const dtoFiles = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("src/api/structures/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
+  const artifacts: IAutoBeTestScenarioArtifacts = await compileTestScenario(
+    ctx,
+    scenario,
+  );
 
   const agentica = new MicroAgentica({
     model: ctx.model,
@@ -82,7 +71,10 @@ async function process<Model extends ILlmSchema.Model>(
     config: {
       ...(ctx.config ?? {}),
     },
-    histories: transformTestProgressHistories(apiFiles, dtoFiles),
+    histories: transformTestWriteHistories({
+      scenario,
+      artifacts,
+    }),
     controllers: [
       createApplication({
         model: ctx.model,
@@ -91,18 +83,11 @@ async function process<Model extends ILlmSchema.Model>(
         },
       }),
     ],
+    tokenUsage: ctx.usage(),
   });
   enforceToolCall(agentica);
 
-  await agentica.conversate(
-    [
-      "Create test code for below scenario:",
-      "",
-      "```json",
-      JSON.stringify(scenario, null, 2),
-      "```",
-    ].join("\n"),
-  );
+  await agentica.conversate("Create e2e test functions.");
   if (pointer.value === null) throw new Error("Failed to create test code.");
   return pointer.value;
 }
@@ -178,7 +163,7 @@ interface ICreateTestCodeProps {
    * #### Execution Strategy
    *
    * - Outline step-by-step test execution flow
-   * - Plan error handling and exception scenarios
+   * - Plan error handling and exception plans
    * - Define cleanup and teardown procedures
    * - Identify dependencies and prerequisites
    *
@@ -188,12 +173,12 @@ interface ICreateTestCodeProps {
    *     1. Prepare valid article data with required fields
    *     2. Execute POST request to create article
    *     3. Validate response structure and data integrity
-   *     4. Test error scenarios (missing fields, invalid data)
+   *     4. Test error plans (missing fields, invalid data)
    *     5. Verify database state changes
-   *     6. Reconsider the plan if it doesn't follow the Test Generation
+   *     6. Reconsider the scenario if it doesn't follow the Test Generation
    *        Guildelines.
    */
-  plan: string;
+  scenario: string;
 
   /**
    * Functional domain classification for test organization.

package/src/orchestrate/test/structures/IAutoBeTestScenarioApplication.ts

@@ -0,0 +1,132 @@
+import { AutoBeOpenApi } from "@autobe/interface";
+
+export interface IAutoBeTestScenarioApplication {
+  /**
+   * Make test scenarios for the given endpoints.
+   *
+   * @param props Properties containing the endpoints and test scenarios.
+   */
+  makeScenario(props: IAutoBeTestScenarioApplication.IProps): void;
+}
+
+export namespace IAutoBeTestScenarioApplication {
+  export interface IProps {
+    /** Array of test scenario groups. */
+    scenarioGroups: IAutoBeTestScenarioApplication.IScenarioGroup[];
+  }
+
+  export interface IScenarioGroup {
+    /** Target API endpoint to test. */
+    endpoint: AutoBeOpenApi.IEndpoint;
+
+    /** Array of test scenarios. */
+    scenarios: IScenario[];
+  }
+
+  /**
+   * Represents a test scenario for a single API operation.
+   *
+   * This interface extends `AutoBeOpenApi.IEndpoint`, inheriting its HTTP
+   * method and path information, and adds two key properties:
+   *
+   * - `draft`: A free-form, human-readable test scenario description for the API
+   *   endpoint.
+   * - `dependsOn`: A list of other API endpoints that must be invoked beforehand
+   *   in order to prepare the context for this test. Each dependency includes
+   *   the purpose of the dependency.
+   *
+   * This structure is intended to help organize test specifications for complex
+   * workflows and ensure that all prerequisites are explicitly declared.
+   */
+  export interface IScenario {
+    /**
+     * A detailed natural language description of how this API endpoint should
+     * be tested. This should include both successful and failure scenarios,
+     * business rule validations, edge cases, and any sequence of steps
+     * necessary to perform the test. A subsequent agent will use this draft to
+     * generate multiple test scenarios.
+     */
+    draft: string;
+
+    /**
+     * Descriptive function name derived from the user scenario.
+     *
+     * The function name serves as a concise, technical identifier that clearly
+     * represents the specific user scenario being described. It should be
+     * immediately understandable and directly correspond to the user situation
+     * without requiring additional context.
+     *
+     * ## Naming Convention
+     *
+     * - Must start with `test_` prefix (mandatory requirement)
+     * - Use snake_case formatting throughout
+     * - Include the primary user action (create, get, update, delete, list, etc.)
+     * - Specify the target resource (user, product, order, profile, etc.)
+     * - Add scenario-specific context (valid_data, invalid_email, not_found,
+     *   etc.)
+     *
+     * ## Content Structure
+     *
+     * Function names should follow this pattern:
+     * `test_[user_action]_[resource]_[scenario_context]`
+     *
+     * Where:
+     *
+     * - `user_action`: What the user is trying to do
+     * - `resource`: What the user is interacting with
+     * - `scenario_context`: The specific situation or condition
+     *
+     * ## User-Focused Examples
+     *
+     * - `test_create_user_profile_with_complete_information` - User providing all
+     *   available profile data
+     * - `test_retrieve_user_profile_when_profile_exists` - User accessing their
+     *   existing profile
+     * - `test_update_user_email_with_valid_new_address` - User changing their
+     *   email to a valid new one
+     * - `test_delete_user_account_when_user_lacks_permission` - User attempting
+     *   account deletion without authorization
+     * - `test_search_user_profiles_with_pagination_preferences` - User browsing
+     *   profiles with specific pagination
+     *
+     * ## Clarity Guidelines
+     *
+     * - Prioritize clarity over brevity
+     * - Avoid technical jargon or implementation terms
+     * - Use terminology that reflects user perspective
+     * - Ensure the name alone conveys the user's intent
+     * - Make it understandable to non-technical stakeholders
+     * - Keep consistent with user scenario description
+     *
+     * ## Single Endpoint Alignment
+     *
+     * Function names must reflect scenarios that:
+     *
+     * - Accomplish user goals through this single endpoint only
+     * - Don't imply dependency on other API operations
+     * - Represent complete user interactions
+     */
+    functionName: string;
+
+    /**
+     * A list of other API endpoints that must be executed before this test
+     * scenario. This helps express dependencies such as data creation or
+     * authentication steps required to reach the intended test state.
+     */
+    dependsOn: IDependsOn[];
+  }
+
+  export interface IDependsOn {
+    /** Target API endpoint that must be executed before the main operation. */
+    endpoint: AutoBeOpenApi.IEndpoint;
+
+    /**
+     * A concise exscenarioation of why this API call is required before
+     * executing the test for the main operation.
+     *
+     * Example: "Creates a category so that a product can be linked to it during
+     * creation."
+     */
+    purpose: string;
+  }
+}