@autobe/agent 0.7.3 → 0.9.0

package/src/orchestrate/test/orchestrateTestProgress.ts

@@ -1,31 +1,39 @@
 import { IAgenticaController, MicroAgentica } from "@agentica/core";
-import { AutoBeTest, AutoBeTestProgressEvent } from "@autobe/interface";
-import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
+import {
+  AutoBeOpenApi,
+  AutoBeTestScenarioEvent,
+  AutoBeTestWriteEvent,
+} from "@autobe/interface";
+import {
+  ILlmApplication,
+  ILlmSchema,
+  OpenApiTypeChecker,
+} from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
+import { enforceToolCall } from "../../utils/enforceToolCall";
 import { transformTestProgressHistories } from "./transformTestProgressHistories";
 
 export async function orchestrateTestProgress<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  scenarios: AutoBeTest.Scenario[],
-): Promise<AutoBeTestProgressEvent[]> {
+  scenarios: AutoBeTestScenarioEvent.IScenario[],
+): Promise<AutoBeTestWriteEvent[]> {
   const start: Date = new Date();
   let complete: number = 0;
 
-  const events: AutoBeTestProgressEvent[] = await Promise.all(
+  const events: AutoBeTestWriteEvent[] = await Promise.all(
     /**
-     * Generate test code for each scenario. Maps through scenarios array to
-     * create individual test code implementations. Each scenario is processed
-     * to generate corresponding test code and progress events.
+     * Generate test code for each scenario. Maps through plans array to create
+     * individual test code implementations. Each scenario is processed to
+     * generate corresponding test code and progress events.
      */
     scenarios.map(async (scenario) => {
       const code: ICreateTestCodeProps = await process(ctx, scenario);
-
-      const event: AutoBeTestProgressEvent = {
-        type: "testProgress",
+      const event: AutoBeTestWriteEvent = {
+        type: "testWrite",
         created_at: start.toISOString(),
         filename: `${code.domain}/${scenario.functionName}.ts`,
         content: code.content,
@@ -53,27 +61,20 @@ export async function orchestrateTestProgress<Model extends ILlmSchema.Model>(
  */
 async function process<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  scenario: AutoBeTest.Scenario,
+  scenario: AutoBeTestScenarioEvent.IScenario,
 ): Promise<ICreateTestCodeProps> {
   const pointer: IPointer<ICreateTestCodeProps | null> = {
     value: null,
   };
-
-  const apiFiles = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("src/api/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
-
-  const dtoFiles = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("src/api/structures/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
+  const document: AutoBeOpenApi.IDocument = filterDocument(
+    scenario,
+    ctx.state().interface!.document,
+  );
+  const files: [string, string][] = Object.entries(
+    await ctx.compiler.interface.compile(document),
+  );
+  const filter = (prefix: string) =>
+    Object.fromEntries(files.filter(([key]) => key.startsWith(prefix)));
 
   const agentica = new MicroAgentica({
     model: ctx.model,
@@ -81,7 +82,12 @@ async function process<Model extends ILlmSchema.Model>(
     config: {
       ...(ctx.config ?? {}),
     },
-    histories: transformTestProgressHistories(apiFiles, dtoFiles),
+    histories: transformTestProgressHistories({
+      scenario: scenario,
+      dto: filter("src/api/structures"),
+      sdk: filter("src/api/functional"),
+      e2e: filter("test/features"),
+    }),
     controllers: [
       createApplication({
         model: ctx.model,
@@ -90,26 +96,61 @@ async function process<Model extends ILlmSchema.Model>(
         },
       }),
     ],
+    tokenUsage: ctx.usage(),
   });
+  enforceToolCall(agentica);
 
-  agentica.on("request", async (event) => {
-    if (event.body.tools) event.body.tool_choice = "required";
-  });
-
-  await agentica.conversate(
-    [
-      "Create test code for below scenario:",
-      "",
-      "```json",
-      JSON.stringify(scenario, null, 2),
-      "```",
-    ].join("\n"),
-  );
-
+  await agentica.conversate("Create e2e test functions.");
   if (pointer.value === null) throw new Error("Failed to create test code.");
   return pointer.value;
 }
 
+export function filterDocument(
+  scenario: AutoBeTestScenarioEvent.IScenario,
+  document: AutoBeOpenApi.IDocument,
+): AutoBeOpenApi.IDocument {
+  const operations: AutoBeOpenApi.IOperation[] = document.operations.filter(
+    (op) => {
+      if (
+        scenario.endpoint.method === op.method &&
+        scenario.endpoint.path === op.path
+      ) {
+        return true;
+      } else if (
+        scenario.dependencies.some(
+          (dp) =>
+            dp.endpoint.method === op.method && dp.endpoint.path === op.path,
+        )
+      ) {
+        return true;
+      }
+    },
+  );
+  const components: AutoBeOpenApi.IComponents = {
+    schemas: {},
+  };
+  const visit = (typeName: string) => {
+    OpenApiTypeChecker.visit({
+      components: document.components,
+      schema: { $ref: `#/components/schemas/${typeName}` },
+      closure: (s) => {
+        if (OpenApiTypeChecker.isReference(s)) {
+          const key: string = s.$ref.split("/").pop()!;
+          components.schemas[key] = document.components.schemas[key];
+        }
+      },
+    });
+  };
+  for (const op of operations) {
+    if (op.requestBody) visit(op.requestBody.typeName);
+    if (op.responseBody) visit(op.responseBody.typeName);
+  }
+  return {
+    operations,
+    components,
+  };
+}
+
 function createApplication<Model extends ILlmSchema.Model>(props: {
   model: Model;
   build: (next: ICreateTestCodeProps) => void;
@@ -181,7 +222,7 @@ interface ICreateTestCodeProps {
    * #### Execution Strategy
    *
    * - Outline step-by-step test execution flow
-   * - Plan error handling and exception scenarios
+   * - Plan error handling and exception plans
    * - Define cleanup and teardown procedures
    * - Identify dependencies and prerequisites
    *
@@ -191,12 +232,12 @@ interface ICreateTestCodeProps {
    *     1. Prepare valid article data with required fields
    *     2. Execute POST request to create article
    *     3. Validate response structure and data integrity
-   *     4. Test error scenarios (missing fields, invalid data)
+   *     4. Test error plans (missing fields, invalid data)
    *     5. Verify database state changes
-   *     6. Reconsider the plan if it doesn't follow the Test Generation
+   *     6. Reconsider the scenario if it doesn't follow the Test Generation
    *        Guildelines.
    */
-  plan: string;
+  scenario: string;
 
   /**
    * Functional domain classification for test organization.

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -1,150 +1,253 @@
-import { IAgenticaController, MicroAgentica } from "@agentica/core";
-import { AutoBeOpenApi, AutoBeTest } from "@autobe/interface";
-import { AutoBeTestScenarioEvent } from "@autobe/interface/src/events/AutoBeTestScenarioEvent";
-import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
+import {
+  IAgenticaController,
+  IAgenticaHistoryJson,
+  MicroAgentica,
+} from "@agentica/core";
+import { AutoBeOpenApi } from "@autobe/interface";
+import { AutoBeTestScenarioEvent } from "@autobe/interface";
+import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
+import { v4 } from "uuid";
 
+import { AutoBeSystemPromptConstant } from "../../constants/AutoBeSystemPromptConstant";
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
-import { transformTestScenarioHistories } from "./transformTestScenarioHistories";
+import { divideArray } from "../../utils/divideArray";
+import { enforceToolCall } from "../../utils/enforceToolCall";
+import { IAutoBeTestScenarioApplication } from "./structures/IAutoBeTestScenarioApplication";
 
 export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
 ): Promise<AutoBeTestScenarioEvent> {
-  const files = Object.entries(ctx.state().interface?.files ?? {})
-    .filter(([filename]) => {
-      return filename.startsWith("test/features/api/");
-    })
-    .reduce<Record<string, string>>((acc, [filename, content]) => {
-      return Object.assign(acc, { [filename]: content });
-    }, {});
-
   const operations = ctx.state().interface?.document.operations ?? [];
-  const endpoints: Omit<AutoBeOpenApi.IOperation, "specification">[] =
-    operations.map((it) => {
-      return {
-        method: it.method,
-        path: it.path,
-        summary: it.summary,
-        description: it.description,
-        parameters: it.parameters,
-        requestBody: it.requestBody,
-        responseBody: it.responseBody,
-      };
-    });
+  if (operations.length === 0) {
+    throw new Error(
+      "Cannot write test scenarios because these are no operations.",
+    );
+  }
 
-  const start: Date = new Date();
-
-  let completed: number = 0;
-
-  const scenarios: AutoBeTest.IScenario[][] = await Promise.all(
-    endpoints.map(async (endpoint, i, arr) => {
-      const endponits = arr.filter((_el, j) => i !== j);
-      const rows: AutoBeTest.IScenario[] = await process(
-        ctx,
-        endpoint,
-        endponits,
-        files,
-      );
-      ctx.dispatch({
-        type: "testScenario",
-        scenarios: rows,
-        total: rows.flatMap((el) => el.scenarios).length,
-        step: ctx.state().test?.step ?? 0,
-        completed,
-        created_at: start.toISOString(),
-      });
-      return rows;
-    }),
-  );
+  const exclude: IAutoBeTestScenarioApplication.IScenarioGroup[] = [];
+  let include: AutoBeOpenApi.IOperation[] = Array.from(operations);
+
+  do {
+    const matrix = divideArray({ array: include, capacity: 30 });
+
+    await Promise.all(
+      matrix.map(async (_include) => {
+        exclude.push(
+          ...(await execute(
+            ctx,
+            operations,
+            _include,
+            exclude.map((x) => x.endpoint),
+          )),
+        );
+      }),
+    );
+
+    include = include.filter((op) => {
+      if (
+        exclude.some(
+          (pg) =>
+            pg.endpoint.method === op.method && pg.endpoint.path === op.path,
+        )
+      ) {
+        return false;
+      }
+      return true;
+    });
+  } while (include.length > 0);
 
   return {
     type: "testScenario",
-    scenarios: scenarios.flat(),
-    total: scenarios.flat().flatMap((el) => el.scenarios).length,
-    step: ctx.state().test?.step ?? 0,
-    completed,
-    created_at: start.toISOString(),
-  };
+    step: ctx.state().analyze?.step ?? 0,
+    scenarios: exclude.flatMap((pg) => {
+      return pg.scenarios.map((plan) => {
+        return {
+          endpoint: pg.endpoint,
+          draft: plan.draft,
+          functionName: plan.functionName,
+          dependencies: plan.dependsOn,
+        } satisfies AutoBeTestScenarioEvent.IScenario;
+      });
+    }),
+    created_at: new Date().toISOString(),
+  } as AutoBeTestScenarioEvent;
 }
 
-async function process<Model extends ILlmSchema.Model>(
+const execute = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  endpoint: AutoBeOpenApi.IEndpoint,
-  endpoints: AutoBeOpenApi.IEndpoint[],
-  files: Record<string, string>,
-): Promise<AutoBeTest.IScenario[]> {
-  const pointer: IPointer<AutoBeTest.IScenario[] | null> = {
-    value: null,
+  ops: AutoBeOpenApi.IOperation[],
+  include: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+  exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+) => {
+  const pointer: IPointer<IAutoBeTestScenarioApplication.IScenarioGroup[]> = {
+    value: [],
   };
-
-  const agentica = new MicroAgentica({
+  const agentica: MicroAgentica<Model> = new MicroAgentica({
     model: ctx.model,
     vendor: ctx.vendor,
     config: {
-      ...(ctx.config ?? { locale: "en-US" }),
-      systemPrompt: {
-        describe: () => {
-          return "Answer only 'completion' or 'failure'.";
-        },
+      ...(ctx.config ?? {}),
+      executor: {
+        describe: null,
       },
     },
     tokenUsage: ctx.usage(),
-    histories: [
-      ...transformTestScenarioHistories(ctx.state(), endpoints, files),
-    ],
+    histories: createHistoryProperties(ops, include, exclude),
     controllers: [
       createApplication({
         model: ctx.model,
         build: (next) => {
-          pointer.value = next.scenarios;
+          pointer.value ??= [];
+          pointer.value.push(...next.scenarioGroups);
         },
       }),
     ],
   });
+  enforceToolCall(agentica);
 
-  agentica.on("request", async (event) => {
-    if (event.body.tools) event.body.tool_choice = "required";
-  });
+  await agentica.conversate(`create test scenarios.`);
+  if (pointer.value.length === 0) {
+    throw new Error("Failed to create test plans.");
+  }
+
+  return pointer.value;
+};
 
-  await agentica.conversate(
-    [
-      "Make User Scenarios for below endpoint:",
+const createHistoryProperties = (
+  operations: AutoBeOpenApi.IOperation[],
+  include: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+  exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
+) => [
+  {
+    id: v4(),
+    created_at: new Date().toISOString(),
+    type: "systemMessage",
+    text: AutoBeSystemPromptConstant.TEST_SCENARIO,
+  } satisfies IAgenticaHistoryJson.ISystemMessage,
+  {
+    id: v4(),
+    created_at: new Date().toISOString(),
+    type: "systemMessage",
+    text: [
+      "Below are the full operations. Please refer to this.",
+      "Your role is to draft all test cases for each given Operation.",
+      "It is also permissible to write multiple test codes on a single endpoint.",
+      "However, rather than meaningless tests, business logic tests should be written and an E2E test situation should be assumed.",
       "",
       "```json",
-      JSON.stringify(endpoint, null, 2),
+      JSON.stringify(
+        operations.map((el) => ({
+          path: el.path,
+          method: el.method,
+          summary: el.summary,
+        })),
+      ),
       "```",
     ].join("\n"),
-  );
-
-  if (pointer.value === null) throw new Error("Failed to make scenarios.");
-  return pointer.value;
-}
+  } satisfies IAgenticaHistoryJson.ISystemMessage,
+  {
+    id: v4(),
+    created_at: new Date().toISOString(),
+    type: "systemMessage",
+    text: [
+      "# Included in Test Plan",
+      include
+        .map((el) => `- ${el.method.toUpperCase()}: ${el.path}`)
+        .join("\n"),
+      "",
+      "# Excluded from Test Plan",
+      "These are the endpoints that have already been used in test codes generated as part of a plan group.",
+      "These endpoints do not need to be tested again.",
+      "However, it is allowed to reference or depend on these endpoints when writing test codes for other purposes.",
+      exclude
+        .map((el) => `- ${el.method.toUpperCase()}: ${el.path}`)
+        .join("\n"),
+    ].join("\n"),
+  } satisfies IAgenticaHistoryJson.ISystemMessage,
+];
 
 function createApplication<Model extends ILlmSchema.Model>(props: {
   model: Model;
-  build: (next: IMakeScenarioProps) => void;
+  build: (next: IAutoBeTestScenarioApplication.IProps) => void;
 }): IAgenticaController.IClass<Model> {
   assertSchemaModel(props.model);
 
   const application: ILlmApplication<Model> = collection[
     props.model
   ] as unknown as ILlmApplication<Model>;
+
+  application.functions[0].validate = (next: unknown): IValidation => {
+    const result: IValidation<IAutoBeTestScenarioApplication.IProps> =
+      typia.validate<IAutoBeTestScenarioApplication.IProps>(next);
+    if (result.success === false) return result;
+
+    const errors: IValidation.IError[] = [];
+    result.data.scenarioGroups.forEach((pg, i, arr) => {
+      arr.forEach((target, j) => {
+        if (
+          i !== j &&
+          target.endpoint.method === pg.endpoint.method &&
+          target.endpoint.path === pg.endpoint.path
+        ) {
+          if (
+            !errors.some(
+              (el) =>
+                el.path !== `planGroups[${j}].path` &&
+                el.value !== target.endpoint.path,
+            )
+          ) {
+            errors.push({
+              path: `planGroups[${j}].path`,
+              expected: `planGroup's {method + path} cannot duplicated.`,
+              value: target.endpoint.path,
+            });
+          }
+
+          if (
+            !errors.some(
+              (el) =>
+                el.path !== `planGroups[${j}].method` &&
+                el.value !== target.endpoint.method,
+            )
+          ) {
+            errors.push({
+              path: `planGroups[${j}].method`,
+              expected: `planGroup's {method + path} cannot duplicated.`,
+              value: target.endpoint.method,
+            });
+          }
+        }
+      });
+    });
+
+    if (errors.length !== 0) {
+      console.log(JSON.stringify(errors, null, 2), "errors");
+      return {
+        success: false,
+        errors,
+        data: next,
+      };
+    }
+
+    return result;
+  };
   return {
     protocol: "class",
-    name: "Make User Scenarios",
+    name: "Make test plans",
     application,
     execute: {
       makeScenario: (next) => {
         props.build(next);
       },
-    } satisfies IApplication,
+    } satisfies IAutoBeTestScenarioApplication,
   };
 }
 
 const claude = typia.llm.application<
-  IApplication,
+  IAutoBeTestScenarioApplication,
   "claude",
   {
     reference: true;
@@ -152,7 +255,7 @@ const claude = typia.llm.application<
 >();
 const collection = {
   chatgpt: typia.llm.application<
-    IApplication,
+    IAutoBeTestScenarioApplication,
     "chatgpt",
     { reference: true }
   >(),
@@ -160,19 +263,5 @@ const collection = {
   llama: claude,
   deepseek: claude,
   "3.1": claude,
-  "3.0": typia.llm.application<IApplication, "3.0">(),
+  "3.0": typia.llm.application<IAutoBeTestScenarioApplication, "3.0">(),
 };
-
-interface IApplication {
-  /**
-   * Make user scenarios for the given endpoints.
-   *
-   * @param props Properties containing the endpoints and user scenarios.
-   */
-  makeScenario(props: IMakeScenarioProps): void;
-}
-
-interface IMakeScenarioProps {
-  /** Array of user scenarios. */
-  scenarios: AutoBeTest.IScenario[];
-}