@autobe/agent 0.24.2 → 0.25.1

package/src/orchestrate/test/histories/transformTestWriteHistories.ts

@@ -7,6 +7,7 @@ import {
   ILlmSchema,
   OpenApi,
 } from "@samchon/openapi";
+import { Singleton } from "tstl";
 import typia from "typia";
 import { v7 } from "uuid";
 
@@ -19,8 +20,11 @@ export async function transformTestWriteHistories<
   Model extends ILlmSchema.Model,
 >(
   ctx: AutoBeContext<Model>,
-  scenario: AutoBeTestScenario,
-  artifacts: IAutoBeTestScenarioArtifacts,
+  props: {
+    instruction: string;
+    scenario: AutoBeTestScenario;
+    artifacts: IAutoBeTestScenarioArtifacts;
+  },
 ): Promise<
   Array<
     IAgenticaHistoryJson.ISystemMessage | IAgenticaHistoryJson.IAssistantMessage
@@ -31,10 +35,7 @@ export async function transformTestWriteHistories<
       id: v7(),
       created_at: new Date().toISOString(),
       type: "systemMessage",
-      text: AutoBeSystemPromptConstant.TEST_WRITE.replace(
-        "{{AutoBeTestScenario}}",
-        JSON.stringify(typia.llm.parameters<AutoBeTestScenario, "llama">()),
-      ),
+      text: systemPrompt.get(),
     },
     {
       id: v7(),
@@ -45,16 +46,31 @@ export async function transformTestWriteHistories<
 
         Make e2e test functions based on the following information.
 
+        ## Instructions
+
+        The following e2e-test-specific instructions were extracted by AI from
+        the user's requirements and conversations. These instructions focus
+        exclusively on test-related aspects such as test data generation strategies,
+        assertion patterns, error handling approaches, and specific validation logic
+        that should be implemented in the test code.
+        
+        Apply these instructions when implementing the e2e test function to ensure
+        the test code aligns with the user's testing requirements and expectations.
+        If any instructions are not relevant to the target test scenario,
+        you may ignore them.
+
+        ${props.instruction}
+
         ## Function Name
 
-        The e2e test function name must be ${JSON.stringify(scenario.functionName)}.
+        The e2e test function name must be ${JSON.stringify(props.scenario.functionName)}.
 
         ## Scenario Plan
 
         Here is the scenario plan what you have to implement.
 
         \`\`\`json
-        ${JSON.stringify(scenario)}
+        ${JSON.stringify(props.scenario)}
         \`\`\`
 
         ## DTO Definitions
@@ -63,7 +79,7 @@ export async function transformTestWriteHistories<
 
         Never use the DTO definitions that are not listed here.
 
-        ${transformTestWriteHistories.structures(artifacts)}
+        ${transformTestWriteHistories.structures(props.artifacts)}
 
         ## API (SDK) Functions
 
@@ -71,14 +87,14 @@ export async function transformTestWriteHistories<
 
         Never use the functions that are not listed here.
 
-        ${transformTestWriteHistories.functional(artifacts)}
+        ${transformTestWriteHistories.functional(props.artifacts)}
 
         ## E2E Mockup Functions
 
         Just reference, and never follow this code as it is.
 
         \`\`\`json
-        ${JSON.stringify(artifacts.e2e)}
+        ${JSON.stringify(props.artifacts.e2e)}
         \`\`\`
 
         ## External Definitions
@@ -103,7 +119,7 @@ export async function transformTestWriteHistories<
         make your implementation code in the import scope.
 
         \`\`\`typescript
-        ${artifacts.template}
+        ${props.artifacts.template}
         \`\`\`
       `,
     },
@@ -144,3 +160,10 @@ export namespace transformTestWriteHistories {
     `;
   }
 }
+
+const systemPrompt = new Singleton(() =>
+  AutoBeSystemPromptConstant.TEST_WRITE.replace(
+    "{{AutoBeTestScenario}}",
+    JSON.stringify(typia.llm.parameters<AutoBeTestScenario, "llama">()),
+  ),
+);

package/src/orchestrate/test/orchestrateTest.ts

@@ -38,7 +38,7 @@ export const orchestrateTest =
       type: "testStart",
       id: v7(),
       created_at: start.toISOString(),
-      reason: props.reason,
+      reason: props.instruction,
       step: ctx.state().analyze?.step ?? 0,
     });
 
@@ -57,31 +57,37 @@ export const orchestrateTest =
       });
 
     // PLAN
-    const scenarios: AutoBeTestScenario[] = await orchestrateTestScenario(ctx);
+    const scenarios: AutoBeTestScenario[] = await orchestrateTestScenario(
+      ctx,
+      props.instruction,
+    );
     if (scenarios.length === 0)
       throw new Error("No scenarios generated. Please check the logs.");
 
     // TEST CODE
-    const written: IAutoBeTestWriteResult[] = await orchestrateTestWrite(
-      ctx,
+    const written: IAutoBeTestWriteResult[] = await orchestrateTestWrite(ctx, {
+      instruction: props.instruction,
       scenarios,
-    );
+    });
     if (written.length === 0)
       throw new Error("No test code written. Please check the logs.");
 
     const corrects: AutoBeTestValidateEvent[] = await orchestrateTestCorrect(
       ctx,
-      written.map((w) => ({
-        scenario: w.scenario,
-        artifacts: w.artifacts,
-        location: w.event.location,
-        script: w.event.final ?? w.event.draft,
-      })),
+      {
+        instruction: props.instruction,
+        functions: written.map((w) => ({
+          scenario: w.scenario,
+          artifacts: w.artifacts,
+          location: w.event.location,
+          script: w.event.final ?? w.event.draft,
+        })),
+      },
     );
 
     // DO COMPILE
     const compiler: IAutoBeCompiler = await ctx.compiler();
-    const compiled: IAutoBeTypeScriptCompileResult =
+    const compileResult: IAutoBeTypeScriptCompileResult =
       await compiler.typescript.compile({
         files: Object.fromEntries([
           ...Object.entries(
@@ -97,7 +103,7 @@ export const orchestrateTest =
       id: v7(),
       created_at: new Date().toISOString(),
       files: corrects.map((s) => s.file),
-      compiled,
+      compiled: compileResult,
       step: ctx.state().interface?.step ?? 0,
       elapsed: new Date().getTime() - start.getTime(),
     });

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -25,11 +25,14 @@ import { IAutoBeTestFunctionFailure } from "./structures/IAutoBeTestFunctionFail
 
 export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  writeResults: IAutoBeTestFunction[],
+  props: {
+    instruction: string;
+    functions: IAutoBeTestFunction[];
+  },
 ): Promise<AutoBeTestValidateEvent[]> => {
   const result: Array<AutoBeTestValidateEvent | null> =
     await executeCachedBatch(
-      writeResults.map((w) => async (promptCacheKey) => {
+      props.functions.map((w) => async (promptCacheKey) => {
         try {
           const compile = (script: string) =>
             compileTestFile(ctx, {
@@ -68,10 +71,13 @@ export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
             );
           return await predicate(
             ctx,
-            transformTestValidateEvent(y, w.artifacts),
-            [],
-            y,
-            promptCacheKey,
+            {
+              function: transformTestValidateEvent(y, w.artifacts),
+              failures: [],
+              validate: y,
+              promptCacheKey,
+              instruction: props.instruction,
+            },
             ctx.retry,
           );
         } catch {
@@ -110,44 +116,54 @@ const compileTestFile = async <Model extends ILlmSchema.Model>(
 
 const predicate = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  content: IAutoBeTestFunction,
-  failures: IAutoBeTestFunctionFailure[],
-  event: AutoBeTestValidateEvent,
-  promptCacheKey: string,
+  props: {
+    function: IAutoBeTestFunction;
+    failures: IAutoBeTestFunctionFailure[];
+    validate: AutoBeTestValidateEvent;
+    promptCacheKey: string;
+    instruction: string;
+  },
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
-  if (event.result.type === "failure") ctx.dispatch(event);
-  return event.result.type === "failure"
-    ? await correct(ctx, content, failures, event, promptCacheKey, life - 1)
-    : event;
+  if (props.validate.result.type === "failure") ctx.dispatch(props.validate);
+  return props.validate.result.type === "failure"
+    ? await correct(ctx, props, life - 1)
+    : props.validate;
 };
 
 const correct = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  content: IAutoBeTestFunction,
-  failures: IAutoBeTestFunctionFailure[],
-  validate: AutoBeTestValidateEvent,
-  promptCacheKey: string,
+  props: {
+    function: IAutoBeTestFunction;
+    failures: IAutoBeTestFunctionFailure[];
+    validate: AutoBeTestValidateEvent;
+    promptCacheKey: string;
+    instruction: string;
+  },
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
-  if (validate.result.type !== "failure") return validate;
-  else if (life < 0) return validate;
+  if (props.validate.result.type !== "failure") return props.validate;
+  else if (life < 0) return props.validate;
 
   const pointer: IPointer<IAutoBeTestCorrectApplication.IProps | null> = {
     value: null,
   };
   const { tokenUsage } = await ctx.conversate({
     source: "testCorrect",
-    histories: await transformTestCorrectHistories(ctx, content, [
-      ...failures,
-      {
-        function: content,
-        failure: validate.result,
-      },
-    ]),
+    histories: await transformTestCorrectHistories(ctx, {
+      instruction: props.instruction,
+      function: props.function,
+      failures: [
+        ...props.failures,
+        {
+          function: props.function,
+          failure: props.validate.result,
+        },
+      ],
+    }),
     controller: createController({
       model: ctx.model,
-      failure: validate.result,
+      failure: props.validate.result,
       build: (next) => {
         pointer.value = next;
       },
@@ -159,18 +175,19 @@ const correct = async <Model extends ILlmSchema.Model>(
       You don't need to explain me anything, but just fix it immediately
       without any hesitation, explanation, and questions.
     `,
-    promptCacheKey,
+    promptCacheKey: props.promptCacheKey,
   });
   if (pointer.value === null) throw new Error("Failed to correct test code.");
 
-  pointer.value.revise.final = await completeTestCode(
-    ctx,
-    content.artifacts,
-    pointer.value.revise.final,
-  );
+  if (pointer.value.revise.final)
+    pointer.value.revise.final = await completeTestCode(
+      ctx,
+      props.function.artifacts,
+      pointer.value.revise.final,
+    );
   pointer.value.draft = await completeTestCode(
     ctx,
-    content.artifacts,
+    props.function.artifacts,
     pointer.value.draft,
   );
 
@@ -178,35 +195,38 @@ const correct = async <Model extends ILlmSchema.Model>(
     type: "testCorrect",
     id: v7(),
     created_at: new Date().toISOString(),
-    file: validate.file,
-    result: validate.result,
+    file: props.validate.file,
+    result: props.validate.result,
     tokenUsage,
     step: ctx.state().analyze?.step ?? 0,
     think: pointer.value.think,
     draft: pointer.value.draft,
     review: pointer.value.revise?.review,
-    final: pointer.value.revise?.final,
+    final: pointer.value.revise?.final ?? undefined,
   } satisfies AutoBeTestCorrectEvent);
-  const newContent: IAutoBeTestFunction = {
-    ...content,
+  const newFunction: IAutoBeTestFunction = {
+    ...props.function,
     script: pointer.value.revise?.final ?? pointer.value.draft,
   };
   const newValidate: AutoBeTestValidateEvent = await compileTestFile(
     ctx,
-    newContent,
+    newFunction,
   );
   return predicate(
     ctx,
-    newContent,
-    [
-      ...failures,
-      {
-        function: content,
-        failure: validate.result,
-      },
-    ],
-    newValidate,
-    promptCacheKey,
+    {
+      function: newFunction,
+      failures: [
+        ...props.failures,
+        {
+          function: props.function,
+          failure: props.validate.result,
+        },
+      ],
+      validate: newValidate,
+      promptCacheKey: props.promptCacheKey,
+      instruction: props.instruction,
+    },
     life,
   );
 };

package/src/orchestrate/test/orchestrateTestCorrectInvalidRequest.ts

@@ -10,6 +10,7 @@ import { v7 } from "uuid";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
+import { completeTestCode } from "./compile/completeTestCode";
 import { transformTestCorrectInvalidRequestHistories } from "./histories/transformTestCorrectInvalidRequestHistories";
 import { IAutoBeTestCorrectInvalidRequestApplication } from "./structures/IAutoBeTestCorrectInvalidRequestApplication";
 import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
@@ -82,6 +83,17 @@ const correct = async <Model extends ILlmSchema.Model>(
   if (pointer.value === null) throw new Error("Failed to correct test code.");
   else if (pointer.value === false) return event; // other's responsibility
 
+  if (pointer.value.revise.final)
+    pointer.value.revise.final = await completeTestCode(
+      ctx,
+      write.artifacts,
+      pointer.value.revise.final,
+    );
+  pointer.value.draft = await completeTestCode(
+    ctx,
+    write.artifacts,
+    pointer.value.draft,
+  );
   ctx.dispatch({
     type: "testCorrect",
     id: v7(),
@@ -97,7 +109,7 @@ const correct = async <Model extends ILlmSchema.Model>(
     think: pointer.value.think,
     draft: pointer.value.draft,
     review: pointer.value.revise?.review,
-    final: pointer.value.revise?.final,
+    final: pointer.value.revise?.final ?? undefined,
   } satisfies AutoBeTestCorrectEvent);
   const newWrite: IAutoBeTestFunction = {
     artifacts: write.artifacts,

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -5,13 +5,18 @@ import {
   AutoBeProgressEventBase,
   AutoBeTestScenario,
 } from "@autobe/interface";
-import { AutoBeEndpointComparator, MapUtil, StringUtil } from "@autobe/utils";
+import {
+  AutoBeOpenApiEndpointComparator,
+  MapUtil,
+  StringUtil,
+} from "@autobe/utils";
 import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
 import { HashMap, IPointer, Pair } from "tstl";
 import typia from "typia";
 import { NamingConvention } from "typia/lib/utils/NamingConvention";
 import { v7 } from "uuid";
 
+import { AutoBeConfigConstant } from "../../constants/AutoBeConfigConstant";
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { divideArray } from "../../utils/divideArray";
@@ -23,6 +28,8 @@ import { IAutoBeTestScenarioAuthorizationRole } from "./structures/IAutoBeTestSc
 
 export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
+  instruction: string,
+  capacity: number = AutoBeConfigConstant.INTERFACE_CAPACITY,
 ): Promise<AutoBeTestScenario[]> {
   const document: AutoBeOpenApi.IDocument | undefined =
     ctx.state().interface?.document;
@@ -44,8 +51,8 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
             op,
           ),
       ),
-      AutoBeEndpointComparator.hashCode,
-      AutoBeEndpointComparator.equals,
+      AutoBeOpenApiEndpointComparator.hashCode,
+      AutoBeOpenApiEndpointComparator.equals,
     );
 
   const endpointNotFound: string = [
@@ -73,7 +80,7 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   do {
     const matrix: AutoBeOpenApi.IOperation[][] = divideArray({
       array: include,
-      capacity: 4,
+      capacity: capacity ?? AutoBeConfigConstant.INTERFACE_CAPACITY,
     });
     await executeCachedBatch(
       matrix.map((include) => async (promptCacheKey) => {
@@ -87,6 +94,7 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
             progress,
             reviewProgress,
             promptCacheKey,
+            instruction,
           })),
         );
       }),
@@ -129,6 +137,7 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
     progress: AutoBeProgressEventBase;
     reviewProgress: AutoBeProgressEventBase;
     promptCacheKey: string;
+    instruction: string;
   },
 ): Promise<IAutoBeTestScenarioApplication.IScenarioGroup[]> => {
   const pointer: IPointer<IAutoBeTestScenarioApplication.IScenarioGroup[]> = {
@@ -140,12 +149,13 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
   try {
     const { tokenUsage } = await ctx.conversate({
       source: "testScenarios",
-      histories: transformTestScenarioHistories(
-        ctx.state(),
-        props.document,
-        props.include,
-        props.exclude,
-      ),
+      histories: transformTestScenarioHistories({
+        state: ctx.state(),
+        document: props.document,
+        include: props.include,
+        exclude: props.exclude,
+        instruction: props.instruction,
+      }),
       controller: createController({
         model: ctx.model,
         endpointNotFound: props.endpointNotFound,
@@ -193,11 +203,11 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
       step: ctx.state().interface?.step ?? 0,
       created_at: new Date().toISOString(),
     });
-    return await orchestrateTestScenarioReview(
-      ctx,
-      pointer.value,
-      props.reviewProgress,
-    );
+    return await orchestrateTestScenarioReview(ctx, {
+      instruction: props.instruction,
+      groups: pointer.value,
+      progress: props.reviewProgress,
+    });
   } catch {
     return [];
   }
@@ -401,8 +411,8 @@ const uniqueScenarioGroups = (
 ): IAutoBeTestScenarioApplication.IScenarioGroup[] =>
   new HashMap(
     groups.map((g) => new Pair(g.endpoint, g)),
-    AutoBeEndpointComparator.hashCode,
-    AutoBeEndpointComparator.equals,
+    AutoBeOpenApiEndpointComparator.hashCode,
+    AutoBeOpenApiEndpointComparator.equals,
   )
     .toJSON()
     .map((it) => it.second);

package/src/orchestrate/test/orchestrateTestScenarioReview.ts

@@ -1,6 +1,6 @@
 import { IAgenticaController } from "@agentica/core";
 import { AutoBeProgressEventBase, AutoBeTestScenario } from "@autobe/interface";
-import { AutoBeEndpointComparator } from "@autobe/utils";
+import { AutoBeOpenApiEndpointComparator } from "@autobe/utils";
 import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
 import { HashMap, IPointer, Pair } from "tstl";
 import typia from "typia";
@@ -16,21 +16,26 @@ export async function orchestrateTestScenarioReview<
   Model extends ILlmSchema.Model,
 >(
   ctx: AutoBeContext<Model>,
-  groups: IAutoBeTestScenarioApplication.IScenarioGroup[],
-  progress: AutoBeProgressEventBase,
+  props: {
+    instruction: string;
+    groups: IAutoBeTestScenarioApplication.IScenarioGroup[];
+    progress: AutoBeProgressEventBase;
+  },
 ): Promise<IAutoBeTestScenarioApplication.IScenarioGroup[]> {
   const res: IAutoBeTestScenarioApplication.IScenarioGroup[] = await review(
     ctx,
-    groups,
-    progress,
+    props,
   );
   return res;
 }
 
 async function review<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  groups: IAutoBeTestScenarioApplication.IScenarioGroup[],
-  progress: AutoBeProgressEventBase,
+  props: {
+    instruction: string;
+    groups: IAutoBeTestScenarioApplication.IScenarioGroup[];
+    progress: AutoBeProgressEventBase;
+  },
 ): Promise<IAutoBeTestScenarioApplication.IScenarioGroup[]> {
   try {
     const pointer: IPointer<IAutoBeTestScenarioReviewApplication.IProps | null> =
@@ -42,9 +47,13 @@ async function review<Model extends ILlmSchema.Model>(
       controller: createController({
         model: ctx.model,
         pointer,
-        originalGroups: groups,
+        originalGroups: props.groups,
+      }),
+      histories: transformTestScenarioReviewHistories({
+        state: ctx.state(),
+        groups: props.groups,
+        instruction: props.instruction,
       }),
-      histories: transformTestScenarioReviewHistories(ctx, groups),
       enforceFunctionCall: true,
       message: "Review the Test Scenario.",
     });
@@ -53,17 +62,17 @@ async function review<Model extends ILlmSchema.Model>(
       throw new Error("Failed to get review result.");
     }
 
-    progress.total = Math.max(
-      progress.total,
-      (progress.completed += pointer.value.scenarioGroups.length),
+    props.progress.total = Math.max(
+      props.progress.total,
+      (props.progress.completed += pointer.value.scenarioGroups.length),
     );
 
     ctx.dispatch({
       type: "testScenariosReview",
       id: v7(),
       tokenUsage,
-      total: progress.total,
-      completed: progress.completed,
+      total: props.progress.total,
+      completed: props.progress.completed,
       scenarios: pointer.value.scenarioGroups
         .map((group) => {
           return group.scenarios.map((s) => {
@@ -77,10 +86,14 @@ async function review<Model extends ILlmSchema.Model>(
       step: ctx.state().interface?.step ?? 0,
       created_at: new Date().toISOString(),
     });
-    return pointer.value.scenarioGroups;
+    // @todo michael: need to investigate scenario removal more gracefully
+    return pointer.value.pass
+      ? // || pointer.value.scenarioGroups.length < props.groups.length
+        props.groups
+      : pointer.value.scenarioGroups;
   } catch {
-    progress.completed += groups.length;
-    return groups;
+    props.progress.completed += props.groups.length;
+    return props.groups;
   }
 }
 
@@ -159,8 +172,8 @@ const uniqueScenarioGroups = (
 ): IAutoBeTestScenarioApplication.IScenarioGroup[] =>
   new HashMap(
     groups.map((g) => new Pair(g.endpoint, g)),
-    AutoBeEndpointComparator.hashCode,
-    AutoBeEndpointComparator.equals,
+    AutoBeOpenApiEndpointComparator.hashCode,
+    AutoBeOpenApiEndpointComparator.equals,
   )
     .toJSON()
     .map((it) => it.second);