@autobe/agent 0.24.1 → 0.25.0

package/src/orchestrate/test/histories/transformTestScenarioReviewHistories.ts

@@ -0,0 +1,159 @@
+import { IAgenticaHistoryJson } from "@agentica/core";
+import { AutoBeOpenApi } from "@autobe/interface";
+import { StringUtil } from "@autobe/utils";
+import { v7 } from "uuid";
+
+import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
+import { AutoBeState } from "../../../context/AutoBeState";
+import { IAutoBeTestScenarioApplication } from "../structures/IAutoBeTestScenarioApplication";
+import { getReferenceIds } from "../utils/getReferenceIds";
+
+export function transformTestScenarioReviewHistories(props: {
+  state: AutoBeState;
+  instruction: string;
+  groups: IAutoBeTestScenarioApplication.IScenarioGroup[];
+}): Array<
+  IAgenticaHistoryJson.ISystemMessage | IAgenticaHistoryJson.IAssistantMessage
+> {
+  interface IRelationship {
+    endpoint: AutoBeOpenApi.IEndpoint;
+    ids: string[];
+  }
+
+  const document: AutoBeOpenApi.IDocument | undefined =
+    props.state.interface?.document;
+  if (document === undefined) {
+    throw new Error(
+      "Cannot review test scenarios because there are no operations.",
+    );
+  }
+
+  const relationships: IRelationship[] = document.operations
+    .map((o) => ({
+      endpoint: {
+        method: o.method,
+        path: o.path,
+      },
+      ids: getReferenceIds({
+        document,
+        operation: o,
+      }),
+    }))
+    .filter((v) => v.ids.length !== 0);
+
+  return [
+    {
+      id: v7(),
+      created_at: new Date().toISOString(),
+      type: "systemMessage",
+      text: AutoBeSystemPromptConstant.TEST_SCENARIO_REVIEW,
+    },
+    {
+      id: v7(),
+      created_at: new Date().toISOString(),
+      type: "assistantMessage",
+      text: StringUtil.trim`
+        ## Instructions
+
+        The following e2e-test-specific instructions were extracted by AI from
+        the user's requirements and conversations. These instructions focus
+        exclusively on test-related aspects such as test coverage priorities,
+        specific edge cases to validate, business logic verification strategies,
+        and critical user workflows that must be tested.
+        
+        Apply these instructions when reviewing test scenarios to ensure the
+        tests align with the user's testing requirements and expectations.
+        If any instructions are not relevant to the target API operations,
+        you may ignore them.
+
+        ${props.instruction}
+
+        ## Available API Operations for Reference
+
+        Below are all available API operations and interface schemas for validation purposes.
+        Match each operation with its corresponding schema.
+
+        \`\`\`json
+        ${JSON.stringify({
+          operations: document.operations,
+        })}
+        \`\`\`
+
+        ## Test Scenario Groups
+
+        Please review the following test scenario groups:
+
+        \`\`\`json
+        ${JSON.stringify(
+          props.groups.map((g) => {
+            return {
+              ...g,
+              scenarios: g.scenarios.map((s) => {
+                const requiredId: string[] = [];
+
+                s.dependencies.forEach((dep) => {
+                  document.operations.forEach((op) => {
+                    if (
+                      g.endpoint.method === op.method &&
+                      g.endpoint.path === op.path
+                    ) {
+                      requiredId.push(
+                        ...getReferenceIds({ document, operation: op }),
+                      );
+                    }
+
+                    if (
+                      op.method === dep.endpoint.method &&
+                      op.path === dep.endpoint.path
+                    ) {
+                      requiredId.push(
+                        ...getReferenceIds({ document, operation: op }),
+                      );
+                    }
+                  });
+                });
+
+                return {
+                  ...s,
+                  requiredIds:
+                    requiredId.length > 0
+                      ? Array.from(new Set(requiredId))
+                      : [],
+                };
+              }),
+            };
+          }),
+        )}
+        \`\`\`
+
+        ## Candidate Dependencies
+    
+        List of candidate dependencies extracted from path parameters and request bodies.
+
+        Apply dependency resolution to the target endpoint from "Included in Test Plan" and to dependencies found recursively from it.
+        For each required ID, locate the operation that creates the resource. Include the creator only if that operation exists in the provided operations list. Do not assume or invent operations. If no creator exists, treat the ID as an external or pre-existing input.
+
+        Dependency resolution steps:
+        1. Starting from the target endpoint, collect required IDs.
+        2. For each ID, search for a creator operation (typically POST).
+        3. If found, add it to the dependency chain in execution order and repeat for its own required IDs.
+        4. Stop when no further creators exist or are needed.
+
+        For each some_entity_id pattern, use the same approach: include a creator only when it is present in the operations list.
+    
+        Endpoint | Required IDs (MUST be created by other APIs)
+        ---------|---------------------------------------------------
+        ${relationships
+          .map((r) =>
+            [
+              `\`${r.endpoint.method} ${r.endpoint.path}\``,
+              r.ids.map((id) => `\`${id}\``).join(", "),
+            ].join(" | "),
+          )
+          .join("\n")}
+
+        Example: If an endpoint requires \`articleId\` and \`POST /articles\` exists, include it in dependencies
+      `,
+    },
+  ];
+}

package/src/orchestrate/test/histories/transformTestWriteHistories.ts

@@ -7,6 +7,7 @@ import {
   ILlmSchema,
   OpenApi,
 } from "@samchon/openapi";
+import { Singleton } from "tstl";
 import typia from "typia";
 import { v7 } from "uuid";
 
@@ -19,8 +20,11 @@ export async function transformTestWriteHistories<
   Model extends ILlmSchema.Model,
 >(
   ctx: AutoBeContext<Model>,
-  scenario: AutoBeTestScenario,
-  artifacts: IAutoBeTestScenarioArtifacts,
+  props: {
+    instruction: string;
+    scenario: AutoBeTestScenario;
+    artifacts: IAutoBeTestScenarioArtifacts;
+  },
 ): Promise<
   Array<
     IAgenticaHistoryJson.ISystemMessage | IAgenticaHistoryJson.IAssistantMessage
@@ -31,10 +35,7 @@ export async function transformTestWriteHistories<
       id: v7(),
       created_at: new Date().toISOString(),
       type: "systemMessage",
-      text: AutoBeSystemPromptConstant.TEST_WRITE.replace(
-        "{{AutoBeTestScenario}}",
-        JSON.stringify(typia.llm.parameters<AutoBeTestScenario, "llama">()),
-      ),
+      text: systemPrompt.get(),
     },
     {
       id: v7(),
@@ -45,16 +46,31 @@ export async function transformTestWriteHistories<
 
         Make e2e test functions based on the following information.
 
+        ## Instructions
+
+        The following e2e-test-specific instructions were extracted by AI from
+        the user's requirements and conversations. These instructions focus
+        exclusively on test-related aspects such as test data generation strategies,
+        assertion patterns, error handling approaches, and specific validation logic
+        that should be implemented in the test code.
+        
+        Apply these instructions when implementing the e2e test function to ensure
+        the test code aligns with the user's testing requirements and expectations.
+        If any instructions are not relevant to the target test scenario,
+        you may ignore them.
+
+        ${props.instruction}
+
         ## Function Name
 
-        The e2e test function name must be ${JSON.stringify(scenario.functionName)}.
+        The e2e test function name must be ${JSON.stringify(props.scenario.functionName)}.
 
         ## Scenario Plan
 
         Here is the scenario plan what you have to implement.
 
         \`\`\`json
-        ${JSON.stringify(scenario)}
+        ${JSON.stringify(props.scenario)}
         \`\`\`
 
         ## DTO Definitions
@@ -63,7 +79,7 @@ export async function transformTestWriteHistories<
 
         Never use the DTO definitions that are not listed here.
 
-        ${transformTestWriteHistories.structures(artifacts)}
+        ${transformTestWriteHistories.structures(props.artifacts)}
 
         ## API (SDK) Functions
 
@@ -71,14 +87,14 @@ export async function transformTestWriteHistories<
 
         Never use the functions that are not listed here.
 
-        ${transformTestWriteHistories.functional(artifacts)}
+        ${transformTestWriteHistories.functional(props.artifacts)}
 
         ## E2E Mockup Functions
 
         Just reference, and never follow this code as it is.
 
         \`\`\`json
-        ${JSON.stringify(artifacts.e2e)}
+        ${JSON.stringify(props.artifacts.e2e)}
         \`\`\`
 
         ## External Definitions
@@ -103,7 +119,7 @@ export async function transformTestWriteHistories<
         make your implementation code in the import scope.
 
         \`\`\`typescript
-        ${artifacts.template}
+        ${props.artifacts.template}
         \`\`\`
       `,
     },
@@ -144,3 +160,10 @@ export namespace transformTestWriteHistories {
     `;
   }
 }
+
+const systemPrompt = new Singleton(() =>
+  AutoBeSystemPromptConstant.TEST_WRITE.replace(
+    "{{AutoBeTestScenario}}",
+    JSON.stringify(typia.llm.parameters<AutoBeTestScenario, "llama">()),
+  ),
+);

package/src/orchestrate/test/orchestrateTest.ts

@@ -38,7 +38,7 @@ export const orchestrateTest =
       type: "testStart",
       id: v7(),
       created_at: start.toISOString(),
-      reason: props.reason,
+      reason: props.instruction,
       step: ctx.state().analyze?.step ?? 0,
     });
 
@@ -57,31 +57,37 @@ export const orchestrateTest =
       });
 
     // PLAN
-    const scenarios: AutoBeTestScenario[] = await orchestrateTestScenario(ctx);
+    const scenarios: AutoBeTestScenario[] = await orchestrateTestScenario(
+      ctx,
+      props.instruction,
+    );
     if (scenarios.length === 0)
       throw new Error("No scenarios generated. Please check the logs.");
 
     // TEST CODE
-    const written: IAutoBeTestWriteResult[] = await orchestrateTestWrite(
-      ctx,
+    const written: IAutoBeTestWriteResult[] = await orchestrateTestWrite(ctx, {
+      instruction: props.instruction,
       scenarios,
-    );
+    });
     if (written.length === 0)
       throw new Error("No test code written. Please check the logs.");
 
     const corrects: AutoBeTestValidateEvent[] = await orchestrateTestCorrect(
       ctx,
-      written.map((w) => ({
-        scenario: w.scenario,
-        artifacts: w.artifacts,
-        location: w.event.location,
-        script: w.event.final ?? w.event.draft,
-      })),
+      {
+        instruction: props.instruction,
+        functions: written.map((w) => ({
+          scenario: w.scenario,
+          artifacts: w.artifacts,
+          location: w.event.location,
+          script: w.event.final ?? w.event.draft,
+        })),
+      },
     );
 
     // DO COMPILE
     const compiler: IAutoBeCompiler = await ctx.compiler();
-    const compiled: IAutoBeTypeScriptCompileResult =
+    const compileResult: IAutoBeTypeScriptCompileResult =
       await compiler.typescript.compile({
         files: Object.fromEntries([
           ...Object.entries(
@@ -97,7 +103,7 @@ export const orchestrateTest =
       id: v7(),
       created_at: new Date().toISOString(),
       files: corrects.map((s) => s.file),
-      compiled,
+      compiled: compileResult,
       step: ctx.state().interface?.step ?? 0,
       elapsed: new Date().getTime() - start.getTime(),
     });

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -25,11 +25,14 @@ import { IAutoBeTestFunctionFailure } from "./structures/IAutoBeTestFunctionFail
 
 export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  writeResults: IAutoBeTestFunction[],
+  props: {
+    instruction: string;
+    functions: IAutoBeTestFunction[];
+  },
 ): Promise<AutoBeTestValidateEvent[]> => {
   const result: Array<AutoBeTestValidateEvent | null> =
     await executeCachedBatch(
-      writeResults.map((w) => async (promptCacheKey) => {
+      props.functions.map((w) => async (promptCacheKey) => {
         try {
           const compile = (script: string) =>
             compileTestFile(ctx, {
@@ -68,10 +71,13 @@ export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
             );
           return await predicate(
             ctx,
-            transformTestValidateEvent(y, w.artifacts),
-            [],
-            y,
-            promptCacheKey,
+            {
+              function: transformTestValidateEvent(y, w.artifacts),
+              failures: [],
+              validate: y,
+              promptCacheKey,
+              instruction: props.instruction,
+            },
             ctx.retry,
           );
         } catch {
@@ -110,44 +116,54 @@ const compileTestFile = async <Model extends ILlmSchema.Model>(
 
 const predicate = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  content: IAutoBeTestFunction,
-  failures: IAutoBeTestFunctionFailure[],
-  event: AutoBeTestValidateEvent,
-  promptCacheKey: string,
+  props: {
+    function: IAutoBeTestFunction;
+    failures: IAutoBeTestFunctionFailure[];
+    validate: AutoBeTestValidateEvent;
+    promptCacheKey: string;
+    instruction: string;
+  },
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
-  if (event.result.type === "failure") ctx.dispatch(event);
-  return event.result.type === "failure"
-    ? await correct(ctx, content, failures, event, promptCacheKey, life - 1)
-    : event;
+  if (props.validate.result.type === "failure") ctx.dispatch(props.validate);
+  return props.validate.result.type === "failure"
+    ? await correct(ctx, props, life - 1)
+    : props.validate;
 };
 
 const correct = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  content: IAutoBeTestFunction,
-  failures: IAutoBeTestFunctionFailure[],
-  validate: AutoBeTestValidateEvent,
-  promptCacheKey: string,
+  props: {
+    function: IAutoBeTestFunction;
+    failures: IAutoBeTestFunctionFailure[];
+    validate: AutoBeTestValidateEvent;
+    promptCacheKey: string;
+    instruction: string;
+  },
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
-  if (validate.result.type !== "failure") return validate;
-  else if (life < 0) return validate;
+  if (props.validate.result.type !== "failure") return props.validate;
+  else if (life < 0) return props.validate;
 
   const pointer: IPointer<IAutoBeTestCorrectApplication.IProps | null> = {
     value: null,
   };
   const { tokenUsage } = await ctx.conversate({
     source: "testCorrect",
-    histories: await transformTestCorrectHistories(ctx, content, [
-      ...failures,
-      {
-        function: content,
-        failure: validate.result,
-      },
-    ]),
+    histories: await transformTestCorrectHistories(ctx, {
+      instruction: props.instruction,
+      function: props.function,
+      failures: [
+        ...props.failures,
+        {
+          function: props.function,
+          failure: props.validate.result,
+        },
+      ],
+    }),
     controller: createController({
       model: ctx.model,
-      failure: validate.result,
+      failure: props.validate.result,
       build: (next) => {
         pointer.value = next;
       },
@@ -159,18 +175,18 @@ const correct = async <Model extends ILlmSchema.Model>(
       You don't need to explain me anything, but just fix it immediately
       without any hesitation, explanation, and questions.
     `,
-    promptCacheKey,
+    promptCacheKey: props.promptCacheKey,
   });
   if (pointer.value === null) throw new Error("Failed to correct test code.");
 
   pointer.value.revise.final = await completeTestCode(
     ctx,
-    content.artifacts,
+    props.function.artifacts,
     pointer.value.revise.final,
   );
   pointer.value.draft = await completeTestCode(
     ctx,
-    content.artifacts,
+    props.function.artifacts,
     pointer.value.draft,
   );
 
@@ -178,8 +194,8 @@ const correct = async <Model extends ILlmSchema.Model>(
     type: "testCorrect",
     id: v7(),
     created_at: new Date().toISOString(),
-    file: validate.file,
-    result: validate.result,
+    file: props.validate.file,
+    result: props.validate.result,
     tokenUsage,
     step: ctx.state().analyze?.step ?? 0,
     think: pointer.value.think,
@@ -187,26 +203,29 @@ const correct = async <Model extends ILlmSchema.Model>(
     review: pointer.value.revise?.review,
     final: pointer.value.revise?.final,
   } satisfies AutoBeTestCorrectEvent);
-  const newContent: IAutoBeTestFunction = {
-    ...content,
+  const newFunction: IAutoBeTestFunction = {
+    ...props.function,
     script: pointer.value.revise?.final ?? pointer.value.draft,
   };
   const newValidate: AutoBeTestValidateEvent = await compileTestFile(
     ctx,
-    newContent,
+    newFunction,
   );
   return predicate(
     ctx,
-    newContent,
-    [
-      ...failures,
-      {
-        function: content,
-        failure: validate.result,
-      },
-    ],
-    newValidate,
-    promptCacheKey,
+    {
+      function: newFunction,
+      failures: [
+        ...props.failures,
+        {
+          function: props.function,
+          failure: props.validate.result,
+        },
+      ],
+      validate: newValidate,
+      promptCacheKey: props.promptCacheKey,
+      instruction: props.instruction,
+    },
     life,
   );
 };

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -17,11 +17,13 @@ import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { divideArray } from "../../utils/divideArray";
 import { executeCachedBatch } from "../../utils/executeCachedBatch";
 import { transformTestScenarioHistories } from "./histories/transformTestScenarioHistories";
+import { orchestrateTestScenarioReview } from "./orchestrateTestScenarioReview";
 import { IAutoBeTestScenarioApplication } from "./structures/IAutoBeTestScenarioApplication";
 import { IAutoBeTestScenarioAuthorizationRole } from "./structures/IAutoBeTestScenarioAuthorizationRole";
 
 export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
+  instruction: string,
 ): Promise<AutoBeTestScenario[]> {
   const document: AutoBeOpenApi.IDocument | undefined =
     ctx.state().interface?.document;
@@ -31,8 +33,6 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
     );
   }
 
-  Object.entries(document.components.schemas);
-
   const dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation> =
     new HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>(
       document.operations.map(
@@ -63,6 +63,10 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
     total: document.operations.length,
     completed: 0,
   };
+  const reviewProgress: AutoBeProgressEventBase = {
+    total: document.operations.length,
+    completed: 0,
+  };
   const exclude: IAutoBeTestScenarioApplication.IScenarioGroup[] = [];
   let include: AutoBeOpenApi.IOperation[] = [...document.operations];
   let trial: number = 0;
@@ -82,7 +86,9 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
             include,
             exclude: exclude.map((x) => x.endpoint),
             progress,
+            reviewProgress,
             promptCacheKey,
+            instruction,
           })),
         );
       }),
@@ -99,6 +105,7 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
       return true;
     });
     progress.total = include.length + exclude.length;
+    reviewProgress.total = include.length + exclude.length;
   } while (include.length > 0 && ++trial < ctx.retry);
 
   return exclude.flatMap((pg) => {
@@ -122,7 +129,9 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
     include: AutoBeOpenApi.IOperation[];
     exclude: AutoBeOpenApi.IEndpoint[];
     progress: AutoBeProgressEventBase;
+    reviewProgress: AutoBeProgressEventBase;
     promptCacheKey: string;
+    instruction: string;
   },
 ): Promise<IAutoBeTestScenarioApplication.IScenarioGroup[]> => {
   const pointer: IPointer<IAutoBeTestScenarioApplication.IScenarioGroup[]> = {
@@ -134,12 +143,13 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
   try {
     const { tokenUsage } = await ctx.conversate({
       source: "testScenarios",
-      histories: transformTestScenarioHistories(
-        ctx.state(),
-        props.document,
-        props.include,
-        props.exclude,
-      ),
+      histories: transformTestScenarioHistories({
+        state: ctx.state(),
+        document: props.document,
+        include: props.include,
+        exclude: props.exclude,
+        instruction: props.instruction,
+      }),
       controller: createController({
         model: ctx.model,
         endpointNotFound: props.endpointNotFound,
@@ -187,7 +197,11 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
       step: ctx.state().interface?.step ?? 0,
       created_at: new Date().toISOString(),
     });
-    return pointer.value;
+    return await orchestrateTestScenarioReview(ctx, {
+      instruction: props.instruction,
+      groups: pointer.value,
+      progress: props.reviewProgress,
+    });
   } catch {
     return [];
   }