@autobe/agent 0.22.0 → 0.23.0

package/src/orchestrate/test/histories/transformTestScenarioHistories.ts

@@ -6,20 +6,36 @@ import { v7 } from "uuid";
 import { AutoBeSystemPromptConstant } from "../../../constants/AutoBeSystemPromptConstant";
 import { AutoBeState } from "../../../context/AutoBeState";
 import { IAutoBeTestScenarioAuthorizationRole } from "../structures/IAutoBeTestScenarioAuthorizationRole";
+import { getReferenceIds } from "../utils/getReferenceIds";
 
 export const transformTestScenarioHistories = (
   state: AutoBeState,
-  entire: AutoBeOpenApi.IOperation[],
+  document: AutoBeOpenApi.IDocument,
   include: AutoBeOpenApi.IOperation[],
   exclude: Pick<AutoBeOpenApi.IOperation, "method" | "path">[],
 ): Array<
   IAgenticaHistoryJson.IAssistantMessage | IAgenticaHistoryJson.ISystemMessage
 > => {
+  interface IRelationship {
+    endpoint: AutoBeOpenApi.IEndpoint;
+    ids: string[];
+  }
   const authorizations: AutoBeInterfaceAuthorization[] =
     state.interface?.authorizations ?? [];
-
   const authorizationRoles: Map<string, IAutoBeTestScenarioAuthorizationRole> =
     new Map();
+  const relationships: IRelationship[] = document.operations
+    .map((o) => ({
+      endpoint: {
+        method: o.method,
+        path: o.path,
+      },
+      ids: getReferenceIds({
+        document,
+        operation: o,
+      }),
+    }))
+    .filter((v) => v.ids.length !== 0);
 
   for (const authorization of authorizations) {
     for (const op of authorization.operations) {
@@ -50,27 +66,22 @@ export const transformTestScenarioHistories = (
       created_at: new Date().toISOString(),
       type: "assistantMessage",
       text: StringUtil.trim`
-        # Operations
+        # API Operations
+
+        Below are the complete API operations.
+        Use this information to understand capabilities and dependency relationships.
+        Generate scenarios only for endpoints listed in "Included in Test Plan".
+        Other operations may be referenced as dependencies only.
 
-        Below are the full operations. Please refer to this.
-        Your role is to draft all test cases for each given Operation.
-        It is also permissible to write multiple test codes on a single endpoint.
-        However, rather than meaningless tests, business logic tests should be written and an E2E test situation should be assumed.
+        You may write multiple scenarios for a single included endpoint.
+        Focus on business-logic-oriented E2E flows rather than trivial CRUD.
 
-        Please carefully analyze each operation to identify all dependencies required for testing.
-        For example, if you want to test liking and then deleting a post,
-        you might think to test post creation, liking, and unlike operations.
-        However, even if not explicitly mentioned, user registration or login are essential prerequisites.
+        Please analyze the operations to identify all dependencies required for testing.
         Pay close attention to IDs and related values in the API,
         and ensure you identify all dependencies between endpoints.
 
         \`\`\`json
-        ${JSON.stringify(
-          entire.map((el) => ({
-            ...el,
-            specification: undefined,
-          })),
-        )}
+        ${JSON.stringify({ operations: document.operations })}
         \`\`\`
       `,
     } satisfies IAgenticaHistoryJson.IAssistantMessage,
@@ -86,11 +97,16 @@ export const transformTestScenarioHistories = (
         When testing endpoints that require authentication, ensure you include the corresponding 
         join/login operations in your test scenario to establish proper authentication context.
 
+        Generate test scenarios only for these included endpoints. Do not create scenarios for excluded endpoints. Operations not listed here may be used only as dependencies.
+
         ${include
           .map((el, i) => {
             const roles = Array.from(authorizationRoles.values()).filter(
               (role) => role.name === el.authorizationRole,
             );
+
+            const requiredIds = getReferenceIds({ document, operation: el });
+
             return StringUtil.trim`
               ## ${i + 1}. ${el.method.toUpperCase()} ${el.path}
 
@@ -108,6 +124,14 @@ export const transformTestScenarioHistories = (
                       .join("\n")
                   : "- None"
               }
+
+              Required IDs:
+              
+              - ${
+                requiredIds.length > 0
+                  ? requiredIds.map((id) => `\`${id}\``).join(", ")
+                  : "None"
+              }
             `;
           })
           .join("\n")}
@@ -123,5 +147,39 @@ export const transformTestScenarioHistories = (
           .join("\n")}
       `,
     } satisfies IAgenticaHistoryJson.IAssistantMessage,
+    {
+      id: v7(),
+      created_at: new Date().toISOString(),
+      type: "assistantMessage",
+      text: StringUtil.trim`
+        # Candidate Dependencies
+    
+        List of candidate dependencies extracted from path parameters and request bodies.
+
+        Apply dependency resolution to the target endpoint from "Included in Test Plan" and to dependencies found recursively from it.
+        For each required ID, locate the operation that creates the resource. Include the creator only if that operation exists in the provided operations list. Do not assume or invent operations. If no creator exists, treat the ID as an external or pre-existing input.
+
+        Dependency resolution steps:
+        1. Starting from the target endpoint, collect required IDs.
+        2. For each ID, search for a creator operation (typically POST).
+        3. If found, add it to the dependency chain in execution order and repeat for its own required IDs.
+        4. Stop when no further creators exist or are needed.
+
+        For each some_entity_id pattern, use the same approach: include a creator only when it is present in the operations list.
+    
+        Endpoint | Required IDs (MUST be created by other APIs)
+        ---------|---------------------------------------------------
+        ${relationships
+          .map((r) =>
+            [
+              `\`${r.endpoint.method} ${r.endpoint.path}\``,
+              r.ids.map((id) => `\`${id}\``).join(", "),
+            ].join(" | "),
+          )
+          .join("\n")}
+
+        Example: If an endpoint requires \`articleId\` and \`POST /articles\` exists, include it in dependencies
+      `,
+    } satisfies IAgenticaHistoryJson.IAssistantMessage,
   ];
 };

package/src/orchestrate/test/histories/transformTestValidateEvent.ts

@@ -0,0 +1,14 @@
+import { AutoBeTestValidateEvent } from "@autobe/interface";
+
+import { IAutoBeTestFunction } from "../structures/IAutoBeTestFunction";
+import { IAutoBeTestScenarioArtifacts } from "../structures/IAutoBeTestScenarioArtifacts";
+
+export const transformTestValidateEvent = (
+  event: AutoBeTestValidateEvent,
+  artifacts: IAutoBeTestScenarioArtifacts,
+): IAutoBeTestFunction => ({
+  scenario: event.file.scenario,
+  artifacts,
+  location: event.file.location,
+  script: event.file.content,
+});

package/src/orchestrate/test/orchestrateTest.ts

@@ -58,22 +58,26 @@ export const orchestrateTest =
 
     // PLAN
     const scenarios: AutoBeTestScenario[] = await orchestrateTestScenario(ctx);
+    if (scenarios.length === 0)
+      throw new Error("No scenarios generated. Please check the logs.");
 
     // TEST CODE
     const written: IAutoBeTestWriteResult[] = await orchestrateTestWrite(
       ctx,
       scenarios,
     );
+    if (written.length === 0)
+      throw new Error("No test code written. Please check the logs.");
+
     const corrects: AutoBeTestValidateEvent[] = await orchestrateTestCorrect(
       ctx,
-      written,
+      written.map((w) => ({
+        scenario: w.scenario,
+        artifacts: w.artifacts,
+        location: w.event.location,
+        script: w.event.final ?? w.event.draft,
+      })),
     );
-    // for (const c of corrects)
-    //   if (c.result.type !== "success")
-    //     c.file.content = c.file.content
-    //       .split("\n")
-    //       .map((s) => `// ${s}`)
-    //       .join("\n");
 
     // DO COMPILE
     const compiler: IAutoBeCompiler = await ctx.compiler();

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -1,11 +1,12 @@
 import { IAgenticaController } from "@agentica/core";
 import {
+  AutoBeTestCorrectEvent,
   AutoBeTestValidateEvent,
   IAutoBeCompiler,
   IAutoBeTypeScriptCompileResult,
 } from "@autobe/interface";
 import { StringUtil } from "@autobe/utils";
-import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
+import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
 import { v7 } from "uuid";
@@ -13,38 +14,64 @@ import { v7 } from "uuid";
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { executeCachedBatch } from "../../utils/executeCachedBatch";
+import { orchestrateCommonCorrectCasting } from "../common/orchestrateCommonCorrectCasting";
 import { completeTestCode } from "./compile/completeTestCode";
 import { transformTestCorrectHistories } from "./histories/transformTestCorrectHistories";
+import { transformTestValidateEvent } from "./histories/transformTestValidateEvent";
+import { orchestrateTestCorrectInvalidRequest } from "./orchestrateTestCorrectInvalidRequest";
 import { IAutoBeTestCorrectApplication } from "./structures/IAutoBeTestCorrectApplication";
 import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
 import { IAutoBeTestFunctionFailure } from "./structures/IAutoBeTestFunctionFailure";
-import { IAutoBeTestScenarioArtifacts } from "./structures/IAutoBeTestScenarioArtifacts";
-import { IAutoBeTestWriteResult } from "./structures/IAutoBeTestWriteResult";
 
 export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  writeResult: IAutoBeTestWriteResult[],
+  writeResults: IAutoBeTestFunction[],
 ): Promise<AutoBeTestValidateEvent[]> => {
   const result: Array<AutoBeTestValidateEvent | null> =
     await executeCachedBatch(
-      writeResult.map((w) => async () => {
+      writeResults.map((w) => async (promptCacheKey) => {
         try {
-          const event: AutoBeTestValidateEvent = await compile(ctx, {
-            artifacts: w.artifacts,
-            scenario: w.scenario,
-            location: w.event.location,
-            script: w.event.final ?? w.event.draft,
-          });
+          const compile = (script: string) =>
+            compileTestFile(ctx, {
+              ...w,
+              script,
+            });
+          const x: AutoBeTestValidateEvent =
+            await orchestrateTestCorrectInvalidRequest(ctx, compile, w);
+          const y: AutoBeTestValidateEvent =
+            await orchestrateCommonCorrectCasting(
+              ctx,
+              {
+                source: "testCorrect",
+                validate: compile,
+                correct: (next) =>
+                  ({
+                    type: "testCorrect",
+                    id: v7(),
+                    created_at: new Date().toISOString(),
+                    file: {
+                      scenario: w.scenario,
+                      location: w.location,
+                      content: next.final ?? next.draft,
+                    },
+                    result: next.failure,
+                    tokenUsage: next.tokenUsage,
+                    think: next.think,
+                    draft: next.draft,
+                    review: next.review,
+                    final: next.final,
+                    step: ctx.state().analyze?.step ?? 0,
+                  }) satisfies AutoBeTestCorrectEvent,
+                script: (event) => event.file.content,
+              },
+              x.file.content,
+            );
           return await predicate(
             ctx,
-            {
-              artifacts: w.artifacts,
-              scenario: w.scenario,
-              location: w.event.location,
-              script: w.event.final ?? w.event.draft,
-            },
+            transformTestValidateEvent(y, w.artifacts),
             [],
-            event,
+            y,
+            promptCacheKey,
             ctx.retry,
           );
         } catch {
@@ -55,7 +82,7 @@ export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   return result.filter((r) => r !== null);
 };
 
-const compile = async <Model extends ILlmSchema.Model>(
+const compileTestFile = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   func: IAutoBeTestFunction,
 ): Promise<AutoBeTestValidateEvent> => {
@@ -86,11 +113,12 @@ const predicate = async <Model extends ILlmSchema.Model>(
   content: IAutoBeTestFunction,
   failures: IAutoBeTestFunctionFailure[],
   event: AutoBeTestValidateEvent,
+  promptCacheKey: string,
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
   if (event.result.type === "failure") ctx.dispatch(event);
   return event.result.type === "failure"
-    ? await correct(ctx, content, failures, event, life - 1)
+    ? await correct(ctx, content, failures, event, promptCacheKey, life - 1)
     : event;
 };
 
@@ -99,10 +127,11 @@ const correct = async <Model extends ILlmSchema.Model>(
   content: IAutoBeTestFunction,
   failures: IAutoBeTestFunctionFailure[],
   validate: AutoBeTestValidateEvent,
+  promptCacheKey: string,
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
   if (validate.result.type !== "failure") return validate;
-  else if (--life <= 0) return validate;
+  else if (life < 0) return validate;
 
   const pointer: IPointer<IAutoBeTestCorrectApplication.IProps | null> = {
     value: null,
@@ -118,7 +147,7 @@ const correct = async <Model extends ILlmSchema.Model>(
     ]),
     controller: createController({
       model: ctx.model,
-      artifacts: content.artifacts,
+      failure: validate.result,
       build: (next) => {
         pointer.value = next;
       },
@@ -130,18 +159,20 @@ const correct = async <Model extends ILlmSchema.Model>(
       You don't need to explain me anything, but just fix it immediately
       without any hesitation, explanation, and questions.
     `,
+    promptCacheKey,
   });
-  if (pointer.value === null) throw new Error("Failed to modify test code.");
+  if (pointer.value === null) throw new Error("Failed to correct test code.");
 
-  const compiler: IAutoBeCompiler = await ctx.compiler();
-  if (pointer.value.revise)
-    pointer.value.revise.final = await compiler.typescript.beautify(
-      pointer.value.revise.final,
-    );
-  else
-    pointer.value.draft = await compiler.typescript.beautify(
-      pointer.value.draft,
-    );
+  pointer.value.revise.final = await completeTestCode(
+    ctx,
+    content.artifacts,
+    pointer.value.revise.final,
+  );
+  pointer.value.draft = await completeTestCode(
+    ctx,
+    content.artifacts,
+    pointer.value.draft,
+  );
 
   ctx.dispatch({
     type: "testCorrect",
@@ -155,12 +186,15 @@ const correct = async <Model extends ILlmSchema.Model>(
     draft: pointer.value.draft,
     review: pointer.value.revise?.review,
     final: pointer.value.revise?.final,
-  });
+  } satisfies AutoBeTestCorrectEvent);
   const newContent: IAutoBeTestFunction = {
     ...content,
     script: pointer.value.revise?.final ?? pointer.value.draft,
   };
-  const newValidate: AutoBeTestValidateEvent = await compile(ctx, newContent);
+  const newValidate: AutoBeTestValidateEvent = await compileTestFile(
+    ctx,
+    newContent,
+  );
   return predicate(
     ctx,
     newContent,
@@ -172,43 +206,80 @@ const correct = async <Model extends ILlmSchema.Model>(
       },
     ],
     newValidate,
+    promptCacheKey,
     life,
   );
 };
 
 const createController = <Model extends ILlmSchema.Model>(props: {
   model: Model;
-  artifacts: IAutoBeTestScenarioArtifacts;
+  failure: IAutoBeTypeScriptCompileResult.IFailure;
   build: (next: IAutoBeTestCorrectApplication.IProps) => void;
 }): IAgenticaController.IClass<Model> => {
   assertSchemaModel(props.model);
 
+  const validate = (
+    input: unknown,
+  ): IValidation<IAutoBeTestCorrectApplication.IProps> => {
+    const result: IValidation<IAutoBeTestCorrectApplication.IProps> =
+      typia.validate<IAutoBeTestCorrectApplication.IProps>(input);
+    // if (result.success === false) return result;
+
+    // const expected: number = props.failure.diagnostics.length;
+    // const actual: number = result.data.think.analyses.length;
+    // if (expected !== actual)
+    //   return {
+    //     success: false,
+    //     errors: [
+    //       {
+    //         path: "$input.think.analyses",
+    //         expected: `Array<IValidation<IAutoBeTypeScriptCompileResult.IDiagnostic>> & MinItems<${expected}> & MaxItems<${expected}>`,
+    //         value: result.data.think.analyses,
+    //         description: StringUtil.trim`
+    //           The 'think.analyses' must contain all the compilation errors.
+
+    //           Therefore, the length of the 'think.analyses' must be not
+    //           ${actual}, but exactly ${expected}, which is equal to the length of
+    //           the 'diagnostics' of the compilation failure.
+    //         `,
+    //       },
+    //     ],
+    //     data: input,
+    //   };
+    return result;
+  };
   const application: ILlmApplication<Model> = collection[
-    props.model
-  ] satisfies ILlmApplication<any> as unknown as ILlmApplication<Model>;
+    props.model === "chatgpt" ? "chatgpt" : "claude"
+  ](
+    validate,
+  ) satisfies ILlmApplication<any> as unknown as ILlmApplication<Model>;
   return {
     protocol: "class",
-    name: "Modify Test Code",
+    name: "correct",
     application,
     execute: {
       rewrite: (next) => {
-        next.draft = completeTestCode(props.artifacts, next.draft);
-        if (next.revise)
-          next.revise.final = completeTestCode(
-            props.artifacts,
-            next.revise.final,
-          );
         props.build(next);
       },
     } satisfies IAutoBeTestCorrectApplication,
   };
 };
 
-const claude = typia.llm.application<IAutoBeTestCorrectApplication, "claude">();
 const collection = {
-  chatgpt: typia.llm.application<IAutoBeTestCorrectApplication, "chatgpt">(),
-  claude,
-  llama: claude,
-  deepseek: claude,
-  "3.1": claude,
+  chatgpt: (validate: Validator) =>
+    typia.llm.application<IAutoBeTestCorrectApplication, "chatgpt">({
+      validate: {
+        rewrite: validate,
+      },
+    }),
+  claude: (validate: Validator) =>
+    typia.llm.application<IAutoBeTestCorrectApplication, "claude">({
+      validate: {
+        rewrite: validate,
+      },
+    }),
 };
+
+type Validator = (
+  input: unknown,
+) => IValidation<IAutoBeTestCorrectApplication.IProps>;

package/src/orchestrate/test/orchestrateTestCorrectInvalidRequest.ts

@@ -0,0 +1,145 @@
+import {
+  AutoBeTestCorrectEvent,
+  AutoBeTestValidateEvent,
+} from "@autobe/interface";
+import { StringUtil } from "@autobe/utils";
+import { ILlmApplication, ILlmController, ILlmSchema } from "@samchon/openapi";
+import { IPointer } from "tstl";
+import typia from "typia";
+import { v7 } from "uuid";
+
+import { AutoBeContext } from "../../context/AutoBeContext";
+import { assertSchemaModel } from "../../context/assertSchemaModel";
+import { transformTestCorrectInvalidRequestHistories } from "./histories/transformTestCorrectInvalidRequestHistories";
+import { IAutoBeTestCorrectInvalidRequestApplication } from "./structures/IAutoBeTestCorrectInvalidRequestApplication";
+import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
+
+type CompileFunction = (script: string) => Promise<AutoBeTestValidateEvent>;
+
+export const orchestrateTestCorrectInvalidRequest = async <
+  Model extends ILlmSchema.Model,
+>(
+  ctx: AutoBeContext<Model>,
+  compile: CompileFunction,
+  write: IAutoBeTestFunction,
+): Promise<AutoBeTestValidateEvent> => {
+  const event: AutoBeTestValidateEvent = await compile(write.script);
+  return await predicate(ctx, compile, write, event, ctx.retry);
+};
+
+const predicate = async <Model extends ILlmSchema.Model>(
+  ctx: AutoBeContext<Model>,
+  compile: CompileFunction,
+  write: IAutoBeTestFunction,
+  event: AutoBeTestValidateEvent,
+  life: number,
+): Promise<AutoBeTestValidateEvent> => {
+  if (event.result.type === "failure") {
+    ctx.dispatch(event);
+    return await correct(ctx, compile, write, event, life - 1);
+  }
+  return event;
+};
+
+const correct = async <Model extends ILlmSchema.Model>(
+  ctx: AutoBeContext<Model>,
+  compile: CompileFunction,
+  write: IAutoBeTestFunction,
+  event: AutoBeTestValidateEvent,
+  life: number,
+): Promise<AutoBeTestValidateEvent> => {
+  if (event.result.type !== "failure") return event;
+  else if (life < 0) return event;
+
+  const pointer: IPointer<
+    IAutoBeTestCorrectInvalidRequestApplication.IProps | false | null
+  > = {
+    value: null,
+  };
+  const { tokenUsage } = await ctx.conversate({
+    source: "testCorrect",
+    histories: await transformTestCorrectInvalidRequestHistories(
+      null!,
+      event.result.diagnostics,
+    ),
+    controller: createController({
+      model: ctx.model,
+      then: (next) => {
+        pointer.value = next;
+      },
+      reject: () => {
+        pointer.value = false;
+      },
+    }),
+    enforceFunctionCall: true,
+    message: StringUtil.trim`
+      Fix the AutoBeTest.IFunction data to resolve the compilation error.
+
+      You don't need to explain me anything, but just fix or give it up 
+      immediately without any hesitation, explanation, and questions.
+    `,
+  });
+  if (pointer.value === null) throw new Error("Failed to correct test code.");
+  else if (pointer.value === false) return event; // other's responsibility
+
+  ctx.dispatch({
+    type: "testCorrect",
+    id: v7(),
+    created_at: new Date().toISOString(),
+    file: {
+      scenario: write.scenario,
+      location: write.location,
+      content: write.script,
+    },
+    result: event.result,
+    tokenUsage,
+    step: ctx.state().analyze?.step ?? 0,
+    think: pointer.value.think,
+    draft: pointer.value.draft,
+    review: pointer.value.revise?.review,
+    final: pointer.value.revise?.final,
+  } satisfies AutoBeTestCorrectEvent);
+  const newWrite: IAutoBeTestFunction = {
+    artifacts: write.artifacts,
+    scenario: write.scenario,
+    location: write.location,
+    script: pointer.value.revise?.final ?? pointer.value.draft,
+  };
+  const newEvent: AutoBeTestValidateEvent = await compile(newWrite.script);
+  return await predicate(ctx, compile, newWrite, newEvent, life - 1);
+};
+
+const createController = <Model extends ILlmSchema.Model>(props: {
+  model: Model;
+  then: (next: IAutoBeTestCorrectInvalidRequestApplication.IProps) => void;
+  reject: () => void;
+}): ILlmController<Model> => {
+  assertSchemaModel(props.model);
+  const application = collection[
+    props.model === "chatgpt" ? "chatgpt" : "claude"
+  ] satisfies ILlmApplication<any> as any as ILlmApplication<Model>;
+  return {
+    protocol: "class",
+    name: "correctInvalidRequest",
+    application,
+    execute: {
+      rewrite: (next) => {
+        props.then(next);
+      },
+      reject: () => {
+        props.reject();
+      },
+    } satisfies IAutoBeTestCorrectInvalidRequestApplication,
+  };
+};
+
+const collection = {
+  chatgpt: typia.llm.application<
+    IAutoBeTestCorrectInvalidRequestApplication,
+    "chatgpt"
+  >(),
+  claude: typia.llm.application<
+    IAutoBeTestCorrectInvalidRequestApplication,
+    "claude"
+  >(),
+};