@autobe/agent 0.22.1 → 0.23.0

package/src/orchestrate/test/orchestrateTest.ts

@@ -58,22 +58,26 @@ export const orchestrateTest =
 
     // PLAN
     const scenarios: AutoBeTestScenario[] = await orchestrateTestScenario(ctx);
+    if (scenarios.length === 0)
+      throw new Error("No scenarios generated. Please check the logs.");
 
     // TEST CODE
     const written: IAutoBeTestWriteResult[] = await orchestrateTestWrite(
       ctx,
       scenarios,
     );
+    if (written.length === 0)
+      throw new Error("No test code written. Please check the logs.");
+
     const corrects: AutoBeTestValidateEvent[] = await orchestrateTestCorrect(
       ctx,
-      written,
+      written.map((w) => ({
+        scenario: w.scenario,
+        artifacts: w.artifacts,
+        location: w.event.location,
+        script: w.event.final ?? w.event.draft,
+      })),
     );
-    // for (const c of corrects)
-    //   if (c.result.type !== "success")
-    //     c.file.content = c.file.content
-    //       .split("\n")
-    //       .map((s) => `// ${s}`)
-    //       .join("\n");
 
     // DO COMPILE
     const compiler: IAutoBeCompiler = await ctx.compiler();

package/src/orchestrate/test/orchestrateTestCorrect.ts

@@ -1,11 +1,12 @@
 import { IAgenticaController } from "@agentica/core";
 import {
+  AutoBeTestCorrectEvent,
   AutoBeTestValidateEvent,
   IAutoBeCompiler,
   IAutoBeTypeScriptCompileResult,
 } from "@autobe/interface";
 import { StringUtil } from "@autobe/utils";
-import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
+import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
 import { v7 } from "uuid";
@@ -13,37 +14,64 @@ import { v7 } from "uuid";
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { executeCachedBatch } from "../../utils/executeCachedBatch";
+import { orchestrateCommonCorrectCasting } from "../common/orchestrateCommonCorrectCasting";
 import { completeTestCode } from "./compile/completeTestCode";
 import { transformTestCorrectHistories } from "./histories/transformTestCorrectHistories";
+import { transformTestValidateEvent } from "./histories/transformTestValidateEvent";
+import { orchestrateTestCorrectInvalidRequest } from "./orchestrateTestCorrectInvalidRequest";
 import { IAutoBeTestCorrectApplication } from "./structures/IAutoBeTestCorrectApplication";
 import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
 import { IAutoBeTestFunctionFailure } from "./structures/IAutoBeTestFunctionFailure";
-import { IAutoBeTestWriteResult } from "./structures/IAutoBeTestWriteResult";
 
 export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
-  writeResult: IAutoBeTestWriteResult[],
+  writeResults: IAutoBeTestFunction[],
 ): Promise<AutoBeTestValidateEvent[]> => {
   const result: Array<AutoBeTestValidateEvent | null> =
     await executeCachedBatch(
-      writeResult.map((w) => async () => {
+      writeResults.map((w) => async (promptCacheKey) => {
         try {
-          const event: AutoBeTestValidateEvent = await compile(ctx, {
-            artifacts: w.artifacts,
-            scenario: w.scenario,
-            location: w.event.location,
-            script: w.event.final ?? w.event.draft,
-          });
+          const compile = (script: string) =>
+            compileTestFile(ctx, {
+              ...w,
+              script,
+            });
+          const x: AutoBeTestValidateEvent =
+            await orchestrateTestCorrectInvalidRequest(ctx, compile, w);
+          const y: AutoBeTestValidateEvent =
+            await orchestrateCommonCorrectCasting(
+              ctx,
+              {
+                source: "testCorrect",
+                validate: compile,
+                correct: (next) =>
+                  ({
+                    type: "testCorrect",
+                    id: v7(),
+                    created_at: new Date().toISOString(),
+                    file: {
+                      scenario: w.scenario,
+                      location: w.location,
+                      content: next.final ?? next.draft,
+                    },
+                    result: next.failure,
+                    tokenUsage: next.tokenUsage,
+                    think: next.think,
+                    draft: next.draft,
+                    review: next.review,
+                    final: next.final,
+                    step: ctx.state().analyze?.step ?? 0,
+                  }) satisfies AutoBeTestCorrectEvent,
+                script: (event) => event.file.content,
+              },
+              x.file.content,
+            );
           return await predicate(
             ctx,
-            {
-              artifacts: w.artifacts,
-              scenario: w.scenario,
-              location: w.event.location,
-              script: w.event.final ?? w.event.draft,
-            },
+            transformTestValidateEvent(y, w.artifacts),
             [],
-            event,
+            y,
+            promptCacheKey,
             ctx.retry,
           );
         } catch {
@@ -54,7 +82,7 @@ export const orchestrateTestCorrect = async <Model extends ILlmSchema.Model>(
   return result.filter((r) => r !== null);
 };
 
-const compile = async <Model extends ILlmSchema.Model>(
+const compileTestFile = async <Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
   func: IAutoBeTestFunction,
 ): Promise<AutoBeTestValidateEvent> => {
@@ -73,7 +101,6 @@ const compile = async <Model extends ILlmSchema.Model>(
       scenario: func.scenario,
       location: func.location,
       content: func.script,
-      // result,
     },
     result,
     created_at: new Date().toISOString(),
@@ -86,11 +113,12 @@ const predicate = async <Model extends ILlmSchema.Model>(
   content: IAutoBeTestFunction,
   failures: IAutoBeTestFunctionFailure[],
   event: AutoBeTestValidateEvent,
+  promptCacheKey: string,
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
   if (event.result.type === "failure") ctx.dispatch(event);
   return event.result.type === "failure"
-    ? await correct(ctx, content, failures, event, life - 1)
+    ? await correct(ctx, content, failures, event, promptCacheKey, life - 1)
     : event;
 };
 
@@ -99,10 +127,11 @@ const correct = async <Model extends ILlmSchema.Model>(
   content: IAutoBeTestFunction,
   failures: IAutoBeTestFunctionFailure[],
   validate: AutoBeTestValidateEvent,
+  promptCacheKey: string,
   life: number,
 ): Promise<AutoBeTestValidateEvent> => {
   if (validate.result.type !== "failure") return validate;
-  else if (--life <= 0) return validate;
+  else if (life < 0) return validate;
 
   const pointer: IPointer<IAutoBeTestCorrectApplication.IProps | null> = {
     value: null,
@@ -118,6 +147,7 @@ const correct = async <Model extends ILlmSchema.Model>(
     ]),
     controller: createController({
       model: ctx.model,
+      failure: validate.result,
       build: (next) => {
         pointer.value = next;
       },
@@ -129,8 +159,9 @@ const correct = async <Model extends ILlmSchema.Model>(
       You don't need to explain me anything, but just fix it immediately
       without any hesitation, explanation, and questions.
     `,
+    promptCacheKey,
   });
-  if (pointer.value === null) throw new Error("Failed to modify test code.");
+  if (pointer.value === null) throw new Error("Failed to correct test code.");
 
   pointer.value.revise.final = await completeTestCode(
     ctx,
@@ -155,12 +186,15 @@ const correct = async <Model extends ILlmSchema.Model>(
     draft: pointer.value.draft,
     review: pointer.value.revise?.review,
     final: pointer.value.revise?.final,
-  });
+  } satisfies AutoBeTestCorrectEvent);
   const newContent: IAutoBeTestFunction = {
     ...content,
     script: pointer.value.revise?.final ?? pointer.value.draft,
   };
-  const newValidate: AutoBeTestValidateEvent = await compile(ctx, newContent);
+  const newValidate: AutoBeTestValidateEvent = await compileTestFile(
+    ctx,
+    newContent,
+  );
   return predicate(
     ctx,
     newContent,
@@ -172,22 +206,56 @@ const correct = async <Model extends ILlmSchema.Model>(
       },
     ],
     newValidate,
+    promptCacheKey,
     life,
   );
 };
 
 const createController = <Model extends ILlmSchema.Model>(props: {
   model: Model;
+  failure: IAutoBeTypeScriptCompileResult.IFailure;
   build: (next: IAutoBeTestCorrectApplication.IProps) => void;
 }): IAgenticaController.IClass<Model> => {
   assertSchemaModel(props.model);
 
+  const validate = (
+    input: unknown,
+  ): IValidation<IAutoBeTestCorrectApplication.IProps> => {
+    const result: IValidation<IAutoBeTestCorrectApplication.IProps> =
+      typia.validate<IAutoBeTestCorrectApplication.IProps>(input);
+    // if (result.success === false) return result;
+
+    // const expected: number = props.failure.diagnostics.length;
+    // const actual: number = result.data.think.analyses.length;
+    // if (expected !== actual)
+    //   return {
+    //     success: false,
+    //     errors: [
+    //       {
+    //         path: "$input.think.analyses",
+    //         expected: `Array<IValidation<IAutoBeTypeScriptCompileResult.IDiagnostic>> & MinItems<${expected}> & MaxItems<${expected}>`,
+    //         value: result.data.think.analyses,
+    //         description: StringUtil.trim`
+    //           The 'think.analyses' must contain all the compilation errors.
+
+    //           Therefore, the length of the 'think.analyses' must be not
+    //           ${actual}, but exactly ${expected}, which is equal to the length of
+    //           the 'diagnostics' of the compilation failure.
+    //         `,
+    //       },
+    //     ],
+    //     data: input,
+    //   };
+    return result;
+  };
   const application: ILlmApplication<Model> = collection[
-    props.model
-  ] satisfies ILlmApplication<any> as unknown as ILlmApplication<Model>;
+    props.model === "chatgpt" ? "chatgpt" : "claude"
+  ](
+    validate,
+  ) satisfies ILlmApplication<any> as unknown as ILlmApplication<Model>;
   return {
     protocol: "class",
-    name: "Modify Test Code",
+    name: "correct",
     application,
     execute: {
       rewrite: (next) => {
@@ -197,11 +265,21 @@ const createController = <Model extends ILlmSchema.Model>(props: {
   };
 };
 
-const claude = typia.llm.application<IAutoBeTestCorrectApplication, "claude">();
 const collection = {
-  chatgpt: typia.llm.application<IAutoBeTestCorrectApplication, "chatgpt">(),
-  claude,
-  llama: claude,
-  deepseek: claude,
-  "3.1": claude,
+  chatgpt: (validate: Validator) =>
+    typia.llm.application<IAutoBeTestCorrectApplication, "chatgpt">({
+      validate: {
+        rewrite: validate,
+      },
+    }),
+  claude: (validate: Validator) =>
+    typia.llm.application<IAutoBeTestCorrectApplication, "claude">({
+      validate: {
+        rewrite: validate,
+      },
+    }),
 };
+
+type Validator = (
+  input: unknown,
+) => IValidation<IAutoBeTestCorrectApplication.IProps>;

package/src/orchestrate/test/orchestrateTestCorrectInvalidRequest.ts

@@ -0,0 +1,145 @@
+import {
+  AutoBeTestCorrectEvent,
+  AutoBeTestValidateEvent,
+} from "@autobe/interface";
+import { StringUtil } from "@autobe/utils";
+import { ILlmApplication, ILlmController, ILlmSchema } from "@samchon/openapi";
+import { IPointer } from "tstl";
+import typia from "typia";
+import { v7 } from "uuid";
+
+import { AutoBeContext } from "../../context/AutoBeContext";
+import { assertSchemaModel } from "../../context/assertSchemaModel";
+import { transformTestCorrectInvalidRequestHistories } from "./histories/transformTestCorrectInvalidRequestHistories";
+import { IAutoBeTestCorrectInvalidRequestApplication } from "./structures/IAutoBeTestCorrectInvalidRequestApplication";
+import { IAutoBeTestFunction } from "./structures/IAutoBeTestFunction";
+
+type CompileFunction = (script: string) => Promise<AutoBeTestValidateEvent>;
+
+export const orchestrateTestCorrectInvalidRequest = async <
+  Model extends ILlmSchema.Model,
+>(
+  ctx: AutoBeContext<Model>,
+  compile: CompileFunction,
+  write: IAutoBeTestFunction,
+): Promise<AutoBeTestValidateEvent> => {
+  const event: AutoBeTestValidateEvent = await compile(write.script);
+  return await predicate(ctx, compile, write, event, ctx.retry);
+};
+
+const predicate = async <Model extends ILlmSchema.Model>(
+  ctx: AutoBeContext<Model>,
+  compile: CompileFunction,
+  write: IAutoBeTestFunction,
+  event: AutoBeTestValidateEvent,
+  life: number,
+): Promise<AutoBeTestValidateEvent> => {
+  if (event.result.type === "failure") {
+    ctx.dispatch(event);
+    return await correct(ctx, compile, write, event, life - 1);
+  }
+  return event;
+};
+
+const correct = async <Model extends ILlmSchema.Model>(
+  ctx: AutoBeContext<Model>,
+  compile: CompileFunction,
+  write: IAutoBeTestFunction,
+  event: AutoBeTestValidateEvent,
+  life: number,
+): Promise<AutoBeTestValidateEvent> => {
+  if (event.result.type !== "failure") return event;
+  else if (life < 0) return event;
+
+  const pointer: IPointer<
+    IAutoBeTestCorrectInvalidRequestApplication.IProps | false | null
+  > = {
+    value: null,
+  };
+  const { tokenUsage } = await ctx.conversate({
+    source: "testCorrect",
+    histories: await transformTestCorrectInvalidRequestHistories(
+      null!,
+      event.result.diagnostics,
+    ),
+    controller: createController({
+      model: ctx.model,
+      then: (next) => {
+        pointer.value = next;
+      },
+      reject: () => {
+        pointer.value = false;
+      },
+    }),
+    enforceFunctionCall: true,
+    message: StringUtil.trim`
+      Fix the AutoBeTest.IFunction data to resolve the compilation error.
+
+      You don't need to explain me anything, but just fix or give it up 
+      immediately without any hesitation, explanation, and questions.
+    `,
+  });
+  if (pointer.value === null) throw new Error("Failed to correct test code.");
+  else if (pointer.value === false) return event; // other's responsibility
+
+  ctx.dispatch({
+    type: "testCorrect",
+    id: v7(),
+    created_at: new Date().toISOString(),
+    file: {
+      scenario: write.scenario,
+      location: write.location,
+      content: write.script,
+    },
+    result: event.result,
+    tokenUsage,
+    step: ctx.state().analyze?.step ?? 0,
+    think: pointer.value.think,
+    draft: pointer.value.draft,
+    review: pointer.value.revise?.review,
+    final: pointer.value.revise?.final,
+  } satisfies AutoBeTestCorrectEvent);
+  const newWrite: IAutoBeTestFunction = {
+    artifacts: write.artifacts,
+    scenario: write.scenario,
+    location: write.location,
+    script: pointer.value.revise?.final ?? pointer.value.draft,
+  };
+  const newEvent: AutoBeTestValidateEvent = await compile(newWrite.script);
+  return await predicate(ctx, compile, newWrite, newEvent, life - 1);
+};
+
+const createController = <Model extends ILlmSchema.Model>(props: {
+  model: Model;
+  then: (next: IAutoBeTestCorrectInvalidRequestApplication.IProps) => void;
+  reject: () => void;
+}): ILlmController<Model> => {
+  assertSchemaModel(props.model);
+  const application = collection[
+    props.model === "chatgpt" ? "chatgpt" : "claude"
+  ] satisfies ILlmApplication<any> as any as ILlmApplication<Model>;
+  return {
+    protocol: "class",
+    name: "correctInvalidRequest",
+    application,
+    execute: {
+      rewrite: (next) => {
+        props.then(next);
+      },
+      reject: () => {
+        props.reject();
+      },
+    } satisfies IAutoBeTestCorrectInvalidRequestApplication,
+  };
+};
+
+const collection = {
+  chatgpt: typia.llm.application<
+    IAutoBeTestCorrectInvalidRequestApplication,
+    "chatgpt"
+  >(),
+  claude: typia.llm.application<
+    IAutoBeTestCorrectInvalidRequestApplication,
+    "claude"
+  >(),
+};

package/src/orchestrate/test/orchestrateTestScenario.ts

@@ -9,6 +9,7 @@ import { AutoBeEndpointComparator, MapUtil, StringUtil } from "@autobe/utils";
 import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
 import { HashMap, IPointer, Pair } from "tstl";
 import typia from "typia";
+import { NamingConvention } from "typia/lib/utils/NamingConvention";
 import { v7 } from "uuid";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
@@ -22,17 +23,19 @@ import { IAutoBeTestScenarioAuthorizationRole } from "./structures/IAutoBeTestSc
 export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
   ctx: AutoBeContext<Model>,
 ): Promise<AutoBeTestScenario[]> {
-  const operations: AutoBeOpenApi.IOperation[] =
-    ctx.state().interface?.document.operations ?? [];
-  if (operations.length === 0) {
+  const document: AutoBeOpenApi.IDocument | undefined =
+    ctx.state().interface?.document;
+  if (document === undefined) {
     throw new Error(
       "Cannot write test scenarios because these are no operations.",
     );
   }
 
+  Object.entries(document.components.schemas);
+
   const dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation> =
     new HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>(
-      operations.map(
+      document.operations.map(
         (op) =>
           new Pair(
             {
@@ -51,15 +54,18 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
     "",
     " method | path ",
     "--------|------",
-    ...operations.map((op) => `\`${op.method}\` | \`${op.path}\``).join("\n"),
+    ...document.operations
+      .map((op) => `\`${op.method}\` | \`${op.path}\``)
+      .join("\n"),
   ].join("\n");
 
   const progress: AutoBeProgressEventBase = {
-    total: operations.length,
+    total: document.operations.length,
     completed: 0,
   };
   const exclude: IAutoBeTestScenarioApplication.IScenarioGroup[] = [];
-  let include: AutoBeOpenApi.IOperation[] = Array.from(operations);
+  let include: AutoBeOpenApi.IOperation[] = [...document.operations];
+  let trial: number = 0;
 
   do {
     const matrix: AutoBeOpenApi.IOperation[][] = divideArray({
@@ -72,7 +78,7 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
           ...(await divideAndConquer(ctx, {
             dict,
             endpointNotFound,
-            entire: operations,
+            document,
             include,
             exclude: exclude.map((x) => x.endpoint),
             progress,
@@ -92,7 +98,8 @@ export async function orchestrateTestScenario<Model extends ILlmSchema.Model>(
       }
       return true;
     });
-  } while (include.length > 0);
+    progress.total = include.length + exclude.length;
+  } while (include.length > 0 && ++trial < ctx.retry);
 
   return exclude.flatMap((pg) => {
     return pg.scenarios.map((plan) => {
@@ -111,7 +118,7 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
   props: {
     dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>;
     endpointNotFound: string;
-    entire: AutoBeOpenApi.IOperation[];
+    document: AutoBeOpenApi.IDocument;
     include: AutoBeOpenApi.IOperation[];
     exclude: AutoBeOpenApi.IEndpoint[];
     progress: AutoBeProgressEventBase;
@@ -129,7 +136,7 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
       source: "testScenarios",
       histories: transformTestScenarioHistories(
         ctx.state(),
-        props.entire,
+        props.document,
         props.include,
         props.exclude,
       ),
@@ -139,6 +146,11 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
         dict: props.dict,
         authorizations,
         build: (next) => {
+          next.scenarioGroups.forEach((sg) =>
+            sg.scenarios.forEach((s) => {
+              s.functionName = NamingConvention.snake(s.functionName);
+            }),
+          );
           pointer.value ??= [];
           pointer.value.push(...next.scenarioGroups);
         },
@@ -148,6 +160,11 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
       message: `Create e2e test scenarios.`,
     });
     if (pointer.value.length === 0) return [];
+
+    props.progress.total = Math.max(
+      props.progress.total,
+      (props.progress.completed += pointer.value.length),
+    );
     ctx.dispatch({
       type: "testScenarios",
       id: v7(),
@@ -165,14 +182,13 @@ const divideAndConquer = async <Model extends ILlmSchema.Model>(
           ),
         )
         .flat(),
-      completed: (props.progress.completed += pointer.value.length),
+      completed: props.progress.completed,
       total: props.progress.total,
       step: ctx.state().interface?.step ?? 0,
       created_at: new Date().toISOString(),
     });
     return pointer.value;
   } catch {
-    console.log("test scenario, failed to function call", props.include);
     return [];
   }
 };

package/src/orchestrate/test/orchestrateTestWrite.ts

@@ -7,6 +7,7 @@ import {
 import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
+import { NamingConvention } from "typia/lib/utils/NamingConvention";
 import { v7 } from "uuid";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
@@ -49,7 +50,7 @@ export async function orchestrateTestWrite<Model extends ILlmSchema.Model>(
           artifacts,
           event,
         };
-      } catch (error) {
+      } catch {
         return null;
       }
     }),
@@ -76,6 +77,7 @@ async function process<Model extends ILlmSchema.Model>(
     controller: createController({
       model: ctx.model,
       build: (next) => {
+        next.domain = NamingConvention.snake(next.domain);
         pointer.value = next;
       },
     }),