@autobe/agent 0.25.4 → 0.25.5

package/src/orchestrate/test/orchestrateTestCorrectInvalidRequest.ts

@@ -3,13 +3,19 @@ import {
   AutoBeTestValidateEvent,
 } from "@autobe/interface";
 import { StringUtil } from "@autobe/utils";
-import { ILlmApplication, ILlmController, ILlmSchema } from "@samchon/openapi";
+import {
+  ILlmApplication,
+  ILlmController,
+  ILlmSchema,
+  IValidation,
+} from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
 import { v7 } from "uuid";
 
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
+import { validateEmptyCode } from "../../utils/validateEmptyCode";
 import { completeTestCode } from "./compile/completeTestCode";
 import { transformTestCorrectInvalidRequestHistories } from "./histories/transformTestCorrectInvalidRequestHistories";
 import { IAutoBeTestCorrectInvalidRequestApplication } from "./structures/IAutoBeTestCorrectInvalidRequestApplication";
@@ -65,6 +71,7 @@ const correct = async <Model extends ILlmSchema.Model>(
     ),
     controller: createController({
       model: ctx.model,
+      functionName: write.scenario.functionName,
       then: (next) => {
         pointer.value = next;
       },
@@ -123,13 +130,31 @@ const correct = async <Model extends ILlmSchema.Model>(
 
 const createController = <Model extends ILlmSchema.Model>(props: {
   model: Model;
+  functionName: string;
   then: (next: IAutoBeTestCorrectInvalidRequestApplication.IProps) => void;
   reject: () => void;
 }): ILlmController<Model> => {
   assertSchemaModel(props.model);
+  const validate: Validator = (input) => {
+    const result: IValidation<IAutoBeTestCorrectInvalidRequestApplication.IProps> =
+      typia.validate<IAutoBeTestCorrectInvalidRequestApplication.IProps>(input);
+    if (result.success === false) return result;
+    const errors: IValidation.IError[] = validateEmptyCode({
+      functionName: props.functionName,
+      draft: result.data.draft,
+      revise: result.data.revise,
+    });
+    return errors.length
+      ? {
+          success: false,
+          errors,
+          data: result.data,
+        }
+      : result;
+  };
   const application = collection[
     props.model === "chatgpt" ? "chatgpt" : "claude"
-  ] satisfies ILlmApplication<any> as any as ILlmApplication<Model>;
+  ](validate) satisfies ILlmApplication<any> as any as ILlmApplication<Model>;
   return {
     protocol: "class",
     name: "correctInvalidRequest",
@@ -146,12 +171,34 @@ const createController = <Model extends ILlmSchema.Model>(props: {
 };
 
 const collection = {
-  chatgpt: typia.llm.application<
-    IAutoBeTestCorrectInvalidRequestApplication,
-    "chatgpt"
-  >(),
-  claude: typia.llm.application<
-    IAutoBeTestCorrectInvalidRequestApplication,
-    "claude"
-  >(),
+  chatgpt: (validate: Validator) =>
+    typia.llm.application<
+      IAutoBeTestCorrectInvalidRequestApplication,
+      "chatgpt"
+    >({
+      validate: {
+        rewrite: validate,
+        reject: () => ({
+          success: true,
+          data: undefined,
+        }),
+      },
+    }),
+  claude: (validate: Validator) =>
+    typia.llm.application<
+      IAutoBeTestCorrectInvalidRequestApplication,
+      "claude"
+    >({
+      validate: {
+        rewrite: validate,
+        reject: () => ({
+          success: true,
+          data: undefined,
+        }),
+      },
+    }),
 };
+
+type Validator = (
+  input: unknown,
+) => IValidation<IAutoBeTestCorrectInvalidRequestApplication.IProps>;

package/src/orchestrate/test/orchestrateTestWrite.ts

@@ -4,7 +4,7 @@ import {
   AutoBeTestScenario,
   AutoBeTestWriteEvent,
 } from "@autobe/interface";
-import { ILlmApplication, ILlmSchema } from "@samchon/openapi";
+import { ILlmApplication, ILlmSchema, IValidation } from "@samchon/openapi";
 import { IPointer } from "tstl";
 import typia from "typia";
 import { NamingConvention } from "typia/lib/utils/NamingConvention";
@@ -13,6 +13,7 @@ import { v7 } from "uuid";
 import { AutoBeContext } from "../../context/AutoBeContext";
 import { assertSchemaModel } from "../../context/assertSchemaModel";
 import { executeCachedBatch } from "../../utils/executeCachedBatch";
+import { validateEmptyCode } from "../../utils/validateEmptyCode";
 import { completeTestCode } from "./compile/completeTestCode";
 import { getTestScenarioArtifacts } from "./compile/getTestScenarioArtifacts";
 import { transformTestWriteHistories } from "./histories/transformTestWriteHistories";
@@ -85,6 +86,7 @@ async function process<Model extends ILlmSchema.Model>(
     }),
     controller: createController({
       model: ctx.model,
+      functionName: props.scenario.functionName,
       build: (next) => {
         next.domain = NamingConvention.snake(next.domain);
         pointer.value = next;
@@ -129,13 +131,33 @@ async function process<Model extends ILlmSchema.Model>(
 
 function createController<Model extends ILlmSchema.Model>(props: {
   model: Model;
+  functionName: string;
   build: (next: IAutoBeTestWriteApplication.IProps) => void;
 }): IAgenticaController.IClass<Model> {
   assertSchemaModel(props.model);
 
+  const validate: Validator = (input) => {
+    const result: IValidation<IAutoBeTestWriteApplication.IProps> =
+      typia.validate<IAutoBeTestWriteApplication.IProps>(input);
+    if (result.success === false) return result;
+    const errors: IValidation.IError[] = validateEmptyCode({
+      functionName: props.functionName,
+      draft: result.data.draft,
+      revise: result.data.revise,
+    });
+    return errors.length
+      ? {
+          success: false,
+          errors,
+          data: result.data,
+        }
+      : result;
+  };
   const application: ILlmApplication<Model> = collection[
-    props.model
-  ] satisfies ILlmApplication<any> as unknown as ILlmApplication<Model>;
+    props.model === "chatgpt" ? "chatgpt" : "claude"
+  ](
+    validate,
+  ) satisfies ILlmApplication<any> as unknown as ILlmApplication<Model>;
   return {
     protocol: "class",
     name: "Create Test Code",
@@ -148,12 +170,21 @@ function createController<Model extends ILlmSchema.Model>(props: {
   };
 }
 
-const claude = typia.llm.application<IAutoBeTestWriteApplication, "claude">();
 const collection = {
-  chatgpt: typia.llm.application<IAutoBeTestWriteApplication, "chatgpt">(),
-  claude,
-  llama: claude,
-  deepseek: claude,
-  "3.1": claude,
-  "3.0": typia.llm.application<IAutoBeTestWriteApplication, "3.0">(),
+  chatgpt: (validate: Validator) =>
+    typia.llm.application<IAutoBeTestWriteApplication, "chatgpt">({
+      validate: {
+        write: validate,
+      },
+    }),
+  claude: (validate: Validator) =>
+    typia.llm.application<IAutoBeTestWriteApplication, "claude">({
+      validate: {
+        write: validate,
+      },
+    }),
 };
+
+type Validator = (
+  input: unknown,
+) => IValidation<IAutoBeTestWriteApplication.IProps>;

package/src/utils/validateEmptyCode.ts

@@ -0,0 +1,41 @@
+import { StringUtil } from "@autobe/utils";
+import { IValidation } from "typia";
+
+export const validateEmptyCode = (props: {
+  functionName: string;
+  draft: string;
+  revise: {
+    final: string | null;
+  };
+}): IValidation.IError[] => {
+  const errors: IValidation.IError[] = [];
+  if (props.draft.includes(props.functionName) === false)
+    errors.push({
+      path: "$input.draft",
+      expected: `string (including function named '${props.functionName}')`,
+      value: props.draft,
+      description: description(props.functionName),
+    });
+  if (
+    props.revise.final !== null &&
+    props.revise.final.includes(props.functionName) === false
+  )
+    errors.push({
+      path: "$input.revise.final",
+      expected: `string (including function named '${props.functionName}')`,
+      value: props.revise.final,
+      description: description(props.functionName),
+    });
+  return errors;
+};
+
+const description = (func: string): string => StringUtil.trim`
+  The function ${func} does not exist in the provided code snippet.
+
+  The first reason of the non-existence is that the code snippet is empty,
+  and the second reason is that AI has written different function name
+  by mistake.
+
+  Please make sure that the code snippet includes the function ${func}.
+  Note that, you never have to write empty code or different function name.
+`;