@autobe/benchmark 0.29.2 → 0.30.0-dev.20260315

package/src/example/AutoBeExampleBenchmark.ts

@@ -1,159 +1,190 @@
-import {
-  AutoBeEvent,
-  AutoBeExampleProject,
-  AutoBePhase,
-  IAutoBeAgent,
-} from "@autobe/interface";
-
-import { IAutoBeExampleBenchmarkState } from "../structures/IAutoBeExampleBenchmarkState";
-import { AutoBeExampleArchiver } from "./AutoBeExampleArchiver";
-
-export namespace AutoBeExampleBenchmark {
-  export interface IContext {
-    createAgent: (
-      props: AutoBeExampleArchiver.IAgentProps,
-    ) => Promise<IAutoBeAgent>;
-  }
-
-  export const execute = async (
-    ctx: IContext,
-    props: {
-      vendors: string[];
-      projects?: AutoBeExampleProject[];
-      phases?: AutoBePhase[];
-      progress: (state: IAutoBeExampleBenchmarkState) => void;
-      on?: (event: AutoBeEvent) => void;
-    },
-  ): Promise<void> => {
-    const state: IAutoBeExampleBenchmarkState = {
-      vendors: props.vendors.map(
-        (vendor): IAutoBeExampleBenchmarkState.IOfVendor => ({
-          name: vendor,
-          projects: PROJECT_SEQUENCE.filter(
-            (p) => !props.projects || props.projects.includes(p),
-          ).map(
-            (project): IAutoBeExampleBenchmarkState.IOfProject => ({
-              name: project,
-              phases: [],
-              success: null,
-              started_at: null,
-              completed_at: null,
-            }),
-          ),
-        }),
-      ),
-    };
-    const report = () => props.progress(state);
-    await Promise.all(
-      state.vendors.map((vendor) =>
-        executeVendor(ctx, {
-          phases: props.phases,
-          vendorState: vendor,
-          on: props.on,
-          report,
-        }),
-      ),
-    );
-  };
-
-  const executeVendor = async (
-    ctx: IContext,
-    props: {
-      vendorState: IAutoBeExampleBenchmarkState.IOfVendor;
-      phases?: AutoBePhase[];
-      report: () => void;
-      on?: (event: AutoBeEvent) => void;
-    },
-  ): Promise<void> => {
-    for (const project of props.vendorState.projects)
-      await executeProject(ctx, {
-        vendor: props.vendorState.name,
-        projectState: project,
-        phases: props.phases,
-        report: props.report,
-        on: props.on,
-      });
-  };
-
-  const executeProject = async (
-    ctx: IContext,
-    props: {
-      vendor: string;
-      projectState: IAutoBeExampleBenchmarkState.IOfProject;
-      phases?: AutoBePhase[];
-      report: () => void;
-      on?: (event: AutoBeEvent) => void;
-    },
-  ): Promise<void> => {
-    props.projectState.started_at = new Date();
-    for (const phase of PHASE_SEQUENCE) {
-      if (props.phases && props.phases.includes(phase) === false) continue;
-      const phaseState: IAutoBeExampleBenchmarkState.IOfPhase = {
-        name: phase,
-        snapshot: null,
-        success: null,
-        started_at: new Date(),
-        completed_at: null,
-        trial: 0,
-      };
-      props.projectState.phases.push(phaseState);
-      for (let i: number = 0; i < 3; ++i) {
-        try {
-          ++phaseState.trial;
-          phaseState.started_at = new Date();
-          phaseState.completed_at = null;
-          const success: boolean = await getArchiver(phase)({
-            vendor: props.vendor,
-            project: props.projectState.name,
-            agent: (next) => ctx.createAgent(next),
-            on: (s) => {
-              const event = s.event;
-              if (
-                event.type !== "jsonValidateError" &&
-                event.type !== "jsonParseError" &&
-                event.type !== "preliminary" &&
-                event.type !== "consentFunctionCall"
-              )
-                phaseState.snapshot = s;
-              props.report();
-              if (props.on) props.on(s.event);
-            },
-          });
-          phaseState.success = success;
-          phaseState.completed_at = new Date();
-          props.report();
-          if (success === true) break;
-        } catch (error) {
-          continue;
-        }
-      }
-      if (phaseState.success === null) break;
-      else if (phaseState.success === false && phaseState.name !== "test")
-        break;
-    }
-    props.projectState.completed_at = new Date();
-    props.projectState.success = props.projectState.phases.every(
-      (phase) => phase.success === true,
-    );
-    props.report();
-  };
-}
-
-const getArchiver = (phase: AutoBePhase) => {
-  if (phase === "analyze") return AutoBeExampleArchiver.archiveAnalyze;
-  else if (phase === "prisma") return AutoBeExampleArchiver.archivePrisma;
-  else if (phase === "interface") return AutoBeExampleArchiver.archiveInterface;
-  else if (phase === "test") return AutoBeExampleArchiver.archiveTest;
-  else if (phase === "realize") return AutoBeExampleArchiver.archiveRealize;
-  phase satisfies never;
-  throw new Error(`Unknown phase: ${phase}`);
-};
-
-const PROJECT_SEQUENCE = ["todo", "bbs", "reddit", "shopping"] as const;
-const PHASE_SEQUENCE = [
-  "analyze",
-  "prisma",
-  "interface",
-  "test",
-  "realize",
-] as const;
+import {
+  AutoBeEvent,
+  AutoBeExampleProject,
+  AutoBePhase,
+  IAutoBeAgent,
+} from "@autobe/interface";
+
+import { IAutoBeExampleBenchmarkState } from "../structures/IAutoBeExampleBenchmarkState";
+import { AutoBeExampleArchiver } from "./AutoBeExampleArchiver";
+
+export namespace AutoBeExampleBenchmark {
+  export interface IContext {
+    createAgent: (
+      props: AutoBeExampleArchiver.IAgentProps,
+    ) => Promise<IAutoBeAgent>;
+  }
+
+  export const execute = async (
+    ctx: IContext,
+    props: {
+      vendors: string[];
+      projects?: AutoBeExampleProject[];
+      phases?: AutoBePhase[];
+      imagePath?: string;
+      progress: (state: IAutoBeExampleBenchmarkState) => void;
+      on?: (
+        event: AutoBeEvent,
+        agent: IAutoBeAgent,
+        context: { vendor: string; project: AutoBeExampleProject },
+      ) => void;
+    },
+  ): Promise<void> => {
+    const state: IAutoBeExampleBenchmarkState = {
+      vendors: props.vendors.map(
+        (vendor): IAutoBeExampleBenchmarkState.IOfVendor => ({
+          name: vendor,
+          projects: PROJECT_SEQUENCE.filter(
+            (p) => !props.projects || props.projects.includes(p),
+          ).map(
+            (project): IAutoBeExampleBenchmarkState.IOfProject => ({
+              name: project,
+              phases: [],
+              success: null,
+              started_at: null,
+              completed_at: null,
+            }),
+          ),
+        }),
+      ),
+    };
+    const report = () => props.progress(state);
+    await Promise.all(
+      state.vendors.map(async (vendor) => {
+        await executeVendor(ctx, {
+          imagePath: props.imagePath,
+          phases: props.phases,
+          vendorState: vendor,
+          on: props.on,
+          report,
+        });
+      }),
+    );
+  };
+
+  const executeVendor = async (
+    ctx: IContext,
+    props: {
+      imagePath?: string;
+      vendorState: IAutoBeExampleBenchmarkState.IOfVendor;
+      phases?: AutoBePhase[];
+      report: () => void;
+      on?: (
+        event: AutoBeEvent,
+        agent: IAutoBeAgent,
+        context: { vendor: string; project: AutoBeExampleProject },
+      ) => void;
+    },
+  ): Promise<void> => {
+    for (const project of props.vendorState.projects)
+      await executeProject(ctx, {
+        imagePath: props.imagePath,
+        vendor: props.vendorState.name,
+        projectState: project,
+        phases: props.phases,
+        report: props.report,
+        on: props.on,
+      });
+  };
+
+  const executeProject = async (
+    ctx: IContext,
+    props: {
+      vendor: string;
+      projectState: IAutoBeExampleBenchmarkState.IOfProject;
+      imagePath?: string;
+      phases?: AutoBePhase[];
+      report: () => void;
+      on?: (
+        event: AutoBeEvent,
+        agent: IAutoBeAgent,
+        context: { vendor: string; project: AutoBeExampleProject },
+      ) => void;
+    },
+  ): Promise<void> => {
+    props.projectState.started_at = new Date();
+    for (const phase of PHASE_SEQUENCE) {
+      if (props.phases && props.phases.includes(phase) === false) continue;
+      const phaseState: IAutoBeExampleBenchmarkState.IOfPhase = {
+        name: phase,
+        snapshot: null,
+        success: null,
+        started_at: new Date(),
+        completed_at: null,
+        count: 0,
+      };
+      props.projectState.phases.push(phaseState);
+      try {
+        phaseState.started_at = new Date();
+        phaseState.completed_at = null;
+        phaseState.count = 0;
+        const success: boolean = await getArchiver(phase)({
+          vendor: props.vendor,
+          project: props.projectState.name,
+          imagePath: props.imagePath,
+          agent: (next) => ctx.createAgent(next),
+          on: (s, agent) => {
+            ++phaseState.count;
+            const event = s.event;
+            if (
+              event.type !== "jsonValidateError" &&
+              event.type !== "jsonParseError" &&
+              event.type !== "preliminary" &&
+              event.type !== "consentFunctionCall"
+            )
+              phaseState.snapshot = s;
+            props.report();
+            if (props.on)
+              props.on(s.event, agent, {
+                vendor: props.vendor,
+                project: props.projectState.name,
+              });
+          },
+        });
+        phaseState.success = success;
+        phaseState.completed_at = new Date();
+        props.report();
+      } catch (error) {
+        console.log(
+          props.vendor,
+          props.projectState.name,
+          phaseState.name,
+          error,
+        );
+        throw error;
+      }
+      if (phaseState.success === null || phaseState.success === false) break;
+    }
+    props.projectState.completed_at = new Date();
+    props.projectState.success = props.projectState.phases.every(
+      (phase) => phase.success === true,
+    );
+    props.report();
+  };
+}
+
+const getArchiver = (phase: AutoBePhase) => {
+  if (phase === "analyze") return AutoBeExampleArchiver.archiveAnalyze;
+  else if (phase === "database") return AutoBeExampleArchiver.archivePrisma;
+  else if (phase === "interface") return AutoBeExampleArchiver.archiveInterface;
+  else if (phase === "test") return AutoBeExampleArchiver.archiveTest;
+  else if (phase === "realize") return AutoBeExampleArchiver.archiveRealize;
+  phase satisfies never;
+  throw new Error(`Unknown phase: ${phase}`);
+};
+
+const PROJECT_SEQUENCE = [
+  "todo",
+  "bbs",
+  "reddit",
+  "shopping",
+  "account",
+  "erp",
+] as const;
+const PHASE_SEQUENCE = [
+  "analyze",
+  "database",
+  "interface",
+  "test",
+  "realize",
+] as const;

package/src/example/{AutoBeExampleDocumentation.ts → AutoBeExampleLogger.ts}

@@ -1,84 +1,85 @@
-import { AutoBeProgressEventBase } from "@autobe/interface";
-import { StringUtil } from "@autobe/utils";
-import typia from "typia";
-
-import { IAutoBeExampleBenchmarkState } from "../structures";
-
-export namespace AutoBeExampleDocumentation {
-  export const markdown = (state: IAutoBeExampleBenchmarkState): string =>
-    StringUtil.trim`
-      # AutoBe Example Benchmark Report
-
-      ${markdownIndex(state)}
-
-      ${state.vendors.map(markdownVendor).join("\n\n")}
-    `;
-
-  const markdownIndex = (
-    state: IAutoBeExampleBenchmarkState,
-  ): string => StringUtil.trim`
-    ## Table of Contents
-
-    ${state.vendors
-      .map(
-        (vendor) =>
-          `- [\`${vendor.name}\`](#${vendor.name
-            .replaceAll("/", "")
-            .replaceAll(":", "")})`,
-      )
-      .join("\n")}
-  `;
-
-  const markdownVendor = (
-    state: IAutoBeExampleBenchmarkState.IOfVendor,
-  ): string => StringUtil.trim`
-    ## \`${state.name}\`
-
-    Project | Phase | State | Elapsed Time
-    :-------|:------|:------|-------------:
-    ${state.projects.map(markdownProject).join("\n")}
-  `;
-
-  const markdownProject = (
-    state: IAutoBeExampleBenchmarkState.IOfProject,
-  ): string => {
-    // yellow circle emoji:
-    const phase: IAutoBeExampleBenchmarkState.IOfPhase | undefined =
-      state.phases.at(-1);
-    return [
-      state.name,
-      !!phase?.name ? `${phase.name} (${phase.trial})` : "-",
-      state.completed_at !== null
-        ? state.success
-          ? "🟢 success"
-          : "🔴 failure"
-        : phase !== undefined && phase.snapshot !== null
-          ? [
-              phase.trial !== 1 ? "🟠" : "🟡",
-              `\`${phase.snapshot.event.type}\``,
-              ...(typia.is<AutoBeProgressEventBase>(phase.snapshot.event)
-                ? [
-                    `(${phase.snapshot.event.completed} of ${phase.snapshot.event.total})`,
-                  ]
-                : []),
-            ].join(" ")
-          : "-",
-      state.started_at !== null
-        ? elapsedTime({
-            started_at: state.started_at,
-            completed_at: state.completed_at,
-          })
-        : "-",
-    ].join(" | ");
-  };
-}
-
-const elapsedTime = (props: {
-  started_at: Date;
-  completed_at: Date | null;
-}): string =>
-  Math.round(
-    ((props.completed_at ?? new Date()).getTime() -
-      props.started_at.getTime()) /
-      1_000,
-  ).toLocaleString() + " sec";
+import { AutoBeProgressEventBase } from "@autobe/interface";
+import { StringUtil } from "@autobe/utils";
+import typia from "typia";
+
+import { IAutoBeExampleBenchmarkState } from "../structures";
+
+export namespace AutoBeExampleLogger {
+  export const markdown = (state: IAutoBeExampleBenchmarkState): string =>
+    StringUtil.trim`
+      # AutoBe Example Benchmark Report
+
+      ${markdownIndex(state)}
+
+      ${state.vendors.map(markdownVendor).join("\n\n")}
+    `;
+
+  const markdownIndex = (
+    state: IAutoBeExampleBenchmarkState,
+  ): string => StringUtil.trim`
+    ## Table of Contents
+
+    ${state.vendors
+      .map(
+        (vendor) =>
+          `- [\`${vendor.name}\`](#${vendor.name
+            .replaceAll("/", "")
+            .replaceAll(":", "")})`,
+      )
+      .join("\n")}
+  `;
+
+  const markdownVendor = (
+    state: IAutoBeExampleBenchmarkState.IOfVendor,
+  ): string => StringUtil.trim`
+    ## \`${state.name}\`
+
+    Project | Phase | State | Count | Elapsed Time
+    :-------|:------|:------|------:|-------------:
+    ${state.projects.map(markdownProject).join("\n")}
+  `;
+
+  const markdownProject = (
+    state: IAutoBeExampleBenchmarkState.IOfProject,
+  ): string => {
+    // yellow circle emoji:
+    const phase: IAutoBeExampleBenchmarkState.IOfPhase | undefined =
+      state.phases.at(-1);
+    return [
+      state.name,
+      phase?.name ?? "-",
+      phase !== undefined && phase.snapshot !== null
+        ? [
+            state.completed_at !== null
+              ? state.success
+                ? "🟢 success"
+                : "🔴 failure"
+              : "🟡",
+            `\`${phase.snapshot.event.type}\``,
+            ...(typia.is<AutoBeProgressEventBase>(phase.snapshot.event)
+              ? [
+                  `(${phase.snapshot.event.completed} of ${phase.snapshot.event.total})`,
+                ]
+              : []),
+          ].join(" ")
+        : "-",
+      phase?.count.toLocaleString() ?? "0",
+      state.started_at !== null
+        ? elapsedTime({
+            started_at: state.started_at,
+            completed_at: state.completed_at,
+          })
+        : "-",
+    ].join(" | ");
+  };
+}
+
+const elapsedTime = (props: {
+  started_at: Date;
+  completed_at: Date | null;
+}): string =>
+  Math.round(
+    ((props.completed_at ?? new Date()).getTime() -
+      props.started_at.getTime()) /
+      1_000,
+  ).toLocaleString() + " sec";