@autobe/benchmark 0.29.0 → 0.29.2

package/src/example/AutoBeExampleBenchmark.ts

@@ -0,0 +1,159 @@
+import {
+  AutoBeEvent,
+  AutoBeExampleProject,
+  AutoBePhase,
+  IAutoBeAgent,
+} from "@autobe/interface";
+
+import { IAutoBeExampleBenchmarkState } from "../structures/IAutoBeExampleBenchmarkState";
+import { AutoBeExampleArchiver } from "./AutoBeExampleArchiver";
+
+export namespace AutoBeExampleBenchmark {
+  export interface IContext {
+    createAgent: (
+      props: AutoBeExampleArchiver.IAgentProps,
+    ) => Promise<IAutoBeAgent>;
+  }
+
+  export const execute = async (
+    ctx: IContext,
+    props: {
+      vendors: string[];
+      projects?: AutoBeExampleProject[];
+      phases?: AutoBePhase[];
+      progress: (state: IAutoBeExampleBenchmarkState) => void;
+      on?: (event: AutoBeEvent) => void;
+    },
+  ): Promise<void> => {
+    const state: IAutoBeExampleBenchmarkState = {
+      vendors: props.vendors.map(
+        (vendor): IAutoBeExampleBenchmarkState.IOfVendor => ({
+          name: vendor,
+          projects: PROJECT_SEQUENCE.filter(
+            (p) => !props.projects || props.projects.includes(p),
+          ).map(
+            (project): IAutoBeExampleBenchmarkState.IOfProject => ({
+              name: project,
+              phases: [],
+              success: null,
+              started_at: null,
+              completed_at: null,
+            }),
+          ),
+        }),
+      ),
+    };
+    const report = () => props.progress(state);
+    await Promise.all(
+      state.vendors.map((vendor) =>
+        executeVendor(ctx, {
+          phases: props.phases,
+          vendorState: vendor,
+          on: props.on,
+          report,
+        }),
+      ),
+    );
+  };
+
+  const executeVendor = async (
+    ctx: IContext,
+    props: {
+      vendorState: IAutoBeExampleBenchmarkState.IOfVendor;
+      phases?: AutoBePhase[];
+      report: () => void;
+      on?: (event: AutoBeEvent) => void;
+    },
+  ): Promise<void> => {
+    for (const project of props.vendorState.projects)
+      await executeProject(ctx, {
+        vendor: props.vendorState.name,
+        projectState: project,
+        phases: props.phases,
+        report: props.report,
+        on: props.on,
+      });
+  };
+
+  const executeProject = async (
+    ctx: IContext,
+    props: {
+      vendor: string;
+      projectState: IAutoBeExampleBenchmarkState.IOfProject;
+      phases?: AutoBePhase[];
+      report: () => void;
+      on?: (event: AutoBeEvent) => void;
+    },
+  ): Promise<void> => {
+    props.projectState.started_at = new Date();
+    for (const phase of PHASE_SEQUENCE) {
+      if (props.phases && props.phases.includes(phase) === false) continue;
+      const phaseState: IAutoBeExampleBenchmarkState.IOfPhase = {
+        name: phase,
+        snapshot: null,
+        success: null,
+        started_at: new Date(),
+        completed_at: null,
+        trial: 0,
+      };
+      props.projectState.phases.push(phaseState);
+      for (let i: number = 0; i < 3; ++i) {
+        try {
+          ++phaseState.trial;
+          phaseState.started_at = new Date();
+          phaseState.completed_at = null;
+          const success: boolean = await getArchiver(phase)({
+            vendor: props.vendor,
+            project: props.projectState.name,
+            agent: (next) => ctx.createAgent(next),
+            on: (s) => {
+              const event = s.event;
+              if (
+                event.type !== "jsonValidateError" &&
+                event.type !== "jsonParseError" &&
+                event.type !== "preliminary" &&
+                event.type !== "consentFunctionCall"
+              )
+                phaseState.snapshot = s;
+              props.report();
+              if (props.on) props.on(s.event);
+            },
+          });
+          phaseState.success = success;
+          phaseState.completed_at = new Date();
+          props.report();
+          if (success === true) break;
+        } catch (error) {
+          continue;
+        }
+      }
+      if (phaseState.success === null) break;
+      else if (phaseState.success === false && phaseState.name !== "test")
+        break;
+    }
+    props.projectState.completed_at = new Date();
+    props.projectState.success = props.projectState.phases.every(
+      (phase) => phase.success === true,
+    );
+    props.report();
+  };
+}
+
+const getArchiver = (phase: AutoBePhase) => {
+  if (phase === "analyze") return AutoBeExampleArchiver.archiveAnalyze;
+  else if (phase === "prisma") return AutoBeExampleArchiver.archivePrisma;
+  else if (phase === "interface") return AutoBeExampleArchiver.archiveInterface;
+  else if (phase === "test") return AutoBeExampleArchiver.archiveTest;
+  else if (phase === "realize") return AutoBeExampleArchiver.archiveRealize;
+  phase satisfies never;
+  throw new Error(`Unknown phase: ${phase}`);
+};
+
+const PROJECT_SEQUENCE = ["todo", "bbs", "reddit", "shopping"] as const;
+const PHASE_SEQUENCE = [
+  "analyze",
+  "prisma",
+  "interface",
+  "test",
+  "realize",
+] as const;

package/src/example/AutoBeExampleDocumentation.ts

@@ -0,0 +1,84 @@
+import { AutoBeProgressEventBase } from "@autobe/interface";
+import { StringUtil } from "@autobe/utils";
+import typia from "typia";
+
+import { IAutoBeExampleBenchmarkState } from "../structures";
+
+export namespace AutoBeExampleDocumentation {
+  export const markdown = (state: IAutoBeExampleBenchmarkState): string =>
+    StringUtil.trim`
+      # AutoBe Example Benchmark Report
+
+      ${markdownIndex(state)}
+
+      ${state.vendors.map(markdownVendor).join("\n\n")}
+    `;
+
+  const markdownIndex = (
+    state: IAutoBeExampleBenchmarkState,
+  ): string => StringUtil.trim`
+    ## Table of Contents
+
+    ${state.vendors
+      .map(
+        (vendor) =>
+          `- [\`${vendor.name}\`](#${vendor.name
+            .replaceAll("/", "")
+            .replaceAll(":", "")})`,
+      )
+      .join("\n")}
+  `;
+
+  const markdownVendor = (
+    state: IAutoBeExampleBenchmarkState.IOfVendor,
+  ): string => StringUtil.trim`
+    ## \`${state.name}\`
+
+    Project | Phase | State | Elapsed Time
+    :-------|:------|:------|-------------:
+    ${state.projects.map(markdownProject).join("\n")}
+  `;
+
+  const markdownProject = (
+    state: IAutoBeExampleBenchmarkState.IOfProject,
+  ): string => {
+    // yellow circle emoji:
+    const phase: IAutoBeExampleBenchmarkState.IOfPhase | undefined =
+      state.phases.at(-1);
+    return [
+      state.name,
+      !!phase?.name ? `${phase.name} (${phase.trial})` : "-",
+      state.completed_at !== null
+        ? state.success
+          ? "🟢 success"
+          : "🔴 failure"
+        : phase !== undefined && phase.snapshot !== null
+          ? [
+              phase.trial !== 1 ? "🟠" : "🟡",
+              `\`${phase.snapshot.event.type}\``,
+              ...(typia.is<AutoBeProgressEventBase>(phase.snapshot.event)
+                ? [
+                    `(${phase.snapshot.event.completed} of ${phase.snapshot.event.total})`,
+                  ]
+                : []),
+            ].join(" ")
+          : "-",
+      state.started_at !== null
+        ? elapsedTime({
+            started_at: state.started_at,
+            completed_at: state.completed_at,
+          })
+        : "-",
+    ].join(" | ");
+  };
+}
+
+const elapsedTime = (props: {
+  started_at: Date;
+  completed_at: Date | null;
+}): string =>
+  Math.round(
+    ((props.completed_at ?? new Date()).getTime() -
+      props.started_at.getTime()) /
+      1_000,
+  ).toLocaleString() + " sec";

package/src/example/AutoBeExampleStorage.ts

@@ -14,6 +14,8 @@ import { Singleton, VariadicSingleton } from "tstl";
 import { v7 } from "uuid";
 
 export namespace AutoBeExampleStorage {
+  export const TEST_ROOT: string = `${__dirname}/../../../../test`;
+
   export const repository = (): string => examples.get();
   export const getDirectory = (props: {
     vendor: string;
@@ -24,7 +26,7 @@ export namespace AutoBeExampleStorage {
   export const save = async (props: {
     vendor: string;
     project: AutoBeExampleProject;
-    files: Record<string, string>;
+    files: Record<string, string | null>;
   }): Promise<void> => {
     await saveWithGzip({
       root: `${getDirectory(props)}`,
@@ -166,52 +168,51 @@ export namespace AutoBeExampleStorage {
     if (replaceSlash) model = model.replaceAll("/", "-");
     return model;
   };
-}
 
-const PROMPT_TEMPLATE = {
-  prisma: "Design the database schema.",
-  interface: "Create the API interface specification.",
-  test: "Make the e2e test functions.",
-  realize: "Implement API functions.",
-};
-const TEST_ROOT: string = `${__dirname}/../../../../test`;
+  const PROMPT_TEMPLATE = {
+    prisma: "Design the database schema.",
+    interface: "Create the API interface specification.",
+    test: "Make the e2e test functions.",
+    realize: "Implement API functions.",
+  };
 
-const examples = new Singleton(() => {
-  const location: string = `${TEST_ROOT}/../../autobe-examples`;
-  if (fs.existsSync(location) === false) {
-    cp.execSync(`git clone https://github.com/wrtnlabs/autobe-examples`, {
-      cwd: `${TEST_ROOT}/../../`,
-      stdio: "inherit",
-    });
-  }
-  cp.execSync("git pull", {
-    cwd: location,
-    stdio: "ignore",
+  const examples = new Singleton(() => {
+    const location: string = `${TEST_ROOT}/../../autobe-examples`;
+    if (fs.existsSync(location) === false)
+      cp.execSync(`git clone https://github.com/wrtnlabs/autobe-examples`, {
+        cwd: `${TEST_ROOT}/../../`,
+        stdio: "inherit",
+      });
+    if (fs.existsSync(`${location}/raw`) === false)
+      fs.mkdirSync(`${location}/raw`);
+    return location;
   });
-  if (fs.existsSync(`${location}/raw`) === false)
-    fs.mkdirSync(`${location}/raw`);
-  return location;
-});
 
-const saveWithGzip = async (props: {
-  root: string;
-  files: Record<string, string>;
-  overwrite?: boolean;
-}): Promise<void> => {
-  if (props.overwrite !== true && fs.existsSync(props.root))
-    await fs.promises.rm(props.root, {
-      recursive: true,
-    });
-  const directory = new VariadicSingleton(async (location: string) => {
-    try {
-      await fs.promises.mkdir(location, {
+  const saveWithGzip = async (props: {
+    root: string;
+    files: Record<string, string | null>;
+    overwrite?: boolean;
+  }): Promise<void> => {
+    if (props.overwrite !== true && fs.existsSync(props.root))
+      await fs.promises.rm(props.root, {
         recursive: true,
       });
-    } catch {}
-  });
-  for (const [key, value] of Object.entries(props.files)) {
-    const file: string = path.resolve(`${props.root}/${key}.gz`);
-    await directory.get(path.dirname(file));
-    await fs.promises.writeFile(file, await CompressUtil.gzip(value ?? ""));
-  }
-};
+    const directory = new VariadicSingleton(async (location: string) => {
+      try {
+        await fs.promises.mkdir(location, {
+          recursive: true,
+        });
+      } catch {}
+    });
+    for (const [key, value] of Object.entries(props.files)) {
+      const file: string = path.resolve(`${props.root}/${key}.gz`);
+      await directory.get(path.dirname(file));
+      if (value !== null)
+        await fs.promises.writeFile(file, await CompressUtil.gzip(value ?? ""));
+      else
+        try {
+          await fs.promises.unlink(file);
+        } catch {}
+    }
+  };
+}

package/src/example/index.ts

@@ -1 +1,4 @@
+export * from "./AutoBeExampleArchiver";
+export * from "./AutoBeExampleBenchmark";
 export * from "./AutoBeExampleStorage";
+export * from "./AutoBeExampleDocumentation";

package/src/replay/AutoBeReplayComputer.ts

@@ -1,4 +1,3 @@
-import { AutoBeProcessAggregateFactory } from "@autobe/agent/src/factory/AutoBeProcessAggregateFactory";
 import {
   AutoBeExampleProject,
   AutoBeHistory,
@@ -6,6 +5,7 @@ import {
   IAutoBePlaygroundBenchmarkScore,
   IAutoBePlaygroundReplay,
 } from "@autobe/interface";
+import { AutoBeProcessAggregateFactory } from "@autobe/utils";
 
 export namespace AutoBeReplayComputer {
   export const SIGNIFICANT_PROJECTS: AutoBeExampleProject[] = [

package/src/replay/AutoBeReplayDocumentation.ts

@@ -15,8 +15,8 @@ export namespace AutoBeReplayDocumentation {
   
       ## Benchmark
   
-      AI Model | Score | FCSR | Status 
-      :--------|------:|-----:|:------:
+      AI Model | Success | Score | FCSR | Status 
+      :--------|---------|------:|-----:|:------:
       ${experiments
         .map((e) =>
           [
@@ -26,6 +26,7 @@ export namespace AutoBeReplayDocumentation {
             )}\`](#${AutoBeExampleStorage.slugModel(e.vendor, false)
               .replaceAll("/", "")
               .replaceAll(".", "")})`,
+            e.replays.filter((r) => r.realize?.success === true).length,
             e.score.aggregate,
             (() => {
               const [x, y] = e.replays

package/src/replay/AutoBeReplayStorage.ts

@@ -1,3 +1,4 @@
+import { CompressUtil } from "@autobe/filesystem";
 import {
   AutoBeEventSnapshot,
   AutoBeExampleProject,
@@ -5,6 +6,7 @@ import {
   AutoBePhase,
   IAutoBePlaygroundReplay,
 } from "@autobe/interface";
+import fs from "fs";
 import typia from "typia";
 
 import { AutoBeExampleStorage } from "../example/AutoBeExampleStorage";
@@ -28,6 +30,22 @@ export namespace AutoBeReplayStorage {
     return replays.filter((r) => r !== null);
   };
 
+  export const getAllSummaries = async (
+    vendor: string,
+    projectFilter?: (project: AutoBeExampleProject) => boolean,
+  ): Promise<IAutoBePlaygroundReplay.ISummary[]> => {
+    const projects: AutoBeExampleProject[] = typia.misc
+      .literals<AutoBeExampleProject>()
+      .filter(projectFilter ?? (() => true));
+    const summaries: Array<IAutoBePlaygroundReplay.ISummary | null> =
+      await Promise.all(
+        projects.map((project) =>
+          AutoBeReplayStorage.getSummary({ vendor, project }),
+        ),
+      );
+    return summaries.filter((s) => s !== null);
+  };
+
   export const get = async (props: {
     vendor: string;
     project: AutoBeExampleProject;
@@ -60,6 +78,17 @@ export namespace AutoBeReplayStorage {
     };
   };
 
+  export const getSummary = async (props: {
+    vendor: string;
+    project: AutoBeExampleProject;
+  }): Promise<IAutoBePlaygroundReplay.ISummary | null> => {
+    const location: string = `${AutoBeExampleStorage.getDirectory(props)}/summary.json.gz`;
+    if (fs.existsSync(location) === false) return null;
+    return JSON.parse(
+      await CompressUtil.gunzip(await fs.promises.readFile(location)),
+    );
+  };
+
   const getHistories = async (props: {
     vendor: string;
     project: AutoBeExampleProject;

package/src/structures/IAutoBeExampleBenchmarkState.ts

@@ -0,0 +1,30 @@
+import {
+  AutoBeEventSnapshot,
+  AutoBeExampleProject,
+  AutoBePhase,
+} from "@autobe/interface";
+
+export interface IAutoBeExampleBenchmarkState {
+  vendors: IAutoBeExampleBenchmarkState.IOfVendor[];
+}
+export namespace IAutoBeExampleBenchmarkState {
+  export interface IOfVendor {
+    name: string;
+    projects: IOfProject[];
+  }
+  export interface IOfProject {
+    name: AutoBeExampleProject;
+    phases: IOfPhase[];
+    success: boolean | null;
+    started_at: Date | null;
+    completed_at: Date | null;
+  }
+  export interface IOfPhase {
+    name: AutoBePhase;
+    snapshot: AutoBeEventSnapshot | null;
+    success: boolean | null;
+    started_at: Date;
+    completed_at: Date | null;
+    trial: number;
+  }
+}

package/src/structures/index.ts

@@ -0,0 +1 @@
+export * from "./IAutoBeExampleBenchmarkState";