@autobe/benchmark 0.29.0 → 0.29.1

package/lib/replay/AutoBeReplayStorage.d.ts

@@ -1,8 +1,13 @@
 import { AutoBeExampleProject, IAutoBePlaygroundReplay } from "@autobe/interface";
 export declare namespace AutoBeReplayStorage {
     const getAll: (vendor: string, projectFilter?: (project: AutoBeExampleProject) => boolean) => Promise<IAutoBePlaygroundReplay[]>;
+    const getAllSummaries: (vendor: string, projectFilter?: (project: AutoBeExampleProject) => boolean) => Promise<IAutoBePlaygroundReplay.ISummary[]>;
     const get: (props: {
         vendor: string;
         project: AutoBeExampleProject;
     }) => Promise<IAutoBePlaygroundReplay | null>;
+    const getSummary: (props: {
+        vendor: string;
+        project: AutoBeExampleProject;
+    }) => Promise<IAutoBePlaygroundReplay.ISummary | null>;
 }

package/lib/replay/AutoBeReplayStorage.js

@@ -13,6 +13,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AutoBeReplayStorage = void 0;
+const filesystem_1 = require("@autobe/filesystem");
+const fs_1 = __importDefault(require("fs"));
 const typia_1 = __importDefault(require("typia"));
 const AutoBeExampleStorage_1 = require("../example/AutoBeExampleStorage");
 var AutoBeReplayStorage;
@@ -27,6 +29,13 @@ var AutoBeReplayStorage;
         })));
         return replays.filter((r) => r !== null);
     });
+    AutoBeReplayStorage.getAllSummaries = (vendor, projectFilter) => __awaiter(this, void 0, void 0, function* () {
+        const projects = typia_1.default.misc
+            .literals()
+            .filter(projectFilter !== null && projectFilter !== void 0 ? projectFilter : (() => true));
+        const summaries = yield Promise.all(projects.map((project) => AutoBeReplayStorage.getSummary({ vendor, project })));
+        return summaries.filter((s) => s !== null);
+    });
     AutoBeReplayStorage.get = (props) => __awaiter(this, void 0, void 0, function* () {
         const histories = yield getHistories(props);
         if (histories === null)
@@ -54,6 +63,12 @@ var AutoBeReplayStorage;
             realize: yield snapshots("realize"),
         };
     });
+    AutoBeReplayStorage.getSummary = (props) => __awaiter(this, void 0, void 0, function* () {
+        const location = `${AutoBeExampleStorage_1.AutoBeExampleStorage.getDirectory(props)}/summary.json.gz`;
+        if (fs_1.default.existsSync(location) === false)
+            return null;
+        return JSON.parse(yield filesystem_1.CompressUtil.gunzip(yield fs_1.default.promises.readFile(location)));
+    });
     const getHistories = (props) => __awaiter(this, void 0, void 0, function* () {
         for (const phase of SEQUENCE) {
             try {

package/lib/replay/AutoBeReplayStorage.js.map

@@ -1 +1 @@
-{"version":3,"file":"AutoBeReplayStorage.js","sourceRoot":"","sources":["../../src/replay/AutoBeReplayStorage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAOA,kDAA0B;AAE1B,0EAAuE;AAEvE,IAAiB,mBAAmB,CAkEnC;AAlED,WAAiB,mBAAmB;IACrB,0BAAM,GAAG,CACpB,MAAc,EACd,aAA0D,EACtB,EAAE;QACtC,MAAM,QAAQ,GAA2B,eAAK,CAAC,IAAI;aAChD,QAAQ,EAAwB;aAChC,MAAM,CAAC,aAAa,aAAb,aAAa,cAAb,aAAa,GAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,MAAM,OAAO,GAA0C,MAAM,OAAO,CAAC,GAAG,CACtE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACjB,mBAAmB,CAAC,GAAG,CAAC;YACtB,MAAM;YACN,OAAO,EAAE,CAAC;SACX,CAAC,CACH,CACF,CAAC;QACF,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAC3C,CAAC,CAAA,CAAC;IAEW,uBAAG,GAAG,CAAO,KAGzB,EAA2C,EAAE;QAC5C,MAAM,SAAS,GAA2B,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;QACpE,IAAI,SAAS,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAEpC,MAAM,SAAS,GAAG,CAChB,KAAkB,EACqB,EAAE;YACzC,IAAI,CAAC;gBACH,OAAO,MAAM,2CAAoB,CAAC,YAAY,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;YAAC,WAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAA,CAAC;QACF,OAAO;YACL,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS;YACT,OAAO,EAAE,MAAM,SAAS,CAAC,SAAS,CAAC;YACnC,MAAM,EAAE,MAAM,SAAS,CAAC,QAAQ,CAAC;YACjC,SAAS,EAAE,MAAM,SAAS,CAAC,WAAW,CAAC;YACvC,IAAI,EAAE,MAAM,SAAS,CAAC,MAAM,CAAC;YAC7B,OAAO,EAAE,MAAM,SAAS,CAAC,SAAS,CAAC;SACpC,CAAC;IACJ,CAAC,CAAA,CAAC;IAEF,MAAM,YAAY,GAAG,CAAO,KAG3B,EAAmC,EAAE;QACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,OAAO,MAAM,2CAAoB,CAAC,YAAY,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;YAAC,WAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAA,CAAC;AACJ,CAAC,EAlEgB,mBAAmB,mCAAnB,mBAAmB,QAkEnC;AAED,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAU,CAAC"}
+{"version":3,"file":"AutoBeReplayStorage.js","sourceRoot":"","sources":["../../src/replay/AutoBeReplayStorage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,mDAAkD;AAQlD,4CAAoB;AACpB,kDAA0B;AAE1B,0EAAuE;AAEvE,IAAiB,mBAAmB,CA6FnC;AA7FD,WAAiB,mBAAmB;IACrB,0BAAM,GAAG,CACpB,MAAc,EACd,aAA0D,EACtB,EAAE;QACtC,MAAM,QAAQ,GAA2B,eAAK,CAAC,IAAI;aAChD,QAAQ,EAAwB;aAChC,MAAM,CAAC,aAAa,aAAb,aAAa,cAAb,aAAa,GAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,MAAM,OAAO,GAA0C,MAAM,OAAO,CAAC,GAAG,CACtE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACjB,mBAAmB,CAAC,GAAG,CAAC;YACtB,MAAM;YACN,OAAO,EAAE,CAAC;SACX,CAAC,CACH,CACF,CAAC;QACF,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAC3C,CAAC,CAAA,CAAC;IAEW,mCAAe,GAAG,CAC7B,MAAc,EACd,aAA0D,EACb,EAAE;QAC/C,MAAM,QAAQ,GAA2B,eAAK,CAAC,IAAI;aAChD,QAAQ,EAAwB;aAChC,MAAM,CAAC,aAAa,aAAb,aAAa,cAAb,aAAa,GAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,MAAM,SAAS,GACb,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACvB,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CACpD,CACF,CAAC;QACJ,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAC7C,CAAC,CAAA,CAAC;IAEW,uBAAG,GAAG,CAAO,KAGzB,EAA2C,EAAE;QAC5C,MAAM,SAAS,GAA2B,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;QACpE,IAAI,SAAS,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAEpC,MAAM,SAAS,GAAG,CAChB,KAAkB,EACqB,EAAE;YACzC,IAAI,CAAC;gBACH,OAAO,MAAM,2CAAoB,CAAC,YAAY,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;YAAC,WAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAA,CAAC;QACF,OAAO;YACL,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS;YACT,OAAO,EAAE,MAAM,SAAS,CAAC,SAAS,CAAC;YACnC,MAAM,EAAE,MAAM,SAAS,CAAC,QAAQ,CAAC;YACjC,SAAS,EAAE,MAAM,SAAS,CAAC,WAAW,CAAC;YACvC,IAAI,EAAE,MAAM,SAAS,CAAC,MAAM,CAAC;YAC7B,OAAO,EAAE,MAAM,SAAS,CAAC,SAAS,CAAC;SACpC,CAAC;IACJ,CAAC,CAAA,CAAC;IAEW,8BAAU,GAAG,CAAO,KAGhC,EAAoD,EAAE;QACrD,MAAM,QAAQ,GAAW,GAAG,2CAAoB,CAAC,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC;QACvF,IAAI,YAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,KAAK;YAAE,OAAO,IAAI,CAAC;QACnD,OAAO,IAAI,CAAC,KAAK,CACf,MAAM,yBAAY,CAAC,MAAM,CAAC,MAAM,YAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAChE,CAAC;IACJ,CAAC,CAAA,CAAC;IAEF,MAAM,YAAY,GAAG,CAAO,KAG3B,EAAmC,EAAE;QACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,OAAO,MAAM,2CAAoB,CAAC,YAAY,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;YAAC,WAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAA,CAAC;AACJ,CAAC,EA7FgB,mBAAmB,mCAAnB,mBAAmB,QA6FnC;AAED,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAU,CAAC"}

package/lib/structures/IAutoBeExampleBenchmarkState.d.ts

@@ -0,0 +1,24 @@
+import { AutoBeEventSnapshot, AutoBeExampleProject, AutoBePhase } from "@autobe/interface";
+export interface IAutoBeExampleBenchmarkState {
+    vendors: IAutoBeExampleBenchmarkState.IOfVendor[];
+}
+export declare namespace IAutoBeExampleBenchmarkState {
+    interface IOfVendor {
+        name: string;
+        projects: IOfProject[];
+    }
+    interface IOfProject {
+        name: AutoBeExampleProject;
+        phases: IOfPhase[];
+        success: boolean | null;
+        started_at: Date;
+        completed_at: Date | null;
+    }
+    interface IOfPhase {
+        name: AutoBePhase;
+        snapshot: AutoBeEventSnapshot | null;
+        success: boolean | null;
+        started_at: Date;
+        completed_at: Date | null;
+    }
+}

package/lib/structures/IAutoBeExampleBenchmarkState.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=IAutoBeExampleBenchmarkState.js.map

package/lib/structures/IAutoBeExampleBenchmarkState.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IAutoBeExampleBenchmarkState.js","sourceRoot":"","sources":["../../src/structures/IAutoBeExampleBenchmarkState.ts"],"names":[],"mappings":""}

package/lib/structures/index.d.ts

@@ -0,0 +1 @@
+export * from "./IAutoBeExampleBenchmarkState";

package/lib/structures/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./IAutoBeExampleBenchmarkState"), exports);
+//# sourceMappingURL=index.js.map

package/lib/structures/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/structures/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,iEAA+C"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autobe/benchmark",
-  "version": "0.29.0",
+  "version": "0.29.1",
   "description": "AI backend server code generator",
   "main": "lib/index.js",
   "keywords": [],
@@ -27,10 +27,9 @@
     "tstl": "^3.0.0",
     "typia": "^10.0.2",
     "uuid": "^11.1.0",
-    "@autobe/agent": "^0.29.0",
-    "@autobe/filesystem": "^0.29.0",
-    "@autobe/interface": "^0.29.0",
-    "@autobe/utils": "^0.29.0"
+    "@autobe/filesystem": "^0.29.1",
+    "@autobe/utils": "^0.29.1",
+    "@autobe/interface": "^0.29.1"
   },
   "devDependencies": {
     "@types/uuid": "^10.0.0",

package/src/example/AutoBeExampleArchiver.ts

@@ -0,0 +1,334 @@
+import { FileSystemIterator } from "@autobe/filesystem";
+import {
+  AutoBeAggregateEventBase,
+  AutoBeEventOfSerializable,
+  AutoBeEventSnapshot,
+  AutoBeExampleProject,
+  AutoBeHistory,
+  AutoBePhase,
+  AutoBePrismaHistory,
+  AutoBeProcessAggregateCollection,
+  AutoBeUserMessageContent,
+  IAutoBeAgent,
+  IAutoBePlaygroundReplay,
+  IAutoBeTokenUsageJson,
+} from "@autobe/interface";
+import {
+  AutoBeProcessAggregateFactory,
+  TokenUsageComputer,
+} from "@autobe/utils";
+import typia from "typia";
+
+import { AutoBeReplayComputer } from "../replay";
+import { AutoBeExampleStorage } from "./AutoBeExampleStorage";
+
+export namespace AutoBeExampleArchiver {
+  export interface IContext {
+    vendor: string;
+    project: AutoBeExampleProject;
+    agent: (props: IAgentProps) => Promise<IAutoBeAgent>;
+    on: (snapshot: AutoBeEventSnapshot) => void;
+  }
+
+  export interface IAgentProps {
+    histories: AutoBeHistory[];
+    tokenUsage: IAutoBeTokenUsageJson;
+  }
+
+  export const archiveAnalyze = (ctx: IContext): Promise<boolean> =>
+    archive(ctx, {
+      phase: "analyze",
+      trial: async (conversate): Promise<boolean> =>
+        (await conversate(
+          await AutoBeExampleStorage.getUserMessage({
+            project: ctx.project,
+            phase: "analyze",
+          }).then((r) => r.contents),
+        )) ||
+        (await conversate(
+          "I'm not familiar with the analyze feature. Please determine everything by yourself, and just show me the analysis report.",
+        )) ||
+        (await conversate(
+          "I already told you to publish the analysis report. Never ask me anything, and just do it right now.",
+        )),
+      predicate: (histories): boolean =>
+        histories.some((h) => h.type === "analyze"),
+    });
+
+  export const archivePrisma = (ctx: IContext): Promise<boolean> =>
+    archive(ctx, {
+      phase: "prisma",
+      trial: getTrial({
+        project: ctx.project,
+        phase: "prisma",
+      }),
+      predicate: (histories): boolean => {
+        const prisma: AutoBePrismaHistory | undefined = histories.find(
+          (h) => h.type === "prisma",
+        );
+        return prisma !== undefined && prisma.compiled.type === "success";
+      },
+    });
+
+  export const archiveInterface = (ctx: IContext): Promise<boolean> =>
+    archive(ctx, {
+      phase: "interface",
+      trial: getTrial({
+        project: ctx.project,
+        phase: "interface",
+      }),
+      predicate: (histories): boolean => {
+        const interfaceHistory: AutoBeHistory | undefined = histories.find(
+          (h) => h.type === "interface",
+        );
+        return (
+          interfaceHistory !== undefined && interfaceHistory.missed.length === 0
+        );
+      },
+    });
+
+  export const archiveTest = (ctx: IContext): Promise<boolean> =>
+    archive(ctx, {
+      phase: "test",
+      trial: getTrial({
+        project: ctx.project,
+        phase: "test",
+      }),
+      predicate: (histories): boolean => {
+        const testHistory: AutoBeHistory | undefined = histories.find(
+          (h) => h.type === "test",
+        );
+        return (
+          testHistory !== undefined && testHistory.compiled.type === "success"
+        );
+      },
+    });
+
+  export const archiveRealize = (ctx: IContext): Promise<boolean> =>
+    archive(ctx, {
+      phase: "realize",
+      trial: getTrial({
+        project: ctx.project,
+        phase: "realize",
+      }),
+      predicate: (histories): boolean => {
+        const realizeHistory: AutoBeHistory | undefined = histories.find(
+          (h) => h.type === "realize",
+        );
+        return (
+          realizeHistory !== undefined &&
+          realizeHistory.compiled.type === "success"
+        );
+      },
+    });
+
+  const archive = async (
+    ctx: IContext,
+    props: {
+      phase: AutoBePhase;
+      trial: (
+        conversate: (
+          input: string | AutoBeUserMessageContent | AutoBeUserMessageContent[],
+        ) => Promise<boolean>,
+      ) => Promise<boolean>;
+      predicate: (histories: AutoBeHistory[]) => boolean;
+    },
+  ): Promise<boolean> => {
+    // INITIALIZE AGENT
+    const asset: IAgentProps = await getAsset({
+      vendor: ctx.vendor,
+      project: ctx.project,
+      phase: props.phase,
+    });
+    const agent: IAutoBeAgent = await ctx.agent(asset);
+    const snapshots: AutoBeEventSnapshot[] = [];
+    for (const type of typia.misc.literals<AutoBeEventOfSerializable.Type>()) {
+      agent.on(type, (e) => {
+        const s: AutoBeEventSnapshot = {
+          event: e,
+          tokenUsage: agent.getTokenUsage(),
+        };
+        ctx.on(s);
+        snapshots.push(s);
+      });
+    }
+
+    const summarize = async (
+      histories: AutoBeHistory[],
+      error: boolean,
+    ): Promise<void> => {
+      const replay: IAutoBePlaygroundReplay = {
+        vendor: ctx.vendor,
+        project: ctx.project,
+        histories,
+        analyze: null,
+        prisma: null,
+        interface: null,
+        test: null,
+        realize: null,
+      };
+      for (const phase of PHASES)
+        if (phase === props.phase) {
+          replay[phase] = snapshots;
+          break;
+        } else
+          replay[phase] = await AutoBeExampleStorage.getSnapshots({
+            vendor: ctx.vendor,
+            project: ctx.project,
+            phase,
+          });
+      const summary: IAutoBePlaygroundReplay.ISummary =
+        AutoBeReplayComputer.summarize(replay);
+      if (error === true) {
+        const aggregates: AutoBeProcessAggregateCollection =
+          AutoBeProcessAggregateFactory.createCollection();
+        for (const { event } of snapshots) {
+          if (typia.is<AutoBeAggregateEventBase>(event) === false) continue;
+          AutoBeProcessAggregateFactory.emplaceEvent(aggregates, event);
+        }
+        summary[props.phase] = {
+          aggregates,
+          success: false,
+          elapsed: 0,
+          commodity: {},
+        };
+      }
+      await AutoBeExampleStorage.save({
+        vendor: ctx.vendor,
+        project: ctx.project,
+        files: {
+          [`summary.json`]: JSON.stringify(summary),
+        },
+      });
+    };
+
+    try {
+      // CONVERSATE
+      const go = async (
+        c: string | AutoBeUserMessageContent | AutoBeUserMessageContent[],
+      ): Promise<boolean> => {
+        const result: AutoBeHistory[] = await agent.conversate(c);
+        return result.some((h) => h.type === props.phase);
+      };
+      const done: boolean = await props.trial(go);
+      if (done === false)
+        throw new Error(
+          `Failed to function calling in the "${props.phase}" phase of the "${ctx.project}" project.`,
+        );
+
+      // AGGREGATE
+      const histories: AutoBeHistory[] = agent.getHistories();
+      try {
+        await FileSystemIterator.save({
+          root: `${AutoBeExampleStorage.TEST_ROOT}/results/${AutoBeExampleStorage.slugModel(ctx.vendor, false)}/${ctx.project}/${props.phase}`,
+          files: {
+            ...(await agent.getFiles()),
+            ...Object.fromEntries(
+              histories
+                .filter(
+                  (h) =>
+                    h.type === "prisma" ||
+                    h.type === "interface" ||
+                    h.type === "test" ||
+                    h.type === "realize",
+                )
+                .map((h) => [`autobe/${h.type}.instruction.md`, h.instruction]),
+            ),
+          },
+        });
+      } catch {}
+      await AutoBeExampleStorage.save({
+        vendor: ctx.vendor,
+        project: ctx.project,
+        files: {
+          [`${props.phase}.histories.json`]: JSON.stringify(histories),
+          [`${props.phase}.snapshots.json`]: JSON.stringify(snapshots),
+          [`${props.phase}.error.json`]: null,
+        },
+      });
+      await summarize([...asset.histories, ...histories], false);
+      return props.predicate(histories);
+    } catch (error) {
+      if (error instanceof Error)
+        await AutoBeExampleStorage.save({
+          vendor: ctx.vendor,
+          project: ctx.project,
+          files: {
+            [`${props.phase}.snapshots.json`]: JSON.stringify(snapshots),
+            [`${props.phase}.error.json`]: JSON.stringify({
+              ...error,
+              name: error.name,
+              message: error.message,
+              stack: error.stack,
+            }),
+          },
+        });
+      await summarize(asset.histories, true);
+      throw error;
+    }
+  };
+
+  const getTrial =
+    (props: { project: AutoBeExampleProject; phase: AutoBePhase }) =>
+    async (
+      conversate: (
+        input: string | AutoBeUserMessageContent | AutoBeUserMessageContent[],
+      ) => Promise<boolean>,
+    ): Promise<boolean> =>
+      (await conversate(
+        await AutoBeExampleStorage.getUserMessage(props).then(
+          (r) => r.contents,
+        ),
+      )) ||
+      (await conversate(
+        "Don't ask me to do that, and just do it right now.",
+      )) ||
+      (await conversate(
+        `I already told you to do ${props.phase} process. Never ask me anything, and just do it right now. Go go go!`,
+      ));
+
+  const getAsset = async (props: {
+    vendor: string;
+    project: AutoBeExampleProject;
+    phase: AutoBePhase;
+  }): Promise<IAgentProps> => {
+    const previous: AutoBePhase | null =
+      PHASES[PHASES.indexOf(props.phase) - 1] ?? null;
+    if (previous === null)
+      return {
+        histories: [],
+        tokenUsage: {
+          aggregate: TokenUsageComputer.zero(),
+          facade: TokenUsageComputer.zero(),
+          analyze: TokenUsageComputer.zero(),
+          prisma: TokenUsageComputer.zero(),
+          interface: TokenUsageComputer.zero(),
+          test: TokenUsageComputer.zero(),
+          realize: TokenUsageComputer.zero(),
+        },
+      };
+    const histories: AutoBeHistory[] = await AutoBeExampleStorage.getHistories({
+      vendor: props.vendor,
+      project: props.project,
+      phase: previous,
+    });
+    const tokenUsage: IAutoBeTokenUsageJson =
+      await AutoBeExampleStorage.getTokenUsage({
+        vendor: props.vendor,
+        project: props.project,
+        phase: previous,
+      });
+    return {
+      histories,
+      tokenUsage,
+    };
+  };
+}
+
+const PHASES: AutoBePhase[] = [
+  "analyze",
+  "prisma",
+  "interface",
+  "test",
+  "realize",
+];

package/src/example/AutoBeExampleBenchmark.ts

@@ -0,0 +1,143 @@
+import {
+  AutoBeEvent,
+  AutoBeExampleProject,
+  AutoBePhase,
+  IAutoBeAgent,
+} from "@autobe/interface";
+
+import { IAutoBeExampleBenchmarkState } from "../structures/IAutoBeExampleBenchmarkState";
+import { AutoBeExampleArchiver } from "./AutoBeExampleArchiver";
+
+export namespace AutoBeExampleBenchmark {
+  export interface IContext {
+    createAgent: (
+      props: AutoBeExampleArchiver.IAgentProps,
+    ) => Promise<IAutoBeAgent>;
+  }
+
+  export const execute = async (
+    ctx: IContext,
+    props: {
+      vendors: string[];
+      projects?: AutoBeExampleProject[];
+      phases?: AutoBePhase[];
+      progress: (state: IAutoBeExampleBenchmarkState) => void;
+      on?: (event: AutoBeEvent) => void;
+    },
+  ): Promise<void> => {
+    const state: IAutoBeExampleBenchmarkState = {
+      vendors: props.vendors.map(
+        (vendor): IAutoBeExampleBenchmarkState.IOfVendor => ({
+          name: vendor,
+          projects: PROJECT_SEQUENCE.filter(
+            (p) => !props.projects || props.projects.includes(p),
+          ).map(
+            (project): IAutoBeExampleBenchmarkState.IOfProject => ({
+              name: project,
+              phases: [],
+              success: null,
+              started_at: new Date(),
+              completed_at: null,
+            }),
+          ),
+        }),
+      ),
+    };
+    const report = () => props.progress(state);
+    await Promise.all(
+      state.vendors.map((vendor) =>
+        executeVendor(ctx, {
+          phases: props.phases,
+          vendorState: vendor,
+          on: props.on,
+          report,
+        }),
+      ),
+    );
+  };
+
+  const executeVendor = async (
+    ctx: IContext,
+    props: {
+      vendorState: IAutoBeExampleBenchmarkState.IOfVendor;
+      phases?: AutoBePhase[];
+      report: () => void;
+      on?: (event: AutoBeEvent) => void;
+    },
+  ): Promise<void> => {
+    for (const project of props.vendorState.projects)
+      await executeProject(ctx, {
+        vendor: props.vendorState.name,
+        projectState: project,
+        phases: props.phases,
+        report: props.report,
+        on: props.on,
+      });
+  };
+
+  const executeProject = async (
+    ctx: IContext,
+    props: {
+      vendor: string;
+      projectState: IAutoBeExampleBenchmarkState.IOfProject;
+      phases?: AutoBePhase[];
+      report: () => void;
+      on?: (event: AutoBeEvent) => void;
+    },
+  ): Promise<void> => {
+    for (const phase of PHASE_SEQUENCE) {
+      if (props.phases && props.phases.includes(phase) === false) continue;
+      const phaseState: IAutoBeExampleBenchmarkState.IOfPhase = {
+        name: phase,
+        snapshot: null,
+        success: null,
+        started_at: new Date(),
+        completed_at: null,
+      };
+      props.projectState.phases.push(phaseState);
+      for (let i: number = 0; i < 3; ++i) {
+        try {
+          phaseState.started_at = new Date();
+          phaseState.completed_at = null;
+          const success: boolean = await getArchiver(phase)({
+            vendor: props.vendor,
+            project: props.projectState.name,
+            agent: (next) => ctx.createAgent(next),
+            on: (s) => {
+              phaseState.snapshot = s;
+              props.report();
+              if (props.on) props.on(s.event);
+            },
+          });
+          phaseState.success = success;
+          phaseState.completed_at = new Date();
+          props.report();
+          if (success === true) break;
+        } catch (error) {
+          continue;
+        }
+      }
+      if (phaseState.success === null) break;
+      else if (phaseState.success === false && phaseState.name !== "test")
+        break;
+    }
+    props.projectState.completed_at = new Date();
+    props.projectState.success = props.projectState.phases.every(
+      (phase) => phase.success === true,
+    );
+    props.report();
+  };
+}
+
+const getArchiver = (phase: AutoBePhase) => {
+  if (phase === "analyze") return AutoBeExampleArchiver.archiveAnalyze;
+  else if (phase === "prisma") return AutoBeExampleArchiver.archivePrisma;
+  else if (phase === "interface") return AutoBeExampleArchiver.archiveInterface;
+  else if (phase === "test") return AutoBeExampleArchiver.archiveTest;
+  else if (phase === "realize") return AutoBeExampleArchiver.archiveRealize;
+  phase satisfies never;
+  throw new Error(`Unknown phase: ${phase}`);
+};
+
+const PROJECT_SEQUENCE = ["todo", "bbs", "reddit", "shopping"] as const;
+const PHASE_SEQUENCE = ["analyze", "interface", "test", "realize"] as const;