@autobe/benchmark 0.27.0

package/lib/replay/AutoBeReplayStorage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutoBeReplayStorage.js","sourceRoot":"","sources":["../../src/replay/AutoBeReplayStorage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAOA,kDAA0B;AAE1B,0EAAuE;AAEvE,IAAiB,mBAAmB,CAkEnC;AAlED,WAAiB,mBAAmB;IACrB,0BAAM,GAAG,CACpB,MAAc,EACd,aAA0D,EACtB,EAAE;QACtC,MAAM,QAAQ,GAA2B,eAAK,CAAC,IAAI;aAChD,QAAQ,EAAwB;aAChC,MAAM,CAAC,aAAa,aAAb,aAAa,cAAb,aAAa,GAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,MAAM,OAAO,GAA0C,MAAM,OAAO,CAAC,GAAG,CACtE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACjB,mBAAmB,CAAC,GAAG,CAAC;YACtB,MAAM;YACN,OAAO,EAAE,CAAC;SACX,CAAC,CACH,CACF,CAAC;QACF,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAC3C,CAAC,CAAA,CAAC;IAEW,uBAAG,GAAG,CAAO,KAGzB,EAA2C,EAAE;QAC5C,MAAM,SAAS,GAA2B,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;QACpE,IAAI,SAAS,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAEpC,MAAM,SAAS,GAAG,CAChB,KAAkB,EACqB,EAAE;YACzC,IAAI,CAAC;gBACH,OAAO,MAAM,2CAAoB,CAAC,YAAY,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;YAAC,WAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAA,CAAC;QACF,OAAO;YACL,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS;YACT,OAAO,EAAE,MAAM,SAAS,CAAC,SAAS,CAAC;YACnC,MAAM,EAAE,MAAM,SAAS,CAAC,QAAQ,CAAC;YACjC,SAAS,EAAE,MAAM,SAAS,CAAC,WAAW,CAAC;YACvC,IAAI,EAAE,MAAM,SAAS,CAAC,MAAM,CAAC;YAC7B,OAAO,EAAE,MAAM,SAAS,CAAC,SAAS,CAAC;SACpC,CAAC;IACJ,CAAC,CAAA,CAAC;IAEF,MAAM,YAAY,GAAG,CAAO,KAG3B,EAAmC,EAAE;QACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,OAAO,MAAM,2CAAoB,CAAC,YAAY,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;YAAC,WAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAA,CAAC;AACJ,CAAC,EAlEgB,mBAAmB,mCAAnB,mBAAmB,QAkEnC;AAED,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAU,CAAC"}

package/lib/replay/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./AutoBeReplayComputer";
+export * from "./AutoBeReplayDocumentation";
+export * from "./AutoBeReplayStorage";

package/lib/replay/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./AutoBeReplayComputer"), exports);
+__exportStar(require("./AutoBeReplayDocumentation"), exports);
+__exportStar(require("./AutoBeReplayStorage"), exports);
+//# sourceMappingURL=index.js.map

package/lib/replay/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/replay/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,yDAAuC;AACvC,8DAA4C;AAC5C,wDAAsC"}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@autobe/benchmark",
+  "version": "0.27.0",
+  "description": "AI backend server code generator",
+  "main": "lib/index.js",
+  "keywords": [],
+  "author": "Wrtn Technologies",
+  "license": "AGPL-3.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/wrtnlabs/autobe"
+  },
+  "bugs": {
+    "url": "https://github.com/wrtnlabs/autobe/issues"
+  },
+  "files": [
+    "lib",
+    "src",
+    "package.json",
+    "LICENSE",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "tstl": "^3.0.0",
+    "typia": "^9.7.2",
+    "uuid": "^11.1.0",
+    "@autobe/agent": "^0.27.0",
+    "@autobe/filesystem": "^0.27.0",
+    "@autobe/utils": "^0.27.0",
+    "@autobe/interface": "^0.27.0"
+  },
+  "devDependencies": {
+    "@types/uuid": "^10.0.0",
+    "rimraf": "^6.0.1",
+    "ts-patch": "^3.3.0",
+    "typescript": "~5.9.2"
+  },
+  "scripts": {
+    "build": "rimraf lib && tsc",
+    "postbuild": "node ../../internals/config/assertBuild.js",
+    "dev": "rimraf lib && tsc --watch"
+  },
+  "typings": "lib/index.d.ts"
+}

package/src/example/AutoBeExampleStorage.ts

@@ -0,0 +1,213 @@
+import { CompressUtil } from "@autobe/filesystem";
+import {
+  AutoBeEventSnapshot,
+  AutoBeExampleProject,
+  AutoBeHistory,
+  AutoBePhase,
+  AutoBeUserMessageHistory,
+  IAutoBeTokenUsageJson,
+} from "@autobe/interface";
+import cp from "child_process";
+import fs from "fs";
+import path from "path";
+import { Singleton, VariadicSingleton } from "tstl";
+import { v7 } from "uuid";
+
+export namespace AutoBeExampleStorage {
+  export const repository = (): string => examples.get();
+  export const getDirectory = (props: {
+    vendor: string;
+    project: string;
+  }): string =>
+    `${examples.get()}/raw/${slugModel(props.vendor, false)}/${props.project}`;
+
+  export const save = async (props: {
+    vendor: string;
+    project: AutoBeExampleProject;
+    files: Record<string, string>;
+  }): Promise<void> => {
+    await saveWithGzip({
+      root: `${getDirectory(props)}`,
+      files: props.files,
+      overwrite: true,
+    });
+  };
+
+  export const getUserMessage = async (props: {
+    project: AutoBeExampleProject;
+    phase: AutoBePhase;
+  }): Promise<AutoBeUserMessageHistory> => {
+    const full: string = `${TEST_ROOT}/scripts/${props.project}/${props.phase}`;
+    if (fs.existsSync(`${full}.md`) === false) {
+      const text: string =
+        props.phase === "analyze"
+          ? await fs.promises.readFile(
+              `${TEST_ROOT}/scripts/${props.project}.md`,
+              "utf8",
+            )
+          : PROMPT_TEMPLATE[props.phase];
+      return {
+        type: "userMessage",
+        id: v7(),
+        created_at: new Date().toISOString(),
+        contents: [
+          {
+            type: "text",
+            text,
+          },
+        ],
+      };
+    }
+    const text: string = await fs.promises.readFile(`${full}.md`, "utf8");
+    return {
+      type: "userMessage",
+      id: v7(),
+      created_at: new Date().toISOString(),
+      contents: [
+        {
+          type: "text",
+          text: text,
+        },
+      ],
+    };
+  };
+
+  export const getVendorModels = async (): Promise<string[]> => {
+    const result: string[] = [];
+    const repoPath: string = repository();
+    for (const vendor of await fs.promises.readdir(repoPath))
+      for (const model of await fs.promises.readdir(`${repoPath}/${vendor}`)) {
+        const stat: fs.Stats = await fs.promises.lstat(
+          `${repoPath}/${vendor}/${model}`,
+        );
+        if (stat.isDirectory() === true) result.push(`${vendor}/${model}`);
+      }
+    return result.sort();
+  };
+
+  export const getHistories = async (props: {
+    vendor: string;
+    project: AutoBeExampleProject;
+    phase: AutoBePhase;
+  }): Promise<AutoBeHistory[]> => {
+    const location: string = `${getDirectory(props)}/${props.phase}.histories.json.gz`;
+    const content: string = await CompressUtil.gunzip(
+      await fs.promises.readFile(location),
+    );
+    return JSON.parse(content);
+  };
+
+  export const getSnapshots = async (props: {
+    vendor: string;
+    project: AutoBeExampleProject;
+    phase: AutoBePhase;
+  }): Promise<AutoBeEventSnapshot[]> => {
+    const location: string = `${getDirectory(props)}/${props.phase}.snapshots.json.gz`;
+    const content: string = await CompressUtil.gunzip(
+      await fs.promises.readFile(location),
+    );
+    return JSON.parse(content);
+  };
+
+  export const getTokenUsage = async (props: {
+    vendor: string;
+    project: AutoBeExampleProject;
+    phase: AutoBePhase;
+  }): Promise<IAutoBeTokenUsageJson> => {
+    const snapshots: AutoBeEventSnapshot[] = await getSnapshots(props);
+    return (
+      snapshots.at(-1)?.tokenUsage ??
+      (() => {
+        const component = (): IAutoBeTokenUsageJson.IComponent => ({
+          total: 0,
+          input: {
+            total: 0,
+            cached: 0,
+          },
+          output: {
+            total: 0,
+            reasoning: 0,
+            accepted_prediction: 0,
+            rejected_prediction: 0,
+          },
+        });
+        return {
+          aggregate: component(),
+          facade: component(),
+          analyze: component(),
+          prisma: component(),
+          interface: component(),
+          test: component(),
+          realize: component(),
+        };
+      })()
+    );
+  };
+
+  export const has = async (props: {
+    vendor: string;
+    project: AutoBeExampleProject;
+    phase: AutoBePhase;
+  }): Promise<boolean> => {
+    return fs.existsSync(
+      `${getDirectory(props)}/${props.phase}.histories.json.gz`,
+    );
+  };
+
+  export const slugModel = (model: string, replaceSlash: boolean): string => {
+    model = model.replaceAll(":", "-");
+    if (replaceSlash) model = model.replaceAll("/", "-");
+    return model;
+  };
+}
+
+const PROMPT_TEMPLATE = {
+  prisma: "Design the database schema.",
+  interface: "Create the API interface specification.",
+  test: "Make the e2e test functions.",
+  realize: "Implement API functions.",
+};
+const TEST_ROOT: string = `${__dirname}/../../../../test`;
+
+const examples = new Singleton(() => {
+  const location: string = `${TEST_ROOT}/repositories/autobe-examples`;
+  if (fs.existsSync(location) === false) {
+    try {
+      fs.mkdirSync(`${TEST_ROOT}/repositories`);
+    } catch {}
+    cp.execSync(`git clone https://github.com/wrtnlabs/autobe-examples`, {
+      cwd: `${TEST_ROOT}/repositories`,
+      stdio: "inherit",
+    });
+  }
+  cp.execSync("git pull", {
+    cwd: location,
+    stdio: "ignore",
+  });
+  if (fs.existsSync(`${location}/raw`) === false)
+    fs.mkdirSync(`${location}/raw`);
+  return location;
+});
+
+const saveWithGzip = async (props: {
+  root: string;
+  files: Record<string, string>;
+  overwrite?: boolean;
+}): Promise<void> => {
+  if (props.overwrite !== true && fs.existsSync(props.root))
+    await fs.promises.rm(props.root, {
+      recursive: true,
+    });
+  const directory = new VariadicSingleton(async (location: string) => {
+    try {
+      await fs.promises.mkdir(location, {
+        recursive: true,
+      });
+    } catch {}
+  });
+  for (const [key, value] of Object.entries(props.files)) {
+    const file: string = path.resolve(`${props.root}/${key}.gz`);
+    await directory.get(path.dirname(file));
+    await fs.promises.writeFile(file, await CompressUtil.gzip(value ?? ""));
+  }
+};

package/src/example/index.ts

@@ -0,0 +1 @@
+export * from "./AutoBeExampleStorage";

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./example";
+export * from "./replay";

package/src/replay/AutoBeReplayComputer.ts

@@ -0,0 +1,189 @@
+import { AutoBeProcessAggregateFactory } from "@autobe/agent/src/factory/AutoBeProcessAggregateFactory";
+import {
+  AutoBeExampleProject,
+  AutoBeHistory,
+  AutoBePhase,
+  IAutoBePlaygroundBenchmarkScore,
+  IAutoBePlaygroundReplay,
+} from "@autobe/interface";
+
+export namespace AutoBeReplayComputer {
+  export const SIGNIFICANT_PROJECTS: AutoBeExampleProject[] = [
+    "todo",
+    "bbs",
+    "reddit",
+    "shopping",
+  ];
+
+  export const emoji = (
+    summaries: IAutoBePlaygroundReplay.ISummary[],
+  ): string => {
+    const success: number = summaries.filter(
+      (s) => s.realize !== null && s.realize.success === true,
+    ).length;
+    if (success >= 3) return "🟢";
+
+    const tested: boolean = !!summaries.find((s) => s.test !== null);
+    return tested ? "🟡" : "❌";
+  };
+
+  export const score = (
+    summaries: IAutoBePlaygroundReplay.ISummary[],
+  ): IAutoBePlaygroundBenchmarkScore => {
+    // list up significant projects
+    summaries = summaries.filter((s) =>
+      ["todo", "bbs", "reddit", "shopping"].includes(s.project),
+    );
+
+    // the formula to compute the benchmark score
+    const compute = (summary: IAutoBePlaygroundReplay.ISummary): number => {
+      const add = (
+        phase: IAutoBePlaygroundReplay.IPhaseState | null,
+        success: number,
+        failure?: number,
+      ): number =>
+        phase !== null
+          ? phase.success === true
+            ? success
+            : (failure ?? success / 2)
+          : 0;
+      return (
+        add(summary.analyze, 10) +
+        add(summary.prisma, 20) +
+        add(summary.interface, 30) +
+        add(summary.test, 20) +
+        add(summary.realize, 20)
+      );
+    };
+    const individual = (project: AutoBeExampleProject): number => {
+      const found = summaries.find((s) => s.project === project);
+      if (found === undefined) return 0;
+      return compute(found);
+    };
+    return {
+      aggregate: summaries.map(compute).reduce((a, b) => a + b, 0) / 4,
+      todo: individual("todo"),
+      bbs: individual("bbs"),
+      reddit: individual("reddit"),
+      shopping: individual("shopping"),
+    };
+  };
+
+  export const summarize = (
+    replay: IAutoBePlaygroundReplay,
+  ): IAutoBePlaygroundReplay.ISummary => {
+    const predicate = <Type extends AutoBePhase>(
+      type: Type,
+      success: (history: AutoBeHistory.Mapper[Type]) => boolean,
+      commodity: (
+        history: AutoBeHistory.Mapper[Type],
+      ) => Record<string, number>,
+    ): IAutoBePlaygroundReplay.IPhaseState | null => {
+      const reversed: AutoBeHistory[] = replay.histories.slice().reverse();
+      const step: number | undefined = reversed.find(
+        (h) => h.type === "analyze",
+      )?.step;
+      if (step === undefined) return null;
+
+      const history: AutoBeHistory.Mapper[Type] | undefined = reversed.find(
+        (h) => h.type === type && h.step === step,
+      ) as AutoBeHistory.Mapper[Type] | undefined;
+      if (history === undefined) return null;
+      return {
+        success: success(history),
+        commodity: commodity(history),
+        elapsed:
+          new Date(history.completed_at).getTime() -
+          new Date(history.created_at).getTime(),
+        aggregates: history.aggregates,
+      };
+    };
+    const phaseStates: Record<
+      AutoBePhase,
+      IAutoBePlaygroundReplay.IPhaseState | null
+    > = {
+      analyze: predicate(
+        "analyze",
+        () => true,
+        (h) => ({
+          actors: h.actors.length,
+          documents: h.files.length,
+        }),
+      ),
+      prisma: predicate(
+        "prisma",
+        (h) => h.compiled.type === "success",
+        (h) => ({
+          namespaces: h.result.data.files.length,
+          models: h.result.data.files.map((f) => f.models).flat().length,
+        }),
+      ),
+      interface: predicate(
+        "interface",
+        (h) => h.missed.length === 0,
+        (h) => ({
+          operations: h.document.operations.length,
+          schemas: Object.keys(h.document.components.schemas).length,
+        }),
+      ),
+      test: predicate(
+        "test",
+        (h) => h.compiled.type === "success",
+        (h) => ({
+          functions: h.files.length,
+          ...(h.compiled.type === "failure"
+            ? {
+                errors: new Set(h.compiled.diagnostics.map((d) => d.file ?? ""))
+                  .size,
+              }
+            : {}),
+        }),
+      ),
+      realize: predicate(
+        "realize",
+        (h) => h.compiled.type === "success",
+        (h) => ({
+          functions: h.functions.length,
+          ...(h.compiled.type === "failure"
+            ? {
+                errors: new Set(h.compiled.diagnostics.map((d) => d.file ?? ""))
+                  .size,
+              }
+            : {}),
+        }),
+      ),
+    };
+    const phase: AutoBePhase | null =
+      (["realize", "test", "interface", "prisma", "analyze"] as const).find(
+        (key) => phaseStates[key] !== null,
+      ) ?? null;
+    return {
+      vendor: replay.vendor,
+      project: replay.project,
+      aggregates: AutoBeProcessAggregateFactory.reduce(
+        replay.histories
+          .filter(
+            (h) =>
+              h.type === "analyze" ||
+              h.type === "prisma" ||
+              h.type === "interface" ||
+              h.type === "test" ||
+              h.type === "realize",
+          )
+          .map((h) => h.aggregates),
+      ),
+      elapsed: replay.histories
+        .filter(
+          (h) => h.type !== "userMessage" && h.type !== "assistantMessage",
+        )
+        .map(
+          (h) =>
+            new Date(h.completed_at).getTime() -
+            new Date(h.created_at).getTime(),
+        )
+        .reduce((a, b) => a + b, 0),
+      ...phaseStates,
+      phase,
+    };
+  };
+}

package/src/replay/AutoBeReplayDocumentation.ts

@@ -0,0 +1,173 @@
+import {
+  AutoBeExampleProject,
+  AutoBePhase,
+  IAutoBePlaygroundBenchmark,
+  IAutoBePlaygroundReplay,
+} from "@autobe/interface";
+import { StringUtil } from "@autobe/utils";
+
+import { AutoBeExampleStorage } from "../example/AutoBeExampleStorage";
+
+export namespace AutoBeReplayDocumentation {
+  export const readme = (experiments: IAutoBePlaygroundBenchmark[]): string => {
+    return StringUtil.trim`
+      # AutoBe Generated Examples
+  
+      ## Benchmark
+  
+      AI Model | Score | FCSR | Status 
+      :--------|------:|-----:|:------:
+      ${experiments
+        .map((e) =>
+          [
+            `[\`${AutoBeExampleStorage.slugModel(
+              e.vendor,
+              false,
+            )}\`](#${AutoBeExampleStorage.slugModel(e.vendor, false)
+              .replaceAll("/", "")
+              .replaceAll(".", "")})`,
+            e.score.aggregate,
+            (() => {
+              const [x, y] = e.replays
+                .map((r) => r.aggregates.total.metric)
+                .map((m) => [m.success, m.attempt])
+                .reduce((a, b) => [a[0] + b[0], a[1] + b[1]], [0, 0]);
+              return y === 0 ? "0%" : Math.floor((x / y) * 100) + "%";
+            })(),
+            e.emoji,
+          ].join(" | "),
+        )
+        .join("\n")}
+
+      - FCSR: Function Calling Success Rate
+      - Status:
+        - 🟢: All projects completed successfully
+        - 🟡: Some projects failed
+        - ❌: All projects failed or not executed
+
+      ${experiments.map(vendor).join("\n\n\n")}
+    `;
+  };
+
+  const vendor = (exp: IAutoBePlaygroundBenchmark): string => {
+    const row = (project: AutoBeExampleProject): string => {
+      const found = exp.replays.find((r) => r.project === project);
+      if (found === undefined)
+        return `\`${project}\` | 0 | ❌ | ❌ | ❌ | ❌ | ❌`;
+      const phase = (
+        state: IAutoBePlaygroundReplay.IPhaseState | null,
+      ): string => {
+        if (state === null) return "❌";
+        else if (state.success === false) return "🟡";
+        else return "🟢";
+      };
+      return [
+        `[\`${found.project}\`](./${exp.vendor}/${found.project}/)`,
+        (exp.score as any)[project],
+        phase(found.analyze),
+        phase(found.prisma),
+        phase(found.interface),
+        phase(found.test),
+        phase(found.realize),
+      ].join(" | ");
+    };
+    return StringUtil.trim`
+      ## \`${exp.vendor}\`
+      
+      Project | Score | Analyze | Prisma | Interface | Test | Realize
+      :-------|------:|:-------:|:------:|:----------|:----:|:-------:
+      ${row("todo")}
+      ${row("bbs")}
+      ${row("reddit")}
+      ${row("shopping")}
+
+      ${exp.replays
+        .map((r) =>
+          project({
+            replay: r,
+            score: (exp.score as any)[r.project],
+          }),
+        )
+        .join("\n\n\n")}
+    `;
+  };
+
+  const project = (props: {
+    replay: IAutoBePlaygroundReplay.ISummary;
+    score: number;
+  }): string => {
+    const phase = (key: AutoBePhase): string => {
+      const title: string = key.charAt(0).toUpperCase() + key.slice(1);
+      const state: IAutoBePlaygroundReplay.IPhaseState | null =
+        props.replay[key];
+      if (state === null) return [`⚪ ${title}`, "", "", "", ""].join(" | ");
+      return [
+        `${state.success === true ? "🟢" : "🔴"} ${title}`,
+        Object.entries(state.commodity)
+          .map(([key, value]) => `\`${key}\`: ${value}`)
+          .join(", "),
+        formatTokens(state.aggregates.total.tokenUsage.total),
+        formatElapsedTime(state.elapsed),
+        Math.floor(
+          (state.aggregates.total.metric.success /
+            state.aggregates.total.metric.attempt) *
+            100,
+        ) + "%",
+      ].join(" | ");
+    };
+    return StringUtil.trim`
+      ### \`${props.replay.vendor}\` - \`${props.replay.project}\`
+
+      - Source Code: ${`[\`${AutoBeExampleStorage.slugModel(
+        props.replay.vendor,
+        false,
+      )}/${props.replay.project}\`](./${AutoBeExampleStorage.slugModel(
+        props.replay.vendor,
+        false,
+      )}/${props.replay.project}/)`}
+      - Score: ${props.score}
+      - Elapsed Time: ${formatElapsedTime(props.replay.elapsed)}
+      - Token Usage: ${formatTokens(
+        props.replay.aggregates.total.tokenUsage.total,
+      )}
+      - Function Calling Success Rate: ${(
+        (props.replay.aggregates.total.metric.success /
+          props.replay.aggregates.total.metric.attempt) *
+        100
+      ).toFixed(2)}%
+
+      Phase | Generated | Token Usage | Elapsed Time | FCSR
+      :-----|:----------|------------:|-------------:|------:
+      ${(["analyze", "prisma", "interface", "test", "realize"] as const)
+        .map((key) => phase(key))
+        .join("\n")}
+    `;
+  };
+}
+
+function formatElapsedTime(ms: number): string {
+  const seconds = Math.floor(ms / 1000);
+  const minutes = Math.floor(seconds / 60);
+  const hours = Math.floor(minutes / 60);
+
+  const s = seconds % 60;
+  const m = minutes % 60;
+  const h = hours;
+
+  if (h > 0) {
+    return `${h}h ${m}m ${s}s`;
+  } else if (m > 0) {
+    return `${m}m ${s}s`;
+  } else {
+    return `${s}s`;
+  }
+}
+
+function formatTokens(num: number): string {
+  if (num >= 1000000) {
+    return `${(num / 1000000).toFixed(2)}M`;
+  } else if (num >= 1000) {
+    return `${(num / 1000).toFixed(1)}K`;
+  }
+  return num.toString();
+}