@autobe/benchmark 0.30.4-dev.20260324 → 0.30.5

package/lib/replay/AutoBeReplayDocumentation.js

@@ -6,13 +6,13 @@ const AutoBeExampleStorage_1 = require("../example/AutoBeExampleStorage");
 var AutoBeReplayDocumentation;
 (function (AutoBeReplayDocumentation) {
     AutoBeReplayDocumentation.readme = (experiments) => {
-        return utils_1.StringUtil.trim `
-      # AutoBe Generated Examples
-  
-      ## Benchmark
-  
-      AI Model | Success | Score | FCSR | Status 
-      :--------|---------|------:|-----:|:------:
+        return utils_1.StringUtil.trim `
+      # AutoBe Generated Examples
+  
+      ## Benchmark
+  
+      AI Model | Success | Score | FCSR | Status 
+      :--------|---------|------:|-----:|:------:
       ${experiments
             .map((e) => [
             `[\`${AutoBeExampleStorage_1.AutoBeExampleStorage.slugModel(e.vendor, false)}\`](#${AutoBeExampleStorage_1.AutoBeExampleStorage.slugModel(e.vendor, false)
@@ -29,15 +29,15 @@ var AutoBeReplayDocumentation;
             })(),
             e.emoji,
         ].join(" | "))
-            .join("\n")}
-
-      - FCSR: Function Calling Success Rate
-      - Status:
-        - 🟢: All projects completed successfully
-        - 🟡: Some projects failed
-        - ❌: All projects failed or not executed
-
-      ${experiments.map(vendor).join("\n\n\n")}
+            .join("\n")}
+
+      - FCSR: Function Calling Success Rate
+      - Status:
+        - 🟢: All projects completed successfully
+        - 🟡: Some projects failed
+        - ❌: All projects failed or not executed
+
+      ${experiments.map(vendor).join("\n\n\n")}
     `;
     };
     const vendor = (exp) => {
@@ -64,23 +64,23 @@ var AutoBeReplayDocumentation;
                 phase(found.realize),
             ].join(" | ");
         };
-        return utils_1.StringUtil.trim `
-      ## \`${exp.vendor}\`
-      
-      Project | Score | Analyze | Prisma | Interface | Test | Realize
-      :-------|------:|:-------:|:------:|:----------|:----:|:-------:
-      ${row("todo")}
-      ${row("reddit")}
-      ${row("shopping")}
-      ${row("erp")}
-
+        return utils_1.StringUtil.trim `
+      ## \`${exp.vendor}\`
+      
+      Project | Score | Analyze | Prisma | Interface | Test | Realize
+      :-------|------:|:-------:|:------:|:----------|:----:|:-------:
+      ${row("todo")}
+      ${row("reddit")}
+      ${row("shopping")}
+      ${row("erp")}
+
       ${exp.replays
             .map((r) => project({
             replay: r,
             // biome-ignore lint: intended
             score: exp.score[r.project],
         }))
-            .join("\n\n\n")}
+            .join("\n\n\n")}
     `;
     };
     const project = (props) => {
@@ -101,22 +101,22 @@ var AutoBeReplayDocumentation;
                     100) + "%",
             ].join(" | ");
         };
-        return utils_1.StringUtil.trim `
-      ### \`${props.replay.vendor}\` - \`${props.replay.project}\`
-
-      - Source Code: ${`[\`${AutoBeExampleStorage_1.AutoBeExampleStorage.slugModel(props.replay.vendor, false)}/${props.replay.project}\`](./${AutoBeExampleStorage_1.AutoBeExampleStorage.slugModel(props.replay.vendor, false)}/${props.replay.project}/)`}
-      - Score: ${props.score}
-      - Elapsed Time: ${formatElapsedTime(props.replay.elapsed)}
-      - Token Usage: ${formatTokens(props.replay.aggregates.total.tokenUsage.total)}
+        return utils_1.StringUtil.trim `
+      ### \`${props.replay.vendor}\` - \`${props.replay.project}\`
+
+      - Source Code: ${`[\`${AutoBeExampleStorage_1.AutoBeExampleStorage.slugModel(props.replay.vendor, false)}/${props.replay.project}\`](./${AutoBeExampleStorage_1.AutoBeExampleStorage.slugModel(props.replay.vendor, false)}/${props.replay.project}/)`}
+      - Score: ${props.score}
+      - Elapsed Time: ${formatElapsedTime(props.replay.elapsed)}
+      - Token Usage: ${formatTokens(props.replay.aggregates.total.tokenUsage.total)}
       - Function Calling Success Rate: ${((props.replay.aggregates.total.metric.success /
             props.replay.aggregates.total.metric.attempt) *
-            100).toFixed(2)}%
-
-      Phase | Generated | Token Usage | Elapsed Time | FCSR
-      :-----|:----------|------------:|-------------:|------:
+            100).toFixed(2)}%
+
+      Phase | Generated | Token Usage | Elapsed Time | FCSR
+      :-----|:----------|------------:|-------------:|------:
       ${["analyze", "database", "interface", "test", "realize"]
             .map((key) => phase(key))
-            .join("\n")}
+            .join("\n")}
     `;
     };
 })(AutoBeReplayDocumentation || (exports.AutoBeReplayDocumentation = AutoBeReplayDocumentation = {}));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autobe/benchmark",
-  "version": "0.30.4-dev.20260324",
+  "version": "0.30.5",
   "description": "AI backend server code generator",
   "main": "lib/index.js",
   "keywords": [],
@@ -27,9 +27,9 @@
     "tstl": "^3.0.0",
     "typia": "^12.0.1",
     "uuid": "^11.1.0",
-    "@autobe/utils": "^0.30.4-dev.20260324",
-    "@autobe/filesystem": "^0.30.4-dev.20260324",
-    "@autobe/interface": "^0.30.4-dev.20260324"
+    "@autobe/filesystem": "^0.30.5",
+    "@autobe/utils": "^0.30.5",
+    "@autobe/interface": "^0.30.5"
   },
   "devDependencies": {
     "@types/uuid": "^10.0.0",

package/src/replay/AutoBeReplayComputer.ts

@@ -1,183 +1,183 @@
-import {
-  AutoBeExampleProject,
-  AutoBeHistory,
-  AutoBePhase,
-  IAutoBePlaygroundBenchmarkScore,
-  IAutoBePlaygroundReplay,
-} from "@autobe/interface";
-import { AutoBeProcessAggregateFactory } from "@autobe/utils";
-import typia from "typia";
-
-export namespace AutoBeReplayComputer {
-  export const SIGNIFICANT_PROJECTS: AutoBeExampleProject[] = [
-    "todo",
-    "reddit",
-    "shopping",
-    "erp",
-  ];
-
-  export const emoji = (
-    summaries: IAutoBePlaygroundReplay.ISummary[],
-  ): string => {
-    const success: number = summaries.filter(
-      (s) => s.realize !== null && s.realize.success === true,
-    ).length;
-    if (success >= 3) return "🟢";
-
-    const tested: boolean = !!summaries.find((s) => s.test !== null);
-    return tested ? "🟡" : "❌";
-  };
-
-  export const score = (
-    summaries: IAutoBePlaygroundReplay.ISummary[],
-  ): IAutoBePlaygroundBenchmarkScore => {
-    // list up significant projects
-    summaries = summaries.filter((s) =>
-      ["todo", "reddit", "shopping", "erp"].includes(s.project),
-    );
-
-    const individual = (project: AutoBeExampleProject): number => {
-      const found = summaries.find((s) => s.project === project);
-      if (found === undefined) return 0;
-      return compute(found);
-    };
-    return {
-      aggregate: round(summaries.map(compute).reduce((a, b) => a + b, 0) / 4),
-      todo: individual("todo"),
-      reddit: individual("reddit"),
-      shopping: individual("shopping"),
-      erp: individual("erp"),
-    };
-  };
-
-  export const summarize = (
-    replay: IAutoBePlaygroundReplay,
-  ): IAutoBePlaygroundReplay.ISummary => {
-    const predicate = <Type extends AutoBePhase>(
-      type: Type,
-      success: (history: AutoBeHistory.Mapper[Type]) => boolean,
-      commodity: (
-        history: AutoBeHistory.Mapper[Type],
-      ) => Record<string, number>,
-    ): IAutoBePlaygroundReplay.IPhaseState | null => {
-      const reversed: AutoBeHistory[] = replay.histories.slice().reverse();
-      const step: number | undefined = reversed.find(
-        (h) => h.type === "analyze",
-      )?.step;
-      if (step === undefined) return null;
-
-      const history: AutoBeHistory.Mapper[Type] | undefined = reversed.find(
-        (h) => h.type === type && h.step === step,
-      ) as AutoBeHistory.Mapper[Type] | undefined;
-      if (history === undefined) return null;
-      return {
-        success: success(history),
-        commodity: commodity(history),
-        elapsed:
-          new Date(history.completed_at).getTime() -
-          new Date(history.created_at).getTime(),
-        aggregates: history.aggregates,
-      };
-    };
-    const phaseStates: Record<
-      AutoBePhase,
-      IAutoBePlaygroundReplay.IPhaseState | null
-    > = {
-      analyze: predicate(
-        "analyze",
-        () => true,
-        (h) => ({
-          actors: h.actors.length,
-          documents: h.files.length,
-        }),
-      ),
-      database: predicate(
-        "database",
-        (h) => h.compiled.type === "success",
-        (h) => ({
-          namespaces: h.result.data.files.length,
-          models: h.result.data.files.map((f) => f.models).flat().length,
-        }),
-      ),
-      interface: predicate(
-        "interface",
-        (h) => h.missed.length === 0,
-        (h) => ({
-          operations: h.document.operations.length,
-          schemas: Object.keys(h.document.components.schemas).length,
-        }),
-      ),
-      test: predicate(
-        "test",
-        (h) => h.compiled.type === "success",
-        (h) => ({
-          functions: h.functions.length,
-          ...(h.compiled.type === "failure"
-            ? {
-                errors: new Set(h.compiled.diagnostics.map((d) => d.file ?? ""))
-                  .size,
-              }
-            : {}),
-        }),
-      ),
-      realize: predicate(
-        "realize",
-        (h) => h.compiled.type === "success",
-        (h) => ({
-          functions: h.functions.length,
-          ...(h.compiled.type === "failure"
-            ? {
-                errors: new Set(h.compiled.diagnostics.map((d) => d.file ?? ""))
-                  .size,
-              }
-            : {}),
-        }),
-      ),
-    };
-    const phase: AutoBePhase | null =
-      (["realize", "test", "interface", "database", "analyze"] as const).find(
-        (key) => phaseStates[key] !== null,
-      ) ?? null;
-    return {
-      vendor: replay.vendor,
-      project: replay.project,
-      ...phaseStates,
-      aggregates: AutoBeProcessAggregateFactory.reduce(
-        Object.values(phaseStates)
-          .filter((p) => p !== null)
-          .map((p) => p.aggregates),
-      ),
-      phase,
-      elapsed: Object.values(phaseStates)
-        .map((p) => p?.elapsed ?? 0)
-        .reduce((a, b) => a + (b ?? 0), 0),
-    };
-  };
-}
-
-const compute = (summary: IAutoBePlaygroundReplay.ISummary): number => {
-  const getScore = (phase: AutoBePhase): number => {
-    const state = summary[phase];
-    if (state === null) return 0;
-
-    const [success, failure] = FORMULA[phase];
-    return state.success === true
-      ? success
-      : success * failure(state.commodity);
-  };
-  return round(sum(typia.misc.literals<AutoBePhase>().map(getScore)));
-};
-const round = (value: number) => Math.round(value * 100) / 100;
-const sum = (targets: number[]): number => targets.reduce((a, b) => a + b, 0);
-
-// for type safety
-const FORMULA: Record<
-  AutoBePhase,
-  [number, (commodity: Record<string, number>) => number]
-> = {
-  analyze: [10, () => 0],
-  database: [20, () => 0.5],
-  interface: [30, () => 0.5],
-  test: [20, (c) => Math.max(0.5, 1 - (c.errors * 3) / c.functions)],
-  realize: [20, (c) => Math.max(0.5, 1 - (c.errors * 3) / c.functions)],
-};
+import {
+  AutoBeExampleProject,
+  AutoBeHistory,
+  AutoBePhase,
+  IAutoBePlaygroundBenchmarkScore,
+  IAutoBePlaygroundReplay,
+} from "@autobe/interface";
+import { AutoBeProcessAggregateFactory } from "@autobe/utils";
+import typia from "typia";
+
+export namespace AutoBeReplayComputer {
+  export const SIGNIFICANT_PROJECTS: AutoBeExampleProject[] = [
+    "todo",
+    "reddit",
+    "shopping",
+    "erp",
+  ];
+
+  export const emoji = (
+    summaries: IAutoBePlaygroundReplay.ISummary[],
+  ): string => {
+    const success: number = summaries.filter(
+      (s) => s.realize !== null && s.realize.success === true,
+    ).length;
+    if (success >= 3) return "🟢";
+
+    const tested: boolean = !!summaries.find((s) => s.test !== null);
+    return tested ? "🟡" : "❌";
+  };
+
+  export const score = (
+    summaries: IAutoBePlaygroundReplay.ISummary[],
+  ): IAutoBePlaygroundBenchmarkScore => {
+    // list up significant projects
+    summaries = summaries.filter((s) =>
+      ["todo", "reddit", "shopping", "erp"].includes(s.project),
+    );
+
+    const individual = (project: AutoBeExampleProject): number => {
+      const found = summaries.find((s) => s.project === project);
+      if (found === undefined) return 0;
+      return compute(found);
+    };
+    return {
+      aggregate: round(summaries.map(compute).reduce((a, b) => a + b, 0) / 4),
+      todo: individual("todo"),
+      reddit: individual("reddit"),
+      shopping: individual("shopping"),
+      erp: individual("erp"),
+    };
+  };
+
+  export const summarize = (
+    replay: IAutoBePlaygroundReplay,
+  ): IAutoBePlaygroundReplay.ISummary => {
+    const predicate = <Type extends AutoBePhase>(
+      type: Type,
+      success: (history: AutoBeHistory.Mapper[Type]) => boolean,
+      commodity: (
+        history: AutoBeHistory.Mapper[Type],
+      ) => Record<string, number>,
+    ): IAutoBePlaygroundReplay.IPhaseState | null => {
+      const reversed: AutoBeHistory[] = replay.histories.slice().reverse();
+      const step: number | undefined = reversed.find(
+        (h) => h.type === "analyze",
+      )?.step;
+      if (step === undefined) return null;
+
+      const history: AutoBeHistory.Mapper[Type] | undefined = reversed.find(
+        (h) => h.type === type && h.step === step,
+      ) as AutoBeHistory.Mapper[Type] | undefined;
+      if (history === undefined) return null;
+      return {
+        success: success(history),
+        commodity: commodity(history),
+        elapsed:
+          new Date(history.completed_at).getTime() -
+          new Date(history.created_at).getTime(),
+        aggregates: history.aggregates,
+      };
+    };
+    const phaseStates: Record<
+      AutoBePhase,
+      IAutoBePlaygroundReplay.IPhaseState | null
+    > = {
+      analyze: predicate(
+        "analyze",
+        () => true,
+        (h) => ({
+          actors: h.actors.length,
+          documents: h.files.length,
+        }),
+      ),
+      database: predicate(
+        "database",
+        (h) => h.compiled.type === "success",
+        (h) => ({
+          namespaces: h.result.data.files.length,
+          models: h.result.data.files.map((f) => f.models).flat().length,
+        }),
+      ),
+      interface: predicate(
+        "interface",
+        (h) => h.missed.length === 0,
+        (h) => ({
+          operations: h.document.operations.length,
+          schemas: Object.keys(h.document.components.schemas).length,
+        }),
+      ),
+      test: predicate(
+        "test",
+        (h) => h.compiled.type === "success",
+        (h) => ({
+          functions: h.functions.length,
+          ...(h.compiled.type === "failure"
+            ? {
+                errors: new Set(h.compiled.diagnostics.map((d) => d.file ?? ""))
+                  .size,
+              }
+            : {}),
+        }),
+      ),
+      realize: predicate(
+        "realize",
+        (h) => h.compiled.type === "success",
+        (h) => ({
+          functions: h.functions.length,
+          ...(h.compiled.type === "failure"
+            ? {
+                errors: new Set(h.compiled.diagnostics.map((d) => d.file ?? ""))
+                  .size,
+              }
+            : {}),
+        }),
+      ),
+    };
+    const phase: AutoBePhase | null =
+      (["realize", "test", "interface", "database", "analyze"] as const).find(
+        (key) => phaseStates[key] !== null,
+      ) ?? null;
+    return {
+      vendor: replay.vendor,
+      project: replay.project,
+      ...phaseStates,
+      aggregates: AutoBeProcessAggregateFactory.reduce(
+        Object.values(phaseStates)
+          .filter((p) => p !== null)
+          .map((p) => p.aggregates),
+      ),
+      phase,
+      elapsed: Object.values(phaseStates)
+        .map((p) => p?.elapsed ?? 0)
+        .reduce((a, b) => a + (b ?? 0), 0),
+    };
+  };
+}
+
+const compute = (summary: IAutoBePlaygroundReplay.ISummary): number => {
+  const getScore = (phase: AutoBePhase): number => {
+    const state = summary[phase];
+    if (state === null) return 0;
+
+    const [success, failure] = FORMULA[phase];
+    return state.success === true
+      ? success
+      : success * failure(state.commodity);
+  };
+  return round(sum(typia.misc.literals<AutoBePhase>().map(getScore)));
+};
+const round = (value: number) => Math.round(value * 100) / 100;
+const sum = (targets: number[]): number => targets.reduce((a, b) => a + b, 0);
+
+// for type safety
+const FORMULA: Record<
+  AutoBePhase,
+  [number, (commodity: Record<string, number>) => number]
+> = {
+  analyze: [10, () => 0],
+  database: [20, () => 0.5],
+  interface: [30, () => 0.5],
+  test: [20, (c) => Math.max(0.5, 1 - (c.errors * 3) / c.functions)],
+  realize: [20, (c) => Math.max(0.5, 1 - (c.errors * 3) / c.functions)],
+};