@autobe/benchmark 0.29.2 → 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/lib/example/AutoBeExampleArchiver.d.ts +2 -1
  2. package/lib/example/AutoBeExampleArchiver.js +41 -19
  3. package/lib/example/AutoBeExampleArchiver.js.map +1 -1
  4. package/lib/example/AutoBeExampleBenchmark.d.ts +5 -1
  5. package/lib/example/AutoBeExampleBenchmark.js +53 -42
  6. package/lib/example/AutoBeExampleBenchmark.js.map +1 -1
  7. package/lib/example/{AutoBeExampleDocumentation.d.ts → AutoBeExampleLogger.d.ts} +1 -1
  8. package/lib/example/{AutoBeExampleDocumentation.js → AutoBeExampleLogger.js} +26 -24
  9. package/lib/example/AutoBeExampleLogger.js.map +1 -0
  10. package/lib/example/AutoBeExampleStorage.d.ts +9 -2
  11. package/lib/example/AutoBeExampleStorage.js +74 -24
  12. package/lib/example/AutoBeExampleStorage.js.map +1 -1
  13. package/lib/example/index.d.ts +1 -1
  14. package/lib/example/index.js +1 -1
  15. package/lib/example/index.js.map +1 -1
  16. package/lib/replay/AutoBeReplayComputer.js +35 -28
  17. package/lib/replay/AutoBeReplayComputer.js.map +1 -1
  18. package/lib/replay/AutoBeReplayDocumentation.js +5 -3
  19. package/lib/replay/AutoBeReplayDocumentation.js.map +1 -1
  20. package/lib/replay/AutoBeReplayStorage.js +10 -3
  21. package/lib/replay/AutoBeReplayStorage.js.map +1 -1
  22. package/lib/structures/IAutoBeExampleBenchmarkState.d.ts +1 -1
  23. package/package.json +5 -6
  24. package/src/example/AutoBeExampleArchiver.ts +73 -29
  25. package/src/example/AutoBeExampleBenchmark.ts +74 -43
  26. package/src/example/{AutoBeExampleDocumentation.ts → AutoBeExampleLogger.ts} +20 -19
  27. package/src/example/AutoBeExampleStorage.ts +97 -26
  28. package/src/example/index.ts +1 -1
  29. package/src/replay/AutoBeReplayComputer.ts +40 -46
  30. package/src/replay/AutoBeReplayDocumentation.ts +5 -3
  31. package/src/replay/AutoBeReplayStorage.ts +10 -3
  32. package/src/structures/IAutoBeExampleBenchmarkState.ts +1 -1
  33. package/lib/example/AutoBeExampleDocumentation.js.map +0 -1
@@ -21,8 +21,13 @@ export namespace AutoBeExampleBenchmark {
21
21
  vendors: string[];
22
22
  projects?: AutoBeExampleProject[];
23
23
  phases?: AutoBePhase[];
24
+ imagePath?: string;
24
25
  progress: (state: IAutoBeExampleBenchmarkState) => void;
25
- on?: (event: AutoBeEvent) => void;
26
+ on?: (
27
+ event: AutoBeEvent,
28
+ agent: IAutoBeAgent,
29
+ context: { vendor: string; project: AutoBeExampleProject },
30
+ ) => void;
26
31
  },
27
32
  ): Promise<void> => {
28
33
  const state: IAutoBeExampleBenchmarkState = {
@@ -45,28 +50,35 @@ export namespace AutoBeExampleBenchmark {
45
50
  };
46
51
  const report = () => props.progress(state);
47
52
  await Promise.all(
48
- state.vendors.map((vendor) =>
49
- executeVendor(ctx, {
53
+ state.vendors.map(async (vendor) => {
54
+ await executeVendor(ctx, {
55
+ imagePath: props.imagePath,
50
56
  phases: props.phases,
51
57
  vendorState: vendor,
52
58
  on: props.on,
53
59
  report,
54
- }),
55
- ),
60
+ });
61
+ }),
56
62
  );
57
63
  };
58
64
 
59
65
  const executeVendor = async (
60
66
  ctx: IContext,
61
67
  props: {
68
+ imagePath?: string;
62
69
  vendorState: IAutoBeExampleBenchmarkState.IOfVendor;
63
70
  phases?: AutoBePhase[];
64
71
  report: () => void;
65
- on?: (event: AutoBeEvent) => void;
72
+ on?: (
73
+ event: AutoBeEvent,
74
+ agent: IAutoBeAgent,
75
+ context: { vendor: string; project: AutoBeExampleProject },
76
+ ) => void;
66
77
  },
67
78
  ): Promise<void> => {
68
79
  for (const project of props.vendorState.projects)
69
80
  await executeProject(ctx, {
81
+ imagePath: props.imagePath,
70
82
  vendor: props.vendorState.name,
71
83
  projectState: project,
72
84
  phases: props.phases,
@@ -80,9 +92,14 @@ export namespace AutoBeExampleBenchmark {
80
92
  props: {
81
93
  vendor: string;
82
94
  projectState: IAutoBeExampleBenchmarkState.IOfProject;
95
+ imagePath?: string;
83
96
  phases?: AutoBePhase[];
84
97
  report: () => void;
85
- on?: (event: AutoBeEvent) => void;
98
+ on?: (
99
+ event: AutoBeEvent,
100
+ agent: IAutoBeAgent,
101
+ context: { vendor: string; project: AutoBeExampleProject },
102
+ ) => void;
86
103
  },
87
104
  ): Promise<void> => {
88
105
  props.projectState.started_at = new Date();
@@ -94,42 +111,49 @@ export namespace AutoBeExampleBenchmark {
94
111
  success: null,
95
112
  started_at: new Date(),
96
113
  completed_at: null,
97
- trial: 0,
114
+ count: 0,
98
115
  };
99
116
  props.projectState.phases.push(phaseState);
100
- for (let i: number = 0; i < 3; ++i) {
101
- try {
102
- ++phaseState.trial;
103
- phaseState.started_at = new Date();
104
- phaseState.completed_at = null;
105
- const success: boolean = await getArchiver(phase)({
106
- vendor: props.vendor,
107
- project: props.projectState.name,
108
- agent: (next) => ctx.createAgent(next),
109
- on: (s) => {
110
- const event = s.event;
111
- if (
112
- event.type !== "jsonValidateError" &&
113
- event.type !== "jsonParseError" &&
114
- event.type !== "preliminary" &&
115
- event.type !== "consentFunctionCall"
116
- )
117
- phaseState.snapshot = s;
118
- props.report();
119
- if (props.on) props.on(s.event);
120
- },
121
- });
122
- phaseState.success = success;
123
- phaseState.completed_at = new Date();
124
- props.report();
125
- if (success === true) break;
126
- } catch (error) {
127
- continue;
128
- }
117
+ try {
118
+ phaseState.started_at = new Date();
119
+ phaseState.completed_at = null;
120
+ phaseState.count = 0;
121
+ const success: boolean = await getArchiver(phase)({
122
+ vendor: props.vendor,
123
+ project: props.projectState.name,
124
+ imagePath: props.imagePath,
125
+ agent: (next) => ctx.createAgent(next),
126
+ on: (s, agent) => {
127
+ ++phaseState.count;
128
+ const event = s.event;
129
+ if (
130
+ event.type !== "jsonValidateError" &&
131
+ event.type !== "jsonParseError" &&
132
+ event.type !== "preliminary" &&
133
+ event.type !== "consentFunctionCall"
134
+ )
135
+ phaseState.snapshot = s;
136
+ props.report();
137
+ if (props.on)
138
+ props.on(s.event, agent, {
139
+ vendor: props.vendor,
140
+ project: props.projectState.name,
141
+ });
142
+ },
143
+ });
144
+ phaseState.success = success;
145
+ phaseState.completed_at = new Date();
146
+ props.report();
147
+ } catch (error) {
148
+ console.log(
149
+ props.vendor,
150
+ props.projectState.name,
151
+ phaseState.name,
152
+ error,
153
+ );
154
+ throw error;
129
155
  }
130
- if (phaseState.success === null) break;
131
- else if (phaseState.success === false && phaseState.name !== "test")
132
- break;
156
+ if (phaseState.success === null || phaseState.success === false) break;
133
157
  }
134
158
  props.projectState.completed_at = new Date();
135
159
  props.projectState.success = props.projectState.phases.every(
@@ -141,7 +165,7 @@ export namespace AutoBeExampleBenchmark {
141
165
 
142
166
  const getArchiver = (phase: AutoBePhase) => {
143
167
  if (phase === "analyze") return AutoBeExampleArchiver.archiveAnalyze;
144
- else if (phase === "prisma") return AutoBeExampleArchiver.archivePrisma;
168
+ else if (phase === "database") return AutoBeExampleArchiver.archivePrisma;
145
169
  else if (phase === "interface") return AutoBeExampleArchiver.archiveInterface;
146
170
  else if (phase === "test") return AutoBeExampleArchiver.archiveTest;
147
171
  else if (phase === "realize") return AutoBeExampleArchiver.archiveRealize;
@@ -149,10 +173,17 @@ const getArchiver = (phase: AutoBePhase) => {
149
173
  throw new Error(`Unknown phase: ${phase}`);
150
174
  };
151
175
 
152
- const PROJECT_SEQUENCE = ["todo", "bbs", "reddit", "shopping"] as const;
176
+ const PROJECT_SEQUENCE = [
177
+ "todo",
178
+ "bbs",
179
+ "reddit",
180
+ "shopping",
181
+ "account",
182
+ "erp",
183
+ ] as const;
153
184
  const PHASE_SEQUENCE = [
154
185
  "analyze",
155
- "prisma",
186
+ "database",
156
187
  "interface",
157
188
  "test",
158
189
  "realize",
@@ -4,7 +4,7 @@ import typia from "typia";
4
4
 
5
5
  import { IAutoBeExampleBenchmarkState } from "../structures";
6
6
 
7
- export namespace AutoBeExampleDocumentation {
7
+ export namespace AutoBeExampleLogger {
8
8
  export const markdown = (state: IAutoBeExampleBenchmarkState): string =>
9
9
  StringUtil.trim`
10
10
  # AutoBe Example Benchmark Report
@@ -34,8 +34,8 @@ export namespace AutoBeExampleDocumentation {
34
34
  ): string => StringUtil.trim`
35
35
  ## \`${state.name}\`
36
36
 
37
- Project | Phase | State | Elapsed Time
38
- :-------|:------|:------|-------------:
37
+ Project | Phase | State | Count | Elapsed Time
38
+ :-------|:------|:------|------:|-------------:
39
39
  ${state.projects.map(markdownProject).join("\n")}
40
40
  `;
41
41
 
@@ -47,22 +47,23 @@ export namespace AutoBeExampleDocumentation {
47
47
  state.phases.at(-1);
48
48
  return [
49
49
  state.name,
50
- !!phase?.name ? `${phase.name} (${phase.trial})` : "-",
51
- state.completed_at !== null
52
- ? state.success
53
- ? "🟢 success"
54
- : "🔴 failure"
55
- : phase !== undefined && phase.snapshot !== null
56
- ? [
57
- phase.trial !== 1 ? "🟠" : "🟡",
58
- `\`${phase.snapshot.event.type}\``,
59
- ...(typia.is<AutoBeProgressEventBase>(phase.snapshot.event)
60
- ? [
61
- `(${phase.snapshot.event.completed} of ${phase.snapshot.event.total})`,
62
- ]
63
- : []),
64
- ].join(" ")
65
- : "-",
50
+ phase?.name ?? "-",
51
+ phase !== undefined && phase.snapshot !== null
52
+ ? [
53
+ state.completed_at !== null
54
+ ? state.success
55
+ ? "🟢 success"
56
+ : "🔴 failure"
57
+ : "🟡",
58
+ `\`${phase.snapshot.event.type}\``,
59
+ ...(typia.is<AutoBeProgressEventBase>(phase.snapshot.event)
60
+ ? [
61
+ `(${phase.snapshot.event.completed} of ${phase.snapshot.event.total})`,
62
+ ]
63
+ : []),
64
+ ].join(" ")
65
+ : "-",
66
+ phase?.count.toLocaleString() ?? "0",
66
67
  state.started_at !== null
67
68
  ? elapsedTime({
68
69
  started_at: state.started_at,
@@ -4,14 +4,14 @@ import {
4
4
  AutoBeExampleProject,
5
5
  AutoBeHistory,
6
6
  AutoBePhase,
7
- AutoBeUserMessageHistory,
7
+ AutoBeUserConversateContent,
8
+ AutoBeUserImageConversateContent,
8
9
  IAutoBeTokenUsageJson,
9
10
  } from "@autobe/interface";
10
11
  import cp from "child_process";
11
12
  import fs from "fs";
12
13
  import path from "path";
13
14
  import { Singleton, VariadicSingleton } from "tstl";
14
- import { v7 } from "uuid";
15
15
 
16
16
  export namespace AutoBeExampleStorage {
17
17
  export const TEST_ROOT: string = `${__dirname}/../../../../test`;
@@ -35,11 +35,88 @@ export namespace AutoBeExampleStorage {
35
35
  });
36
36
  };
37
37
 
38
+ export const load = async <T>(props: {
39
+ vendor: string;
40
+ project: AutoBeExampleProject;
41
+ file: string;
42
+ }): Promise<T | null> => {
43
+ const location: string = `${getDirectory(props)}/${props.file}.gz`;
44
+ if (fs.existsSync(location) === false) return null;
45
+ const content: string = await CompressUtil.gunzip(
46
+ await fs.promises.readFile(location),
47
+ );
48
+ return JSON.parse(content) as T;
49
+ };
50
+
51
+ export const loadImages = async (
52
+ imagePath: string,
53
+ ): Promise<AutoBeUserConversateContent[]> => {
54
+ const stat: fs.Stats = await fs.promises.lstat(imagePath);
55
+
56
+ const filePaths: string[] = await (async () => {
57
+ if (stat.isFile() === true) {
58
+ if (imagePath.endsWith(".png") === false)
59
+ throw new Error("Image Format must be .png");
60
+ return [imagePath];
61
+ }
62
+ const files: string[] = await fs.promises.readdir(imagePath);
63
+ return files
64
+ .filter((f) => f.endsWith(".png"))
65
+ .map((f) => path.join(imagePath, f));
66
+ })();
67
+
68
+ return Promise.all(
69
+ filePaths.map(async (filePath) => {
70
+ const extension: string = path.extname(filePath).toLowerCase().slice(1);
71
+ const base64Data: string = `data:image/${extension};base64,${await fs.promises.readFile(filePath, "base64")}`;
72
+ return {
73
+ type: "image",
74
+ image: {
75
+ type: "base64",
76
+ data: base64Data,
77
+ } satisfies AutoBeUserImageConversateContent.IBase64,
78
+ } satisfies AutoBeUserConversateContent;
79
+ }),
80
+ );
81
+ };
82
+
38
83
  export const getUserMessage = async (props: {
39
84
  project: AutoBeExampleProject;
40
85
  phase: AutoBePhase;
41
- }): Promise<AutoBeUserMessageHistory> => {
86
+ imagePath?: string;
87
+ }): Promise<AutoBeUserConversateContent[]> => {
88
+ const imageMessages: AutoBeUserConversateContent[] = props.imagePath
89
+ ? await loadImages(props.imagePath)
90
+ : [];
91
+
42
92
  const full: string = `${TEST_ROOT}/scripts/${props.project}/${props.phase}`;
93
+ if (props.project === "account" && props.phase === "analyze") {
94
+ const files: string[] = await fs.promises.readdir(full);
95
+ const contents: AutoBeUserConversateContent[] = await Promise.all(
96
+ files.map(async (filename) => {
97
+ const filePath = path.join(full, filename);
98
+ const extension = filename.split(".").pop() ?? "unknown";
99
+ const base64Data = `data:image/${extension};base64,${await fs.promises.readFile(filePath, "base64")}`;
100
+
101
+ return {
102
+ type: "image",
103
+ image: {
104
+ type: "base64",
105
+ data: base64Data,
106
+ },
107
+ } satisfies AutoBeUserConversateContent;
108
+ }),
109
+ );
110
+ return [
111
+ ...contents,
112
+ ...imageMessages,
113
+ {
114
+ type: "text",
115
+ text: "Convert the images into a planning document.",
116
+ },
117
+ ];
118
+ }
119
+
43
120
  if (fs.existsSync(`${full}.md`) === false) {
44
121
  const text: string =
45
122
  props.phase === "analyze"
@@ -48,30 +125,24 @@ export namespace AutoBeExampleStorage {
48
125
  "utf8",
49
126
  )
50
127
  : PROMPT_TEMPLATE[props.phase];
51
- return {
52
- type: "userMessage",
53
- id: v7(),
54
- created_at: new Date().toISOString(),
55
- contents: [
56
- {
57
- type: "text",
58
- text,
59
- },
60
- ],
61
- };
62
- }
63
- const text: string = await fs.promises.readFile(`${full}.md`, "utf8");
64
- return {
65
- type: "userMessage",
66
- id: v7(),
67
- created_at: new Date().toISOString(),
68
- contents: [
128
+ return [
129
+ ...imageMessages,
69
130
  {
70
131
  type: "text",
71
- text: text,
132
+ text,
72
133
  },
73
- ],
74
- };
134
+ ];
135
+ }
136
+
137
+ const text: string = await fs.promises.readFile(`${full}.md`, "utf8");
138
+
139
+ return [
140
+ ...imageMessages,
141
+ {
142
+ type: "text",
143
+ text,
144
+ },
145
+ ];
75
146
  };
76
147
 
77
148
  export const getVendorModels = async (): Promise<string[]> => {
@@ -144,7 +215,7 @@ export namespace AutoBeExampleStorage {
144
215
  aggregate: component(),
145
216
  facade: component(),
146
217
  analyze: component(),
147
- prisma: component(),
218
+ database: component(),
148
219
  interface: component(),
149
220
  test: component(),
150
221
  realize: component(),
@@ -170,7 +241,7 @@ export namespace AutoBeExampleStorage {
170
241
  };
171
242
 
172
243
  const PROMPT_TEMPLATE = {
173
- prisma: "Design the database schema.",
244
+ database: "Design the database schema.",
174
245
  interface: "Create the API interface specification.",
175
246
  test: "Make the e2e test functions.",
176
247
  realize: "Implement API functions.",
@@ -1,4 +1,4 @@
1
1
  export * from "./AutoBeExampleArchiver";
2
2
  export * from "./AutoBeExampleBenchmark";
3
3
  export * from "./AutoBeExampleStorage";
4
- export * from "./AutoBeExampleDocumentation";
4
+ export * from "./AutoBeExampleLogger";
@@ -6,6 +6,7 @@ import {
6
6
  IAutoBePlaygroundReplay,
7
7
  } from "@autobe/interface";
8
8
  import { AutoBeProcessAggregateFactory } from "@autobe/utils";
9
+ import typia from "typia";
9
10
 
10
11
  export namespace AutoBeReplayComputer {
11
12
  export const SIGNIFICANT_PROJECTS: AutoBeExampleProject[] = [
@@ -35,33 +36,13 @@ export namespace AutoBeReplayComputer {
35
36
  ["todo", "bbs", "reddit", "shopping"].includes(s.project),
36
37
  );
37
38
 
38
- // the formula to compute the benchmark score
39
- const compute = (summary: IAutoBePlaygroundReplay.ISummary): number => {
40
- const add = (
41
- phase: IAutoBePlaygroundReplay.IPhaseState | null,
42
- success: number,
43
- failure?: number,
44
- ): number =>
45
- phase !== null
46
- ? phase.success === true
47
- ? success
48
- : (failure ?? success / 2)
49
- : 0;
50
- return (
51
- add(summary.analyze, 10) +
52
- add(summary.prisma, 20) +
53
- add(summary.interface, 30) +
54
- add(summary.test, 20) +
55
- add(summary.realize, 20)
56
- );
57
- };
58
39
  const individual = (project: AutoBeExampleProject): number => {
59
40
  const found = summaries.find((s) => s.project === project);
60
41
  if (found === undefined) return 0;
61
42
  return compute(found);
62
43
  };
63
44
  return {
64
- aggregate: summaries.map(compute).reduce((a, b) => a + b, 0) / 4,
45
+ aggregate: round(summaries.map(compute).reduce((a, b) => a + b, 0) / 4),
65
46
  todo: individual("todo"),
66
47
  bbs: individual("bbs"),
67
48
  reddit: individual("reddit"),
@@ -110,8 +91,8 @@ export namespace AutoBeReplayComputer {
110
91
  documents: h.files.length,
111
92
  }),
112
93
  ),
113
- prisma: predicate(
114
- "prisma",
94
+ database: predicate(
95
+ "database",
115
96
  (h) => h.compiled.type === "success",
116
97
  (h) => ({
117
98
  namespaces: h.result.data.files.length,
@@ -130,7 +111,7 @@ export namespace AutoBeReplayComputer {
130
111
  "test",
131
112
  (h) => h.compiled.type === "success",
132
113
  (h) => ({
133
- functions: h.files.length,
114
+ functions: h.functions.length,
134
115
  ...(h.compiled.type === "failure"
135
116
  ? {
136
117
  errors: new Set(h.compiled.diagnostics.map((d) => d.file ?? ""))
@@ -154,36 +135,49 @@ export namespace AutoBeReplayComputer {
154
135
  ),
155
136
  };
156
137
  const phase: AutoBePhase | null =
157
- (["realize", "test", "interface", "prisma", "analyze"] as const).find(
138
+ (["realize", "test", "interface", "database", "analyze"] as const).find(
158
139
  (key) => phaseStates[key] !== null,
159
140
  ) ?? null;
160
141
  return {
161
142
  vendor: replay.vendor,
162
143
  project: replay.project,
144
+ ...phaseStates,
163
145
  aggregates: AutoBeProcessAggregateFactory.reduce(
164
- replay.histories
165
- .filter(
166
- (h) =>
167
- h.type === "analyze" ||
168
- h.type === "prisma" ||
169
- h.type === "interface" ||
170
- h.type === "test" ||
171
- h.type === "realize",
172
- )
173
- .map((h) => h.aggregates),
146
+ Object.values(phaseStates)
147
+ .filter((p) => p !== null)
148
+ .map((p) => p.aggregates),
174
149
  ),
175
- elapsed: replay.histories
176
- .filter(
177
- (h) => h.type !== "userMessage" && h.type !== "assistantMessage",
178
- )
179
- .map(
180
- (h) =>
181
- new Date(h.completed_at).getTime() -
182
- new Date(h.created_at).getTime(),
183
- )
184
- .reduce((a, b) => a + b, 0),
185
- ...phaseStates,
186
150
  phase,
151
+ elapsed: Object.values(phaseStates)
152
+ .map((p) => p?.elapsed ?? 0)
153
+ .reduce((a, b) => a + (b ?? 0), 0),
187
154
  };
188
155
  };
189
156
  }
157
+
158
+ const compute = (summary: IAutoBePlaygroundReplay.ISummary): number => {
159
+ const getScore = (phase: AutoBePhase): number => {
160
+ const state = summary[phase];
161
+ if (state === null) return 0;
162
+
163
+ const [success, failure] = FORMULA[phase];
164
+ return state.success === true
165
+ ? success
166
+ : success * failure(state.commodity);
167
+ };
168
+ return round(sum(typia.misc.literals<AutoBePhase>().map(getScore)));
169
+ };
170
+ const round = (value: number) => Math.round(value * 100) / 100;
171
+ const sum = (targets: number[]): number => targets.reduce((a, b) => a + b, 0);
172
+
173
+ // for type safety
174
+ const FORMULA: Record<
175
+ AutoBePhase,
176
+ [number, (commodity: Record<string, number>) => number]
177
+ > = {
178
+ analyze: [10, () => 0],
179
+ database: [20, () => 0.5],
180
+ interface: [30, () => 0.5],
181
+ test: [20, (c) => Math.max(0.5, 1 - (c.errors * 3) / c.functions)],
182
+ realize: [20, (c) => Math.max(0.5, 1 - (c.errors * 3) / c.functions)],
183
+ };
@@ -64,9 +64,10 @@ export namespace AutoBeReplayDocumentation {
64
64
  };
65
65
  return [
66
66
  `[\`${found.project}\`](./${exp.vendor}/${found.project}/)`,
67
+ // biome-ignore lint: intended
67
68
  (exp.score as any)[project],
68
69
  phase(found.analyze),
69
- phase(found.prisma),
70
+ phase(found.database),
70
71
  phase(found.interface),
71
72
  phase(found.test),
72
73
  phase(found.realize),
@@ -86,6 +87,7 @@ export namespace AutoBeReplayDocumentation {
86
87
  .map((r) =>
87
88
  project({
88
89
  replay: r,
90
+ // biome-ignore lint: intended
89
91
  score: (exp.score as any)[r.project],
90
92
  }),
91
93
  )
@@ -103,7 +105,7 @@ export namespace AutoBeReplayDocumentation {
103
105
  props.replay[key];
104
106
  if (state === null) return [`⚪ ${title}`, "", "", "", ""].join(" | ");
105
107
  return [
106
- `${state.success === true ? "🟢" : "🔴"} ${title}`,
108
+ `${state.success === true ? "🟢" : Object.keys(state.commodity).length ? "🟡" : "🔴"} ${title}`,
107
109
  Object.entries(state.commodity)
108
110
  .map(([key, value]) => `\`${key}\`: ${value}`)
109
111
  .join(", "),
@@ -139,7 +141,7 @@ export namespace AutoBeReplayDocumentation {
139
141
 
140
142
  Phase | Generated | Token Usage | Elapsed Time | FCSR
141
143
  :-----|:----------|------------:|-------------:|------:
142
- ${(["analyze", "prisma", "interface", "test", "realize"] as const)
144
+ ${(["analyze", "database", "interface", "test", "realize"] as const)
143
145
  .map((key) => phase(key))
144
146
  .join("\n")}
145
147
  `;
@@ -71,7 +71,7 @@ export namespace AutoBeReplayStorage {
71
71
  project: props.project,
72
72
  histories,
73
73
  analyze: await snapshots("analyze"),
74
- prisma: await snapshots("prisma"),
74
+ database: await snapshots("database"),
75
75
  interface: await snapshots("interface"),
76
76
  test: await snapshots("test"),
77
77
  realize: await snapshots("realize"),
@@ -84,9 +84,10 @@ export namespace AutoBeReplayStorage {
84
84
  }): Promise<IAutoBePlaygroundReplay.ISummary | null> => {
85
85
  const location: string = `${AutoBeExampleStorage.getDirectory(props)}/summary.json.gz`;
86
86
  if (fs.existsSync(location) === false) return null;
87
- return JSON.parse(
87
+ const replay: IAutoBePlaygroundReplay.ISummary = JSON.parse(
88
88
  await CompressUtil.gunzip(await fs.promises.readFile(location)),
89
89
  );
90
+ return replay;
90
91
  };
91
92
 
92
93
  const getHistories = async (props: {
@@ -106,4 +107,10 @@ export namespace AutoBeReplayStorage {
106
107
  };
107
108
  }
108
109
 
109
- const SEQUENCE = ["realize", "test", "interface", "prisma", "analyze"] as const;
110
+ const SEQUENCE = [
111
+ "realize",
112
+ "test",
113
+ "interface",
114
+ "database",
115
+ "analyze",
116
+ ] as const;
@@ -25,6 +25,6 @@ export namespace IAutoBeExampleBenchmarkState {
25
25
  success: boolean | null;
26
26
  started_at: Date;
27
27
  completed_at: Date | null;
28
- trial: number;
28
+ count: number;
29
29
  }
30
30
  }
@@ -1 +0,0 @@
1
- {"version":3,"file":"AutoBeExampleDocumentation.js","sourceRoot":"","sources":["../../src/example/AutoBeExampleDocumentation.ts"],"names":[],"mappings":";;;;;;AACA,yCAA2C;AAC3C,kDAA0B;AAI1B,IAAiB,0BAA0B,CAmE1C;AAnED,WAAiB,0BAA0B;IAC5B,mCAAQ,GAAG,CAAC,KAAmC,EAAU,EAAE,CACtE,kBAAU,CAAC,IAAI,CAAA;;;QAGX,aAAa,CAAC,KAAK,CAAC;;QAEpB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;KACjD,CAAC;IAEJ,MAAM,aAAa,GAAG,CACpB,KAAmC,EAC3B,EAAE,CAAC,kBAAU,CAAC,IAAI,CAAA;;;MAGxB,KAAK,CAAC,OAAO;SACZ,GAAG,CACF,CAAC,MAAM,EAAE,EAAE,CACT,QAAQ,MAAM,CAAC,IAAI,QAAQ,MAAM,CAAC,IAAI;SACnC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC;SACnB,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAC5B;SACA,IAAI,CAAC,IAAI,CAAC;GACd,CAAC;IAEF,MAAM,cAAc,GAAG,CACrB,KAA6C,EACrC,EAAE,CAAC,kBAAU,CAAC,IAAI,CAAA;WACnB,KAAK,CAAC,IAAI;;;;MAIf,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;GACjD,CAAC;IAEF,MAAM,eAAe,GAAG,CACtB,KAA8C,EACtC,EAAE;QACV,uBAAuB;QACvB,MAAM,KAAK,GACT,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO;YACL,KAAK,CAAC,IAAI;YACV,CAAC,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,CAAA,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG;YACtD,KAAK,CAAC,YAAY,KAAK,IAAI;gBACzB,CAAC,CAAC,KAAK,CAAC,OAAO;oBACb,CAAC,CAAC,YAAY;oBACd,CAAC,CAAC,YAAY;gBAChB,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;oBAC9C,CAAC,CAAC;wBACE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;wBAC/B,KAAK,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI;wBAClC,GAAG,CAAC,eAAK,CAAC,EAAE,CAA0B,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;4BACzD,CAAC,CAAC;gCACE,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG;6BACvE;4BACH,CAAC,CAAC,EAAE,CAAC;qBACR,CAAC,IAAI,CAAC,GAAG,CAAC;oBACb,CAAC,CAAC,GAAG;YACT,KAAK,CAAC,UAAU,KAAK,IAAI;gBACvB,CAAC,CAAC,WAAW,CAAC;oBACV,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;iBACjC,CAAC;gBACJ,CAAC,CAAC,GAAG;SACR,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC,EAnEgB,0BAA0B,0CAA1B,0BAA0B,QAmE1C;AAED,MAAM,WAAW,GAAG,CAAC,KAGpB,EAAU,EAAE;;IACX,OAAA,IAAI,CAAC,KAAK,CACR,CAAC,CAAC,MAAA,KAAK,CAAC,YAAY,mCAAI,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;QAC3C,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAC3B,IAAK,CACR,CAAC,cAAc,EAAE,GAAG,MAAM,CAAA;CAAA,CAAC"}