@autobe/benchmark 0.29.2 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/example/AutoBeExampleArchiver.d.ts +2 -1
- package/lib/example/AutoBeExampleArchiver.js +41 -19
- package/lib/example/AutoBeExampleArchiver.js.map +1 -1
- package/lib/example/AutoBeExampleBenchmark.d.ts +5 -1
- package/lib/example/AutoBeExampleBenchmark.js +53 -42
- package/lib/example/AutoBeExampleBenchmark.js.map +1 -1
- package/lib/example/{AutoBeExampleDocumentation.d.ts → AutoBeExampleLogger.d.ts} +1 -1
- package/lib/example/{AutoBeExampleDocumentation.js → AutoBeExampleLogger.js} +26 -24
- package/lib/example/AutoBeExampleLogger.js.map +1 -0
- package/lib/example/AutoBeExampleStorage.d.ts +9 -2
- package/lib/example/AutoBeExampleStorage.js +74 -24
- package/lib/example/AutoBeExampleStorage.js.map +1 -1
- package/lib/example/index.d.ts +1 -1
- package/lib/example/index.js +1 -1
- package/lib/example/index.js.map +1 -1
- package/lib/replay/AutoBeReplayComputer.js +35 -28
- package/lib/replay/AutoBeReplayComputer.js.map +1 -1
- package/lib/replay/AutoBeReplayDocumentation.js +5 -3
- package/lib/replay/AutoBeReplayDocumentation.js.map +1 -1
- package/lib/replay/AutoBeReplayStorage.js +10 -3
- package/lib/replay/AutoBeReplayStorage.js.map +1 -1
- package/lib/structures/IAutoBeExampleBenchmarkState.d.ts +1 -1
- package/package.json +5 -6
- package/src/example/AutoBeExampleArchiver.ts +73 -29
- package/src/example/AutoBeExampleBenchmark.ts +74 -43
- package/src/example/{AutoBeExampleDocumentation.ts → AutoBeExampleLogger.ts} +20 -19
- package/src/example/AutoBeExampleStorage.ts +97 -26
- package/src/example/index.ts +1 -1
- package/src/replay/AutoBeReplayComputer.ts +40 -46
- package/src/replay/AutoBeReplayDocumentation.ts +5 -3
- package/src/replay/AutoBeReplayStorage.ts +10 -3
- package/src/structures/IAutoBeExampleBenchmarkState.ts +1 -1
- package/lib/example/AutoBeExampleDocumentation.js.map +0 -1
|
@@ -21,8 +21,13 @@ export namespace AutoBeExampleBenchmark {
|
|
|
21
21
|
vendors: string[];
|
|
22
22
|
projects?: AutoBeExampleProject[];
|
|
23
23
|
phases?: AutoBePhase[];
|
|
24
|
+
imagePath?: string;
|
|
24
25
|
progress: (state: IAutoBeExampleBenchmarkState) => void;
|
|
25
|
-
on?: (
|
|
26
|
+
on?: (
|
|
27
|
+
event: AutoBeEvent,
|
|
28
|
+
agent: IAutoBeAgent,
|
|
29
|
+
context: { vendor: string; project: AutoBeExampleProject },
|
|
30
|
+
) => void;
|
|
26
31
|
},
|
|
27
32
|
): Promise<void> => {
|
|
28
33
|
const state: IAutoBeExampleBenchmarkState = {
|
|
@@ -45,28 +50,35 @@ export namespace AutoBeExampleBenchmark {
|
|
|
45
50
|
};
|
|
46
51
|
const report = () => props.progress(state);
|
|
47
52
|
await Promise.all(
|
|
48
|
-
state.vendors.map((vendor) =>
|
|
49
|
-
executeVendor(ctx, {
|
|
53
|
+
state.vendors.map(async (vendor) => {
|
|
54
|
+
await executeVendor(ctx, {
|
|
55
|
+
imagePath: props.imagePath,
|
|
50
56
|
phases: props.phases,
|
|
51
57
|
vendorState: vendor,
|
|
52
58
|
on: props.on,
|
|
53
59
|
report,
|
|
54
|
-
})
|
|
55
|
-
),
|
|
60
|
+
});
|
|
61
|
+
}),
|
|
56
62
|
);
|
|
57
63
|
};
|
|
58
64
|
|
|
59
65
|
const executeVendor = async (
|
|
60
66
|
ctx: IContext,
|
|
61
67
|
props: {
|
|
68
|
+
imagePath?: string;
|
|
62
69
|
vendorState: IAutoBeExampleBenchmarkState.IOfVendor;
|
|
63
70
|
phases?: AutoBePhase[];
|
|
64
71
|
report: () => void;
|
|
65
|
-
on?: (
|
|
72
|
+
on?: (
|
|
73
|
+
event: AutoBeEvent,
|
|
74
|
+
agent: IAutoBeAgent,
|
|
75
|
+
context: { vendor: string; project: AutoBeExampleProject },
|
|
76
|
+
) => void;
|
|
66
77
|
},
|
|
67
78
|
): Promise<void> => {
|
|
68
79
|
for (const project of props.vendorState.projects)
|
|
69
80
|
await executeProject(ctx, {
|
|
81
|
+
imagePath: props.imagePath,
|
|
70
82
|
vendor: props.vendorState.name,
|
|
71
83
|
projectState: project,
|
|
72
84
|
phases: props.phases,
|
|
@@ -80,9 +92,14 @@ export namespace AutoBeExampleBenchmark {
|
|
|
80
92
|
props: {
|
|
81
93
|
vendor: string;
|
|
82
94
|
projectState: IAutoBeExampleBenchmarkState.IOfProject;
|
|
95
|
+
imagePath?: string;
|
|
83
96
|
phases?: AutoBePhase[];
|
|
84
97
|
report: () => void;
|
|
85
|
-
on?: (
|
|
98
|
+
on?: (
|
|
99
|
+
event: AutoBeEvent,
|
|
100
|
+
agent: IAutoBeAgent,
|
|
101
|
+
context: { vendor: string; project: AutoBeExampleProject },
|
|
102
|
+
) => void;
|
|
86
103
|
},
|
|
87
104
|
): Promise<void> => {
|
|
88
105
|
props.projectState.started_at = new Date();
|
|
@@ -94,42 +111,49 @@ export namespace AutoBeExampleBenchmark {
|
|
|
94
111
|
success: null,
|
|
95
112
|
started_at: new Date(),
|
|
96
113
|
completed_at: null,
|
|
97
|
-
|
|
114
|
+
count: 0,
|
|
98
115
|
};
|
|
99
116
|
props.projectState.phases.push(phaseState);
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
|
|
117
|
+
try {
|
|
118
|
+
phaseState.started_at = new Date();
|
|
119
|
+
phaseState.completed_at = null;
|
|
120
|
+
phaseState.count = 0;
|
|
121
|
+
const success: boolean = await getArchiver(phase)({
|
|
122
|
+
vendor: props.vendor,
|
|
123
|
+
project: props.projectState.name,
|
|
124
|
+
imagePath: props.imagePath,
|
|
125
|
+
agent: (next) => ctx.createAgent(next),
|
|
126
|
+
on: (s, agent) => {
|
|
127
|
+
++phaseState.count;
|
|
128
|
+
const event = s.event;
|
|
129
|
+
if (
|
|
130
|
+
event.type !== "jsonValidateError" &&
|
|
131
|
+
event.type !== "jsonParseError" &&
|
|
132
|
+
event.type !== "preliminary" &&
|
|
133
|
+
event.type !== "consentFunctionCall"
|
|
134
|
+
)
|
|
135
|
+
phaseState.snapshot = s;
|
|
136
|
+
props.report();
|
|
137
|
+
if (props.on)
|
|
138
|
+
props.on(s.event, agent, {
|
|
139
|
+
vendor: props.vendor,
|
|
140
|
+
project: props.projectState.name,
|
|
141
|
+
});
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
phaseState.success = success;
|
|
145
|
+
phaseState.completed_at = new Date();
|
|
146
|
+
props.report();
|
|
147
|
+
} catch (error) {
|
|
148
|
+
console.log(
|
|
149
|
+
props.vendor,
|
|
150
|
+
props.projectState.name,
|
|
151
|
+
phaseState.name,
|
|
152
|
+
error,
|
|
153
|
+
);
|
|
154
|
+
throw error;
|
|
129
155
|
}
|
|
130
|
-
if (phaseState.success === null) break;
|
|
131
|
-
else if (phaseState.success === false && phaseState.name !== "test")
|
|
132
|
-
break;
|
|
156
|
+
if (phaseState.success === null || phaseState.success === false) break;
|
|
133
157
|
}
|
|
134
158
|
props.projectState.completed_at = new Date();
|
|
135
159
|
props.projectState.success = props.projectState.phases.every(
|
|
@@ -141,7 +165,7 @@ export namespace AutoBeExampleBenchmark {
|
|
|
141
165
|
|
|
142
166
|
const getArchiver = (phase: AutoBePhase) => {
|
|
143
167
|
if (phase === "analyze") return AutoBeExampleArchiver.archiveAnalyze;
|
|
144
|
-
else if (phase === "
|
|
168
|
+
else if (phase === "database") return AutoBeExampleArchiver.archivePrisma;
|
|
145
169
|
else if (phase === "interface") return AutoBeExampleArchiver.archiveInterface;
|
|
146
170
|
else if (phase === "test") return AutoBeExampleArchiver.archiveTest;
|
|
147
171
|
else if (phase === "realize") return AutoBeExampleArchiver.archiveRealize;
|
|
@@ -149,10 +173,17 @@ const getArchiver = (phase: AutoBePhase) => {
|
|
|
149
173
|
throw new Error(`Unknown phase: ${phase}`);
|
|
150
174
|
};
|
|
151
175
|
|
|
152
|
-
const PROJECT_SEQUENCE = [
|
|
176
|
+
const PROJECT_SEQUENCE = [
|
|
177
|
+
"todo",
|
|
178
|
+
"bbs",
|
|
179
|
+
"reddit",
|
|
180
|
+
"shopping",
|
|
181
|
+
"account",
|
|
182
|
+
"erp",
|
|
183
|
+
] as const;
|
|
153
184
|
const PHASE_SEQUENCE = [
|
|
154
185
|
"analyze",
|
|
155
|
-
"
|
|
186
|
+
"database",
|
|
156
187
|
"interface",
|
|
157
188
|
"test",
|
|
158
189
|
"realize",
|
|
@@ -4,7 +4,7 @@ import typia from "typia";
|
|
|
4
4
|
|
|
5
5
|
import { IAutoBeExampleBenchmarkState } from "../structures";
|
|
6
6
|
|
|
7
|
-
export namespace
|
|
7
|
+
export namespace AutoBeExampleLogger {
|
|
8
8
|
export const markdown = (state: IAutoBeExampleBenchmarkState): string =>
|
|
9
9
|
StringUtil.trim`
|
|
10
10
|
# AutoBe Example Benchmark Report
|
|
@@ -34,8 +34,8 @@ export namespace AutoBeExampleDocumentation {
|
|
|
34
34
|
): string => StringUtil.trim`
|
|
35
35
|
## \`${state.name}\`
|
|
36
36
|
|
|
37
|
-
Project | Phase | State | Elapsed Time
|
|
38
|
-
|
|
37
|
+
Project | Phase | State | Count | Elapsed Time
|
|
38
|
+
:-------|:------|:------|------:|-------------:
|
|
39
39
|
${state.projects.map(markdownProject).join("\n")}
|
|
40
40
|
`;
|
|
41
41
|
|
|
@@ -47,22 +47,23 @@ export namespace AutoBeExampleDocumentation {
|
|
|
47
47
|
state.phases.at(-1);
|
|
48
48
|
return [
|
|
49
49
|
state.name,
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
?
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
50
|
+
phase?.name ?? "-",
|
|
51
|
+
phase !== undefined && phase.snapshot !== null
|
|
52
|
+
? [
|
|
53
|
+
state.completed_at !== null
|
|
54
|
+
? state.success
|
|
55
|
+
? "🟢 success"
|
|
56
|
+
: "🔴 failure"
|
|
57
|
+
: "🟡",
|
|
58
|
+
`\`${phase.snapshot.event.type}\``,
|
|
59
|
+
...(typia.is<AutoBeProgressEventBase>(phase.snapshot.event)
|
|
60
|
+
? [
|
|
61
|
+
`(${phase.snapshot.event.completed} of ${phase.snapshot.event.total})`,
|
|
62
|
+
]
|
|
63
|
+
: []),
|
|
64
|
+
].join(" ")
|
|
65
|
+
: "-",
|
|
66
|
+
phase?.count.toLocaleString() ?? "0",
|
|
66
67
|
state.started_at !== null
|
|
67
68
|
? elapsedTime({
|
|
68
69
|
started_at: state.started_at,
|
|
@@ -4,14 +4,14 @@ import {
|
|
|
4
4
|
AutoBeExampleProject,
|
|
5
5
|
AutoBeHistory,
|
|
6
6
|
AutoBePhase,
|
|
7
|
-
|
|
7
|
+
AutoBeUserConversateContent,
|
|
8
|
+
AutoBeUserImageConversateContent,
|
|
8
9
|
IAutoBeTokenUsageJson,
|
|
9
10
|
} from "@autobe/interface";
|
|
10
11
|
import cp from "child_process";
|
|
11
12
|
import fs from "fs";
|
|
12
13
|
import path from "path";
|
|
13
14
|
import { Singleton, VariadicSingleton } from "tstl";
|
|
14
|
-
import { v7 } from "uuid";
|
|
15
15
|
|
|
16
16
|
export namespace AutoBeExampleStorage {
|
|
17
17
|
export const TEST_ROOT: string = `${__dirname}/../../../../test`;
|
|
@@ -35,11 +35,88 @@ export namespace AutoBeExampleStorage {
|
|
|
35
35
|
});
|
|
36
36
|
};
|
|
37
37
|
|
|
38
|
+
export const load = async <T>(props: {
|
|
39
|
+
vendor: string;
|
|
40
|
+
project: AutoBeExampleProject;
|
|
41
|
+
file: string;
|
|
42
|
+
}): Promise<T | null> => {
|
|
43
|
+
const location: string = `${getDirectory(props)}/${props.file}.gz`;
|
|
44
|
+
if (fs.existsSync(location) === false) return null;
|
|
45
|
+
const content: string = await CompressUtil.gunzip(
|
|
46
|
+
await fs.promises.readFile(location),
|
|
47
|
+
);
|
|
48
|
+
return JSON.parse(content) as T;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
export const loadImages = async (
|
|
52
|
+
imagePath: string,
|
|
53
|
+
): Promise<AutoBeUserConversateContent[]> => {
|
|
54
|
+
const stat: fs.Stats = await fs.promises.lstat(imagePath);
|
|
55
|
+
|
|
56
|
+
const filePaths: string[] = await (async () => {
|
|
57
|
+
if (stat.isFile() === true) {
|
|
58
|
+
if (imagePath.endsWith(".png") === false)
|
|
59
|
+
throw new Error("Image Format must be .png");
|
|
60
|
+
return [imagePath];
|
|
61
|
+
}
|
|
62
|
+
const files: string[] = await fs.promises.readdir(imagePath);
|
|
63
|
+
return files
|
|
64
|
+
.filter((f) => f.endsWith(".png"))
|
|
65
|
+
.map((f) => path.join(imagePath, f));
|
|
66
|
+
})();
|
|
67
|
+
|
|
68
|
+
return Promise.all(
|
|
69
|
+
filePaths.map(async (filePath) => {
|
|
70
|
+
const extension: string = path.extname(filePath).toLowerCase().slice(1);
|
|
71
|
+
const base64Data: string = `data:image/${extension};base64,${await fs.promises.readFile(filePath, "base64")}`;
|
|
72
|
+
return {
|
|
73
|
+
type: "image",
|
|
74
|
+
image: {
|
|
75
|
+
type: "base64",
|
|
76
|
+
data: base64Data,
|
|
77
|
+
} satisfies AutoBeUserImageConversateContent.IBase64,
|
|
78
|
+
} satisfies AutoBeUserConversateContent;
|
|
79
|
+
}),
|
|
80
|
+
);
|
|
81
|
+
};
|
|
82
|
+
|
|
38
83
|
export const getUserMessage = async (props: {
|
|
39
84
|
project: AutoBeExampleProject;
|
|
40
85
|
phase: AutoBePhase;
|
|
41
|
-
|
|
86
|
+
imagePath?: string;
|
|
87
|
+
}): Promise<AutoBeUserConversateContent[]> => {
|
|
88
|
+
const imageMessages: AutoBeUserConversateContent[] = props.imagePath
|
|
89
|
+
? await loadImages(props.imagePath)
|
|
90
|
+
: [];
|
|
91
|
+
|
|
42
92
|
const full: string = `${TEST_ROOT}/scripts/${props.project}/${props.phase}`;
|
|
93
|
+
if (props.project === "account" && props.phase === "analyze") {
|
|
94
|
+
const files: string[] = await fs.promises.readdir(full);
|
|
95
|
+
const contents: AutoBeUserConversateContent[] = await Promise.all(
|
|
96
|
+
files.map(async (filename) => {
|
|
97
|
+
const filePath = path.join(full, filename);
|
|
98
|
+
const extension = filename.split(".").pop() ?? "unknown";
|
|
99
|
+
const base64Data = `data:image/${extension};base64,${await fs.promises.readFile(filePath, "base64")}`;
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
type: "image",
|
|
103
|
+
image: {
|
|
104
|
+
type: "base64",
|
|
105
|
+
data: base64Data,
|
|
106
|
+
},
|
|
107
|
+
} satisfies AutoBeUserConversateContent;
|
|
108
|
+
}),
|
|
109
|
+
);
|
|
110
|
+
return [
|
|
111
|
+
...contents,
|
|
112
|
+
...imageMessages,
|
|
113
|
+
{
|
|
114
|
+
type: "text",
|
|
115
|
+
text: "Convert the images into a planning document.",
|
|
116
|
+
},
|
|
117
|
+
];
|
|
118
|
+
}
|
|
119
|
+
|
|
43
120
|
if (fs.existsSync(`${full}.md`) === false) {
|
|
44
121
|
const text: string =
|
|
45
122
|
props.phase === "analyze"
|
|
@@ -48,30 +125,24 @@ export namespace AutoBeExampleStorage {
|
|
|
48
125
|
"utf8",
|
|
49
126
|
)
|
|
50
127
|
: PROMPT_TEMPLATE[props.phase];
|
|
51
|
-
return
|
|
52
|
-
|
|
53
|
-
id: v7(),
|
|
54
|
-
created_at: new Date().toISOString(),
|
|
55
|
-
contents: [
|
|
56
|
-
{
|
|
57
|
-
type: "text",
|
|
58
|
-
text,
|
|
59
|
-
},
|
|
60
|
-
],
|
|
61
|
-
};
|
|
62
|
-
}
|
|
63
|
-
const text: string = await fs.promises.readFile(`${full}.md`, "utf8");
|
|
64
|
-
return {
|
|
65
|
-
type: "userMessage",
|
|
66
|
-
id: v7(),
|
|
67
|
-
created_at: new Date().toISOString(),
|
|
68
|
-
contents: [
|
|
128
|
+
return [
|
|
129
|
+
...imageMessages,
|
|
69
130
|
{
|
|
70
131
|
type: "text",
|
|
71
|
-
text
|
|
132
|
+
text,
|
|
72
133
|
},
|
|
73
|
-
]
|
|
74
|
-
}
|
|
134
|
+
];
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const text: string = await fs.promises.readFile(`${full}.md`, "utf8");
|
|
138
|
+
|
|
139
|
+
return [
|
|
140
|
+
...imageMessages,
|
|
141
|
+
{
|
|
142
|
+
type: "text",
|
|
143
|
+
text,
|
|
144
|
+
},
|
|
145
|
+
];
|
|
75
146
|
};
|
|
76
147
|
|
|
77
148
|
export const getVendorModels = async (): Promise<string[]> => {
|
|
@@ -144,7 +215,7 @@ export namespace AutoBeExampleStorage {
|
|
|
144
215
|
aggregate: component(),
|
|
145
216
|
facade: component(),
|
|
146
217
|
analyze: component(),
|
|
147
|
-
|
|
218
|
+
database: component(),
|
|
148
219
|
interface: component(),
|
|
149
220
|
test: component(),
|
|
150
221
|
realize: component(),
|
|
@@ -170,7 +241,7 @@ export namespace AutoBeExampleStorage {
|
|
|
170
241
|
};
|
|
171
242
|
|
|
172
243
|
const PROMPT_TEMPLATE = {
|
|
173
|
-
|
|
244
|
+
database: "Design the database schema.",
|
|
174
245
|
interface: "Create the API interface specification.",
|
|
175
246
|
test: "Make the e2e test functions.",
|
|
176
247
|
realize: "Implement API functions.",
|
package/src/example/index.ts
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
IAutoBePlaygroundReplay,
|
|
7
7
|
} from "@autobe/interface";
|
|
8
8
|
import { AutoBeProcessAggregateFactory } from "@autobe/utils";
|
|
9
|
+
import typia from "typia";
|
|
9
10
|
|
|
10
11
|
export namespace AutoBeReplayComputer {
|
|
11
12
|
export const SIGNIFICANT_PROJECTS: AutoBeExampleProject[] = [
|
|
@@ -35,33 +36,13 @@ export namespace AutoBeReplayComputer {
|
|
|
35
36
|
["todo", "bbs", "reddit", "shopping"].includes(s.project),
|
|
36
37
|
);
|
|
37
38
|
|
|
38
|
-
// the formula to compute the benchmark score
|
|
39
|
-
const compute = (summary: IAutoBePlaygroundReplay.ISummary): number => {
|
|
40
|
-
const add = (
|
|
41
|
-
phase: IAutoBePlaygroundReplay.IPhaseState | null,
|
|
42
|
-
success: number,
|
|
43
|
-
failure?: number,
|
|
44
|
-
): number =>
|
|
45
|
-
phase !== null
|
|
46
|
-
? phase.success === true
|
|
47
|
-
? success
|
|
48
|
-
: (failure ?? success / 2)
|
|
49
|
-
: 0;
|
|
50
|
-
return (
|
|
51
|
-
add(summary.analyze, 10) +
|
|
52
|
-
add(summary.prisma, 20) +
|
|
53
|
-
add(summary.interface, 30) +
|
|
54
|
-
add(summary.test, 20) +
|
|
55
|
-
add(summary.realize, 20)
|
|
56
|
-
);
|
|
57
|
-
};
|
|
58
39
|
const individual = (project: AutoBeExampleProject): number => {
|
|
59
40
|
const found = summaries.find((s) => s.project === project);
|
|
60
41
|
if (found === undefined) return 0;
|
|
61
42
|
return compute(found);
|
|
62
43
|
};
|
|
63
44
|
return {
|
|
64
|
-
aggregate: summaries.map(compute).reduce((a, b) => a + b, 0) / 4,
|
|
45
|
+
aggregate: round(summaries.map(compute).reduce((a, b) => a + b, 0) / 4),
|
|
65
46
|
todo: individual("todo"),
|
|
66
47
|
bbs: individual("bbs"),
|
|
67
48
|
reddit: individual("reddit"),
|
|
@@ -110,8 +91,8 @@ export namespace AutoBeReplayComputer {
|
|
|
110
91
|
documents: h.files.length,
|
|
111
92
|
}),
|
|
112
93
|
),
|
|
113
|
-
|
|
114
|
-
"
|
|
94
|
+
database: predicate(
|
|
95
|
+
"database",
|
|
115
96
|
(h) => h.compiled.type === "success",
|
|
116
97
|
(h) => ({
|
|
117
98
|
namespaces: h.result.data.files.length,
|
|
@@ -130,7 +111,7 @@ export namespace AutoBeReplayComputer {
|
|
|
130
111
|
"test",
|
|
131
112
|
(h) => h.compiled.type === "success",
|
|
132
113
|
(h) => ({
|
|
133
|
-
functions: h.
|
|
114
|
+
functions: h.functions.length,
|
|
134
115
|
...(h.compiled.type === "failure"
|
|
135
116
|
? {
|
|
136
117
|
errors: new Set(h.compiled.diagnostics.map((d) => d.file ?? ""))
|
|
@@ -154,36 +135,49 @@ export namespace AutoBeReplayComputer {
|
|
|
154
135
|
),
|
|
155
136
|
};
|
|
156
137
|
const phase: AutoBePhase | null =
|
|
157
|
-
(["realize", "test", "interface", "
|
|
138
|
+
(["realize", "test", "interface", "database", "analyze"] as const).find(
|
|
158
139
|
(key) => phaseStates[key] !== null,
|
|
159
140
|
) ?? null;
|
|
160
141
|
return {
|
|
161
142
|
vendor: replay.vendor,
|
|
162
143
|
project: replay.project,
|
|
144
|
+
...phaseStates,
|
|
163
145
|
aggregates: AutoBeProcessAggregateFactory.reduce(
|
|
164
|
-
|
|
165
|
-
.filter(
|
|
166
|
-
|
|
167
|
-
h.type === "analyze" ||
|
|
168
|
-
h.type === "prisma" ||
|
|
169
|
-
h.type === "interface" ||
|
|
170
|
-
h.type === "test" ||
|
|
171
|
-
h.type === "realize",
|
|
172
|
-
)
|
|
173
|
-
.map((h) => h.aggregates),
|
|
146
|
+
Object.values(phaseStates)
|
|
147
|
+
.filter((p) => p !== null)
|
|
148
|
+
.map((p) => p.aggregates),
|
|
174
149
|
),
|
|
175
|
-
elapsed: replay.histories
|
|
176
|
-
.filter(
|
|
177
|
-
(h) => h.type !== "userMessage" && h.type !== "assistantMessage",
|
|
178
|
-
)
|
|
179
|
-
.map(
|
|
180
|
-
(h) =>
|
|
181
|
-
new Date(h.completed_at).getTime() -
|
|
182
|
-
new Date(h.created_at).getTime(),
|
|
183
|
-
)
|
|
184
|
-
.reduce((a, b) => a + b, 0),
|
|
185
|
-
...phaseStates,
|
|
186
150
|
phase,
|
|
151
|
+
elapsed: Object.values(phaseStates)
|
|
152
|
+
.map((p) => p?.elapsed ?? 0)
|
|
153
|
+
.reduce((a, b) => a + (b ?? 0), 0),
|
|
187
154
|
};
|
|
188
155
|
};
|
|
189
156
|
}
|
|
157
|
+
|
|
158
|
+
const compute = (summary: IAutoBePlaygroundReplay.ISummary): number => {
|
|
159
|
+
const getScore = (phase: AutoBePhase): number => {
|
|
160
|
+
const state = summary[phase];
|
|
161
|
+
if (state === null) return 0;
|
|
162
|
+
|
|
163
|
+
const [success, failure] = FORMULA[phase];
|
|
164
|
+
return state.success === true
|
|
165
|
+
? success
|
|
166
|
+
: success * failure(state.commodity);
|
|
167
|
+
};
|
|
168
|
+
return round(sum(typia.misc.literals<AutoBePhase>().map(getScore)));
|
|
169
|
+
};
|
|
170
|
+
const round = (value: number) => Math.round(value * 100) / 100;
|
|
171
|
+
const sum = (targets: number[]): number => targets.reduce((a, b) => a + b, 0);
|
|
172
|
+
|
|
173
|
+
// for type safety
|
|
174
|
+
const FORMULA: Record<
|
|
175
|
+
AutoBePhase,
|
|
176
|
+
[number, (commodity: Record<string, number>) => number]
|
|
177
|
+
> = {
|
|
178
|
+
analyze: [10, () => 0],
|
|
179
|
+
database: [20, () => 0.5],
|
|
180
|
+
interface: [30, () => 0.5],
|
|
181
|
+
test: [20, (c) => Math.max(0.5, 1 - (c.errors * 3) / c.functions)],
|
|
182
|
+
realize: [20, (c) => Math.max(0.5, 1 - (c.errors * 3) / c.functions)],
|
|
183
|
+
};
|
|
@@ -64,9 +64,10 @@ export namespace AutoBeReplayDocumentation {
|
|
|
64
64
|
};
|
|
65
65
|
return [
|
|
66
66
|
`[\`${found.project}\`](./${exp.vendor}/${found.project}/)`,
|
|
67
|
+
// biome-ignore lint: intended
|
|
67
68
|
(exp.score as any)[project],
|
|
68
69
|
phase(found.analyze),
|
|
69
|
-
phase(found.
|
|
70
|
+
phase(found.database),
|
|
70
71
|
phase(found.interface),
|
|
71
72
|
phase(found.test),
|
|
72
73
|
phase(found.realize),
|
|
@@ -86,6 +87,7 @@ export namespace AutoBeReplayDocumentation {
|
|
|
86
87
|
.map((r) =>
|
|
87
88
|
project({
|
|
88
89
|
replay: r,
|
|
90
|
+
// biome-ignore lint: intended
|
|
89
91
|
score: (exp.score as any)[r.project],
|
|
90
92
|
}),
|
|
91
93
|
)
|
|
@@ -103,7 +105,7 @@ export namespace AutoBeReplayDocumentation {
|
|
|
103
105
|
props.replay[key];
|
|
104
106
|
if (state === null) return [`⚪ ${title}`, "", "", "", ""].join(" | ");
|
|
105
107
|
return [
|
|
106
|
-
`${state.success === true ? "🟢" : "🔴"} ${title}`,
|
|
108
|
+
`${state.success === true ? "🟢" : Object.keys(state.commodity).length ? "🟡" : "🔴"} ${title}`,
|
|
107
109
|
Object.entries(state.commodity)
|
|
108
110
|
.map(([key, value]) => `\`${key}\`: ${value}`)
|
|
109
111
|
.join(", "),
|
|
@@ -139,7 +141,7 @@ export namespace AutoBeReplayDocumentation {
|
|
|
139
141
|
|
|
140
142
|
Phase | Generated | Token Usage | Elapsed Time | FCSR
|
|
141
143
|
:-----|:----------|------------:|-------------:|------:
|
|
142
|
-
${(["analyze", "
|
|
144
|
+
${(["analyze", "database", "interface", "test", "realize"] as const)
|
|
143
145
|
.map((key) => phase(key))
|
|
144
146
|
.join("\n")}
|
|
145
147
|
`;
|
|
@@ -71,7 +71,7 @@ export namespace AutoBeReplayStorage {
|
|
|
71
71
|
project: props.project,
|
|
72
72
|
histories,
|
|
73
73
|
analyze: await snapshots("analyze"),
|
|
74
|
-
|
|
74
|
+
database: await snapshots("database"),
|
|
75
75
|
interface: await snapshots("interface"),
|
|
76
76
|
test: await snapshots("test"),
|
|
77
77
|
realize: await snapshots("realize"),
|
|
@@ -84,9 +84,10 @@ export namespace AutoBeReplayStorage {
|
|
|
84
84
|
}): Promise<IAutoBePlaygroundReplay.ISummary | null> => {
|
|
85
85
|
const location: string = `${AutoBeExampleStorage.getDirectory(props)}/summary.json.gz`;
|
|
86
86
|
if (fs.existsSync(location) === false) return null;
|
|
87
|
-
|
|
87
|
+
const replay: IAutoBePlaygroundReplay.ISummary = JSON.parse(
|
|
88
88
|
await CompressUtil.gunzip(await fs.promises.readFile(location)),
|
|
89
89
|
);
|
|
90
|
+
return replay;
|
|
90
91
|
};
|
|
91
92
|
|
|
92
93
|
const getHistories = async (props: {
|
|
@@ -106,4 +107,10 @@ export namespace AutoBeReplayStorage {
|
|
|
106
107
|
};
|
|
107
108
|
}
|
|
108
109
|
|
|
109
|
-
const SEQUENCE = [
|
|
110
|
+
const SEQUENCE = [
|
|
111
|
+
"realize",
|
|
112
|
+
"test",
|
|
113
|
+
"interface",
|
|
114
|
+
"database",
|
|
115
|
+
"analyze",
|
|
116
|
+
] as const;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"AutoBeExampleDocumentation.js","sourceRoot":"","sources":["../../src/example/AutoBeExampleDocumentation.ts"],"names":[],"mappings":";;;;;;AACA,yCAA2C;AAC3C,kDAA0B;AAI1B,IAAiB,0BAA0B,CAmE1C;AAnED,WAAiB,0BAA0B;IAC5B,mCAAQ,GAAG,CAAC,KAAmC,EAAU,EAAE,CACtE,kBAAU,CAAC,IAAI,CAAA;;;QAGX,aAAa,CAAC,KAAK,CAAC;;QAEpB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;KACjD,CAAC;IAEJ,MAAM,aAAa,GAAG,CACpB,KAAmC,EAC3B,EAAE,CAAC,kBAAU,CAAC,IAAI,CAAA;;;MAGxB,KAAK,CAAC,OAAO;SACZ,GAAG,CACF,CAAC,MAAM,EAAE,EAAE,CACT,QAAQ,MAAM,CAAC,IAAI,QAAQ,MAAM,CAAC,IAAI;SACnC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC;SACnB,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAC5B;SACA,IAAI,CAAC,IAAI,CAAC;GACd,CAAC;IAEF,MAAM,cAAc,GAAG,CACrB,KAA6C,EACrC,EAAE,CAAC,kBAAU,CAAC,IAAI,CAAA;WACnB,KAAK,CAAC,IAAI;;;;MAIf,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;GACjD,CAAC;IAEF,MAAM,eAAe,GAAG,CACtB,KAA8C,EACtC,EAAE;QACV,uBAAuB;QACvB,MAAM,KAAK,GACT,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO;YACL,KAAK,CAAC,IAAI;YACV,CAAC,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,CAAA,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG;YACtD,KAAK,CAAC,YAAY,KAAK,IAAI;gBACzB,CAAC,CAAC,KAAK,CAAC,OAAO;oBACb,CAAC,CAAC,YAAY;oBACd,CAAC,CAAC,YAAY;gBAChB,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;oBAC9C,CAAC,CAAC;wBACE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;wBAC/B,KAAK,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI;wBAClC,GAAG,CAAC,eAAK,CAAC,EAAE,CAA0B,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;4BACzD,CAAC,CAAC;gCACE,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG;6BACvE;4BACH,CAAC,CAAC,EAAE,CAAC;qBACR,CAAC,IAAI,CAAC,GAAG,CAAC;oBACb,CAAC,CAAC,GAAG;YACT,KAAK,CAAC,UAAU,KAAK,IAAI;gBACvB,CAAC,CAAC,WAAW,CAAC;oBACV,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;iBACjC,CAAC;gBACJ,CAAC,CAAC,GAAG;SACR,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC,EAnEgB,0BAA0B,0CAA1B,0BAA0B,QAmE1C;AAED,MAAM,WAAW,GAAG,CAAC,KAGpB,EAAU,EAAE;;IACX,OAAA,IAAI,CAAC,KAAK,CACR,CAAC,CAAC,MAAA,KAAK,CAAC,YAAY,mCAAI,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;QAC3C,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAC3B,IAAK,CACR,CAAC,cAAc,EAAE,GAAG,MAAM,CAAA;CAAA,CAAC"}
|