@autobe/benchmark 0.29.1 → 0.29.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/example/AutoBeExampleArchiver.d.ts +1 -0
- package/lib/example/AutoBeExampleArchiver.js +2 -0
- package/lib/example/AutoBeExampleArchiver.js.map +1 -1
- package/lib/example/AutoBeExampleBenchmark.js +17 -3
- package/lib/example/AutoBeExampleBenchmark.js.map +1 -1
- package/lib/example/AutoBeExampleDocumentation.d.ts +4 -0
- package/lib/example/AutoBeExampleDocumentation.js +70 -0
- package/lib/example/AutoBeExampleDocumentation.js.map +1 -0
- package/lib/example/index.d.ts +1 -0
- package/lib/example/index.js +1 -0
- package/lib/example/index.js.map +1 -1
- package/lib/replay/AutoBeReplayDocumentation.js +3 -2
- package/lib/replay/AutoBeReplayDocumentation.js.map +1 -1
- package/lib/structures/IAutoBeExampleBenchmarkState.d.ts +2 -1
- package/package.json +4 -4
- package/src/example/AutoBeExampleArchiver.ts +3 -0
- package/src/example/AutoBeExampleBenchmark.ts +19 -3
- package/src/example/AutoBeExampleDocumentation.ts +84 -0
- package/src/example/index.ts +1 -0
- package/src/replay/AutoBeReplayDocumentation.ts +3 -2
- package/src/structures/IAutoBeExampleBenchmarkState.ts +2 -1
|
@@ -200,6 +200,7 @@ var AutoBeExampleArchiver;
|
|
|
200
200
|
const previous = (_a = PHASES[PHASES.indexOf(props.phase) - 1]) !== null && _a !== void 0 ? _a : null;
|
|
201
201
|
if (previous === null)
|
|
202
202
|
return {
|
|
203
|
+
vendor: props.vendor,
|
|
203
204
|
histories: [],
|
|
204
205
|
tokenUsage: {
|
|
205
206
|
aggregate: utils_1.TokenUsageComputer.zero(),
|
|
@@ -222,6 +223,7 @@ var AutoBeExampleArchiver;
|
|
|
222
223
|
phase: previous,
|
|
223
224
|
});
|
|
224
225
|
return {
|
|
226
|
+
vendor: props.vendor,
|
|
225
227
|
histories,
|
|
226
228
|
tokenUsage,
|
|
227
229
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AutoBeExampleArchiver.js","sourceRoot":"","sources":["../../src/example/AutoBeExampleArchiver.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,mDAAwD;AAexD,yCAGuB;AACvB,kDAA0B;AAE1B,sCAAiD;AACjD,iEAA8D;AAE9D,IAAiB,qBAAqB,
|
|
1
|
+
{"version":3,"file":"AutoBeExampleArchiver.js","sourceRoot":"","sources":["../../src/example/AutoBeExampleArchiver.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,mDAAwD;AAexD,yCAGuB;AACvB,kDAA0B;AAE1B,sCAAiD;AACjD,iEAA8D;AAE9D,IAAiB,qBAAqB,CAgTrC;AAhTD,WAAiB,qBAAqB;IAcvB,oCAAc,GAAG,CAAC,GAAa,EAAoB,EAAE,CAChE,OAAO,CAAC,GAAG,EAAE;QACX,KAAK,EAAE,SAAS;QAChB,KAAK,EAAE,CAAO,UAAU,EAAoB,EAAE;YAC5C,OAAA,CAAC,MAAM,UAAU,CACf,MAAM,2CAAoB,CAAC,cAAc,CAAC;gBACxC,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,KAAK,EAAE,SAAS;aACjB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAC3B,CAAC;gBACF,CAAC,MAAM,UAAU,CACf,2HAA2H,CAC5H,CAAC;gBACF,CAAC,MAAM,UAAU,CACf,qGAAqG,CACtG,CAAC,CAAA;UAAA;QACJ,SAAS,EAAE,CAAC,SAAS,EAAW,EAAE,CAChC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;KAC9C,CAAC,CAAC;IAEQ,mCAAa,GAAG,CAAC,GAAa,EAAoB,EAAE,CAC/D,OAAO,CAAC,GAAG,EAAE;QACX,KAAK,EAAE,QAAQ;QACf,KAAK,EAAE,QAAQ,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,KAAK,EAAE,QAAQ;SAChB,CAAC;QACF,SAAS,EAAE,CAAC,SAAS,EAAW,EAAE;YAChC,MAAM,MAAM,GAAoC,SAAS,CAAC,IAAI,CAC5D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAC3B,CAAC;YACF,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC;QACpE,CAAC;KACF,CAAC,CAAC;IAEQ,sCAAgB,GAAG,CAAC,GAAa,EAAoB,EAAE,CAClE,OAAO,CAAC,GAAG,EAAE;QACX,KAAK,EAAE,WAAW;QAClB,KAAK,EAAE,QAAQ,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,KAAK,EAAE,WAAW;SACnB,CAAC;QACF,SAAS,EAAE,CAAC,SAAS,EAAW,EAAE;YAChC,MAAM,gBAAgB,GAA8B,SAAS,CAAC,IAAI,CAChE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAC9B,CAAC;YACF,OAAO,CACL,gBAAgB,KAAK,SAAS,IAAI,gBAAgB,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CACvE,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;IAEQ,iCAAW,GAAG,CAAC,GAAa,EAAoB,EAAE,CAC7D,OAAO,CAAC,GAAG,EAAE;QACX,KAAK,EAAE,MAAM;QACb,KAAK,EAAE,QAAQ,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,KAAK,EAAE,MAAM;SACd,CAAC;QACF,SAAS,EAAE,CAAC,SAAS,EAAW,EAAE;YAChC,MAAM,WAAW,GAA8B,SAAS,CAAC,IAAI,CAC3D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CACzB,CAAC;YACF,OAAO,CACL,WAAW,KAAK,SAAS,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,CACrE,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;IAEQ,oCAAc,GAAG,CAAC,GAAa,EAAoB,EAAE,CAChE,OAAO,CAAC,GAAG,EAAE;QACX,KAAK,EAAE,SAAS;QAChB,KAAK,EAAE,QAAQ,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,KAAK,EAAE,SAAS;SACjB,CAAC;QACF,SAAS,EAAE,CAAC,SAAS,EAAW,EAAE;YAChC,MAAM,cAAc,GAA8B,SAAS,CAAC,IAAI,CAC9D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAC5B,CAAC;YACF,OAAO,CACL,cAAc,KAAK,SAAS;gBAC5B,cAAc,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,CAC3C,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;IAEL,MAAM,OAAO,GAAG,CACd,GAAa,EACb,KAQC,EACiB,EAAE;QACpB,mBAAmB;QACnB,MAAM,KAAK,GAAgB,MAAM,QAAQ,CAAC;YACxC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;QACH,MAAM,KAAK,GAAiB,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,SAAS,GAA0B,EAAE,CAAC;QAC5C,KAAK,MAAM,IAAI,IAAI,eAAK,CAAC,IAAI,CAAC,QAAQ,EAAkC,EAAE,CAAC;YACzE,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE;gBACnB,MAAM,CAAC,GAAwB;oBAC7B,KAAK,EAAE,CAAC;oBACR,UAAU,EAAE,KAAK,CAAC,aAAa,EAAE;iBAClC,CAAC;gBACF,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACV,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,CAChB,SAA0B,EAC1B,KAAc,EACC,EAAE;YACjB,MAAM,MAAM,GAA4B;gBACtC,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,SAAS;gBACT,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI;gBACf,IAAI,EAAE,IAAI;gBACV,OAAO,EAAE,IAAI;aACd,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,MAAM;gBACxB,IAAI,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;oBAC1B,MAAM;gBACR,CAAC;;oBACC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,2CAAoB,CAAC,YAAY,CAAC;wBACtD,MAAM,EAAE,GAAG,CAAC,MAAM;wBAClB,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,KAAK;qBACN,CAAC,CAAC;YACP,MAAM,OAAO,GACX,6BAAoB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,UAAU,GACd,qCAA6B,CAAC,gBAAgB,EAAE,CAAC;gBACnD,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,SAAS,EAAE,CAAC;oBAClC,IAAI,eAAK,CAAC,EAAE,CAA2B,KAAK,CAAC,KAAK,KAAK;wBAAE,SAAS;oBAClE,qCAA6B,CAAC,YAAY,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;gBAChE,CAAC;gBACD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG;oBACrB,UAAU;oBACV,OAAO,EAAE,KAAK;oBACd,OAAO,EAAE,CAAC;oBACV,SAAS,EAAE,EAAE;iBACd,CAAC;YACJ,CAAC;YACD,MAAM,2CAAoB,CAAC,IAAI,CAAC;gBAC9B,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,KAAK,EAAE;oBACL,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;iBAC1C;aACF,CAAC,CAAC;QACL,CAAC,CAAA,CAAC;QAEF,IAAI,CAAC;YACH,aAAa;YACb,MAAM,EAAE,GAAG,CACT,CAAiE,EAC/C,EAAE;gBACpB,MAAM,MAAM,GAAoB,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC1D,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC;YACpD,CAAC,CAAA,CAAC;YACF,MAAM,IAAI,GAAY,MAAM,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC5C,IAAI,IAAI,KAAK,KAAK;gBAChB,MAAM,IAAI,KAAK,CACb,sCAAsC,KAAK,CAAC,KAAK,mBAAmB,GAAG,CAAC,OAAO,YAAY,CAC5F,CAAC;YAEJ,YAAY;YACZ,MAAM,SAAS,GAAoB,KAAK,CAAC,YAAY,EAAE,CAAC;YACxD,IAAI,CAAC;gBACH,MAAM,+BAAkB,CAAC,IAAI,CAAC;oBAC5B,IAAI,EAAE,GAAG,2CAAoB,CAAC,SAAS,YAAY,2CAAoB,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE;oBACpI,KAAK,kCACA,CAAC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC,GACxB,MAAM,CAAC,WAAW,CACnB,SAAS;yBACN,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,QAAQ;wBACnB,CAAC,CAAC,IAAI,KAAK,WAAW;wBACtB,CAAC,CAAC,IAAI,KAAK,MAAM;wBACjB,CAAC,CAAC,IAAI,KAAK,SAAS,CACvB;yBACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,iBAAiB,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAClE,CACF;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,WAAM,CAAC,CAAA,CAAC;YACV,MAAM,2CAAoB,CAAC,IAAI,CAAC;gBAC9B,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,KAAK,EAAE;oBACL,CAAC,GAAG,KAAK,CAAC,KAAK,iBAAiB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;oBAC5D,CAAC,GAAG,KAAK,CAAC,KAAK,iBAAiB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;oBAC5D,CAAC,GAAG,KAAK,CAAC,KAAK,aAAa,CAAC,EAAE,IAAI;iBACpC;aACF,CAAC,CAAC;YACH,MAAM,SAAS,CAAC,CAAC,GAAG,KAAK,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;YAC3D,OAAO,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK;gBACxB,MAAM,2CAAoB,CAAC,IAAI,CAAC;oBAC9B,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,KAAK,EAAE;wBACL,CAAC,GAAG,KAAK,CAAC,KAAK,iBAAiB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;wBAC5D,CAAC,GAAG,KAAK,CAAC,KAAK,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,iCACxC,KAAK,KACR,IAAI,EAAE,KAAK,CAAC,IAAI,EAChB,OAAO,EAAE,KAAK,CAAC,OAAO,EACtB,KAAK,EAAE,KAAK,CAAC,KAAK,IAClB;qBACH;iBACF,CAAC,CAAC;YACL,MAAM,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACvC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC,CAAA,CAAC;IAEF,MAAM,QAAQ,GACZ,CAAC,KAA4D,EAAE,EAAE,CACjE,CACE,UAEqB,EACH,EAAE;QACpB,OAAA,CAAC,MAAM,UAAU,CACf,MAAM,2CAAoB,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,IAAI,CACnD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAClB,CACF,CAAC;YACF,CAAC,MAAM,UAAU,CACf,oDAAoD,CACrD,CAAC;YACF,CAAC,MAAM,UAAU,CACf,4BAA4B,KAAK,CAAC,KAAK,sEAAsE,CAC9G,CAAC,CAAA;MAAA,CAAC;IAEP,MAAM,QAAQ,GAAG,CAAO,KAIvB,EAAwB,EAAE;;QACzB,MAAM,QAAQ,GACZ,MAAA,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,mCAAI,IAAI,CAAC;QAClD,IAAI,QAAQ,KAAK,IAAI;YACnB,OAAO;gBACL,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,SAAS,EAAE,EAAE;gBACb,UAAU,EAAE;oBACV,SAAS,EAAE,0BAAkB,CAAC,IAAI,EAAE;oBACpC,MAAM,EAAE,0BAAkB,CAAC,IAAI,EAAE;oBACjC,OAAO,EAAE,0BAAkB,CAAC,IAAI,EAAE;oBAClC,MAAM,EAAE,0BAAkB,CAAC,IAAI,EAAE;oBACjC,SAAS,EAAE,0BAAkB,CAAC,IAAI,EAAE;oBACpC,IAAI,EAAE,0BAAkB,CAAC,IAAI,EAAE;oBAC/B,OAAO,EAAE,0BAAkB,CAAC,IAAI,EAAE;iBACnC;aACF,CAAC;QACJ,MAAM,SAAS,GAAoB,MAAM,2CAAoB,CAAC,YAAY,CAAC;YACzE,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;QACH,MAAM,UAAU,GACd,MAAM,2CAAoB,CAAC,aAAa,CAAC;YACvC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;QACL,OAAO;YACL,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,SAAS;YACT,UAAU;SACX,CAAC;IACJ,CAAC,CAAA,CAAC;AACJ,CAAC,EAhTgB,qBAAqB,qCAArB,qBAAqB,QAgTrC;AAED,MAAM,MAAM,GAAkB;IAC5B,SAAS;IACT,QAAQ;IACR,WAAW;IACX,MAAM;IACN,SAAS;CACV,CAAC"}
|
|
@@ -21,7 +21,7 @@ var AutoBeExampleBenchmark;
|
|
|
21
21
|
name: project,
|
|
22
22
|
phases: [],
|
|
23
23
|
success: null,
|
|
24
|
-
started_at:
|
|
24
|
+
started_at: null,
|
|
25
25
|
completed_at: null,
|
|
26
26
|
})),
|
|
27
27
|
})),
|
|
@@ -45,6 +45,7 @@ var AutoBeExampleBenchmark;
|
|
|
45
45
|
});
|
|
46
46
|
});
|
|
47
47
|
const executeProject = (ctx, props) => __awaiter(this, void 0, void 0, function* () {
|
|
48
|
+
props.projectState.started_at = new Date();
|
|
48
49
|
for (const phase of PHASE_SEQUENCE) {
|
|
49
50
|
if (props.phases && props.phases.includes(phase) === false)
|
|
50
51
|
continue;
|
|
@@ -54,10 +55,12 @@ var AutoBeExampleBenchmark;
|
|
|
54
55
|
success: null,
|
|
55
56
|
started_at: new Date(),
|
|
56
57
|
completed_at: null,
|
|
58
|
+
trial: 0,
|
|
57
59
|
};
|
|
58
60
|
props.projectState.phases.push(phaseState);
|
|
59
61
|
for (let i = 0; i < 3; ++i) {
|
|
60
62
|
try {
|
|
63
|
+
++phaseState.trial;
|
|
61
64
|
phaseState.started_at = new Date();
|
|
62
65
|
phaseState.completed_at = null;
|
|
63
66
|
const success = yield getArchiver(phase)({
|
|
@@ -65,7 +68,12 @@ var AutoBeExampleBenchmark;
|
|
|
65
68
|
project: props.projectState.name,
|
|
66
69
|
agent: (next) => ctx.createAgent(next),
|
|
67
70
|
on: (s) => {
|
|
68
|
-
|
|
71
|
+
const event = s.event;
|
|
72
|
+
if (event.type !== "jsonValidateError" &&
|
|
73
|
+
event.type !== "jsonParseError" &&
|
|
74
|
+
event.type !== "preliminary" &&
|
|
75
|
+
event.type !== "consentFunctionCall")
|
|
76
|
+
phaseState.snapshot = s;
|
|
69
77
|
props.report();
|
|
70
78
|
if (props.on)
|
|
71
79
|
props.on(s.event);
|
|
@@ -106,5 +114,11 @@ const getArchiver = (phase) => {
|
|
|
106
114
|
throw new Error(`Unknown phase: ${phase}`);
|
|
107
115
|
};
|
|
108
116
|
const PROJECT_SEQUENCE = ["todo", "bbs", "reddit", "shopping"];
|
|
109
|
-
const PHASE_SEQUENCE = [
|
|
117
|
+
const PHASE_SEQUENCE = [
|
|
118
|
+
"analyze",
|
|
119
|
+
"prisma",
|
|
120
|
+
"interface",
|
|
121
|
+
"test",
|
|
122
|
+
"realize",
|
|
123
|
+
];
|
|
110
124
|
//# sourceMappingURL=AutoBeExampleBenchmark.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AutoBeExampleBenchmark.js","sourceRoot":"","sources":["../../src/example/AutoBeExampleBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;AAQA,mEAAgE;AAEhE,IAAiB,sBAAsB,
|
|
1
|
+
{"version":3,"file":"AutoBeExampleBenchmark.js","sourceRoot":"","sources":["../../src/example/AutoBeExampleBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;AAQA,mEAAgE;AAEhE,IAAiB,sBAAsB,CAiItC;AAjID,WAAiB,sBAAsB;IAOxB,8BAAO,GAAG,CACrB,GAAa,EACb,KAMC,EACc,EAAE;QACjB,MAAM,KAAK,GAAiC;YAC1C,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CACxB,CAAC,MAAM,EAA0C,EAAE,CAAC,CAAC;gBACnD,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,gBAAgB,CAAC,MAAM,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CACrD,CAAC,GAAG,CACH,CAAC,OAAO,EAA2C,EAAE,CAAC,CAAC;oBACrD,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI;oBACb,UAAU,EAAE,IAAI;oBAChB,YAAY,EAAE,IAAI;iBACnB,CAAC,CACH;aACF,CAAC,CACH;SACF,CAAC;QACF,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAC3B,aAAa,CAAC,GAAG,EAAE;YACjB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,WAAW,EAAE,MAAM;YACnB,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,MAAM;SACP,CAAC,CACH,CACF,CAAC;IACJ,CAAC,CAAA,CAAC;IAEF,MAAM,aAAa,GAAG,CACpB,GAAa,EACb,KAKC,EACc,EAAE;QACjB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,QAAQ;YAC9C,MAAM,cAAc,CAAC,GAAG,EAAE;gBACxB,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI;gBAC9B,YAAY,EAAE,OAAO;gBACrB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,EAAE,EAAE,KAAK,CAAC,EAAE;aACb,CAAC,CAAC;IACP,CAAC,CAAA,CAAC;IAEF,MAAM,cAAc,GAAG,CACrB,GAAa,EACb,KAMC,EACc,EAAE;QACjB,KAAK,CAAC,YAAY,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,KAAK;gBAAE,SAAS;YACrE,MAAM,UAAU,GAA0C;gBACxD,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,IAAI;gBACb,UAAU,EAAE,IAAI,IAAI,EAAE;gBACtB,YAAY,EAAE,IAAI;gBAClB,KAAK,EAAE,CAAC;aACT,CAAC;YACF,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC3C,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC;oBACH,EAAE,UAAU,CAAC,KAAK,CAAC;oBACnB,UAAU,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;oBACnC,UAAU,CAAC,YAAY,GAAG,IAAI,CAAC;oBAC/B,MAAM,OAAO,GAAY,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;wBAChD,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI;wBAChC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;wBACtC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE;4BACR,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;4BACtB,IACE,KAAK,CAAC,IAAI,KAAK,mBAAmB;gCAClC,KAAK,CAAC,IAAI,KAAK,gBAAgB;gCAC/B,KAAK,CAAC,IAAI,KAAK,aAAa;gCAC5B,KAAK,CAAC,IAAI,KAAK,qBAAqB;gCAEpC,UAAU,CAAC,QAAQ,GAAG,CAAC,CAAC;4BAC1B,KAAK,CAAC,MAAM,EAAE,CAAC;4BACf,IAAI,KAAK,CAAC,EAAE;gCAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;wBAClC,CAAC;qBACF,CAAC,CAAC;oBACH,UAAU,CAAC,OAAO,GAAG,OAAO,CAAC;oBAC7B,UAAU,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;oBACrC,KAAK,CAAC,MAAM,EAAE,CAAC;oBACf,IAAI,OAAO,KAAK,IAAI;wBAAE,MAAM;gBAC9B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,SAAS;gBACX,CAAC;YACH,CAAC;YACD,IAAI,UAAU,CAAC,OAAO,KAAK,IAAI;gBAAE,MAAM;iBAClC,IAAI,UAAU,CAAC,OAAO,KAAK,KAAK,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM;gBACjE,MAAM;QACV,CAAC;QACD,KAAK,CAAC,YAAY,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,KAAK,CAAC,YAAY,CAAC,OAAO,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAC1D,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAClC,CAAC;QACF,KAAK,CAAC,MAAM,EAAE,CAAC;IACjB,CAAC,CAAA,CAAC;AACJ,CAAC,EAjIgB,sBAAsB,sCAAtB,sBAAsB,QAiItC;AAED,MAAM,WAAW,GAAG,CAAC,KAAkB,EAAE,EAAE;IACzC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,6CAAqB,CAAC,cAAc,CAAC;SAChE,IAAI,KAAK,KAAK,QAAQ;QAAE,OAAO,6CAAqB,CAAC,aAAa,CAAC;SACnE,IAAI,KAAK,KAAK,WAAW;QAAE,OAAO,6CAAqB,CAAC,gBAAgB,CAAC;SACzE,IAAI,KAAK,KAAK,MAAM;QAAE,OAAO,6CAAqB,CAAC,WAAW,CAAC;SAC/D,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,6CAAqB,CAAC,cAAc,CAAC;IAC1E,KAAqB,CAAC;IACtB,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,CAAU,CAAC;AACxE,MAAM,cAAc,GAAG;IACrB,SAAS;IACT,QAAQ;IACR,WAAW;IACX,MAAM;IACN,SAAS;CACD,CAAC"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.AutoBeExampleDocumentation = void 0;
|
|
7
|
+
const utils_1 = require("@autobe/utils");
|
|
8
|
+
const typia_1 = __importDefault(require("typia"));
|
|
9
|
+
var AutoBeExampleDocumentation;
|
|
10
|
+
(function (AutoBeExampleDocumentation) {
|
|
11
|
+
AutoBeExampleDocumentation.markdown = (state) => utils_1.StringUtil.trim `
|
|
12
|
+
# AutoBe Example Benchmark Report
|
|
13
|
+
|
|
14
|
+
${markdownIndex(state)}
|
|
15
|
+
|
|
16
|
+
${state.vendors.map(markdownVendor).join("\n\n")}
|
|
17
|
+
`;
|
|
18
|
+
const markdownIndex = (state) => utils_1.StringUtil.trim `
|
|
19
|
+
## Table of Contents
|
|
20
|
+
|
|
21
|
+
${state.vendors
|
|
22
|
+
.map((vendor) => `- [\`${vendor.name}\`](#${vendor.name
|
|
23
|
+
.replaceAll("/", "")
|
|
24
|
+
.replaceAll(":", "")})`)
|
|
25
|
+
.join("\n")}
|
|
26
|
+
`;
|
|
27
|
+
const markdownVendor = (state) => utils_1.StringUtil.trim `
|
|
28
|
+
## \`${state.name}\`
|
|
29
|
+
|
|
30
|
+
Project | Phase | State | Elapsed Time
|
|
31
|
+
:-------|:------|:------|-------------:
|
|
32
|
+
${state.projects.map(markdownProject).join("\n")}
|
|
33
|
+
`;
|
|
34
|
+
const markdownProject = (state) => {
|
|
35
|
+
// yellow circle emoji:
|
|
36
|
+
const phase = state.phases.at(-1);
|
|
37
|
+
return [
|
|
38
|
+
state.name,
|
|
39
|
+
!!(phase === null || phase === void 0 ? void 0 : phase.name) ? `${phase.name} (${phase.trial})` : "-",
|
|
40
|
+
state.completed_at !== null
|
|
41
|
+
? state.success
|
|
42
|
+
? "🟢 success"
|
|
43
|
+
: "🔴 failure"
|
|
44
|
+
: phase !== undefined && phase.snapshot !== null
|
|
45
|
+
? [
|
|
46
|
+
phase.trial !== 1 ? "🟠" : "🟡",
|
|
47
|
+
`\`${phase.snapshot.event.type}\``,
|
|
48
|
+
...(typia_1.default.is(phase.snapshot.event)
|
|
49
|
+
? [
|
|
50
|
+
`(${phase.snapshot.event.completed} of ${phase.snapshot.event.total})`,
|
|
51
|
+
]
|
|
52
|
+
: []),
|
|
53
|
+
].join(" ")
|
|
54
|
+
: "-",
|
|
55
|
+
state.started_at !== null
|
|
56
|
+
? elapsedTime({
|
|
57
|
+
started_at: state.started_at,
|
|
58
|
+
completed_at: state.completed_at,
|
|
59
|
+
})
|
|
60
|
+
: "-",
|
|
61
|
+
].join(" | ");
|
|
62
|
+
};
|
|
63
|
+
})(AutoBeExampleDocumentation || (exports.AutoBeExampleDocumentation = AutoBeExampleDocumentation = {}));
|
|
64
|
+
const elapsedTime = (props) => {
|
|
65
|
+
var _a;
|
|
66
|
+
return Math.round((((_a = props.completed_at) !== null && _a !== void 0 ? _a : new Date()).getTime() -
|
|
67
|
+
props.started_at.getTime()) /
|
|
68
|
+
1000).toLocaleString() + " sec";
|
|
69
|
+
};
|
|
70
|
+
//# sourceMappingURL=AutoBeExampleDocumentation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AutoBeExampleDocumentation.js","sourceRoot":"","sources":["../../src/example/AutoBeExampleDocumentation.ts"],"names":[],"mappings":";;;;;;AACA,yCAA2C;AAC3C,kDAA0B;AAI1B,IAAiB,0BAA0B,CAmE1C;AAnED,WAAiB,0BAA0B;IAC5B,mCAAQ,GAAG,CAAC,KAAmC,EAAU,EAAE,CACtE,kBAAU,CAAC,IAAI,CAAA;;;QAGX,aAAa,CAAC,KAAK,CAAC;;QAEpB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;KACjD,CAAC;IAEJ,MAAM,aAAa,GAAG,CACpB,KAAmC,EAC3B,EAAE,CAAC,kBAAU,CAAC,IAAI,CAAA;;;MAGxB,KAAK,CAAC,OAAO;SACZ,GAAG,CACF,CAAC,MAAM,EAAE,EAAE,CACT,QAAQ,MAAM,CAAC,IAAI,QAAQ,MAAM,CAAC,IAAI;SACnC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC;SACnB,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAC5B;SACA,IAAI,CAAC,IAAI,CAAC;GACd,CAAC;IAEF,MAAM,cAAc,GAAG,CACrB,KAA6C,EACrC,EAAE,CAAC,kBAAU,CAAC,IAAI,CAAA;WACnB,KAAK,CAAC,IAAI;;;;MAIf,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;GACjD,CAAC;IAEF,MAAM,eAAe,GAAG,CACtB,KAA8C,EACtC,EAAE;QACV,uBAAuB;QACvB,MAAM,KAAK,GACT,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO;YACL,KAAK,CAAC,IAAI;YACV,CAAC,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,CAAA,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG;YACtD,KAAK,CAAC,YAAY,KAAK,IAAI;gBACzB,CAAC,CAAC,KAAK,CAAC,OAAO;oBACb,CAAC,CAAC,YAAY;oBACd,CAAC,CAAC,YAAY;gBAChB,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;oBAC9C,CAAC,CAAC;wBACE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;wBAC/B,KAAK,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI;wBAClC,GAAG,CAAC,eAAK,CAAC,EAAE,CAA0B,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;4BACzD,CAAC,CAAC;gCACE,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG;6BACvE;4BACH,CAAC,CAAC,EAAE,CAAC;qBACR,CAAC,IAAI,CAAC,GAAG,CAAC;oBACb,CAAC,CAAC,GAAG;YACT,KAAK,CAAC,UAAU,KAAK,IAAI;gBACvB,CAAC,CAAC,WAAW,CAAC;oBACV,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;iBACjC,CAAC;gBACJ,CAAC,CAAC,GAAG;SACR,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC,EAnEgB,0BAA0B,0CAA1B,0BAA0B,QAmE1C;AAED,MAAM,WAAW,GAAG,CAAC,KAGpB,EAAU,EAAE;;IACX,OAAA,IAAI,CAAC,KAAK,CACR,CAAC,CAAC,MAAA,KAAK,CAAC,YAAY,mCAAI,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;QAC3C,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAC3B,IAAK,CACR,CAAC,cAAc,EAAE,GAAG,MAAM,CAAA;CAAA,CAAC"}
|
package/lib/example/index.d.ts
CHANGED
package/lib/example/index.js
CHANGED
|
@@ -17,4 +17,5 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
17
17
|
__exportStar(require("./AutoBeExampleArchiver"), exports);
|
|
18
18
|
__exportStar(require("./AutoBeExampleBenchmark"), exports);
|
|
19
19
|
__exportStar(require("./AutoBeExampleStorage"), exports);
|
|
20
|
+
__exportStar(require("./AutoBeExampleDocumentation"), exports);
|
|
20
21
|
//# sourceMappingURL=index.js.map
|
package/lib/example/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/example/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0DAAwC;AACxC,2DAAyC;AACzC,yDAAuC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/example/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0DAAwC;AACxC,2DAAyC;AACzC,yDAAuC;AACvC,+DAA6C"}
|
|
@@ -11,13 +11,14 @@ var AutoBeReplayDocumentation;
|
|
|
11
11
|
|
|
12
12
|
## Benchmark
|
|
13
13
|
|
|
14
|
-
AI Model | Score | FCSR | Status
|
|
15
|
-
|
|
14
|
+
AI Model | Success | Score | FCSR | Status
|
|
15
|
+
:--------|---------|------:|-----:|:------:
|
|
16
16
|
${experiments
|
|
17
17
|
.map((e) => [
|
|
18
18
|
`[\`${AutoBeExampleStorage_1.AutoBeExampleStorage.slugModel(e.vendor, false)}\`](#${AutoBeExampleStorage_1.AutoBeExampleStorage.slugModel(e.vendor, false)
|
|
19
19
|
.replaceAll("/", "")
|
|
20
20
|
.replaceAll(".", "")})`,
|
|
21
|
+
e.replays.filter((r) => { var _a; return ((_a = r.realize) === null || _a === void 0 ? void 0 : _a.success) === true; }).length,
|
|
21
22
|
e.score.aggregate,
|
|
22
23
|
(() => {
|
|
23
24
|
const [x, y] = e.replays
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AutoBeReplayDocumentation.js","sourceRoot":"","sources":["../../src/replay/AutoBeReplayDocumentation.ts"],"names":[],"mappings":";;;AAMA,yCAA2C;AAE3C,0EAAuE;AAEvE,IAAiB,yBAAyB,
|
|
1
|
+
{"version":3,"file":"AutoBeReplayDocumentation.js","sourceRoot":"","sources":["../../src/replay/AutoBeReplayDocumentation.ts"],"names":[],"mappings":";;;AAMA,yCAA2C;AAE3C,0EAAuE;AAEvE,IAAiB,yBAAyB,CAwIzC;AAxID,WAAiB,yBAAyB;IAC3B,gCAAM,GAAG,CAAC,WAAyC,EAAU,EAAE;QAC1E,OAAO,kBAAU,CAAC,IAAI,CAAA;;;;;;;QAOlB,WAAW;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACT;YACE,MAAM,2CAAoB,CAAC,SAAS,CAClC,CAAC,CAAC,MAAM,EACR,KAAK,CACN,QAAQ,2CAAoB,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC;iBACrD,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC;iBACnB,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG;YACzB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,WAAC,OAAA,CAAA,MAAA,CAAC,CAAC,OAAO,0CAAE,OAAO,MAAK,IAAI,CAAA,EAAA,CAAC,CAAC,MAAM;YAC3D,CAAC,CAAC,KAAK,CAAC,SAAS;YACjB,CAAC,GAAG,EAAE;gBACJ,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO;qBACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC;qBACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;qBAClC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACxD,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;YAC1D,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,KAAK;SACR,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;aACA,IAAI,CAAC,IAAI,CAAC;;;;;;;;QAQX,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;KACzC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,CAAC,GAA+B,EAAU,EAAE;QACzD,MAAM,GAAG,GAAG,CAAC,OAA6B,EAAU,EAAE;YACpD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;YAC7D,IAAI,KAAK,KAAK,SAAS;gBACrB,OAAO,KAAK,OAAO,4BAA4B,CAAC;YAClD,MAAM,KAAK,GAAG,CACZ,KAAiD,EACzC,EAAE;gBACV,IAAI,KAAK,KAAK,IAAI;oBAAE,OAAO,GAAG,CAAC;qBAC1B,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK;oBAAE,OAAO,IAAI,CAAC;;oBACzC,OAAO,IAAI,CAAC;YACnB,CAAC,CAAC;YACF,OAAO;gBACL,MAAM,KAAK,CAAC,OAAO,SAAS,GAAG,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI;gBAC1D,GAAG,CAAC,KAAa,CAAC,OAAO,CAAC;gBAC3B,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;gBACpB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;gBACnB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;gBACtB,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;gBACjB,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;aACrB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC;QACF,OAAO,kBAAU,CAAC,IAAI,CAAA;aACb,GAAG,CAAC,MAAM;;;;QAIf,GAAG,CAAC,MAAM,CAAC;QACX,GAAG,CAAC,KAAK,CAAC;QACV,GAAG,CAAC,QAAQ,CAAC;QACb,GAAG,CAAC,UAAU,CAAC;;QAEf,GAAG,CAAC,OAAO;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACT,OAAO,CAAC;YACN,MAAM,EAAE,CAAC;YACT,KAAK,EAAG,GAAG,CAAC,KAAa,CAAC,CAAC,CAAC,OAAO,CAAC;SACrC,CAAC,CACH;aACA,IAAI,CAAC,QAAQ,CAAC;KAClB,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,KAGhB,EAAU,EAAE;QACX,MAAM,KAAK,GAAG,CAAC,GAAgB,EAAU,EAAE;YACzC,MAAM,KAAK,GAAW,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjE,MAAM,KAAK,GACT,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACpB,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,CAAC,KAAK,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtE,OAAO;gBACL,GAAG,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,KAAK,EAAE;gBAClD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC;qBAC5B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,OAAO,KAAK,EAAE,CAAC;qBAC7C,IAAI,CAAC,IAAI,CAAC;gBACb,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrD,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC;gBAChC,IAAI,CAAC,KAAK,CACR,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO;oBACpC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;oBACtC,GAAG,CACN,GAAG,GAAG;aACR,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC;QACF,OAAO,kBAAU,CAAC,IAAI,CAAA;cACZ,KAAK,CAAC,MAAM,CAAC,MAAM,UAAU,KAAK,CAAC,MAAM,CAAC,OAAO;;uBAExC,MAAM,2CAAoB,CAAC,SAAS,CACnD,KAAK,CAAC,MAAM,CAAC,MAAM,EACnB,KAAK,CACN,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,SAAS,2CAAoB,CAAC,SAAS,CAC9D,KAAK,CAAC,MAAM,CAAC,MAAM,EACnB,KAAK,CACN,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI;iBAClB,KAAK,CAAC,KAAK;wBACJ,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;uBACxC,YAAY,CAC3B,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAC/C;yCACkC,CACjC,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO;YAC3C,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;YAC/C,GAAG,CACJ,CAAC,OAAO,CAAC,CAAC,CAAC;;;;QAIT,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,CAAW;aAC/D,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;aACxB,IAAI,CAAC,IAAI,CAAC;KACd,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC,EAxIgB,yBAAyB,yCAAzB,yBAAyB,QAwIzC;AAED,SAAS,iBAAiB,CAAC,EAAU;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IAEvC,MAAM,CAAC,GAAG,OAAO,GAAG,EAAE,CAAC;IACvB,MAAM,CAAC,GAAG,OAAO,GAAG,EAAE,CAAC;IACvB,MAAM,CAAC,GAAG,KAAK,CAAC;IAEhB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC;IAC7B,CAAC;SAAM,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACjB,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,CAAC,GAAG,CAAC;IACjB,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;QACnB,OAAO,GAAG,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAC1C,CAAC;SAAM,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACvC,CAAC;IACD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC"}
|
|
@@ -11,7 +11,7 @@ export declare namespace IAutoBeExampleBenchmarkState {
|
|
|
11
11
|
name: AutoBeExampleProject;
|
|
12
12
|
phases: IOfPhase[];
|
|
13
13
|
success: boolean | null;
|
|
14
|
-
started_at: Date;
|
|
14
|
+
started_at: Date | null;
|
|
15
15
|
completed_at: Date | null;
|
|
16
16
|
}
|
|
17
17
|
interface IOfPhase {
|
|
@@ -20,5 +20,6 @@ export declare namespace IAutoBeExampleBenchmarkState {
|
|
|
20
20
|
success: boolean | null;
|
|
21
21
|
started_at: Date;
|
|
22
22
|
completed_at: Date | null;
|
|
23
|
+
trial: number;
|
|
23
24
|
}
|
|
24
25
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@autobe/benchmark",
|
|
3
|
-
"version": "0.29.
|
|
3
|
+
"version": "0.29.2",
|
|
4
4
|
"description": "AI backend server code generator",
|
|
5
5
|
"main": "lib/index.js",
|
|
6
6
|
"keywords": [],
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"tstl": "^3.0.0",
|
|
28
28
|
"typia": "^10.0.2",
|
|
29
29
|
"uuid": "^11.1.0",
|
|
30
|
-
"@autobe/filesystem": "^0.29.
|
|
31
|
-
"@autobe/utils": "^0.29.
|
|
32
|
-
"@autobe/interface": "^0.29.
|
|
30
|
+
"@autobe/filesystem": "^0.29.2",
|
|
31
|
+
"@autobe/utils": "^0.29.2",
|
|
32
|
+
"@autobe/interface": "^0.29.2"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
35
|
"@types/uuid": "^10.0.0",
|
|
@@ -31,6 +31,7 @@ export namespace AutoBeExampleArchiver {
|
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
export interface IAgentProps {
|
|
34
|
+
vendor: string;
|
|
34
35
|
histories: AutoBeHistory[];
|
|
35
36
|
tokenUsage: IAutoBeTokenUsageJson;
|
|
36
37
|
}
|
|
@@ -296,6 +297,7 @@ export namespace AutoBeExampleArchiver {
|
|
|
296
297
|
PHASES[PHASES.indexOf(props.phase) - 1] ?? null;
|
|
297
298
|
if (previous === null)
|
|
298
299
|
return {
|
|
300
|
+
vendor: props.vendor,
|
|
299
301
|
histories: [],
|
|
300
302
|
tokenUsage: {
|
|
301
303
|
aggregate: TokenUsageComputer.zero(),
|
|
@@ -319,6 +321,7 @@ export namespace AutoBeExampleArchiver {
|
|
|
319
321
|
phase: previous,
|
|
320
322
|
});
|
|
321
323
|
return {
|
|
324
|
+
vendor: props.vendor,
|
|
322
325
|
histories,
|
|
323
326
|
tokenUsage,
|
|
324
327
|
};
|
|
@@ -36,7 +36,7 @@ export namespace AutoBeExampleBenchmark {
|
|
|
36
36
|
name: project,
|
|
37
37
|
phases: [],
|
|
38
38
|
success: null,
|
|
39
|
-
started_at:
|
|
39
|
+
started_at: null,
|
|
40
40
|
completed_at: null,
|
|
41
41
|
}),
|
|
42
42
|
),
|
|
@@ -85,6 +85,7 @@ export namespace AutoBeExampleBenchmark {
|
|
|
85
85
|
on?: (event: AutoBeEvent) => void;
|
|
86
86
|
},
|
|
87
87
|
): Promise<void> => {
|
|
88
|
+
props.projectState.started_at = new Date();
|
|
88
89
|
for (const phase of PHASE_SEQUENCE) {
|
|
89
90
|
if (props.phases && props.phases.includes(phase) === false) continue;
|
|
90
91
|
const phaseState: IAutoBeExampleBenchmarkState.IOfPhase = {
|
|
@@ -93,10 +94,12 @@ export namespace AutoBeExampleBenchmark {
|
|
|
93
94
|
success: null,
|
|
94
95
|
started_at: new Date(),
|
|
95
96
|
completed_at: null,
|
|
97
|
+
trial: 0,
|
|
96
98
|
};
|
|
97
99
|
props.projectState.phases.push(phaseState);
|
|
98
100
|
for (let i: number = 0; i < 3; ++i) {
|
|
99
101
|
try {
|
|
102
|
+
++phaseState.trial;
|
|
100
103
|
phaseState.started_at = new Date();
|
|
101
104
|
phaseState.completed_at = null;
|
|
102
105
|
const success: boolean = await getArchiver(phase)({
|
|
@@ -104,7 +107,14 @@ export namespace AutoBeExampleBenchmark {
|
|
|
104
107
|
project: props.projectState.name,
|
|
105
108
|
agent: (next) => ctx.createAgent(next),
|
|
106
109
|
on: (s) => {
|
|
107
|
-
|
|
110
|
+
const event = s.event;
|
|
111
|
+
if (
|
|
112
|
+
event.type !== "jsonValidateError" &&
|
|
113
|
+
event.type !== "jsonParseError" &&
|
|
114
|
+
event.type !== "preliminary" &&
|
|
115
|
+
event.type !== "consentFunctionCall"
|
|
116
|
+
)
|
|
117
|
+
phaseState.snapshot = s;
|
|
108
118
|
props.report();
|
|
109
119
|
if (props.on) props.on(s.event);
|
|
110
120
|
},
|
|
@@ -140,4 +150,10 @@ const getArchiver = (phase: AutoBePhase) => {
|
|
|
140
150
|
};
|
|
141
151
|
|
|
142
152
|
const PROJECT_SEQUENCE = ["todo", "bbs", "reddit", "shopping"] as const;
|
|
143
|
-
const PHASE_SEQUENCE = [
|
|
153
|
+
const PHASE_SEQUENCE = [
|
|
154
|
+
"analyze",
|
|
155
|
+
"prisma",
|
|
156
|
+
"interface",
|
|
157
|
+
"test",
|
|
158
|
+
"realize",
|
|
159
|
+
] as const;
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { AutoBeProgressEventBase } from "@autobe/interface";
|
|
2
|
+
import { StringUtil } from "@autobe/utils";
|
|
3
|
+
import typia from "typia";
|
|
4
|
+
|
|
5
|
+
import { IAutoBeExampleBenchmarkState } from "../structures";
|
|
6
|
+
|
|
7
|
+
export namespace AutoBeExampleDocumentation {
|
|
8
|
+
export const markdown = (state: IAutoBeExampleBenchmarkState): string =>
|
|
9
|
+
StringUtil.trim`
|
|
10
|
+
# AutoBe Example Benchmark Report
|
|
11
|
+
|
|
12
|
+
${markdownIndex(state)}
|
|
13
|
+
|
|
14
|
+
${state.vendors.map(markdownVendor).join("\n\n")}
|
|
15
|
+
`;
|
|
16
|
+
|
|
17
|
+
const markdownIndex = (
|
|
18
|
+
state: IAutoBeExampleBenchmarkState,
|
|
19
|
+
): string => StringUtil.trim`
|
|
20
|
+
## Table of Contents
|
|
21
|
+
|
|
22
|
+
${state.vendors
|
|
23
|
+
.map(
|
|
24
|
+
(vendor) =>
|
|
25
|
+
`- [\`${vendor.name}\`](#${vendor.name
|
|
26
|
+
.replaceAll("/", "")
|
|
27
|
+
.replaceAll(":", "")})`,
|
|
28
|
+
)
|
|
29
|
+
.join("\n")}
|
|
30
|
+
`;
|
|
31
|
+
|
|
32
|
+
const markdownVendor = (
|
|
33
|
+
state: IAutoBeExampleBenchmarkState.IOfVendor,
|
|
34
|
+
): string => StringUtil.trim`
|
|
35
|
+
## \`${state.name}\`
|
|
36
|
+
|
|
37
|
+
Project | Phase | State | Elapsed Time
|
|
38
|
+
:-------|:------|:------|-------------:
|
|
39
|
+
${state.projects.map(markdownProject).join("\n")}
|
|
40
|
+
`;
|
|
41
|
+
|
|
42
|
+
const markdownProject = (
|
|
43
|
+
state: IAutoBeExampleBenchmarkState.IOfProject,
|
|
44
|
+
): string => {
|
|
45
|
+
// yellow circle emoji:
|
|
46
|
+
const phase: IAutoBeExampleBenchmarkState.IOfPhase | undefined =
|
|
47
|
+
state.phases.at(-1);
|
|
48
|
+
return [
|
|
49
|
+
state.name,
|
|
50
|
+
!!phase?.name ? `${phase.name} (${phase.trial})` : "-",
|
|
51
|
+
state.completed_at !== null
|
|
52
|
+
? state.success
|
|
53
|
+
? "🟢 success"
|
|
54
|
+
: "🔴 failure"
|
|
55
|
+
: phase !== undefined && phase.snapshot !== null
|
|
56
|
+
? [
|
|
57
|
+
phase.trial !== 1 ? "🟠" : "🟡",
|
|
58
|
+
`\`${phase.snapshot.event.type}\``,
|
|
59
|
+
...(typia.is<AutoBeProgressEventBase>(phase.snapshot.event)
|
|
60
|
+
? [
|
|
61
|
+
`(${phase.snapshot.event.completed} of ${phase.snapshot.event.total})`,
|
|
62
|
+
]
|
|
63
|
+
: []),
|
|
64
|
+
].join(" ")
|
|
65
|
+
: "-",
|
|
66
|
+
state.started_at !== null
|
|
67
|
+
? elapsedTime({
|
|
68
|
+
started_at: state.started_at,
|
|
69
|
+
completed_at: state.completed_at,
|
|
70
|
+
})
|
|
71
|
+
: "-",
|
|
72
|
+
].join(" | ");
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const elapsedTime = (props: {
|
|
77
|
+
started_at: Date;
|
|
78
|
+
completed_at: Date | null;
|
|
79
|
+
}): string =>
|
|
80
|
+
Math.round(
|
|
81
|
+
((props.completed_at ?? new Date()).getTime() -
|
|
82
|
+
props.started_at.getTime()) /
|
|
83
|
+
1_000,
|
|
84
|
+
).toLocaleString() + " sec";
|
package/src/example/index.ts
CHANGED
|
@@ -15,8 +15,8 @@ export namespace AutoBeReplayDocumentation {
|
|
|
15
15
|
|
|
16
16
|
## Benchmark
|
|
17
17
|
|
|
18
|
-
AI Model | Score | FCSR | Status
|
|
19
|
-
|
|
18
|
+
AI Model | Success | Score | FCSR | Status
|
|
19
|
+
:--------|---------|------:|-----:|:------:
|
|
20
20
|
${experiments
|
|
21
21
|
.map((e) =>
|
|
22
22
|
[
|
|
@@ -26,6 +26,7 @@ export namespace AutoBeReplayDocumentation {
|
|
|
26
26
|
)}\`](#${AutoBeExampleStorage.slugModel(e.vendor, false)
|
|
27
27
|
.replaceAll("/", "")
|
|
28
28
|
.replaceAll(".", "")})`,
|
|
29
|
+
e.replays.filter((r) => r.realize?.success === true).length,
|
|
29
30
|
e.score.aggregate,
|
|
30
31
|
(() => {
|
|
31
32
|
const [x, y] = e.replays
|
|
@@ -16,7 +16,7 @@ export namespace IAutoBeExampleBenchmarkState {
|
|
|
16
16
|
name: AutoBeExampleProject;
|
|
17
17
|
phases: IOfPhase[];
|
|
18
18
|
success: boolean | null;
|
|
19
|
-
started_at: Date;
|
|
19
|
+
started_at: Date | null;
|
|
20
20
|
completed_at: Date | null;
|
|
21
21
|
}
|
|
22
22
|
export interface IOfPhase {
|
|
@@ -25,5 +25,6 @@ export namespace IAutoBeExampleBenchmarkState {
|
|
|
25
25
|
success: boolean | null;
|
|
26
26
|
started_at: Date;
|
|
27
27
|
completed_at: Date | null;
|
|
28
|
+
trial: number;
|
|
28
29
|
}
|
|
29
30
|
}
|