@agentica/benchmark 0.12.21 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -33
- package/lib/AgenticaCallBenchmark.d.ts +12 -6
- package/lib/AgenticaCallBenchmark.js +24 -18
- package/lib/AgenticaCallBenchmark.js.map +1 -1
- package/lib/AgenticaSelectBenchmark.d.ts +12 -6
- package/lib/AgenticaSelectBenchmark.js +14 -12
- package/lib/AgenticaSelectBenchmark.js.map +1 -1
- package/lib/index.mjs +315 -236
- package/lib/index.mjs.map +1 -1
- package/lib/internal/AgenticaBenchmarkPredicator.d.ts +38 -29
- package/lib/internal/AgenticaBenchmarkPredicator.js +100 -84
- package/lib/internal/AgenticaBenchmarkPredicator.js.map +1 -1
- package/lib/internal/AgenticaBenchmarkUtil.d.ts +21 -6
- package/lib/internal/AgenticaBenchmarkUtil.js +39 -33
- package/lib/internal/AgenticaBenchmarkUtil.js.map +1 -1
- package/lib/internal/AgenticaCallBenchmarkReporter.d.ts +6 -5
- package/lib/internal/AgenticaCallBenchmarkReporter.js +130 -126
- package/lib/internal/AgenticaCallBenchmarkReporter.js.map +1 -1
- package/lib/internal/AgenticaPromptReporter.d.ts +13 -5
- package/lib/internal/AgenticaPromptReporter.js +45 -41
- package/lib/internal/AgenticaPromptReporter.js.map +1 -1
- package/lib/internal/AgenticaSelectBenchmarkReporter.d.ts +3 -1
- package/lib/internal/AgenticaSelectBenchmarkReporter.js +153 -150
- package/lib/internal/AgenticaSelectBenchmarkReporter.js.map +1 -1
- package/lib/structures/IAgenticaBenchmarkExpected.d.ts +8 -2
- package/lib/structures/IAgenticaCallBenchmarkEvent.d.ts +9 -3
- package/lib/structures/IAgenticaCallBenchmarkResult.d.ts +10 -4
- package/lib/structures/IAgenticaCallBenchmarkScenario.d.ts +8 -2
- package/lib/structures/IAgenticaSelectBenchmarkEvent.d.ts +9 -3
- package/lib/structures/IAgenticaSelectBenchmarkResult.d.ts +10 -4
- package/lib/structures/IAgenticaSelectBenchmarkScenario.d.ts +8 -2
- package/lib/utils/MathUtil.d.ts +15 -3
- package/lib/utils/MathUtil.js +15 -4
- package/lib/utils/MathUtil.js.map +1 -1
- package/package.json +12 -10
- package/src/AgenticaCallBenchmark.ts +64 -45
- package/src/AgenticaSelectBenchmark.ts +42 -30
- package/src/internal/AgenticaBenchmarkPredicator.ts +208 -186
- package/src/internal/AgenticaBenchmarkUtil.ts +58 -40
- package/src/internal/AgenticaCallBenchmarkReporter.ts +180 -182
- package/src/internal/AgenticaPromptReporter.ts +46 -33
- package/src/internal/AgenticaSelectBenchmarkReporter.ts +205 -203
- package/src/structures/IAgenticaBenchmarkExpected.ts +9 -2
- package/src/structures/IAgenticaCallBenchmarkEvent.ts +9 -3
- package/src/structures/IAgenticaCallBenchmarkResult.ts +10 -4
- package/src/structures/IAgenticaCallBenchmarkScenario.ts +8 -2
- package/src/structures/IAgenticaSelectBenchmarkEvent.ts +9 -3
- package/src/structures/IAgenticaSelectBenchmarkResult.ts +10 -4
- package/src/structures/IAgenticaSelectBenchmarkScenario.ts +8 -2
- package/src/utils/MathUtil.ts +16 -3
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AgenticaCallBenchmarkReporter = void 0;
|
|
4
|
+
exports.markdown = markdown;
|
|
4
5
|
const MathUtil_1 = require("../utils/MathUtil");
|
|
5
6
|
const AgenticaBenchmarkUtil_1 = require("./AgenticaBenchmarkUtil");
|
|
6
7
|
const AgenticaPromptReporter_1 = require("./AgenticaPromptReporter");
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
exports.AgenticaCallBenchmarkReporter = {
|
|
9
|
+
markdown,
|
|
10
|
+
};
|
|
11
|
+
function markdown(result) {
|
|
12
|
+
return Object.fromEntries([
|
|
10
13
|
["./README.md", writeIndex(result)],
|
|
11
14
|
...result.experiments
|
|
12
|
-
.map(
|
|
15
|
+
.map(exp => [
|
|
13
16
|
[`./${exp.scenario.name}/README.md`, writeExperimentIndex(exp)],
|
|
14
17
|
...exp.events.map((event, i) => [
|
|
15
18
|
`./${exp.scenario.name}/${i + 1}.${event.type}.md`,
|
|
@@ -18,127 +21,128 @@ var AgenticaCallBenchmarkReporter;
|
|
|
18
21
|
])
|
|
19
22
|
.flat(),
|
|
20
23
|
]);
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
].join(" | ")),
|
|
60
|
-
].join("\n");
|
|
61
|
-
};
|
|
62
|
-
const writeExperimentIndex = (exp) => {
|
|
63
|
-
return [
|
|
64
|
-
`# ${exp.scenario.name}`,
|
|
65
|
-
"## Summary",
|
|
66
|
-
` - Scenarios: #${exp.events.length.toLocaleString()}`,
|
|
67
|
-
` - Success: ${exp.events.filter((e) => e.type === "success").length}`,
|
|
68
|
-
` - Failure: ${exp.events.filter((e) => e.type === "failure").length}`,
|
|
69
|
-
` - Average Time: ${MathUtil_1.MathUtil.round(exp.events
|
|
70
|
-
.map((e) => e.completed_at.getTime() - e.started_at.getTime())
|
|
24
|
+
}
|
|
25
|
+
function writeIndex(result) {
|
|
26
|
+
const events = result.experiments
|
|
27
|
+
.map(r => r.events)
|
|
28
|
+
.flat();
|
|
29
|
+
const average = events
|
|
30
|
+
.map(e => e.completed_at.getTime() - e.started_at.getTime())
|
|
31
|
+
.reduce((a, b) => a + b, 0) / events.length;
|
|
32
|
+
const aggregate = result.usage.aggregate;
|
|
33
|
+
return [
|
|
34
|
+
"# LLM Function Call Benchmark",
|
|
35
|
+
"## Summary",
|
|
36
|
+
` - Aggregation:`,
|
|
37
|
+
` - Scenarios: #${result.experiments.length.toLocaleString()}`,
|
|
38
|
+
` - Trial: ${events.length}`,
|
|
39
|
+
` - Success: ${events.filter(e => e.type === "success").length}`,
|
|
40
|
+
` - Failure: ${events.filter(e => e.type === "failure").length}`,
|
|
41
|
+
` - Average Time: ${MathUtil_1.MathUtil.round(average).toLocaleString()} ms`,
|
|
42
|
+
` - Token Usage`,
|
|
43
|
+
` - Total: ${aggregate.total.toLocaleString()}`,
|
|
44
|
+
` - Input`,
|
|
45
|
+
` - Total: ${aggregate.input.total.toLocaleString()}`,
|
|
46
|
+
` - Cached: ${aggregate.input.cached.toLocaleString()}`,
|
|
47
|
+
` - Output:`,
|
|
48
|
+
` - Total: ${aggregate.output.total.toLocaleString()}`,
|
|
49
|
+
` - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`,
|
|
50
|
+
` - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`,
|
|
51
|
+
` - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`,
|
|
52
|
+
"",
|
|
53
|
+
"## Experiments",
|
|
54
|
+
" Name | Select | Call | Time/Avg ",
|
|
55
|
+
":-----|:-------|:-----|----------:",
|
|
56
|
+
...result.experiments.map(exp => [
|
|
57
|
+
`[${exp.scenario.name}](./${exp.scenario.name}/README.md)`,
|
|
58
|
+
drawStatus(exp.events, e => e.type !== "error" && e.select === true),
|
|
59
|
+
drawStatus(exp.events, e => e.type !== "error" && e.call === true),
|
|
60
|
+
`${MathUtil_1.MathUtil.round(exp.events
|
|
61
|
+
.map(e => e.completed_at.getTime() - e.started_at.getTime())
|
|
71
62
|
.reduce((a, b) => a + b, 0) / exp.events.length).toLocaleString()} ms`,
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
exp.
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
63
|
+
].join(" | ")),
|
|
64
|
+
].join("\n");
|
|
65
|
+
}
|
|
66
|
+
function writeExperimentIndex(exp) {
|
|
67
|
+
return [
|
|
68
|
+
`# ${exp.scenario.name}`,
|
|
69
|
+
"## Summary",
|
|
70
|
+
` - Scenarios: #${exp.events.length.toLocaleString()}`,
|
|
71
|
+
` - Success: ${exp.events.filter(e => e.type === "success").length}`,
|
|
72
|
+
` - Failure: ${exp.events.filter(e => e.type === "failure").length}`,
|
|
73
|
+
` - Average Time: ${MathUtil_1.MathUtil.round(exp.events
|
|
74
|
+
.map(e => e.completed_at.getTime() - e.started_at.getTime())
|
|
75
|
+
.reduce((a, b) => a + b, 0) / exp.events.length).toLocaleString()} ms`,
|
|
76
|
+
"",
|
|
77
|
+
"## Events",
|
|
78
|
+
" Name | Type | Time",
|
|
79
|
+
":-----|:-----|----:",
|
|
80
|
+
...exp.events.map((e, i) => [
|
|
81
|
+
`[${i + 1}.](./${i + 1}.${e.type}.md)`,
|
|
82
|
+
e.type,
|
|
83
|
+
`${MathUtil_1.MathUtil.round(e.completed_at.getTime() - e.started_at.getTime())} ms`,
|
|
84
|
+
].join(" | ")),
|
|
85
|
+
"",
|
|
86
|
+
"## Scenario",
|
|
87
|
+
"### User Prompt",
|
|
88
|
+
exp.scenario.text,
|
|
89
|
+
"",
|
|
90
|
+
"### Expected",
|
|
91
|
+
"```json",
|
|
92
|
+
JSON.stringify(AgenticaBenchmarkUtil_1.AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected), null, 2),
|
|
93
|
+
"```",
|
|
94
|
+
].join("\n");
|
|
95
|
+
}
|
|
96
|
+
function writeExperimentEvent(event, index) {
|
|
97
|
+
return [
|
|
98
|
+
`# ${index + 1}. ${event.type}`,
|
|
99
|
+
"## Summary",
|
|
100
|
+
` - Name: ${event.scenario.name}`,
|
|
101
|
+
` - Type: ${event.type}`,
|
|
102
|
+
` - Time: ${MathUtil_1.MathUtil.round(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`,
|
|
103
|
+
...(event.type !== "error"
|
|
104
|
+
? [
|
|
105
|
+
` - Select: ${event.select ? "✅" : "❌"}`,
|
|
106
|
+
` - Call: ${event.call ? "✅" : "❌"}`,
|
|
107
|
+
]
|
|
108
|
+
: []),
|
|
109
|
+
` - Token Usage:`,
|
|
110
|
+
` - Total: ${JSON.stringify(event.usage.aggregate.total)}`,
|
|
111
|
+
` - Input`,
|
|
112
|
+
` - Total: ${event.usage.aggregate.input.total}`,
|
|
113
|
+
` - Cached: ${event.usage.aggregate.input.cached}`,
|
|
114
|
+
` - Output:`,
|
|
115
|
+
` - Total: ${event.usage.aggregate.output.total}`,
|
|
116
|
+
` - Accepted Prediction: ${event.usage.aggregate.output.accepted_prediction}`,
|
|
117
|
+
` - Reasoning: ${event.usage.aggregate.output.reasoning}`,
|
|
118
|
+
` - Rejected Prediction: ${event.usage.aggregate.output.rejected_prediction}`,
|
|
119
|
+
"",
|
|
120
|
+
"## Scenario",
|
|
121
|
+
"### User Prompt",
|
|
122
|
+
event.scenario.text,
|
|
123
|
+
"",
|
|
124
|
+
"### Expected",
|
|
125
|
+
"```json",
|
|
126
|
+
JSON.stringify(AgenticaBenchmarkUtil_1.AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected), null, 2),
|
|
127
|
+
"```",
|
|
128
|
+
"",
|
|
129
|
+
"## Prompt Histories",
|
|
130
|
+
...event.prompts.map(AgenticaPromptReporter_1.AgenticaPromptReporter.markdown),
|
|
131
|
+
"",
|
|
132
|
+
...(event.type === "error"
|
|
133
|
+
? [
|
|
134
|
+
"## Error",
|
|
135
|
+
"```json",
|
|
136
|
+
JSON.stringify(AgenticaBenchmarkUtil_1.AgenticaBenchmarkUtil.errorToJson(event.error), null, 2),
|
|
137
|
+
"```",
|
|
138
|
+
]
|
|
139
|
+
: []),
|
|
140
|
+
].join("\n");
|
|
141
|
+
}
|
|
142
|
+
function drawStatus(events, success) {
|
|
143
|
+
const count = Math.floor((events.filter(success).length / events.length) * 10);
|
|
144
|
+
// @TODO use String.prototype.padStart, padEnd or String.prototype.repeat
|
|
145
|
+
return (Array.from({ length: count }).fill("■").join("")
|
|
146
|
+
+ Array.from({ length: 10 - count }).fill("□").join(""));
|
|
147
|
+
}
|
|
144
148
|
//# sourceMappingURL=AgenticaCallBenchmarkReporter.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgenticaCallBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaCallBenchmarkReporter.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"AgenticaCallBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaCallBenchmarkReporter.ts"],"names":[],"mappings":";;;AAmBA,4BAaC;AArBD,gDAA6C;AAC7C,mEAAgE;AAChE,qEAAkE;AAErD,QAAA,6BAA6B,GAAG;IAC3C,QAAQ;CACT,CAAC;AAEF,SAAgB,QAAQ,CAAiC,MAA2C;IAClG,OAAO,MAAM,CAAC,WAAW,CAAC;QACxB,CAAC,aAAa,EAAE,UAAU,CAAQ,MAAM,CAAC,CAAC;QAC1C,GAAG,MAAM,CAAC,WAAW;aAClB,GAAG,CAAqB,GAAG,CAAC,EAAE,CAAC;YAC9B,CAAC,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,YAAY,EAAE,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC/D,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAmB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK;gBAClD,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC;aAC/B,CAAC;SACH,CAAC;aACD,IAAI,EAAE;KACV,CAAC,CAAC;AACL,CAAC;AAED,SAAS,UAAU,CAAiC,MAA2C;IAC7F,MAAM,MAAM,GAAyC,MAAM,CAAC,WAAW;SACpE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;SAClB,IAAI,EAAE,CAAC;IACV,MAAM,OAAO,GACT,MAAM;SACL,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;SAC3D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAChD,MAAM,SAAS,GAAkC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;IACxE,OAAO;QACL,+BAA+B;QAC/B,YAAY;QACZ,kBAAkB;QAClB,qBAAqB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;QACjE,gBAAgB,MAAM,CAAC,MAAM,EAAE;QAC/B,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACnE,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACnE,uBAAuB,mBAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE,KAAK;QACpE,iBAAiB;QACjB,gBAAgB,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAClD,aAAa;QACb,kBAAkB,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC1D,mBAAmB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;QAC5D,eAAe;QACf,kBAAkB,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC3D,sBAAsB,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;QACnE,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,EAAE;QACF,gBAAgB;QAChB,mCAAmC;QACnC,oCAAoC;QACpC,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAC9B;YACE,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC,IAAI,aAAa;YAC1D,UAAU,CACR,GAAG,CAAC,MAAM,EACV,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAC7C;YACD,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC;YAClE,GAAG,mBAAQ,CAAC,KAAK,CACf,GAAG,CAAC,MAAM;iBACP,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;iBAC3D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;SACxB,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;KACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB,CAAiC,GAAoD;IAChH,OAAO;QACL,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE;QACxB,YAAY;QACZ,mBAAmB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;QACvD,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACrE,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACrE,qBAAqB,mBAAQ,CAAC,KAAK,CACjC,GAAG,CAAC,MAAM;aACP,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aAC3D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;QACvB,EAAE;QACF,WAAW;QACX,qBAAqB;QACrB,qBAAqB;QACrB,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB;YACE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM;YACtC,CAAC,CAAC,IAAI;YACN,GAAG,mBAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK;SAC1E,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;QACD,EAAE;QACF,aAAa;QACb,iBAAiB;QACjB,GAAG,CAAC,QAAQ,CAAC,IAAI;QACjB,EAAE;QACF,cAAc;QACd,SAAS;QACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3D,IAAI,EACJ,CAAC,CACF;QACD,KAAK;KACN,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB,CAAiC,KAAyC,EAAE,KAAa;IACpH,OAAO;QACL,KAAK,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,EAAE;QAC/B,YAAY;QACZ,aAAa,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE;QAClC,aAAa,KAAK,CAAC,IAAI,EAAE;QACzB,aAAa,mBAAQ,CAAC,KAAK,CACzB,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC1D,CAAC,cAAc,EAAE,KAAK;QACvB,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;YACxB,CAAC,CAAC;gBACE,eAAe,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE;gBACzC,aAAa,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE;aACtC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,kBAAkB;QAClB,gBAAgB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE;QAC7D,aAAa;QACb,kBAAkB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE;QACrD,mBAAmB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE;QACvD,eAAe;QACf,kBAAkB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE;QACtD,gCAAgC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,mBAAmB,EAAE;QAClF,sBAAsB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE;QAC9D,gCAAgC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,mBAAmB,EAAE;QAElF,EAAE;QACF,aAAa;QACb,iBAAiB;QACjB,KAAK,CAAC,QAAQ,CAAC,IAAI;QACnB,EAAE;QACF,cAAc;QACd,SAAS;QACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC7D,IAAI,EACJ,CAAC,CACF;QACD,KAAK;QACL,EAAE;QACF,qBAAqB;QACrB,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,+CAAsB,CAAC,QAAQ,CAAC;QACrD,EAAE;QACF,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;YACxB,CAAC,CAAC;gBACE,UAAU;gBACV,SAAS;gBACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,EAC9C,IAAI,EACJ,CAAC,CACF;gBACD,KAAK;aACN;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAiC,MAA4C,EAAE,OAA2D;IAC3J,MAAM,KAAK,GAAW,IAAI,CAAC,KAAK,CAC9B,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CACrD,CAAC;IACF,yEAAyE;IACzE,OAAO,CACL,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;UAC9C,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CACxD,CAAC;AACJ,CAAC"}
|
|
@@ -1,5 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @module
|
|
3
|
+
* This file contains functions to work with AgenticaPromptReporter.
|
|
4
|
+
*
|
|
5
|
+
* @author Wrtn Technologies
|
|
6
|
+
*/
|
|
7
|
+
import type { AgenticaPrompt } from "@agentica/core";
|
|
8
|
+
import type { ILlmSchema } from "@samchon/openapi";
|
|
9
|
+
export declare const AgenticaPromptReporter: {
|
|
10
|
+
markdown: typeof markdown;
|
|
11
|
+
};
|
|
12
|
+
declare function markdown<Model extends ILlmSchema.Model>(p: AgenticaPrompt<Model>): string;
|
|
13
|
+
export {};
|
|
@@ -1,49 +1,53 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AgenticaPromptReporter = void 0;
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
.map((s) => {
|
|
14
|
-
var _a;
|
|
15
|
-
return [
|
|
16
|
-
`#### ${s.operation.name}`,
|
|
17
|
-
` - controller: ${s.operation.controller.name}`,
|
|
18
|
-
` - function: ${s.operation.function.name}`,
|
|
19
|
-
` - reason: ${s.reason}`,
|
|
20
|
-
"",
|
|
21
|
-
...(!!((_a = s.operation.function.description) === null || _a === void 0 ? void 0 : _a.length)
|
|
22
|
-
? [s.operation.function.description, ""]
|
|
23
|
-
: []),
|
|
24
|
-
];
|
|
25
|
-
})
|
|
26
|
-
.flat(),
|
|
27
|
-
].join("\n");
|
|
28
|
-
else if (p.type === "describe")
|
|
29
|
-
return [
|
|
30
|
-
"### Describe",
|
|
31
|
-
...p.executes.map((e) => ` - ${e.operation.name}`),
|
|
32
|
-
"",
|
|
33
|
-
...p.text.split("\n").map((s) => `> ${s}`),
|
|
34
|
-
"",
|
|
35
|
-
].join("\n");
|
|
4
|
+
exports.AgenticaPromptReporter = {
|
|
5
|
+
markdown,
|
|
6
|
+
};
|
|
7
|
+
function markdown(p) {
|
|
8
|
+
// @TODO use switch statement
|
|
9
|
+
if (p.type === "text") {
|
|
10
|
+
return [`### Text (${p.role})`, p.text, ""].join("\n");
|
|
11
|
+
}
|
|
12
|
+
else if (p.type === "select" || p.type === "cancel") {
|
|
36
13
|
return [
|
|
37
|
-
"
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
14
|
+
`### ${p.type === "select" ? "Select" : "Cancel"}`,
|
|
15
|
+
...p.selections
|
|
16
|
+
.flatMap((s) => {
|
|
17
|
+
var _a, _b;
|
|
18
|
+
const functionDescriptionCount = (_b = (_a = s.operation.function.description) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0;
|
|
19
|
+
return [
|
|
20
|
+
`#### ${s.operation.name}`,
|
|
21
|
+
` - controller: ${s.operation.controller.name}`,
|
|
22
|
+
` - function: ${s.operation.function.name}`,
|
|
23
|
+
` - reason: ${s.reason}`,
|
|
24
|
+
"",
|
|
25
|
+
...(functionDescriptionCount > 0
|
|
26
|
+
? [s.operation.function.description, ""]
|
|
27
|
+
: []),
|
|
28
|
+
];
|
|
29
|
+
}),
|
|
30
|
+
].join("\n");
|
|
31
|
+
}
|
|
32
|
+
else if (p.type === "describe") {
|
|
33
|
+
return [
|
|
34
|
+
"### Describe",
|
|
35
|
+
...p.executes.map(e => ` - ${e.operation.name}`),
|
|
41
36
|
"",
|
|
42
|
-
"
|
|
43
|
-
JSON.stringify(p.arguments, null, 2),
|
|
44
|
-
"```",
|
|
37
|
+
...p.text.split("\n").map(s => `> ${s}`),
|
|
45
38
|
"",
|
|
46
39
|
].join("\n");
|
|
47
|
-
}
|
|
48
|
-
|
|
40
|
+
}
|
|
41
|
+
return [
|
|
42
|
+
"### Execute",
|
|
43
|
+
` - name: ${p.operation.name}`,
|
|
44
|
+
` - controller: ${p.operation.controller.name}`,
|
|
45
|
+
` - function: ${p.operation.function.name}`,
|
|
46
|
+
"",
|
|
47
|
+
"```json",
|
|
48
|
+
JSON.stringify(p.arguments, null, 2),
|
|
49
|
+
"```",
|
|
50
|
+
"",
|
|
51
|
+
].join("\n");
|
|
52
|
+
}
|
|
49
53
|
//# sourceMappingURL=AgenticaPromptReporter.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgenticaPromptReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaPromptReporter.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"AgenticaPromptReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaPromptReporter.ts"],"names":[],"mappings":";;;AASa,QAAA,sBAAsB,GAAG;IACpC,QAAQ;CACT,CAAC;AAEF,SAAS,QAAQ,CAAiC,CAAwB;IACxE,6BAA6B;IAC7B,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,aAAa,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzD,CAAC;SACI,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACpD,OAAO;YACL,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE;YAClD,GAAG,CAAC,CAAC,UAAU;iBACZ,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;;gBACb,MAAM,wBAAwB,GAAG,MAAA,MAAA,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,0CAAE,MAAM,mCAAI,CAAC,CAAC;gBAE/E,OAAO;oBACL,QAAQ,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE;oBAC1B,mBAAmB,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE;oBAChD,iBAAiB,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE;oBAC5C,eAAe,CAAC,CAAC,MAAM,EAAE;oBACzB,EAAE;oBACF,GAAG,CAAC,wBAAwB,GAAG,CAAC;wBAC9B,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;wBACxC,CAAC,CAAC,EAAE,CAAC;iBACR,CAAC;YACJ,CAAC,CAAC;SACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;SACI,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC/B,OAAO;YACL,cAAc;YACd,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YACjD,EAAE;YACF,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YACxC,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IACD,OAAO;QACL,aAAa;QACb,aAAa,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE;QAC/B,mBAAmB,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE;QAChD,iBAAiB,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE;QAC5C,EAAE;QACF,SAAS;QACT,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,KAAK;QACL,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC"}
|
|
@@ -1 +1,3 @@
|
|
|
1
|
-
|
|
1
|
+
import type { ILlmSchema } from "@samchon/openapi";
|
|
2
|
+
import type { IAgenticaSelectBenchmarkResult } from "../structures/IAgenticaSelectBenchmarkResult";
|
|
3
|
+
export declare function markdown<Model extends ILlmSchema.Model>(result: IAgenticaSelectBenchmarkResult<Model>): Record<string, string>;
|