@agentica/benchmark 0.19.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -300
- package/lib/index.mjs +3 -3
- package/lib/index.mjs.map +1 -1
- package/lib/internal/AgenticaSelectBenchmarkReporter.js +3 -1
- package/lib/internal/AgenticaSelectBenchmarkReporter.js.map +1 -1
- package/package.json +4 -4
- package/src/internal/AgenticaSelectBenchmarkReporter.ts +3 -1
package/README.md
CHANGED
|
@@ -1,335 +1,165 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Agentica, AI Function Calling Framework
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
<!-- https://github.com/user-attachments/assets/5326cc59-5129-470d-abcb-c3f458b5c488 -->
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
[](https://www.npmjs.com/package/@agentica/benchmark)
|
|
7
|
-
[](https://www.npmjs.com/package/@agentica/benchmark)
|
|
8
|
-
[](https://github.com/wrtnlabs/agentica/actions?query=workflow%3Abuild)
|
|
9
|
-
|
|
10
|
-
Benchmark program of `Agentica`.
|
|
11
|
-
|
|
12
|
-
`Agentica` is the simplest Agentic AI library specialized in **LLM Function Calling**, and `@agentica/benchmark` is the benchmark tool of such Agentic AI library. It supports two quantitive benchmark tools `AgenticaSelectBenchmark` and `AgenticaCallBenchmark` which can measure function calling's selecting and calling qualities.
|
|
13
|
-
|
|
14
|
-
Here is an example report generated by `@agentica/benchmark` measuring function calling quality of "Shopping Mall" scenario. Below measured benchmark scenario is exactly same with the recorded video, and you can find that every function calling has succeeded without any error.
|
|
5
|
+
|
|
15
6
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
> - Repository: https://github.com/wrtnlabs/shopping-backend
|
|
7
|
+
[](https://github.com/wrtnlabs/agentica/blob/master/LICENSE)
|
|
8
|
+
[](https://www.npmjs.com/package/@agentica/core)
|
|
9
|
+
[](https://www.npmjs.com/package/@agentica/core)
|
|
10
|
+
[](https://github.com/wrtnlabs/agentica/actions?query=workflow%3Abuild)
|
|
11
|
+
[](https://wrtnlabs.io/agentica/)
|
|
12
|
+
[](https://discord.gg/aMhRmzkqCx)
|
|
23
13
|
|
|
24
|
-
|
|
14
|
+
Agentic AI framework specialized in AI Function Calling.
|
|
25
15
|
|
|
26
|
-
|
|
16
|
+
Don't be afraid of AI agent development. Just list functions from three protocols below. This is everything you should do for AI agent development.
|
|
27
17
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
```
|
|
18
|
+
- TypeScript Class
|
|
19
|
+
- Swagger/OpenAPI Document
|
|
20
|
+
- MCP (Model Context Protocol) Server
|
|
32
21
|
|
|
33
|
-
|
|
22
|
+
Wanna make an e-commerce agent? Bring in e-commerce functions. Need a newspaper agent? Get API functions from the newspaper company. Just prepare any functions that you need, then it becomes an AI agent.
|
|
34
23
|
|
|
35
|
-
|
|
24
|
+
Are you a TypeScript developer? Then you're already an AI developer. Familiar with backend development? You're already well-versed in AI development. Anyone who can make functions can make AI agents.
|
|
36
25
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
By the way, as `typia` is a transformer library analyzing TypeScript source code in the compilation level, it needs additional setup command `npx typia setup`.
|
|
40
|
-
|
|
41
|
-
### Function selecting Benchmark
|
|
26
|
+
<!-- eslint-skip -->
|
|
42
27
|
|
|
43
28
|
```typescript
|
|
44
|
-
import
|
|
45
|
-
import path from "node:path";
|
|
46
|
-
|
|
47
|
-
import { AgenticaSelectBenchmark } from "@agentica/benchmark";
|
|
48
|
-
import { Agentica, IAgenticaOperation } from "@agentica/core";
|
|
49
|
-
import { HttpLlm, IHttpConnection, OpenApi } from "@samchon/openapi";
|
|
29
|
+
import { Agentica, assertHttpLlmApplication } from "@agentica/core";
|
|
50
30
|
import OpenAI from "openai";
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
31
|
+
import typia from "typia";
|
|
32
|
+
|
|
33
|
+
import { MobileFileSystem } from "./services/MobileFileSystem";
|
|
34
|
+
|
|
35
|
+
const agent = new Agentica({
|
|
36
|
+
vendor: {
|
|
37
|
+
api: new OpenAI({ apiKey: "********" }),
|
|
38
|
+
model: "gpt-4o-mini",
|
|
39
|
+
},
|
|
40
|
+
controllers: [
|
|
41
|
+
// functions from TypeScript class
|
|
42
|
+
{
|
|
43
|
+
protocol: "http",
|
|
44
|
+
application: typia.llm.application<MobileFileSystem, "chatgpt">(),
|
|
45
|
+
execute: new MobileFileSystem(),
|
|
61
46
|
},
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
},
|
|
75
|
-
},
|
|
76
|
-
],
|
|
77
|
-
});
|
|
78
|
-
|
|
79
|
-
// DO BENCHMARK
|
|
80
|
-
const find = (method: OpenApi.Method, path: string): IAgenticaOperation => {
|
|
81
|
-
const found = agent
|
|
82
|
-
.getOperations()
|
|
83
|
-
.find(
|
|
84
|
-
op =>
|
|
85
|
-
op.protocol === "http"
|
|
86
|
-
&& op.function.method === method
|
|
87
|
-
&& op.function.path === path,
|
|
88
|
-
);
|
|
89
|
-
if (!found) {
|
|
90
|
-
throw new Error(`Operation not found: ${method} ${path}`);
|
|
91
|
-
}
|
|
92
|
-
return found;
|
|
93
|
-
};
|
|
94
|
-
const benchmark: AgenticaSelectBenchmark<"chatgpt">
|
|
95
|
-
= new AgenticaSelectBenchmark({
|
|
96
|
-
agent,
|
|
97
|
-
config: {
|
|
98
|
-
repeat: 4,
|
|
47
|
+
// functions from Swagger/OpenAPI
|
|
48
|
+
{
|
|
49
|
+
protocol: "http",
|
|
50
|
+
application: assertHttpLlmApplication({
|
|
51
|
+
model: "chatgpt",
|
|
52
|
+
document: await fetch(
|
|
53
|
+
"https://shopping-be.wrtn.ai/editor/swagger.json",
|
|
54
|
+
).then(r => r.json()),
|
|
55
|
+
}),
|
|
56
|
+
connection: {
|
|
57
|
+
host: "https://shopping-be.wrtn.ai",
|
|
58
|
+
headers: { Authorization: "Bearer ********" },
|
|
99
59
|
},
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
"I wanna see every sales in the shopping mall",
|
|
105
|
-
"",
|
|
106
|
-
"And then show me the detailed information about the Macbook.",
|
|
107
|
-
"",
|
|
108
|
-
"After that, select the most expensive stock",
|
|
109
|
-
"from the Macbook, and put it into my shopping cart.",
|
|
110
|
-
"And take the shopping cart to the order.",
|
|
111
|
-
"",
|
|
112
|
-
"At last, I'll publish it by cash payment, and my address is",
|
|
113
|
-
"",
|
|
114
|
-
" - country: South Korea",
|
|
115
|
-
" - city/province: Seoul",
|
|
116
|
-
" - department: Wrtn Apartment",
|
|
117
|
-
" - Possession: 101-1411",
|
|
118
|
-
].join("\n"),
|
|
119
|
-
expected: {
|
|
120
|
-
type: "array",
|
|
121
|
-
items: [
|
|
122
|
-
{
|
|
123
|
-
type: "standalone",
|
|
124
|
-
operation: find("patch", "/shoppings/customers/sales"),
|
|
125
|
-
},
|
|
126
|
-
{
|
|
127
|
-
type: "standalone",
|
|
128
|
-
operation: find("get", "/shoppings/customers/sales/{id}"),
|
|
129
|
-
},
|
|
130
|
-
{
|
|
131
|
-
type: "anyOf",
|
|
132
|
-
anyOf: [
|
|
133
|
-
{
|
|
134
|
-
type: "standalone",
|
|
135
|
-
operation: find("post", "/shoppings/customers/orders"),
|
|
136
|
-
},
|
|
137
|
-
{
|
|
138
|
-
type: "standalone",
|
|
139
|
-
operation: find("post", "/shoppings/customers/orders/direct"),
|
|
140
|
-
},
|
|
141
|
-
],
|
|
142
|
-
},
|
|
143
|
-
{
|
|
144
|
-
type: "standalone",
|
|
145
|
-
operation: find(
|
|
146
|
-
"post",
|
|
147
|
-
"/shoppings/customers/orders/{orderId}/publish",
|
|
148
|
-
),
|
|
149
|
-
},
|
|
150
|
-
],
|
|
151
|
-
},
|
|
152
|
-
},
|
|
153
|
-
],
|
|
154
|
-
});
|
|
155
|
-
await benchmark.execute();
|
|
156
|
-
|
|
157
|
-
// REPORT
|
|
158
|
-
const docs: Record<string, string> = benchmark.report();
|
|
159
|
-
const root: string = `docs/benchmarks/call`;
|
|
160
|
-
|
|
161
|
-
await rmdir(root);
|
|
162
|
-
for (const [key, value] of Object.entries(docs)) {
|
|
163
|
-
await mkdir(path.join(root, key.split("/").slice(0, -1).join("/")));
|
|
164
|
-
await fs.promises.writeFile(path.join(root, key), value, "utf8");
|
|
165
|
-
}
|
|
166
|
-
}
|
|
60
|
+
},
|
|
61
|
+
],
|
|
62
|
+
});
|
|
63
|
+
await agent.conversate("I wanna buy MacBook Pro");
|
|
167
64
|
```
|
|
168
65
|
|
|
169
|
-
|
|
170
|
-
>
|
|
171
|
-
> - [Benchmark Report](https://github.com/wrtnlabs/agentica/tree/main/test/examples/benchmarks/select)
|
|
172
|
-
> - Swagger Document: https://shopping-be.wrtn.ai/editor
|
|
173
|
-
> - Repository: https://github.com/wrtnlabs/shopping-backend
|
|
66
|
+
## 📦 Setup
|
|
174
67
|
|
|
175
|
-
|
|
68
|
+
```bash
|
|
69
|
+
$ npx agentica start <directory>
|
|
70
|
+
|
|
71
|
+
----------------------------------------
|
|
72
|
+
Agentica Setup Wizard
|
|
73
|
+
----------------------------------------
|
|
74
|
+
? Package Manager (use arrow keys)
|
|
75
|
+
> npm
|
|
76
|
+
pnpm
|
|
77
|
+
yarn (berry is not supported)
|
|
78
|
+
? Project Type
|
|
79
|
+
NodeJS Agent Server
|
|
80
|
+
> NestJS Agent Server
|
|
81
|
+
React Client Application
|
|
82
|
+
Standalone Application
|
|
83
|
+
? Embedded Controllers (multi-selectable)
|
|
84
|
+
(none)
|
|
85
|
+
Google Calendar
|
|
86
|
+
Google News
|
|
87
|
+
> Github
|
|
88
|
+
Reddit
|
|
89
|
+
Slack
|
|
90
|
+
...
|
|
91
|
+
```
|
|
176
92
|
|
|
177
|
-
|
|
93
|
+
The setup wizard helps you create a new project tailored to your needs.
|
|
178
94
|
|
|
179
|
-
|
|
95
|
+
For reference, when selecting a project type, any option other than "Standalone Application" will implement the [WebSocket Protocol](https://wrtnlabs.io/agentica/docs/websocket/) for client-server communication.
|
|
180
96
|
|
|
181
|
-
|
|
97
|
+
For comprehensive setup instructions, visit our [Getting Started](https://wrtnlabs.io/agentica/docs/) guide.
|
|
182
98
|
|
|
183
|
-
|
|
184
|
-
>
|
|
185
|
-
> - [Benchmark Report](https://github.com/wrtnlabs/agentica/tree/main/test/examples/benchmarks/call)
|
|
186
|
-
> - Swagger Document: https://shopping-be.wrtn.ai/editor
|
|
187
|
-
> - Repository: https://github.com/wrtnlabs/shopping-backend
|
|
99
|
+
## 💻 Playground
|
|
188
100
|
|
|
189
|
-
|
|
190
|
-
import fs from "node:fs";
|
|
191
|
-
import path from "node:path";
|
|
101
|
+
Experience Agentica firsthand through our [interactive playground](https://wrtnlabs.io/agentica/playground) before installing.
|
|
192
102
|
|
|
193
|
-
|
|
194
|
-
import { Agentica, IAgenticaOperation } from "@agentica/core";
|
|
195
|
-
import { HttpLlm, IHttpConnection, OpenApi } from "@samchon/openapi";
|
|
196
|
-
import OpenAI from "openai";
|
|
103
|
+
Our demonstrations showcase the power and simplicity of Agentica's function calling capabilities across different integration methods.
|
|
197
104
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
model: "chatgpt",
|
|
202
|
-
vendor: {
|
|
203
|
-
api: new OpenAI({
|
|
204
|
-
apiKey: "YOUR_OPENAI_API_KEY",
|
|
205
|
-
}),
|
|
206
|
-
model: "gpt-4o-mini",
|
|
207
|
-
},
|
|
208
|
-
controllers: [
|
|
209
|
-
{
|
|
210
|
-
protocol: "http",
|
|
211
|
-
name: "shopping",
|
|
212
|
-
application: HttpLlm.application({
|
|
213
|
-
model: "chatgpt",
|
|
214
|
-
document: await fetch(
|
|
215
|
-
"https://shopping-be.wrtn.ai/editor/swagger.json",
|
|
216
|
-
).then(res => res.json()),
|
|
217
|
-
}),
|
|
218
|
-
connection: {
|
|
219
|
-
host: "https://shopping-be.wrtn.ai",
|
|
220
|
-
},
|
|
221
|
-
},
|
|
222
|
-
],
|
|
223
|
-
});
|
|
224
|
-
|
|
225
|
-
// DO BENCHMARK
|
|
226
|
-
const find = (method: OpenApi.Method, path: string): IAgenticaOperation => {
|
|
227
|
-
const found = agent
|
|
228
|
-
.getOperations()
|
|
229
|
-
.find(
|
|
230
|
-
op =>
|
|
231
|
-
op.protocol === "http"
|
|
232
|
-
&& op.function.method === method
|
|
233
|
-
&& op.function.path === path,
|
|
234
|
-
);
|
|
235
|
-
if (!found) {
|
|
236
|
-
throw new Error(`Operation not found: ${method} ${path}`);
|
|
237
|
-
}
|
|
238
|
-
return found;
|
|
239
|
-
};
|
|
240
|
-
const benchmark: AgenticaSelectBenchmark<"chatgpt">
|
|
241
|
-
= new AgenticaSelectBenchmark({
|
|
242
|
-
agent,
|
|
243
|
-
config: {
|
|
244
|
-
repeat: 4,
|
|
245
|
-
},
|
|
246
|
-
scenarios: [
|
|
247
|
-
{
|
|
248
|
-
name: "order",
|
|
249
|
-
text: [
|
|
250
|
-
"I wanna see every sales in the shopping mall",
|
|
251
|
-
"",
|
|
252
|
-
"And then show me the detailed information about the Macbook.",
|
|
253
|
-
"",
|
|
254
|
-
"After that, select the most expensive stock",
|
|
255
|
-
"from the Macbook, and put it into my shopping cart.",
|
|
256
|
-
"And take the shopping cart to the order.",
|
|
257
|
-
"",
|
|
258
|
-
"At last, I'll publish it by cash payment, and my address is",
|
|
259
|
-
"",
|
|
260
|
-
" - country: South Korea",
|
|
261
|
-
" - city/province: Seoul",
|
|
262
|
-
" - department: Wrtn Apartment",
|
|
263
|
-
" - Possession: 101-1411",
|
|
264
|
-
].join("\n"),
|
|
265
|
-
expected: {
|
|
266
|
-
type: "array",
|
|
267
|
-
items: [
|
|
268
|
-
{
|
|
269
|
-
type: "standalone",
|
|
270
|
-
operation: find("patch", "/shoppings/customers/sales"),
|
|
271
|
-
},
|
|
272
|
-
{
|
|
273
|
-
type: "standalone",
|
|
274
|
-
operation: find("get", "/shoppings/customers/sales/{id}"),
|
|
275
|
-
},
|
|
276
|
-
{
|
|
277
|
-
type: "anyOf",
|
|
278
|
-
anyOf: [
|
|
279
|
-
{
|
|
280
|
-
type: "standalone",
|
|
281
|
-
operation: find("post", "/shoppings/customers/orders"),
|
|
282
|
-
},
|
|
283
|
-
{
|
|
284
|
-
type: "standalone",
|
|
285
|
-
operation: find("post", "/shoppings/customers/orders/direct"),
|
|
286
|
-
},
|
|
287
|
-
],
|
|
288
|
-
},
|
|
289
|
-
{
|
|
290
|
-
type: "standalone",
|
|
291
|
-
operation: find(
|
|
292
|
-
"post",
|
|
293
|
-
"/shoppings/customers/orders/{orderId}/publish",
|
|
294
|
-
),
|
|
295
|
-
},
|
|
296
|
-
],
|
|
297
|
-
},
|
|
298
|
-
},
|
|
299
|
-
],
|
|
300
|
-
});
|
|
301
|
-
await benchmark.execute();
|
|
302
|
-
|
|
303
|
-
// REPORT
|
|
304
|
-
const docs: Record<string, string> = benchmark.report();
|
|
305
|
-
const root: string = `docs/benchmarks/call`;
|
|
306
|
-
|
|
307
|
-
await rmdir(root);
|
|
308
|
-
for (const [key, value] of Object.entries(docs)) {
|
|
309
|
-
await mkdir(path.join(root, key.split("/").slice(0, -1).join("/")));
|
|
310
|
-
await fs.promises.writeFile(path.join(root, key), value, "utf8");
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
```
|
|
105
|
+
- [TypeScript Class](https://wrtnlabs.io/agentica/playground/bbs)
|
|
106
|
+
- [Swagger/OpenAPI Document](https://wrtnlabs.io/agentica/playground/swagger)
|
|
107
|
+
- [Enterprise E-commerce Agent](https://wrtnlabs.io/agentica/playground/shopping)
|
|
314
108
|
|
|
315
|
-
|
|
109
|
+
<!--
|
|
110
|
+
@todo this section would be changed after making tutorial playground
|
|
111
|
+
-->
|
|
316
112
|
|
|
317
|
-
|
|
113
|
+
## 📚 Documentation Resources
|
|
318
114
|
|
|
319
|
-
|
|
115
|
+
Find comprehensive resources at our [official website](https://wrtnlabs.io/agentica).
|
|
320
116
|
|
|
321
|
-
|
|
117
|
+
- [Home](https://wrtnlabs.io/agentica)
|
|
118
|
+
- [Guide Documents](https://wrtnlabs.io/agentica/docs)
|
|
119
|
+
- [Tutorial](https://wrtnlabs.io/agentica/tutorial)
|
|
120
|
+
- [API Documents](https://wrtnlabs.io/agentica/api)
|
|
121
|
+
- [Youtube](https://www.youtube.com/@wrtnlabs)
|
|
122
|
+
- [Paper](https://wrtnlabs.io/agentica/paper)
|
|
322
123
|
|
|
323
|
-
|
|
124
|
+
## 🌟 Why Agentica?
|
|
324
125
|
|
|
325
|
-
|
|
126
|
+
```mermaid
|
|
127
|
+
flowchart
|
|
128
|
+
subgraph "JSON Schema Specification"
|
|
129
|
+
schemav4("JSON Schema v4 ~ v7") --upgrades--> emended[["OpenAPI v3.1 (emended)"]]
|
|
130
|
+
schema2910("JSON Schema 2019-03") --upgrades--> emended
|
|
131
|
+
schema2020("JSON Schema 2020-12") --emends--> emended
|
|
132
|
+
end
|
|
133
|
+
subgraph "Agentica"
|
|
134
|
+
emended --"Artificial Intelligence"--> fc{{"AI Function Calling"}}
|
|
135
|
+
fc --"OpenAI"--> chatgpt("ChatGPT")
|
|
136
|
+
fc --"Google"--> gemini("Gemini")
|
|
137
|
+
fc --"Anthropic"--> claude("Claude")
|
|
138
|
+
fc --"High-Flyer"--> deepseek("DeepSeek")
|
|
139
|
+
fc --"Meta"--> llama("Llama")
|
|
140
|
+
chatgpt --"3.1"--> custom(["Custom JSON Schema"])
|
|
141
|
+
gemini --"3.0"--> custom(["Custom JSON Schema"])
|
|
142
|
+
claude --"3.1"--> standard(["Standard JSON Schema"])
|
|
143
|
+
deepseek --"3.1"--> standard
|
|
144
|
+
llama --"3.1"--> standard
|
|
145
|
+
end
|
|
146
|
+
```
|
|
326
147
|
|
|
327
|
-
|
|
148
|
+
Agentica enhances AI function calling by the following strategies:
|
|
328
149
|
|
|
329
|
-
|
|
150
|
+
- [**Compiler Driven Development**](https://wrtnlabs.io/agentica/docs/concepts/compiler-driven-development): constructs function calling schema automatically by compiler skills without hand-writing.
|
|
151
|
+
- [**JSON Schema Conversion**](https://wrtnlabs.io/agentica/docs/core/vendor/#schema-specification): automatically handles specification differences between LLM vendors, ensuring seamless integration regardless of your chosen AI model.
|
|
152
|
+
- [**Validation Feedback**](https://wrtnlabs.io/agentica/docs/concepts/function-calling#validation-feedback): detects and corrects AI mistakes in argument composition, dramatically reducing errors and improving reliability.
|
|
153
|
+
- [**Selector Agent**](https://wrtnlabs.io/agentica/docs/concepts/function-calling#orchestration-strategy): filtering candidate functions to minimize context usage, optimize performance, and reduce token consumption.
|
|
330
154
|
|
|
331
|
-
|
|
155
|
+
Thanks to these innovations, Agentica makes AI function calling easier, safer, and more accurate than before. Development becomes more intuitive since you only need to prepare functions relevant to your specific use case, and scaling your agent's capabilities is as simple as adding or removing functions.
|
|
332
156
|
|
|
333
|
-
|
|
157
|
+
In 2023, when OpenAI announced function calling, many predicted that function calling-driven AI development would become the mainstream. However, in reality, due to the difficulty and instability of function calling, the trend in AI development became agent workflow. Agent workflow, which is inflexible and must be created for specific purposes, has conquered the AI agent ecosystem.
|
|
158
|
+
By the way, as Agentica has resolved the difficulty and instability problems of function calling, the time has come to embrace function-driven AI development once again.
|
|
334
159
|
|
|
335
|
-
|
|
160
|
+
| Type | Workflow | Vanilla Function Calling | Agentica Function Calling |
|
|
161
|
+
| ----------- | ------------- | ------------------------ | ------------------------- |
|
|
162
|
+
| Purpose | ❌ Specific | 🟢 General | 🟢 General |
|
|
163
|
+
| Difficulty | ❌ Difficult | ❌ Difficult | 🟢 Easy |
|
|
164
|
+
| Stability | 🟢 Stable | ❌ Unstable | 🟢 Stable |
|
|
165
|
+
| Flexibility | ❌ Inflexible | 🟢 Flexible | 🟢 Flexible |
|
package/lib/index.mjs
CHANGED
|
@@ -418,7 +418,7 @@ function writeIndex(result) {
|
|
|
418
418
|
const events = result.experiments.map((r => r.events)).flat();
|
|
419
419
|
const average = events.map((e => e.completed_at.getTime() - e.started_at.getTime())).reduce(((a, b) => a + b), 0) / events.length;
|
|
420
420
|
const aggregate = result.usage.aggregate;
|
|
421
|
-
return [ "# LLM Function Selection Benchmark", "## Summary", ` - Aggregation:`, ` - Scenarios: #${result.experiments.length.toLocaleString()}`, ` - Trial: ${events.length}`, ` - Success: ${events.filter((e => e.type === "success")).length}`, ` - Failure: ${events.filter((e => e.type === "failure")).length}`, ` - Average Time: ${MathUtil.round(average).toLocaleString()} ms`, ` - Token Usage`, ` - Total: ${aggregate.total.toLocaleString()}`, ` - Input`, ` - Total: ${aggregate.input.total.toLocaleString()}`, ` - Cached: ${aggregate.input.cached.toLocaleString()}`, ` - Output:`, ` - Total: ${aggregate.output.total.toLocaleString()}`, ` - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`, ` - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`, ` - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`, "", "## Experiments", " Name | Status | Time/Avg ", ":-----|:-------|----------:", ...result.experiments.map((exp => [ `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`, (() => {
|
|
421
|
+
return [ "# LLM Function Selection Benchmark", "## Summary", ` - Aggregation:`, ` - Scenarios: #${result.experiments.length.toLocaleString()}`, ` - Trial: ${events.length}`, ` - Success: ${events.filter((e => e.type === "success")).length}`, ` - Failure: ${events.filter((e => e.type === "failure")).length}`, ` - Error: ${events.filter((e => e.type === "error")).length}`, ` - Average Time: ${MathUtil.round(average).toLocaleString()} ms`, ` - Token Usage`, ` - Total: ${aggregate.total.toLocaleString()}`, ` - Input`, ` - Total: ${aggregate.input.total.toLocaleString()}`, ` - Cached: ${aggregate.input.cached.toLocaleString()}`, ` - Output:`, ` - Total: ${aggregate.output.total.toLocaleString()}`, ` - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`, ` - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`, ` - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`, "", "## Experiments", " Name | Status | Time/Avg ", ":-----|:-------|----------:", ...result.experiments.map((exp => [ `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`, (() => {
|
|
422
422
|
const success = Math.floor(exp.events.filter((e => e.type === "success")).length / exp.events.length * 10);
|
|
423
423
|
return Array.from({
|
|
424
424
|
length: success
|
|
@@ -430,11 +430,11 @@ function writeIndex(result) {
|
|
|
430
430
|
|
|
431
431
|
function writeExperimentIndex(exp) {
|
|
432
432
|
const aggregate = exp.usage.aggregate;
|
|
433
|
-
return [ `# ${exp.scenario.name}`, "## Summary", " - Aggregation:", ` - Trial: ${exp.events.length}`, ` - Success: ${exp.events.filter((e => e.type === "success")).length}`, ` - Failure: ${exp.events.filter((e => e.type === "failure")).length}`, ` - Average Time: ${MathUtil.round(exp.events.map((event => event.completed_at.getTime() - event.started_at.getTime())).reduce(((a, b) => a + b), 0) / exp.events.length).toLocaleString()} ms`, ` - Token Usage`, ` - Total: ${aggregate.total.toLocaleString()}`, ` - Input`, ` - Total: ${aggregate.input.total.toLocaleString()}`, ` - Cached: ${aggregate.input.cached.toLocaleString()}`, ` - Output:`, ` - Total: ${aggregate.output.total.toLocaleString()}`, ` - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`, ` - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`, ` - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`, "", "## Events", " No | Type | Time", "---:|:-----|----:", ...exp.events.map(((e, i) => [ `[${i + 1}.](./${i + 1}.${e.type}.md)`, e.type, `${MathUtil.round(e.completed_at.getTime() - e.started_at.getTime())} ms` ].join(" | "))), "", "## Scenario", "### User Prompt", exp.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected), null, 2), "```" ].join("\n");
|
|
433
|
+
return [ `# ${exp.scenario.name}`, "## Summary", " - Aggregation:", ` - Trial: ${exp.events.length}`, ` - Success: ${exp.events.filter((e => e.type === "success")).length}`, ` - Failure: ${exp.events.filter((e => e.type === "failure")).length}`, ` - Error: ${exp.events.filter((e => e.type === "error")).length}`, ` - Average Time: ${MathUtil.round(exp.events.map((event => event.completed_at.getTime() - event.started_at.getTime())).reduce(((a, b) => a + b), 0) / exp.events.length).toLocaleString()} ms`, ` - Token Usage`, ` - Total: ${aggregate.total.toLocaleString()}`, ` - Input`, ` - Total: ${aggregate.input.total.toLocaleString()}`, ` - Cached: ${aggregate.input.cached.toLocaleString()}`, ` - Output:`, ` - Total: ${aggregate.output.total.toLocaleString()}`, ` - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`, ` - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`, ` - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`, "", "## Events", " No | Type | Time", "---:|:-----|----:", ...exp.events.map(((e, i) => [ `[${i + 1}.](./${i + 1}.${e.type}.md)`, e.type, `${MathUtil.round(e.completed_at.getTime() - e.started_at.getTime())} ms` ].join(" | "))), "", "## Scenario", "### User Prompt", exp.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected), null, 2), "```" ].join("\n");
|
|
434
434
|
}
|
|
435
435
|
|
|
436
436
|
function writeExperimentEvent(event, index) {
|
|
437
|
-
return [ `# ${index + 1}. ${event.type}`, `## Summary`, ` - Name: ${event.scenario.name}`, ` - Type: ${event.type}`, ` - Time: ${(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`, ...event.type !== "error" ? [ " - Token Usage", ` - Total: ${event.usage.aggregate.toLocaleString()}`, ` - Prompt`, ` - Total: ${event.usage.aggregate.input.total.toLocaleString()}`, ` - Cached: ${event.usage.aggregate.input.cached.toLocaleString()}`, ` - Completion:`, ` - Total: ${event.usage.aggregate.output.total.toLocaleString()}`, ` - Reasoning: ${event.usage.aggregate.output.reasoning.toLocaleString()}`, ` - Accepted Prediction: ${event.usage.aggregate.output.accepted_prediction.toLocaleString()}`, ` - Rejected Prediction: ${event.usage.aggregate.output.rejected_prediction.toLocaleString()}` ] : [], "", "## Scenario", "### User Prompt", event.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected), null, 2), "```", "", ...event.type === "success" || event.type === "failure" ? [ "## Result", ...event.selected.map((s => [ `### ${s.operation.name}`, ` - Controller: \`${s.operation.controller.name}\``, ` - Function: \`${s.operation.function.name}\``, ` - Reason: ${s.reason}`, "", ...s.operation.function.description !== undefined && s.operation.function.description !== "" ? [ s.operation.function.description, "" ] : [] ].join("\n"))) ] : [], ...event.type === "error" ? [ "## Error", "```json", AgenticaBenchmarkUtil.errorToJson(JSON.stringify(event.error, null, 2)), "```", "" ] : [] ].join("\n");
|
|
437
|
+
return [ `# ${index + 1}. ${event.type}`, `## Summary`, ` - Name: ${event.scenario.name}`, ` - Type: ${event.type}`, ` - Time: ${(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`, ...event.type !== "error" ? [ " - Token Usage", ` - Total: ${event.usage.aggregate.total.toLocaleString()}`, ` - Prompt`, ` - Total: ${event.usage.aggregate.input.total.toLocaleString()}`, ` - Cached: ${event.usage.aggregate.input.cached.toLocaleString()}`, ` - Completion:`, ` - Total: ${event.usage.aggregate.output.total.toLocaleString()}`, ` - Reasoning: ${event.usage.aggregate.output.reasoning.toLocaleString()}`, ` - Accepted Prediction: ${event.usage.aggregate.output.accepted_prediction.toLocaleString()}`, ` - Rejected Prediction: ${event.usage.aggregate.output.rejected_prediction.toLocaleString()}` ] : [], "", "## Scenario", "### User Prompt", event.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected), null, 2), "```", "", ...event.type === "success" || event.type === "failure" ? [ "## Result", ...event.selected.map((s => [ `### ${s.operation.name}`, ` - Controller: \`${s.operation.controller.name}\``, ` - Function: \`${s.operation.function.name}\``, ` - Reason: ${s.reason}`, "", ...s.operation.function.description !== undefined && s.operation.function.description !== "" ? [ s.operation.function.description, "" ] : [] ].join("\n"))) ] : [], ...event.type === "error" ? [ "## Error", "```json", AgenticaBenchmarkUtil.errorToJson(JSON.stringify(event.error, null, 2)), "```", "" ] : [] ].join("\n");
|
|
438
438
|
}
|
|
439
439
|
|
|
440
440
|
class AgenticaSelectBenchmark {
|
package/lib/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.mjs","sources":["../src/internal/AgenticaBenchmarkPredicator.ts","../src/utils/MathUtil.ts","../src/internal/AgenticaBenchmarkUtil.ts","../src/internal/AgenticaPromptReporter.ts","../src/internal/AgenticaCallBenchmarkReporter.ts","../src/AgenticaCallBenchmark.ts","../src/internal/AgenticaSelectBenchmarkReporter.ts","../src/AgenticaSelectBenchmark.ts"],"sourcesContent":[null,null,null,null,null,null,null,null],"names":["AgenticaBenchmarkPredicator","isNext","success","async","agent","last","getHistories","at","llmVendor","getVendor","isTextPrompt","type","role","consent","functions","result","api","chat","completions","create","model","messages","content","join","text","tools","function","name","description","parameters","tool_choice","parallel_tool_calls","options","toolCall","choices","message","tool_calls","filter","tc","undefined","input","arguments","reply","props","successInner","call","expected","overrideOperations","operations","strict","take","targetIterator","items","Symbol","iterator","targeted","next","done","length","value","slice","target","operation","some","op","anyOf","callResult","allOf","map","every","r","reduce","acc","Math","max","MathUtil","round","floor","AgenticaBenchmarkUtil","errorToJson","expectedToJson","error","Error","stack","AgenticaPromptReporter","markdown","p","selections","flatMap","s","functionDescriptionCount","controller","reason","executes","e","split","JSON","stringify","AgenticaCallBenchmarkReporter","Object","fromEntries","writeIndex","experiments","exp","scenario","writeExperimentIndex","events","event","i","writeExperimentEvent","flat","average","completed_at","getTime","started_at","a","b","aggregate","usage","toLocaleString","total","cached","output","reasoning","accepted_prediction","rejected_prediction","drawStatus","select","index","prompts","count","Array","from","fill","AgenticaCallBenchmark","constructor","this","agent_","scenarios_","scenarios","config_","repeat","config","simultaneous","result_","execute","listener","Date","semaphore","Semaphore","task","Promise","all","acquire","step","release","cur","AgenticaTokenUsage","plus","zero","report","clone","out","getTokenUsage","conversate","AgenticaSelectBenchmarkReporter","selected","AgenticaSelectBenchmark","histories_","context","getContext","prompt","factory","createTextHistory","executor","TypeError","histories","orchestrate","ready","dispatch","assistantPrompts"],"mappings":";;;;;;;;AAeO,MAAMA,8BAA8B;IACzCC;IACAC;;;AAmCFC,eAAeF,OAAuCG;IACpD,MAAMC,OAA2CD,MAC9CE,eACAC,IAAG;IAQN,MAAMC,YAAYJ,MAAMK;IACxB,MAAMC,eAAeL,MAAMM,SAAS,UAAUN,KAAKO,SAAS;IAC5D,KAAKF,cAAc;QACjB,OAAO;;IAGT,MAAMG,UAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAGrCC,UAAU;IACd,MAAMC,eAAsCP,UAAUQ,IAAIC,KAAKC,YAAYC,OACzE;QACEC,OAAOZ,UAAUY;QACjBC,UAAU,EACR;YACET,MAAM;YACNU,SAAS,EACP,iCACA,IACA,uDACA,gEACA,qDACAC,KAAK;WAET;YACEX,MAAM;YACNU,SAASjB,KAAKmB;;QAGlBC,OAAO,EACL;YACEd,MAAM;YACNe,UAAU;gBACRC,MAAMd,QAAQc;gBACdC,aAAaf,QAAQe;gBACrBC,YAAYhB,QAAQgB;;;QAI1BC,aAAa;QACbC,qBAAqB;OAEvBvB,UAAUwB;IAGZ,MAAMC,YACJlB,OAAOmB,QAAQ,IAAIC,QAAQC,cAAc,IACzCC,QACAC,MAAMA,GAAG3B,SAAS,cAAc2B,GAAGZ,SAASC,SAASd,QAAQc,SAC3D;IAEJ,IAAIM,aAAaM,WAAW;QAC1B,OAAO;;IAGT,MAAMC;;;;;;;SAA0CP,SAASP,SAASe;IAClE,OAAOD,UAAU,OAAOA,MAAME,QAAQ;AACxC;;AAWM,SAAUxC,QAAwCyC;IAoBtD,OAAOC,aAAaD,OAAO5B;AAC7B;;AAEA,SAAS6B,aAA6CD;IAQpD,MAAME,OAAO,CACXC,UACAC,uBAEAH,aAAa;QACXE;QACAE,YAAYD,sBAAsBJ,MAAMK;QACxCC,QAAQN,MAAMM;;IAGlB,QAAQN,MAAMG,SAASnC;MACrB,KAAK;QAAS;YACZ,IAAIuC,OAAO;YACX,MAAMC,iBAAiBR,MAAMG,SAASM,MAAMC,OAAOC;YACnD,IAAIC,WAAWJ,eAAeK;YAE9B,OAAO,MAAM;gBACX,IAAID,SAASE,SAAS,MAAM;oBAC1B,OAAO;wBACL1C,QAAQ;wBACRmC;;;gBAGJ,IAAIA,QAAQP,MAAMK,WAAWU,QAAQ;oBACnC,OAAO;wBAAE3C,QAAQ;;;gBAGnB,MAAMA,SAAS8B,KAAKU,SAASI,OAAOhB,MAAMK,WAAWY,MAAMV;gBAC3D,KAAKnC,OAAOA,QAAQ;oBAClB,IAAI4B,MAAMM,WAAW,MAAM;wBACzB,OAAO;4BAAElC,QAAQ;;;oBAEnBmC,QAAQ;oBACR;;gBAGFA,QAAQnC,OAAOmC;gBACfK,WAAWJ,eAAeK;;;;MAG9B,KAAK;QAAc;YACjB,MAAMK,SAASlB,MAAMG,SAASgB;YAC9B,MAAM/C,SAAS4B,MAAMK,WAAWe,MAAKC,MAAMA,GAAGrC,SAASkC,OAAOlC;YAC9D,IAAIZ,QAAQ;gBACV,OAAO;oBAAEA;oBAAQmC,MAAM;;;YAEzB,OAAO;gBACLnC;;;;MAGJ,KAAK;QACH,KAAK,MAAM+B,YAAYH,MAAMG,SAASmB,OAAO;YAC3C,MAAMC,aAAarB,KAAKC;YACxB,IAAIoB,WAAWnD,QAAQ;gBACrB,OAAOmD;;;QAIX,OAAO;YAAEnD,QAAQ;;;MACnB,KAAK;QAAS;YAQZ,MAAMA,SAAS4B,MAAMG,SAASqB,MAAMC,KAAItB,YAAYD,KAAKC;YACzD,IAAI/B,OAAOsD,OAAMC,KAAKA,EAAEvD,UAAS;gBAC/B,OAAO;oBACLA,QAAQ;oBACRmC,MAAMnC,OAAOwD,QAAO,CAACC,KAAKF,MAAMG,KAAKC,IAAIF,KAAKF,EAAEpB,QAAO;;;YAI3D,OAAO;gBACLnC,QAAQ;;;;AAIhB;;AC3OO,MAAM4D,WAAW;IAOtBC,OAAQjB,SAA0Bc,KAAKI,MAAMlB,QAAQ,OAAO;;;ACJvD,MAAMmB,wBAAwB;IACnCC;IACAC;;;AAGF,SAASD,YAAeE;IAOtB,IAAIA,iBAAiBC,OAAO;QAC1B,OAAO;eACFD;YACHtD,MAAMsD,MAAMtD;YACZQ,SAAS8C,MAAM9C;YACfgD,OAAOF,MAAME;;;IAGjB,OAAOF;AACT;;AAEA,SAASD,eAA+ClC;IACtD,IAAIA,SAASnC,SAAS,cAAc;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfmD,WAAW;gBACTnC,MAAMmB,SAASgB,UAAUnC;gBACzBC,aAAakB,SAASgB,UAAUpC,SAASE;;;WAI1C,IAAIkB,SAASnC,SAAS,SAAS;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfyC,OAAON,SAASM,MAAMgB,IAAIY;;WAGzB,IAAIlC,SAASnC,SAAS,SAAS;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfwD,OAAOrB,SAASqB,MAAMC,IAAIY;;WAGzB;QACH,OAAO;YACLrE,MAAMmC,SAASnC;YACfsD,OAAOnB,SAASmB,MAAMG,IAAIY;;;AAGhC;;ACpDO,MAAMI,yBAAyB;cACpCC;;;AAGF,SAASA,WAAyCC;IAEhD,IAAIA,EAAE3E,SAAS,QAAQ;QACrB,OAAO,EAAC,aAAa2E,EAAE1E,SAAS0E,EAAE9D,MAAM,KAAID,KAAK;WAE9C,IAAI+D,EAAE3E,SAAS,YAAY2E,EAAE3E,SAAS,UAAU;QACnD,OAAO,EACL,OAAO2E,EAAE3E,SAAS,WAAW,WAAW,eACrC2E,EAAEC,WACFC,SAASC;YACR,MAAMC,2BAA2BD,EAAE3B,UAAUpC,SAASE,aAAa8B,UAAU;YAE7E,OAAO,EACL,QAAQ+B,EAAE3B,UAAUnC,QACpB,mBAAmB8D,EAAE3B,UAAU6B,WAAWhE,QAC1C,iBAAiB8D,EAAE3B,UAAUpC,SAASC,QACtC,eAAe8D,EAAEG,UACjB,OACIF,2BAA2B,IAC3B,EAACD,EAAE3B,UAAUpC,SAASE,aAAa,OACnC;AACL,cAELL,KAAK;WAEJ,IAAI+D,EAAE3E,SAAS,YAAY;QAC9B,OAAO,EACL,mBACG2E,EAAEO,SAASzB,KAAI0B,KAAK,OAAOA,EAAEhC,UAAUnC,UAC1C,OACG2D,EAAE9D,KAAKuE,MAAM,MAAM3B,KAAIqB,KAAK,KAAKA,OACpC,KACAlE,KAAK;;IAET,OAAO,EACL,eACA,aAAa+D,EAAExB,UAAUnC,QACzB,mBAAmB2D,EAAExB,UAAU6B,WAAWhE,QAC1C,iBAAiB2D,EAAExB,UAAUpC,SAASC,QACtC,IACA,WACAqE,KAAKC,UAAUX,EAAE7C,WAAW,MAAM,IAClC,OACA,KACAlB,KAAK;AACT;;ACzCO,MAAM2E,gCAAgC;cAC3Cb;;;AAGI,SAAUA,WAAyCtE;IACvD,OAAOoF,OAAOC,YAAY,EACxB,EAAC,eAAeC,aAAkBtF,cAC/BA,OAAOuF,YACPlC,KAAwBmC,OAAO,EAC9B,EAAC,KAAKA,IAAIC,SAAS7E,kBAAkB8E,uBAAqBF,WACvDA,IAAIG,OAAOtC,KAAsB,CAACuC,OAAOC,MAAM,EAChD,KAAKL,IAAIC,SAAS7E,QAAQiF,IAAI,KAAKD,MAAMhG,WACzCkG,uBAAqBF,OAAOC,WAG/BE;AAEP;;AAEA,SAAST,aAA2CtF;IAClD,MAAM2F,SAA+C3F,OAAOuF,YACzDlC,KAAIE,KAAKA,EAAEoC,SACXI;IACH,MAAMC,UACFL,OACCtC,KAAI0B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKV,OAAOhD;IACzC,MAAM2D,YAA2CtG,OAAOuG,MAAMD;IAC9D,OAAO,EACL,iCACA,cACA,oBACA,qBAAqBtG,OAAOuF,YAAY5C,OAAO6D,oBAC/C,gBAAgBb,OAAOhD,UACvB,kBAAkBgD,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC3D,kBAAkBgD,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC3D,uBAAuBiB,SAASC,MAAMmC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU7E,MAAMgF,MAAMD,oBACxC,mBAAmBF,UAAU7E,MAAMiF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,qCACA,yCACGxG,OAAOuF,YAAYlC,KAAImC,OACxB,EACE,IAAIA,IAAIC,SAAS7E,WAAW4E,IAAIC,SAAS7E,mBACzCmG,WACEvB,IAAIG,SACJZ,KAAKA,EAAEnF,SAAS,WAAWmF,EAAEiC,WAAW,QAE1CD,WAAWvB,IAAIG,SAAQZ,KAAKA,EAAEnF,SAAS,WAAWmF,EAAEjD,SAAS,QAC7D,GAAG8B,SAASC,MACV2B,IAAIG,OACDtC,KAAI0B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOhD,QAC3C6D,wBACFhG,KAAK,WAETA,KAAK;AACT;;AAEA,SAASkF,uBAAqDF;IAC5D,OAAO,EACL,KAAKA,IAAIC,SAAS7E,QAClB,cACA,mBAAmB4E,IAAIG,OAAOhD,OAAO6D,oBACrC,gBAAgBhB,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC7D,gBAAgB6C,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC7D,qBAAqBiB,SAASC,MAC5B2B,IAAIG,OACDtC,KAAI0B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOhD,QAC3C6D,uBACF,IACA,aACA,uBACA,0BACGhB,IAAIG,OAAOtC,KAAI,CAAC0B,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAEnF,YAC5BmF,EAAEnF,MACF,GAAGgE,SAASC,MAAMkB,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,kBAC1D1F,KAAK,UAET,IACA,eACA,mBACAgF,IAAIC,SAAShF,MACb,IACA,gBACA,WACAwE,KAAKC,UACHnB,sBAAsBE,eAAeuB,IAAIC,SAAS1D,WAClD,MACA,IAEF,QACAvB,KAAK;AACT;;AAEA,SAASsF,uBAAqDF,OAA2CqB;IACvG,OAAO,EACL,KAAKA,QAAQ,MAAMrB,MAAMhG,QACzB,cACA,aAAagG,MAAMH,SAAS7E,QAC5B,aAAagF,MAAMhG,QACnB,aAAagE,SAASC,MACpB+B,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,WAChDM,0BACEZ,MAAMhG,SAAS,UACf,EACE,eAAegG,MAAMoB,SAAS,MAAM,OACpC,aAAapB,MAAM9D,OAAO,MAAM,UAElC,IACJ,oBACA,gBAAgBmD,KAAKC,UAAUU,MAAMW,MAAMD,UAAUG,UACrD,eACA,kBAAkBb,MAAMW,MAAMD,UAAU7E,MAAMgF,SAC9C,mBAAmBb,MAAMW,MAAMD,UAAU7E,MAAMiF,UAC/C,iBACA,kBAAkBd,MAAMW,MAAMD,UAAUK,OAAOF,SAC/C,gCAAgCb,MAAMW,MAAMD,UAAUK,OAAOE,uBAC7D,sBAAsBjB,MAAMW,MAAMD,UAAUK,OAAOC,aACnD,gCAAgChB,MAAMW,MAAMD,UAAUK,OAAOG,uBAE7D,IACA,eACA,mBACAlB,MAAMH,SAAShF,MACf,IACA,gBACA,WACAwE,KAAKC,UACHnB,sBAAsBE,eAAe2B,MAAMH,SAAS1D,WACpD,MACA,IAEF,OACA,IACA,0BACG6D,MAAMsB,QAAQ7D,IAAIgB,uBAAuBC,WAC5C,OACIsB,MAAMhG,SAAS,UACf,EACE,YACA,WACAqF,KAAKC,UACHnB,sBAAsBC,YAAY4B,MAAM1B,QACxC,MACA,IAEF,UAEF,KACJ1D,KAAK;AACT;;AAEA,SAASuG,WAA2CpB,QAA8CxG;IAChG,MAAMgI,QAAgBzD,KAAKI,MACxB6B,OAAOrE,OAAOnC,SAASwD,SAASgD,OAAOhD,SAAU;IAGpD,OACEyE,MAAMC,KAAK;QAAE1E,QAAQwE;OAASG,KAAK,KAAK9G,KAAK,MAC3C4G,MAAMC,KAAK;QAAE1E,QAAQ,KAAKwE;OAASG,KAAK,KAAK9G,KAAK;AAExD;;MCxJa+G;IAWX,WAAAC,CAAmB5F;QACjB6F,KAAKC,SAAS9F,MAAMvC;QACpBoI,KAAKE,aAAa/F,MAAMgG,UAAU/E;QAClC4E,KAAKI,UAAU;YACbC,QAAQlG,MAAMmG,QAAQD,UAAU;YAChCE,cAAcpG,MAAMmG,QAAQC,gBAAgB;YAC5ClI,SAAS8B,MAAMmG,QAAQjI,WAAW;;QAEpC2H,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMhC,aAAmB,IAAIiC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMO,OAAOd,KAAKE,WAAWtE,KAAIjE,MAAOqG;YACtC,MAAME,eACI6C,QAAQC,IACdrB,MAAMC,KAAK;gBAAE1E,QAAQ8E,KAAKI,QAAQC;eAAUzE,KAAIjE;sBACxCiJ,UAAUK;gBAChB,MAAM3D,UACI0C,KAAKkB,KAAKlD;sBACd4C,UAAUO;gBAEhB,IAAIT,aAAa3G,WAAW;oBAC1B2G,SAASpD;;gBAGX,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAY,OAAOZ,OACJrE,QAAOyD,KAAKA,EAAEnF,SAAS,UACvByD,KAAI0B,KAAKA,EAAEwB,QACX/C,QAAO,CAACC,KAAKoF,QAAQC,mBAAmBC,KAAKtF,KAAKoF,OAAMC,mBAAmBE;;AAC/E;QAEH,MAAMzD,oBACIiD,QAAQC,IAAIF;QACtB,OAAQd,KAAKQ,UAAU;YACrB1C;YACAY;YACAF,cAAc,IAAImC;YAClB7B,OAAOhB,YACJlC,KAAIkB,KAAKA,EAAEgC,QACX/C,QAAO,CAACC,KAAKoF,QAAQC,mBAAmBC,KAAKtF,KAAKoF,OAAMC,mBAAmBE;;;IAsB3E,MAAAC;QACL,IAAIxB,KAAKQ,YAAY,MAAM;YACzB,MAAM,IAAI9D,MAAM;;QAElB,OAAOgB,8BAA8Bb,SAASmD,KAAKQ;;IAG7C,UAAMU,CACZlD;QAEA,MAAMpG,QAAyBoI,KAAKC,OAAOwB;QAC3C,MAAM/C,aAAmB,IAAIiC;QAC7B,MAAMjJ,UAAU,MACdF,4BAA4BE,QAAQ;YAClC4C,UAAU0D,SAAS1D;YACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,YACvByD,KAAIkB,KAAKA,EAAExB;YACdb,QAAQ;;QAEZ,MAAMiH,MAAM;YACV,MAAMnC,SAAS/H,4BAA4BE,QAAQ;gBACjD4C,UAAU0D,SAAS1D;gBACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,WACvByD,KAAIkB,KAAKA,EAAEC,aACXuB,OACA1C,KAAIkB,KAAKA,EAAExB;gBACdb,QAAQ;;YAEV,MAAMJ,OAAO3C;YACb,OAAO;gBACLS,MAAOkC,OAAO,YAAY;gBAC1B2D;gBACAuB;gBACAlF;gBACAoF,SAAS7H,MAAME;gBACfgH,OAAOlH,MAAM+J;gBACbjD;gBACAF,cAAc,IAAImC;;AACmC;QAGzD;kBACQ/I,MAAMgK,WAAW5D,SAAShF;YAChC,IAAItB,WAAW;gBACb,OAAOgK;;YAGT,KAAK,IAAItD,IAAY,GAAGA,IAAI4B,KAAKI,QAAQ/H,WAAW+F,GAAG;gBACrD,MAAMpD,aACIxD,4BAA4BC,OAAOG;gBAC7C,IAAIoD,SAAS,MAAM;oBACjB;;sBAGIpD,MAAMgK,WAAW5G;gBACvB,IAAItD,WAAW;oBACb,OAAOgK;;;YAGX,OAAOA;UAET,OAAOjF;YACL,OAAO;gBACLtE,MAAM;gBACN6F;gBACAyB,SAAS7H,MAAME;gBACfgH,OAAOlH,MAAM+J;gBACblF;gBACAiC;gBACAF,cAAc,IAAImC;;;;;;AChMnB,MAAMkB,kCAAkC;IAC7ChF;;;AAGI,SAAUA,SAAyCtE;IACvD,MAAMuC,WAAW,EACf,EAAC,eAAe+C,WAAWtF,cACxBA,OAAOuF,YACPlC,KAAwBmC,OAAO,EAC9B,EAAC,KAAKA,IAAIC,SAAS7E,kBAAkB8E,qBAAqBF,WACvDA,IAAIG,OAAOtC,KAAsB,CAACuC,OAAOC,MAAM,EAChD,KAAKL,IAAIC,SAAS7E,QAAQiF,IAAI,KAAKD,MAAMhG,WACzCkG,qBAAqBF,OAAOC,WAG/BE;IAGL,OAAOX,OAAOC,YAAY9C;AAC5B;;AAEA,SAAS+C,WAA2CtF;IAClD,MAAM2F,SAAiD3F,OAAOuF,YAC3DlC,KAAIE,KAAKA,EAAEoC,SACXI;IACH,MAAMC,UACAL,OACCtC,KAAI0B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKV,OAAOhD;IAC3C,MAAM2D,YAA2CtG,OAAOuG,MAAMD;IAC9D,OAAO,EACL,sCACA,cACA,oBACA,qBAAqBtG,OAAOuF,YAAY5C,OAAO6D,oBAC/C,gBAAgBb,OAAOhD,UACvB,kBAAkBgD,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC3D,kBAAkBgD,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC3D,uBAAuBiB,SAASC,MAAMmC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU7E,MAAMgF,MAAMD,oBACxC,mBAAmBF,UAAU7E,MAAMiF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,+BACA,kCACGxG,OAAOuF,YAAYlC,KAAImC,OACxB,EACE,IAAIA,IAAIC,SAAS7E,WAAW4E,IAAIC,SAAS7E,mBACzC;QACE,MAAMzB,UAAkBuE,KAAKI,MAC1B0B,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,SAC1C6C,IAAIG,OAAOhD,SACb;QAEJ,OACEyE,MAAMC,KAAK;YAAE1E,QAAQxD;WAAWmI,KAAK,KAAK9G,KAAK,MAC7C4G,MAAMC,KAAK;YAAE1E,QAAQ,KAAKxD;WAAWmI,KAAK,KAAK9G,KAAK;AAEzD,MAVD,IAWA,GAAGoD,SAASC,MACV2B,IAAIG,OACDtC,KACCuC,SACEA,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,YAEnD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOhD,QAC3C6D,wBACFhG,KAAK,WAETA,KAAK;AACT;;AAEA,SAASkF,qBAAqDF;IAC5D,MAAMc,YAA2Cd,IAAIe,MAAMD;IAC3D,OAAO,EACL,KAAKd,IAAIC,SAAS7E,QAClB,cACA,oBACA,gBAAgB4E,IAAIG,OAAOhD,UAC3B,kBAAkB6C,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC/D,kBAAkB6C,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC/D,uBAAuBiB,SAASC,MAC9B2B,IAAIG,OACDtC,KACCuC,SACEA,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,YAEnD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOhD,QAC3C6D,uBACF,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU7E,MAAMgF,MAAMD,oBACxC,mBAAmBF,UAAU7E,MAAMiF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,aACA,qBACA,wBACGhB,IAAIG,OAAOtC,KAAI,CAAC0B,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAEnF,YAC5BmF,EAAEnF,MACF,GAAGgE,SAASC,MAAMkB,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,kBAE1D1F,KAAK,UAET,IACA,eACA,mBACAgF,IAAIC,SAAShF,MACb,IACA,gBACA,WACAwE,KAAKC,UACHnB,sBAAsBE,eAAeuB,IAAIC,SAAS1D,WAClD,MACA,IAEF,QACAvB,KAAK;AACT;;AAEA,SAASsF,qBAAqDF,OAA6CqB;IACzG,OAAO,EACL,KAAKA,QAAQ,MAAMrB,MAAMhG,QACzB,cACA,aAAagG,MAAMH,SAAS7E,QAC5B,aAAagF,MAAMhG,QACnB,cAAcgG,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,WAAWM,0BACrEZ,MAAMhG,SAAS,UACf,EACE,mBACA,gBAAgBgG,MAAMW,MAAMD,UAAUE,oBACtC,gBACA,kBAAkBZ,MAAMW,MAAMD,UAAU7E,MAAMgF,MAAMD,oBACpD,mBAAmBZ,MAAMW,MAAMD,UAAU7E,MAAMiF,OAAOF,oBACtD,qBACA,kBAAkBZ,MAAMW,MAAMD,UAAUK,OAAOF,MAAMD,oBACrD,sBAAsBZ,MAAMW,MAAMD,UAAUK,OAAOC,UAAUJ,oBAC7D,gCAAgCZ,MAAMW,MAAMD,UAAUK,OAAOE,oBAAoBL,oBACjF,gCAAgCZ,MAAMW,MAAMD,UAAUK,OAAOG,oBAAoBN,uBAEnF,IACJ,IACA,eACA,mBACAZ,MAAMH,SAAShF,MACf,IACA,gBACA,WACAwE,KAAKC,UACHnB,sBAAsBE,eAAe2B,MAAMH,SAAS1D,WACpD,MACA,IAEF,OACA,OACI6D,MAAMhG,SAAS,aAAagG,MAAMhG,SAAS,YAC3C,EACE,gBACGgG,MAAM2D,SAASlG,KAAIqB,KACpB,EACE,OAAOA,EAAE3B,UAAUnC,QACnB,qBAAqB8D,EAAE3B,UAAU6B,WAAWhE,UAC5C,mBAAmB8D,EAAE3B,UAAUpC,SAASC,UACxC,eAAe8D,EAAEG,UACjB,OACIH,EAAE3B,UAAUpC,SAASE,gBAAgBW,aAAakD,EAAE3B,UAAUpC,SAASE,gBAAgB,KACvF,EAAC6D,EAAE3B,UAAUpC,SAASE,aAAa,OACnC,KACJL,KAAK,YAGX,OACAoF,MAAMhG,SAAS,UACf,EACE,YACA,WACAmE,sBAAsBC,YACpBiB,KAAKC,UAAUU,MAAM1B,OAAO,MAAM,KAEpC,OACA,OAEF,KACJ1D,KAAK;AACT;;MC/KagJ;IAYX,WAAAhC,CAAmB5F;QACjB6F,KAAKC,SAAS9F,MAAMvC;QACpBoI,KAAKE,aAAa/F,MAAMgG,UAAU/E;QAClC4E,KAAKI,UAAU;YACbC,QAAQlG,MAAMmG,QAAQD,UAAU;YAChCE,cAAcpG,MAAMmG,QAAQC,gBAAgB;;QAE9CP,KAAKgC,aAAa7H,MAAMvC,MAAME,eAAesD;QAC7C4E,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMhC,aAAmB,IAAIiC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMzC,oBACIiD,QAAQC,IACdhB,KAAKE,WAAWtE,KAAIjE,MAAOqG;YACzB,MAAME,eACI6C,QAAQC,IACdrB,MAAMC,KAAK;gBAAE1E,QAAQ8E,KAAKI,QAAQC;eAAUzE,KAAIjE;sBACxCiJ,UAAUK;gBAChB,MAAM3D,UACI0C,KAAKkB,KAAKlD;sBACd4C,UAAUO;gBAChB,IAAIT,aAAa3G,WAAW;oBAC1B2G,SAASpD;;gBAEX,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAY,OAAOZ,OACJrE,QAAOyD,KAAKA,EAAEnF,SAAS,UACvByD,KAAI0B,KAAKA,EAAEwB,QACX/C,QAAO,CAACC,KAAKoF,QAAQC,mBAAmBC,KAAKtF,KAAKoF,OAAMC,mBAAmBE;;AAC/E;QAGP,OAAQvB,KAAKQ,UAAU;YACrB1C;YACAY;YACAF,cAAc,IAAImC;YAClB7B,OAAOhB,YACJlC,KAAIkB,KAAKA,EAAEgC,QACX/C,QAAO,CAACC,KAAKoF,QAAQC,mBAAmBC,KAAKtF,KAAKoF,OAAMC,mBAAmBE;;;IAuB3E,MAAAC;QACL,IAAIxB,KAAKQ,YAAY,MAAM;YACzB,MAAM,IAAI9D,MAAM;;QAElB,OAAOmF,gCAAgChF,SAASmD,KAAKQ;;IAG/C,UAAMU,CACZlD;QAEA,MAAMU,aAAmB,IAAIiC;QAC7B;YACE,MAAM7B,QAA4BuC,mBAAmBE;YACrD,MAAMU,UAAUjC,KAAKC,OAAOiC,WAAW;gBAErCC,QAAQC,QAAQC,kBAAkB;oBAChCjK,MAAM;oBACNY,MAAMgF,SAAShF;;gBAEjB8F;;YAEF,WAAWmD,QAAQ3B,QAAQgC,aAAa,YAAY;gBAClD,MAAM,IAAIC,UAAU;;YAGtB,MAAMC,mBACKP,QAAQ3B,QAAQgC,UAAU/C,UAAUkD,YAAYlD,QAAQ;mBAC5D0C;gBACHO,WAAWxC,KAAKgC,WAAW5G;gBAC3BuB,OAAO;gBACP+F,OAAO,MAAM;gBACbC,UAAUhL;;YAEd,MAAMmK,WAAgDU,UACnD3I,QAAOiD,KAAKA,EAAE3E,SAAS,WACvByD,KAAIkB,KAAKA,EAAEC,aACXuB;YACH,OAAO;gBACLnG,MAAMX,4BAA4BE,QAAQ;oBACxC4C,UAAU0D,SAAS1D;oBACnBE,YAAYsH,SAASlG,KAAIqB,KAAKA,EAAE3B;qBAE9B,YACA;gBACJ0C;gBACA8D;gBACAhD;gBACA8D,kBAAkBJ,UACf3I,QAAOiD,KAAKA,EAAE3E,SAAS,SACvB0B,QACEiD,KAA6CA,EAAE1E,SAAS;gBAE7DsG;gBACAF,cAAc,IAAImC;;UAKtB,OAAOlE;YACL,OAAO;gBACLtE,MAAM;gBACN6F;gBACAvB;gBACAiC;gBACAF,cAAc,IAAImC;;;;;;"}
|
|
1
|
+
{"version":3,"file":"index.mjs","sources":["../src/internal/AgenticaBenchmarkPredicator.ts","../src/utils/MathUtil.ts","../src/internal/AgenticaBenchmarkUtil.ts","../src/internal/AgenticaPromptReporter.ts","../src/internal/AgenticaCallBenchmarkReporter.ts","../src/AgenticaCallBenchmark.ts","../src/internal/AgenticaSelectBenchmarkReporter.ts","../src/AgenticaSelectBenchmark.ts"],"sourcesContent":[null,null,null,null,null,null,null,null],"names":["AgenticaBenchmarkPredicator","isNext","success","async","agent","last","getHistories","at","llmVendor","getVendor","isTextPrompt","type","role","consent","functions","result","api","chat","completions","create","model","messages","content","join","text","tools","function","name","description","parameters","tool_choice","parallel_tool_calls","options","toolCall","choices","message","tool_calls","filter","tc","undefined","input","arguments","reply","props","successInner","call","expected","overrideOperations","operations","strict","take","targetIterator","items","Symbol","iterator","targeted","next","done","length","value","slice","target","operation","some","op","anyOf","callResult","allOf","map","every","r","reduce","acc","Math","max","MathUtil","round","floor","AgenticaBenchmarkUtil","errorToJson","expectedToJson","error","Error","stack","AgenticaPromptReporter","markdown","p","selections","flatMap","s","functionDescriptionCount","controller","reason","executes","e","split","JSON","stringify","AgenticaCallBenchmarkReporter","Object","fromEntries","writeIndex","experiments","exp","scenario","writeExperimentIndex","events","event","i","writeExperimentEvent","flat","average","completed_at","getTime","started_at","a","b","aggregate","usage","toLocaleString","total","cached","output","reasoning","accepted_prediction","rejected_prediction","drawStatus","select","index","prompts","count","Array","from","fill","AgenticaCallBenchmark","constructor","this","agent_","scenarios_","scenarios","config_","repeat","config","simultaneous","result_","execute","listener","Date","semaphore","Semaphore","task","Promise","all","acquire","step","release","cur","AgenticaTokenUsage","plus","zero","report","clone","out","getTokenUsage","conversate","AgenticaSelectBenchmarkReporter","selected","AgenticaSelectBenchmark","histories_","context","getContext","prompt","factory","createTextHistory","executor","TypeError","histories","orchestrate","ready","dispatch","assistantPrompts"],"mappings":";;;;;;;;AAeO,MAAMA,8BAA8B;IACzCC;IACAC;;;AAmCFC,eAAeF,OAAuCG;IACpD,MAAMC,OAA2CD,MAC9CE,eACAC,IAAG;IAQN,MAAMC,YAAYJ,MAAMK;IACxB,MAAMC,eAAeL,MAAMM,SAAS,UAAUN,KAAKO,SAAS;IAC5D,KAAKF,cAAc;QACjB,OAAO;;IAGT,MAAMG,UAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAGrCC,UAAU;IACd,MAAMC,eAAsCP,UAAUQ,IAAIC,KAAKC,YAAYC,OACzE;QACEC,OAAOZ,UAAUY;QACjBC,UAAU,EACR;YACET,MAAM;YACNU,SAAS,EACP,iCACA,IACA,uDACA,gEACA,qDACAC,KAAK;WAET;YACEX,MAAM;YACNU,SAASjB,KAAKmB;;QAGlBC,OAAO,EACL;YACEd,MAAM;YACNe,UAAU;gBACRC,MAAMd,QAAQc;gBACdC,aAAaf,QAAQe;gBACrBC,YAAYhB,QAAQgB;;;QAI1BC,aAAa;QACbC,qBAAqB;OAEvBvB,UAAUwB;IAGZ,MAAMC,YACJlB,OAAOmB,QAAQ,IAAIC,QAAQC,cAAc,IACzCC,QACAC,MAAMA,GAAG3B,SAAS,cAAc2B,GAAGZ,SAASC,SAASd,QAAQc,SAC3D;IAEJ,IAAIM,aAAaM,WAAW;QAC1B,OAAO;;IAGT,MAAMC;;;;;;;SAA0CP,SAASP,SAASe;IAClE,OAAOD,UAAU,OAAOA,MAAME,QAAQ;AACxC;;AAWM,SAAUxC,QAAwCyC;IAoBtD,OAAOC,aAAaD,OAAO5B;AAC7B;;AAEA,SAAS6B,aAA6CD;IAQpD,MAAME,OAAO,CACXC,UACAC,uBAEAH,aAAa;QACXE;QACAE,YAAYD,sBAAsBJ,MAAMK;QACxCC,QAAQN,MAAMM;;IAGlB,QAAQN,MAAMG,SAASnC;MACrB,KAAK;QAAS;YACZ,IAAIuC,OAAO;YACX,MAAMC,iBAAiBR,MAAMG,SAASM,MAAMC,OAAOC;YACnD,IAAIC,WAAWJ,eAAeK;YAE9B,OAAO,MAAM;gBACX,IAAID,SAASE,SAAS,MAAM;oBAC1B,OAAO;wBACL1C,QAAQ;wBACRmC;;;gBAGJ,IAAIA,QAAQP,MAAMK,WAAWU,QAAQ;oBACnC,OAAO;wBAAE3C,QAAQ;;;gBAGnB,MAAMA,SAAS8B,KAAKU,SAASI,OAAOhB,MAAMK,WAAWY,MAAMV;gBAC3D,KAAKnC,OAAOA,QAAQ;oBAClB,IAAI4B,MAAMM,WAAW,MAAM;wBACzB,OAAO;4BAAElC,QAAQ;;;oBAEnBmC,QAAQ;oBACR;;gBAGFA,QAAQnC,OAAOmC;gBACfK,WAAWJ,eAAeK;;;;MAG9B,KAAK;QAAc;YACjB,MAAMK,SAASlB,MAAMG,SAASgB;YAC9B,MAAM/C,SAAS4B,MAAMK,WAAWe,MAAKC,MAAMA,GAAGrC,SAASkC,OAAOlC;YAC9D,IAAIZ,QAAQ;gBACV,OAAO;oBAAEA;oBAAQmC,MAAM;;;YAEzB,OAAO;gBACLnC;;;;MAGJ,KAAK;QACH,KAAK,MAAM+B,YAAYH,MAAMG,SAASmB,OAAO;YAC3C,MAAMC,aAAarB,KAAKC;YACxB,IAAIoB,WAAWnD,QAAQ;gBACrB,OAAOmD;;;QAIX,OAAO;YAAEnD,QAAQ;;;MACnB,KAAK;QAAS;YAQZ,MAAMA,SAAS4B,MAAMG,SAASqB,MAAMC,KAAItB,YAAYD,KAAKC;YACzD,IAAI/B,OAAOsD,OAAMC,KAAKA,EAAEvD,UAAS;gBAC/B,OAAO;oBACLA,QAAQ;oBACRmC,MAAMnC,OAAOwD,QAAO,CAACC,KAAKF,MAAMG,KAAKC,IAAIF,KAAKF,EAAEpB,QAAO;;;YAI3D,OAAO;gBACLnC,QAAQ;;;;AAIhB;;AC3OO,MAAM4D,WAAW;IAOtBC,OAAQjB,SAA0Bc,KAAKI,MAAMlB,QAAQ,OAAO;;;ACJvD,MAAMmB,wBAAwB;IACnCC;IACAC;;;AAGF,SAASD,YAAeE;IAOtB,IAAIA,iBAAiBC,OAAO;QAC1B,OAAO;eACFD;YACHtD,MAAMsD,MAAMtD;YACZQ,SAAS8C,MAAM9C;YACfgD,OAAOF,MAAME;;;IAGjB,OAAOF;AACT;;AAEA,SAASD,eAA+ClC;IACtD,IAAIA,SAASnC,SAAS,cAAc;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfmD,WAAW;gBACTnC,MAAMmB,SAASgB,UAAUnC;gBACzBC,aAAakB,SAASgB,UAAUpC,SAASE;;;WAI1C,IAAIkB,SAASnC,SAAS,SAAS;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfyC,OAAON,SAASM,MAAMgB,IAAIY;;WAGzB,IAAIlC,SAASnC,SAAS,SAAS;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfwD,OAAOrB,SAASqB,MAAMC,IAAIY;;WAGzB;QACH,OAAO;YACLrE,MAAMmC,SAASnC;YACfsD,OAAOnB,SAASmB,MAAMG,IAAIY;;;AAGhC;;ACpDO,MAAMI,yBAAyB;cACpCC;;;AAGF,SAASA,WAAyCC;IAEhD,IAAIA,EAAE3E,SAAS,QAAQ;QACrB,OAAO,EAAC,aAAa2E,EAAE1E,SAAS0E,EAAE9D,MAAM,KAAID,KAAK;WAE9C,IAAI+D,EAAE3E,SAAS,YAAY2E,EAAE3E,SAAS,UAAU;QACnD,OAAO,EACL,OAAO2E,EAAE3E,SAAS,WAAW,WAAW,eACrC2E,EAAEC,WACFC,SAASC;YACR,MAAMC,2BAA2BD,EAAE3B,UAAUpC,SAASE,aAAa8B,UAAU;YAE7E,OAAO,EACL,QAAQ+B,EAAE3B,UAAUnC,QACpB,mBAAmB8D,EAAE3B,UAAU6B,WAAWhE,QAC1C,iBAAiB8D,EAAE3B,UAAUpC,SAASC,QACtC,eAAe8D,EAAEG,UACjB,OACIF,2BAA2B,IAC3B,EAACD,EAAE3B,UAAUpC,SAASE,aAAa,OACnC;AACL,cAELL,KAAK;WAEJ,IAAI+D,EAAE3E,SAAS,YAAY;QAC9B,OAAO,EACL,mBACG2E,EAAEO,SAASzB,KAAI0B,KAAK,OAAOA,EAAEhC,UAAUnC,UAC1C,OACG2D,EAAE9D,KAAKuE,MAAM,MAAM3B,KAAIqB,KAAK,KAAKA,OACpC,KACAlE,KAAK;;IAET,OAAO,EACL,eACA,aAAa+D,EAAExB,UAAUnC,QACzB,mBAAmB2D,EAAExB,UAAU6B,WAAWhE,QAC1C,iBAAiB2D,EAAExB,UAAUpC,SAASC,QACtC,IACA,WACAqE,KAAKC,UAAUX,EAAE7C,WAAW,MAAM,IAClC,OACA,KACAlB,KAAK;AACT;;ACzCO,MAAM2E,gCAAgC;cAC3Cb;;;AAGI,SAAUA,WAAyCtE;IACvD,OAAOoF,OAAOC,YAAY,EACxB,EAAC,eAAeC,aAAkBtF,cAC/BA,OAAOuF,YACPlC,KAAwBmC,OAAO,EAC9B,EAAC,KAAKA,IAAIC,SAAS7E,kBAAkB8E,uBAAqBF,WACvDA,IAAIG,OAAOtC,KAAsB,CAACuC,OAAOC,MAAM,EAChD,KAAKL,IAAIC,SAAS7E,QAAQiF,IAAI,KAAKD,MAAMhG,WACzCkG,uBAAqBF,OAAOC,WAG/BE;AAEP;;AAEA,SAAST,aAA2CtF;IAClD,MAAM2F,SAA+C3F,OAAOuF,YACzDlC,KAAIE,KAAKA,EAAEoC,SACXI;IACH,MAAMC,UACFL,OACCtC,KAAI0B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKV,OAAOhD;IACzC,MAAM2D,YAA2CtG,OAAOuG,MAAMD;IAC9D,OAAO,EACL,iCACA,cACA,oBACA,qBAAqBtG,OAAOuF,YAAY5C,OAAO6D,oBAC/C,gBAAgBb,OAAOhD,UACvB,kBAAkBgD,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC3D,kBAAkBgD,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC3D,uBAAuBiB,SAASC,MAAMmC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU7E,MAAMgF,MAAMD,oBACxC,mBAAmBF,UAAU7E,MAAMiF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,qCACA,yCACGxG,OAAOuF,YAAYlC,KAAImC,OACxB,EACE,IAAIA,IAAIC,SAAS7E,WAAW4E,IAAIC,SAAS7E,mBACzCmG,WACEvB,IAAIG,SACJZ,KAAKA,EAAEnF,SAAS,WAAWmF,EAAEiC,WAAW,QAE1CD,WAAWvB,IAAIG,SAAQZ,KAAKA,EAAEnF,SAAS,WAAWmF,EAAEjD,SAAS,QAC7D,GAAG8B,SAASC,MACV2B,IAAIG,OACDtC,KAAI0B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOhD,QAC3C6D,wBACFhG,KAAK,WAETA,KAAK;AACT;;AAEA,SAASkF,uBAAqDF;IAC5D,OAAO,EACL,KAAKA,IAAIC,SAAS7E,QAClB,cACA,mBAAmB4E,IAAIG,OAAOhD,OAAO6D,oBACrC,gBAAgBhB,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC7D,gBAAgB6C,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC7D,qBAAqBiB,SAASC,MAC5B2B,IAAIG,OACDtC,KAAI0B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOhD,QAC3C6D,uBACF,IACA,aACA,uBACA,0BACGhB,IAAIG,OAAOtC,KAAI,CAAC0B,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAEnF,YAC5BmF,EAAEnF,MACF,GAAGgE,SAASC,MAAMkB,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,kBAC1D1F,KAAK,UAET,IACA,eACA,mBACAgF,IAAIC,SAAShF,MACb,IACA,gBACA,WACAwE,KAAKC,UACHnB,sBAAsBE,eAAeuB,IAAIC,SAAS1D,WAClD,MACA,IAEF,QACAvB,KAAK;AACT;;AAEA,SAASsF,uBAAqDF,OAA2CqB;IACvG,OAAO,EACL,KAAKA,QAAQ,MAAMrB,MAAMhG,QACzB,cACA,aAAagG,MAAMH,SAAS7E,QAC5B,aAAagF,MAAMhG,QACnB,aAAagE,SAASC,MACpB+B,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,WAChDM,0BACEZ,MAAMhG,SAAS,UACf,EACE,eAAegG,MAAMoB,SAAS,MAAM,OACpC,aAAapB,MAAM9D,OAAO,MAAM,UAElC,IACJ,oBACA,gBAAgBmD,KAAKC,UAAUU,MAAMW,MAAMD,UAAUG,UACrD,eACA,kBAAkBb,MAAMW,MAAMD,UAAU7E,MAAMgF,SAC9C,mBAAmBb,MAAMW,MAAMD,UAAU7E,MAAMiF,UAC/C,iBACA,kBAAkBd,MAAMW,MAAMD,UAAUK,OAAOF,SAC/C,gCAAgCb,MAAMW,MAAMD,UAAUK,OAAOE,uBAC7D,sBAAsBjB,MAAMW,MAAMD,UAAUK,OAAOC,aACnD,gCAAgChB,MAAMW,MAAMD,UAAUK,OAAOG,uBAE7D,IACA,eACA,mBACAlB,MAAMH,SAAShF,MACf,IACA,gBACA,WACAwE,KAAKC,UACHnB,sBAAsBE,eAAe2B,MAAMH,SAAS1D,WACpD,MACA,IAEF,OACA,IACA,0BACG6D,MAAMsB,QAAQ7D,IAAIgB,uBAAuBC,WAC5C,OACIsB,MAAMhG,SAAS,UACf,EACE,YACA,WACAqF,KAAKC,UACHnB,sBAAsBC,YAAY4B,MAAM1B,QACxC,MACA,IAEF,UAEF,KACJ1D,KAAK;AACT;;AAEA,SAASuG,WAA2CpB,QAA8CxG;IAChG,MAAMgI,QAAgBzD,KAAKI,MACxB6B,OAAOrE,OAAOnC,SAASwD,SAASgD,OAAOhD,SAAU;IAGpD,OACEyE,MAAMC,KAAK;QAAE1E,QAAQwE;OAASG,KAAK,KAAK9G,KAAK,MAC3C4G,MAAMC,KAAK;QAAE1E,QAAQ,KAAKwE;OAASG,KAAK,KAAK9G,KAAK;AAExD;;MCxJa+G;IAWX,WAAAC,CAAmB5F;QACjB6F,KAAKC,SAAS9F,MAAMvC;QACpBoI,KAAKE,aAAa/F,MAAMgG,UAAU/E;QAClC4E,KAAKI,UAAU;YACbC,QAAQlG,MAAMmG,QAAQD,UAAU;YAChCE,cAAcpG,MAAMmG,QAAQC,gBAAgB;YAC5ClI,SAAS8B,MAAMmG,QAAQjI,WAAW;;QAEpC2H,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMhC,aAAmB,IAAIiC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMO,OAAOd,KAAKE,WAAWtE,KAAIjE,MAAOqG;YACtC,MAAME,eACI6C,QAAQC,IACdrB,MAAMC,KAAK;gBAAE1E,QAAQ8E,KAAKI,QAAQC;eAAUzE,KAAIjE;sBACxCiJ,UAAUK;gBAChB,MAAM3D,UACI0C,KAAKkB,KAAKlD;sBACd4C,UAAUO;gBAEhB,IAAIT,aAAa3G,WAAW;oBAC1B2G,SAASpD;;gBAGX,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAY,OAAOZ,OACJrE,QAAOyD,KAAKA,EAAEnF,SAAS,UACvByD,KAAI0B,KAAKA,EAAEwB,QACX/C,QAAO,CAACC,KAAKoF,QAAQC,mBAAmBC,KAAKtF,KAAKoF,OAAMC,mBAAmBE;;AAC/E;QAEH,MAAMzD,oBACIiD,QAAQC,IAAIF;QACtB,OAAQd,KAAKQ,UAAU;YACrB1C;YACAY;YACAF,cAAc,IAAImC;YAClB7B,OAAOhB,YACJlC,KAAIkB,KAAKA,EAAEgC,QACX/C,QAAO,CAACC,KAAKoF,QAAQC,mBAAmBC,KAAKtF,KAAKoF,OAAMC,mBAAmBE;;;IAsB3E,MAAAC;QACL,IAAIxB,KAAKQ,YAAY,MAAM;YACzB,MAAM,IAAI9D,MAAM;;QAElB,OAAOgB,8BAA8Bb,SAASmD,KAAKQ;;IAG7C,UAAMU,CACZlD;QAEA,MAAMpG,QAAyBoI,KAAKC,OAAOwB;QAC3C,MAAM/C,aAAmB,IAAIiC;QAC7B,MAAMjJ,UAAU,MACdF,4BAA4BE,QAAQ;YAClC4C,UAAU0D,SAAS1D;YACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,YACvByD,KAAIkB,KAAKA,EAAExB;YACdb,QAAQ;;QAEZ,MAAMiH,MAAM;YACV,MAAMnC,SAAS/H,4BAA4BE,QAAQ;gBACjD4C,UAAU0D,SAAS1D;gBACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,WACvByD,KAAIkB,KAAKA,EAAEC,aACXuB,OACA1C,KAAIkB,KAAKA,EAAExB;gBACdb,QAAQ;;YAEV,MAAMJ,OAAO3C;YACb,OAAO;gBACLS,MAAOkC,OAAO,YAAY;gBAC1B2D;gBACAuB;gBACAlF;gBACAoF,SAAS7H,MAAME;gBACfgH,OAAOlH,MAAM+J;gBACbjD;gBACAF,cAAc,IAAImC;;AACmC;QAGzD;kBACQ/I,MAAMgK,WAAW5D,SAAShF;YAChC,IAAItB,WAAW;gBACb,OAAOgK;;YAGT,KAAK,IAAItD,IAAY,GAAGA,IAAI4B,KAAKI,QAAQ/H,WAAW+F,GAAG;gBACrD,MAAMpD,aACIxD,4BAA4BC,OAAOG;gBAC7C,IAAIoD,SAAS,MAAM;oBACjB;;sBAGIpD,MAAMgK,WAAW5G;gBACvB,IAAItD,WAAW;oBACb,OAAOgK;;;YAGX,OAAOA;UAET,OAAOjF;YACL,OAAO;gBACLtE,MAAM;gBACN6F;gBACAyB,SAAS7H,MAAME;gBACfgH,OAAOlH,MAAM+J;gBACblF;gBACAiC;gBACAF,cAAc,IAAImC;;;;;;AChMnB,MAAMkB,kCAAkC;IAC7ChF;;;AAGI,SAAUA,SAAyCtE;IACvD,MAAMuC,WAAW,EACf,EAAC,eAAe+C,WAAWtF,cACxBA,OAAOuF,YACPlC,KAAwBmC,OAAO,EAC9B,EAAC,KAAKA,IAAIC,SAAS7E,kBAAkB8E,qBAAqBF,WACvDA,IAAIG,OAAOtC,KAAsB,CAACuC,OAAOC,MAAM,EAChD,KAAKL,IAAIC,SAAS7E,QAAQiF,IAAI,KAAKD,MAAMhG,WACzCkG,qBAAqBF,OAAOC,WAG/BE;IAGL,OAAOX,OAAOC,YAAY9C;AAC5B;;AAEA,SAAS+C,WAA2CtF;IAClD,MAAM2F,SAAiD3F,OAAOuF,YAC3DlC,KAAIE,KAAKA,EAAEoC,SACXI;IACH,MAAMC,UACAL,OACCtC,KAAI0B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKV,OAAOhD;IAC3C,MAAM2D,YAA2CtG,OAAOuG,MAAMD;IAC9D,OAAO,EACL,sCACA,cACA,oBACA,qBAAqBtG,OAAOuF,YAAY5C,OAAO6D,oBAC/C,gBAAgBb,OAAOhD,UACvB,kBAAkBgD,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC3D,kBAAkBgD,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC3D,gBAAgBgD,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,UAAS+C,UACvD,uBAAuBiB,SAASC,MAAMmC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU7E,MAAMgF,MAAMD,oBACxC,mBAAmBF,UAAU7E,MAAMiF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,+BACA,kCACGxG,OAAOuF,YAAYlC,KAAImC,OACxB,EACE,IAAIA,IAAIC,SAAS7E,WAAW4E,IAAIC,SAAS7E,mBACzC;QACE,MAAMzB,UAAkBuE,KAAKI,MAC1B0B,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,SAC1C6C,IAAIG,OAAOhD,SACb;QAEJ,OACEyE,MAAMC,KAAK;YAAE1E,QAAQxD;WAAWmI,KAAK,KAAK9G,KAAK,MAC7C4G,MAAMC,KAAK;YAAE1E,QAAQ,KAAKxD;WAAWmI,KAAK,KAAK9G,KAAK;AAEzD,MAVD,IAWA,GAAGoD,SAASC,MACV2B,IAAIG,OACDtC,KACCuC,SACEA,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,YAEnD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOhD,QAC3C6D,wBACFhG,KAAK,WAETA,KAAK;AACT;;AAEA,SAASkF,qBAAqDF;IAC5D,MAAMc,YAA2Cd,IAAIe,MAAMD;IAC3D,OAAO,EACL,KAAKd,IAAIC,SAAS7E,QAClB,cACA,oBACA,gBAAgB4E,IAAIG,OAAOhD,UAC3B,kBAAkB6C,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC/D,kBAAkB6C,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,YAAW+C,UAC/D,gBAAgB6C,IAAIG,OAAOrE,QAAOyD,KAAKA,EAAEnF,SAAS,UAAS+C,UAC3D,uBAAuBiB,SAASC,MAC9B2B,IAAIG,OACDtC,KACCuC,SACEA,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,YAEnD1C,QAAO,CAAC4C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOhD,QAC3C6D,uBACF,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU7E,MAAMgF,MAAMD,oBACxC,mBAAmBF,UAAU7E,MAAMiF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,aACA,qBACA,wBACGhB,IAAIG,OAAOtC,KAAI,CAAC0B,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAEnF,YAC5BmF,EAAEnF,MACF,GAAGgE,SAASC,MAAMkB,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,kBAE1D1F,KAAK,UAET,IACA,eACA,mBACAgF,IAAIC,SAAShF,MACb,IACA,gBACA,WACAwE,KAAKC,UACHnB,sBAAsBE,eAAeuB,IAAIC,SAAS1D,WAClD,MACA,IAEF,QACAvB,KAAK;AACT;;AAEA,SAASsF,qBAAqDF,OAA6CqB;IACzG,OAAO,EACL,KAAKA,QAAQ,MAAMrB,MAAMhG,QACzB,cACA,aAAagG,MAAMH,SAAS7E,QAC5B,aAAagF,MAAMhG,QACnB,cAAcgG,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,WAAWM,0BACrEZ,MAAMhG,SAAS,UACf,EACE,mBACA,gBAAgBgG,MAAMW,MAAMD,UAAUG,MAAMD,oBAC5C,gBACA,kBAAkBZ,MAAMW,MAAMD,UAAU7E,MAAMgF,MAAMD,oBACpD,mBAAmBZ,MAAMW,MAAMD,UAAU7E,MAAMiF,OAAOF,oBACtD,qBACA,kBAAkBZ,MAAMW,MAAMD,UAAUK,OAAOF,MAAMD,oBACrD,sBAAsBZ,MAAMW,MAAMD,UAAUK,OAAOC,UAAUJ,oBAC7D,gCAAgCZ,MAAMW,MAAMD,UAAUK,OAAOE,oBAAoBL,oBACjF,gCAAgCZ,MAAMW,MAAMD,UAAUK,OAAOG,oBAAoBN,uBAEnF,IACJ,IACA,eACA,mBACAZ,MAAMH,SAAShF,MACf,IACA,gBACA,WACAwE,KAAKC,UACHnB,sBAAsBE,eAAe2B,MAAMH,SAAS1D,WACpD,MACA,IAEF,OACA,OACI6D,MAAMhG,SAAS,aAAagG,MAAMhG,SAAS,YAC3C,EACE,gBACGgG,MAAM2D,SAASlG,KAAIqB,KACpB,EACE,OAAOA,EAAE3B,UAAUnC,QACnB,qBAAqB8D,EAAE3B,UAAU6B,WAAWhE,UAC5C,mBAAmB8D,EAAE3B,UAAUpC,SAASC,UACxC,eAAe8D,EAAEG,UACjB,OACIH,EAAE3B,UAAUpC,SAASE,gBAAgBW,aAAakD,EAAE3B,UAAUpC,SAASE,gBAAgB,KACvF,EAAC6D,EAAE3B,UAAUpC,SAASE,aAAa,OACnC,KACJL,KAAK,YAGX,OACAoF,MAAMhG,SAAS,UACf,EACE,YACA,WACAmE,sBAAsBC,YACpBiB,KAAKC,UAAUU,MAAM1B,OAAO,MAAM,KAEpC,OACA,OAEF,KACJ1D,KAAK;AACT;;MCjLagJ;IAYX,WAAAhC,CAAmB5F;QACjB6F,KAAKC,SAAS9F,MAAMvC;QACpBoI,KAAKE,aAAa/F,MAAMgG,UAAU/E;QAClC4E,KAAKI,UAAU;YACbC,QAAQlG,MAAMmG,QAAQD,UAAU;YAChCE,cAAcpG,MAAMmG,QAAQC,gBAAgB;;QAE9CP,KAAKgC,aAAa7H,MAAMvC,MAAME,eAAesD;QAC7C4E,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMhC,aAAmB,IAAIiC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMzC,oBACIiD,QAAQC,IACdhB,KAAKE,WAAWtE,KAAIjE,MAAOqG;YACzB,MAAME,eACI6C,QAAQC,IACdrB,MAAMC,KAAK;gBAAE1E,QAAQ8E,KAAKI,QAAQC;eAAUzE,KAAIjE;sBACxCiJ,UAAUK;gBAChB,MAAM3D,UACI0C,KAAKkB,KAAKlD;sBACd4C,UAAUO;gBAChB,IAAIT,aAAa3G,WAAW;oBAC1B2G,SAASpD;;gBAEX,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAY,OAAOZ,OACJrE,QAAOyD,KAAKA,EAAEnF,SAAS,UACvByD,KAAI0B,KAAKA,EAAEwB,QACX/C,QAAO,CAACC,KAAKoF,QAAQC,mBAAmBC,KAAKtF,KAAKoF,OAAMC,mBAAmBE;;AAC/E;QAGP,OAAQvB,KAAKQ,UAAU;YACrB1C;YACAY;YACAF,cAAc,IAAImC;YAClB7B,OAAOhB,YACJlC,KAAIkB,KAAKA,EAAEgC,QACX/C,QAAO,CAACC,KAAKoF,QAAQC,mBAAmBC,KAAKtF,KAAKoF,OAAMC,mBAAmBE;;;IAuB3E,MAAAC;QACL,IAAIxB,KAAKQ,YAAY,MAAM;YACzB,MAAM,IAAI9D,MAAM;;QAElB,OAAOmF,gCAAgChF,SAASmD,KAAKQ;;IAG/C,UAAMU,CACZlD;QAEA,MAAMU,aAAmB,IAAIiC;QAC7B;YACE,MAAM7B,QAA4BuC,mBAAmBE;YACrD,MAAMU,UAAUjC,KAAKC,OAAOiC,WAAW;gBAErCC,QAAQC,QAAQC,kBAAkB;oBAChCjK,MAAM;oBACNY,MAAMgF,SAAShF;;gBAEjB8F;;YAEF,WAAWmD,QAAQ3B,QAAQgC,aAAa,YAAY;gBAClD,MAAM,IAAIC,UAAU;;YAGtB,MAAMC,mBACKP,QAAQ3B,QAAQgC,UAAU/C,UAAUkD,YAAYlD,QAAQ;mBAC5D0C;gBACHO,WAAWxC,KAAKgC,WAAW5G;gBAC3BuB,OAAO;gBACP+F,OAAO,MAAM;gBACbC,UAAUhL;;YAEd,MAAMmK,WAAgDU,UACnD3I,QAAOiD,KAAKA,EAAE3E,SAAS,WACvByD,KAAIkB,KAAKA,EAAEC,aACXuB;YACH,OAAO;gBACLnG,MAAMX,4BAA4BE,QAAQ;oBACxC4C,UAAU0D,SAAS1D;oBACnBE,YAAYsH,SAASlG,KAAIqB,KAAKA,EAAE3B;qBAE9B,YACA;gBACJ0C;gBACA8D;gBACAhD;gBACA8D,kBAAkBJ,UACf3I,QAAOiD,KAAKA,EAAE3E,SAAS,SACvB0B,QACEiD,KAA6CA,EAAE1E,SAAS;gBAE7DsG;gBACAF,cAAc,IAAImC;;UAKtB,OAAOlE;YACL,OAAO;gBACLtE,MAAM;gBACN6F;gBACAvB;gBACAiC;gBACAF,cAAc,IAAImC;;;;;;"}
|
|
@@ -41,6 +41,7 @@ function writeIndex(result) {
|
|
|
41
41
|
` - Trial: ${events.length}`,
|
|
42
42
|
` - Success: ${events.filter(e => e.type === "success").length}`,
|
|
43
43
|
` - Failure: ${events.filter(e => e.type === "failure").length}`,
|
|
44
|
+
` - Error: ${events.filter(e => e.type === "error").length}`,
|
|
44
45
|
` - Average Time: ${MathUtil_1.MathUtil.round(average).toLocaleString()} ms`,
|
|
45
46
|
` - Token Usage`,
|
|
46
47
|
` - Total: ${aggregate.total.toLocaleString()}`,
|
|
@@ -80,6 +81,7 @@ function writeExperimentIndex(exp) {
|
|
|
80
81
|
` - Trial: ${exp.events.length}`,
|
|
81
82
|
` - Success: ${exp.events.filter(e => e.type === "success").length}`,
|
|
82
83
|
` - Failure: ${exp.events.filter(e => e.type === "failure").length}`,
|
|
84
|
+
` - Error: ${exp.events.filter(e => e.type === "error").length}`,
|
|
83
85
|
` - Average Time: ${MathUtil_1.MathUtil.round(exp.events
|
|
84
86
|
.map(event => event.completed_at.getTime() - event.started_at.getTime())
|
|
85
87
|
.reduce((a, b) => a + b, 0) / exp.events.length).toLocaleString()} ms`,
|
|
@@ -123,7 +125,7 @@ function writeExperimentEvent(event, index) {
|
|
|
123
125
|
...(event.type !== "error"
|
|
124
126
|
? [
|
|
125
127
|
" - Token Usage",
|
|
126
|
-
` - Total: ${event.usage.aggregate.toLocaleString()}`,
|
|
128
|
+
` - Total: ${event.usage.aggregate.total.toLocaleString()}`,
|
|
127
129
|
` - Prompt`,
|
|
128
130
|
` - Total: ${event.usage.aggregate.input.total.toLocaleString()}`,
|
|
129
131
|
` - Cached: ${event.usage.aggregate.input.cached.toLocaleString()}`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgenticaSelectBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaSelectBenchmarkReporter.ts"],"names":[],"mappings":";;;AAuBA,4BAeC;AA1BD,gDAA6C;AAE7C,mEAAgE;AAEhE;;GAEG;AACU,QAAA,+BAA+B,GAAG;IAC7C,QAAQ;CACT,CAAC;AAEF,SAAgB,QAAQ,CAAiC,MAA6C;IACpG,MAAM,QAAQ,GAAG;QACf,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACnC,GAAG,MAAM,CAAC,WAAW;aAClB,GAAG,CAAqB,GAAG,CAAC,EAAE,CAAC;YAC9B,CAAC,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,YAAY,EAAE,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC/D,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAmB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK;gBAClD,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC;aAC/B,CAAC;SACH,CAAC;aACD,IAAI,EAAE;KACmB,CAAC;IAE/B,OAAO,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,UAAU,CAAiC,MAA6C;IAC/F,MAAM,MAAM,GAA2C,MAAM,CAAC,WAAW;SACtE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;SAClB,IAAI,EAAE,CAAC;IACV,MAAM,OAAO,GACP,MAAM;SACL,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;SAC3D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAClD,MAAM,SAAS,GAAkC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;IACxE,OAAO;QACL,oCAAoC;QACpC,YAAY;QACZ,kBAAkB;QAClB,qBAAqB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;QACjE,gBAAgB,MAAM,CAAC,MAAM,EAAE;QAC/B,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACnE,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACnE,uBAAuB,mBAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE,KAAK;QACpE,iBAAiB;QACjB,gBAAgB,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAClD,aAAa;QACb,kBAAkB,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC1D,mBAAmB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;QAC5D,eAAe;QACf,kBAAkB,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC3D,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,sBAAsB,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;QACnE,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,EAAE;QACF,gBAAgB;QAChB,6BAA6B;QAC7B,6BAA6B;QAC7B,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAC9B;YACE,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC,IAAI,aAAa;YAC1D,CAAC,GAAG,EAAE;gBACJ,MAAM,OAAO,GAAW,IAAI,CAAC,KAAK,CAChC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM;sBAChD,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;sBACpB,EAAE,CACL,CAAC;gBACF,OAAO,CACL,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;sBAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAC1D,CAAC;YACJ,CAAC,CAAC,EAAE;YACJ,GAAG,mBAAQ,CAAC,KAAK,CACf,GAAG,CAAC,MAAM;iBACP,GAAG,CACF,KAAK,CAAC,EAAE,CACN,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC5D;iBACA,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;SACxB,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;KACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB,CAAiC,GAAsD;IAClH,MAAM,SAAS,GAAkC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC;IACrE,OAAO;QACL,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE;QACxB,YAAY;QACZ,kBAAkB;QAClB,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE;QACnC,kBAAkB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACvE,kBAAkB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACvE,uBAAuB,mBAAQ,CAAC,KAAK,CACnC,GAAG,CAAC,MAAM;aACP,GAAG,CACF,KAAK,CAAC,EAAE,CACN,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC5D;aACA,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;QACvB,iBAAiB;QACjB,gBAAgB,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAClD,aAAa;QACb,kBAAkB,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC1D,mBAAmB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;QAC5D,eAAe;QACf,kBAAkB,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC3D,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,sBAAsB,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;QACnE,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,EAAE;QACF,WAAW;QACX,mBAAmB;QACnB,mBAAmB;QACnB,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB;YACE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM;YACtC,CAAC,CAAC,IAAI;YACN,GAAG,mBAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CACnE,KAAK;SACN,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;QACD,EAAE;QACF,aAAa;QACb,iBAAiB;QACjB,GAAG,CAAC,QAAQ,CAAC,IAAI;QACjB,EAAE;QACF,cAAc;QACd,SAAS;QACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3D,IAAI,EACJ,CAAC,CACF;QACD,KAAK;KACN,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB,CAAiC,KAA2C,EAAE,KAAa;IACtH,OAAO;QACL,KAAK,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,EAAE;QAC/B,YAAY;QACZ,aAAa,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE;QAClC,aAAa,KAAK,CAAC,IAAI,EAAE;QACzB,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,cAAc,EAAE,KAAK;QAC9F,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;YACxB,CAAC,CAAC;gBACE,iBAAiB;gBACjB,gBAAgB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;
|
|
1
|
+
{"version":3,"file":"AgenticaSelectBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaSelectBenchmarkReporter.ts"],"names":[],"mappings":";;;AAuBA,4BAeC;AA1BD,gDAA6C;AAE7C,mEAAgE;AAEhE;;GAEG;AACU,QAAA,+BAA+B,GAAG;IAC7C,QAAQ;CACT,CAAC;AAEF,SAAgB,QAAQ,CAAiC,MAA6C;IACpG,MAAM,QAAQ,GAAG;QACf,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACnC,GAAG,MAAM,CAAC,WAAW;aAClB,GAAG,CAAqB,GAAG,CAAC,EAAE,CAAC;YAC9B,CAAC,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,YAAY,EAAE,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC/D,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAmB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK;gBAClD,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC;aAC/B,CAAC;SACH,CAAC;aACD,IAAI,EAAE;KACmB,CAAC;IAE/B,OAAO,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,UAAU,CAAiC,MAA6C;IAC/F,MAAM,MAAM,GAA2C,MAAM,CAAC,WAAW;SACtE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;SAClB,IAAI,EAAE,CAAC;IACV,MAAM,OAAO,GACP,MAAM;SACL,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;SAC3D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAClD,MAAM,SAAS,GAAkC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;IACxE,OAAO;QACL,oCAAoC;QACpC,YAAY;QACZ,kBAAkB;QAClB,qBAAqB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;QACjE,gBAAgB,MAAM,CAAC,MAAM,EAAE;QAC/B,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACnE,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACnE,gBAAgB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE;QAC/D,uBAAuB,mBAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE,KAAK;QACpE,iBAAiB;QACjB,gBAAgB,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAClD,aAAa;QACb,kBAAkB,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC1D,mBAAmB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;QAC5D,eAAe;QACf,kBAAkB,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC3D,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,sBAAsB,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;QACnE,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,EAAE;QACF,gBAAgB;QAChB,6BAA6B;QAC7B,6BAA6B;QAC7B,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAC9B;YACE,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC,IAAI,aAAa;YAC1D,CAAC,GAAG,EAAE;gBACJ,MAAM,OAAO,GAAW,IAAI,CAAC,KAAK,CAChC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM;sBAChD,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;sBACpB,EAAE,CACL,CAAC;gBACF,OAAO,CACL,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;sBAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAC1D,CAAC;YACJ,CAAC,CAAC,EAAE;YACJ,GAAG,mBAAQ,CAAC,KAAK,CACf,GAAG,CAAC,MAAM;iBACP,GAAG,CACF,KAAK,CAAC,EAAE,CACN,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC5D;iBACA,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;SACxB,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;KACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB,CAAiC,GAAsD;IAClH,MAAM,SAAS,GAAkC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC;IACrE,OAAO;QACL,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE;QACxB,YAAY;QACZ,kBAAkB;QAClB,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE;QACnC,kBAAkB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACvE,kBAAkB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QACvE,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE;QACnE,uBAAuB,mBAAQ,CAAC,KAAK,CACnC,GAAG,CAAC,MAAM;aACP,GAAG,CACF,KAAK,CAAC,EAAE,CACN,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC5D;aACA,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;QACvB,iBAAiB;QACjB,gBAAgB,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAClD,aAAa;QACb,kBAAkB,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC1D,mBAAmB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;QAC5D,eAAe;QACf,kBAAkB,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;QAC3D,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,sBAAsB,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;QACnE,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;QACvF,EAAE;QACF,WAAW;QACX,mBAAmB;QACnB,mBAAmB;QACnB,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB;YACE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM;YACtC,CAAC,CAAC,IAAI;YACN,GAAG,mBAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CACnE,KAAK;SACN,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;QACD,EAAE;QACF,aAAa;QACb,iBAAiB;QACjB,GAAG,CAAC,QAAQ,CAAC,IAAI;QACjB,EAAE;QACF,cAAc;QACd,SAAS;QACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3D,IAAI,EACJ,CAAC,CACF;QACD,KAAK;KACN,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB,CAAiC,KAA2C,EAAE,KAAa;IACtH,OAAO;QACL,KAAK,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,EAAE;QAC/B,YAAY;QACZ,aAAa,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE;QAClC,aAAa,KAAK,CAAC,IAAI,EAAE;QACzB,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,cAAc,EAAE,KAAK;QAC9F,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;YACxB,CAAC,CAAC;gBACE,iBAAiB;gBACjB,gBAAgB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;gBAC9D,cAAc;gBACd,kBAAkB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;gBACtE,mBAAmB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;gBACxE,mBAAmB;gBACnB,kBAAkB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;gBACvE,sBAAsB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;gBAC/E,gCAAgC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;gBACnG,gCAAgC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;aACpG;YACH,CAAC,CAAC,EAAE,CAAC;QACP,EAAE;QACF,aAAa;QACb,iBAAiB;QACjB,KAAK,CAAC,QAAQ,CAAC,IAAI;QACnB,EAAE;QACF,cAAc;QACd,SAAS;QACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC7D,IAAI,EACJ,CAAC,CACF;QACD,KAAK;QACL,EAAE;QACF,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;YACtD,CAAC,CAAC;gBACE,WAAW;gBACX,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CACxB;oBACE,OAAO,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE;oBACzB,qBAAqB,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,IAAI;oBACpD,mBAAmB,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,IAAI;oBAChD,eAAe,CAAC,CAAC,MAAM,EAAE;oBACzB,EAAE;oBACF,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,KAAK,EAAE;wBAC3F,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;wBACxC,CAAC,CAAC,EAAE,CAAC;iBACR,CAAC,IAAI,CAAC,IAAI,CAAC,CACb;aACF;YACH,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;YACxB,CAAC,CAAC;gBACE,UAAU;gBACV,SAAS;gBACT,6CAAqB,CAAC,WAAW,CAC/B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CACrC;gBACD,KAAK;gBACL,EAAE;aACH;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentica/benchmark",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.21.0",
|
|
4
4
|
"description": "Agentic AI Library specialized in LLM Function Calling",
|
|
5
5
|
"author": "Wrtn Technologies",
|
|
6
6
|
"license": "MIT",
|
|
@@ -35,14 +35,14 @@
|
|
|
35
35
|
"src"
|
|
36
36
|
],
|
|
37
37
|
"peerDependencies": {
|
|
38
|
-
"@agentica/core": "^0.
|
|
38
|
+
"@agentica/core": "^0.21.0"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@samchon/openapi": "^4.
|
|
41
|
+
"@samchon/openapi": "^4.2.0",
|
|
42
42
|
"openai": "^4.80.0",
|
|
43
43
|
"tstl": "^3.0.0",
|
|
44
44
|
"typia": "^9.0.1",
|
|
45
|
-
"@agentica/core": "^0.
|
|
45
|
+
"@agentica/core": "^0.21.0"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
48
|
"@rollup/plugin-terser": "^0.4.4",
|
|
@@ -55,6 +55,7 @@ function writeIndex<Model extends ILlmSchema.Model>(result: IAgenticaSelectBench
|
|
|
55
55
|
` - Trial: ${events.length}`,
|
|
56
56
|
` - Success: ${events.filter(e => e.type === "success").length}`,
|
|
57
57
|
` - Failure: ${events.filter(e => e.type === "failure").length}`,
|
|
58
|
+
` - Error: ${events.filter(e => e.type === "error").length}`,
|
|
58
59
|
` - Average Time: ${MathUtil.round(average).toLocaleString()} ms`,
|
|
59
60
|
` - Token Usage`,
|
|
60
61
|
` - Total: ${aggregate.total.toLocaleString()}`,
|
|
@@ -106,6 +107,7 @@ function writeExperimentIndex<Model extends ILlmSchema.Model>(exp: IAgenticaSele
|
|
|
106
107
|
` - Trial: ${exp.events.length}`,
|
|
107
108
|
` - Success: ${exp.events.filter(e => e.type === "success").length}`,
|
|
108
109
|
` - Failure: ${exp.events.filter(e => e.type === "failure").length}`,
|
|
110
|
+
` - Error: ${exp.events.filter(e => e.type === "error").length}`,
|
|
109
111
|
` - Average Time: ${MathUtil.round(
|
|
110
112
|
exp.events
|
|
111
113
|
.map(
|
|
@@ -162,7 +164,7 @@ function writeExperimentEvent<Model extends ILlmSchema.Model>(event: IAgenticaSe
|
|
|
162
164
|
...(event.type !== "error"
|
|
163
165
|
? [
|
|
164
166
|
" - Token Usage",
|
|
165
|
-
` - Total: ${event.usage.aggregate.toLocaleString()}`,
|
|
167
|
+
` - Total: ${event.usage.aggregate.total.toLocaleString()}`,
|
|
166
168
|
` - Prompt`,
|
|
167
169
|
` - Total: ${event.usage.aggregate.input.total.toLocaleString()}`,
|
|
168
170
|
` - Cached: ${event.usage.aggregate.input.cached.toLocaleString()}`,
|