@agentica/benchmark 0.10.4 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/AgenticaCallBenchmark.js +8 -6
- package/lib/AgenticaCallBenchmark.js.map +1 -1
- package/lib/AgenticaSelectBenchmark.js +8 -9
- package/lib/AgenticaSelectBenchmark.js.map +1 -1
- package/lib/index.mjs +15 -16
- package/lib/index.mjs.map +1 -1
- package/lib/internal/AgenticaBenchmarkPredicator.d.ts +2 -2
- package/lib/internal/AgenticaBenchmarkPredicator.js.map +1 -1
- package/lib/internal/AgenticaCallBenchmarkReporter.js.map +1 -1
- package/lib/internal/AgenticaPromptReporter.d.ts +2 -2
- package/lib/internal/AgenticaPromptReporter.js +12 -12
- package/lib/internal/AgenticaPromptReporter.js.map +1 -1
- package/lib/internal/AgenticaSelectBenchmarkReporter.js +6 -4
- package/lib/internal/AgenticaSelectBenchmarkReporter.js.map +1 -1
- package/lib/structures/IAgenticaBenchmarkExpected.d.ts +2 -2
- package/lib/structures/IAgenticaCallBenchmarkEvent.d.ts +3 -3
- package/lib/structures/IAgenticaCallBenchmarkResult.d.ts +3 -3
- package/lib/structures/IAgenticaSelectBenchmarkEvent.d.ts +7 -7
- package/lib/structures/IAgenticaSelectBenchmarkResult.d.ts +3 -3
- package/package.json +2 -2
- package/src/AgenticaCallBenchmark.ts +8 -13
- package/src/AgenticaSelectBenchmark.ts +17 -25
- package/src/internal/AgenticaBenchmarkPredicator.ts +4 -8
- package/src/internal/AgenticaCallBenchmarkReporter.ts +2 -2
- package/src/internal/AgenticaPromptReporter.ts +14 -14
- package/src/internal/AgenticaSelectBenchmarkReporter.ts +9 -7
- package/src/structures/IAgenticaBenchmarkExpected.ts +2 -2
- package/src/structures/IAgenticaCallBenchmarkEvent.ts +3 -3
- package/src/structures/IAgenticaCallBenchmarkResult.ts +3 -3
- package/src/structures/IAgenticaSelectBenchmarkEvent.ts +9 -9
- package/src/structures/IAgenticaSelectBenchmarkResult.ts +3 -3
|
@@ -10,7 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.AgenticaCallBenchmark = void 0;
|
|
13
|
-
const
|
|
13
|
+
const core_1 = require("@agentica/core");
|
|
14
14
|
const tstl_1 = require("tstl");
|
|
15
15
|
const AgenticaBenchmarkPredicator_1 = require("./internal/AgenticaBenchmarkPredicator");
|
|
16
16
|
const AgenticaCallBenchmarkReporter_1 = require("./internal/AgenticaCallBenchmarkReporter");
|
|
@@ -86,7 +86,7 @@ class AgenticaCallBenchmark {
|
|
|
86
86
|
usage: events
|
|
87
87
|
.filter((e) => e.type !== "error")
|
|
88
88
|
.map((e) => e.usage)
|
|
89
|
-
.reduce(
|
|
89
|
+
.reduce(core_1.AgenticaTokenUsage.plus, core_1.AgenticaTokenUsage.zero()),
|
|
90
90
|
};
|
|
91
91
|
})));
|
|
92
92
|
return (this.result_ = {
|
|
@@ -95,7 +95,7 @@ class AgenticaCallBenchmark {
|
|
|
95
95
|
completed_at: new Date(),
|
|
96
96
|
usage: experiments
|
|
97
97
|
.map((p) => p.usage)
|
|
98
|
-
.reduce(
|
|
98
|
+
.reduce(core_1.AgenticaTokenUsage.plus, core_1.AgenticaTokenUsage.zero()),
|
|
99
99
|
});
|
|
100
100
|
});
|
|
101
101
|
}
|
|
@@ -130,7 +130,8 @@ class AgenticaCallBenchmark {
|
|
|
130
130
|
expected: scenario.expected,
|
|
131
131
|
operations: agent
|
|
132
132
|
.getPromptHistories()
|
|
133
|
-
.filter((p) => p.type === "execute")
|
|
133
|
+
.filter((p) => p.type === "execute")
|
|
134
|
+
.map((p) => p.operation),
|
|
134
135
|
strict: false,
|
|
135
136
|
});
|
|
136
137
|
const out = () => {
|
|
@@ -139,8 +140,9 @@ class AgenticaCallBenchmark {
|
|
|
139
140
|
operations: agent
|
|
140
141
|
.getPromptHistories()
|
|
141
142
|
.filter((p) => p.type === "select")
|
|
142
|
-
.map((p) => p.
|
|
143
|
-
.flat()
|
|
143
|
+
.map((p) => p.selections)
|
|
144
|
+
.flat()
|
|
145
|
+
.map((p) => p.operation),
|
|
144
146
|
strict: false,
|
|
145
147
|
});
|
|
146
148
|
const call = success();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgenticaCallBenchmark.js","sourceRoot":"","sources":["../src/AgenticaCallBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"AgenticaCallBenchmark.js","sourceRoot":"","sources":["../src/AgenticaCallBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,yCAA8D;AAE9D,+BAAiC;AAGjC,wFAAqF;AACrF,4FAAyF;AAKzF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAa,qBAAqB;IAMhC;;;;OAIG;IACH,YAAmB,KAA0C;;QAC3D,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG;YACb,MAAM,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,MAAM,mCAAI,EAAE;YAClC,YAAY,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,YAAY,mCAAI,EAAE;YAC9C,OAAO,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,OAAO,mCAAI,CAAC;SACpC,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACU,OAAO,CAClB,QAA8D;;YAE9D,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,MAAM,SAAS,GAAc,IAAI,gBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACtE,MAAM,WAAW,GACf,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAO,QAAQ,EAAE,EAAE;gBACrC,MAAM,MAAM,GACV,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAS,EAAE;oBACpD,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC1B,MAAM,CAAC,GACL,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC5B,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC1B,IAAI,QAAQ,KAAK,SAAS;wBAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;oBACxC,OAAO,CAAC,CAAC;gBACX,CAAC,CAAA,CAAC,CACH,CAAC;gBACJ,OAAO;oBACL,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,MAAM;yBACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;yBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;yBACnB,MAAM,CAAC,yBAAkB,CAAC,IAAI,EAAE,yBAAkB,CAAC,IAAI,EAAE,CAAC;iBAC9D,CAAC;YACJ,CAAC,CAAA,CAAC,CACH,CAAC;YACJ,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG;gBACrB,WAAW;gBACX,UAAU;gBACV,YAAY,EAAE,IAAI,IAAI,EAAE;gBACxB,KAAK,EAAE,WAAW;qBACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;qBACnB,MAAM,CAAC,yBAAkB,CAAC,IAAI,EAAE,yBAAkB,CAAC,IAAI,EAAE,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;KAAA;IAED;;;;;;;;;;;;;;;;;OAiBG;IACI,MAAM;QACX,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,OAAO,6DAA6B,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9D,CAAC;IAEa,IAAI,CAChB,QAA+C;;YAE/C,MAAM,KAAK,GAAoB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACnD,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB,yDAA2B,CAAC,OAAO,CAAC;gBAClC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,UAAU,EAAE,KAAK;qBACd,kBAAkB,EAAE;qBACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;qBACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC1B,MAAM,EAAE,KAAK;aACd,CAAC,CAAC;YACL,MAAM,GAAG,GAAG,GAAuC,EAAE;gBACnD,MAAM,MAAM,GAAG,yDAA2B,CAAC,OAAO,CAAC;oBACjD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,UAAU,EAAE,KAAK;yBACd,kBAAkB,EAAE;yBACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;yBAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;yBACxB,IAAI,EAAE;yBACN,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;oBAC1B,MAAM,EAAE,KAAK;iBACd,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;gBACvB,OAAO;oBACL,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAc;oBACjD,QAAQ;oBACR,MAAM;oBACN,IAAI;oBACJ,OAAO,EAAE,KAAK,CAAC,kBAAkB,EAAE;oBACnC,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE;oBAC5B,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBAC6B,CAAC;YAC1D,CAAC,CAAC;YAEF,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACtC,IAAI,OAAO,EAAE;oBAAE,OAAO,GAAG,EAAE,CAAC;gBAC5B,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;oBACtD,MAAM,IAAI,GACR,MAAM,yDAA2B,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAClD,IAAI,IAAI,KAAK,IAAI;wBAAE,MAAM;oBAEzB,MAAM,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;oBAC7B,IAAI,OAAO,EAAE;wBAAE,OAAO,GAAG,EAAE,CAAC;gBAC9B,CAAC;gBACD,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,QAAQ;oBACR,OAAO,EAAE,KAAK,CAAC,kBAAkB,EAAE;oBACnC,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE;oBAC5B,KAAK;oBACL,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBACzB,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;CACF;AAnKD,sDAmKC"}
|
|
@@ -10,8 +10,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.AgenticaSelectBenchmark = void 0;
|
|
13
|
+
const core_1 = require("@agentica/core");
|
|
13
14
|
const ChatGptSelectFunctionAgent_1 = require("@agentica/core/src/chatgpt/ChatGptSelectFunctionAgent");
|
|
14
|
-
const AgenticaTokenUsageAggregator_1 = require("@agentica/core/src/internal/AgenticaTokenUsageAggregator");
|
|
15
15
|
const tstl_1 = require("tstl");
|
|
16
16
|
const AgenticaBenchmarkPredicator_1 = require("./internal/AgenticaBenchmarkPredicator");
|
|
17
17
|
const AgenticaSelectBenchmarkReporter_1 = require("./internal/AgenticaSelectBenchmarkReporter");
|
|
@@ -84,7 +84,7 @@ class AgenticaSelectBenchmark {
|
|
|
84
84
|
usage: events
|
|
85
85
|
.filter((e) => e.type !== "error")
|
|
86
86
|
.map((e) => e.usage)
|
|
87
|
-
.reduce(
|
|
87
|
+
.reduce(core_1.AgenticaTokenUsage.plus, core_1.AgenticaTokenUsage.zero()),
|
|
88
88
|
};
|
|
89
89
|
})));
|
|
90
90
|
return (this.result_ = {
|
|
@@ -93,7 +93,7 @@ class AgenticaSelectBenchmark {
|
|
|
93
93
|
completed_at: new Date(),
|
|
94
94
|
usage: experiments
|
|
95
95
|
.map((p) => p.usage)
|
|
96
|
-
.reduce(
|
|
96
|
+
.reduce(core_1.AgenticaTokenUsage.plus, core_1.AgenticaTokenUsage.zero()),
|
|
97
97
|
});
|
|
98
98
|
});
|
|
99
99
|
}
|
|
@@ -125,23 +125,22 @@ class AgenticaSelectBenchmark {
|
|
|
125
125
|
return __awaiter(this, void 0, void 0, function* () {
|
|
126
126
|
const started_at = new Date();
|
|
127
127
|
try {
|
|
128
|
-
const usage =
|
|
128
|
+
const usage = core_1.AgenticaTokenUsage.zero();
|
|
129
129
|
const prompts = yield ChatGptSelectFunctionAgent_1.ChatGptSelectFunctionAgent.execute(Object.assign(Object.assign({}, this.agent_.getContext({
|
|
130
|
-
prompt: {
|
|
131
|
-
type: "text",
|
|
130
|
+
prompt: new core_1.AgenticaTextPrompt({
|
|
132
131
|
role: "user",
|
|
133
132
|
text: scenario.text,
|
|
134
|
-
},
|
|
133
|
+
}),
|
|
135
134
|
usage,
|
|
136
135
|
})), { histories: this.histories_.slice(), stack: [], ready: () => true, dispatch: () => __awaiter(this, void 0, void 0, function* () { }) }));
|
|
137
136
|
const selected = prompts
|
|
138
137
|
.filter((p) => p.type === "select")
|
|
139
|
-
.map((p) => p.
|
|
138
|
+
.map((p) => p.selections)
|
|
140
139
|
.flat();
|
|
141
140
|
return {
|
|
142
141
|
type: AgenticaBenchmarkPredicator_1.AgenticaBenchmarkPredicator.success({
|
|
143
142
|
expected: scenario.expected,
|
|
144
|
-
operations: selected,
|
|
143
|
+
operations: selected.map((s) => s.operation),
|
|
145
144
|
})
|
|
146
145
|
? "success"
|
|
147
146
|
: "failure",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgenticaSelectBenchmark.js","sourceRoot":"","sources":["../src/AgenticaSelectBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"AgenticaSelectBenchmark.js","sourceRoot":"","sources":["../src/AgenticaSelectBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,yCAOwB;AACxB,sGAAmG;AAEnG,+BAAiC;AAGjC,wFAAqF;AACrF,gGAA6F;AAK7F;;;;;;;;;;;;;;;;GAgBG;AACH,MAAa,uBAAuB;IAOlC;;;;OAIG;IACH,YAAmB,KAA4C;;QAC7D,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG;YACb,MAAM,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,MAAM,mCAAI,EAAE;YAClC,YAAY,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,YAAY,mCAAI,EAAE;SAC/C,CAAC;QACF,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACU,OAAO,CAClB,QAAgE;;YAEhE,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,MAAM,SAAS,GAAc,IAAI,gBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACtE,MAAM,WAAW,GACf,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAO,QAAQ,EAAE,EAAE;gBACrC,MAAM,MAAM,GACV,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAS,EAAE;oBACpD,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC1B,MAAM,CAAC,GACL,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC5B,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC1B,IAAI,QAAQ,KAAK,SAAS;wBAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;oBACxC,OAAO,CAAC,CAAC;gBACX,CAAC,CAAA,CAAC,CACH,CAAC;gBACJ,OAAO;oBACL,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,MAAM;yBACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;yBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;yBACnB,MAAM,CAAC,yBAAkB,CAAC,IAAI,EAAE,yBAAkB,CAAC,IAAI,EAAE,CAAC;iBAC9D,CAAC;YACJ,CAAC,CAAA,CAAC,CACH,CAAC;YACJ,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG;gBACrB,WAAW;gBACX,UAAU;gBACV,YAAY,EAAE,IAAI,IAAI,EAAE;gBACxB,KAAK,EAAE,WAAW;qBACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;qBACnB,MAAM,CAAC,yBAAkB,CAAC,IAAI,EAAE,yBAAkB,CAAC,IAAI,EAAE,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;KAAA;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACI,MAAM;QACX,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,OAAO,iEAA+B,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChE,CAAC;IAEa,IAAI,CAChB,QAAiD;;YAEjD,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,MAAM,KAAK,GAAuB,yBAAkB,CAAC,IAAI,EAAE,CAAC;gBAC5D,MAAM,OAAO,GACX,MAAM,uDAA0B,CAAC,OAAO,CAAC,gCACpC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;oBACxB,MAAM,EAAE,IAAI,yBAAkB,CAAC;wBAC7B,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,QAAQ,CAAC,IAAI;qBACpB,CAAC;oBACF,KAAK;iBACN,CAAC,KACF,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAClC,KAAK,EAAE,EAAE,EACT,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,EACjB,QAAQ,EAAE,GAAS,EAAE,gDAAE,CAAC,CAAA,GACQ,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAwC,OAAO;qBAC1D,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;qBAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;qBACxB,IAAI,EAAE,CAAC;gBACV,OAAO;oBACL,IAAI,EAAE,yDAA2B,CAAC,OAAO,CAAC;wBACxC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;qBAC7C,CAAC;wBACA,CAAC,CAAC,SAAS;wBACX,CAAC,CAAC,SAAS;oBACb,QAAQ;oBACR,QAAQ;oBACR,KAAK;oBACL,gBAAgB,EAAE,OAAO;yBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;yBAChC,MAAM,CACL,CAAC,CAAC,EAAwC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CACpE;oBACH,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBAGuB,CAAC;YACpD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,QAAQ;oBACR,KAAK;oBACL,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBAC6B,CAAC;YAC1D,CAAC;QACH,CAAC;KAAA;CACF;AA7JD,0DA6JC"}
|
package/lib/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaTokenUsage, AgenticaTextPrompt } from "@agentica/core";
|
|
2
2
|
|
|
3
3
|
import { Semaphore } from "tstl";
|
|
4
4
|
|
|
@@ -234,8 +234,8 @@ var AgenticaPromptReporter;
|
|
|
234
234
|
|
|
235
235
|
(function(AgenticaPromptReporter) {
|
|
236
236
|
AgenticaPromptReporter.markdown = p => {
|
|
237
|
-
if (p.type === "text") return [ `### Text (${p.role})`, p.text, "" ].join("\n"); else if (p.type === "select" || p.type === "cancel") return [ `### ${p.type === "select" ? "Select" : "Cancel"}`, ...p.
|
|
238
|
-
return [ "### Execute", ` - name: ${p.name}`, ` - controller: ${p.controller.name}`, ` - function: ${p.function.name}`, "", "```json", JSON.stringify(p.arguments, null, 2), "```", "" ].join("\n");
|
|
237
|
+
if (p.type === "text") return [ `### Text (${p.role})`, p.text, "" ].join("\n"); else if (p.type === "select" || p.type === "cancel") return [ `### ${p.type === "select" ? "Select" : "Cancel"}`, ...p.selections.map((s => [ `#### ${s.operation.name}`, ` - controller: ${s.operation.controller.name}`, ` - function: ${s.operation.function.name}`, ` - reason: ${s.reason}`, "", ...!!s.operation.function.description?.length ? [ s.operation.function.description, "" ] : [] ])).flat() ].join("\n"); else if (p.type === "describe") return [ "### Describe", ...p.executes.map((e => ` - ${e.operation.name}`)), "", ...p.text.split("\n").map((s => `> ${s}`)), "" ].join("\n");
|
|
238
|
+
return [ "### Execute", ` - name: ${p.operation.name}`, ` - controller: ${p.operation.controller.name}`, ` - function: ${p.operation.function.name}`, "", "```json", JSON.stringify(p.arguments, null, 2), "```", "" ].join("\n");
|
|
239
239
|
};
|
|
240
240
|
})(AgenticaPromptReporter || (AgenticaPromptReporter = {}));
|
|
241
241
|
|
|
@@ -282,14 +282,14 @@ class AgenticaCallBenchmark {
|
|
|
282
282
|
return {
|
|
283
283
|
scenario,
|
|
284
284
|
events,
|
|
285
|
-
usage: events.filter((e => e.type !== "error")).map((e => e.usage)).reduce(
|
|
285
|
+
usage: events.filter((e => e.type !== "error")).map((e => e.usage)).reduce(AgenticaTokenUsage.plus, AgenticaTokenUsage.zero())
|
|
286
286
|
};
|
|
287
287
|
})));
|
|
288
288
|
return this.result_ = {
|
|
289
289
|
experiments,
|
|
290
290
|
started_at,
|
|
291
291
|
completed_at: new Date,
|
|
292
|
-
usage: experiments.map((p => p.usage)).reduce(
|
|
292
|
+
usage: experiments.map((p => p.usage)).reduce(AgenticaTokenUsage.plus, AgenticaTokenUsage.zero())
|
|
293
293
|
};
|
|
294
294
|
}
|
|
295
295
|
report() {
|
|
@@ -301,13 +301,13 @@ class AgenticaCallBenchmark {
|
|
|
301
301
|
const started_at = new Date;
|
|
302
302
|
const success = () => AgenticaBenchmarkPredicator.success({
|
|
303
303
|
expected: scenario.expected,
|
|
304
|
-
operations: agent.getPromptHistories().filter((p => p.type === "execute")),
|
|
304
|
+
operations: agent.getPromptHistories().filter((p => p.type === "execute")).map((p => p.operation)),
|
|
305
305
|
strict: false
|
|
306
306
|
});
|
|
307
307
|
const out = () => {
|
|
308
308
|
const select = AgenticaBenchmarkPredicator.success({
|
|
309
309
|
expected: scenario.expected,
|
|
310
|
-
operations: agent.getPromptHistories().filter((p => p.type === "select")).map((p => p.
|
|
310
|
+
operations: agent.getPromptHistories().filter((p => p.type === "select")).map((p => p.selections)).flat().map((p => p.operation)),
|
|
311
311
|
strict: false
|
|
312
312
|
});
|
|
313
313
|
const call = success();
|
|
@@ -363,7 +363,7 @@ var AgenticaSelectBenchmarkReporter;
|
|
|
363
363
|
const aggregate = exp.usage.aggregate;
|
|
364
364
|
return [ `# ${exp.scenario.name}`, "## Summary", " - Aggregation:", ` - Trial: ${exp.events.length}`, ` - Success: ${exp.events.filter((e => e.type === "success")).length}`, ` - Failure: ${exp.events.filter((e => e.type === "failure")).length}`, ` - Average Time: ${MathUtil.round(exp.events.map((event => event.completed_at.getTime() - event.started_at.getTime())).reduce(((a, b) => a + b), 0) / exp.events.length).toLocaleString()} ms`, ` - Token Usage`, ` - Total: ${aggregate.total.toLocaleString()}`, ` - Input`, ` - Total: ${aggregate.input.total.toLocaleString()}`, ` - Cached: ${aggregate.input.cached.toLocaleString()}`, ` - Output:`, ` - Total: ${aggregate.output.total.toLocaleString()}`, ` - Accepted Prediction: ${aggregate.output.accepted_prediction.toLocaleString()}`, ` - Reasoning: ${aggregate.output.reasoning.toLocaleString()}`, ` - Rejected Prediction: ${aggregate.output.rejected_prediction.toLocaleString()}`, "", "## Events", " No | Type | Time", "---:|:-----|----:", ...exp.events.map(((e, i) => [ `[${i + 1}.](./${i + 1}.${e.type}.md)`, e.type, MathUtil.round(e.completed_at.getTime() - e.started_at.getTime()) + " ms" ].join(" | "))), "", "## Scenario", "### User Prompt", exp.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected), null, 2), "```" ].join("\n");
|
|
365
365
|
};
|
|
366
|
-
const writeExperimentEvent = (event, index) => [ `# ${index + 1}. ${event.type}`, `## Summary`, ` - Name: ${event.scenario.name}`, ` - Type: ${event.type}`, ` - Time: ${(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`, ...event.type !== "error" ? [ " - Token Usage", ` - Total: ${event.usage.aggregate.toLocaleString()}`, ` - Prompt`, ` - Total: ${event.usage.aggregate.input.total.toLocaleString()}`, ` - Cached: ${event.usage.aggregate.input.cached.toLocaleString()}`, ` - Completion:`, ` - Total: ${event.usage.aggregate.output.total.toLocaleString()}`, ` - Reasoning: ${event.usage.aggregate.output.reasoning.toLocaleString()}`, ` - Accepted Prediction: ${event.usage.aggregate.output.accepted_prediction.toLocaleString()}`, ` - Rejected Prediction: ${event.usage.aggregate.output.rejected_prediction.toLocaleString()}` ] : [], "", "## Scenario", "### User Prompt", event.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected), null, 2), "```", "", ...event.type === "success" || event.type === "failure" ? [ "## Result", ...event.selected.map((s => [ `### ${s.name}`, ` - Controller: \`${s.controller.name}\``, ` - Function: \`${s.function.name}\``, ` - Reason: ${s.reason}`, "", ...s.function.description ? [ s.function.description, "" ] : [] ].join("\n"))) ] : [], ...event.type === "error" ? [ "## Error", "```json", AgenticaBenchmarkUtil.errorToJson(JSON.stringify(event.error, null, 2)), "```", "" ] : [] ].join("\n");
|
|
366
|
+
const writeExperimentEvent = (event, index) => [ `# ${index + 1}. ${event.type}`, `## Summary`, ` - Name: ${event.scenario.name}`, ` - Type: ${event.type}`, ` - Time: ${(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`, ...event.type !== "error" ? [ " - Token Usage", ` - Total: ${event.usage.aggregate.toLocaleString()}`, ` - Prompt`, ` - Total: ${event.usage.aggregate.input.total.toLocaleString()}`, ` - Cached: ${event.usage.aggregate.input.cached.toLocaleString()}`, ` - Completion:`, ` - Total: ${event.usage.aggregate.output.total.toLocaleString()}`, ` - Reasoning: ${event.usage.aggregate.output.reasoning.toLocaleString()}`, ` - Accepted Prediction: ${event.usage.aggregate.output.accepted_prediction.toLocaleString()}`, ` - Rejected Prediction: ${event.usage.aggregate.output.rejected_prediction.toLocaleString()}` ] : [], "", "## Scenario", "### User Prompt", event.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected), null, 2), "```", "", ...event.type === "success" || event.type === "failure" ? [ "## Result", ...event.selected.map((s => [ `### ${s.operation.name}`, ` - Controller: \`${s.operation.controller.name}\``, ` - Function: \`${s.operation.function.name}\``, ` - Reason: ${s.reason}`, "", ...s.operation.function.description ? [ s.operation.function.description, "" ] : [] ].join("\n"))) ] : [], ...event.type === "error" ? [ "## Error", "```json", AgenticaBenchmarkUtil.errorToJson(JSON.stringify(event.error, null, 2)), "```", "" ] : [] ].join("\n");
|
|
367
367
|
})(AgenticaSelectBenchmarkReporter || (AgenticaSelectBenchmarkReporter = {}));
|
|
368
368
|
|
|
369
369
|
class AgenticaSelectBenchmark {
|
|
@@ -391,14 +391,14 @@ class AgenticaSelectBenchmark {
|
|
|
391
391
|
return {
|
|
392
392
|
scenario,
|
|
393
393
|
events,
|
|
394
|
-
usage: events.filter((e => e.type !== "error")).map((e => e.usage)).reduce(
|
|
394
|
+
usage: events.filter((e => e.type !== "error")).map((e => e.usage)).reduce(AgenticaTokenUsage.plus, AgenticaTokenUsage.zero())
|
|
395
395
|
};
|
|
396
396
|
})));
|
|
397
397
|
return this.result_ = {
|
|
398
398
|
experiments,
|
|
399
399
|
started_at,
|
|
400
400
|
completed_at: new Date,
|
|
401
|
-
usage: experiments.map((p => p.usage)).reduce(
|
|
401
|
+
usage: experiments.map((p => p.usage)).reduce(AgenticaTokenUsage.plus, AgenticaTokenUsage.zero())
|
|
402
402
|
};
|
|
403
403
|
}
|
|
404
404
|
report() {
|
|
@@ -408,14 +408,13 @@ class AgenticaSelectBenchmark {
|
|
|
408
408
|
async step(scenario) {
|
|
409
409
|
const started_at = new Date;
|
|
410
410
|
try {
|
|
411
|
-
const usage =
|
|
411
|
+
const usage = AgenticaTokenUsage.zero();
|
|
412
412
|
const prompts = await ChatGptSelectFunctionAgent.execute({
|
|
413
413
|
...this.agent_.getContext({
|
|
414
|
-
prompt: {
|
|
415
|
-
type: "text",
|
|
414
|
+
prompt: new AgenticaTextPrompt({
|
|
416
415
|
role: "user",
|
|
417
416
|
text: scenario.text
|
|
418
|
-
},
|
|
417
|
+
}),
|
|
419
418
|
usage
|
|
420
419
|
}),
|
|
421
420
|
histories: this.histories_.slice(),
|
|
@@ -423,11 +422,11 @@ class AgenticaSelectBenchmark {
|
|
|
423
422
|
ready: () => true,
|
|
424
423
|
dispatch: async () => {}
|
|
425
424
|
});
|
|
426
|
-
const selected = prompts.filter((p => p.type === "select")).map((p => p.
|
|
425
|
+
const selected = prompts.filter((p => p.type === "select")).map((p => p.selections)).flat();
|
|
427
426
|
return {
|
|
428
427
|
type: AgenticaBenchmarkPredicator.success({
|
|
429
428
|
expected: scenario.expected,
|
|
430
|
-
operations: selected
|
|
429
|
+
operations: selected.map((s => s.operation))
|
|
431
430
|
}) ? "success" : "failure",
|
|
432
431
|
scenario,
|
|
433
432
|
selected,
|
package/lib/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.mjs","sources":["../src/internal/AgenticaBenchmarkPredicator.ts","../src/utils/MathUtil.ts","../src/internal/AgenticaBenchmarkUtil.ts","../src/internal/AgenticaPromptReporter.ts","../src/internal/AgenticaCallBenchmarkReporter.ts","../src/AgenticaCallBenchmark.ts","../src/internal/AgenticaSelectBenchmarkReporter.ts","../src/AgenticaSelectBenchmark.ts"],"sourcesContent":[null,null,null,null,null,null,null,null],"names":["AgenticaBenchmarkPredicator","isNext","async","agent","last","getPromptHistories","at","type","role","consent","functions","result","vendor","api","chat","completions","create","model","messages","content","join","text","tools","function","name","description","parameters","tool_choice","parallel_tool_calls","options","toolCall","choices","message","tool_calls","filter","tc","undefined","input","JSON","parse","arguments","_io0","reply","success","props","successInner","call","expected","overrideOperations","operations","strict","take","targetIterator","items","Symbol","iterator","targeted","next","done","length","value","slice","target","operation","some","op","anyOf","callResult","allOf","map","every","r","reduce","acc","Math","max","MathUtil","round","floor","AgenticaBenchmarkUtil","errorToJson","error","Error","stack","expectedToJson","AgenticaPromptReporter","markdown","p","controller","reason","flat","executions","e","split","s","stringify","AgenticaCallBenchmarkReporter","Object","fromEntries","writeIndex","experiments","exp","scenario","writeExperimentIndex","events","event","i","writeExperimentEvent","average","completed_at","getTime","started_at","a","b","aggregate","usage","toLocaleString","total","cached","output","reasoning","accepted_prediction","rejected_prediction","drawStatus","select","index","prompts","count","Array","fill","AgenticaCallBenchmark","constructor","this","agent_","scenarios_","scenarios","config_","repeat","config","simultaneous","result_","execute","listener","Date","semaphore","Semaphore","Promise","all","acquire","step","release","AgenticaTokenUsageAggregator","plus","zero","report","clone","out","getTokenUsage","conversate","AgenticaSelectBenchmarkReporter","selected","AgenticaSelectBenchmark","histories_","ChatGptSelectFunctionAgent","getContext","prompt","histories","ready","dispatch","assistantPrompts"],"mappings":";;;;;;;;;;AAOM,IAAWA;;CAAjB,SAAiBA;IACFA,4BAAAC,SAASC,MACpBC;QAEA,MAAMC,OAA2CD,MAC9CE,qBACAC,IAAG;QACN,IAAIF,MAAMG,SAAS,UAAUH,KAAKI,SAAS,aAAa,OAAO;QAE/D,MAAMC,UAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAGrCC,UAAU;QACd,MAAMC,eAAsCR,MAC1C,SACAS,OAAOC,IAAIC,KAAKC,YAAYC,OAC5B;YACEC,OAAOd,MAAM,SAASS,OAAOK;YAC7BC,UAAU,EACR;gBACEV,MAAM;gBACNW,SAAS,EACP,iCACA,IACA,uDACA,gEACA,qDACAC,KAAK;eAET;gBACEZ,MAAM;gBACNW,SAASf,KAAKiB;;YAGlBC,OAAO,EACL;gBACEf,MAAM;gBACNgB,UAAU;oBACRC,MAAMf,QAAQe;oBACdC,aAAahB,QAAQgB;oBACrBC,YAAYjB,QAAQiB;;;YAI1BC,aAAa;YACbC,qBAAqB;WAEvBzB,MAAM,SAASS,OAAOiB;QAExB,MAAMC,YACJnB,OAAOoB,QAAQ,IAAIC,QAAQC,cAAc,IACzCC,QACCC,MAAOA,GAAG5B,SAAS,cAAc4B,GAAGZ,SAASC,SAASf,QAAQe,SAC7D;QACJ,IAAIM,aAAaM,WAAW,OAAO;QACnC,MAAMC,QAAuBC,KAAKC,MAAMT,SAASP,SAASiB;QAC1D,OAAO;YAAS,MAAAC,OAAAJ,SAAA,oBAAAA,MAAAlB,WAAA,oBAAAkB,MAAAK;YAAA,OAAAL,SAAA,oBAAAA,SAAA,SAAAA,SAAAI,KAAAJ;AAAA,UAAT,GAASA,SAASA,MAAMK,QAAQ;AAAI;IAYhC1C,4BAAA2C,UAA2CC,SAqBzCC,aAAaD,OAAOjC;IAEnC,MAAMkC,eACJD;QASA,MAAME,OAAO,CACXC,UACAC,uBAIAH,aAAa;YACXE;YACAE,YAAYD,sBAAsBJ,MAAMK;YACxCC,QAAQN,MAAMM;;QAGlB,QAAQN,MAAMG,SAASxC;UACrB,KAAK;YAAS;gBACZ,IAAI4C,OAAO;gBACX,MAAMC,iBAAiBR,MAAMG,SAASM,MAAMC,OAAOC;gBACnD,IAAIC,WAAWJ,eAAeK;gBAE9B,OAAO,MAAM;oBACX,IAAID,SAASE,MAAM;wBACjB,OAAO;4BACL/C,QAAQ;4BACRwC;;;oBAGJ,IAAIA,QAAQP,MAAMK,WAAWU,QAAQ;wBACnC,OAAO;4BAAEhD,QAAQ;;;oBAGnB,MAAMA,SAASmC,KAAKU,SAASI,OAAOhB,MAAMK,WAAWY,MAAMV;oBAC3D,KAAKxC,OAAOA,QAAQ;wBAClB,KAAKiC,MAAMM,QAAQ;4BACjBC,QAAQ;4BACR;;wBAEF,OAAO;4BAAExC,QAAQ;;;oBAGnBwC,QAAQxC,OAAOwC;oBACfK,WAAWJ,eAAeK;;;;UAG9B,KAAK;YAAc;gBACjB,MAAMK,SAASlB,MAAMG,SAASgB;gBAC9B,MAAMpD,SAASiC,MAAMK,WAAWe,MAAMC,MAAOA,GAAGzC,SAASsC,OAAOtC;gBAChE,IAAIb,QAAQ;oBACV,OAAO;wBAAEA;wBAAQwC,MAAM;;;gBAEzB,OAAO;oBACLxC;;;;UAGJ,KAAK;YACH,KAAK,MAAMoC,YAAYH,MAAMG,SAASmB,OAAO;gBAC3C,MAAMC,aAAarB,KAAKC;gBACxB,IAAIoB,WAAWxD,QAAQ;oBACrB,OAAOwD;;;YAIX,OAAO;gBAAExD,QAAQ;;;UACnB,KAAK;YAAS;gBAQZ,MAAMA,SAASiC,MAAMG,SAASqB,MAAMC,KAAKtB,YAAaD,KAAKC;gBAC3D,IAAIpC,OAAO2D,OAAOC,KAAMA,EAAE5D,UAAS;oBACjC,OAAO;wBACLA,QAAQ;wBACRwC,MAAMxC,OAAO6D,QAAO,CAACC,KAAKF,MAAMG,KAAKC,IAAIF,KAAKF,EAAEpB,QAAO;;;gBAI3D,OAAO;oBACLxC,QAAQ;;;;;AAKjB,EAxLD,CAAiBX,gCAAAA,8BAwLhB,CAAA;;AC/LK,IAAW4E;;CAAjB,SAAiBA;IACFA,SAAAC,QAASjB,SAA0Bc,KAAKI,MAAMlB,QAAQ,OAAO;AAC3E,EAFD,CAAiBgB,aAAAA,WAEhB,CAAA;;ACEK,IAAWG;;CAAjB,SAAiBA;IACFA,sBAAAC,cAAeC;QAC1B,IAAIA,iBAAiBC,OACnB,OAAO;eACFD;YACHzD,MAAMyD,MAAMzD;YACZQ,SAASiD,MAAMjD;YACfmD,OAAOF,MAAME;;QAEjB,OAAOF;AAAK;IAGDF,sBAAAK,iBACXrC;QAEA,IAAIA,SAASxC,SAAS,cACpB,OAAO;YACLA,MAAMwC,SAASxC;YACfwD,WAAW;gBACTvC,MAAMuB,SAASgB,UAAUvC;gBACzBC,aAAasB,SAASgB,UAAUxC,SAASE;;gBAG1C,IAAIsB,SAASxC,SAAS,SACzB,OAAO;YACLA,MAAMwC,SAASxC;YACf8C,OAAON,SAASM,MAAMgB,IAAIU,sBAAAK;gBAEzB,IAAIrC,SAASxC,SAAS,SACzB,OAAO;YACLA,MAAMwC,SAASxC;YACf6D,OAAOrB,SAASqB,MAAMC,IAAIU,sBAAAK;gBAG5B,OAAO;YACL7E,MAAMwC,SAASxC;YACf2D,OAAOnB,SAASmB,MAAMG,IAAIU,sBAAAK;;AAC3B;AAEN,EAvCD,CAAiBL,0BAAAA,wBAuChB,CAAA;;ACxCK,IAAWM;;CAAjB,SAAiBA;IACFA,uBAAAC,WACXC;QAEA,IAAIA,EAAEhF,SAAS,QACb,OAAO,EAAC,aAAagF,EAAE/E,SAAS+E,EAAElE,MAAM,KAAID,KAAK,YAC9C,IAAImE,EAAEhF,SAAS,YAAYgF,EAAEhF,SAAS,UACzC,OAAO,EACL,OAAOgF,EAAEhF,SAAS,WAAW,WAAW,eACrCgF,EAAEtC,WACFoB,KAAKJ,MAAO,EACX,QAAQA,GAAGzC,QACX,mBAAmByC,GAAGuB,WAAWhE,QACjC,iBAAiByC,GAAG1C,SAASC,QAC7B,eAAeyC,GAAGwB,UAClB,SACMxB,GAAG1C,SAASE,aAAakC,SAC3B,EAACM,GAAG1C,SAASE,aAAa,OAC1B,OAELiE,SACHtE,KAAK,YACJ,IAAImE,EAAEhF,SAAS,YAClB,OAAO,EACL,mBACGgF,EAAEI,WAAWtB,KAAKuB,KAAM,OAAOA,EAAEpE,UACpC,OACG+D,EAAElE,KAAKwE,MAAM,MAAMxB,KAAKyB,KAAM,KAAKA,OACtC,KACA1E,KAAK;QACT,OAAO,EACL,eACA,aAAamE,EAAE/D,QACf,mBAAmB+D,EAAEC,WAAWhE,QAChC,iBAAiB+D,EAAEhE,SAASC,QAC5B,IACA,WACAc,KAAKyD,UAAUR,EAAE/C,WAAW,MAAM,IAClC,OACA,KACApB,KAAK;AAAK;AAEf,EA1CD,CAAiBiE,2BAAAA,yBA0ChB,CAAA;;ACpCK,IAAWW;;CAAjB,SAAiBA;IACFA,8BAAQV,WACnB3E,UAEAsF,OAAOC,YAAY,EACjB,EAAC,eAAeC,WAAkBxF,cAC/BA,OAAOyF,YACP/B,KAAKgC,OAAQ,EACZ,EAAC,KAAKA,IAAIC,SAAS9E,kBAAkB+E,qBAAqBF,WACvDA,IAAIG,OAAOnC,KAAI,CAACoC,OAAOC,MAAM,EAC9B,KAAKL,IAAIC,SAAS9E,QAAQkF,IAAI,KAAKD,MAAMlG,WACzCoG,qBAAqBF,OAAOC,WAG/BhB;IAGP,MAAMS,aACJxF;QAEA,MAAM6F,SAA+C7F,OAAOyF,YACzD/B,KAAKE,KAAMA,EAAEiC,SACbd;QACH,MAAMkB,UACJJ,OACGnC,KAAKuB,KAAMA,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,YACnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKT,OAAO7C;QACzC,MAAMuD,YAA4CvG,OAAOwG,MAAMD;QAC/D,OAAO,EACL,iCACA,cACA,oBACA,qBAAqBvG,OAAOyF,YAAYzC,OAAOyD,oBAC/C,gBAAgBZ,OAAO7C,UACvB,kBAAkB6C,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC7D,kBAAkB6C,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC7D,uBAAuBiB,SAASC,MAAM+B,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU7E,MAAMgF,MAAMD,oBACxC,mBAAmBF,UAAU7E,MAAMiF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,qCACA,yCACGzG,OAAOyF,YAAY/B,KAAKgC,OACzB,EACE,IAAIA,IAAIC,SAAS9E,WAAW6E,IAAIC,SAAS9E,mBACzCmG,WACEtB,IAAIG,SACHZ,KAAMA,EAAErF,SAAS,WAAWqF,EAAEgC,WAAW,QAE5CD,WAAWtB,IAAIG,SAASZ,KAAMA,EAAErF,SAAS,WAAWqF,EAAE9C,SAAS,QAC/D,GAAG8B,SAASC,MACVwB,IAAIG,OACDnC,KAAKuB,KAAMA,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,YACnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO7C,QAC3CyD,wBACFhG,KAAK,WAETA,KAAK;AAAK;IAGd,MAAMmF,uBACJF,OAEO,EACL,KAAKA,IAAIC,SAAS9E,QAClB,cACA,mBAAmB6E,IAAIG,OAAO7C,OAAOyD,oBACrC,gBAAgBf,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC/D,gBAAgB0C,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC/D,qBAAqBiB,SAASC,MAC5BwB,IAAIG,OACDnC,KAAKuB,KAAMA,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,YACnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO7C,QAC3CyD,uBACF,IACA,aACA,uBACA,0BACGf,IAAIG,OAAOnC,KAAI,CAACuB,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAErF,YAC5BqF,EAAErF,MACF,GAAGqE,SAASC,MAAMe,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,kBAC1D1F,KAAK,UAET,IACA,eACA,mBACAiF,IAAIC,SAASjF,MACb,IACA,gBACA,WACAiB,KAAKyD,UACHhB,sBAAsBK,eAAeiB,IAAIC,SAASvD,WAClD,MACA,IAEF,QACA3B,KAAK;IAGT,MAAMuF,uBAAuB,CAC3BF,OACAoB,UAEO,EACL,KAAKA,QAAQ,MAAMpB,MAAMlG,QACzB,cACA,aAAakG,MAAMH,SAAS9E,QAC5B,aAAaiF,MAAMlG,QACnB,aAAaqE,SAASC,MACpB4B,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,WAChDM,0BACEX,MAAMlG,SAAS,UACf,EACE,eAAekG,MAAMmB,SAAS,MAAM,OACpC,aAAanB,MAAM3D,OAAO,MAAM,UAElC,IACJ,oBAAoB2D,MAAMU,MAAMC,oBAChC,IACA,eACA,mBACAX,MAAMH,SAASjF,MACf,IACA,gBACA,WACAiB,KAAKyD,UACHhB,sBAAsBK,eAAeqB,MAAMH,SAASvD,WACpD,MACA,IAEF,OACA,IACA,0BACG0D,MAAMqB,QAAQzD,IAAIgB,uBAAuBC,WAC5C,OACImB,MAAMlG,SAAS,UACf,EACE,YACA,WACA+B,KAAKyD,UACHhB,sBAAsBC,YAAYyB,MAAMxB,QACxC,MACA,IAEF,UAEF,KACJ7D,KAAK;IAGT,MAAMuG,aAAa,CACjBnB,QACA7D;QAEA,MAAMoF,QAAgBrD,KAAKI,MACxB0B,OAAOtE,OAAOS,SAASgB,SAAS6C,OAAO7C,SAAU;QAEpD,OACE,IAAIqE,MAAMD,OAAOE,KAAK,KAAK7G,KAAK,MAChC,IAAI4G,MAAM,KAAKD,OAAOE,KAAK,KAAK7G,KAAK;AAAG;AAG7C,EA7KD,CAAiB4E,kCAAAA,gCA6KhB,CAAA;;MCtJYkC;IAWX,WAAAC,CAAmBvF;QACjBwF,KAAKC,SAASzF,MAAMzC;QACpBiI,KAAKE,aAAa1F,MAAM2F,UAAU1E;QAClCuE,KAAKI,UAAU;YACbC,QAAQ7F,MAAM8F,QAAQD,UAAU;YAChCE,cAAc/F,MAAM8F,QAAQC,gBAAgB;YAC5ClI,SAASmC,MAAM8F,QAAQjI,WAAW;;QAEpC2H,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAM/B,aAAmB,IAAIgC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMvC,oBACE8C,QAAQC,IACZf,KAAKE,WAAWjE,KAAInE,MAAOoG;YACzB,MAAME,eACE0C,QAAQC,IACZ,IAAInB,MAAMI,KAAKI,QAAQC,QAAQR,KAAK,GAAG5D,KAAInE;sBACnC8I,UAAUI;gBAChB,MAAMxD,UACEwC,KAAKiB,KAAK/C;sBACZ0C,UAAUM;gBAChB,IAAIR,aAAa1G,WAAW0G,SAASlD;gBACrC,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAW,OAAOX,OACJtE,QAAQ0D,KAAMA,EAAErF,SAAS,UACzB8D,KAAKuB,KAAMA,EAAEuB,QACb3C,OACC+E,6BAA6BC,MAC7BD,6BAA6BE;;AAElC;QAGP,OAAQrB,KAAKQ,UAAU;YACrBxC;YACAW;YACAF,cAAc,IAAIkC;YAClB5B,OAAOf,YACJ/B,KAAKkB,KAAMA,EAAE4B,QACb3C,OACC+E,6BAA6BC,MAC7BD,6BAA6BE;;;IAuB9B,MAAAC;QACL,IAAItB,KAAKQ,YAAY,MACnB,MAAM,IAAI1D,MAAM;QAClB,OAAOc,8BAA8BV,SAAS8C,KAAKQ;;IAG7C,UAAMS,CACZ/C;QAEA,MAAMnG,QAAyBiI,KAAKC,OAAOsB;QAC3C,MAAM5C,aAAmB,IAAIgC;QAC7B,MAAMpG,UAAU,MACd3C,4BAA4B2C,QAAQ;YAClCI,UAAUuD,SAASvD;YACnBE,YAAY9C,MACTE,qBACA6B,QAAQqD,KAAMA,EAAEhF,SAAS;YAC5B2C,QAAQ;;QAEZ,MAAM0G,MAAM;YACV,MAAMhC,SAAS5H,4BAA4B2C,QAAQ;gBACjDI,UAAUuD,SAASvD;gBACnBE,YAAY9C,MACTE,qBACA6B,QAAQqD,KAAMA,EAAEhF,SAAS,WACzB8D,KAAKkB,KAAMA,EAAEtC,aACbyC;gBACHxC,QAAQ;;YAEV,MAAMJ,OAAOH;YACb,OAAO;gBACLpC,MAAOuC,OAAO,YAAY;gBAC1BwD;gBACAsB;gBACA9E;gBACAgF,SAAS3H,MAAME;gBACf8G,OAAOhH,MAAM0J;gBACb9C;gBACAF,cAAc,IAAIkC;;AACmC;QAGzD;kBACQ5I,MAAM2J,WAAWxD,SAASjF;YAChC,IAAIsB,WAAW,OAAOiH;YACtB,KAAK,IAAIlD,IAAY,GAAGA,IAAI0B,KAAKI,QAAQ/H,WAAWiG,GAAG;gBACrD,MAAMjD,aACEzD,4BAA4BC,OAAOE;gBAC3C,IAAIsD,SAAS,MAAM;sBAEbtD,MAAM2J,WAAWrG;gBACvB,IAAId,WAAW,OAAOiH;;YAExB,OAAOA;UACP,OAAO3E;YACP,OAAO;gBACL1E,MAAM;gBACN+F;gBACAwB,SAAS3H,MAAME;gBACf8G,OAAOhH,MAAM0J;gBACb5E;gBACA8B;gBACAF,cAAc,IAAIkC;;;;;;ACxLpB,IAAWgB;;CAAjB,SAAiBA;IACFA,gCAAQzE,WACnB3E,UAEAsF,OAAOC,YAAY,EACjB,EAAC,eAAeC,WAAWxF,cACxBA,OAAOyF,YACP/B,KAAKgC,OAAQ,EACZ,EAAC,KAAKA,IAAIC,SAAS9E,kBAAkB+E,qBAAqBF,WACvDA,IAAIG,OAAOnC,KAAI,CAACoC,OAAOC,MAAM,EAC9B,KAAKL,IAAIC,SAAS9E,QAAQkF,IAAI,KAAKD,MAAMlG,WACzCoG,qBAAqBF,OAAOC,WAG/BhB;IAGP,MAAMS,aACJxF;QAEA,MAAM6F,SAAiD7F,OAAOyF,YAC3D/B,KAAKE,KAAMA,EAAEiC,SACbd;QACH,MAAMkB,UACJJ,OACGnC,KAAKuB,KAAMA,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,YACnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKT,OAAO7C;QACzC,MAAMuD,YAA4CvG,OAAOwG,MAAMD;QAC/D,OAAO,EACL,sCACA,cACA,oBACA,qBAAqBvG,OAAOyF,YAAYzC,OAAOyD,oBAC/C,gBAAgBZ,OAAO7C,UACvB,kBAAkB6C,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC7D,kBAAkB6C,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC7D,uBAAuBiB,SAASC,MAAM+B,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU7E,MAAMgF,MAAMD,oBACxC,mBAAmBF,UAAU7E,MAAMiF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,+BACA,kCACGzG,OAAOyF,YAAY/B,KAAKgC,OACzB,EACE,IAAIA,IAAIC,SAAS9E,WAAW6E,IAAIC,SAAS9E,mBACzC;YACE,MAAMmB,UAAkB+B,KAAKI,MAC1BuB,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,SAC9C0C,IAAIG,OAAO7C,SACX;YAEJ,OACE,IAAIqE,MAAMrF,SAASsF,KAAK,KAAK7G,KAAK,MAClC,IAAI4G,MAAM,KAAKrF,SAASsF,KAAK,KAAK7G,KAAK;AAE1C,UAVD,IAWAwD,SAASC,MACPwB,IAAIG,OACDnC,KACEoC,SACCA,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,YAEnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO7C,QAC3CyD,mBAAmB,QACrBhG,KAAK,WAETA,KAAK;AAAK;IAGd,MAAMmF,uBACJF;QAEA,MAAMa,YAA4Cb,IAAIc,MAAMD;QAC5D,OAAO,EACL,KAAKb,IAAIC,SAAS9E,QAClB,cACA,oBACA,gBAAgB6E,IAAIG,OAAO7C,UAC3B,kBAAkB0C,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UACjE,kBAAkB0C,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UACjE,uBAAuBiB,SAASC,MAC9BwB,IAAIG,OACDnC,KACEoC,SACCA,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,YAEnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO7C,QAC3CyD,uBACF,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU7E,MAAMgF,MAAMD,oBACxC,mBAAmBF,UAAU7E,MAAMiF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,aACA,qBACA,wBACGf,IAAIG,OAAOnC,KAAI,CAACuB,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAErF,YAC5BqF,EAAErF,MACFqE,SAASC,MAAMe,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,aACrD,QACF1F,KAAK,UAET,IACA,eACA,mBACAiF,IAAIC,SAASjF,MACb,IACA,gBACA,WACAiB,KAAKyD,UACHhB,sBAAsBK,eAAeiB,IAAIC,SAASvD,WAClD,MACA,IAEF,QACA3B,KAAK;AAAK;IAGd,MAAMuF,uBAAuB,CAC3BF,OACAoB,UAEO,EACL,KAAKA,QAAQ,MAAMpB,MAAMlG,QACzB,cACA,aAAakG,MAAMH,SAAS9E,QAC5B,aAAaiF,MAAMlG,QACnB,cAAckG,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,WAAWM,0BACrEX,MAAMlG,SAAS,UACf,EACE,mBACA,gBAAgBkG,MAAMU,MAAMD,UAAUE,oBACtC,gBACA,kBAAkBX,MAAMU,MAAMD,UAAU7E,MAAMgF,MAAMD,oBACpD,mBAAmBX,MAAMU,MAAMD,UAAU7E,MAAMiF,OAAOF,oBACtD,qBACA,kBAAkBX,MAAMU,MAAMD,UAAUK,OAAOF,MAAMD,oBACrD,sBAAsBX,MAAMU,MAAMD,UAAUK,OAAOC,UAAUJ,oBAC7D,gCAAgCX,MAAMU,MAAMD,UAAUK,OAAOE,oBAAoBL,oBACjF,gCAAgCX,MAAMU,MAAMD,UAAUK,OAAOG,oBAAoBN,uBAEnF,IACJ,IACA,eACA,mBACAX,MAAMH,SAASjF,MACf,IACA,gBACA,WACAiB,KAAKyD,UACHhB,sBAAsBK,eAAeqB,MAAMH,SAASvD,WACpD,MACA,IAEF,OACA,OACI0D,MAAMlG,SAAS,aAAakG,MAAMlG,SAAS,YAC3C,EACE,gBACGkG,MAAMuD,SAAS3F,KAAKyB,KACrB,EACE,OAAOA,EAAEtE,QACT,qBAAqBsE,EAAEN,WAAWhE,UAClC,mBAAmBsE,EAAEvE,SAASC,UAC9B,eAAesE,EAAEL,UACjB,OACIK,EAAEvE,SAASE,cAAc,EAACqE,EAAEvE,SAASE,aAAa,OAAM,KAC5DL,KAAK,YAGX,OACAqF,MAAMlG,SAAS,UACf,EACE,YACA,WACAwE,sBAAsBC,YACpB1C,KAAKyD,UAAUU,MAAMxB,OAAO,MAAM,KAEpC,OACA,OAEF,KACJ7D,KAAK;AAEV,EAzMD,CAAiB2I,oCAAAA,kCAyMhB,CAAA;;MChLYE;IAYX,WAAA9B,CAAmBvF;QACjBwF,KAAKC,SAASzF,MAAMzC;QACpBiI,KAAKE,aAAa1F,MAAM2F,UAAU1E;QAClCuE,KAAKI,UAAU;YACbC,QAAQ7F,MAAM8F,QAAQD,UAAU;YAChCE,cAAc/F,MAAM8F,QAAQC,gBAAgB;;QAE9CP,KAAK8B,aAAatH,MAAMzC,MAAME,qBAAqBwD;QACnDuE,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAM/B,aAAmB,IAAIgC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMvC,oBACE8C,QAAQC,IACZf,KAAKE,WAAWjE,KAAInE,MAAOoG;YACzB,MAAME,eACE0C,QAAQC,IACZ,IAAInB,MAAMI,KAAKI,QAAQC,QAAQR,KAAK,GAAG5D,KAAInE;sBACnC8I,UAAUI;gBAChB,MAAMxD,UACEwC,KAAKiB,KAAK/C;sBACZ0C,UAAUM;gBAChB,IAAIR,aAAa1G,WAAW0G,SAASlD;gBACrC,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAW,OAAOX,OACJtE,QAAQ0D,KAAMA,EAAErF,SAAS,UACzB8D,KAAKuB,KAAMA,EAAEuB,QACb3C,OACC+E,6BAA6BC,MAC7BD,6BAA6BE;;AAElC;QAGP,OAAQrB,KAAKQ,UAAU;YACrBxC;YACAW;YACAF,cAAc,IAAIkC;YAClB5B,OAAOf,YACJ/B,KAAKkB,KAAMA,EAAE4B,QACb3C,OACC+E,6BAA6BC,MAC7BD,6BAA6BE;;;IAwB9B,MAAAC;QACL,IAAItB,KAAKQ,YAAY,MACnB,MAAM,IAAI1D,MAAM;QAClB,OAAO6E,gCAAgCzE,SAAS8C,KAAKQ;;IAG/C,UAAMS,CACZ/C;QAEA,MAAMS,aAAmB,IAAIgC;QAC7B;YACE,MAAM5B,QAA6BoC,6BAA6BE;YAChE,MAAM3B,gBACEqC,2BAA2BtB,QAAQ;mBACpCT,KAAKC,OAAO+B,WAAW;oBACxBC,QAAQ;wBACN9J,MAAM;wBACNC,MAAM;wBACNa,MAAMiF,SAASjF;;oBAEjB8F;;gBAEFmD,WAAWlC,KAAK8B,WAAWrG;gBAC3BsB,OAAO;gBACPoF,OAAO,MAAM;gBACbC,UAAUtK;;YAEd,MAAM8J,WAAiDlC,QACpD5F,QAAQqD,KAAMA,EAAEhF,SAAS,WACzB8D,KAAKkB,KAAMA,EAAEtC,aACbyC;YACH,OAAO;gBACLnF,MAAMP,4BAA4B2C,QAAQ;oBACxCI,UAAUuD,SAASvD;oBACnBE,YAAY+G;qBAEV,YACA;gBACJ1D;gBACA0D;gBACA7C;gBACAsD,kBAAkB3C,QACf5F,QAAQqD,KAAMA,EAAEhF,SAAS,SACzB2B,QACEqD,KACCA,EAAE/E,SAAS;gBAEjBuG;gBACAF,cAAc,IAAIkC;;UAIpB,OAAO9D;YACP,OAAO;gBACL1E,MAAM;gBACN+F;gBACArB;gBACA8B;gBACAF,cAAc,IAAIkC;;;;;;"}
|
|
1
|
+
{"version":3,"file":"index.mjs","sources":["../src/internal/AgenticaBenchmarkPredicator.ts","../src/utils/MathUtil.ts","../src/internal/AgenticaBenchmarkUtil.ts","../src/internal/AgenticaPromptReporter.ts","../src/internal/AgenticaCallBenchmarkReporter.ts","../src/AgenticaCallBenchmark.ts","../src/internal/AgenticaSelectBenchmarkReporter.ts","../src/AgenticaSelectBenchmark.ts"],"sourcesContent":[null,null,null,null,null,null,null,null],"names":["AgenticaBenchmarkPredicator","isNext","async","agent","last","getPromptHistories","at","type","role","consent","functions","result","vendor","api","chat","completions","create","model","messages","content","join","text","tools","function","name","description","parameters","tool_choice","parallel_tool_calls","options","toolCall","choices","message","tool_calls","filter","tc","undefined","input","JSON","parse","arguments","_io0","reply","success","props","successInner","call","expected","overrideOperations","operations","strict","take","targetIterator","items","Symbol","iterator","targeted","next","done","length","value","slice","target","operation","some","op","anyOf","callResult","allOf","map","every","r","reduce","acc","Math","max","MathUtil","round","floor","AgenticaBenchmarkUtil","errorToJson","error","Error","stack","expectedToJson","AgenticaPromptReporter","markdown","p","selections","s","controller","reason","flat","executes","e","split","stringify","AgenticaCallBenchmarkReporter","Object","fromEntries","writeIndex","experiments","exp","scenario","writeExperimentIndex","events","event","i","writeExperimentEvent","average","completed_at","getTime","started_at","a","b","aggregate","usage","toLocaleString","total","cached","output","reasoning","accepted_prediction","rejected_prediction","drawStatus","select","index","prompts","count","Array","fill","AgenticaCallBenchmark","constructor","this","agent_","scenarios_","scenarios","config_","repeat","config","simultaneous","result_","execute","listener","Date","semaphore","Semaphore","Promise","all","acquire","step","release","AgenticaTokenUsage","plus","zero","report","clone","out","getTokenUsage","conversate","AgenticaSelectBenchmarkReporter","selected","AgenticaSelectBenchmark","histories_","ChatGptSelectFunctionAgent","getContext","prompt","AgenticaTextPrompt","histories","ready","dispatch","assistantPrompts"],"mappings":";;;;;;;;;;AAOM,IAAWA;;CAAjB,SAAiBA;IACFA,4BAAAC,SAASC,MACpBC;QAEA,MAAMC,OAA0CD,MAC7CE,qBACAC,IAAG;QACN,IAAIF,MAAMG,SAAS,UAAUH,KAAKI,SAAS,aAAa,OAAO;QAE/D,MAAMC,UAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAGrCC,UAAU;QACd,MAAMC,eAAsCR,MAC1C,SACAS,OAAOC,IAAIC,KAAKC,YAAYC,OAC5B;YACEC,OAAOd,MAAM,SAASS,OAAOK;YAC7BC,UAAU,EACR;gBACEV,MAAM;gBACNW,SAAS,EACP,iCACA,IACA,uDACA,gEACA,qDACAC,KAAK;eAET;gBACEZ,MAAM;gBACNW,SAASf,KAAKiB;;YAGlBC,OAAO,EACL;gBACEf,MAAM;gBACNgB,UAAU;oBACRC,MAAMf,QAAQe;oBACdC,aAAahB,QAAQgB;oBACrBC,YAAYjB,QAAQiB;;;YAI1BC,aAAa;YACbC,qBAAqB;WAEvBzB,MAAM,SAASS,OAAOiB;QAExB,MAAMC,YACJnB,OAAOoB,QAAQ,IAAIC,QAAQC,cAAc,IACzCC,QACCC,MAAOA,GAAG5B,SAAS,cAAc4B,GAAGZ,SAASC,SAASf,QAAQe,SAC7D;QACJ,IAAIM,aAAaM,WAAW,OAAO;QACnC,MAAMC,QAAuBC,KAAKC,MAAMT,SAASP,SAASiB;QAC1D,OAAO;YAAS,MAAAC,OAAAJ,SAAA,oBAAAA,MAAAlB,WAAA,oBAAAkB,MAAAK;YAAA,OAAAL,SAAA,oBAAAA,SAAA,SAAAA,SAAAI,KAAAJ;AAAA,UAAT,GAASA,SAASA,MAAMK,QAAQ;AAAI;IAYhC1C,4BAAA2C,UAA2CC,SAmBzCC,aAAaD,OAAOjC;IAEnC,MAAMkC,eACJD;QASA,MAAME,OAAO,CACXC,UACAC,uBAEAH,aAAa;YACXE;YACAE,YAAYD,sBAAsBJ,MAAMK;YACxCC,QAAQN,MAAMM;;QAGlB,QAAQN,MAAMG,SAASxC;UACrB,KAAK;YAAS;gBACZ,IAAI4C,OAAO;gBACX,MAAMC,iBAAiBR,MAAMG,SAASM,MAAMC,OAAOC;gBACnD,IAAIC,WAAWJ,eAAeK;gBAE9B,OAAO,MAAM;oBACX,IAAID,SAASE,MAAM;wBACjB,OAAO;4BACL/C,QAAQ;4BACRwC;;;oBAGJ,IAAIA,QAAQP,MAAMK,WAAWU,QAAQ;wBACnC,OAAO;4BAAEhD,QAAQ;;;oBAGnB,MAAMA,SAASmC,KAAKU,SAASI,OAAOhB,MAAMK,WAAWY,MAAMV;oBAC3D,KAAKxC,OAAOA,QAAQ;wBAClB,KAAKiC,MAAMM,QAAQ;4BACjBC,QAAQ;4BACR;;wBAEF,OAAO;4BAAExC,QAAQ;;;oBAGnBwC,QAAQxC,OAAOwC;oBACfK,WAAWJ,eAAeK;;;;UAG9B,KAAK;YAAc;gBACjB,MAAMK,SAASlB,MAAMG,SAASgB;gBAC9B,MAAMpD,SAASiC,MAAMK,WAAWe,MAAMC,MAAOA,GAAGzC,SAASsC,OAAOtC;gBAChE,IAAIb,QAAQ;oBACV,OAAO;wBAAEA;wBAAQwC,MAAM;;;gBAEzB,OAAO;oBACLxC;;;;UAGJ,KAAK;YACH,KAAK,MAAMoC,YAAYH,MAAMG,SAASmB,OAAO;gBAC3C,MAAMC,aAAarB,KAAKC;gBACxB,IAAIoB,WAAWxD,QAAQ;oBACrB,OAAOwD;;;YAIX,OAAO;gBAAExD,QAAQ;;;UACnB,KAAK;YAAS;gBAQZ,MAAMA,SAASiC,MAAMG,SAASqB,MAAMC,KAAKtB,YAAaD,KAAKC;gBAC3D,IAAIpC,OAAO2D,OAAOC,KAAMA,EAAE5D,UAAS;oBACjC,OAAO;wBACLA,QAAQ;wBACRwC,MAAMxC,OAAO6D,QAAO,CAACC,KAAKF,MAAMG,KAAKC,IAAIF,KAAKF,EAAEpB,QAAO;;;gBAI3D,OAAO;oBACLxC,QAAQ;;;;;AAKjB,EApLD,CAAiBX,gCAAAA,8BAoLhB,CAAA;;AC3LK,IAAW4E;;CAAjB,SAAiBA;IACFA,SAAAC,QAASjB,SAA0Bc,KAAKI,MAAMlB,QAAQ,OAAO;AAC3E,EAFD,CAAiBgB,aAAAA,WAEhB,CAAA;;ACEK,IAAWG;;CAAjB,SAAiBA;IACFA,sBAAAC,cAAeC;QAC1B,IAAIA,iBAAiBC,OACnB,OAAO;eACFD;YACHzD,MAAMyD,MAAMzD;YACZQ,SAASiD,MAAMjD;YACfmD,OAAOF,MAAME;;QAEjB,OAAOF;AAAK;IAGDF,sBAAAK,iBACXrC;QAEA,IAAIA,SAASxC,SAAS,cACpB,OAAO;YACLA,MAAMwC,SAASxC;YACfwD,WAAW;gBACTvC,MAAMuB,SAASgB,UAAUvC;gBACzBC,aAAasB,SAASgB,UAAUxC,SAASE;;gBAG1C,IAAIsB,SAASxC,SAAS,SACzB,OAAO;YACLA,MAAMwC,SAASxC;YACf8C,OAAON,SAASM,MAAMgB,IAAIU,sBAAAK;gBAEzB,IAAIrC,SAASxC,SAAS,SACzB,OAAO;YACLA,MAAMwC,SAASxC;YACf6D,OAAOrB,SAASqB,MAAMC,IAAIU,sBAAAK;gBAG5B,OAAO;YACL7E,MAAMwC,SAASxC;YACf2D,OAAOnB,SAASmB,MAAMG,IAAIU,sBAAAK;;AAC3B;AAEN,EAvCD,CAAiBL,0BAAAA,wBAuChB,CAAA;;ACxCK,IAAWM;;CAAjB,SAAiBA;IACFA,uBAAAC,WACXC;QAEA,IAAIA,EAAEhF,SAAS,QACb,OAAO,EAAC,aAAagF,EAAE/E,SAAS+E,EAAElE,MAAM,KAAID,KAAK,YAC9C,IAAImE,EAAEhF,SAAS,YAAYgF,EAAEhF,SAAS,UACzC,OAAO,EACL,OAAOgF,EAAEhF,SAAS,WAAW,WAAW,eACrCgF,EAAEC,WACFnB,KAAKoB,KAAM,EACV,QAAQA,EAAE1B,UAAUvC,QACpB,mBAAmBiE,EAAE1B,UAAU2B,WAAWlE,QAC1C,iBAAiBiE,EAAE1B,UAAUxC,SAASC,QACtC,eAAeiE,EAAEE,UACjB,SACMF,EAAE1B,UAAUxC,SAASE,aAAakC,SACpC,EAAC8B,EAAE1B,UAAUxC,SAASE,aAAa,OACnC,OAELmE,SACHxE,KAAK,YACJ,IAAImE,EAAEhF,SAAS,YAClB,OAAO,EACL,mBACGgF,EAAEM,SAASxB,KAAKyB,KAAM,OAAOA,EAAE/B,UAAUvC,UAC5C,OACG+D,EAAElE,KAAK0E,MAAM,MAAM1B,KAAKoB,KAAM,KAAKA,OACtC,KACArE,KAAK;QACT,OAAO,EACL,eACA,aAAamE,EAAExB,UAAUvC,QACzB,mBAAmB+D,EAAExB,UAAU2B,WAAWlE,QAC1C,iBAAiB+D,EAAExB,UAAUxC,SAASC,QACtC,IACA,WACAc,KAAK0D,UAAUT,EAAE/C,WAAW,MAAM,IAClC,OACA,KACApB,KAAK;AAAK;AAEf,EA1CD,CAAiBiE,2BAAAA,yBA0ChB,CAAA;;ACpCK,IAAWY;;CAAjB,SAAiBA;IACFA,8BAAQX,WACnB3E,UAEAuF,OAAOC,YAAY,EACjB,EAAC,eAAeC,WAAkBzF,cAC/BA,OAAO0F,YACPhC,KAAKiC,OAAQ,EACZ,EAAC,KAAKA,IAAIC,SAAS/E,kBAAkBgF,qBAAqBF,WACvDA,IAAIG,OAAOpC,KAAI,CAACqC,OAAOC,MAAM,EAC9B,KAAKL,IAAIC,SAAS/E,QAAQmF,IAAI,KAAKD,MAAMnG,WACzCqG,qBAAqBF,OAAOC,WAG/Bf;IAGP,MAAMQ,aACJzF;QAEA,MAAM8F,SAA+C9F,OAAO0F,YACzDhC,KAAKE,KAAMA,EAAEkC,SACbb;QACH,MAAMiB,UACJJ,OACGpC,KAAKyB,KAAMA,EAAEgB,aAAaC,YAAYjB,EAAEkB,WAAWD,YACnDvC,QAAO,CAACyC,GAAGC,MAAMD,IAAIC,IAAG,KAAKT,OAAO9C;QACzC,MAAMwD,YAA2CxG,OAAOyG,MAAMD;QAC9D,OAAO,EACL,iCACA,cACA,oBACA,qBAAqBxG,OAAO0F,YAAY1C,OAAO0D,oBAC/C,gBAAgBZ,OAAO9C,UACvB,kBAAkB8C,OAAOvE,QAAQ4D,KAAMA,EAAEvF,SAAS,YAAWoD,UAC7D,kBAAkB8C,OAAOvE,QAAQ4D,KAAMA,EAAEvF,SAAS,YAAWoD,UAC7D,uBAAuBiB,SAASC,MAAMgC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU9E,MAAMiF,MAAMD,oBACxC,mBAAmBF,UAAU9E,MAAMkF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,qCACA,yCACG1G,OAAO0F,YAAYhC,KAAKiC,OACzB,EACE,IAAIA,IAAIC,SAAS/E,WAAW8E,IAAIC,SAAS/E,mBACzCoG,WACEtB,IAAIG,SACHX,KAAMA,EAAEvF,SAAS,WAAWuF,EAAE+B,WAAW,QAE5CD,WAAWtB,IAAIG,SAASX,KAAMA,EAAEvF,SAAS,WAAWuF,EAAEhD,SAAS,QAC/D,GAAG8B,SAASC,MACVyB,IAAIG,OACDpC,KAAKyB,KAAMA,EAAEgB,aAAaC,YAAYjB,EAAEkB,WAAWD,YACnDvC,QAAO,CAACyC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO9C,QAC3C0D,wBACFjG,KAAK,WAETA,KAAK;AAAK;IAGd,MAAMoF,uBACJF,OAEO,EACL,KAAKA,IAAIC,SAAS/E,QAClB,cACA,mBAAmB8E,IAAIG,OAAO9C,OAAO0D,oBACrC,gBAAgBf,IAAIG,OAAOvE,QAAQ4D,KAAMA,EAAEvF,SAAS,YAAWoD,UAC/D,gBAAgB2C,IAAIG,OAAOvE,QAAQ4D,KAAMA,EAAEvF,SAAS,YAAWoD,UAC/D,qBAAqBiB,SAASC,MAC5ByB,IAAIG,OACDpC,KAAKyB,KAAMA,EAAEgB,aAAaC,YAAYjB,EAAEkB,WAAWD,YACnDvC,QAAO,CAACyC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO9C,QAC3C0D,uBACF,IACA,aACA,uBACA,0BACGf,IAAIG,OAAOpC,KAAI,CAACyB,GAAGa,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKb,EAAEvF,YAC5BuF,EAAEvF,MACF,GAAGqE,SAASC,MAAMiB,EAAEgB,aAAaC,YAAYjB,EAAEkB,WAAWD,kBAC1D3F,KAAK,UAET,IACA,eACA,mBACAkF,IAAIC,SAASlF,MACb,IACA,gBACA,WACAiB,KAAK0D,UACHjB,sBAAsBK,eAAekB,IAAIC,SAASxD,WAClD,MACA,IAEF,QACA3B,KAAK;IAGT,MAAMwF,uBAAuB,CAC3BF,OACAoB,UAEO,EACL,KAAKA,QAAQ,MAAMpB,MAAMnG,QACzB,cACA,aAAamG,MAAMH,SAAS/E,QAC5B,aAAakF,MAAMnG,QACnB,aAAaqE,SAASC,MACpB6B,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,WAChDM,0BACEX,MAAMnG,SAAS,UACf,EACE,eAAemG,MAAMmB,SAAS,MAAM,OACpC,aAAanB,MAAM5D,OAAO,MAAM,UAElC,IACJ,oBAAoB4D,MAAMU,MAAMC,oBAChC,IACA,eACA,mBACAX,MAAMH,SAASlF,MACf,IACA,gBACA,WACAiB,KAAK0D,UACHjB,sBAAsBK,eAAesB,MAAMH,SAASxD,WACpD,MACA,IAEF,OACA,IACA,0BACG2D,MAAMqB,QAAQ1D,IAAIgB,uBAAuBC,WAC5C,OACIoB,MAAMnG,SAAS,UACf,EACE,YACA,WACA+B,KAAK0D,UACHjB,sBAAsBC,YAAY0B,MAAMzB,QACxC,MACA,IAEF,UAEF,KACJ7D,KAAK;IAGT,MAAMwG,aAAa,CACjBnB,QACA9D;QAEA,MAAMqF,QAAgBtD,KAAKI,MACxB2B,OAAOvE,OAAOS,SAASgB,SAAS8C,OAAO9C,SAAU;QAEpD,OACE,IAAIsE,MAAMD,OAAOE,KAAK,KAAK9G,KAAK,MAChC,IAAI6G,MAAM,KAAKD,OAAOE,KAAK,KAAK9G,KAAK;AAAG;AAG7C,EA7KD,CAAiB6E,kCAAAA,gCA6KhB,CAAA;;MCvJYkC;IAWX,WAAAC,CAAmBxF;QACjByF,KAAKC,SAAS1F,MAAMzC;QACpBkI,KAAKE,aAAa3F,MAAM4F,UAAU3E;QAClCwE,KAAKI,UAAU;YACbC,QAAQ9F,MAAM+F,QAAQD,UAAU;YAChCE,cAAchG,MAAM+F,QAAQC,gBAAgB;YAC5CnI,SAASmC,MAAM+F,QAAQlI,WAAW;;QAEpC4H,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAM/B,aAAmB,IAAIgC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMvC,oBACE8C,QAAQC,IACZf,KAAKE,WAAWlE,KAAInE,MAAOqG;YACzB,MAAME,eACE0C,QAAQC,IACZ,IAAInB,MAAMI,KAAKI,QAAQC,QAAQR,KAAK,GAAG7D,KAAInE;sBACnC+I,UAAUI;gBAChB,MAAMvD,UACEuC,KAAKiB,KAAK/C;sBACZ0C,UAAUM;gBAChB,IAAIR,aAAa3G,WAAW2G,SAASjD;gBACrC,OAAOA;AAAC;YAGd,OAAO;gBACLS;gBACAE;gBACAW,OAAOX,OACJvE,QAAQ4D,KAAMA,EAAEvF,SAAS,UACzB8D,KAAKyB,KAAMA,EAAEsB,QACb5C,OAAOgF,mBAAmBC,MAAMD,mBAAmBE;;AACvD;QAGP,OAAQrB,KAAKQ,UAAU;YACrBxC;YACAW;YACAF,cAAc,IAAIkC;YAClB5B,OAAOf,YACJhC,KAAKkB,KAAMA,EAAE6B,QACb5C,OAAOgF,mBAAmBC,MAAMD,mBAAmBE;;;IAsBnD,MAAAC;QACL,IAAItB,KAAKQ,YAAY,MACnB,MAAM,IAAI3D,MAAM;QAClB,OAAOe,8BAA8BX,SAAS+C,KAAKQ;;IAG7C,UAAMS,CACZ/C;QAEA,MAAMpG,QAAyBkI,KAAKC,OAAOsB;QAC3C,MAAM5C,aAAmB,IAAIgC;QAC7B,MAAMrG,UAAU,MACd3C,4BAA4B2C,QAAQ;YAClCI,UAAUwD,SAASxD;YACnBE,YAAY9C,MACTE,qBACA6B,QAAQqD,KAAMA,EAAEhF,SAAS,YACzB8D,KAAKkB,KAAMA,EAAExB;YAChBb,QAAQ;;QAEZ,MAAM2G,MAAM;YACV,MAAMhC,SAAS7H,4BAA4B2C,QAAQ;gBACjDI,UAAUwD,SAASxD;gBACnBE,YAAY9C,MACTE,qBACA6B,QAAQqD,KAAMA,EAAEhF,SAAS,WACzB8D,KAAKkB,KAAMA,EAAEC,aACbI,OACAvB,KAAKkB,KAAMA,EAAExB;gBAChBb,QAAQ;;YAEV,MAAMJ,OAAOH;YACb,OAAO;gBACLpC,MAAOuC,OAAO,YAAY;gBAC1ByD;gBACAsB;gBACA/E;gBACAiF,SAAS5H,MAAME;gBACf+G,OAAOjH,MAAM2J;gBACb9C;gBACAF,cAAc,IAAIkC;;AACmC;QAGzD;kBACQ7I,MAAM4J,WAAWxD,SAASlF;YAChC,IAAIsB,WAAW,OAAOkH;YACtB,KAAK,IAAIlD,IAAY,GAAGA,IAAI0B,KAAKI,QAAQhI,WAAWkG,GAAG;gBACrD,MAAMlD,aACEzD,4BAA4BC,OAAOE;gBAC3C,IAAIsD,SAAS,MAAM;sBAEbtD,MAAM4J,WAAWtG;gBACvB,IAAId,WAAW,OAAOkH;;YAExB,OAAOA;UACP,OAAO5E;YACP,OAAO;gBACL1E,MAAM;gBACNgG;gBACAwB,SAAS5H,MAAME;gBACf+G,OAAOjH,MAAM2J;gBACb7E;gBACA+B;gBACAF,cAAc,IAAIkC;;;;;;ACnLpB,IAAWgB;;CAAjB,SAAiBA;IACFA,gCAAQ1E,WACnB3E,UAEAuF,OAAOC,YAAY,EACjB,EAAC,eAAeC,WAAWzF,cACxBA,OAAO0F,YACPhC,KAAKiC,OAAQ,EACZ,EAAC,KAAKA,IAAIC,SAAS/E,kBAAkBgF,qBAAqBF,WACvDA,IAAIG,OAAOpC,KAAI,CAACqC,OAAOC,MAAM,EAC9B,KAAKL,IAAIC,SAAS/E,QAAQmF,IAAI,KAAKD,MAAMnG,WACzCqG,qBAAqBF,OAAOC,WAG/Bf;IAGP,MAAMQ,aACJzF;QAEA,MAAM8F,SAAiD9F,OAAO0F,YAC3DhC,KAAKE,KAAMA,EAAEkC,SACbb;QACH,MAAMiB,UACJJ,OACGpC,KAAKyB,KAAMA,EAAEgB,aAAaC,YAAYjB,EAAEkB,WAAWD,YACnDvC,QAAO,CAACyC,GAAGC,MAAMD,IAAIC,IAAG,KAAKT,OAAO9C;QACzC,MAAMwD,YAA2CxG,OAAOyG,MAAMD;QAC9D,OAAO,EACL,sCACA,cACA,oBACA,qBAAqBxG,OAAO0F,YAAY1C,OAAO0D,oBAC/C,gBAAgBZ,OAAO9C,UACvB,kBAAkB8C,OAAOvE,QAAQ4D,KAAMA,EAAEvF,SAAS,YAAWoD,UAC7D,kBAAkB8C,OAAOvE,QAAQ4D,KAAMA,EAAEvF,SAAS,YAAWoD,UAC7D,uBAAuBiB,SAASC,MAAMgC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU9E,MAAMiF,MAAMD,oBACxC,mBAAmBF,UAAU9E,MAAMkF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,+BACA,kCACG1G,OAAO0F,YAAYhC,KAAKiC,OACzB,EACE,IAAIA,IAAIC,SAAS/E,WAAW8E,IAAIC,SAAS/E,mBACzC;YACE,MAAMmB,UAAkB+B,KAAKI,MAC1BwB,IAAIG,OAAOvE,QAAQ4D,KAAMA,EAAEvF,SAAS,YAAWoD,SAC9C2C,IAAIG,OAAO9C,SACX;YAEJ,OACE,IAAIsE,MAAMtF,SAASuF,KAAK,KAAK9G,KAAK,MAClC,IAAI6G,MAAM,KAAKtF,SAASuF,KAAK,KAAK9G,KAAK;AAE1C,UAVD,IAWAwD,SAASC,MACPyB,IAAIG,OACDpC,KACEqC,SACCA,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,YAEnDvC,QAAO,CAACyC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO9C,QAC3C0D,mBAAmB,QACrBjG,KAAK,WAETA,KAAK;AAAK;IAGd,MAAMoF,uBACJF;QAEA,MAAMa,YAA2Cb,IAAIc,MAAMD;QAC3D,OAAO,EACL,KAAKb,IAAIC,SAAS/E,QAClB,cACA,oBACA,gBAAgB8E,IAAIG,OAAO9C,UAC3B,kBAAkB2C,IAAIG,OAAOvE,QAAQ4D,KAAMA,EAAEvF,SAAS,YAAWoD,UACjE,kBAAkB2C,IAAIG,OAAOvE,QAAQ4D,KAAMA,EAAEvF,SAAS,YAAWoD,UACjE,uBAAuBiB,SAASC,MAC9ByB,IAAIG,OACDpC,KACEqC,SACCA,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,YAEnDvC,QAAO,CAACyC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO9C,QAC3C0D,uBACF,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU9E,MAAMiF,MAAMD,oBACxC,mBAAmBF,UAAU9E,MAAMkF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,aACA,qBACA,wBACGf,IAAIG,OAAOpC,KAAI,CAACyB,GAAGa,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKb,EAAEvF,YAC5BuF,EAAEvF,MACFqE,SAASC,MAAMiB,EAAEgB,aAAaC,YAAYjB,EAAEkB,WAAWD,aACrD,QACF3F,KAAK,UAET,IACA,eACA,mBACAkF,IAAIC,SAASlF,MACb,IACA,gBACA,WACAiB,KAAK0D,UACHjB,sBAAsBK,eAAekB,IAAIC,SAASxD,WAClD,MACA,IAEF,QACA3B,KAAK;AAAK;IAGd,MAAMwF,uBAAuB,CAC3BF,OACAoB,UAEO,EACL,KAAKA,QAAQ,MAAMpB,MAAMnG,QACzB,cACA,aAAamG,MAAMH,SAAS/E,QAC5B,aAAakF,MAAMnG,QACnB,cAAcmG,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,WAAWM,0BACrEX,MAAMnG,SAAS,UACf,EACE,mBACA,gBAAgBmG,MAAMU,MAAMD,UAAUE,oBACtC,gBACA,kBAAkBX,MAAMU,MAAMD,UAAU9E,MAAMiF,MAAMD,oBACpD,mBAAmBX,MAAMU,MAAMD,UAAU9E,MAAMkF,OAAOF,oBACtD,qBACA,kBAAkBX,MAAMU,MAAMD,UAAUK,OAAOF,MAAMD,oBACrD,sBAAsBX,MAAMU,MAAMD,UAAUK,OAAOC,UAAUJ,oBAC7D,gCAAgCX,MAAMU,MAAMD,UAAUK,OAAOE,oBAAoBL,oBACjF,gCAAgCX,MAAMU,MAAMD,UAAUK,OAAOG,oBAAoBN,uBAEnF,IACJ,IACA,eACA,mBACAX,MAAMH,SAASlF,MACf,IACA,gBACA,WACAiB,KAAK0D,UACHjB,sBAAsBK,eAAesB,MAAMH,SAASxD,WACpD,MACA,IAEF,OACA,OACI2D,MAAMnG,SAAS,aAAamG,MAAMnG,SAAS,YAC3C,EACE,gBACGmG,MAAMuD,SAAS5F,KAAKoB,KACrB,EACE,OAAOA,EAAE1B,UAAUvC,QACnB,qBAAqBiE,EAAE1B,UAAU2B,WAAWlE,UAC5C,mBAAmBiE,EAAE1B,UAAUxC,SAASC,UACxC,eAAeiE,EAAEE,UACjB,OACIF,EAAE1B,UAAUxC,SAASE,cACrB,EAACgE,EAAE1B,UAAUxC,SAASE,aAAa,OACnC,KACJL,KAAK,YAGX,OACAsF,MAAMnG,SAAS,UACf,EACE,YACA,WACAwE,sBAAsBC,YACpB1C,KAAK0D,UAAUU,MAAMzB,OAAO,MAAM,KAEpC,OACA,OAEF,KACJ7D,KAAK;AAEV,EA3MD,CAAiB4I,oCAAAA,kCA2MhB,CAAA;;MClLYE;IAYX,WAAA9B,CAAmBxF;QACjByF,KAAKC,SAAS1F,MAAMzC;QACpBkI,KAAKE,aAAa3F,MAAM4F,UAAU3E;QAClCwE,KAAKI,UAAU;YACbC,QAAQ9F,MAAM+F,QAAQD,UAAU;YAChCE,cAAchG,MAAM+F,QAAQC,gBAAgB;;QAE9CP,KAAK8B,aAAavH,MAAMzC,MAAME,qBAAqBwD;QACnDwE,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAM/B,aAAmB,IAAIgC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMvC,oBACE8C,QAAQC,IACZf,KAAKE,WAAWlE,KAAInE,MAAOqG;YACzB,MAAME,eACE0C,QAAQC,IACZ,IAAInB,MAAMI,KAAKI,QAAQC,QAAQR,KAAK,GAAG7D,KAAInE;sBACnC+I,UAAUI;gBAChB,MAAMvD,UACEuC,KAAKiB,KAAK/C;sBACZ0C,UAAUM;gBAChB,IAAIR,aAAa3G,WAAW2G,SAASjD;gBACrC,OAAOA;AAAC;YAGd,OAAO;gBACLS;gBACAE;gBACAW,OAAOX,OACJvE,QAAQ4D,KAAMA,EAAEvF,SAAS,UACzB8D,KAAKyB,KAAMA,EAAEsB,QACb5C,OAAOgF,mBAAmBC,MAAMD,mBAAmBE;;AACvD;QAGP,OAAQrB,KAAKQ,UAAU;YACrBxC;YACAW;YACAF,cAAc,IAAIkC;YAClB5B,OAAOf,YACJhC,KAAKkB,KAAMA,EAAE6B,QACb5C,OAAOgF,mBAAmBC,MAAMD,mBAAmBE;;;IAuBnD,MAAAC;QACL,IAAItB,KAAKQ,YAAY,MACnB,MAAM,IAAI3D,MAAM;QAClB,OAAO8E,gCAAgC1E,SAAS+C,KAAKQ;;IAG/C,UAAMS,CACZ/C;QAEA,MAAMS,aAAmB,IAAIgC;QAC7B;YACE,MAAM5B,QAA4BoC,mBAAmBE;YACrD,MAAM3B,gBACEqC,2BAA2BtB,QAAQ;mBACpCT,KAAKC,OAAO+B,WAAW;oBACxBC,QAAQ,IAAIC,mBAAmB;wBAC7B/J,MAAM;wBACNa,MAAMkF,SAASlF;;oBAEjB+F;;gBAEFoD,WAAWnC,KAAK8B,WAAWtG;gBAC3BsB,OAAO;gBACPsF,OAAO,MAAM;gBACbC,UAAUxK;;YAEd,MAAM+J,WAAgDlC,QACnD7F,QAAQqD,KAAMA,EAAEhF,SAAS,WACzB8D,KAAKkB,KAAMA,EAAEC,aACbI;YACH,OAAO;gBACLrF,MAAMP,4BAA4B2C,QAAQ;oBACxCI,UAAUwD,SAASxD;oBACnBE,YAAYgH,SAAS5F,KAAKoB,KAAMA,EAAE1B;qBAEhC,YACA;gBACJwC;gBACA0D;gBACA7C;gBACAuD,kBAAkB5C,QACf7F,QAAQqD,KAAMA,EAAEhF,SAAS,SACzB2B,QACEqD,KAA4CA,EAAE/E,SAAS;gBAE5DwG;gBACAF,cAAc,IAAIkC;;UAIpB,OAAO/D;YACP,OAAO;gBACL1E,MAAM;gBACNgG;gBACAtB;gBACA+B;gBACAF,cAAc,IAAIkC;;;;;;"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agentica,
|
|
1
|
+
import { Agentica, AgenticaOperation } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
import { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
|
|
4
4
|
export declare namespace AgenticaBenchmarkPredicator {
|
|
@@ -22,7 +22,7 @@ export declare namespace AgenticaBenchmarkPredicator {
|
|
|
22
22
|
/**
|
|
23
23
|
* Specified operations.
|
|
24
24
|
*/
|
|
25
|
-
operations: Array<
|
|
25
|
+
operations: Array<AgenticaOperation<Model>>;
|
|
26
26
|
/**
|
|
27
27
|
* If it's `false`, check the array and let it go even if there's something wrong between them.
|
|
28
28
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgenticaBenchmarkPredicator.js","sourceRoot":"","sources":["../../src/internal/AgenticaBenchmarkPredicator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGA,kDAA0B;AAI1B,IAAiB,2BAA2B,
|
|
1
|
+
{"version":3,"file":"AgenticaBenchmarkPredicator.js","sourceRoot":"","sources":["../../src/internal/AgenticaBenchmarkPredicator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGA,kDAA0B;AAI1B,IAAiB,2BAA2B,CAoL3C;AApLD,WAAiB,2BAA2B;IAC7B,kCAAM,GAAG,CACpB,KAAsB,EACE,EAAE;;QAC1B,MAAM,IAAI,GAAsC,KAAK;aAClD,kBAAkB,EAAE;aACpB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACV,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,MAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW;YAAE,OAAO,IAAI,CAAC;QAEpE,MAAM,OAAO,GAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAGrC,SAAS,CAAC,CAAC,CAAE,CAAC;QAClB,MAAM,MAAM,GAA0B,MAAM,KAAK,CAC/C,OAAO,CACR,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAClC;YACE,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK;YAClC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE;wBACP,+BAA+B;wBAC/B,EAAE;wBACF,qDAAqD;wBACrD,8DAA8D;wBAC9D,kDAAkD;qBACnD,CAAC,IAAI,CAAC,IAAI,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,IAAI,CAAC,IAAI;iBACnB;aACF;YACD,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,UAAU;oBAChB,QAAQ,EAAE;wBACR,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,UAAU,EAAE,OAAO,CAAC,UAAiC;qBACtD;iBACF;aACF;YACD,WAAW,EAAE,UAAU;YACvB,mBAAmB,EAAE,KAAK;SAC3B,EACD,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAC9B,CAAC;QACF,MAAM,QAAQ,GAAqD,MAAA,CACjE,MAAA,MAAA,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,OAAO,CAAC,UAAU,mCAAI,EAAE,CAC5C,CAAC,MAAM,CACN,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,UAAU,IAAI,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CACpE,0CAAG,CAAC,CAAC,CAAC;QACP,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACxC,MAAM,KAAK,GAAkB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACrE,OAAO,sLAAS,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC,CAAA,CAAC;IAEF;;;;;;;;OAQG;IACU,mCAAO,GAAG,CAAiC,KAmBvD,EAAW,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAE1C,MAAM,YAAY,GAAG,CACnB,KAA2C,EAQvC,EAAE;QACN,MAAM,IAAI,GAAG,CACX,QAA2C,EAC3C,kBAAoD,EACpD,EAAE,CACF,YAAY,CAAC;YACX,QAAQ;YACR,UAAU,EAAE,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,KAAK,CAAC,UAAU;YAClD,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CAAC;QAEL,QAAQ,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC5B,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,IAAI,IAAI,GAAG,CAAC,CAAC;gBACb,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/D,IAAI,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;gBAErC,OAAO,IAAI,EAAE,CAAC;oBACZ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;wBAClB,OAAO;4BACL,MAAM,EAAE,IAAI;4BACZ,IAAI;yBACL,CAAC;oBACJ,CAAC;oBACD,IAAI,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;wBACpC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;oBAC3B,CAAC;oBAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;oBAClE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;wBACnB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;4BAClB,IAAI,IAAI,CAAC,CAAC;4BACV,SAAS;wBACX,CAAC;wBACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;oBAC3B,CAAC;oBAED,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;oBACpB,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;gBACnC,CAAC;YACH,CAAC;YACD,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACxC,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;gBACtE,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC7B,CAAC;gBACD,OAAO;oBACL,MAAM;iBACP,CAAC;YACJ,CAAC;YACD,KAAK,OAAO;gBACV,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;oBAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAClC,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;wBACtB,OAAO,UAAU,CAAC;oBACpB,CAAC;gBACH,CAAC;gBAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAC3B,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb;;;;;;mBAMG;gBACH,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACtE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;oBAClC,OAAO;wBACL,MAAM,EAAE,IAAI;wBACZ,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;qBAC1D,CAAC;gBACJ,CAAC;gBAED,OAAO;oBACL,MAAM,EAAE,KAAK;iBACd,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC;AACJ,CAAC,EApLgB,2BAA2B,2CAA3B,2BAA2B,QAoL3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgenticaCallBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaCallBenchmarkReporter.ts"],"names":[],"mappings":";;;AAKA,gDAA6C;AAC7C,mEAAgE;AAChE,qEAAkE;AAElE,IAAiB,6BAA6B,CA6K7C;AA7KD,WAAiB,6BAA6B;IAC/B,sCAAQ,GAAG,CACtB,MAA2C,EACnB,EAAE,CAC1B,MAAM,CAAC,WAAW,CAAC;QACjB,CAAC,aAAa,EAAE,UAAU,CAAQ,MAAM,CAAC,CAAC;QAC1C,GAAG,MAAM,CAAC,WAAW;aAClB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;YACZ,CAAC,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,YAAY,EAAE,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC/D,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK;gBAClD,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC;aAC/B,CAAC;SACH,CAAC;aACD,IAAI,EAAE;KACV,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,CACjB,MAA2C,EACnC,EAAE;QACV,MAAM,MAAM,GAAyC,MAAM,CAAC,WAAW;aACpE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,IAAI,EAAE,CAAC;QACV,MAAM,OAAO,GACX,MAAM;aACH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAChD,MAAM,SAAS,
|
|
1
|
+
{"version":3,"file":"AgenticaCallBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaCallBenchmarkReporter.ts"],"names":[],"mappings":";;;AAKA,gDAA6C;AAC7C,mEAAgE;AAChE,qEAAkE;AAElE,IAAiB,6BAA6B,CA6K7C;AA7KD,WAAiB,6BAA6B;IAC/B,sCAAQ,GAAG,CACtB,MAA2C,EACnB,EAAE,CAC1B,MAAM,CAAC,WAAW,CAAC;QACjB,CAAC,aAAa,EAAE,UAAU,CAAQ,MAAM,CAAC,CAAC;QAC1C,GAAG,MAAM,CAAC,WAAW;aAClB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;YACZ,CAAC,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,YAAY,EAAE,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC/D,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK;gBAClD,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC;aAC/B,CAAC;SACH,CAAC;aACD,IAAI,EAAE;KACV,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,CACjB,MAA2C,EACnC,EAAE;QACV,MAAM,MAAM,GAAyC,MAAM,CAAC,WAAW;aACpE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,IAAI,EAAE,CAAC;QACV,MAAM,OAAO,GACX,MAAM;aACH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAChD,MAAM,SAAS,GAAkC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;QACxE,OAAO;YACL,+BAA+B;YAC/B,YAAY;YACZ,kBAAkB;YAClB,qBAAqB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YACjE,gBAAgB,MAAM,CAAC,MAAM,EAAE;YAC/B,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACrE,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACrE,uBAAuB,mBAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE,KAAK;YACpE,iBAAiB;YACjB,gBAAgB,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClD,aAAa;YACb,kBAAkB,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC1D,mBAAmB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YAC5D,eAAe;YACf,kBAAkB,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC3D,sBAAsB,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;YACnE,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YACvF,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YACvF,EAAE;YACF,gBAAgB;YAChB,mCAAmC;YACnC,oCAAoC;YACpC,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAChC;gBACE,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC,IAAI,aAAa;gBAC1D,UAAU,CACR,GAAG,CAAC,MAAM,EACV,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAC/C;gBACD,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC;gBACpE,GAAG,mBAAQ,CAAC,KAAK,CACf,GAAG,CAAC,MAAM;qBACP,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;qBAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;aACxB,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;SACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,oBAAoB,GAAG,CAC3B,GAAoD,EAC5C,EAAE;QACV,OAAO;YACL,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE;YACxB,YAAY;YACZ,mBAAmB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YACvD,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACvE,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACvE,qBAAqB,mBAAQ,CAAC,KAAK,CACjC,GAAG,CAAC,MAAM;iBACP,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;iBAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;YACvB,EAAE;YACF,WAAW;YACX,qBAAqB;YACrB,qBAAqB;YACrB,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB;gBACE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM;gBACtC,CAAC,CAAC,IAAI;gBACN,GAAG,mBAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK;aAC1E,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;YACD,EAAE;YACF,aAAa;YACb,iBAAiB;YACjB,GAAG,CAAC,QAAQ,CAAC,IAAI;YACjB,EAAE;YACF,cAAc;YACd,SAAS;YACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3D,IAAI,EACJ,CAAC,CACF;YACD,KAAK;SACN,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,oBAAoB,GAAG,CAC3B,KAAyC,EACzC,KAAa,EACL,EAAE;QACV,OAAO;YACL,KAAK,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,EAAE;YAC/B,YAAY;YACZ,aAAa,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE;YAClC,aAAa,KAAK,CAAC,IAAI,EAAE;YACzB,aAAa,mBAAQ,CAAC,KAAK,CACzB,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC1D,CAAC,cAAc,EAAE,KAAK;YACvB,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;gBACxB,CAAC,CAAC;oBACE,eAAe,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE;oBACzC,aAAa,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE;iBACtC;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,oBAAoB,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClD,EAAE;YACF,aAAa;YACb,iBAAiB;YACjB,KAAK,CAAC,QAAQ,CAAC,IAAI;YACnB,EAAE;YACF,cAAc;YACd,SAAS;YACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC7D,IAAI,EACJ,CAAC,CACF;YACD,KAAK;YACL,EAAE;YACF,qBAAqB;YACrB,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,+CAAsB,CAAC,QAAQ,CAAC;YACrD,EAAE;YACF,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;gBACxB,CAAC,CAAC;oBACE,UAAU;oBACV,SAAS;oBACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,EAC9C,IAAI,EACJ,CAAC,CACF;oBACD,KAAK;iBACN;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,CACjB,MAA4C,EAC5C,OAA2D,EACnD,EAAE;QACV,MAAM,KAAK,GAAW,IAAI,CAAC,KAAK,CAC9B,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CACrD,CAAC;QACF,OAAO,CACL,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CACzC,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC,EA7KgB,6BAA6B,6CAA7B,6BAA6B,QA6K7C"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaPrompt } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
export declare namespace AgenticaPromptReporter {
|
|
4
|
-
const markdown: <Model extends ILlmSchema.Model>(p:
|
|
4
|
+
const markdown: <Model extends ILlmSchema.Model>(p: AgenticaPrompt<Model>) => string;
|
|
5
5
|
}
|
|
@@ -9,17 +9,17 @@ var AgenticaPromptReporter;
|
|
|
9
9
|
else if (p.type === "select" || p.type === "cancel")
|
|
10
10
|
return [
|
|
11
11
|
`### ${p.type === "select" ? "Select" : "Cancel"}`,
|
|
12
|
-
...p.
|
|
13
|
-
.map((
|
|
12
|
+
...p.selections
|
|
13
|
+
.map((s) => {
|
|
14
14
|
var _a;
|
|
15
15
|
return [
|
|
16
|
-
`#### ${
|
|
17
|
-
` - controller: ${
|
|
18
|
-
` - function: ${
|
|
19
|
-
` - reason: ${
|
|
16
|
+
`#### ${s.operation.name}`,
|
|
17
|
+
` - controller: ${s.operation.controller.name}`,
|
|
18
|
+
` - function: ${s.operation.function.name}`,
|
|
19
|
+
` - reason: ${s.reason}`,
|
|
20
20
|
"",
|
|
21
|
-
...(!!((_a =
|
|
22
|
-
? [
|
|
21
|
+
...(!!((_a = s.operation.function.description) === null || _a === void 0 ? void 0 : _a.length)
|
|
22
|
+
? [s.operation.function.description, ""]
|
|
23
23
|
: []),
|
|
24
24
|
];
|
|
25
25
|
})
|
|
@@ -28,16 +28,16 @@ var AgenticaPromptReporter;
|
|
|
28
28
|
else if (p.type === "describe")
|
|
29
29
|
return [
|
|
30
30
|
"### Describe",
|
|
31
|
-
...p.
|
|
31
|
+
...p.executes.map((e) => ` - ${e.operation.name}`),
|
|
32
32
|
"",
|
|
33
33
|
...p.text.split("\n").map((s) => `> ${s}`),
|
|
34
34
|
"",
|
|
35
35
|
].join("\n");
|
|
36
36
|
return [
|
|
37
37
|
"### Execute",
|
|
38
|
-
` - name: ${p.name}`,
|
|
39
|
-
` - controller: ${p.controller.name}`,
|
|
40
|
-
` - function: ${p.function.name}`,
|
|
38
|
+
` - name: ${p.operation.name}`,
|
|
39
|
+
` - controller: ${p.operation.controller.name}`,
|
|
40
|
+
` - function: ${p.operation.function.name}`,
|
|
41
41
|
"",
|
|
42
42
|
"```json",
|
|
43
43
|
JSON.stringify(p.arguments, null, 2),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgenticaPromptReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaPromptReporter.ts"],"names":[],"mappings":";;;AAGA,IAAiB,sBAAsB,CA0CtC;AA1CD,WAAiB,sBAAsB;IACxB,+BAAQ,GAAG,CACtB,
|
|
1
|
+
{"version":3,"file":"AgenticaPromptReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaPromptReporter.ts"],"names":[],"mappings":";;;AAGA,IAAiB,sBAAsB,CA0CtC;AA1CD,WAAiB,sBAAsB;IACxB,+BAAQ,GAAG,CACtB,CAAwB,EAChB,EAAE;QACV,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YACnB,OAAO,CAAC,aAAa,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACpD,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ;YACjD,OAAO;gBACL,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE;gBAClD,GAAG,CAAC,CAAC,UAAU;qBACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;;oBAAC,OAAA;wBACV,QAAQ,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE;wBAC1B,mBAAmB,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE;wBAChD,iBAAiB,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE;wBAC5C,eAAe,CAAC,CAAC,MAAM,EAAE;wBACzB,EAAE;wBACF,GAAG,CAAC,CAAC,CAAC,CAAA,MAAA,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,0CAAE,MAAM,CAAA;4BAC5C,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;4BACxC,CAAC,CAAC,EAAE,CAAC;qBACR,CAAA;iBAAA,CAAC;qBACD,IAAI,EAAE;aACV,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACV,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU;YAC5B,OAAO;gBACL,cAAc;gBACd,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnD,EAAE;gBACF,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1C,EAAE;aACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,OAAO;YACL,aAAa;YACb,aAAa,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE;YAC/B,mBAAmB,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE;YAChD,iBAAiB,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE;YAC5C,EAAE;YACF,SAAS;YACT,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACpC,KAAK;YACL,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,EA1CgB,sBAAsB,sCAAtB,sBAAsB,QA0CtC"}
|
|
@@ -144,12 +144,14 @@ var AgenticaSelectBenchmarkReporter;
|
|
|
144
144
|
? [
|
|
145
145
|
"## Result",
|
|
146
146
|
...event.selected.map((s) => [
|
|
147
|
-
`### ${s.name}`,
|
|
148
|
-
` - Controller: \`${s.controller.name}\``,
|
|
149
|
-
` - Function: \`${s.function.name}\``,
|
|
147
|
+
`### ${s.operation.name}`,
|
|
148
|
+
` - Controller: \`${s.operation.controller.name}\``,
|
|
149
|
+
` - Function: \`${s.operation.function.name}\``,
|
|
150
150
|
` - Reason: ${s.reason}`,
|
|
151
151
|
"",
|
|
152
|
-
...(s.
|
|
152
|
+
...(s.operation.function.description
|
|
153
|
+
? [s.operation.function.description, ""]
|
|
154
|
+
: []),
|
|
153
155
|
].join("\n")),
|
|
154
156
|
]
|
|
155
157
|
: []),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgenticaSelectBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaSelectBenchmarkReporter.ts"],"names":[],"mappings":";;;AAKA,gDAA6C;AAC7C,mEAAgE;AAEhE;;GAEG;AACH,IAAiB,+BAA+B,
|
|
1
|
+
{"version":3,"file":"AgenticaSelectBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaSelectBenchmarkReporter.ts"],"names":[],"mappings":";;;AAKA,gDAA6C;AAC7C,mEAAgE;AAEhE;;GAEG;AACH,IAAiB,+BAA+B,CA2M/C;AA3MD,WAAiB,+BAA+B;IACjC,wCAAQ,GAAG,CACtB,MAA6C,EACrB,EAAE,CAC1B,MAAM,CAAC,WAAW,CAAC;QACjB,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACnC,GAAG,MAAM,CAAC,WAAW;aAClB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;YACZ,CAAC,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,YAAY,EAAE,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC/D,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK;gBAClD,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC;aAC/B,CAAC;SACH,CAAC;aACD,IAAI,EAAE;KACV,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,CACjB,MAA6C,EACrC,EAAE;QACV,MAAM,MAAM,GAA2C,MAAM,CAAC,WAAW;aACtE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,IAAI,EAAE,CAAC;QACV,MAAM,OAAO,GACX,MAAM;aACH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAChD,MAAM,SAAS,GAAkC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;QACxE,OAAO;YACL,oCAAoC;YACpC,YAAY;YACZ,kBAAkB;YAClB,qBAAqB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YACjE,gBAAgB,MAAM,CAAC,MAAM,EAAE;YAC/B,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACrE,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACrE,uBAAuB,mBAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE,KAAK;YACpE,iBAAiB;YACjB,gBAAgB,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClD,aAAa;YACb,kBAAkB,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC1D,mBAAmB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YAC5D,eAAe;YACf,kBAAkB,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC3D,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YACvF,sBAAsB,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;YACnE,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YACvF,EAAE;YACF,gBAAgB;YAChB,6BAA6B;YAC7B,6BAA6B;YAC7B,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAChC;gBACE,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC,IAAI,aAAa;gBAC1D,CAAC,GAAG,EAAE;oBACJ,MAAM,OAAO,GAAW,IAAI,CAAC,KAAK,CAChC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM;wBACpD,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;wBAClB,EAAE,CACL,CAAC;oBACF,OAAO,CACL,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;wBACrC,IAAI,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAC3C,CAAC;gBACJ,CAAC,CAAC,EAAE;gBACJ,mBAAQ,CAAC,KAAK,CACZ,GAAG,CAAC,MAAM;qBACP,GAAG,CACF,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC5D;qBACA,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,GAAG,KAAK;aAC3B,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;SACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,oBAAoB,GAAG,CAC3B,GAAsD,EAC9C,EAAE;QACV,MAAM,SAAS,GAAkC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC;QACrE,OAAO;YACL,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE;YACxB,YAAY;YACZ,kBAAkB;YAClB,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE;YACnC,kBAAkB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACzE,kBAAkB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACzE,uBAAuB,mBAAQ,CAAC,KAAK,CACnC,GAAG,CAAC,MAAM;iBACP,GAAG,CACF,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC5D;iBACA,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;YACvB,iBAAiB;YACjB,gBAAgB,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClD,aAAa;YACb,kBAAkB,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC1D,mBAAmB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YAC5D,eAAe;YACf,kBAAkB,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC3D,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YACvF,sBAAsB,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;YACnE,gCAAgC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YACvF,EAAE;YACF,WAAW;YACX,mBAAmB;YACnB,mBAAmB;YACnB,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB;gBACE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM;gBACtC,CAAC,CAAC,IAAI;gBACN,mBAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;oBAC/D,KAAK;aACR,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;YACD,EAAE;YACF,aAAa;YACb,iBAAiB;YACjB,GAAG,CAAC,QAAQ,CAAC,IAAI;YACjB,EAAE;YACF,cAAc;YACd,SAAS;YACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3D,IAAI,EACJ,CAAC,CACF;YACD,KAAK;SACN,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,oBAAoB,GAAG,CAC3B,KAA2C,EAC3C,KAAa,EACL,EAAE;QACV,OAAO;YACL,KAAK,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,EAAE;YAC/B,YAAY;YACZ,aAAa,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE;YAClC,aAAa,KAAK,CAAC,IAAI,EAAE;YACzB,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,cAAc,EAAE,KAAK;YAC9F,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;gBACxB,CAAC,CAAC;oBACE,iBAAiB;oBACjB,gBAAgB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;oBACxD,cAAc;oBACd,kBAAkB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;oBACtE,mBAAmB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;oBACxE,mBAAmB;oBACnB,kBAAkB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;oBACvE,sBAAsB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;oBAC/E,gCAAgC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;oBACnG,gCAAgC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;iBACpG;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,EAAE;YACF,aAAa;YACb,iBAAiB;YACjB,KAAK,CAAC,QAAQ,CAAC,IAAI;YACnB,EAAE;YACF,cAAc;YACd,SAAS;YACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC7D,IAAI,EACJ,CAAC,CACF;YACD,KAAK;YACL,EAAE;YACF,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;gBACtD,CAAC,CAAC;oBACE,WAAW;oBACX,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1B;wBACE,OAAO,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE;wBACzB,qBAAqB,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,IAAI;wBACpD,mBAAmB,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,IAAI;wBAChD,eAAe,CAAC,CAAC,MAAM,EAAE;wBACzB,EAAE;wBACF,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW;4BAClC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;4BACxC,CAAC,CAAC,EAAE,CAAC;qBACR,CAAC,IAAI,CAAC,IAAI,CAAC,CACb;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;gBACxB,CAAC,CAAC;oBACE,UAAU;oBACV,SAAS;oBACT,6CAAqB,CAAC,WAAW,CAC/B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CACrC;oBACD,KAAK;oBACL,EAAE;iBACH;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,EA3MgB,+BAA+B,+CAA/B,+BAA+B,QA2M/C"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaOperation } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
/**
|
|
4
4
|
* Expected operation determinant.
|
|
@@ -40,6 +40,6 @@ export declare namespace IAgenticaBenchmarkExpected {
|
|
|
40
40
|
*/
|
|
41
41
|
interface IStandalone<Model extends ILlmSchema.Model> {
|
|
42
42
|
type: "standalone";
|
|
43
|
-
operation:
|
|
43
|
+
operation: AgenticaOperation<Model>;
|
|
44
44
|
}
|
|
45
45
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaPrompt, AgenticaTokenUsage } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
import { IAgenticaCallBenchmarkScenario } from "./IAgenticaCallBenchmarkScenario";
|
|
4
4
|
/**
|
|
@@ -78,11 +78,11 @@ export declare namespace IAgenticaCallBenchmarkEvent {
|
|
|
78
78
|
*
|
|
79
79
|
* List of prompts occurred during the benchmark testing.
|
|
80
80
|
*/
|
|
81
|
-
prompts:
|
|
81
|
+
prompts: AgenticaPrompt<Model>[];
|
|
82
82
|
/**
|
|
83
83
|
* Usage of the token during the benchmark.
|
|
84
84
|
*/
|
|
85
|
-
usage:
|
|
85
|
+
usage: AgenticaTokenUsage;
|
|
86
86
|
/**
|
|
87
87
|
* When the benchmark testing started.
|
|
88
88
|
*/
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaTokenUsage } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
import { IAgenticaCallBenchmarkEvent } from "./IAgenticaCallBenchmarkEvent";
|
|
4
4
|
import { IAgenticaCallBenchmarkScenario } from "./IAgenticaCallBenchmarkScenario";
|
|
@@ -27,7 +27,7 @@ export interface IAgenticaCallBenchmarkResult<Model extends ILlmSchema.Model> {
|
|
|
27
27
|
/**
|
|
28
28
|
* Aggregated token usage information.
|
|
29
29
|
*/
|
|
30
|
-
usage:
|
|
30
|
+
usage: AgenticaTokenUsage;
|
|
31
31
|
/**
|
|
32
32
|
* Start time of the benchmark.
|
|
33
33
|
*/
|
|
@@ -58,6 +58,6 @@ export declare namespace IAgenticaCallBenchmarkResult {
|
|
|
58
58
|
/**
|
|
59
59
|
* LLM token usage information.
|
|
60
60
|
*/
|
|
61
|
-
usage:
|
|
61
|
+
usage: AgenticaTokenUsage;
|
|
62
62
|
}
|
|
63
63
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaOperationSelection, AgenticaTextPrompt, AgenticaTokenUsage } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
import { IAgenticaSelectBenchmarkScenario } from "./IAgenticaSelectBenchmarkScenario";
|
|
4
4
|
/**
|
|
@@ -35,15 +35,15 @@ export declare namespace IAgenticaSelectBenchmarkEvent {
|
|
|
35
35
|
/**
|
|
36
36
|
* Usage of the token during the benchmark.
|
|
37
37
|
*/
|
|
38
|
-
usage:
|
|
38
|
+
usage: AgenticaTokenUsage;
|
|
39
39
|
/**
|
|
40
40
|
* Selected operations in the benchmark.
|
|
41
41
|
*/
|
|
42
|
-
selected:
|
|
42
|
+
selected: AgenticaOperationSelection<Model>[];
|
|
43
43
|
/**
|
|
44
44
|
* Prompt messages from the assistant.
|
|
45
45
|
*/
|
|
46
|
-
assistantPrompts:
|
|
46
|
+
assistantPrompts: AgenticaTextPrompt<"assistant">[];
|
|
47
47
|
}
|
|
48
48
|
/**
|
|
49
49
|
* Failure event type.
|
|
@@ -55,15 +55,15 @@ export declare namespace IAgenticaSelectBenchmarkEvent {
|
|
|
55
55
|
/**
|
|
56
56
|
* Usage of the token during the benchmark.
|
|
57
57
|
*/
|
|
58
|
-
usage:
|
|
58
|
+
usage: AgenticaTokenUsage;
|
|
59
59
|
/**
|
|
60
60
|
* Selected operations in the benchmark.
|
|
61
61
|
*/
|
|
62
|
-
selected:
|
|
62
|
+
selected: AgenticaOperationSelection<Model>[];
|
|
63
63
|
/**
|
|
64
64
|
* Prompt messages from the assistant.
|
|
65
65
|
*/
|
|
66
|
-
assistantPrompts:
|
|
66
|
+
assistantPrompts: AgenticaTextPrompt<"assistant">[];
|
|
67
67
|
}
|
|
68
68
|
/**
|
|
69
69
|
* Error event type.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaTokenUsage } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
import { IAgenticaSelectBenchmarkEvent } from "./IAgenticaSelectBenchmarkEvent";
|
|
4
4
|
import { IAgenticaSelectBenchmarkScenario } from "./IAgenticaSelectBenchmarkScenario";
|
|
@@ -27,7 +27,7 @@ export interface IAgenticaSelectBenchmarkResult<Model extends ILlmSchema.Model>
|
|
|
27
27
|
/**
|
|
28
28
|
* Aggregated token usage information.
|
|
29
29
|
*/
|
|
30
|
-
usage:
|
|
30
|
+
usage: AgenticaTokenUsage;
|
|
31
31
|
/**
|
|
32
32
|
* Start time of the benchmark.
|
|
33
33
|
*/
|
|
@@ -58,6 +58,6 @@ export declare namespace IAgenticaSelectBenchmarkResult {
|
|
|
58
58
|
/**
|
|
59
59
|
* LLM token usage information.
|
|
60
60
|
*/
|
|
61
|
-
usage:
|
|
61
|
+
usage: AgenticaTokenUsage;
|
|
62
62
|
}
|
|
63
63
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentica/benchmark",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.1",
|
|
4
4
|
"main": "lib/index.js",
|
|
5
5
|
"description": "Agentic AI Library specialized in LLM Function Calling",
|
|
6
6
|
"scripts": {
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"src"
|
|
38
38
|
],
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@agentica/core": "^0.
|
|
40
|
+
"@agentica/core": "^0.11.1",
|
|
41
41
|
"@samchon/openapi": "^3.0.0",
|
|
42
42
|
"openai": "^4.80.0",
|
|
43
43
|
"tstl": "^3.0.0",
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { Agentica } from "@agentica/core";
|
|
2
|
-
import { AgenticaTokenUsageAggregator } from "@agentica/core/src/internal/AgenticaTokenUsageAggregator";
|
|
1
|
+
import { Agentica, AgenticaTokenUsage } from "@agentica/core";
|
|
3
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
4
3
|
import { Semaphore } from "tstl";
|
|
5
4
|
import { tags } from "typia";
|
|
@@ -93,10 +92,7 @@ export class AgenticaCallBenchmark<Model extends ILlmSchema.Model> {
|
|
|
93
92
|
usage: events
|
|
94
93
|
.filter((e) => e.type !== "error")
|
|
95
94
|
.map((e) => e.usage)
|
|
96
|
-
.reduce(
|
|
97
|
-
AgenticaTokenUsageAggregator.plus,
|
|
98
|
-
AgenticaTokenUsageAggregator.zero(),
|
|
99
|
-
),
|
|
95
|
+
.reduce(AgenticaTokenUsage.plus, AgenticaTokenUsage.zero()),
|
|
100
96
|
};
|
|
101
97
|
}),
|
|
102
98
|
);
|
|
@@ -106,10 +102,7 @@ export class AgenticaCallBenchmark<Model extends ILlmSchema.Model> {
|
|
|
106
102
|
completed_at: new Date(),
|
|
107
103
|
usage: experiments
|
|
108
104
|
.map((p) => p.usage)
|
|
109
|
-
.reduce(
|
|
110
|
-
AgenticaTokenUsageAggregator.plus,
|
|
111
|
-
AgenticaTokenUsageAggregator.zero(),
|
|
112
|
-
),
|
|
105
|
+
.reduce(AgenticaTokenUsage.plus, AgenticaTokenUsage.zero()),
|
|
113
106
|
});
|
|
114
107
|
}
|
|
115
108
|
|
|
@@ -147,7 +140,8 @@ export class AgenticaCallBenchmark<Model extends ILlmSchema.Model> {
|
|
|
147
140
|
expected: scenario.expected,
|
|
148
141
|
operations: agent
|
|
149
142
|
.getPromptHistories()
|
|
150
|
-
.filter((p) => p.type === "execute")
|
|
143
|
+
.filter((p) => p.type === "execute")
|
|
144
|
+
.map((p) => p.operation),
|
|
151
145
|
strict: false,
|
|
152
146
|
});
|
|
153
147
|
const out = (): IAgenticaCallBenchmarkEvent<Model> => {
|
|
@@ -156,8 +150,9 @@ export class AgenticaCallBenchmark<Model extends ILlmSchema.Model> {
|
|
|
156
150
|
operations: agent
|
|
157
151
|
.getPromptHistories()
|
|
158
152
|
.filter((p) => p.type === "select")
|
|
159
|
-
.map((p) => p.
|
|
160
|
-
.flat()
|
|
153
|
+
.map((p) => p.selections)
|
|
154
|
+
.flat()
|
|
155
|
+
.map((p) => p.operation),
|
|
161
156
|
strict: false,
|
|
162
157
|
});
|
|
163
158
|
const call = success();
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
Agentica,
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
AgenticaContext,
|
|
4
|
+
AgenticaOperationSelection,
|
|
5
|
+
AgenticaPrompt,
|
|
6
|
+
AgenticaTextPrompt,
|
|
7
|
+
AgenticaTokenUsage,
|
|
7
8
|
} from "@agentica/core";
|
|
8
9
|
import { ChatGptSelectFunctionAgent } from "@agentica/core/src/chatgpt/ChatGptSelectFunctionAgent";
|
|
9
|
-
import { AgenticaTokenUsageAggregator } from "@agentica/core/src/internal/AgenticaTokenUsageAggregator";
|
|
10
10
|
import { ILlmSchema } from "@samchon/openapi";
|
|
11
11
|
import { Semaphore } from "tstl";
|
|
12
12
|
import { tags } from "typia";
|
|
@@ -38,7 +38,7 @@ export class AgenticaSelectBenchmark<Model extends ILlmSchema.Model> {
|
|
|
38
38
|
private agent_: Agentica<Model>;
|
|
39
39
|
private scenarios_: IAgenticaSelectBenchmarkScenario<Model>[];
|
|
40
40
|
private config_: AgenticaSelectBenchmark.IConfig;
|
|
41
|
-
private histories_:
|
|
41
|
+
private histories_: AgenticaPrompt<Model>[];
|
|
42
42
|
private result_: IAgenticaSelectBenchmarkResult<Model> | null;
|
|
43
43
|
|
|
44
44
|
/**
|
|
@@ -98,10 +98,7 @@ export class AgenticaSelectBenchmark<Model extends ILlmSchema.Model> {
|
|
|
98
98
|
usage: events
|
|
99
99
|
.filter((e) => e.type !== "error")
|
|
100
100
|
.map((e) => e.usage)
|
|
101
|
-
.reduce(
|
|
102
|
-
AgenticaTokenUsageAggregator.plus,
|
|
103
|
-
AgenticaTokenUsageAggregator.zero(),
|
|
104
|
-
),
|
|
101
|
+
.reduce(AgenticaTokenUsage.plus, AgenticaTokenUsage.zero()),
|
|
105
102
|
};
|
|
106
103
|
}),
|
|
107
104
|
);
|
|
@@ -111,10 +108,7 @@ export class AgenticaSelectBenchmark<Model extends ILlmSchema.Model> {
|
|
|
111
108
|
completed_at: new Date(),
|
|
112
109
|
usage: experiments
|
|
113
110
|
.map((p) => p.usage)
|
|
114
|
-
.reduce(
|
|
115
|
-
AgenticaTokenUsageAggregator.plus,
|
|
116
|
-
AgenticaTokenUsageAggregator.zero(),
|
|
117
|
-
),
|
|
111
|
+
.reduce(AgenticaTokenUsage.plus, AgenticaTokenUsage.zero()),
|
|
118
112
|
});
|
|
119
113
|
}
|
|
120
114
|
|
|
@@ -148,30 +142,29 @@ export class AgenticaSelectBenchmark<Model extends ILlmSchema.Model> {
|
|
|
148
142
|
): Promise<IAgenticaSelectBenchmarkEvent<Model>> {
|
|
149
143
|
const started_at: Date = new Date();
|
|
150
144
|
try {
|
|
151
|
-
const usage:
|
|
152
|
-
const prompts:
|
|
145
|
+
const usage: AgenticaTokenUsage = AgenticaTokenUsage.zero();
|
|
146
|
+
const prompts: AgenticaPrompt<Model>[] =
|
|
153
147
|
await ChatGptSelectFunctionAgent.execute({
|
|
154
148
|
...this.agent_.getContext({
|
|
155
|
-
prompt: {
|
|
156
|
-
type: "text",
|
|
149
|
+
prompt: new AgenticaTextPrompt({
|
|
157
150
|
role: "user",
|
|
158
151
|
text: scenario.text,
|
|
159
|
-
},
|
|
152
|
+
}),
|
|
160
153
|
usage,
|
|
161
154
|
}),
|
|
162
155
|
histories: this.histories_.slice(),
|
|
163
156
|
stack: [],
|
|
164
157
|
ready: () => true,
|
|
165
158
|
dispatch: async () => {},
|
|
166
|
-
} satisfies
|
|
167
|
-
const selected:
|
|
159
|
+
} satisfies AgenticaContext<Model>);
|
|
160
|
+
const selected: AgenticaOperationSelection<Model>[] = prompts
|
|
168
161
|
.filter((p) => p.type === "select")
|
|
169
|
-
.map((p) => p.
|
|
162
|
+
.map((p) => p.selections)
|
|
170
163
|
.flat();
|
|
171
164
|
return {
|
|
172
165
|
type: AgenticaBenchmarkPredicator.success({
|
|
173
166
|
expected: scenario.expected,
|
|
174
|
-
operations: selected,
|
|
167
|
+
operations: selected.map((s) => s.operation),
|
|
175
168
|
})
|
|
176
169
|
? "success"
|
|
177
170
|
: "failure",
|
|
@@ -181,8 +174,7 @@ export class AgenticaSelectBenchmark<Model extends ILlmSchema.Model> {
|
|
|
181
174
|
assistantPrompts: prompts
|
|
182
175
|
.filter((p) => p.type === "text")
|
|
183
176
|
.filter(
|
|
184
|
-
(p): p is
|
|
185
|
-
p.role === "assistant",
|
|
177
|
+
(p): p is AgenticaTextPrompt<"assistant"> => p.role === "assistant",
|
|
186
178
|
),
|
|
187
179
|
started_at,
|
|
188
180
|
completed_at: new Date(),
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agentica,
|
|
1
|
+
import { Agentica, AgenticaOperation, AgenticaPrompt } from "@agentica/core";
|
|
2
2
|
import { ILlmFunction, ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
import OpenAI from "openai";
|
|
4
4
|
import typia from "typia";
|
|
@@ -9,7 +9,7 @@ export namespace AgenticaBenchmarkPredicator {
|
|
|
9
9
|
export const isNext = async <Model extends ILlmSchema.Model>(
|
|
10
10
|
agent: Agentica<Model>,
|
|
11
11
|
): Promise<string | null> => {
|
|
12
|
-
const last:
|
|
12
|
+
const last: AgenticaPrompt<Model> | undefined = agent
|
|
13
13
|
.getPromptHistories()
|
|
14
14
|
.at(-1);
|
|
15
15
|
if (last?.type !== "text" || last.role !== "assistant") return null;
|
|
@@ -84,9 +84,7 @@ export namespace AgenticaBenchmarkPredicator {
|
|
|
84
84
|
/**
|
|
85
85
|
* Specified operations.
|
|
86
86
|
*/
|
|
87
|
-
operations: Array<
|
|
88
|
-
IAgenticaOperation<Model> | IAgenticaPrompt.IExecute<Model>
|
|
89
|
-
>;
|
|
87
|
+
operations: Array<AgenticaOperation<Model>>;
|
|
90
88
|
|
|
91
89
|
/**
|
|
92
90
|
* If it's `false`, check the array and let it go even if there's something wrong between them.
|
|
@@ -108,9 +106,7 @@ export namespace AgenticaBenchmarkPredicator {
|
|
|
108
106
|
} => {
|
|
109
107
|
const call = (
|
|
110
108
|
expected: IAgenticaBenchmarkExpected<Model>,
|
|
111
|
-
overrideOperations?: Array<
|
|
112
|
-
IAgenticaOperation<Model> | IAgenticaPrompt.IExecute<Model>
|
|
113
|
-
>,
|
|
109
|
+
overrideOperations?: Array<AgenticaOperation<Model>>,
|
|
114
110
|
) =>
|
|
115
111
|
successInner({
|
|
116
112
|
expected,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaTokenUsage } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
|
|
4
4
|
import { IAgenticaCallBenchmarkEvent } from "../structures/IAgenticaCallBenchmarkEvent";
|
|
@@ -34,7 +34,7 @@ export namespace AgenticaCallBenchmarkReporter {
|
|
|
34
34
|
events
|
|
35
35
|
.map((e) => e.completed_at.getTime() - e.started_at.getTime())
|
|
36
36
|
.reduce((a, b) => a + b, 0) / events.length;
|
|
37
|
-
const aggregate:
|
|
37
|
+
const aggregate: AgenticaTokenUsage.IComponent = result.usage.aggregate;
|
|
38
38
|
return [
|
|
39
39
|
"# LLM Function Call Benchmark",
|
|
40
40
|
"## Summary",
|
|
@@ -1,24 +1,24 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaPrompt } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
|
|
4
4
|
export namespace AgenticaPromptReporter {
|
|
5
5
|
export const markdown = <Model extends ILlmSchema.Model>(
|
|
6
|
-
p:
|
|
6
|
+
p: AgenticaPrompt<Model>,
|
|
7
7
|
): string => {
|
|
8
8
|
if (p.type === "text")
|
|
9
9
|
return [`### Text (${p.role})`, p.text, ""].join("\n");
|
|
10
10
|
else if (p.type === "select" || p.type === "cancel")
|
|
11
11
|
return [
|
|
12
12
|
`### ${p.type === "select" ? "Select" : "Cancel"}`,
|
|
13
|
-
...p.
|
|
14
|
-
.map((
|
|
15
|
-
`#### ${
|
|
16
|
-
` - controller: ${
|
|
17
|
-
` - function: ${
|
|
18
|
-
` - reason: ${
|
|
13
|
+
...p.selections
|
|
14
|
+
.map((s) => [
|
|
15
|
+
`#### ${s.operation.name}`,
|
|
16
|
+
` - controller: ${s.operation.controller.name}`,
|
|
17
|
+
` - function: ${s.operation.function.name}`,
|
|
18
|
+
` - reason: ${s.reason}`,
|
|
19
19
|
"",
|
|
20
|
-
...(!!
|
|
21
|
-
? [
|
|
20
|
+
...(!!s.operation.function.description?.length
|
|
21
|
+
? [s.operation.function.description, ""]
|
|
22
22
|
: []),
|
|
23
23
|
])
|
|
24
24
|
.flat(),
|
|
@@ -26,16 +26,16 @@ export namespace AgenticaPromptReporter {
|
|
|
26
26
|
else if (p.type === "describe")
|
|
27
27
|
return [
|
|
28
28
|
"### Describe",
|
|
29
|
-
...p.
|
|
29
|
+
...p.executes.map((e) => ` - ${e.operation.name}`),
|
|
30
30
|
"",
|
|
31
31
|
...p.text.split("\n").map((s) => `> ${s}`),
|
|
32
32
|
"",
|
|
33
33
|
].join("\n");
|
|
34
34
|
return [
|
|
35
35
|
"### Execute",
|
|
36
|
-
` - name: ${p.name}`,
|
|
37
|
-
` - controller: ${p.controller.name}`,
|
|
38
|
-
` - function: ${p.function.name}`,
|
|
36
|
+
` - name: ${p.operation.name}`,
|
|
37
|
+
` - controller: ${p.operation.controller.name}`,
|
|
38
|
+
` - function: ${p.operation.function.name}`,
|
|
39
39
|
"",
|
|
40
40
|
"```json",
|
|
41
41
|
JSON.stringify(p.arguments, null, 2),
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaTokenUsage } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
|
|
4
4
|
import { IAgenticaSelectBenchmarkEvent } from "../structures/IAgenticaSelectBenchmarkEvent";
|
|
@@ -36,7 +36,7 @@ export namespace AgenticaSelectBenchmarkReporter {
|
|
|
36
36
|
events
|
|
37
37
|
.map((e) => e.completed_at.getTime() - e.started_at.getTime())
|
|
38
38
|
.reduce((a, b) => a + b, 0) / events.length;
|
|
39
|
-
const aggregate:
|
|
39
|
+
const aggregate: AgenticaTokenUsage.IComponent = result.usage.aggregate;
|
|
40
40
|
return [
|
|
41
41
|
"# LLM Function Selection Benchmark",
|
|
42
42
|
"## Summary",
|
|
@@ -90,7 +90,7 @@ export namespace AgenticaSelectBenchmarkReporter {
|
|
|
90
90
|
const writeExperimentIndex = <Model extends ILlmSchema.Model>(
|
|
91
91
|
exp: IAgenticaSelectBenchmarkResult.IExperiment<Model>,
|
|
92
92
|
): string => {
|
|
93
|
-
const aggregate:
|
|
93
|
+
const aggregate: AgenticaTokenUsage.IComponent = exp.usage.aggregate;
|
|
94
94
|
return [
|
|
95
95
|
`# ${exp.scenario.name}`,
|
|
96
96
|
"## Summary",
|
|
@@ -187,12 +187,14 @@ export namespace AgenticaSelectBenchmarkReporter {
|
|
|
187
187
|
"## Result",
|
|
188
188
|
...event.selected.map((s) =>
|
|
189
189
|
[
|
|
190
|
-
`### ${s.name}`,
|
|
191
|
-
` - Controller: \`${s.controller.name}\``,
|
|
192
|
-
` - Function: \`${s.function.name}\``,
|
|
190
|
+
`### ${s.operation.name}`,
|
|
191
|
+
` - Controller: \`${s.operation.controller.name}\``,
|
|
192
|
+
` - Function: \`${s.operation.function.name}\``,
|
|
193
193
|
` - Reason: ${s.reason}`,
|
|
194
194
|
"",
|
|
195
|
-
...(s.
|
|
195
|
+
...(s.operation.function.description
|
|
196
|
+
? [s.operation.function.description, ""]
|
|
197
|
+
: []),
|
|
196
198
|
].join("\n"),
|
|
197
199
|
),
|
|
198
200
|
]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaOperation } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
|
|
4
4
|
/**
|
|
@@ -63,6 +63,6 @@ export namespace IAgenticaBenchmarkExpected {
|
|
|
63
63
|
*/
|
|
64
64
|
export interface IStandalone<Model extends ILlmSchema.Model> {
|
|
65
65
|
type: "standalone";
|
|
66
|
-
operation:
|
|
66
|
+
operation: AgenticaOperation<Model>;
|
|
67
67
|
}
|
|
68
68
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaPrompt, AgenticaTokenUsage } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
|
|
4
4
|
import { IAgenticaCallBenchmarkScenario } from "./IAgenticaCallBenchmarkScenario";
|
|
@@ -93,12 +93,12 @@ export namespace IAgenticaCallBenchmarkEvent {
|
|
|
93
93
|
*
|
|
94
94
|
* List of prompts occurred during the benchmark testing.
|
|
95
95
|
*/
|
|
96
|
-
prompts:
|
|
96
|
+
prompts: AgenticaPrompt<Model>[];
|
|
97
97
|
|
|
98
98
|
/**
|
|
99
99
|
* Usage of the token during the benchmark.
|
|
100
100
|
*/
|
|
101
|
-
usage:
|
|
101
|
+
usage: AgenticaTokenUsage;
|
|
102
102
|
|
|
103
103
|
/**
|
|
104
104
|
* When the benchmark testing started.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaTokenUsage } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
|
|
4
4
|
import { IAgenticaCallBenchmarkEvent } from "./IAgenticaCallBenchmarkEvent";
|
|
@@ -30,7 +30,7 @@ export interface IAgenticaCallBenchmarkResult<Model extends ILlmSchema.Model> {
|
|
|
30
30
|
/**
|
|
31
31
|
* Aggregated token usage information.
|
|
32
32
|
*/
|
|
33
|
-
usage:
|
|
33
|
+
usage: AgenticaTokenUsage;
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
36
|
* Start time of the benchmark.
|
|
@@ -65,6 +65,6 @@ export namespace IAgenticaCallBenchmarkResult {
|
|
|
65
65
|
/**
|
|
66
66
|
* LLM token usage information.
|
|
67
67
|
*/
|
|
68
|
-
usage:
|
|
68
|
+
usage: AgenticaTokenUsage;
|
|
69
69
|
}
|
|
70
70
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
AgenticaOperationSelection,
|
|
3
|
+
AgenticaTextPrompt,
|
|
4
|
+
AgenticaTokenUsage,
|
|
5
5
|
} from "@agentica/core";
|
|
6
6
|
import { ILlmSchema } from "@samchon/openapi";
|
|
7
7
|
|
|
@@ -45,17 +45,17 @@ export namespace IAgenticaSelectBenchmarkEvent {
|
|
|
45
45
|
/**
|
|
46
46
|
* Usage of the token during the benchmark.
|
|
47
47
|
*/
|
|
48
|
-
usage:
|
|
48
|
+
usage: AgenticaTokenUsage;
|
|
49
49
|
|
|
50
50
|
/**
|
|
51
51
|
* Selected operations in the benchmark.
|
|
52
52
|
*/
|
|
53
|
-
selected:
|
|
53
|
+
selected: AgenticaOperationSelection<Model>[];
|
|
54
54
|
|
|
55
55
|
/**
|
|
56
56
|
* Prompt messages from the assistant.
|
|
57
57
|
*/
|
|
58
|
-
assistantPrompts:
|
|
58
|
+
assistantPrompts: AgenticaTextPrompt<"assistant">[];
|
|
59
59
|
}
|
|
60
60
|
|
|
61
61
|
/**
|
|
@@ -69,17 +69,17 @@ export namespace IAgenticaSelectBenchmarkEvent {
|
|
|
69
69
|
/**
|
|
70
70
|
* Usage of the token during the benchmark.
|
|
71
71
|
*/
|
|
72
|
-
usage:
|
|
72
|
+
usage: AgenticaTokenUsage;
|
|
73
73
|
|
|
74
74
|
/**
|
|
75
75
|
* Selected operations in the benchmark.
|
|
76
76
|
*/
|
|
77
|
-
selected:
|
|
77
|
+
selected: AgenticaOperationSelection<Model>[];
|
|
78
78
|
|
|
79
79
|
/**
|
|
80
80
|
* Prompt messages from the assistant.
|
|
81
81
|
*/
|
|
82
|
-
assistantPrompts:
|
|
82
|
+
assistantPrompts: AgenticaTextPrompt<"assistant">[];
|
|
83
83
|
}
|
|
84
84
|
|
|
85
85
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AgenticaTokenUsage } from "@agentica/core";
|
|
2
2
|
import { ILlmSchema } from "@samchon/openapi";
|
|
3
3
|
|
|
4
4
|
import { IAgenticaSelectBenchmarkEvent } from "./IAgenticaSelectBenchmarkEvent";
|
|
@@ -32,7 +32,7 @@ export interface IAgenticaSelectBenchmarkResult<
|
|
|
32
32
|
/**
|
|
33
33
|
* Aggregated token usage information.
|
|
34
34
|
*/
|
|
35
|
-
usage:
|
|
35
|
+
usage: AgenticaTokenUsage;
|
|
36
36
|
|
|
37
37
|
/**
|
|
38
38
|
* Start time of the benchmark.
|
|
@@ -67,6 +67,6 @@ export namespace IAgenticaSelectBenchmarkResult {
|
|
|
67
67
|
/**
|
|
68
68
|
* LLM token usage information.
|
|
69
69
|
*/
|
|
70
|
-
usage:
|
|
70
|
+
usage: AgenticaTokenUsage;
|
|
71
71
|
}
|
|
72
72
|
}
|