@agentica/benchmark 0.7.0-dev.20250224-2

package/lib/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./AgenticaCallBenchmark"), exports);
+__exportStar(require("./AgenticaSelectBenchmark"), exports);
+//# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0DAAwC;AAExC,4DAA0C"}

package/lib/index.mjs

@@ -0,0 +1,449 @@
+import { Semaphore } from "tstl";
+
+import "typia";
+
+import { ChatGptSelectFunctionAgent } from "@agentica/core/src/chatgpt/ChatGptSelectFunctionAgent";
+
+var AgenticaBenchmarkPredicator;
+
+(function(AgenticaBenchmarkPredicator) {
+    AgenticaBenchmarkPredicator.isNext = async agent => {
+        const last = agent.getPromptHistories().at(-1);
+        if (last?.type !== "text" || last.role !== "assistant") return null;
+        const consent = {
+            functions: [ {
+                name: "consent",
+                parameters: {
+                    description: "Properties for asking the user's consent",
+                    type: "object",
+                    properties: {
+                        content: {
+                            description: "Reason of the message implying what the AI agent wants\nto do at the next step after the user's consent.",
+                            type: "string"
+                        },
+                        reply: {
+                            title: "Recommended reply message for the user",
+                            description: "Recommended reply message for the user.\n\nThe message what AI agent wants the user to reply\naccepting the AI agent's next job suggestion.",
+                            type: "string"
+                        }
+                    },
+                    required: [ "content", "reply" ],
+                    additionalProperties: false,
+                    $defs: {}
+                },
+                description: "Ask user to consent for what the AI agent wants to do next.\n\nIf AI agent wants to do some function calling at next,\nbut it needs the user's consent about the function calling to do,\nthen call this tool function."
+            } ]
+        }.functions[0];
+        const result = await agent["props"].provider.api.chat.completions.create({
+            model: agent["props"].provider.model,
+            messages: [ {
+                role: "system",
+                content: [ "You are an helpful assistant.", "", "If what the assistant said seems like to asking for", "user's consent about some function calling at the next step,", "use the tools appropriately to step to the next." ].join("\n")
+            }, {
+                role: "assistant",
+                content: last.text
+            } ],
+            tools: [ {
+                type: "function",
+                function: {
+                    name: consent.name,
+                    description: consent.description,
+                    parameters: consent.parameters
+                }
+            } ],
+            tool_choice: "required",
+            parallel_tool_calls: false
+        }, agent["props"].provider.options);
+        const toolCall = (result.choices[0]?.message.tool_calls ?? []).filter((tc => tc.type === "function" && tc.function.name === consent.name))?.[0];
+        if (toolCall === undefined) return null;
+        const input = JSON.parse(toolCall.function.arguments);
+        return (() => {
+            const _io0 = input => "string" === typeof input.content && "string" === typeof input.reply;
+            return input => "object" === typeof input && null !== input && _io0(input);
+        })()(input) ? input.reply : null;
+    };
+    AgenticaBenchmarkPredicator.success = props => successInner(props).result;
+    const successInner = props => {
+        const call = (expected, overrideOperations) => successInner({
+            expected,
+            operations: overrideOperations ?? props.operations,
+            strict: props.strict
+        });
+        switch (props.expected.type) {
+          case "array":
+            {
+                let take = 0;
+                const targetIterator = props.expected.items[Symbol.iterator]();
+                let targeted = targetIterator.next();
+                while (true) {
+                    if (targeted.done) {
+                        return {
+                            result: true,
+                            take
+                        };
+                    }
+                    if (take >= props.operations.length) {
+                        return {
+                            result: false
+                        };
+                    }
+                    const result = call(targeted.value, props.operations.slice(take));
+                    if (!result.result) {
+                        if (!props.strict) {
+                            take += 1;
+                            continue;
+                        }
+                        return {
+                            result: false
+                        };
+                    }
+                    take += result.take;
+                    targeted = targetIterator.next();
+                }
+            }
+
+          case "standalone":
+            {
+                const target = props.expected.operation;
+                const result = props.operations.some((op => op.name === target.name));
+                if (result) {
+                    return {
+                        result,
+                        take: 1
+                    };
+                }
+                return {
+                    result
+                };
+            }
+
+          case "anyOf":
+            for (const expected of props.expected.anyOf) {
+                const callResult = call(expected);
+                if (callResult.result) {
+                    return callResult;
+                }
+            }
+            return {
+                result: false
+            };
+
+          case "allOf":
+            {
+                const result = props.expected.allOf.map((expected => call(expected)));
+                if (result.every((r => r.result))) {
+                    return {
+                        result: true,
+                        take: result.reduce(((acc, r) => Math.max(acc, r.take)), 0)
+                    };
+                }
+                return {
+                    result: false
+                };
+            }
+        }
+    };
+})(AgenticaBenchmarkPredicator || (AgenticaBenchmarkPredicator = {}));
+
+var MathUtil;
+
+(function(MathUtil) {
+    MathUtil.round = value => Math.floor(value * 100) / 100;
+})(MathUtil || (MathUtil = {}));
+
+var AgenticaBenchmarkUtil;
+
+(function(AgenticaBenchmarkUtil) {
+    AgenticaBenchmarkUtil.errorToJson = error => {
+        if (error instanceof Error) return {
+            ...error,
+            name: error.name,
+            message: error.message,
+            stack: error.stack
+        };
+        return error;
+    };
+    AgenticaBenchmarkUtil.expectedToJson = expected => {
+        if (expected.type === "standalone") return {
+            type: expected.type,
+            operation: {
+                name: expected.operation.name,
+                description: expected.operation.function.description
+            }
+        }; else if (expected.type === "array") return {
+            type: expected.type,
+            items: expected.items.map(AgenticaBenchmarkUtil.expectedToJson)
+        }; else if (expected.type === "allOf") return {
+            type: expected.type,
+            allOf: expected.allOf.map(AgenticaBenchmarkUtil.expectedToJson)
+        }; else return {
+            type: expected.type,
+            anyOf: expected.anyOf.map(AgenticaBenchmarkUtil.expectedToJson)
+        };
+    };
+})(AgenticaBenchmarkUtil || (AgenticaBenchmarkUtil = {}));
+
+var AgenticaPromptReporter;
+
+(function(AgenticaPromptReporter) {
+    AgenticaPromptReporter.markdown = p => {
+        if (p.type === "text") return [ `### Text (${p.role})`, p.text, "" ].join("\n"); else if (p.type === "select" || p.type === "cancel") return [ `### ${p.type === "select" ? "Select" : "Cancel"}`, ...p.operations.map((op => [ `#### ${op.name}`, `  - controller: ${op.controller.name}`, `  - function: ${op.function.name}`, `  - reason: ${op.reason}`, "", ...!!op.function.description?.length ? [ op.function.description, "" ] : [] ])).flat() ].join("\n"); else if (p.type === "describe") return [ "### Describe", ...p.executions.map((e => `  - ${e.name}`)), "", ...p.text.split("\n").map((s => `> ${s}`)), "" ].join("\n");
+        return [ "### Execute", `  - name: ${p.name}`, `  - controller: ${p.controller.name}`, `  - function: ${p.function.name}`, "", "```json", JSON.stringify(p.arguments, null, 2), "```", "" ].join("\n");
+    };
+})(AgenticaPromptReporter || (AgenticaPromptReporter = {}));
+
+var AgenticaCallBenchmarkReporter;
+
+(function(AgenticaCallBenchmarkReporter) {
+    AgenticaCallBenchmarkReporter.markdown = result => Object.fromEntries([ [ "./README.md", writeIndex(result) ], ...result.experiments.map((exp => [ [ `./${exp.scenario.name}/README.md`, writeExperimentIndex(exp) ], ...exp.events.map(((event, i) => [ `./${exp.scenario.name}/${i + 1}.${event.type}.md`, writeExperimentEvent(event, i) ])) ])).flat() ]);
+    const writeIndex = result => {
+        const events = result.experiments.map((r => r.events)).flat();
+        const average = events.map((e => e.completed_at.getTime() - e.started_at.getTime())).reduce(((a, b) => a + b), 0) / events.length;
+        return [ "# LLM Function Call Benchmark", "## Summary", `  - Aggregation:`, `    - Scenarios: #${result.experiments.length.toLocaleString()}`, `    - Trial: ${events.length}`, `    - Success: ${events.filter((e => e.type === "success")).length}`, `    - Failure: ${events.filter((e => e.type === "failure")).length}`, `    - Average Time: ${MathUtil.round(average).toLocaleString()} ms`, `  - Token Usage`, `    - Total: ${result.usage.total.toLocaleString()}`, `    - Prompt`, `      - Total: ${result.usage.prompt.total.toLocaleString()}`, `      - Audio: ${result.usage.prompt.audio.toLocaleString()}`, `      - Cached: ${result.usage.prompt.cached.toLocaleString()}`, `    - Completion:`, `      - Total: ${result.usage.completion.total.toLocaleString()}`, `      - Accepted Prediction: ${result.usage.completion.accepted_prediction.toLocaleString()}`, `      - Audio: ${result.usage.completion.audio.toLocaleString()}`, `      - Reasoning: ${result.usage.completion.reasoning.toLocaleString()}`, `      - Rejected Prediction: ${result.usage.completion.rejected_prediction.toLocaleString()}`, "", "## Experiments", " Name | Select | Call | Time/Avg ", ":-----|:-------|:-----|----------:", ...result.experiments.map((exp => [ `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`, drawStatus(exp.events, (e => e.type !== "error" && e.select === true)), drawStatus(exp.events, (e => e.type !== "error" && e.call === true)), `${MathUtil.round(exp.events.map((e => e.completed_at.getTime() - e.started_at.getTime())).reduce(((a, b) => a + b), 0) / exp.events.length).toLocaleString()} ms` ].join(" | "))) ].join("\n");
+    };
+    const writeExperimentIndex = exp => [ `# ${exp.scenario.name}`, "## Summary", `  - Scenarios: #${exp.events.length.toLocaleString()}`, `  - Success: ${exp.events.filter((e => e.type === "success")).length}`, `  - Failure: ${exp.events.filter((e => e.type === "failure")).length}`, `  - Average Time: ${MathUtil.round(exp.events.map((e => e.completed_at.getTime() - e.started_at.getTime())).reduce(((a, b) => a + b), 0) / exp.events.length).toLocaleString()} ms`, "", "## Events", " Name | Type | Time", ":-----|:-----|----:", ...exp.events.map(((e, i) => [ `[${i + 1}.](./${i + 1}.${e.type}.md)`, e.type, `${MathUtil.round(e.completed_at.getTime() - e.started_at.getTime())} ms` ].join(" | "))), "", "## Scenario", "### User Prompt", exp.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected), null, 2), "```" ].join("\n");
+    const writeExperimentEvent = (event, index) => [ `# ${index}. ${event.type}`, "## Summary", `  - Name: ${event.scenario.name}`, `  - Type: ${event.type}`, `  - Time: ${MathUtil.round(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`, ...event.type !== "error" ? [ `  - Select: ${event.select ? "✅" : "❌"}`, `  - Call: ${event.call ? "✅" : "❌"}` ] : [], `  - Token Usage: ${event.usage.toLocaleString()}`, "", "## Scenario", "### User Prompt", event.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected), null, 2), "```", "", "## Prompt Histories", ...event.prompts.map(AgenticaPromptReporter.markdown), "", ...event.type === "error" ? [ "## Error", "```json", JSON.stringify(AgenticaBenchmarkUtil.errorToJson(event.error), null, 2), "```" ] : [] ].join("\n");
+    const drawStatus = (events, success) => {
+        const count = events.filter(success).length;
+        return new Array(count).fill("■").join("") + new Array(10 - count).fill("□").join("");
+    };
+})(AgenticaCallBenchmarkReporter || (AgenticaCallBenchmarkReporter = {}));
+
+var TokenUsageComputer;
+
+(function(TokenUsageComputer) {
+    TokenUsageComputer.zero = () => ({
+        total: 0,
+        prompt: {
+            total: 0,
+            audio: 0,
+            cached: 0
+        },
+        completion: {
+            total: 0,
+            accepted_prediction: 0,
+            audio: 0,
+            reasoning: 0,
+            rejected_prediction: 0
+        }
+    });
+    TokenUsageComputer.plus = (a, b) => ({
+        total: a.total + b.total,
+        prompt: {
+            total: a.prompt.total + b.prompt.total,
+            audio: a.prompt.audio + b.prompt.audio,
+            cached: a.prompt.cached + b.prompt.cached
+        },
+        completion: {
+            total: a.completion.total + b.completion.total,
+            accepted_prediction: a.completion.accepted_prediction + b.completion.accepted_prediction,
+            audio: a.completion.audio + b.completion.audio,
+            reasoning: a.completion.reasoning + b.completion.reasoning,
+            rejected_prediction: a.completion.rejected_prediction + b.completion.rejected_prediction
+        }
+    });
+})(TokenUsageComputer || (TokenUsageComputer = {}));
+
+class AgenticaCallBenchmark {
+    constructor(props) {
+        this.agent_ = props.agent;
+        this.scenarios_ = props.scenarios.slice();
+        this.config_ = {
+            repeat: props.config?.repeat ?? 10,
+            simultaneous: props.config?.simultaneous ?? 10,
+            consent: props.config?.consent ?? 3
+        };
+        this.result_ = null;
+    }
+    async execute(listener) {
+        const started_at = new Date;
+        const semaphore = new Semaphore(this.config_.simultaneous);
+        const experiments = await Promise.all(this.scenarios_.map((async scenario => {
+            const events = await Promise.all(new Array(this.config_.repeat).fill(0).map((async () => {
+                await semaphore.acquire();
+                const e = await this.step(scenario);
+                await semaphore.release();
+                if (listener !== undefined) listener(e);
+                return e;
+            })));
+            return {
+                scenario,
+                events,
+                usage: events.filter((e => e.type !== "error")).map((e => e.usage)).reduce(TokenUsageComputer.plus, TokenUsageComputer.zero())
+            };
+        })));
+        return this.result_ = {
+            experiments,
+            started_at,
+            completed_at: new Date,
+            usage: experiments.map((p => p.usage)).reduce(TokenUsageComputer.plus, TokenUsageComputer.zero())
+        };
+    }
+    report() {
+        if (this.result_ === null) throw new Error("Benchmark is not executed yet.");
+        return AgenticaCallBenchmarkReporter.markdown(this.result_);
+    }
+    async step(scenario) {
+        const agent = this.agent_.clone();
+        const started_at = new Date;
+        const success = () => AgenticaBenchmarkPredicator.success({
+            expected: scenario.expected,
+            operations: agent.getPromptHistories().filter((p => p.type === "execute")),
+            strict: false
+        });
+        const out = () => {
+            const select = AgenticaBenchmarkPredicator.success({
+                expected: scenario.expected,
+                operations: agent.getPromptHistories().filter((p => p.type === "select")).map((p => p.operations)).flat(),
+                strict: false
+            });
+            const call = success();
+            return {
+                type: call ? "success" : "failure",
+                scenario,
+                select,
+                call,
+                prompts: agent.getPromptHistories(),
+                usage: agent.getTokenUsage(),
+                started_at,
+                completed_at: new Date
+            };
+        };
+        try {
+            await agent.conversate(scenario.text);
+            if (success()) return out();
+            for (let i = 0; i < this.config_.consent; ++i) {
+                const next = await AgenticaBenchmarkPredicator.isNext(agent);
+                if (next === null) break;
+                await agent.conversate(next);
+                if (success()) return out();
+            }
+            return out();
+        } catch (error) {
+            return {
+                type: "error",
+                scenario,
+                prompts: agent.getPromptHistories(),
+                usage: agent.getTokenUsage(),
+                error,
+                started_at,
+                completed_at: new Date
+            };
+        }
+    }
+}
+
+var AgenticaSelectBenchmarkReporter;
+
+(function(AgenticaSelectBenchmarkReporter) {
+    AgenticaSelectBenchmarkReporter.markdown = result => Object.fromEntries([ [ "./README.md", writeIndex(result) ], ...result.experiments.map((exp => [ [ `./${exp.scenario.name}/README.md`, writeExperimentIndex(exp) ], ...exp.events.map(((event, i) => [ `./${exp.scenario.name}/${i + 1}.${event.type}.md`, writeExperimentEvent(event, i) ])) ])).flat() ]);
+    const writeIndex = result => {
+        const events = result.experiments.map((r => r.events)).flat();
+        const average = events.map((e => e.completed_at.getTime() - e.started_at.getTime())).reduce(((a, b) => a + b), 0) / events.length;
+        return [ "# LLM Function Selection Benchmark", "## Summary", `  - Aggregation:`, `    - Scenarios: #${result.experiments.length.toLocaleString()}`, `    - Trial: ${events.length}`, `    - Success: ${events.filter((e => e.type === "success")).length}`, `    - Failure: ${events.filter((e => e.type === "failure")).length}`, `    - Average Time: ${MathUtil.round(average).toLocaleString()} ms`, `  - Token Usage`, `    - Total: ${result.usage.total.toLocaleString()}`, `    - Prompt`, `      - Total: ${result.usage.prompt.total.toLocaleString()}`, `      - Audio: ${result.usage.prompt.audio.toLocaleString()}`, `      - Cached: ${result.usage.prompt.cached.toLocaleString()}`, `    - Completion:`, `      - Total: ${result.usage.completion.total.toLocaleString()}`, `      - Accepted Prediction: ${result.usage.completion.accepted_prediction.toLocaleString()}`, `      - Audio: ${result.usage.completion.audio.toLocaleString()}`, `      - Reasoning: ${result.usage.completion.reasoning.toLocaleString()}`, `      - Rejected Prediction: ${result.usage.completion.rejected_prediction.toLocaleString()}`, "", "## Experiments", " Name | Status | Time/Avg  ", ":-----|:-------|----------:", ...result.experiments.map((exp => [ `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`, (() => {
+            const success = Math.floor(exp.events.filter((e => e.type === "success")).length / exp.events.length * 10);
+            return new Array(success).fill("■").join("") + new Array(10 - success).fill("□").join("");
+        })(), MathUtil.round(exp.events.map((event => event.completed_at.getTime() - event.started_at.getTime())).reduce(((a, b) => a + b), 0) / exp.events.length).toLocaleString() + " ms" ].join(" | "))) ].join("\n");
+    };
+    const writeExperimentIndex = exp => [ `# ${exp.scenario.name}`, "## Summary", "  - Aggregation:", `    - Trial: ${exp.events.length}`, `    - Success: ${exp.events.filter((e => e.type === "success")).length}`, `    - Failure: ${exp.events.filter((e => e.type === "failure")).length}`, `    - Average Time: ${MathUtil.round(exp.events.map((event => event.completed_at.getTime() - event.started_at.getTime())).reduce(((a, b) => a + b), 0) / exp.events.length).toLocaleString()} ms`, "  - Token Usage", `    - Total: ${exp.usage.total.toLocaleString()}`, `    - Prompt`, `      - Total: ${exp.usage.prompt.total.toLocaleString()}`, `      - Audio: ${exp.usage.prompt.audio.toLocaleString()}`, `      - Cached: ${exp.usage.prompt.cached.toLocaleString()}`, `    - Completion:`, `      - Total: ${exp.usage.completion.total.toLocaleString()}`, `      - Accepted Prediction: ${exp.usage.completion.accepted_prediction.toLocaleString()}`, `      - Audio: ${exp.usage.completion.audio.toLocaleString()}`, `      - Reasoning: ${exp.usage.completion.reasoning.toLocaleString()}`, `      - Rejected Prediction: ${exp.usage.completion.rejected_prediction.toLocaleString()}`, "", "## Events", " No | Type | Time", "---:|:-----|----:", ...exp.events.map(((e, i) => [ `[${i + 1}.](./${i + 1}.${e.type}.md)`, e.type, MathUtil.round(e.completed_at.getTime() - e.started_at.getTime()) + " ms" ].join(" | "))), "", "## Scenario", "### User Prompt", exp.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected), null, 2), "```" ].join("\n");
+    const writeExperimentEvent = (event, index) => [ `# ${index}. ${event.type}`, `## Summary`, `  - Name: ${event.scenario.name}`, `  - Type: ${event.type}`, `  - Time: ${(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`, ...event.type !== "error" ? [ "  - Token Usage", `    - Total: ${event.usage.total.toLocaleString()}`, `    - Prompt`, `      - Total: ${event.usage.prompt.total.toLocaleString()}`, `      - Audio: ${event.usage.prompt.audio.toLocaleString()}`, `      - Cached: ${event.usage.prompt.cached.toLocaleString()}`, `    - Completion:`, `      - Total: ${event.usage.completion.total.toLocaleString()}`, `      - Accepted Prediction: ${event.usage.completion.accepted_prediction.toLocaleString()}`, `      - Audio: ${event.usage.completion.audio.toLocaleString()}`, `      - Reasoning: ${event.usage.completion.reasoning.toLocaleString()}`, `      - Rejected Prediction: ${event.usage.completion.rejected_prediction.toLocaleString()}` ] : [], "", "## Scenario", "### User Prompt", event.scenario.text, "", "### Expected", "```json", JSON.stringify(AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected), null, 2), "```", "", ...event.type === "success" || event.type === "failure" ? [ "## Result", ...event.selected.map((s => [ `### ${s.name}`, `  - Controller: \`${s.controller.name}\``, `  - Function: \`${s.function.name}\``, `  - Reason: ${s.reason}`, "", ...s.function.description ? [ s.function.description, "" ] : [] ].join("\n"))) ] : [], ...event.type === "error" ? [ "## Error", "```json", AgenticaBenchmarkUtil.errorToJson(JSON.stringify(event.error, null, 2)), "```", "" ] : [] ].join("\n");
+})(AgenticaSelectBenchmarkReporter || (AgenticaSelectBenchmarkReporter = {}));
+
+class AgenticaSelectBenchmark {
+    constructor(props) {
+        this.agent_ = props.agent;
+        this.scenarios_ = props.scenarios.slice();
+        this.config_ = {
+            repeat: props.config?.repeat ?? 10,
+            simultaneous: props.config?.simultaneous ?? 10
+        };
+        this.histories_ = props.agent.getPromptHistories().slice();
+        this.result_ = null;
+    }
+    async execute(listener) {
+        const started_at = new Date;
+        const semaphore = new Semaphore(this.config_.simultaneous);
+        const experiments = await Promise.all(this.scenarios_.map((async scenario => {
+            const events = await Promise.all(new Array(this.config_.repeat).fill(0).map((async () => {
+                await semaphore.acquire();
+                const e = await this.step(scenario);
+                await semaphore.release();
+                if (listener !== undefined) listener(e);
+                return e;
+            })));
+            return {
+                scenario,
+                events,
+                usage: events.filter((e => e.type !== "error")).map((e => e.usage)).reduce(TokenUsageComputer.plus, TokenUsageComputer.zero())
+            };
+        })));
+        return this.result_ = {
+            experiments,
+            started_at,
+            completed_at: new Date,
+            usage: experiments.map((p => p.usage)).reduce(TokenUsageComputer.plus, TokenUsageComputer.zero())
+        };
+    }
+    report() {
+        if (this.result_ === null) throw new Error("Benchmark is not executed yet.");
+        return AgenticaSelectBenchmarkReporter.markdown(this.result_);
+    }
+    async step(scenario) {
+        const started_at = new Date;
+        try {
+            const usage = {
+                total: 0,
+                prompt: {
+                    total: 0,
+                    audio: 0,
+                    cached: 0
+                },
+                completion: {
+                    total: 0,
+                    accepted_prediction: 0,
+                    audio: 0,
+                    reasoning: 0,
+                    rejected_prediction: 0
+                }
+            };
+            const prompts = await ChatGptSelectFunctionAgent.execute({
+                ...this.agent_.getContext({
+                    prompt: {
+                        type: "text",
+                        role: "user",
+                        text: scenario.text
+                    },
+                    usage
+                }),
+                histories: this.histories_.slice(),
+                stack: [],
+                ready: () => true,
+                dispatch: async () => {}
+            });
+            const selected = prompts.filter((p => p.type === "select")).map((p => p.operations)).flat();
+            return {
+                type: AgenticaBenchmarkPredicator.success({
+                    expected: scenario.expected,
+                    operations: selected
+                }) ? "success" : "failure",
+                scenario,
+                selected,
+                usage,
+                assistantPrompts: prompts.filter((p => p.type === "text")).filter((p => p.role === "assistant")),
+                started_at,
+                completed_at: new Date
+            };
+        } catch (error) {
+            return {
+                type: "error",
+                scenario,
+                error,
+                started_at,
+                completed_at: new Date
+            };
+        }
+    }
+}
+
+export { AgenticaCallBenchmark, AgenticaSelectBenchmark };
+//# sourceMappingURL=index.mjs.map

package/lib/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","sources":["../src/internal/AgenticaBenchmarkPredicator.ts","../src/utils/MathUtil.ts","../src/internal/AgenticaBenchmarkUtil.ts","../src/internal/AgenticaPromptReporter.ts","../src/internal/AgenticaCallBenchmarkReporter.ts","../src/utils/TokenUsageComputer.ts","../src/AgenticaCallBenchmark.ts","../src/internal/AgenticaSelectBenchmarkReporter.ts","../src/AgenticaSelectBenchmark.ts"],"sourcesContent":[null,null,null,null,null,null,null,null,null],"names":["AgenticaBenchmarkPredicator","isNext","async","agent","last","getPromptHistories","at","type","role","consent","functions","result","provider","api","chat","completions","create","model","messages","content","join","text","tools","function","name","description","parameters","tool_choice","parallel_tool_calls","options","toolCall","choices","message","tool_calls","filter","tc","undefined","input","JSON","parse","arguments","_io0","reply","success","props","successInner","call","expected","overrideOperations","operations","strict","take","targetIterator","items","Symbol","iterator","targeted","next","done","length","value","slice","target","operation","some","op","anyOf","callResult","allOf","map","every","r","reduce","acc","Math","max","MathUtil","round","floor","AgenticaBenchmarkUtil","errorToJson","error","Error","stack","expectedToJson","AgenticaPromptReporter","markdown","p","controller","reason","flat","executions","e","split","s","stringify","AgenticaCallBenchmarkReporter","Object","fromEntries","writeIndex","experiments","exp","scenario","writeExperimentIndex","events","event","i","writeExperimentEvent","average","completed_at","getTime","started_at","a","b","toLocaleString","usage","total","prompt","audio","cached","completion","accepted_prediction","reasoning","rejected_prediction","drawStatus","select","index","prompts","count","Array","fill","TokenUsageComputer","zero","plus","AgenticaCallBenchmark","constructor","this","agent_","scenarios_","scenarios","config_","repeat","config","simultaneous","result_","execute","listener","Date","semaphore","Semaphore","Promise","all","acquire","step","release","report","clone","out","getTokenUsage","conversate","AgenticaSelectBenchmarkReporter","selected","AgenticaSelectBenchmark","histories_","ChatGptSelectFunctionAgent","getContext","histories","ready","dispatch","assistantPrompts"],"mappings":";;;;;;AAOM,IAAWA;;CAAjB,SAAiBA;IACFA,4BAAAC,SAASC,MAAOC;QAC3B,MAAMC,OAAoCD,MAAME,qBAAqBC,IAAG;QACxE,IAAIF,MAAMG,SAAS,UAAUH,KAAKI,SAAS,aAAa,OAAO;QAE/D,MAAMC,UAAmC;;;;;;;;;;;;;;;;;;;;;;;UAGrCC,UAAU;QACd,MAAMC,eAAsCR,MAC1C,SACAS,SAASC,IAAIC,KAAKC,YAAYC,OAC9B;YACEC,OAAOd,MAAM,SAASS,SAASK;YAC/BC,UAAU,EACR;gBACEV,MAAM;gBACNW,SAAS,EACP,iCACA,IACA,uDACA,gEACA,qDACAC,KAAK;eAET;gBACEZ,MAAM;gBACNW,SAASf,KAAKiB;;YAGlBC,OAAO,EACL;gBACEf,MAAM;gBACNgB,UAAU;oBACRC,MAAMf,QAAQe;oBACdC,aAAahB,QAAQgB;oBACrBC,YAAYjB,QAAQiB;;;YAI1BC,aAAa;YACbC,qBAAqB;WAEvBzB,MAAM,SAASS,SAASiB;QAE1B,MAAMC,YACJnB,OAAOoB,QAAQ,IAAIC,QAAQC,cAAc,IACzCC,QACCC,MAAOA,GAAG5B,SAAS,cAAc4B,GAAGZ,SAASC,SAASf,QAAQe,SAC7D;QACJ,IAAIM,aAAaM,WAAW,OAAO;QACnC,MAAMC,QAAuBC,KAAKC,MAAMT,SAASP,SAASiB;QAC1D,OAAO;YAAS,MAAAC,OAAAJ,SAAA,oBAAAA,MAAAlB,WAAA,oBAAAkB,MAAAK;YAAA,OAAAL,SAAA,oBAAAA,SAAA,SAAAA,SAAAI,KAAAJ;AAAA,UAAT,GAASA,SAASA,MAAMK,QAAQ;AAAI;IAYhC1C,4BAAA2C,UAAWC,SAmBTC,aAAaD,OAAOjC;IAEnC,MAAMkC,eACJD;QASA,MAAME,OAAO,CACXC,UACAC,uBAEAH,aAAa;YACXE;YACAE,YAAYD,sBAAsBJ,MAAMK;YACxCC,QAAQN,MAAMM;;QAGlB,QAAQN,MAAMG,SAASxC;UACrB,KAAK;YAAS;gBACZ,IAAI4C,OAAO;gBACX,MAAMC,iBAAiBR,MAAMG,SAASM,MAAMC,OAAOC;gBACnD,IAAIC,WAAWJ,eAAeK;gBAE9B,OAAO,MAAM;oBACX,IAAID,SAASE,MAAM;wBACjB,OAAO;4BACL/C,QAAQ;4BACRwC;;;oBAGJ,IAAIA,QAAQP,MAAMK,WAAWU,QAAQ;wBACnC,OAAO;4BAAEhD,QAAQ;;;oBAGnB,MAAMA,SAASmC,KAAKU,SAASI,OAAOhB,MAAMK,WAAWY,MAAMV;oBAC3D,KAAKxC,OAAOA,QAAQ;wBAClB,KAAKiC,MAAMM,QAAQ;4BACjBC,QAAQ;4BACR;;wBAEF,OAAO;4BAAExC,QAAQ;;;oBAGnBwC,QAAQxC,OAAOwC;oBACfK,WAAWJ,eAAeK;;;;UAG9B,KAAK;YAAc;gBACjB,MAAMK,SAASlB,MAAMG,SAASgB;gBAC9B,MAAMpD,SAASiC,MAAMK,WAAWe,MAAMC,MAAOA,GAAGzC,SAASsC,OAAOtC;gBAChE,IAAIb,QAAQ;oBACV,OAAO;wBAAEA;wBAAQwC,MAAM;;;gBAEzB,OAAO;oBACLxC;;;;UAGJ,KAAK;YACH,KAAK,MAAMoC,YAAYH,MAAMG,SAASmB,OAAO;gBAC3C,MAAMC,aAAarB,KAAKC;gBACxB,IAAIoB,WAAWxD,QAAQ;oBACrB,OAAOwD;;;YAIX,OAAO;gBAAExD,QAAQ;;;UACnB,KAAK;YAAS;gBAQZ,MAAMA,SAASiC,MAAMG,SAASqB,MAAMC,KAAKtB,YAAaD,KAAKC;gBAC3D,IAAIpC,OAAO2D,OAAOC,KAAMA,EAAE5D,UAAS;oBACjC,OAAO;wBACLA,QAAQ;wBACRwC,MAAMxC,OAAO6D,QAAO,CAACC,KAAKF,MAAMG,KAAKC,IAAIF,KAAKF,EAAEpB,QAAO;;;gBAI3D,OAAO;oBACLxC,QAAQ;;;;;AAKjB,EAhLD,CAAiBX,gCAAAA,8BAgLhB,CAAA;;ACvLK,IAAW4E;;CAAjB,SAAiBA;IACFA,SAAAC,QAASjB,SAA0Bc,KAAKI,MAAMlB,QAAQ,OAAO;AAC3E,EAFD,CAAiBgB,aAAAA,WAEhB,CAAA;;ACAK,IAAWG;;CAAjB,SAAiBA;IACFA,sBAAAC,cAAeC;QAC1B,IAAIA,iBAAiBC,OACnB,OAAO;eACFD;YACHzD,MAAMyD,MAAMzD;YACZQ,SAASiD,MAAMjD;YACfmD,OAAOF,MAAME;;QAEjB,OAAOF;AAAK;IAGDF,sBAAAK,iBAAkBrC;QAC7B,IAAIA,SAASxC,SAAS,cACpB,OAAO;YACLA,MAAMwC,SAASxC;YACfwD,WAAW;gBACTvC,MAAMuB,SAASgB,UAAUvC;gBACzBC,aAAasB,SAASgB,UAAUxC,SAASE;;gBAG1C,IAAIsB,SAASxC,SAAS,SACzB,OAAO;YACLA,MAAMwC,SAASxC;YACf8C,OAAON,SAASM,MAAMgB,IAAIU,sBAAAK;gBAEzB,IAAIrC,SAASxC,SAAS,SACzB,OAAO;YACLA,MAAMwC,SAASxC;YACf6D,OAAOrB,SAASqB,MAAMC,IAAIU,sBAAAK;gBAG5B,OAAO;YACL7E,MAAMwC,SAASxC;YACf2D,OAAOnB,SAASmB,MAAMG,IAAIU,sBAAAK;;AAC3B;AAEN,EArCD,CAAiBL,0BAAAA,wBAqChB,CAAA;;ACrCK,IAAWM;;CAAjB,SAAiBA;IACFA,uBAAAC,WAAYC;QACvB,IAAIA,EAAEhF,SAAS,QACb,OAAO,EAAC,aAAagF,EAAE/E,SAAS+E,EAAElE,MAAM,KAAID,KAAK,YAC9C,IAAImE,EAAEhF,SAAS,YAAYgF,EAAEhF,SAAS,UACzC,OAAO,EACL,OAAOgF,EAAEhF,SAAS,WAAW,WAAW,eACrCgF,EAAEtC,WACFoB,KAAKJ,MAAO,EACX,QAAQA,GAAGzC,QACX,mBAAmByC,GAAGuB,WAAWhE,QACjC,iBAAiByC,GAAG1C,SAASC,QAC7B,eAAeyC,GAAGwB,UAClB,SACMxB,GAAG1C,SAASE,aAAakC,SAC3B,EAACM,GAAG1C,SAASE,aAAa,OAC1B,OAELiE,SACHtE,KAAK,YACJ,IAAImE,EAAEhF,SAAS,YAClB,OAAO,EACL,mBACGgF,EAAEI,WAAWtB,KAAKuB,KAAM,OAAOA,EAAEpE,UACpC,OACG+D,EAAElE,KAAKwE,MAAM,MAAMxB,KAAKyB,KAAM,KAAKA,OACtC,KACA1E,KAAK;QACT,OAAO,EACL,eACA,aAAamE,EAAE/D,QACf,mBAAmB+D,EAAEC,WAAWhE,QAChC,iBAAiB+D,EAAEhE,SAASC,QAC5B,IACA,WACAc,KAAKyD,UAAUR,EAAE/C,WAAW,MAAM,IAClC,OACA,KACApB,KAAK;AAAK;AAEf,EAxCD,CAAiBiE,2BAAAA,yBAwChB,CAAA;;ACpCK,IAAWW;;CAAjB,SAAiBA;IACFA,8BAAQV,WACnB3E,UAEAsF,OAAOC,YAAY,EACjB,EAAC,eAAeC,WAAWxF,cACxBA,OAAOyF,YACP/B,KAAKgC,OAAQ,EACZ,EAAC,KAAKA,IAAIC,SAAS9E,kBAAkB+E,qBAAqBF,WACvDA,IAAIG,OAAOnC,KAAI,CAACoC,OAAOC,MAAM,EAC9B,KAAKL,IAAIC,SAAS9E,QAAQkF,IAAI,KAAKD,MAAMlG,WACzCoG,qBAAqBF,OAAOC,WAG/BhB;IAGP,MAAMS,aAAcxF;QAClB,MAAM6F,SAAwC7F,OAAOyF,YAClD/B,KAAKE,KAAMA,EAAEiC,SACbd;QACH,MAAMkB,UACJJ,OACGnC,KAAKuB,KAAMA,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,YACnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKT,OAAO7C;QACzC,OAAO,EACL,iCACA,cACA,oBACA,qBAAqBhD,OAAOyF,YAAYzC,OAAOuD,oBAC/C,gBAAgBV,OAAO7C,UACvB,kBAAkB6C,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC7D,kBAAkB6C,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC7D,uBAAuBiB,SAASC,MAAM+B,SAASM,uBAC/C,mBACA,gBAAgBvG,OAAOwG,MAAMC,MAAMF,oBACnC,gBACA,kBAAkBvG,OAAOwG,MAAME,OAAOD,MAAMF,oBAC5C,kBAAkBvG,OAAOwG,MAAME,OAAOC,MAAMJ,oBAC5C,mBAAmBvG,OAAOwG,MAAME,OAAOE,OAAOL,oBAC9C,qBACA,kBAAkBvG,OAAOwG,MAAMK,WAAWJ,MAAMF,oBAChD,gCAAgCvG,OAAOwG,MAAMK,WAAWC,oBAAoBP,oBAC5E,kBAAkBvG,OAAOwG,MAAMK,WAAWF,MAAMJ,oBAChD,sBAAsBvG,OAAOwG,MAAMK,WAAWE,UAAUR,oBACxD,gCAAgCvG,OAAOwG,MAAMK,WAAWG,oBAAoBT,oBAC5E,IACA,kBACA,qCACA,yCACGvG,OAAOyF,YAAY/B,KAAKgC,OACzB,EACE,IAAIA,IAAIC,SAAS9E,WAAW6E,IAAIC,SAAS9E,mBACzCoG,WACEvB,IAAIG,SACHZ,KAAMA,EAAErF,SAAS,WAAWqF,EAAEiC,WAAW,QAE5CD,WAAWvB,IAAIG,SAASZ,KAAMA,EAAErF,SAAS,WAAWqF,EAAE9C,SAAS,QAC/D,GAAG8B,SAASC,MACVwB,IAAIG,OACDnC,KAAKuB,KAAMA,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,YACnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO7C,QAC3CuD,wBACF9F,KAAK,WAETA,KAAK;AAAK;IAGd,MAAMmF,uBACJF,OAEO,EACL,KAAKA,IAAIC,SAAS9E,QAClB,cACA,mBAAmB6E,IAAIG,OAAO7C,OAAOuD,oBACrC,gBAAgBb,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC/D,gBAAgB0C,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC/D,qBAAqBiB,SAASC,MAC5BwB,IAAIG,OACDnC,KAAKuB,KAAMA,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,YACnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO7C,QAC3CuD,uBACF,IACA,aACA,uBACA,0BACGb,IAAIG,OAAOnC,KAAI,CAACuB,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAErF,YAC5BqF,EAAErF,MACF,GAAGqE,SAASC,MAAMe,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,kBAC1D1F,KAAK,UAET,IACA,eACA,mBACAiF,IAAIC,SAASjF,MACb,IACA,gBACA,WACAiB,KAAKyD,UACHhB,sBAAsBK,eAAeiB,IAAIC,SAASvD,WAClD,MACA,IAEF,QACA3B,KAAK;IAGT,MAAMuF,uBAAuB,CAC3BF,OACAqB,UAEO,EACL,KAAKA,UAAUrB,MAAMlG,QACrB,cACA,aAAakG,MAAMH,SAAS9E,QAC5B,aAAaiF,MAAMlG,QACnB,aAAaqE,SAASC,MACpB4B,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,WAChDI,0BACET,MAAMlG,SAAS,UACf,EACE,eAAekG,MAAMoB,SAAS,MAAM,OACpC,aAAapB,MAAM3D,OAAO,MAAM,UAElC,IACJ,oBAAoB2D,MAAMU,MAAMD,oBAChC,IACA,eACA,mBACAT,MAAMH,SAASjF,MACf,IACA,gBACA,WACAiB,KAAKyD,UACHhB,sBAAsBK,eAAeqB,MAAMH,SAASvD,WACpD,MACA,IAEF,OACA,IACA,0BACG0D,MAAMsB,QAAQ1D,IAAIgB,uBAAuBC,WAC5C,OACImB,MAAMlG,SAAS,UACf,EACE,YACA,WACA+B,KAAKyD,UACHhB,sBAAsBC,YAAYyB,MAAMxB,QACxC,MACA,IAEF,UAEF,KACJ7D,KAAK;IAGT,MAAMwG,aAAa,CACjBpB,QACA7D;QAEA,MAAMqF,QAAgBxB,OAAOtE,OAAOS,SAASgB;QAC7C,OACE,IAAIsE,MAAMD,OAAOE,KAAK,KAAK9G,KAAK,MAChC,IAAI6G,MAAM,KAAKD,OAAOE,KAAK,KAAK9G,KAAK;AAAG;AAG7C,EA1KD,CAAiB4E,kCAAAA,gCA0KhB,CAAA;;AC9KK,IAAWmC;;CAAjB,SAAiBA;IACFA,mBAAAC,OAAO,OAA4B;QAC9ChB,OAAO;QACPC,QAAQ;YACND,OAAO;YACPE,OAAO;YACPC,QAAQ;;QAEVC,YAAY;YACVJ,OAAO;YACPK,qBAAqB;YACrBH,OAAO;YACPI,WAAW;YACXC,qBAAqB;;;IAIZQ,mBAAIE,OAAG,CAClBrB,GACAC,OACyB;QACzBG,OAAOJ,EAAEI,QAAQH,EAAEG;QACnBC,QAAQ;YACND,OAAOJ,EAAEK,OAAOD,QAAQH,EAAEI,OAAOD;YACjCE,OAAON,EAAEK,OAAOC,QAAQL,EAAEI,OAAOC;YACjCC,QAAQP,EAAEK,OAAOE,SAASN,EAAEI,OAAOE;;QAErCC,YAAY;YACVJ,OAAOJ,EAAEQ,WAAWJ,QAAQH,EAAEO,WAAWJ;YACzCK,qBACET,EAAEQ,WAAWC,sBAAsBR,EAAEO,WAAWC;YAClDH,OAAON,EAAEQ,WAAWF,QAAQL,EAAEO,WAAWF;YACzCI,WAAWV,EAAEQ,WAAWE,YAAYT,EAAEO,WAAWE;YACjDC,qBACEX,EAAEQ,WAAWG,sBAAsBV,EAAEO,WAAWG;;;AAGvD,EArCD,CAAiBQ,uBAAAA,qBAqChB,CAAA;;MCRYG;IAWX,WAAAC,CAAmB3F;QACjB4F,KAAKC,SAAS7F,MAAMzC;QACpBqI,KAAKE,aAAa9F,MAAM+F,UAAU9E;QAClC2E,KAAKI,UAAU;YACbC,QAAQjG,MAAMkG,QAAQD,UAAU;YAChCE,cAAcnG,MAAMkG,QAAQC,gBAAgB;YAC5CtI,SAASmC,MAAMkG,QAAQrI,WAAW;;QAEpC+H,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMnC,aAAmB,IAAIoC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAM3C,oBACEkD,QAAQC,IACZf,KAAKE,WAAWrE,KAAInE,MAAOoG;YACzB,MAAME,eAA8C8C,QAAQC,IAC1D,IAAItB,MAAMO,KAAKI,QAAQC,QAAQX,KAAK,GAAG7D,KAAInE;sBACnCkJ,UAAUI;gBAChB,MAAM5D,UAAuC4C,KAAKiB,KAAKnD;sBACjD8C,UAAUM;gBAChB,IAAIR,aAAa9G,WAAW8G,SAAStD;gBACrC,OAAOA;AAAC;YAGZ,OAAO;gBACLU;gBACAE;gBACAW,OAAOX,OACJtE,QAAQ0D,KAAMA,EAAErF,SAAS,UACzB8D,KAAKuB,KAAMA,EAAEuB,QACb3C,OAAO2D,mBAAmBE,MAAMF,mBAAmBC;;AACvD;QAGP,OAAQI,KAAKQ,UAAU;YACrB5C;YACAW;YACAF,cAAc,IAAIsC;YAClBhC,OAAOf,YACJ/B,KAAKkB,KAAMA,EAAE4B,QACb3C,OAAO2D,mBAAmBE,MAAMF,mBAAmBC;;;IAsBnD,MAAAuB;QACL,IAAInB,KAAKQ,YAAY,MACnB,MAAM,IAAI9D,MAAM;QAClB,OAAOc,8BAA8BV,SAASkD,KAAKQ;;IAG7C,UAAMS,CACZnD;QAEA,MAAMnG,QAAkBqI,KAAKC,OAAOmB;QACpC,MAAM7C,aAAmB,IAAIoC;QAC7B,MAAMxG,UAAU,MACd3C,4BAA4B2C,QAAQ;YAClCI,UAAUuD,SAASvD;YACnBE,YAAY9C,MACTE,qBACA6B,QAAQqD,KAAMA,EAAEhF,SAAS;YAC5B2C,QAAQ;;QAEZ,MAAM2G,MAAM;YACV,MAAMhC,SAAS7H,4BAA4B2C,QAAQ;gBACjDI,UAAUuD,SAASvD;gBACnBE,YAAY9C,MACTE,qBACA6B,QAAQqD,KAAMA,EAAEhF,SAAS,WACzB8D,KAAKkB,KAAMA,EAAEtC,aACbyC;gBACHxC,QAAQ;;YAEV,MAAMJ,OAAOH;YACb,OAAO;gBACLpC,MAAOuC,OAAO,YAAY;gBAC1BwD;gBACAuB;gBACA/E;gBACAiF,SAAS5H,MAAME;gBACf8G,OAAOhH,MAAM2J;gBACb/C;gBACAF,cAAc,IAAIsC;;AAC4B;QAGlD;kBACQhJ,MAAM4J,WAAWzD,SAASjF;YAChC,IAAIsB,WAAW,OAAOkH;YACtB,KAAK,IAAInD,IAAY,GAAGA,IAAI8B,KAAKI,QAAQnI,WAAWiG,GAAG;gBACrD,MAAMjD,aACEzD,4BAA4BC,OAAOE;gBAC3C,IAAIsD,SAAS,MAAM;sBAEbtD,MAAM4J,WAAWtG;gBACvB,IAAId,WAAW,OAAOkH;;YAExB,OAAOA;UACP,OAAO5E;YACP,OAAO;gBACL1E,MAAM;gBACN+F;gBACAyB,SAAS5H,MAAME;gBACf8G,OAAOhH,MAAM2J;gBACb7E;gBACA8B;gBACAF,cAAc,IAAIsC;;;;;;AClLpB,IAAWa;;CAAjB,SAAiBA;IACFA,gCAAQ1E,WACnB3E,UAEAsF,OAAOC,YAAY,EACjB,EAAC,eAAeC,WAAWxF,cACxBA,OAAOyF,YACP/B,KAAKgC,OAAQ,EACZ,EAAC,KAAKA,IAAIC,SAAS9E,kBAAkB+E,qBAAqBF,WACvDA,IAAIG,OAAOnC,KAAI,CAACoC,OAAOC,MAAM,EAC9B,KAAKL,IAAIC,SAAS9E,QAAQkF,IAAI,KAAKD,MAAMlG,WACzCoG,qBAAqBF,OAAOC,WAG/BhB;IAGP,MAAMS,aAAcxF;QAClB,MAAM6F,SAA0C7F,OAAOyF,YACpD/B,KAAKE,KAAMA,EAAEiC,SACbd;QACH,MAAMkB,UACJJ,OACGnC,KAAKuB,KAAMA,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,YACnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKT,OAAO7C;QACzC,OAAO,EACL,sCACA,cACA,oBACA,qBAAqBhD,OAAOyF,YAAYzC,OAAOuD,oBAC/C,gBAAgBV,OAAO7C,UACvB,kBAAkB6C,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC7D,kBAAkB6C,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UAC7D,uBAAuBiB,SAASC,MAAM+B,SAASM,uBAC/C,mBACA,gBAAgBvG,OAAOwG,MAAMC,MAAMF,oBACnC,gBACA,kBAAkBvG,OAAOwG,MAAME,OAAOD,MAAMF,oBAC5C,kBAAkBvG,OAAOwG,MAAME,OAAOC,MAAMJ,oBAC5C,mBAAmBvG,OAAOwG,MAAME,OAAOE,OAAOL,oBAC9C,qBACA,kBAAkBvG,OAAOwG,MAAMK,WAAWJ,MAAMF,oBAChD,gCAAgCvG,OAAOwG,MAAMK,WAAWC,oBAAoBP,oBAC5E,kBAAkBvG,OAAOwG,MAAMK,WAAWF,MAAMJ,oBAChD,sBAAsBvG,OAAOwG,MAAMK,WAAWE,UAAUR,oBACxD,gCAAgCvG,OAAOwG,MAAMK,WAAWG,oBAAoBT,oBAC5E,IACA,kBACA,+BACA,kCACGvG,OAAOyF,YAAY/B,KAAKgC,OACzB,EACE,IAAIA,IAAIC,SAAS9E,WAAW6E,IAAIC,SAAS9E,mBACzC;YACE,MAAMmB,UAAkB+B,KAAKI,MAC1BuB,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,SAC9C0C,IAAIG,OAAO7C,SACX;YAEJ,OACE,IAAIsE,MAAMtF,SAASuF,KAAK,KAAK9G,KAAK,MAClC,IAAI6G,MAAM,KAAKtF,SAASuF,KAAK,KAAK9G,KAAK;AAE1C,UAVD,IAWAwD,SAASC,MACPwB,IAAIG,OACDnC,KACEoC,SACCA,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,YAEnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO7C,QAC3CuD,mBAAmB,QACrB9F,KAAK,WAETA,KAAK;AAAK;IAGd,MAAMmF,uBACJF,OAEO,EACL,KAAKA,IAAIC,SAAS9E,QAClB,cACA,oBACA,gBAAgB6E,IAAIG,OAAO7C,UAC3B,kBAAkB0C,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UACjE,kBAAkB0C,IAAIG,OAAOtE,QAAQ0D,KAAMA,EAAErF,SAAS,YAAWoD,UACjE,uBAAuBiB,SAASC,MAC9BwB,IAAIG,OACDnC,KACEoC,SACCA,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,YAEnDtC,QAAO,CAACwC,GAAGC,MAAMD,IAAIC,IAAG,KAAKZ,IAAIG,OAAO7C,QAC3CuD,uBACF,mBACA,gBAAgBb,IAAIc,MAAMC,MAAMF,oBAChC,gBACA,kBAAkBb,IAAIc,MAAME,OAAOD,MAAMF,oBACzC,kBAAkBb,IAAIc,MAAME,OAAOC,MAAMJ,oBACzC,mBAAmBb,IAAIc,MAAME,OAAOE,OAAOL,oBAC3C,qBACA,kBAAkBb,IAAIc,MAAMK,WAAWJ,MAAMF,oBAC7C,gCAAgCb,IAAIc,MAAMK,WAAWC,oBAAoBP,oBACzE,kBAAkBb,IAAIc,MAAMK,WAAWF,MAAMJ,oBAC7C,sBAAsBb,IAAIc,MAAMK,WAAWE,UAAUR,oBACrD,gCAAgCb,IAAIc,MAAMK,WAAWG,oBAAoBT,oBACzE,IACA,aACA,qBACA,wBACGb,IAAIG,OAAOnC,KAAI,CAACuB,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAErF,YAC5BqF,EAAErF,MACFqE,SAASC,MAAMe,EAAEiB,aAAaC,YAAYlB,EAAEmB,WAAWD,aACrD,QACF1F,KAAK,UAET,IACA,eACA,mBACAiF,IAAIC,SAASjF,MACb,IACA,gBACA,WACAiB,KAAKyD,UACHhB,sBAAsBK,eAAeiB,IAAIC,SAASvD,WAClD,MACA,IAEF,QACA3B,KAAK;IAGT,MAAMuF,uBAAuB,CAC3BF,OACAqB,UAEO,EACL,KAAKA,UAAUrB,MAAMlG,QACrB,cACA,aAAakG,MAAMH,SAAS9E,QAC5B,aAAaiF,MAAMlG,QACnB,cAAckG,MAAMI,aAAaC,YAAYL,MAAMM,WAAWD,WAAWI,0BACrET,MAAMlG,SAAS,UACf,EACE,mBACA,gBAAgBkG,MAAMU,MAAMC,MAAMF,oBAClC,gBACA,kBAAkBT,MAAMU,MAAME,OAAOD,MAAMF,oBAC3C,kBAAkBT,MAAMU,MAAME,OAAOC,MAAMJ,oBAC3C,mBAAmBT,MAAMU,MAAME,OAAOE,OAAOL,oBAC7C,qBACA,kBAAkBT,MAAMU,MAAMK,WAAWJ,MAAMF,oBAC/C,gCAAgCT,MAAMU,MAAMK,WAAWC,oBAAoBP,oBAC3E,kBAAkBT,MAAMU,MAAMK,WAAWF,MAAMJ,oBAC/C,sBAAsBT,MAAMU,MAAMK,WAAWE,UAAUR,oBACvD,gCAAgCT,MAAMU,MAAMK,WAAWG,oBAAoBT,uBAE7E,IACJ,IACA,eACA,mBACAT,MAAMH,SAASjF,MACf,IACA,gBACA,WACAiB,KAAKyD,UACHhB,sBAAsBK,eAAeqB,MAAMH,SAASvD,WACpD,MACA,IAEF,OACA,OACI0D,MAAMlG,SAAS,aAAakG,MAAMlG,SAAS,YAC3C,EACE,gBACGkG,MAAMwD,SAAS5F,KAAKyB,KACrB,EACE,OAAOA,EAAEtE,QACT,qBAAqBsE,EAAEN,WAAWhE,UAClC,mBAAmBsE,EAAEvE,SAASC,UAC9B,eAAesE,EAAEL,UACjB,OACIK,EAAEvE,SAASE,cAAc,EAACqE,EAAEvE,SAASE,aAAa,OAAM,KAC5DL,KAAK,YAGX,OACAqF,MAAMlG,SAAS,UACf,EACE,YACA,WACAwE,sBAAsBC,YACpB1C,KAAKyD,UAAUU,MAAMxB,OAAO,MAAM,KAEpC,OACA,OAEF,KACJ7D,KAAK;AAEV,EA3MD,CAAiB4I,oCAAAA,kCA2MhB,CAAA;;MChLYE;IAYX,WAAA3B,CAAmB3F;QACjB4F,KAAKC,SAAS7F,MAAMzC;QACpBqI,KAAKE,aAAa9F,MAAM+F,UAAU9E;QAClC2E,KAAKI,UAAU;YACbC,QAAQjG,MAAMkG,QAAQD,UAAU;YAChCE,cAAcnG,MAAMkG,QAAQC,gBAAgB;;QAE9CP,KAAK2B,aAAavH,MAAMzC,MAAME,qBAAqBwD;QACnD2E,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMnC,aAAmB,IAAIoC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAM3C,oBACEkD,QAAQC,IACZf,KAAKE,WAAWrE,KAAInE,MAAOoG;YACzB,MAAME,eAAgD8C,QAAQC,IAC5D,IAAItB,MAAMO,KAAKI,QAAQC,QAAQX,KAAK,GAAG7D,KAAInE;sBACnCkJ,UAAUI;gBAChB,MAAM5D,UACE4C,KAAKiB,KAAKnD;sBACZ8C,UAAUM;gBAChB,IAAIR,aAAa9G,WAAW8G,SAAStD;gBACrC,OAAOA;AAAC;YAGZ,OAAO;gBACLU;gBACAE;gBACAW,OAAOX,OACJtE,QAAQ0D,KAAMA,EAAErF,SAAS,UACzB8D,KAAKuB,KAAMA,EAAEuB,QACb3C,OAAO2D,mBAAmBE,MAAMF,mBAAmBC;;AACvD;QAGP,OAAQI,KAAKQ,UAAU;YACrB5C;YACAW;YACAF,cAAc,IAAIsC;YAClBhC,OAAOf,YACJ/B,KAAKkB,KAAMA,EAAE4B,QACb3C,OAAO2D,mBAAmBE,MAAMF,mBAAmBC;;;IAuBnD,MAAAuB;QACL,IAAInB,KAAKQ,YAAY,MACnB,MAAM,IAAI9D,MAAM;QAClB,OAAO8E,gCAAgC1E,SAASkD,KAAKQ;;IAG/C,UAAMS,CACZnD;QAEA,MAAMS,aAAmB,IAAIoC;QAC7B;YACE,MAAMhC,QAA6B;gBACjCC,OAAO;gBACPC,QAAQ;oBACND,OAAO;oBACPE,OAAO;oBACPC,QAAQ;;gBAEVC,YAAY;oBACVJ,OAAO;oBACPK,qBAAqB;oBACrBH,OAAO;oBACPI,WAAW;oBACXC,qBAAqB;;;YAGzB,MAAMI,gBACEqC,2BAA2BnB,QAAQ;mBACpCT,KAAKC,OAAO4B,WAAW;oBACxBhD,QAAQ;wBACN9G,MAAM;wBACNC,MAAM;wBACNa,MAAMiF,SAASjF;;oBAEjB8F;;gBAEFmD,WAAW9B,KAAK2B,WAAWtG;gBAC3BsB,OAAO;gBACPoF,OAAO,MAAM;gBACbC,UAAUtK;;YAEd,MAAM+J,WAA0ClC,QAC7C7F,QAAQqD,KAAMA,EAAEhF,SAAS,WACzB8D,KAAKkB,KAAMA,EAAEtC,aACbyC;YACH,OAAO;gBACLnF,MAAMP,4BAA4B2C,QAAQ;oBACxCI,UAAUuD,SAASvD;oBACnBE,YAAYgH;qBAEV,YACA;gBACJ3D;gBACA2D;gBACA9C;gBACAsD,kBAAkB1C,QACf7F,QAAQqD,KAAMA,EAAEhF,SAAS,SACzB2B,QACEqD,KACCA,EAAE/E,SAAS;gBAEjBuG;gBACAF,cAAc,IAAIsC;;UAIpB,OAAOlE;YACP,OAAO;gBACL1E,MAAM;gBACN+F;gBACArB;gBACA8B;gBACAF,cAAc,IAAIsC;;;;;;"}

package/lib/internal/AgenticaBenchmarkPredicator.d.ts

@@ -0,0 +1,32 @@
+import { Agentica, IAgenticaOperation, IAgenticaPrompt } from "@agentica/core";
+import { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
+export declare namespace AgenticaBenchmarkPredicator {
+    const isNext: (agent: Agentica) => Promise<string | null>;
+    /**
+     * Check if the called operations match the expected operations.
+     *
+     * @param props Properties for checking the match of the called operations
+     * and the expected operations
+     *
+     * @returns `true` if the called operations match the expected operations,
+     * otherwise `false`.
+     */
+    const success: (props: {
+        /**
+         * Expected operations to be called.
+         *
+         * For 'allOf' within an 'array', the next expected element starts checking from the element that follows the last called element in 'allOf'.
+         */
+        expected: IAgenticaBenchmarkExpected;
+        /**
+         * Specified operations.
+         */
+        operations: Array<IAgenticaOperation | IAgenticaPrompt.IExecute>;
+        /**
+         * If it's `false`, check the array and let it go even if there's something wrong between them.
+         *
+         * @default `false`
+         */
+        strict?: boolean;
+    }) => boolean;
+}