@agentica/benchmark 0.7.0-dev.20250224-2

package/lib/internal/AgenticaBenchmarkPredicator.js

@@ -0,0 +1,179 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgenticaBenchmarkPredicator = void 0;
+const typia_1 = __importDefault(require("typia"));
+var AgenticaBenchmarkPredicator;
+(function (AgenticaBenchmarkPredicator) {
+    AgenticaBenchmarkPredicator.isNext = (agent) => __awaiter(this, void 0, void 0, function* () {
+        var _a, _b, _c;
+        const last = agent.getPromptHistories().at(-1);
+        if ((last === null || last === void 0 ? void 0 : last.type) !== "text" || last.role !== "assistant")
+            return null;
+        const consent = {
+            model: "chatgpt",
+            options: {
+                reference: false,
+                strict: false,
+                separate: null
+            },
+            functions: [
+                {
+                    name: "consent",
+                    parameters: {
+                        description: "Properties for asking the user's consent",
+                        type: "object",
+                        properties: {
+                            content: {
+                                description: "Reason of the message implying what the AI agent wants\nto do at the next step after the user's consent.",
+                                type: "string"
+                            },
+                            reply: {
+                                title: "Recommended reply message for the user",
+                                description: "Recommended reply message for the user.\n\nThe message what AI agent wants the user to reply\naccepting the AI agent's next job suggestion.",
+                                type: "string"
+                            }
+                        },
+                        required: [
+                            "content",
+                            "reply"
+                        ],
+                        additionalProperties: false,
+                        $defs: {}
+                    },
+                    description: "Ask user to consent for what the AI agent wants to do next.\n\nIf AI agent wants to do some function calling at next,\nbut it needs the user's consent about the function calling to do,\nthen call this tool function."
+                }
+            ]
+        }.functions[0];
+        const result = yield agent["props"].provider.api.chat.completions.create({
+            model: agent["props"].provider.model,
+            messages: [
+                {
+                    role: "system",
+                    content: [
+                        "You are an helpful assistant.",
+                        "",
+                        "If what the assistant said seems like to asking for",
+                        "user's consent about some function calling at the next step,",
+                        "use the tools appropriately to step to the next.",
+                    ].join("\n"),
+                },
+                {
+                    role: "assistant",
+                    content: last.text,
+                },
+            ],
+            tools: [
+                {
+                    type: "function",
+                    function: {
+                        name: consent.name,
+                        description: consent.description,
+                        parameters: consent.parameters,
+                    },
+                },
+            ],
+            tool_choice: "required",
+            parallel_tool_calls: false,
+        }, agent["props"].provider.options);
+        const toolCall = (_c = ((_b = (_a = result.choices[0]) === null || _a === void 0 ? void 0 : _a.message.tool_calls) !== null && _b !== void 0 ? _b : []).filter((tc) => tc.type === "function" && tc.function.name === consent.name)) === null || _c === void 0 ? void 0 : _c[0];
+        if (toolCall === undefined)
+            return null;
+        const input = JSON.parse(toolCall.function.arguments);
+        return (() => { const _io0 = input => "string" === typeof input.content && "string" === typeof input.reply; return input => "object" === typeof input && null !== input && _io0(input); })()(input) ? input.reply : null;
+    });
+    /**
+     * Check if the called operations match the expected operations.
+     *
+     * @param props Properties for checking the match of the called operations
+     * and the expected operations
+     *
+     * @returns `true` if the called operations match the expected operations,
+     * otherwise `false`.
+     */
+    AgenticaBenchmarkPredicator.success = (props) => successInner(props).result;
+    const successInner = (props) => {
+        const call = (expected, overrideOperations) => successInner({
+            expected,
+            operations: overrideOperations !== null && overrideOperations !== void 0 ? overrideOperations : props.operations,
+            strict: props.strict,
+        });
+        switch (props.expected.type) {
+            case "array": {
+                let take = 0;
+                const targetIterator = props.expected.items[Symbol.iterator]();
+                let targeted = targetIterator.next();
+                while (true) {
+                    if (targeted.done) {
+                        return {
+                            result: true,
+                            take,
+                        };
+                    }
+                    if (take >= props.operations.length) {
+                        return { result: false };
+                    }
+                    const result = call(targeted.value, props.operations.slice(take));
+                    if (!result.result) {
+                        if (!props.strict) {
+                            take += 1;
+                            continue;
+                        }
+                        return { result: false };
+                    }
+                    take += result.take;
+                    targeted = targetIterator.next();
+                }
+            }
+            case "standalone": {
+                const target = props.expected.operation;
+                const result = props.operations.some((op) => op.name === target.name);
+                if (result) {
+                    return { result, take: 1 };
+                }
+                return {
+                    result,
+                };
+            }
+            case "anyOf":
+                for (const expected of props.expected.anyOf) {
+                    const callResult = call(expected);
+                    if (callResult.result) {
+                        return callResult;
+                    }
+                }
+                return { result: false };
+            case "allOf": {
+                /**
+                 * @example
+                 * expected = [4, 2];
+                 * called = [1, 2, 3, 4, 5];
+                 *
+                 * { result: true, take: 3 };
+                 */
+                const result = props.expected.allOf.map((expected) => call(expected));
+                if (result.every((r) => r.result)) {
+                    return {
+                        result: true,
+                        take: result.reduce((acc, r) => Math.max(acc, r.take), 0),
+                    };
+                }
+                return {
+                    result: false,
+                };
+            }
+        }
+    };
+})(AgenticaBenchmarkPredicator || (exports.AgenticaBenchmarkPredicator = AgenticaBenchmarkPredicator = {}));
+//# sourceMappingURL=AgenticaBenchmarkPredicator.js.map

package/lib/internal/AgenticaBenchmarkPredicator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgenticaBenchmarkPredicator.js","sourceRoot":"","sources":["../../src/internal/AgenticaBenchmarkPredicator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAGA,kDAA0B;AAI1B,IAAiB,2BAA2B,CAgL3C;AAhLD,WAAiB,2BAA2B;IAC7B,kCAAM,GAAG,CAAO,KAAe,EAA0B,EAAE;;QACtE,MAAM,IAAI,GAAgC,KAAK,CAAC,kBAAkB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,MAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW;YAAE,OAAO,IAAI,CAAC;QAEpE,MAAM,OAAO,GAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAGrC,SAAS,CAAC,CAAC,CAAE,CAAC;QAClB,MAAM,MAAM,GAA0B,MAAM,KAAK,CAC/C,OAAO,CACR,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CACpC;YACE,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,KAAK;YACpC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE;wBACP,+BAA+B;wBAC/B,EAAE;wBACF,qDAAqD;wBACrD,8DAA8D;wBAC9D,kDAAkD;qBACnD,CAAC,IAAI,CAAC,IAAI,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,IAAI,CAAC,IAAI;iBACnB;aACF;YACD,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,UAAU;oBAChB,QAAQ,EAAE;wBACR,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,UAAU,EAAE,OAAO,CAAC,UAAiC;qBACtD;iBACF;aACF;YACD,WAAW,EAAE,UAAU;YACvB,mBAAmB,EAAE,KAAK;SAC3B,EACD,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAChC,CAAC;QACF,MAAM,QAAQ,GAAqD,MAAA,CACjE,MAAA,MAAA,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,OAAO,CAAC,UAAU,mCAAI,EAAE,CAC5C,CAAC,MAAM,CACN,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,UAAU,IAAI,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CACpE,0CAAG,CAAC,CAAC,CAAC;QACP,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACxC,MAAM,KAAK,GAAkB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACrE,OAAO,sLAAS,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC,CAAA,CAAC;IAEF;;;;;;;;OAQG;IACU,mCAAO,GAAG,CAAC,KAmBvB,EAAW,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAE1C,MAAM,YAAY,GAAG,CACnB,KAAoC,EAQhC,EAAE;QACN,MAAM,IAAI,GAAG,CACX,QAAoC,EACpC,kBAAyE,EACzE,EAAE,CACF,YAAY,CAAC;YACX,QAAQ;YACR,UAAU,EAAE,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,KAAK,CAAC,UAAU;YAClD,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CAAC;QAEL,QAAQ,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC5B,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,IAAI,IAAI,GAAG,CAAC,CAAC;gBACb,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/D,IAAI,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;gBAErC,OAAO,IAAI,EAAE,CAAC;oBACZ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;wBAClB,OAAO;4BACL,MAAM,EAAE,IAAI;4BACZ,IAAI;yBACL,CAAC;oBACJ,CAAC;oBACD,IAAI,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;wBACpC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;oBAC3B,CAAC;oBAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;oBAClE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;wBACnB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;4BAClB,IAAI,IAAI,CAAC,CAAC;4BACV,SAAS;wBACX,CAAC;wBACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;oBAC3B,CAAC;oBAED,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;oBACpB,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;gBACnC,CAAC;YACH,CAAC;YACD,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACxC,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;gBACtE,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC7B,CAAC;gBACD,OAAO;oBACL,MAAM;iBACP,CAAC;YACJ,CAAC;YACD,KAAK,OAAO;gBACV,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;oBAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAClC,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;wBACtB,OAAO,UAAU,CAAC;oBACpB,CAAC;gBACH,CAAC;gBAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAC3B,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb;;;;;;mBAMG;gBACH,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACtE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;oBAClC,OAAO;wBACL,MAAM,EAAE,IAAI;wBACZ,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;qBAC1D,CAAC;gBACJ,CAAC;gBAED,OAAO;oBACL,MAAM,EAAE,KAAK;iBACd,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC;AACJ,CAAC,EAhLgB,2BAA2B,2CAA3B,2BAA2B,QAgL3C"}

package/lib/internal/AgenticaBenchmarkUtil.d.ts

@@ -0,0 +1,5 @@
+import { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
+export declare namespace AgenticaBenchmarkUtil {
+    const errorToJson: (error: any) => any;
+    const expectedToJson: (expected: IAgenticaBenchmarkExpected) => any;
+}

package/lib/internal/AgenticaBenchmarkUtil.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgenticaBenchmarkUtil = void 0;
+var AgenticaBenchmarkUtil;
+(function (AgenticaBenchmarkUtil) {
+    AgenticaBenchmarkUtil.errorToJson = (error) => {
+        if (error instanceof Error)
+            return Object.assign(Object.assign({}, error), { name: error.name, message: error.message, stack: error.stack });
+        return error;
+    };
+    AgenticaBenchmarkUtil.expectedToJson = (expected) => {
+        if (expected.type === "standalone")
+            return {
+                type: expected.type,
+                operation: {
+                    name: expected.operation.name,
+                    description: expected.operation.function.description,
+                },
+            };
+        else if (expected.type === "array")
+            return {
+                type: expected.type,
+                items: expected.items.map(AgenticaBenchmarkUtil.expectedToJson),
+            };
+        else if (expected.type === "allOf")
+            return {
+                type: expected.type,
+                allOf: expected.allOf.map(AgenticaBenchmarkUtil.expectedToJson),
+            };
+        else
+            return {
+                type: expected.type,
+                anyOf: expected.anyOf.map(AgenticaBenchmarkUtil.expectedToJson),
+            };
+    };
+})(AgenticaBenchmarkUtil || (exports.AgenticaBenchmarkUtil = AgenticaBenchmarkUtil = {}));
+//# sourceMappingURL=AgenticaBenchmarkUtil.js.map

package/lib/internal/AgenticaBenchmarkUtil.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgenticaBenchmarkUtil.js","sourceRoot":"","sources":["../../src/internal/AgenticaBenchmarkUtil.ts"],"names":[],"mappings":";;;AAEA,IAAiB,qBAAqB,CAqCrC;AArCD,WAAiB,qBAAqB;IACvB,iCAAW,GAAG,CAAC,KAAU,EAAO,EAAE;QAC7C,IAAI,KAAK,YAAY,KAAK;YACxB,uCACK,KAAK,KACR,IAAI,EAAE,KAAK,CAAC,IAAI,EAChB,OAAO,EAAE,KAAK,CAAC,OAAO,EACtB,KAAK,EAAE,KAAK,CAAC,KAAK,IAClB;QACJ,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEW,oCAAc,GAAG,CAAC,QAAoC,EAAO,EAAE;QAC1E,IAAI,QAAQ,CAAC,IAAI,KAAK,YAAY;YAChC,OAAO;gBACL,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,SAAS,EAAE;oBACT,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI;oBAC7B,WAAW,EAAE,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW;iBACrD;aACF,CAAC;aACC,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO;YAChC,OAAO;gBACL,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAA,cAAc,CAAC;aAC1C,CAAC;aACC,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO;YAChC,OAAO;gBACL,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAA,cAAc,CAAC;aAC1C,CAAC;;YAEF,OAAO;gBACL,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAA,cAAc,CAAC;aAC1C,CAAC;IACN,CAAC,CAAC;AACJ,CAAC,EArCgB,qBAAqB,qCAArB,qBAAqB,QAqCrC"}

package/lib/internal/AgenticaCallBenchmarkReporter.d.ts

@@ -0,0 +1,4 @@
+import { IAgenticaCallBenchmarkResult } from "../structures/IAgenticaCallBenchmarkResult";
+export declare namespace AgenticaCallBenchmarkReporter {
+    const markdown: (result: IAgenticaCallBenchmarkResult) => Record<string, string>;
+}

package/lib/internal/AgenticaCallBenchmarkReporter.js

@@ -0,0 +1,136 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgenticaCallBenchmarkReporter = void 0;
+const MathUtil_1 = require("../utils/MathUtil");
+const AgenticaBenchmarkUtil_1 = require("./AgenticaBenchmarkUtil");
+const AgenticaPromptReporter_1 = require("./AgenticaPromptReporter");
+var AgenticaCallBenchmarkReporter;
+(function (AgenticaCallBenchmarkReporter) {
+    AgenticaCallBenchmarkReporter.markdown = (result) => Object.fromEntries([
+        ["./README.md", writeIndex(result)],
+        ...result.experiments
+            .map((exp) => [
+            [`./${exp.scenario.name}/README.md`, writeExperimentIndex(exp)],
+            ...exp.events.map((event, i) => [
+                `./${exp.scenario.name}/${i + 1}.${event.type}.md`,
+                writeExperimentEvent(event, i),
+            ]),
+        ])
+            .flat(),
+    ]);
+    const writeIndex = (result) => {
+        const events = result.experiments
+            .map((r) => r.events)
+            .flat();
+        const average = events
+            .map((e) => e.completed_at.getTime() - e.started_at.getTime())
+            .reduce((a, b) => a + b, 0) / events.length;
+        return [
+            "# LLM Function Call Benchmark",
+            "## Summary",
+            `  - Aggregation:`,
+            `    - Scenarios: #${result.experiments.length.toLocaleString()}`,
+            `    - Trial: ${events.length}`,
+            `    - Success: ${events.filter((e) => e.type === "success").length}`,
+            `    - Failure: ${events.filter((e) => e.type === "failure").length}`,
+            `    - Average Time: ${MathUtil_1.MathUtil.round(average).toLocaleString()} ms`,
+            `  - Token Usage`,
+            `    - Total: ${result.usage.total.toLocaleString()}`,
+            `    - Prompt`,
+            `      - Total: ${result.usage.prompt.total.toLocaleString()}`,
+            `      - Audio: ${result.usage.prompt.audio.toLocaleString()}`,
+            `      - Cached: ${result.usage.prompt.cached.toLocaleString()}`,
+            `    - Completion:`,
+            `      - Total: ${result.usage.completion.total.toLocaleString()}`,
+            `      - Accepted Prediction: ${result.usage.completion.accepted_prediction.toLocaleString()}`,
+            `      - Audio: ${result.usage.completion.audio.toLocaleString()}`,
+            `      - Reasoning: ${result.usage.completion.reasoning.toLocaleString()}`,
+            `      - Rejected Prediction: ${result.usage.completion.rejected_prediction.toLocaleString()}`,
+            "",
+            "## Experiments",
+            " Name | Select | Call | Time/Avg ",
+            ":-----|:-------|:-----|----------:",
+            ...result.experiments.map((exp) => [
+                `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`,
+                drawStatus(exp.events, (e) => e.type !== "error" && e.select === true),
+                drawStatus(exp.events, (e) => e.type !== "error" && e.call === true),
+                `${MathUtil_1.MathUtil.round(exp.events
+                    .map((e) => e.completed_at.getTime() - e.started_at.getTime())
+                    .reduce((a, b) => a + b, 0) / exp.events.length).toLocaleString()} ms`,
+            ].join(" | ")),
+        ].join("\n");
+    };
+    const writeExperimentIndex = (exp) => {
+        return [
+            `# ${exp.scenario.name}`,
+            "## Summary",
+            `  - Scenarios: #${exp.events.length.toLocaleString()}`,
+            `  - Success: ${exp.events.filter((e) => e.type === "success").length}`,
+            `  - Failure: ${exp.events.filter((e) => e.type === "failure").length}`,
+            `  - Average Time: ${MathUtil_1.MathUtil.round(exp.events
+                .map((e) => e.completed_at.getTime() - e.started_at.getTime())
+                .reduce((a, b) => a + b, 0) / exp.events.length).toLocaleString()} ms`,
+            "",
+            "## Events",
+            " Name | Type | Time",
+            ":-----|:-----|----:",
+            ...exp.events.map((e, i) => [
+                `[${i + 1}.](./${i + 1}.${e.type}.md)`,
+                e.type,
+                `${MathUtil_1.MathUtil.round(e.completed_at.getTime() - e.started_at.getTime())} ms`,
+            ].join(" | ")),
+            "",
+            "## Scenario",
+            "### User Prompt",
+            exp.scenario.text,
+            "",
+            "### Expected",
+            "```json",
+            JSON.stringify(AgenticaBenchmarkUtil_1.AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected), null, 2),
+            "```",
+        ].join("\n");
+    };
+    const writeExperimentEvent = (event, index) => {
+        return [
+            `# ${index}. ${event.type}`,
+            "## Summary",
+            `  - Name: ${event.scenario.name}`,
+            `  - Type: ${event.type}`,
+            `  - Time: ${MathUtil_1.MathUtil.round(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`,
+            ...(event.type !== "error"
+                ? [
+                    `  - Select: ${event.select ? "✅" : "❌"}`,
+                    `  - Call: ${event.call ? "✅" : "❌"}`,
+                ]
+                : []),
+            `  - Token Usage: ${event.usage.toLocaleString()}`,
+            "",
+            "## Scenario",
+            "### User Prompt",
+            event.scenario.text,
+            "",
+            "### Expected",
+            "```json",
+            JSON.stringify(AgenticaBenchmarkUtil_1.AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected), null, 2),
+            "```",
+            "",
+            "## Prompt Histories",
+            ...event.prompts.map(AgenticaPromptReporter_1.AgenticaPromptReporter.markdown),
+            "",
+            ...(event.type === "error"
+                ? [
+                    "## Error",
+                    "```json",
+                    JSON.stringify(AgenticaBenchmarkUtil_1.AgenticaBenchmarkUtil.errorToJson(event.error), null, 2),
+                    "```",
+                ]
+                : []),
+        ].join("\n");
+    };
+    const drawStatus = (events, success) => {
+        const count = events.filter(success).length;
+        return (new Array(count).fill("■").join("") +
+            new Array(10 - count).fill("□").join(""));
+    };
+})(AgenticaCallBenchmarkReporter || (exports.AgenticaCallBenchmarkReporter = AgenticaCallBenchmarkReporter = {}));
+//# sourceMappingURL=AgenticaCallBenchmarkReporter.js.map

package/lib/internal/AgenticaCallBenchmarkReporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgenticaCallBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaCallBenchmarkReporter.ts"],"names":[],"mappings":";;;AAEA,gDAA6C;AAC7C,mEAAgE;AAChE,qEAAkE;AAElE,IAAiB,6BAA6B,CA0K7C;AA1KD,WAAiB,6BAA6B;IAC/B,sCAAQ,GAAG,CACtB,MAAoC,EACZ,EAAE,CAC1B,MAAM,CAAC,WAAW,CAAC;QACjB,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACnC,GAAG,MAAM,CAAC,WAAW;aAClB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;YACZ,CAAC,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,YAAY,EAAE,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC/D,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK;gBAClD,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC;aAC/B,CAAC;SACH,CAAC;aACD,IAAI,EAAE;KACV,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,CAAC,MAAoC,EAAU,EAAE;QAClE,MAAM,MAAM,GAAkC,MAAM,CAAC,WAAW;aAC7D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,IAAI,EAAE,CAAC;QACV,MAAM,OAAO,GACX,MAAM;aACH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAChD,OAAO;YACL,+BAA+B;YAC/B,YAAY;YACZ,kBAAkB;YAClB,qBAAqB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YACjE,gBAAgB,MAAM,CAAC,MAAM,EAAE;YAC/B,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACrE,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACrE,uBAAuB,mBAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE,KAAK;YACpE,iBAAiB;YACjB,gBAAgB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YACrD,cAAc;YACd,kBAAkB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC9D,kBAAkB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC9D,mBAAmB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YAChE,mBAAmB;YACnB,kBAAkB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClE,gCAAgC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YAC9F,kBAAkB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClE,sBAAsB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;YAC1E,gCAAgC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YAC9F,EAAE;YACF,gBAAgB;YAChB,mCAAmC;YACnC,oCAAoC;YACpC,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAChC;gBACE,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC,IAAI,aAAa;gBAC1D,UAAU,CACR,GAAG,CAAC,MAAM,EACV,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAC/C;gBACD,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC;gBACpE,GAAG,mBAAQ,CAAC,KAAK,CACf,GAAG,CAAC,MAAM;qBACP,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;qBAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;aACxB,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;SACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,oBAAoB,GAAG,CAC3B,GAA6C,EACrC,EAAE;QACV,OAAO;YACL,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE;YACxB,YAAY;YACZ,mBAAmB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YACvD,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACvE,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACvE,qBAAqB,mBAAQ,CAAC,KAAK,CACjC,GAAG,CAAC,MAAM;iBACP,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;iBAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;YACvB,EAAE;YACF,WAAW;YACX,qBAAqB;YACrB,qBAAqB;YACrB,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB;gBACE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM;gBACtC,CAAC,CAAC,IAAI;gBACN,GAAG,mBAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK;aAC1E,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;YACD,EAAE;YACF,aAAa;YACb,iBAAiB;YACjB,GAAG,CAAC,QAAQ,CAAC,IAAI;YACjB,EAAE;YACF,cAAc;YACd,SAAS;YACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3D,IAAI,EACJ,CAAC,CACF;YACD,KAAK;SACN,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,oBAAoB,GAAG,CAC3B,KAAkC,EAClC,KAAa,EACL,EAAE;QACV,OAAO;YACL,KAAK,KAAK,KAAK,KAAK,CAAC,IAAI,EAAE;YAC3B,YAAY;YACZ,aAAa,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE;YAClC,aAAa,KAAK,CAAC,IAAI,EAAE;YACzB,aAAa,mBAAQ,CAAC,KAAK,CACzB,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC1D,CAAC,cAAc,EAAE,KAAK;YACvB,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;gBACxB,CAAC,CAAC;oBACE,eAAe,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE;oBACzC,aAAa,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE;iBACtC;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,oBAAoB,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClD,EAAE;YACF,aAAa;YACb,iBAAiB;YACjB,KAAK,CAAC,QAAQ,CAAC,IAAI;YACnB,EAAE;YACF,cAAc;YACd,SAAS;YACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC7D,IAAI,EACJ,CAAC,CACF;YACD,KAAK;YACL,EAAE;YACF,qBAAqB;YACrB,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,+CAAsB,CAAC,QAAQ,CAAC;YACrD,EAAE;YACF,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;gBACxB,CAAC,CAAC;oBACE,UAAU;oBACV,SAAS;oBACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,EAC9C,IAAI,EACJ,CAAC,CACF;oBACD,KAAK;iBACN;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,CACjB,MAAqC,EACrC,OAAoD,EAC5C,EAAE;QACV,MAAM,KAAK,GAAW,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QACpD,OAAO,CACL,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CACzC,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC,EA1KgB,6BAA6B,6CAA7B,6BAA6B,QA0K7C"}

package/lib/internal/AgenticaPromptReporter.d.ts

@@ -0,0 +1,4 @@
+import { IAgenticaPrompt } from "@agentica/core";
+export declare namespace AgenticaPromptReporter {
+    const markdown: (p: IAgenticaPrompt) => string;
+}

package/lib/internal/AgenticaPromptReporter.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgenticaPromptReporter = void 0;
+var AgenticaPromptReporter;
+(function (AgenticaPromptReporter) {
+    AgenticaPromptReporter.markdown = (p) => {
+        if (p.type === "text")
+            return [`### Text (${p.role})`, p.text, ""].join("\n");
+        else if (p.type === "select" || p.type === "cancel")
+            return [
+                `### ${p.type === "select" ? "Select" : "Cancel"}`,
+                ...p.operations
+                    .map((op) => {
+                    var _a;
+                    return [
+                        `#### ${op.name}`,
+                        `  - controller: ${op.controller.name}`,
+                        `  - function: ${op.function.name}`,
+                        `  - reason: ${op.reason}`,
+                        "",
+                        ...(!!((_a = op.function.description) === null || _a === void 0 ? void 0 : _a.length)
+                            ? [op.function.description, ""]
+                            : []),
+                    ];
+                })
+                    .flat(),
+            ].join("\n");
+        else if (p.type === "describe")
+            return [
+                "### Describe",
+                ...p.executions.map((e) => `  - ${e.name}`),
+                "",
+                ...p.text.split("\n").map((s) => `> ${s}`),
+                "",
+            ].join("\n");
+        return [
+            "### Execute",
+            `  - name: ${p.name}`,
+            `  - controller: ${p.controller.name}`,
+            `  - function: ${p.function.name}`,
+            "",
+            "```json",
+            JSON.stringify(p.arguments, null, 2),
+            "```",
+            "",
+        ].join("\n");
+    };
+})(AgenticaPromptReporter || (exports.AgenticaPromptReporter = AgenticaPromptReporter = {}));
+//# sourceMappingURL=AgenticaPromptReporter.js.map

package/lib/internal/AgenticaPromptReporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgenticaPromptReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaPromptReporter.ts"],"names":[],"mappings":";;;AAEA,IAAiB,sBAAsB,CAwCtC;AAxCD,WAAiB,sBAAsB;IACxB,+BAAQ,GAAG,CAAC,CAAkB,EAAU,EAAE;QACrD,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YACnB,OAAO,CAAC,aAAa,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACpD,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ;YACjD,OAAO;gBACL,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE;gBAClD,GAAG,CAAC,CAAC,UAAU;qBACZ,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;;oBAAC,OAAA;wBACX,QAAQ,EAAE,CAAC,IAAI,EAAE;wBACjB,mBAAmB,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE;wBACvC,iBAAiB,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE;wBACnC,eAAe,EAAE,CAAC,MAAM,EAAE;wBAC1B,EAAE;wBACF,GAAG,CAAC,CAAC,CAAC,CAAA,MAAA,EAAE,CAAC,QAAQ,CAAC,WAAW,0CAAE,MAAM,CAAA;4BACnC,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;4BAC/B,CAAC,CAAC,EAAE,CAAC;qBACR,CAAA;iBAAA,CAAC;qBACD,IAAI,EAAE;aACV,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACV,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU;YAC5B,OAAO;gBACL,cAAc;gBACd,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3C,EAAE;gBACF,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1C,EAAE;aACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,OAAO;YACL,aAAa;YACb,aAAa,CAAC,CAAC,IAAI,EAAE;YACrB,mBAAmB,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE;YACtC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE;YAClC,EAAE;YACF,SAAS;YACT,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACpC,KAAK;YACL,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,EAxCgB,sBAAsB,sCAAtB,sBAAsB,QAwCtC"}

package/lib/internal/AgenticaSelectBenchmarkReporter.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/internal/AgenticaSelectBenchmarkReporter.js

@@ -0,0 +1,172 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgenticaSelectBenchmarkReporter = void 0;
+const MathUtil_1 = require("../utils/MathUtil");
+const AgenticaBenchmarkUtil_1 = require("./AgenticaBenchmarkUtil");
+/**
+ * @internal
+ */
+var AgenticaSelectBenchmarkReporter;
+(function (AgenticaSelectBenchmarkReporter) {
+    AgenticaSelectBenchmarkReporter.markdown = (result) => Object.fromEntries([
+        ["./README.md", writeIndex(result)],
+        ...result.experiments
+            .map((exp) => [
+            [`./${exp.scenario.name}/README.md`, writeExperimentIndex(exp)],
+            ...exp.events.map((event, i) => [
+                `./${exp.scenario.name}/${i + 1}.${event.type}.md`,
+                writeExperimentEvent(event, i),
+            ]),
+        ])
+            .flat(),
+    ]);
+    const writeIndex = (result) => {
+        const events = result.experiments
+            .map((r) => r.events)
+            .flat();
+        const average = events
+            .map((e) => e.completed_at.getTime() - e.started_at.getTime())
+            .reduce((a, b) => a + b, 0) / events.length;
+        return [
+            "# LLM Function Selection Benchmark",
+            "## Summary",
+            `  - Aggregation:`,
+            `    - Scenarios: #${result.experiments.length.toLocaleString()}`,
+            `    - Trial: ${events.length}`,
+            `    - Success: ${events.filter((e) => e.type === "success").length}`,
+            `    - Failure: ${events.filter((e) => e.type === "failure").length}`,
+            `    - Average Time: ${MathUtil_1.MathUtil.round(average).toLocaleString()} ms`,
+            `  - Token Usage`,
+            `    - Total: ${result.usage.total.toLocaleString()}`,
+            `    - Prompt`,
+            `      - Total: ${result.usage.prompt.total.toLocaleString()}`,
+            `      - Audio: ${result.usage.prompt.audio.toLocaleString()}`,
+            `      - Cached: ${result.usage.prompt.cached.toLocaleString()}`,
+            `    - Completion:`,
+            `      - Total: ${result.usage.completion.total.toLocaleString()}`,
+            `      - Accepted Prediction: ${result.usage.completion.accepted_prediction.toLocaleString()}`,
+            `      - Audio: ${result.usage.completion.audio.toLocaleString()}`,
+            `      - Reasoning: ${result.usage.completion.reasoning.toLocaleString()}`,
+            `      - Rejected Prediction: ${result.usage.completion.rejected_prediction.toLocaleString()}`,
+            "",
+            "## Experiments",
+            " Name | Status | Time/Avg  ",
+            ":-----|:-------|----------:",
+            ...result.experiments.map((exp) => [
+                `[${exp.scenario.name}](./${exp.scenario.name}/README.md)`,
+                (() => {
+                    const success = Math.floor((exp.events.filter((e) => e.type === "success").length /
+                        exp.events.length) *
+                        10);
+                    return (new Array(success).fill("■").join("") +
+                        new Array(10 - success).fill("□").join(""));
+                })(),
+                MathUtil_1.MathUtil.round(exp.events
+                    .map((event) => event.completed_at.getTime() - event.started_at.getTime())
+                    .reduce((a, b) => a + b, 0) / exp.events.length).toLocaleString() + " ms",
+            ].join(" | ")),
+        ].join("\n");
+    };
+    const writeExperimentIndex = (exp) => {
+        return [
+            `# ${exp.scenario.name}`,
+            "## Summary",
+            "  - Aggregation:",
+            `    - Trial: ${exp.events.length}`,
+            `    - Success: ${exp.events.filter((e) => e.type === "success").length}`,
+            `    - Failure: ${exp.events.filter((e) => e.type === "failure").length}`,
+            `    - Average Time: ${MathUtil_1.MathUtil.round(exp.events
+                .map((event) => event.completed_at.getTime() - event.started_at.getTime())
+                .reduce((a, b) => a + b, 0) / exp.events.length).toLocaleString()} ms`,
+            "  - Token Usage",
+            `    - Total: ${exp.usage.total.toLocaleString()}`,
+            `    - Prompt`,
+            `      - Total: ${exp.usage.prompt.total.toLocaleString()}`,
+            `      - Audio: ${exp.usage.prompt.audio.toLocaleString()}`,
+            `      - Cached: ${exp.usage.prompt.cached.toLocaleString()}`,
+            `    - Completion:`,
+            `      - Total: ${exp.usage.completion.total.toLocaleString()}`,
+            `      - Accepted Prediction: ${exp.usage.completion.accepted_prediction.toLocaleString()}`,
+            `      - Audio: ${exp.usage.completion.audio.toLocaleString()}`,
+            `      - Reasoning: ${exp.usage.completion.reasoning.toLocaleString()}`,
+            `      - Rejected Prediction: ${exp.usage.completion.rejected_prediction.toLocaleString()}`,
+            "",
+            "## Events",
+            " No | Type | Time",
+            "---:|:-----|----:",
+            ...exp.events.map((e, i) => [
+                `[${i + 1}.](./${i + 1}.${e.type}.md)`,
+                e.type,
+                MathUtil_1.MathUtil.round(e.completed_at.getTime() - e.started_at.getTime()) +
+                    " ms",
+            ].join(" | ")),
+            "",
+            "## Scenario",
+            "### User Prompt",
+            exp.scenario.text,
+            "",
+            "### Expected",
+            "```json",
+            JSON.stringify(AgenticaBenchmarkUtil_1.AgenticaBenchmarkUtil.expectedToJson(exp.scenario.expected), null, 2),
+            "```",
+        ].join("\n");
+    };
+    const writeExperimentEvent = (event, index) => {
+        return [
+            `# ${index}. ${event.type}`,
+            `## Summary`,
+            `  - Name: ${event.scenario.name}`,
+            `  - Type: ${event.type}`,
+            `  - Time: ${(event.completed_at.getTime() - event.started_at.getTime()).toLocaleString()} ms`,
+            ...(event.type !== "error"
+                ? [
+                    "  - Token Usage",
+                    `    - Total: ${event.usage.total.toLocaleString()}`,
+                    `    - Prompt`,
+                    `      - Total: ${event.usage.prompt.total.toLocaleString()}`,
+                    `      - Audio: ${event.usage.prompt.audio.toLocaleString()}`,
+                    `      - Cached: ${event.usage.prompt.cached.toLocaleString()}`,
+                    `    - Completion:`,
+                    `      - Total: ${event.usage.completion.total.toLocaleString()}`,
+                    `      - Accepted Prediction: ${event.usage.completion.accepted_prediction.toLocaleString()}`,
+                    `      - Audio: ${event.usage.completion.audio.toLocaleString()}`,
+                    `      - Reasoning: ${event.usage.completion.reasoning.toLocaleString()}`,
+                    `      - Rejected Prediction: ${event.usage.completion.rejected_prediction.toLocaleString()}`,
+                ]
+                : []),
+            "",
+            "## Scenario",
+            "### User Prompt",
+            event.scenario.text,
+            "",
+            "### Expected",
+            "```json",
+            JSON.stringify(AgenticaBenchmarkUtil_1.AgenticaBenchmarkUtil.expectedToJson(event.scenario.expected), null, 2),
+            "```",
+            "",
+            ...(event.type === "success" || event.type === "failure"
+                ? [
+                    "## Result",
+                    ...event.selected.map((s) => [
+                        `### ${s.name}`,
+                        `  - Controller: \`${s.controller.name}\``,
+                        `  - Function: \`${s.function.name}\``,
+                        `  - Reason: ${s.reason}`,
+                        "",
+                        ...(s.function.description ? [s.function.description, ""] : []),
+                    ].join("\n")),
+                ]
+                : []),
+            ...(event.type === "error"
+                ? [
+                    "## Error",
+                    "```json",
+                    AgenticaBenchmarkUtil_1.AgenticaBenchmarkUtil.errorToJson(JSON.stringify(event.error, null, 2)),
+                    "```",
+                    "",
+                ]
+                : []),
+        ].join("\n");
+    };
+})(AgenticaSelectBenchmarkReporter || (exports.AgenticaSelectBenchmarkReporter = AgenticaSelectBenchmarkReporter = {}));
+//# sourceMappingURL=AgenticaSelectBenchmarkReporter.js.map

package/lib/internal/AgenticaSelectBenchmarkReporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgenticaSelectBenchmarkReporter.js","sourceRoot":"","sources":["../../src/internal/AgenticaSelectBenchmarkReporter.ts"],"names":[],"mappings":";;;AAEA,gDAA6C;AAC7C,mEAAgE;AAEhE;;GAEG;AACH,IAAiB,+BAA+B,CA2M/C;AA3MD,WAAiB,+BAA+B;IACjC,wCAAQ,GAAG,CACtB,MAAsC,EACd,EAAE,CAC1B,MAAM,CAAC,WAAW,CAAC;QACjB,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACnC,GAAG,MAAM,CAAC,WAAW;aAClB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;YACZ,CAAC,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,YAAY,EAAE,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC/D,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK;gBAClD,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC;aAC/B,CAAC;SACH,CAAC;aACD,IAAI,EAAE;KACV,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,CAAC,MAAsC,EAAU,EAAE;QACpE,MAAM,MAAM,GAAoC,MAAM,CAAC,WAAW;aAC/D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,IAAI,EAAE,CAAC;QACV,MAAM,OAAO,GACX,MAAM;aACH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAChD,OAAO;YACL,oCAAoC;YACpC,YAAY;YACZ,kBAAkB;YAClB,qBAAqB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YACjE,gBAAgB,MAAM,CAAC,MAAM,EAAE;YAC/B,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACrE,kBAAkB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACrE,uBAAuB,mBAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE,KAAK;YACpE,iBAAiB;YACjB,gBAAgB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YACrD,cAAc;YACd,kBAAkB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC9D,kBAAkB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC9D,mBAAmB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YAChE,mBAAmB;YACnB,kBAAkB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClE,gCAAgC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YAC9F,kBAAkB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClE,sBAAsB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;YAC1E,gCAAgC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YAC9F,EAAE;YACF,gBAAgB;YAChB,6BAA6B;YAC7B,6BAA6B;YAC7B,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAChC;gBACE,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC,IAAI,aAAa;gBAC1D,CAAC,GAAG,EAAE;oBACJ,MAAM,OAAO,GAAW,IAAI,CAAC,KAAK,CAChC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM;wBACpD,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;wBAClB,EAAE,CACL,CAAC;oBACF,OAAO,CACL,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;wBACrC,IAAI,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAC3C,CAAC;gBACJ,CAAC,CAAC,EAAE;gBACJ,mBAAQ,CAAC,KAAK,CACZ,GAAG,CAAC,MAAM;qBACP,GAAG,CACF,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC5D;qBACA,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,GAAG,KAAK;aAC3B,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;SACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,oBAAoB,GAAG,CAC3B,GAA+C,EACvC,EAAE;QACV,OAAO;YACL,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE;YACxB,YAAY;YACZ,kBAAkB;YAClB,gBAAgB,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE;YACnC,kBAAkB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACzE,kBAAkB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;YACzE,uBAAuB,mBAAQ,CAAC,KAAK,CACnC,GAAG,CAAC,MAAM;iBACP,GAAG,CACF,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAC5D;iBACA,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAClD,CAAC,cAAc,EAAE,KAAK;YACvB,iBAAiB;YACjB,gBAAgB,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAClD,cAAc;YACd,kBAAkB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC3D,kBAAkB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC3D,mBAAmB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;YAC7D,mBAAmB;YACnB,kBAAkB,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC/D,gCAAgC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YAC3F,kBAAkB,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;YAC/D,sBAAsB,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;YACvE,gCAAgC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;YAC3F,EAAE;YACF,WAAW;YACX,mBAAmB;YACnB,mBAAmB;YACnB,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB;gBACE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM;gBACtC,CAAC,CAAC,IAAI;gBACN,mBAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;oBAC/D,KAAK;aACR,CAAC,IAAI,CAAC,KAAK,CAAC,CACd;YACD,EAAE;YACF,aAAa;YACb,iBAAiB;YACjB,GAAG,CAAC,QAAQ,CAAC,IAAI;YACjB,EAAE;YACF,cAAc;YACd,SAAS;YACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3D,IAAI,EACJ,CAAC,CACF;YACD,KAAK;SACN,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,oBAAoB,GAAG,CAC3B,KAAoC,EACpC,KAAa,EACL,EAAE;QACV,OAAO;YACL,KAAK,KAAK,KAAK,KAAK,CAAC,IAAI,EAAE;YAC3B,YAAY;YACZ,aAAa,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE;YAClC,aAAa,KAAK,CAAC,IAAI,EAAE;YACzB,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,cAAc,EAAE,KAAK;YAC9F,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;gBACxB,CAAC,CAAC;oBACE,iBAAiB;oBACjB,gBAAgB,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;oBACpD,cAAc;oBACd,kBAAkB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;oBAC7D,kBAAkB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;oBAC7D,mBAAmB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE;oBAC/D,mBAAmB;oBACnB,kBAAkB,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;oBACjE,gCAAgC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;oBAC7F,kBAAkB,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE;oBACjE,sBAAsB,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;oBACzE,gCAAgC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,cAAc,EAAE,EAAE;iBAC9F;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,EAAE;YACF,aAAa;YACb,iBAAiB;YACjB,KAAK,CAAC,QAAQ,CAAC,IAAI;YACnB,EAAE;YACF,cAAc;YACd,SAAS;YACT,IAAI,CAAC,SAAS,CACZ,6CAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC7D,IAAI,EACJ,CAAC,CACF;YACD,KAAK;YACL,EAAE;YACF,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;gBACtD,CAAC,CAAC;oBACE,WAAW;oBACX,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1B;wBACE,OAAO,CAAC,CAAC,IAAI,EAAE;wBACf,qBAAqB,CAAC,CAAC,UAAU,CAAC,IAAI,IAAI;wBAC1C,mBAAmB,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI;wBACtC,eAAe,CAAC,CAAC,MAAM,EAAE;wBACzB,EAAE;wBACF,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;qBAChE,CAAC,IAAI,CAAC,IAAI,CAAC,CACb;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;gBACxB,CAAC,CAAC;oBACE,UAAU;oBACV,SAAS;oBACT,6CAAqB,CAAC,WAAW,CAC/B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CACrC;oBACD,KAAK;oBACL,EAAE;iBACH;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,EA3MgB,+BAA+B,+CAA/B,+BAA+B,QA2M/C"}