@agentica/benchmark 0.7.0-dev.20250224-2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Wrtn Technologies
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/lib/AgenticaCallBenchmark.d.ts

@@ -0,0 +1,137 @@
+import { Agentica } from "@agentica/core";
+import { tags } from "typia";
+import { IAgenticaCallBenchmarkEvent } from "./structures/IAgenticaCallBenchmarkEvent";
+import { IAgenticaCallBenchmarkResult } from "./structures/IAgenticaCallBenchmarkResult";
+import { IAgenticaCallBenchmarkScenario } from "./structures/IAgenticaCallBenchmarkScenario";
+/**
+ * LLM function calling selection benchmark.
+ *
+ * `AgenticaCallBenchmark` is a class for the benchmark of the
+ * LLM (Large Model Language) function calling part. It utilizes both
+ * `selector` and `caller` agents and tests whether the expected
+ * {@link IAgenticaOperation operations} are properly selected and
+ * called from the given
+ * {@link IAgenticaCallBenchmarkScenario scenarios}.
+ *
+ * Note that, this `AgenticaCallBenchmark` consumes a lot of time and
+ * LLM token costs because it needs the whole process of the
+ * {@link Agentica} class with a lot of repetitions. If you don't want
+ * such a heavy benchmark, consider to using
+ * {@link AgenticaSelectBenchmark} instead. In my experience,
+ * {@link Agentica} does not fail to function calling, so the function
+ * selection benchmark is much economical.
+ *
+ * @author Samchon
+ */
+export declare class AgenticaCallBenchmark {
+    private agent_;
+    private scenarios_;
+    private config_;
+    private result_;
+    /**
+     * Initializer Constructor.
+     *
+     * @param props Properties of the selection benchmark
+     */
+    constructor(props: AgenticaCallBenchmark.IProps);
+    /**
+     * Execute the benchmark.
+     *
+     * Execute the benchmark of the LLM function calling, and returns
+     * the result of the benchmark.
+     *
+     * If you wanna see progress of the benchmark, you can pass a callback
+     * function as the argument of the `listener`. The callback function
+     * would be called whenever a benchmark event is occurred.
+     *
+     * Also, you can publish a markdown format report by calling
+     * the {@link report} function after the benchmark execution.
+     *
+     * @param listener Callback function listening the benchmark events
+     * @returns Results of the function calling benchmark
+     */
+    execute(listener?: (event: IAgenticaCallBenchmarkEvent) => void): Promise<IAgenticaCallBenchmarkResult>;
+    /**
+     * Report the benchmark result as markdown files.
+     *
+     * Report the benchmark result {@link execute}d by
+     * `AgenticaCallBenchmark` as markdown files, and returns a dictionary
+     * object of the markdown reporting files. The key of the dictionary
+     * would be file name, and the value would be the markdown content.
+     *
+     * For reference, the markdown files are composed like below:
+     *
+     * - `./README.md`
+     * - `./scenario-1/README.md`
+     * - `./scenario-1/1.success.md`
+     * - `./scenario-1/2.failure.md`
+     * - `./scenario-1/3.error.md`
+     *
+     * @returns Dictionary of markdown files.
+     */
+    report(): Record<string, string>;
+    private step;
+}
+export declare namespace AgenticaCallBenchmark {
+    /**
+     * Properties of the {@link AgenticaCallBenchmark} constructor.
+     */
+    interface IProps {
+        /**
+         * AI agent instance.
+         */
+        agent: Agentica;
+        /**
+         * List of scenarios what you expect.
+         */
+        scenarios: IAgenticaCallBenchmarkScenario[];
+        /**
+         * Configuration for the benchmark.
+         */
+        config?: Partial<IConfig>;
+    }
+    /**
+     * Configuration for the benchmark.
+     *
+     * `AgenticaSelectBenchmark.IConfig` is a data structure which
+     * represents a configuration for the benchmark, especially the
+     * capacity information of the benchmark execution.
+     */
+    interface IConfig {
+        /**
+         * Repeat count.
+         *
+         * The number of repeating count for the benchmark execution
+         * for each scenario.
+         *
+         * @default 10
+         */
+        repeat: number & tags.Type<"uint32"> & tags.Minimum<1>;
+        /**
+         * Simultaneous count.
+         *
+         * The number of simultaneous count for the parallel benchmark
+         * execution.
+         *
+         * If you configure this property greater than `1`, the benchmark
+         * for each scenario would be executed in parallel in the given
+         * count.
+         *
+         * @default 10
+         */
+        simultaneous: number & tags.Type<"uint32"> & tags.Minimum<1>;
+        /**
+         * Number of consents.
+         *
+         * AI agent sometimes asks user to consent to the function
+         * calling, and perform it at the next step.
+         *
+         * This property represents the number of consents to allow.
+         * If the number of consents from the AI agent exceeds the
+         * configured value, the benchmark will be failed.
+         *
+         * @default 3
+         */
+        consent: number;
+    }
+}

package/lib/AgenticaCallBenchmark.js

@@ -0,0 +1,187 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgenticaCallBenchmark = void 0;
+const tstl_1 = require("tstl");
+const AgenticaBenchmarkPredicator_1 = require("./internal/AgenticaBenchmarkPredicator");
+const AgenticaCallBenchmarkReporter_1 = require("./internal/AgenticaCallBenchmarkReporter");
+const TokenUsageComputer_1 = require("./utils/TokenUsageComputer");
+/**
+ * LLM function calling selection benchmark.
+ *
+ * `AgenticaCallBenchmark` is a class for the benchmark of the
+ * LLM (Large Model Language) function calling part. It utilizes both
+ * `selector` and `caller` agents and tests whether the expected
+ * {@link IAgenticaOperation operations} are properly selected and
+ * called from the given
+ * {@link IAgenticaCallBenchmarkScenario scenarios}.
+ *
+ * Note that, this `AgenticaCallBenchmark` consumes a lot of time and
+ * LLM token costs because it needs the whole process of the
+ * {@link Agentica} class with a lot of repetitions. If you don't want
+ * such a heavy benchmark, consider to using
+ * {@link AgenticaSelectBenchmark} instead. In my experience,
+ * {@link Agentica} does not fail to function calling, so the function
+ * selection benchmark is much economical.
+ *
+ * @author Samchon
+ */
+class AgenticaCallBenchmark {
+    /**
+     * Initializer Constructor.
+     *
+     * @param props Properties of the selection benchmark
+     */
+    constructor(props) {
+        var _a, _b, _c, _d, _e, _f;
+        this.agent_ = props.agent;
+        this.scenarios_ = props.scenarios.slice();
+        this.config_ = {
+            repeat: (_b = (_a = props.config) === null || _a === void 0 ? void 0 : _a.repeat) !== null && _b !== void 0 ? _b : 10,
+            simultaneous: (_d = (_c = props.config) === null || _c === void 0 ? void 0 : _c.simultaneous) !== null && _d !== void 0 ? _d : 10,
+            consent: (_f = (_e = props.config) === null || _e === void 0 ? void 0 : _e.consent) !== null && _f !== void 0 ? _f : 3,
+        };
+        this.result_ = null;
+    }
+    /**
+     * Execute the benchmark.
+     *
+     * Execute the benchmark of the LLM function calling, and returns
+     * the result of the benchmark.
+     *
+     * If you wanna see progress of the benchmark, you can pass a callback
+     * function as the argument of the `listener`. The callback function
+     * would be called whenever a benchmark event is occurred.
+     *
+     * Also, you can publish a markdown format report by calling
+     * the {@link report} function after the benchmark execution.
+     *
+     * @param listener Callback function listening the benchmark events
+     * @returns Results of the function calling benchmark
+     */
+    execute(listener) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const started_at = new Date();
+            const semaphore = new tstl_1.Semaphore(this.config_.simultaneous);
+            const experiments = yield Promise.all(this.scenarios_.map((scenario) => __awaiter(this, void 0, void 0, function* () {
+                const events = yield Promise.all(new Array(this.config_.repeat).fill(0).map(() => __awaiter(this, void 0, void 0, function* () {
+                    yield semaphore.acquire();
+                    const e = yield this.step(scenario);
+                    yield semaphore.release();
+                    if (listener !== undefined)
+                        listener(e);
+                    return e;
+                })));
+                return {
+                    scenario,
+                    events,
+                    usage: events
+                        .filter((e) => e.type !== "error")
+                        .map((e) => e.usage)
+                        .reduce(TokenUsageComputer_1.TokenUsageComputer.plus, TokenUsageComputer_1.TokenUsageComputer.zero()),
+                };
+            })));
+            return (this.result_ = {
+                experiments,
+                started_at,
+                completed_at: new Date(),
+                usage: experiments
+                    .map((p) => p.usage)
+                    .reduce(TokenUsageComputer_1.TokenUsageComputer.plus, TokenUsageComputer_1.TokenUsageComputer.zero()),
+            });
+        });
+    }
+    /**
+     * Report the benchmark result as markdown files.
+     *
+     * Report the benchmark result {@link execute}d by
+     * `AgenticaCallBenchmark` as markdown files, and returns a dictionary
+     * object of the markdown reporting files. The key of the dictionary
+     * would be file name, and the value would be the markdown content.
+     *
+     * For reference, the markdown files are composed like below:
+     *
+     * - `./README.md`
+     * - `./scenario-1/README.md`
+     * - `./scenario-1/1.success.md`
+     * - `./scenario-1/2.failure.md`
+     * - `./scenario-1/3.error.md`
+     *
+     * @returns Dictionary of markdown files.
+     */
+    report() {
+        if (this.result_ === null)
+            throw new Error("Benchmark is not executed yet.");
+        return AgenticaCallBenchmarkReporter_1.AgenticaCallBenchmarkReporter.markdown(this.result_);
+    }
+    step(scenario) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const agent = this.agent_.clone();
+            const started_at = new Date();
+            const success = () => AgenticaBenchmarkPredicator_1.AgenticaBenchmarkPredicator.success({
+                expected: scenario.expected,
+                operations: agent
+                    .getPromptHistories()
+                    .filter((p) => p.type === "execute"),
+                strict: false,
+            });
+            const out = () => {
+                const select = AgenticaBenchmarkPredicator_1.AgenticaBenchmarkPredicator.success({
+                    expected: scenario.expected,
+                    operations: agent
+                        .getPromptHistories()
+                        .filter((p) => p.type === "select")
+                        .map((p) => p.operations)
+                        .flat(),
+                    strict: false,
+                });
+                const call = success();
+                return {
+                    type: (call ? "success" : "failure"),
+                    scenario,
+                    select,
+                    call,
+                    prompts: agent.getPromptHistories(),
+                    usage: agent.getTokenUsage(),
+                    started_at,
+                    completed_at: new Date(),
+                };
+            };
+            try {
+                yield agent.conversate(scenario.text);
+                if (success())
+                    return out();
+                for (let i = 0; i < this.config_.consent; ++i) {
+                    const next = yield AgenticaBenchmarkPredicator_1.AgenticaBenchmarkPredicator.isNext(agent);
+                    if (next === null)
+                        break;
+                    yield agent.conversate(next);
+                    if (success())
+                        return out();
+                }
+                return out();
+            }
+            catch (error) {
+                return {
+                    type: "error",
+                    scenario,
+                    prompts: agent.getPromptHistories(),
+                    usage: agent.getTokenUsage(),
+                    error,
+                    started_at,
+                    completed_at: new Date(),
+                };
+            }
+        });
+    }
+}
+exports.AgenticaCallBenchmark = AgenticaCallBenchmark;
+//# sourceMappingURL=AgenticaCallBenchmark.js.map

package/lib/AgenticaCallBenchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgenticaCallBenchmark.js","sourceRoot":"","sources":["../src/AgenticaCallBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,+BAAiC;AAGjC,wFAAqF;AACrF,4FAAyF;AAIzF,mEAAgE;AAEhE;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAa,qBAAqB;IAMhC;;;;OAIG;IACH,YAAmB,KAAmC;;QACpD,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG;YACb,MAAM,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,MAAM,mCAAI,EAAE;YAClC,YAAY,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,YAAY,mCAAI,EAAE;YAC9C,OAAO,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,OAAO,mCAAI,CAAC;SACpC,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACU,OAAO,CAClB,QAAuD;;YAEvD,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,MAAM,SAAS,GAAc,IAAI,gBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACtE,MAAM,WAAW,GACf,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAO,QAAQ,EAAE,EAAE;gBACrC,MAAM,MAAM,GAAkC,MAAM,OAAO,CAAC,GAAG,CAC7D,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAS,EAAE;oBACpD,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC1B,MAAM,CAAC,GAAgC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACjE,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC1B,IAAI,QAAQ,KAAK,SAAS;wBAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;oBACxC,OAAO,CAAC,CAAC;gBACX,CAAC,CAAA,CAAC,CACH,CAAC;gBACF,OAAO;oBACL,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,MAAM;yBACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;yBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;yBACnB,MAAM,CAAC,uCAAkB,CAAC,IAAI,EAAE,uCAAkB,CAAC,IAAI,EAAE,CAAC;iBAC9D,CAAC;YACJ,CAAC,CAAA,CAAC,CACH,CAAC;YACJ,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG;gBACrB,WAAW;gBACX,UAAU;gBACV,YAAY,EAAE,IAAI,IAAI,EAAE;gBACxB,KAAK,EAAE,WAAW;qBACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;qBACnB,MAAM,CAAC,uCAAkB,CAAC,IAAI,EAAE,uCAAkB,CAAC,IAAI,EAAE,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;KAAA;IAED;;;;;;;;;;;;;;;;;OAiBG;IACI,MAAM;QACX,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,OAAO,6DAA6B,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9D,CAAC;IAEa,IAAI,CAChB,QAAwC;;YAExC,MAAM,KAAK,GAAa,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC5C,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB,yDAA2B,CAAC,OAAO,CAAC;gBAClC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,UAAU,EAAE,KAAK;qBACd,kBAAkB,EAAE;qBACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;gBACtC,MAAM,EAAE,KAAK;aACd,CAAC,CAAC;YACL,MAAM,GAAG,GAAG,GAAgC,EAAE;gBAC5C,MAAM,MAAM,GAAG,yDAA2B,CAAC,OAAO,CAAC;oBACjD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,UAAU,EAAE,KAAK;yBACd,kBAAkB,EAAE;yBACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;yBAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;yBACxB,IAAI,EAAE;oBACT,MAAM,EAAE,KAAK;iBACd,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;gBACvB,OAAO;oBACL,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAc;oBACjD,QAAQ;oBACR,MAAM;oBACN,IAAI;oBACJ,OAAO,EAAE,KAAK,CAAC,kBAAkB,EAAE;oBACnC,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE;oBAC5B,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBACsB,CAAC;YACnD,CAAC,CAAC;YAEF,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACtC,IAAI,OAAO,EAAE;oBAAE,OAAO,GAAG,EAAE,CAAC;gBAC5B,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;oBACtD,MAAM,IAAI,GACR,MAAM,yDAA2B,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAClD,IAAI,IAAI,KAAK,IAAI;wBAAE,MAAM;oBAEzB,MAAM,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;oBAC7B,IAAI,OAAO,EAAE;wBAAE,OAAO,GAAG,EAAE,CAAC;gBAC9B,CAAC;gBACD,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,QAAQ;oBACR,OAAO,EAAE,KAAK,CAAC,kBAAkB,EAAE;oBACnC,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE;oBAC5B,KAAK;oBACL,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBACzB,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;CACF;AA/JD,sDA+JC"}

package/lib/AgenticaSelectBenchmark.d.ts

@@ -0,0 +1,123 @@
+import { Agentica } from "@agentica/core";
+import { tags } from "typia";
+import { IAgenticaSelectBenchmarkEvent } from "./structures/IAgenticaSelectBenchmarkEvent";
+import { IAgenticaSelectBenchmarkResult } from "./structures/IAgenticaSelectBenchmarkResult";
+import { IAgenticaSelectBenchmarkScenario } from "./structures/IAgenticaSelectBenchmarkScenario";
+/**
+ * LLM function calling selection benchmark.
+ *
+ * `AgenticaSelectBenchmark` is a class for the benchmark of the
+ * LLM (Large Model Language) function calling's selection part.
+ * It utilizes the `selector` agent and tests whether the expected
+ * {@link IAgenticaOperation operations} are properly selected from
+ * the given {@link IAgenticaSelectBenchmarkScenario scenarios}.
+ *
+ * Note that, this `AgenticaSelectBenchmark` class measures only the
+ * selection benchmark, testing whether the `selector` agent can select
+ * candidate functions to call as expected. Therefore, it does not test
+ * about the actual function calling which is done by the `executor` agent.
+ * If you want that feature, use {@link AgenticaCallBenchmark} class instead.
+ *
+ * @author Samchon
+ */
+export declare class AgenticaSelectBenchmark {
+    private agent_;
+    private scenarios_;
+    private config_;
+    private histories_;
+    private result_;
+    /**
+     * Initializer Constructor.
+     *
+     * @param props Properties of the selection benchmark
+     */
+    constructor(props: AgenticaSelectBenchmark.IProps);
+    /**
+     * Execute the benchmark.
+     *
+     * Execute the benchmark of the LLM function selection, and returns
+     * the result of the benchmark.
+     *
+     * If you wanna see progress of the benchmark, you can pass a callback
+     * function as the argument of the `listener`. The callback function
+     * would be called whenever a benchmark event is occurred.
+     *
+     * Also, you can publish a markdown format report by calling
+     * the {@link report} function after the benchmark execution.
+     *
+     * @param listener Callback function listening the benchmark events
+     * @returns Results of the function selection benchmark
+     */
+    execute(listener?: (event: IAgenticaSelectBenchmarkEvent) => void): Promise<IAgenticaSelectBenchmarkResult>;
+    /**
+     * Report the benchmark result as markdown files.
+     *
+     * Report the benchmark result {@link execute}d by
+     * `AgenticaSelectBenchmark` as markdown files, and returns a
+     * dictionary object of the markdown reporting files. The key of
+     * the dictionary would be file name, and the value would be the
+     * markdown content.
+     *
+     * For reference, the markdown files are composed like below:
+     *
+     * - `./README.md`
+     * - `./scenario-1/README.md`
+     * - `./scenario-1/1.success.md`
+     * - `./scenario-1/2.failure.md`
+     * - `./scenario-1/3.error.md`
+     *
+     * @returns Dictionary of markdown files.
+     */
+    report(): Record<string, string>;
+    private step;
+}
+export declare namespace AgenticaSelectBenchmark {
+    /**
+     * Properties of the {@link AgenticaSelectBenchmark} constructor.
+     */
+    interface IProps {
+        /**
+         * AI agent instance.
+         */
+        agent: Agentica;
+        /**
+         * List of scenarios what you expect.
+         */
+        scenarios: IAgenticaSelectBenchmarkScenario[];
+        /**
+         * Configuration for the benchmark.
+         */
+        config?: Partial<IConfig>;
+    }
+    /**
+     * Configuration for the benchmark.
+     *
+     * `AgenticaSelectBenchmark.IConfig` is a data structure which
+     * represents a configuration for the benchmark, especially the
+     * capacity information of the benchmark execution.
+     */
+    interface IConfig {
+        /**
+         * Repeat count.
+         *
+         * The number of repeating count for the benchmark execution
+         * for each scenario.
+         *
+         * @default 10
+         */
+        repeat: number & tags.Type<"uint32"> & tags.Minimum<1>;
+        /**
+         * Simultaneous count.
+         *
+         * The number of simultaneous count for the parallel benchmark
+         * execution.
+         *
+         * If you configure this property greater than `1`, the benchmark
+         * for each scenario would be executed in parallel in the given
+         * count.
+         *
+         * @default 10
+         */
+        simultaneous: number & tags.Type<"uint32"> & tags.Minimum<1>;
+    }
+}

package/lib/AgenticaSelectBenchmark.js

@@ -0,0 +1,185 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgenticaSelectBenchmark = void 0;
+const ChatGptSelectFunctionAgent_1 = require("@agentica/core/src/chatgpt/ChatGptSelectFunctionAgent");
+const tstl_1 = require("tstl");
+const AgenticaBenchmarkPredicator_1 = require("./internal/AgenticaBenchmarkPredicator");
+const AgenticaSelectBenchmarkReporter_1 = require("./internal/AgenticaSelectBenchmarkReporter");
+const TokenUsageComputer_1 = require("./utils/TokenUsageComputer");
+/**
+ * LLM function calling selection benchmark.
+ *
+ * `AgenticaSelectBenchmark` is a class for the benchmark of the
+ * LLM (Large Model Language) function calling's selection part.
+ * It utilizes the `selector` agent and tests whether the expected
+ * {@link IAgenticaOperation operations} are properly selected from
+ * the given {@link IAgenticaSelectBenchmarkScenario scenarios}.
+ *
+ * Note that, this `AgenticaSelectBenchmark` class measures only the
+ * selection benchmark, testing whether the `selector` agent can select
+ * candidate functions to call as expected. Therefore, it does not test
+ * about the actual function calling which is done by the `executor` agent.
+ * If you want that feature, use {@link AgenticaCallBenchmark} class instead.
+ *
+ * @author Samchon
+ */
+class AgenticaSelectBenchmark {
+    /**
+     * Initializer Constructor.
+     *
+     * @param props Properties of the selection benchmark
+     */
+    constructor(props) {
+        var _a, _b, _c, _d;
+        this.agent_ = props.agent;
+        this.scenarios_ = props.scenarios.slice();
+        this.config_ = {
+            repeat: (_b = (_a = props.config) === null || _a === void 0 ? void 0 : _a.repeat) !== null && _b !== void 0 ? _b : 10,
+            simultaneous: (_d = (_c = props.config) === null || _c === void 0 ? void 0 : _c.simultaneous) !== null && _d !== void 0 ? _d : 10,
+        };
+        this.histories_ = props.agent.getPromptHistories().slice();
+        this.result_ = null;
+    }
+    /**
+     * Execute the benchmark.
+     *
+     * Execute the benchmark of the LLM function selection, and returns
+     * the result of the benchmark.
+     *
+     * If you wanna see progress of the benchmark, you can pass a callback
+     * function as the argument of the `listener`. The callback function
+     * would be called whenever a benchmark event is occurred.
+     *
+     * Also, you can publish a markdown format report by calling
+     * the {@link report} function after the benchmark execution.
+     *
+     * @param listener Callback function listening the benchmark events
+     * @returns Results of the function selection benchmark
+     */
+    execute(listener) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const started_at = new Date();
+            const semaphore = new tstl_1.Semaphore(this.config_.simultaneous);
+            const experiments = yield Promise.all(this.scenarios_.map((scenario) => __awaiter(this, void 0, void 0, function* () {
+                const events = yield Promise.all(new Array(this.config_.repeat).fill(0).map(() => __awaiter(this, void 0, void 0, function* () {
+                    yield semaphore.acquire();
+                    const e = yield this.step(scenario);
+                    yield semaphore.release();
+                    if (listener !== undefined)
+                        listener(e);
+                    return e;
+                })));
+                return {
+                    scenario,
+                    events,
+                    usage: events
+                        .filter((e) => e.type !== "error")
+                        .map((e) => e.usage)
+                        .reduce(TokenUsageComputer_1.TokenUsageComputer.plus, TokenUsageComputer_1.TokenUsageComputer.zero()),
+                };
+            })));
+            return (this.result_ = {
+                experiments,
+                started_at,
+                completed_at: new Date(),
+                usage: experiments
+                    .map((p) => p.usage)
+                    .reduce(TokenUsageComputer_1.TokenUsageComputer.plus, TokenUsageComputer_1.TokenUsageComputer.zero()),
+            });
+        });
+    }
+    /**
+     * Report the benchmark result as markdown files.
+     *
+     * Report the benchmark result {@link execute}d by
+     * `AgenticaSelectBenchmark` as markdown files, and returns a
+     * dictionary object of the markdown reporting files. The key of
+     * the dictionary would be file name, and the value would be the
+     * markdown content.
+     *
+     * For reference, the markdown files are composed like below:
+     *
+     * - `./README.md`
+     * - `./scenario-1/README.md`
+     * - `./scenario-1/1.success.md`
+     * - `./scenario-1/2.failure.md`
+     * - `./scenario-1/3.error.md`
+     *
+     * @returns Dictionary of markdown files.
+     */
+    report() {
+        if (this.result_ === null)
+            throw new Error("Benchmark is not executed yet.");
+        return AgenticaSelectBenchmarkReporter_1.AgenticaSelectBenchmarkReporter.markdown(this.result_);
+    }
+    step(scenario) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const started_at = new Date();
+            try {
+                const usage = {
+                    total: 0,
+                    prompt: {
+                        total: 0,
+                        audio: 0,
+                        cached: 0,
+                    },
+                    completion: {
+                        total: 0,
+                        accepted_prediction: 0,
+                        audio: 0,
+                        reasoning: 0,
+                        rejected_prediction: 0,
+                    },
+                };
+                const prompts = yield ChatGptSelectFunctionAgent_1.ChatGptSelectFunctionAgent.execute(Object.assign(Object.assign({}, this.agent_.getContext({
+                    prompt: {
+                        type: "text",
+                        role: "user",
+                        text: scenario.text,
+                    },
+                    usage,
+                })), { histories: this.histories_.slice(), stack: [], ready: () => true, dispatch: () => __awaiter(this, void 0, void 0, function* () { }) }));
+                const selected = prompts
+                    .filter((p) => p.type === "select")
+                    .map((p) => p.operations)
+                    .flat();
+                return {
+                    type: AgenticaBenchmarkPredicator_1.AgenticaBenchmarkPredicator.success({
+                        expected: scenario.expected,
+                        operations: selected,
+                    })
+                        ? "success"
+                        : "failure",
+                    scenario,
+                    selected,
+                    usage,
+                    assistantPrompts: prompts
+                        .filter((p) => p.type === "text")
+                        .filter((p) => p.role === "assistant"),
+                    started_at,
+                    completed_at: new Date(),
+                };
+            }
+            catch (error) {
+                return {
+                    type: "error",
+                    scenario,
+                    error,
+                    started_at,
+                    completed_at: new Date(),
+                };
+            }
+        });
+    }
+}
+exports.AgenticaSelectBenchmark = AgenticaSelectBenchmark;
+//# sourceMappingURL=AgenticaSelectBenchmark.js.map

package/lib/AgenticaSelectBenchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgenticaSelectBenchmark.js","sourceRoot":"","sources":["../src/AgenticaSelectBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;AAOA,sGAAmG;AACnG,+BAAiC;AAGjC,wFAAqF;AACrF,gGAA6F;AAI7F,mEAAgE;AAEhE;;;;;;;;;;;;;;;;GAgBG;AACH,MAAa,uBAAuB;IAOlC;;;;OAIG;IACH,YAAmB,KAAqC;;QACtD,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG;YACb,MAAM,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,MAAM,mCAAI,EAAE;YAClC,YAAY,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,YAAY,mCAAI,EAAE;SAC/C,CAAC;QACF,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACU,OAAO,CAClB,QAAyD;;YAEzD,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,MAAM,SAAS,GAAc,IAAI,gBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACtE,MAAM,WAAW,GACf,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAO,QAAQ,EAAE,EAAE;gBACrC,MAAM,MAAM,GAAoC,MAAM,OAAO,CAAC,GAAG,CAC/D,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAS,EAAE;oBACpD,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC1B,MAAM,CAAC,GACL,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC5B,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC1B,IAAI,QAAQ,KAAK,SAAS;wBAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;oBACxC,OAAO,CAAC,CAAC;gBACX,CAAC,CAAA,CAAC,CACH,CAAC;gBACF,OAAO;oBACL,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,MAAM;yBACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;yBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;yBACnB,MAAM,CAAC,uCAAkB,CAAC,IAAI,EAAE,uCAAkB,CAAC,IAAI,EAAE,CAAC;iBAC9D,CAAC;YACJ,CAAC,CAAA,CAAC,CACH,CAAC;YACJ,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG;gBACrB,WAAW;gBACX,UAAU;gBACV,YAAY,EAAE,IAAI,IAAI,EAAE;gBACxB,KAAK,EAAE,WAAW;qBACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;qBACnB,MAAM,CAAC,uCAAkB,CAAC,IAAI,EAAE,uCAAkB,CAAC,IAAI,EAAE,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;KAAA;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACI,MAAM;QACX,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,OAAO,iEAA+B,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChE,CAAC;IAEa,IAAI,CAChB,QAA0C;;YAE1C,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,MAAM,KAAK,GAAwB;oBACjC,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE;wBACN,KAAK,EAAE,CAAC;wBACR,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;qBACV;oBACD,UAAU,EAAE;wBACV,KAAK,EAAE,CAAC;wBACR,mBAAmB,EAAE,CAAC;wBACtB,KAAK,EAAE,CAAC;wBACR,SAAS,EAAE,CAAC;wBACZ,mBAAmB,EAAE,CAAC;qBACvB;iBACF,CAAC;gBACF,MAAM,OAAO,GACX,MAAM,uDAA0B,CAAC,OAAO,CAAC,gCACpC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;oBACxB,MAAM,EAAE;wBACN,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,QAAQ,CAAC,IAAI;qBACpB;oBACD,KAAK;iBACN,CAAC,KACF,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAClC,KAAK,EAAE,EAAE,EACT,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,EACjB,QAAQ,EAAE,GAAS,EAAE,gDAAE,CAAC,CAAA,GACE,CAAC,CAAC;gBAChC,MAAM,QAAQ,GAAkC,OAAO;qBACpD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;qBAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;qBACxB,IAAI,EAAE,CAAC;gBACV,OAAO;oBACL,IAAI,EAAE,yDAA2B,CAAC,OAAO,CAAC;wBACxC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,UAAU,EAAE,QAAQ;qBACrB,CAAC;wBACA,CAAC,CAAC,SAAS;wBACX,CAAC,CAAC,SAAS;oBACb,QAAQ;oBACR,QAAQ;oBACR,KAAK;oBACL,gBAAgB,EAAE,OAAO;yBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;yBAChC,MAAM,CACL,CAAC,CAAC,EAA2C,EAAE,CAC7C,CAAC,CAAC,IAAI,KAAK,WAAW,CACzB;oBACH,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBAGgB,CAAC;YAC7C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,QAAQ;oBACR,KAAK;oBACL,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBACsB,CAAC;YACnD,CAAC;QACH,CAAC;KAAA;CACF;AA5KD,0DA4KC"}

package/lib/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./AgenticaCallBenchmark";
+export * from "./AgenticaSelectBenchmark";