@agentica/benchmark 0.22.0 → 0.23.0

package/README.md

@@ -26,7 +26,7 @@ Are you a TypeScript developer? Then you're already an AI developer. Familiar wi
 <!-- eslint-skip -->
 
 ```typescript
-import { Agentica, assertHttpLlmApplication } from "@agentica/core";
+import { Agentica, assertHttpController } from "@agentica/core";
 import OpenAI from "openai";
 import typia from "typia";
 
@@ -40,24 +40,23 @@ const agent = new Agentica({
   controllers: [
     // functions from TypeScript class
     {
-      protocol: "http",
+      protocol: "class",
+      name: "filesystem",
       application: typia.llm.application<MobileFileSystem, "chatgpt">(),
       execute: new MobileFileSystem(),
     },
     // functions from Swagger/OpenAPI
-    {
-      protocol: "http",
-      application: assertHttpLlmApplication({
-        model: "chatgpt",
-        document: await fetch(
-          "https://shopping-be.wrtn.ai/editor/swagger.json",
-        ).then(r => r.json()),
-      }),
+    assertHttpController({
+      name: "shopping",
+      model: "chatgpt",
+      document: await fetch(
+        "https://shopping-be.wrtn.ai/editor/swagger.json",
+      ).then(r => r.json()),
       connection: {
         host: "https://shopping-be.wrtn.ai",
         headers: { Authorization: "Bearer ********" },
       },
-    },
+    }),
   ],
 });
 await agent.conversate("I wanna buy MacBook Pro");

package/lib/MicroAgenticaCallBenchmark.d.ts

@@ -0,0 +1,138 @@
+import type { MicroAgentica } from "@agentica/core";
+import type { ILlmSchema } from "@samchon/openapi";
+import type { tags } from "typia";
+import type { IAgenticaCallBenchmarkEvent } from "./structures/IAgenticaCallBenchmarkEvent";
+import type { IAgenticaCallBenchmarkResult } from "./structures/IAgenticaCallBenchmarkResult";
+import type { IAgenticaCallBenchmarkScenario } from "./structures/IAgenticaCallBenchmarkScenario";
+/**
+ * LLM function calling selection benchmark.
+ *
+ * `AgenticaCallBenchmark` is a class for the benchmark of the
+ * LLM (Large Model Language) function calling part. It utilizes both
+ * `selector` and `caller` agents and tests whether the expected
+ * {@link IAgenticaOperation operations} are properly selected and
+ * called from the given
+ * {@link IAgenticaCallBenchmarkScenario scenarios}.
+ *
+ * Note that, this `MicroAgenticaCallBenchmark` consumes a lot of time and
+ * LLM token costs because it needs the whole process of the
+ * {@link MicroAgentica} class with a lot of repetitions. If you don't want
+ * such a heavy benchmark, consider to using
+ * {@link AgenticaSelectBenchmark} instead. In my experience,
+ * {@link MicroAgentica} does not fail to function calling, so the function
+ * selection benchmark is much economical.
+ *
+ * @author Samchon
+ */
+export declare class MicroAgenticaCallBenchmark<Model extends ILlmSchema.Model> {
+    private agent_;
+    private scenarios_;
+    private config_;
+    private result_;
+    /**
+     * Initializer Constructor.
+     *
+     * @param props Properties of the selection benchmark
+     */
+    constructor(props: MicroAgenticaCallBenchmark.IProps<Model>);
+    /**
+     * Execute the benchmark.
+     *
+     * Execute the benchmark of the LLM function calling, and returns
+     * the result of the benchmark.
+     *
+     * If you wanna see progress of the benchmark, you can pass a callback
+     * function as the argument of the `listener`. The callback function
+     * would be called whenever a benchmark event is occurred.
+     *
+     * Also, you can publish a markdown format report by calling
+     * the {@link report} function after the benchmark execution.
+     *
+     * @param listener Callback function listening the benchmark events
+     * @returns Results of the function calling benchmark
+     */
+    execute(listener?: (event: IAgenticaCallBenchmarkEvent<Model>) => void): Promise<IAgenticaCallBenchmarkResult<Model>>;
+    /**
+     * Report the benchmark result as markdown files.
+     *
+     * Report the benchmark result {@link execute}d by
+     * `AgenticaCallBenchmark` as markdown files, and returns a dictionary
+     * object of the markdown reporting files. The key of the dictionary
+     * would be file name, and the value would be the markdown content.
+     *
+     * For reference, the markdown files are composed like below:
+     *
+     * - `./README.md`
+     * - `./scenario-1/README.md`
+     * - `./scenario-1/1.success.md`
+     * - `./scenario-1/2.failure.md`
+     * - `./scenario-1/3.error.md`
+     *
+     * @returns Dictionary of markdown files.
+     */
+    report(): Record<string, string>;
+    private step;
+}
+export declare namespace MicroAgenticaCallBenchmark {
+    /**
+     * Properties of the {@link MicroAgenticaCallBenchmark} constructor.
+     */
+    interface IProps<Model extends ILlmSchema.Model> {
+        /**
+         * AI agent instance.
+         */
+        agent: MicroAgentica<Model>;
+        /**
+         * List of scenarios what you expect.
+         */
+        scenarios: IAgenticaCallBenchmarkScenario<Model>[];
+        /**
+         * Configuration for the benchmark.
+         */
+        config?: Partial<IConfig>;
+    }
+    /**
+     * Configuration for the benchmark.
+     *
+     * `AgenticaSelectBenchmark.IConfig` is a data structure which
+     * represents a configuration for the benchmark, especially the
+     * capacity information of the benchmark execution.
+     */
+    interface IConfig {
+        /**
+         * Repeat count.
+         *
+         * The number of repeating count for the benchmark execution
+         * for each scenario.
+         *
+         * @default 10
+         */
+        repeat: number & tags.Type<"uint32"> & tags.Minimum<1>;
+        /**
+         * Simultaneous count.
+         *
+         * The number of simultaneous count for the parallel benchmark
+         * execution.
+         *
+         * If you configure this property greater than `1`, the benchmark
+         * for each scenario would be executed in parallel in the given
+         * count.
+         *
+         * @default 10
+         */
+        simultaneous: number & tags.Type<"uint32"> & tags.Minimum<1>;
+        /**
+         * Number of consents.
+         *
+         * AI agent sometimes asks user to consent to the function
+         * calling, and perform it at the next step.
+         *
+         * This property represents the number of consents to allow.
+         * If the number of consents from the AI agent exceeds the
+         * configured value, the benchmark will be failed.
+         *
+         * @default 3
+         */
+        consent: number;
+    }
+}

package/lib/MicroAgenticaCallBenchmark.js

@@ -0,0 +1,199 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MicroAgenticaCallBenchmark = void 0;
+/**
+ * @module
+ * This file contains the implementation of the AgenticaCallBenchmark class.
+ *
+ * @author Wrtn Technologies
+ */
+const core_1 = require("@agentica/core");
+const tstl_1 = require("tstl");
+const AgenticaBenchmarkPredicator_1 = require("./internal/AgenticaBenchmarkPredicator");
+const AgenticaCallBenchmarkReporter_1 = require("./internal/AgenticaCallBenchmarkReporter");
+/**
+ * LLM function calling selection benchmark.
+ *
+ * `AgenticaCallBenchmark` is a class for the benchmark of the
+ * LLM (Large Model Language) function calling part. It utilizes both
+ * `selector` and `caller` agents and tests whether the expected
+ * {@link IAgenticaOperation operations} are properly selected and
+ * called from the given
+ * {@link IAgenticaCallBenchmarkScenario scenarios}.
+ *
+ * Note that, this `MicroAgenticaCallBenchmark` consumes a lot of time and
+ * LLM token costs because it needs the whole process of the
+ * {@link MicroAgentica} class with a lot of repetitions. If you don't want
+ * such a heavy benchmark, consider to using
+ * {@link AgenticaSelectBenchmark} instead. In my experience,
+ * {@link MicroAgentica} does not fail to function calling, so the function
+ * selection benchmark is much economical.
+ *
+ * @author Samchon
+ */
+class MicroAgenticaCallBenchmark {
+    /**
+     * Initializer Constructor.
+     *
+     * @param props Properties of the selection benchmark
+     */
+    constructor(props) {
+        var _a, _b, _c, _d, _e, _f;
+        this.agent_ = props.agent;
+        this.scenarios_ = props.scenarios.slice();
+        this.config_ = {
+            repeat: (_b = (_a = props.config) === null || _a === void 0 ? void 0 : _a.repeat) !== null && _b !== void 0 ? _b : 10,
+            simultaneous: (_d = (_c = props.config) === null || _c === void 0 ? void 0 : _c.simultaneous) !== null && _d !== void 0 ? _d : 10,
+            consent: (_f = (_e = props.config) === null || _e === void 0 ? void 0 : _e.consent) !== null && _f !== void 0 ? _f : 3,
+        };
+        this.result_ = null;
+    }
+    /**
+     * Execute the benchmark.
+     *
+     * Execute the benchmark of the LLM function calling, and returns
+     * the result of the benchmark.
+     *
+     * If you wanna see progress of the benchmark, you can pass a callback
+     * function as the argument of the `listener`. The callback function
+     * would be called whenever a benchmark event is occurred.
+     *
+     * Also, you can publish a markdown format report by calling
+     * the {@link report} function after the benchmark execution.
+     *
+     * @param listener Callback function listening the benchmark events
+     * @returns Results of the function calling benchmark
+     */
+    execute(listener) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const started_at = new Date();
+            const semaphore = new tstl_1.Semaphore(this.config_.simultaneous);
+            const task = this.scenarios_.map((scenario) => __awaiter(this, void 0, void 0, function* () {
+                const events = yield Promise.all(Array.from({ length: this.config_.repeat }).map(() => __awaiter(this, void 0, void 0, function* () {
+                    yield semaphore.acquire();
+                    const e = yield this.step(scenario);
+                    yield semaphore.release();
+                    if (listener !== undefined) {
+                        listener(e);
+                    }
+                    return e;
+                })));
+                return {
+                    scenario,
+                    events,
+                    usage: events
+                        .filter(e => e.type !== "error")
+                        .map(e => e.usage)
+                        .reduce((acc, cur) => core_1.AgenticaTokenUsage.plus(acc, cur), core_1.AgenticaTokenUsage.zero()),
+                };
+            }));
+            const experiments = yield Promise.all(task);
+            return (this.result_ = {
+                experiments,
+                started_at,
+                completed_at: new Date(),
+                usage: experiments
+                    .map(p => p.usage)
+                    .reduce((acc, cur) => core_1.AgenticaTokenUsage.plus(acc, cur), core_1.AgenticaTokenUsage.zero()),
+            });
+        });
+    }
+    /**
+     * Report the benchmark result as markdown files.
+     *
+     * Report the benchmark result {@link execute}d by
+     * `AgenticaCallBenchmark` as markdown files, and returns a dictionary
+     * object of the markdown reporting files. The key of the dictionary
+     * would be file name, and the value would be the markdown content.
+     *
+     * For reference, the markdown files are composed like below:
+     *
+     * - `./README.md`
+     * - `./scenario-1/README.md`
+     * - `./scenario-1/1.success.md`
+     * - `./scenario-1/2.failure.md`
+     * - `./scenario-1/3.error.md`
+     *
+     * @returns Dictionary of markdown files.
+     */
+    report() {
+        if (this.result_ === null) {
+            throw new Error("Benchmark is not executed yet.");
+        }
+        return AgenticaCallBenchmarkReporter_1.AgenticaCallBenchmarkReporter.markdown(this.result_);
+    }
+    step(scenario) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const agent = this.agent_.clone();
+            const started_at = new Date();
+            const success = () => AgenticaBenchmarkPredicator_1.AgenticaBenchmarkPredicator.success({
+                expected: scenario.expected,
+                operations: agent
+                    .getHistories()
+                    .filter(p => p.type === "execute")
+                    .map(p => p.operation),
+                strict: false,
+            });
+            const out = () => {
+                const select = AgenticaBenchmarkPredicator_1.AgenticaBenchmarkPredicator.success({
+                    expected: scenario.expected,
+                    operations: agent
+                        .getHistories()
+                        .filter(p => p.type === "execute")
+                        .map(p => p.operation),
+                    strict: false,
+                });
+                const call = success();
+                return {
+                    type: (call ? "success" : "failure"),
+                    scenario,
+                    select,
+                    call,
+                    prompts: agent.getHistories(),
+                    usage: agent.getTokenUsage(),
+                    started_at,
+                    completed_at: new Date(),
+                };
+            };
+            try {
+                yield agent.conversate(scenario.text);
+                if (success()) {
+                    return out();
+                }
+                for (let i = 0; i < this.config_.consent; ++i) {
+                    const next = yield AgenticaBenchmarkPredicator_1.AgenticaBenchmarkPredicator.isNext(agent);
+                    if (next === null) {
+                        break;
+                    }
+                    yield agent.conversate(next);
+                    if (success()) {
+                        return out();
+                    }
+                }
+                return out();
+            }
+            catch (error) {
+                return {
+                    type: "error",
+                    scenario,
+                    prompts: agent.getHistories(),
+                    usage: agent.getTokenUsage(),
+                    error,
+                    started_at,
+                    completed_at: new Date(),
+                };
+            }
+        });
+    }
+}
+exports.MicroAgenticaCallBenchmark = MicroAgenticaCallBenchmark;
+//# sourceMappingURL=MicroAgenticaCallBenchmark.js.map

package/lib/MicroAgenticaCallBenchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MicroAgenticaCallBenchmark.js","sourceRoot":"","sources":["../src/MicroAgenticaCallBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;AAIA;;;;;GAKG;AACH,yCAAoD;AACpD,+BAAiC;AAMjC,wFAAqF;AACrF,4FAAyF;AAEzF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAa,0BAA0B;IAMrC;;;;OAIG;IACH,YAAmB,KAA+C;;QAChE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG;YACb,MAAM,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,MAAM,mCAAI,EAAE;YAClC,YAAY,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,YAAY,mCAAI,EAAE;YAC9C,OAAO,EAAE,MAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,OAAO,mCAAI,CAAC;SACpC,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACU,OAAO,CAClB,QAA8D;;YAE9D,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,MAAM,SAAS,GAAc,IAAI,gBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACtE,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAO,QAAQ,EAAE,EAAE;gBAClD,MAAM,MAAM,GACR,MAAM,OAAO,CAAC,GAAG,CACjB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,GAAS,EAAE;oBACzD,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC1B,MAAM,CAAC,GACH,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC9B,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBAE1B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;wBAC3B,QAAQ,CAAC,CAAC,CAAC,CAAC;oBACd,CAAC;oBAED,OAAO,CAAC,CAAC;gBACX,CAAC,CAAA,CAAC,CACH,CAAC;gBACJ,OAAO;oBACL,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,MAAM;yBACV,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;yBAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;yBACjB,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,yBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,yBAAkB,CAAC,IAAI,EAAE,CAAC;iBACtF,CAAC;YACJ,CAAC,CAAA,CAAC,CAAC;YACH,MAAM,WAAW,GACb,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG;gBACrB,WAAW;gBACX,UAAU;gBACV,YAAY,EAAE,IAAI,IAAI,EAAE;gBACxB,KAAK,EAAE,WAAW;qBACf,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;qBACjB,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,yBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,yBAAkB,CAAC,IAAI,EAAE,CAAC;aACtF,CAAC,CAAC;QACL,CAAC;KAAA;IAED;;;;;;;;;;;;;;;;;OAiBG;IACI,MAAM;QACX,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QACD,OAAO,6DAA6B,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9D,CAAC;IAEa,IAAI,CAChB,QAA+C;;YAE/C,MAAM,KAAK,GAAyB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACxD,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;YACpC,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB,yDAA2B,CAAC,OAAO,CAAC;gBAClC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,UAAU,EAAE,KAAK;qBACd,YAAY,EAAE;qBACd,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;qBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;gBACxB,MAAM,EAAE,KAAK;aACd,CAAC,CAAC;YACL,MAAM,GAAG,GAAG,GAAuC,EAAE;gBACnD,MAAM,MAAM,GAAG,yDAA2B,CAAC,OAAO,CAAC;oBACjD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,UAAU,EAAE,KAAK;yBACd,YAAY,EAAE;yBACd,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;yBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;oBACxB,MAAM,EAAE,KAAK;iBACd,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;gBACvB,OAAO;oBACL,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAc;oBACjD,QAAQ;oBACR,MAAM;oBACN,IAAI;oBACJ,OAAO,EAAE,KAAK,CAAC,YAAY,EAAE;oBAC7B,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE;oBAC5B,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBAC6B,CAAC;YAC1D,CAAC,CAAC;YAEF,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACtC,IAAI,OAAO,EAAE,EAAE,CAAC;oBACd,OAAO,GAAG,EAAE,CAAC;gBACf,CAAC;gBAED,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;oBACtD,MAAM,IAAI,GACN,MAAM,yDAA2B,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACpD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;wBAClB,MAAM;oBACR,CAAC;oBAED,MAAM,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;oBAC7B,IAAI,OAAO,EAAE,EAAE,CAAC;wBACd,OAAO,GAAG,EAAE,CAAC;oBACf,CAAC;gBACH,CAAC;gBACD,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,QAAQ;oBACR,OAAO,EAAE,KAAK,CAAC,YAAY,EAAE;oBAC7B,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE;oBAC5B,KAAK;oBACL,UAAU;oBACV,YAAY,EAAE,IAAI,IAAI,EAAE;iBACzB,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;CACF;AA7KD,gEA6KC"}

package/lib/index.d.ts

@@ -1,2 +1,3 @@
 export * from "./AgenticaCallBenchmark";
+export * from "./MicroAgenticaCallBenchmark";
 export * from "./AgenticaSelectBenchmark";

package/lib/index.js

@@ -15,5 +15,6 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./AgenticaCallBenchmark"), exports);
+__exportStar(require("./MicroAgenticaCallBenchmark"), exports);
 __exportStar(require("./AgenticaSelectBenchmark"), exports);
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0DAAwC;AAExC,4DAA0C"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0DAAwC;AACxC,+DAA6C;AAC7C,4DAA0C"}

package/lib/index.mjs

@@ -413,6 +413,108 @@ class AgenticaCallBenchmark {
     }
 }
 
+class MicroAgenticaCallBenchmark {
+    constructor(props) {
+        this.agent_ = props.agent;
+        this.scenarios_ = props.scenarios.slice();
+        this.config_ = {
+            repeat: props.config?.repeat ?? 10,
+            simultaneous: props.config?.simultaneous ?? 10,
+            consent: props.config?.consent ?? 3
+        };
+        this.result_ = null;
+    }
+    async execute(listener) {
+        const started_at = new Date;
+        const semaphore = new Semaphore(this.config_.simultaneous);
+        const task = this.scenarios_.map((async scenario => {
+            const events = await Promise.all(Array.from({
+                length: this.config_.repeat
+            }).map((async () => {
+                await semaphore.acquire();
+                const e = await this.step(scenario);
+                await semaphore.release();
+                if (listener !== undefined) {
+                    listener(e);
+                }
+                return e;
+            })));
+            return {
+                scenario,
+                events,
+                usage: events.filter((e => e.type !== "error")).map((e => e.usage)).reduce(((acc, cur) => AgenticaTokenUsage.plus(acc, cur)), AgenticaTokenUsage.zero())
+            };
+        }));
+        const experiments = await Promise.all(task);
+        return this.result_ = {
+            experiments,
+            started_at,
+            completed_at: new Date,
+            usage: experiments.map((p => p.usage)).reduce(((acc, cur) => AgenticaTokenUsage.plus(acc, cur)), AgenticaTokenUsage.zero())
+        };
+    }
+    report() {
+        if (this.result_ === null) {
+            throw new Error("Benchmark is not executed yet.");
+        }
+        return AgenticaCallBenchmarkReporter.markdown(this.result_);
+    }
+    async step(scenario) {
+        const agent = this.agent_.clone();
+        const started_at = new Date;
+        const success = () => AgenticaBenchmarkPredicator.success({
+            expected: scenario.expected,
+            operations: agent.getHistories().filter((p => p.type === "execute")).map((p => p.operation)),
+            strict: false
+        });
+        const out = () => {
+            const select = AgenticaBenchmarkPredicator.success({
+                expected: scenario.expected,
+                operations: agent.getHistories().filter((p => p.type === "execute")).map((p => p.operation)),
+                strict: false
+            });
+            const call = success();
+            return {
+                type: call ? "success" : "failure",
+                scenario,
+                select,
+                call,
+                prompts: agent.getHistories(),
+                usage: agent.getTokenUsage(),
+                started_at,
+                completed_at: new Date
+            };
+        };
+        try {
+            await agent.conversate(scenario.text);
+            if (success()) {
+                return out();
+            }
+            for (let i = 0; i < this.config_.consent; ++i) {
+                const next = await AgenticaBenchmarkPredicator.isNext(agent);
+                if (next === null) {
+                    break;
+                }
+                await agent.conversate(next);
+                if (success()) {
+                    return out();
+                }
+            }
+            return out();
+        } catch (error) {
+            return {
+                type: "error",
+                scenario,
+                prompts: agent.getHistories(),
+                usage: agent.getTokenUsage(),
+                error,
+                started_at,
+                completed_at: new Date
+            };
+        }
+    }
+}
+
 const AgenticaSelectBenchmarkReporter = {
     markdown
 };
@@ -538,5 +640,5 @@ class AgenticaSelectBenchmark {
     }
 }
 
-export { AgenticaCallBenchmark, AgenticaSelectBenchmark };
+export { AgenticaCallBenchmark, AgenticaSelectBenchmark, MicroAgenticaCallBenchmark };
 //# sourceMappingURL=index.mjs.map

package/lib/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","sources":["../src/internal/AgenticaBenchmarkPredicator.ts","../src/utils/MathUtil.ts","../src/internal/AgenticaBenchmarkUtil.ts","../src/internal/AgenticaPromptReporter.ts","../src/internal/AgenticaCallBenchmarkReporter.ts","../src/AgenticaCallBenchmark.ts","../src/internal/AgenticaSelectBenchmarkReporter.ts","../src/AgenticaSelectBenchmark.ts"],"sourcesContent":[null,null,null,null,null,null,null,null],"names":["AgenticaBenchmarkPredicator","isNext","success","async","agent","last","getHistories","at","llmVendor","getVendor","isTextPrompt","type","role","consent","functions","result","api","chat","completions","create","model","messages","content","join","text","tools","function","name","description","parameters","tool_choice","parallel_tool_calls","options","toolCall","choices","message","tool_calls","filter","tc","undefined","input","arguments","reply","props","successInner","call","expected","overrideOperations","operations","strict","take","targetIterator","items","Symbol","iterator","targeted","next","done","length","value","slice","target","operation","some","op","anyOf","callResult","allOf","map","every","r","reduce","acc","Math","max","MathUtil","round","floor","AgenticaBenchmarkUtil","errorToJson","expectedToJson","error","Error","stack","AgenticaPromptReporter","markdown","p","contents","selections","flatMap","s","functionDescriptionCount","controller","reason","executes","e","split","JSON","stringify","AgenticaCallBenchmarkReporter","Object","fromEntries","writeIndex","experiments","exp","scenario","writeExperimentIndex","events","event","i","writeExperimentEvent","flat","average","completed_at","getTime","started_at","a","b","aggregate","usage","toLocaleString","total","cached","output","reasoning","accepted_prediction","rejected_prediction","drawStatus","select","index","prompts","count","Array","from","fill","AgenticaCallBenchmark","constructor","this","agent_","scenarios_","scenarios","config_","repeat","config","simultaneous","result_","execute","listener","Date","semaphore","Semaphore","task","Promise","all","acquire","step","release","cur","AgenticaTokenUsage","plus","zero","report","clone","out","getTokenUsage","conversate","AgenticaSelectBenchmarkReporter","selected","AgenticaSelectBenchmark","histories_","context","getContext","prompt","factory","createUserInputHistory","executor","TypeError","histories","orchestrate","ready","dispatch","assistantPrompts"],"mappings":";;;;;;;;AAeO,MAAMA,8BAA8B;IACzCC;IACAC;;;AAmCFC,eAAeF,OAAuCG;IACpD,MAAMC,OAA2CD,MAC9CE,eACAC,IAAG;IAQN,MAAMC,YAAYJ,MAAMK;IACxB,MAAMC,eAAeL,MAAMM,SAAS,UAAUN,KAAKO,SAAS;IAC5D,KAAKF,cAAc;QACjB,OAAO;;IAGT,MAAMG,UAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAGrCC,UAAU;IACd,MAAMC,eAAsCP,UAAUQ,IAAIC,KAAKC,YAAYC,OACzE;QACEC,OAAOZ,UAAUY;QACjBC,UAAU,EACR;YACET,MAAM;YACNU,SAAS,EACP,iCACA,IACA,uDACA,gEACA,qDACAC,KAAK;WAET;YACEX,MAAM;YACNU,SAASjB,KAAKmB;;QAGlBC,OAAO,EACL;YACEd,MAAM;YACNe,UAAU;gBACRC,MAAMd,QAAQc;gBACdC,aAAaf,QAAQe;gBACrBC,YAAYhB,QAAQgB;;;QAI1BC,aAAa;QACbC,qBAAqB;OAEvBvB,UAAUwB;IAGZ,MAAMC,YACJlB,OAAOmB,QAAQ,IAAIC,QAAQC,cAAc,IACzCC,QACAC,MAAMA,GAAG3B,SAAS,cAAc2B,GAAGZ,SAASC,SAASd,QAAQc,SAC3D;IAEJ,IAAIM,aAAaM,WAAW;QAC1B,OAAO;;IAGT,MAAMC;;;;;;;SAA0CP,SAASP,SAASe;IAClE,OAAOD,UAAU,OAAOA,MAAME,QAAQ;AACxC;;AAWM,SAAUxC,QAAwCyC;IAoBtD,OAAOC,aAAaD,OAAO5B;AAC7B;;AAEA,SAAS6B,aAA6CD;IAQpD,MAAME,OAAO,CACXC,UACAC,uBAEAH,aAAa;QACXE;QACAE,YAAYD,sBAAsBJ,MAAMK;QACxCC,QAAQN,MAAMM;;IAGlB,QAAQN,MAAMG,SAASnC;MACrB,KAAK;QAAS;YACZ,IAAIuC,OAAO;YACX,MAAMC,iBAAiBR,MAAMG,SAASM,MAAMC,OAAOC;YACnD,IAAIC,WAAWJ,eAAeK;YAE9B,OAAO,MAAM;gBACX,IAAID,SAASE,SAAS,MAAM;oBAC1B,OAAO;wBACL1C,QAAQ;wBACRmC;;;gBAGJ,IAAIA,QAAQP,MAAMK,WAAWU,QAAQ;oBACnC,OAAO;wBAAE3C,QAAQ;;;gBAGnB,MAAMA,SAAS8B,KAAKU,SAASI,OAAOhB,MAAMK,WAAWY,MAAMV;gBAC3D,KAAKnC,OAAOA,QAAQ;oBAClB,IAAI4B,MAAMM,WAAW,MAAM;wBACzB,OAAO;4BAAElC,QAAQ;;;oBAEnBmC,QAAQ;oBACR;;gBAGFA,QAAQnC,OAAOmC;gBACfK,WAAWJ,eAAeK;;;;MAG9B,KAAK;QAAc;YACjB,MAAMK,SAASlB,MAAMG,SAASgB;YAC9B,MAAM/C,SAAS4B,MAAMK,WAAWe,MAAKC,MAAMA,GAAGrC,SAASkC,OAAOlC;YAC9D,IAAIZ,QAAQ;gBACV,OAAO;oBAAEA;oBAAQmC,MAAM;;;YAEzB,OAAO;gBACLnC;;;;MAGJ,KAAK;QACH,KAAK,MAAM+B,YAAYH,MAAMG,SAASmB,OAAO;YAC3C,MAAMC,aAAarB,KAAKC;YACxB,IAAIoB,WAAWnD,QAAQ;gBACrB,OAAOmD;;;QAIX,OAAO;YAAEnD,QAAQ;;;MACnB,KAAK;QAAS;YAQZ,MAAMA,SAAS4B,MAAMG,SAASqB,MAAMC,KAAItB,YAAYD,KAAKC;YACzD,IAAI/B,OAAOsD,OAAMC,KAAKA,EAAEvD,UAAS;gBAC/B,OAAO;oBACLA,QAAQ;oBACRmC,MAAMnC,OAAOwD,QAAO,CAACC,KAAKF,MAAMG,KAAKC,IAAIF,KAAKF,EAAEpB,QAAO;;;YAI3D,OAAO;gBACLnC,QAAQ;;;;AAIhB;;AC3OO,MAAM4D,WAAW;IAOtBC,OAAQjB,SAA0Bc,KAAKI,MAAMlB,QAAQ,OAAO;;;ACJvD,MAAMmB,wBAAwB;IACnCC;IACAC;;;AAGF,SAASD,YAAeE;IAOtB,IAAIA,iBAAiBC,OAAO;QAC1B,OAAO;eACFD;YACHtD,MAAMsD,MAAMtD;YACZQ,SAAS8C,MAAM9C;YACfgD,OAAOF,MAAME;;;IAGjB,OAAOF;AACT;;AAEA,SAASD,eAA+ClC;IACtD,IAAIA,SAASnC,SAAS,cAAc;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfmD,WAAW;gBACTnC,MAAMmB,SAASgB,UAAUnC;gBACzBC,aAAakB,SAASgB,UAAUpC,SAASE;;;WAI1C,IAAIkB,SAASnC,SAAS,SAAS;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfyC,OAAON,SAASM,MAAMgB,IAAIY;;WAGzB,IAAIlC,SAASnC,SAAS,SAAS;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfwD,OAAOrB,SAASqB,MAAMC,IAAIY;;WAGzB;QACH,OAAO;YACLrE,MAAMmC,SAASnC;YACfsD,OAAOnB,SAASmB,MAAMG,IAAIY;;;AAGhC;;ACpDO,MAAMI,yBAAyB;cACpCC;;;AAGF,SAASA,WAAyCC;IAChD,IAAIA,EAAE3E,SAAS,cAAc;QAC3B,OAAO,EAAC,kBAAkB2E,EAAEC,UAAU,KAAIhE,KAAK;;IAEjD,IAAI+D,EAAE3E,SAAS,QAAQ;QACrB,OAAO,EAAC,aAAa2E,EAAE1E,SAAS0E,EAAE9D,MAAM,KAAID,KAAK;;IAGnD,IAAI+D,EAAE3E,SAAS,YAAY2E,EAAE3E,SAAS,UAAU;QAC9C,OAAO,EACL,OAAO2E,EAAE3E,SAAS,WAAW,WAAW,eACrC2E,EAAEE,WACFC,SAASC;YACR,MAAMC,2BAA2BD,EAAE5B,UAAUpC,SAASE,aAAa8B,UAAU;YAE7E,OAAO,EACL,QAAQgC,EAAE5B,UAAUnC,QACpB,mBAAmB+D,EAAE5B,UAAU8B,WAAWjE,QAC1C,iBAAiB+D,EAAE5B,UAAUpC,SAASC,QACtC,eAAe+D,EAAEG,UACjB,OACIF,2BAA2B,IAC3B,EAACD,EAAE5B,UAAUpC,SAASE,aAAa,OACnC;AACL,cAELL,KAAK;;IAGT,IAAI+D,EAAE3E,SAAS,YAAY;QACzB,OAAO,EACL,mBACG2E,EAAEQ,SAAS1B,KAAI2B,KAAK,OAAOA,EAAEjC,UAAUnC,UAC1C,OACG2D,EAAE9D,KAAKwE,MAAM,MAAM5B,KAAIsB,KAAK,KAAKA,OACpC,KACAnE,KAAK;;IAGT,IAAI+D,EAAE3E,SAAS,WAAW;QACxB,OAAO,EACL,eACA,aAAa2E,EAAExB,UAAUnC,QACzB,mBAAmB2D,EAAExB,UAAU8B,WAAWjE,QAC1C,iBAAiB2D,EAAExB,UAAUpC,SAASC,QACtC,IACA,WACAsE,KAAKC,UAAUZ,EAAE7C,WAAW,MAAM,IAClC,OACA,KACAlB,KAAK;;IAIT,MAAM,IAAI2D,MAAM;AAClB;;ACnDO,MAAMiB,gCAAgC;cAC3Cd;;;AAGI,SAAUA,WAAyCtE;IACvD,OAAOqF,OAAOC,YAAY,EACxB,EAAC,eAAeC,aAAkBvF,cAC/BA,OAAOwF,YACPnC,KAAwBoC,OAAO,EAC9B,EAAC,KAAKA,IAAIC,SAAS9E,kBAAkB+E,uBAAqBF,WACvDA,IAAIG,OAAOvC,KAAsB,CAACwC,OAAOC,MAAM,EAChD,KAAKL,IAAIC,SAAS9E,QAAQkF,IAAI,KAAKD,MAAMjG,WACzCmG,uBAAqBF,OAAOC,WAG/BE;AAEP;;AAEA,SAAST,aAA2CvF;IAClD,MAAM4F,SAA+C5F,OAAOwF,YACzDnC,KAAIE,KAAKA,EAAEqC,SACXI;IACH,MAAMC,UACFL,OACCvC,KAAI2B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKV,OAAOjD;IACzC,MAAM4D,YAA2CvG,OAAOwG,MAAMD;IAC9D,OAAO,EACL,iCACA,cACA,oBACA,qBAAqBvG,OAAOwF,YAAY7C,OAAO8D,oBAC/C,gBAAgBb,OAAOjD,UACvB,kBAAkBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC3D,kBAAkBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC3D,uBAAuBiB,SAASC,MAAMoC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU9E,MAAMiF,MAAMD,oBACxC,mBAAmBF,UAAU9E,MAAMkF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,qCACA,yCACGzG,OAAOwF,YAAYnC,KAAIoC,OACxB,EACE,IAAIA,IAAIC,SAAS9E,WAAW6E,IAAIC,SAAS9E,mBACzCoG,WACEvB,IAAIG,SACJZ,KAAKA,EAAEpF,SAAS,WAAWoF,EAAEiC,WAAW,QAE1CD,WAAWvB,IAAIG,SAAQZ,KAAKA,EAAEpF,SAAS,WAAWoF,EAAElD,SAAS,QAC7D,GAAG8B,SAASC,MACV4B,IAAIG,OACDvC,KAAI2B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOjD,QAC3C8D,wBACFjG,KAAK,WAETA,KAAK;AACT;;AAEA,SAASmF,uBAAqDF;IAC5D,OAAO,EACL,KAAKA,IAAIC,SAAS9E,QAClB,cACA,mBAAmB6E,IAAIG,OAAOjD,OAAO8D,oBACrC,gBAAgBhB,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC7D,gBAAgB8C,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC7D,qBAAqBiB,SAASC,MAC5B4B,IAAIG,OACDvC,KAAI2B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOjD,QAC3C8D,uBACF,IACA,aACA,uBACA,0BACGhB,IAAIG,OAAOvC,KAAI,CAAC2B,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAEpF,YAC5BoF,EAAEpF,MACF,GAAGgE,SAASC,MAAMmB,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,kBAC1D3F,KAAK,UAET,IACA,eACA,mBACAiF,IAAIC,SAASjF,MACb,IACA,gBACA,WACAyE,KAAKC,UACHpB,sBAAsBE,eAAewB,IAAIC,SAAS3D,WAClD,MACA,IAEF,QACAvB,KAAK;AACT;;AAEA,SAASuF,uBAAqDF,OAA2CqB;IACvG,OAAO,EACL,KAAKA,QAAQ,MAAMrB,MAAMjG,QACzB,cACA,aAAaiG,MAAMH,SAAS9E,QAC5B,aAAaiF,MAAMjG,QACnB,aAAagE,SAASC,MACpBgC,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,WAChDM,0BACEZ,MAAMjG,SAAS,UACf,EACE,eAAeiG,MAAMoB,SAAS,MAAM,OACpC,aAAapB,MAAM/D,OAAO,MAAM,UAElC,IACJ,oBACA,gBAAgBoD,KAAKC,UAAUU,MAAMW,MAAMD,UAAUG,UACrD,eACA,kBAAkBb,MAAMW,MAAMD,UAAU9E,MAAMiF,SAC9C,mBAAmBb,MAAMW,MAAMD,UAAU9E,MAAMkF,UAC/C,iBACA,kBAAkBd,MAAMW,MAAMD,UAAUK,OAAOF,SAC/C,gCAAgCb,MAAMW,MAAMD,UAAUK,OAAOE,uBAC7D,sBAAsBjB,MAAMW,MAAMD,UAAUK,OAAOC,aACnD,gCAAgChB,MAAMW,MAAMD,UAAUK,OAAOG,uBAE7D,IACA,eACA,mBACAlB,MAAMH,SAASjF,MACf,IACA,gBACA,WACAyE,KAAKC,UACHpB,sBAAsBE,eAAe4B,MAAMH,SAAS3D,WACpD,MACA,IAEF,OACA,IACA,0BACG8D,MAAMsB,QAAQ9D,IAAIgB,uBAAuBC,WAC5C,OACIuB,MAAMjG,SAAS,UACf,EACE,YACA,WACAsF,KAAKC,UACHpB,sBAAsBC,YAAY6B,MAAM3B,QACxC,MACA,IAEF,UAEF,KACJ1D,KAAK;AACT;;AAEA,SAASwG,WAA2CpB,QAA8CzG;IAChG,MAAMiI,QAAgB1D,KAAKI,MACxB8B,OAAOtE,OAAOnC,SAASwD,SAASiD,OAAOjD,SAAU;IAGpD,OACE0E,MAAMC,KAAK;QAAE3E,QAAQyE;OAASG,KAAK,KAAK/G,KAAK,MAC3C6G,MAAMC,KAAK;QAAE3E,QAAQ,KAAKyE;OAASG,KAAK,KAAK/G,KAAK;AAExD;;MCxJagH;IAWX,WAAAC,CAAmB7F;QACjB8F,KAAKC,SAAS/F,MAAMvC;QACpBqI,KAAKE,aAAahG,MAAMiG,UAAUhF;QAClC6E,KAAKI,UAAU;YACbC,QAAQnG,MAAMoG,QAAQD,UAAU;YAChCE,cAAcrG,MAAMoG,QAAQC,gBAAgB;YAC5CnI,SAAS8B,MAAMoG,QAAQlI,WAAW;;QAEpC4H,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMhC,aAAmB,IAAIiC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMO,OAAOd,KAAKE,WAAWvE,KAAIjE,MAAOsG;YACtC,MAAME,eACI6C,QAAQC,IACdrB,MAAMC,KAAK;gBAAE3E,QAAQ+E,KAAKI,QAAQC;eAAU1E,KAAIjE;sBACxCkJ,UAAUK;gBAChB,MAAM3D,UACI0C,KAAKkB,KAAKlD;sBACd4C,UAAUO;gBAEhB,IAAIT,aAAa5G,WAAW;oBAC1B4G,SAASpD;;gBAGX,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAY,OAAOZ,OACJtE,QAAO0D,KAAKA,EAAEpF,SAAS,UACvByD,KAAI2B,KAAKA,EAAEwB,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;AAC/E;QAEH,MAAMzD,oBACIiD,QAAQC,IAAIF;QACtB,OAAQd,KAAKQ,UAAU;YACrB1C;YACAY;YACAF,cAAc,IAAImC;YAClB7B,OAAOhB,YACJnC,KAAIkB,KAAKA,EAAEiC,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;;IAsB3E,MAAAC;QACL,IAAIxB,KAAKQ,YAAY,MAAM;YACzB,MAAM,IAAI/D,MAAM;;QAElB,OAAOiB,8BAA8Bd,SAASoD,KAAKQ;;IAG7C,UAAMU,CACZlD;QAEA,MAAMrG,QAAyBqI,KAAKC,OAAOwB;QAC3C,MAAM/C,aAAmB,IAAIiC;QAC7B,MAAMlJ,UAAU,MACdF,4BAA4BE,QAAQ;YAClC4C,UAAU2D,SAAS3D;YACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,YACvByD,KAAIkB,KAAKA,EAAExB;YACdb,QAAQ;;QAEZ,MAAMkH,MAAM;YACV,MAAMnC,SAAShI,4BAA4BE,QAAQ;gBACjD4C,UAAU2D,SAAS3D;gBACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,WACvByD,KAAIkB,KAAKA,EAAEE,aACXuB,OACA3C,KAAIkB,KAAKA,EAAExB;gBACdb,QAAQ;;YAEV,MAAMJ,OAAO3C;YACb,OAAO;gBACLS,MAAOkC,OAAO,YAAY;gBAC1B4D;gBACAuB;gBACAnF;gBACAqF,SAAS9H,MAAME;gBACfiH,OAAOnH,MAAMgK;gBACbjD;gBACAF,cAAc,IAAImC;;AACmC;QAGzD;kBACQhJ,MAAMiK,WAAW5D,SAASjF;YAChC,IAAItB,WAAW;gBACb,OAAOiK;;YAGT,KAAK,IAAItD,IAAY,GAAGA,IAAI4B,KAAKI,QAAQhI,WAAWgG,GAAG;gBACrD,MAAMrD,aACIxD,4BAA4BC,OAAOG;gBAC7C,IAAIoD,SAAS,MAAM;oBACjB;;sBAGIpD,MAAMiK,WAAW7G;gBACvB,IAAItD,WAAW;oBACb,OAAOiK;;;YAGX,OAAOA;UAET,OAAOlF;YACL,OAAO;gBACLtE,MAAM;gBACN8F;gBACAyB,SAAS9H,MAAME;gBACfiH,OAAOnH,MAAMgK;gBACbnF;gBACAkC;gBACAF,cAAc,IAAImC;;;;;;AChMnB,MAAMkB,kCAAkC;IAC7CjF;;;AAGI,SAAUA,SAAyCtE;IACvD,MAAMuC,WAAW,EACf,EAAC,eAAegD,WAAWvF,cACxBA,OAAOwF,YACPnC,KAAwBoC,OAAO,EAC9B,EAAC,KAAKA,IAAIC,SAAS9E,kBAAkB+E,qBAAqBF,WACvDA,IAAIG,OAAOvC,KAAsB,CAACwC,OAAOC,MAAM,EAChD,KAAKL,IAAIC,SAAS9E,QAAQkF,IAAI,KAAKD,MAAMjG,WACzCmG,qBAAqBF,OAAOC,WAG/BE;IAGL,OAAOX,OAAOC,YAAY/C;AAC5B;;AAEA,SAASgD,WAA2CvF;IAClD,MAAM4F,SAAiD5F,OAAOwF,YAC3DnC,KAAIE,KAAKA,EAAEqC,SACXI;IACH,MAAMC,UACAL,OACCvC,KAAI2B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKV,OAAOjD;IAC3C,MAAM4D,YAA2CvG,OAAOwG,MAAMD;IAC9D,OAAO,EACL,sCACA,cACA,oBACA,qBAAqBvG,OAAOwF,YAAY7C,OAAO8D,oBAC/C,gBAAgBb,OAAOjD,UACvB,kBAAkBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC3D,kBAAkBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC3D,gBAAgBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,UAAS+C,UACvD,uBAAuBiB,SAASC,MAAMoC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU9E,MAAMiF,MAAMD,oBACxC,mBAAmBF,UAAU9E,MAAMkF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,+BACA,kCACGzG,OAAOwF,YAAYnC,KAAIoC,OACxB,EACE,IAAIA,IAAIC,SAAS9E,WAAW6E,IAAIC,SAAS9E,mBACzC;QACE,MAAMzB,UAAkBuE,KAAKI,MAC1B2B,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,SAC1C8C,IAAIG,OAAOjD,SACb;QAEJ,OACE0E,MAAMC,KAAK;YAAE3E,QAAQxD;WAAWoI,KAAK,KAAK/G,KAAK,MAC7C6G,MAAMC,KAAK;YAAE3E,QAAQ,KAAKxD;WAAWoI,KAAK,KAAK/G,KAAK;AAEzD,MAVD,IAWA,GAAGoD,SAASC,MACV4B,IAAIG,OACDvC,KACCwC,SACEA,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,YAEnD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOjD,QAC3C8D,wBACFjG,KAAK,WAETA,KAAK;AACT;;AAEA,SAASmF,qBAAqDF;IAC5D,MAAMc,YAA2Cd,IAAIe,MAAMD;IAC3D,OAAO,EACL,KAAKd,IAAIC,SAAS9E,QAClB,cACA,oBACA,gBAAgB6E,IAAIG,OAAOjD,UAC3B,kBAAkB8C,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC/D,kBAAkB8C,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC/D,gBAAgB8C,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,UAAS+C,UAC3D,uBAAuBiB,SAASC,MAC9B4B,IAAIG,OACDvC,KACCwC,SACEA,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,YAEnD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOjD,QAC3C8D,uBACF,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU9E,MAAMiF,MAAMD,oBACxC,mBAAmBF,UAAU9E,MAAMkF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,aACA,qBACA,wBACGhB,IAAIG,OAAOvC,KAAI,CAAC2B,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAEpF,YAC5BoF,EAAEpF,MACF,GAAGgE,SAASC,MAAMmB,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,kBAE1D3F,KAAK,UAET,IACA,eACA,mBACAiF,IAAIC,SAASjF,MACb,IACA,gBACA,WACAyE,KAAKC,UACHpB,sBAAsBE,eAAewB,IAAIC,SAAS3D,WAClD,MACA,IAEF,QACAvB,KAAK;AACT;;AAEA,SAASuF,qBAAqDF,OAA6CqB;IACzG,OAAO,EACL,KAAKA,QAAQ,MAAMrB,MAAMjG,QACzB,cACA,aAAaiG,MAAMH,SAAS9E,QAC5B,aAAaiF,MAAMjG,QACnB,cAAciG,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,WAAWM,0BACrEZ,MAAMjG,SAAS,UACf,EACE,mBACA,gBAAgBiG,MAAMW,MAAMD,UAAUG,MAAMD,oBAC5C,gBACA,kBAAkBZ,MAAMW,MAAMD,UAAU9E,MAAMiF,MAAMD,oBACpD,mBAAmBZ,MAAMW,MAAMD,UAAU9E,MAAMkF,OAAOF,oBACtD,qBACA,kBAAkBZ,MAAMW,MAAMD,UAAUK,OAAOF,MAAMD,oBACrD,sBAAsBZ,MAAMW,MAAMD,UAAUK,OAAOC,UAAUJ,oBAC7D,gCAAgCZ,MAAMW,MAAMD,UAAUK,OAAOE,oBAAoBL,oBACjF,gCAAgCZ,MAAMW,MAAMD,UAAUK,OAAOG,oBAAoBN,uBAEnF,IACJ,IACA,eACA,mBACAZ,MAAMH,SAASjF,MACf,IACA,gBACA,WACAyE,KAAKC,UACHpB,sBAAsBE,eAAe4B,MAAMH,SAAS3D,WACpD,MACA,IAEF,OACA,OACI8D,MAAMjG,SAAS,aAAaiG,MAAMjG,SAAS,YAC3C,EACE,gBACGiG,MAAM2D,SAASnG,KAAIsB,KACpB,EACE,OAAOA,EAAE5B,UAAUnC,QACnB,qBAAqB+D,EAAE5B,UAAU8B,WAAWjE,UAC5C,mBAAmB+D,EAAE5B,UAAUpC,SAASC,UACxC,eAAe+D,EAAEG,UACjB,OACIH,EAAE5B,UAAUpC,SAASE,gBAAgBW,aAAamD,EAAE5B,UAAUpC,SAASE,gBAAgB,KACvF,EAAC8D,EAAE5B,UAAUpC,SAASE,aAAa,OACnC,KACJL,KAAK,YAGX,OACAqF,MAAMjG,SAAS,UACf,EACE,YACA,WACAmE,sBAAsBC,YACpBkB,KAAKC,UAAUU,MAAM3B,OAAO,MAAM,KAEpC,OACA,OAEF,KACJ1D,KAAK;AACT;;MClLaiJ;IAYX,WAAAhC,CAAmB7F;QACjB8F,KAAKC,SAAS/F,MAAMvC;QACpBqI,KAAKE,aAAahG,MAAMiG,UAAUhF;QAClC6E,KAAKI,UAAU;YACbC,QAAQnG,MAAMoG,QAAQD,UAAU;YAChCE,cAAcrG,MAAMoG,QAAQC,gBAAgB;;QAE9CP,KAAKgC,aAAa9H,MAAMvC,MAAME,eAAesD;QAC7C6E,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMhC,aAAmB,IAAIiC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMzC,oBACIiD,QAAQC,IACdhB,KAAKE,WAAWvE,KAAIjE,MAAOsG;YACzB,MAAME,eACI6C,QAAQC,IACdrB,MAAMC,KAAK;gBAAE3E,QAAQ+E,KAAKI,QAAQC;eAAU1E,KAAIjE;sBACxCkJ,UAAUK;gBAChB,MAAM3D,UACI0C,KAAKkB,KAAKlD;sBACd4C,UAAUO;gBAChB,IAAIT,aAAa5G,WAAW;oBAC1B4G,SAASpD;;gBAEX,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAY,OAAOZ,OACJtE,QAAO0D,KAAKA,EAAEpF,SAAS,UACvByD,KAAI2B,KAAKA,EAAEwB,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;AAC/E;QAGP,OAAQvB,KAAKQ,UAAU;YACrB1C;YACAY;YACAF,cAAc,IAAImC;YAClB7B,OAAOhB,YACJnC,KAAIkB,KAAKA,EAAEiC,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;;IAuB3E,MAAAC;QACL,IAAIxB,KAAKQ,YAAY,MAAM;YACzB,MAAM,IAAI/D,MAAM;;QAElB,OAAOoF,gCAAgCjF,SAASoD,KAAKQ;;IAG/C,UAAMU,CACZlD;QAEA,MAAMU,aAAmB,IAAIiC;QAC7B;YACE,MAAM7B,QAA4BuC,mBAAmBE;YACrD,MAAMU,UAAUjC,KAAKC,OAAOiC,WAAW;gBACrCC,QAAQC,QAAQC,uBAAuB;oBACrCvF,UAAU,EAAC;wBACT5E,MAAM;wBACNa,MAAMiF,SAASjF;;;gBAGnB+F;;YAEF,WAAWmD,QAAQ3B,QAAQgC,aAAa,YAAY;gBAClD,MAAM,IAAIC,UAAU;;YAGtB,MAAMC,mBACKP,QAAQ3B,QAAQgC,UAAU/C,UAAUkD,YAAYlD,QAAQ;mBAC5D0C;gBACHO,WAAWxC,KAAKgC,WAAW7G;gBAC3BuB,OAAO;gBACPgG,OAAO,MAAM;gBACbC,UAAUjL;;YAEd,MAAMoK,WAAgDU,UACnD5I,QAAOiD,KAAKA,EAAE3E,SAAS,WACvByD,KAAIkB,KAAKA,EAAEE,aACXuB;YACH,OAAO;gBACLpG,MAAMX,4BAA4BE,QAAQ;oBACxC4C,UAAU2D,SAAS3D;oBACnBE,YAAYuH,SAASnG,KAAIsB,KAAKA,EAAE5B;qBAE9B,YACA;gBACJ2C;gBACA8D;gBACAhD;gBACA8D,kBAAkBJ,UAEf5I,QAAOiD,KAAKA,EAAE3E,SAAS;gBAC1BwG;gBACAF,cAAc,IAAImC;;UAKtB,OAAOnE;YACL,OAAO;gBACLtE,MAAM;gBACN8F;gBACAxB;gBACAkC;gBACAF,cAAc,IAAImC;;;;;;"}
+{"version":3,"file":"index.mjs","sources":["../src/internal/AgenticaBenchmarkPredicator.ts","../src/utils/MathUtil.ts","../src/internal/AgenticaBenchmarkUtil.ts","../src/internal/AgenticaPromptReporter.ts","../src/internal/AgenticaCallBenchmarkReporter.ts","../src/AgenticaCallBenchmark.ts","../src/MicroAgenticaCallBenchmark.ts","../src/internal/AgenticaSelectBenchmarkReporter.ts","../src/AgenticaSelectBenchmark.ts"],"sourcesContent":[null,null,null,null,null,null,null,null,null],"names":["AgenticaBenchmarkPredicator","isNext","success","async","agent","last","getHistories","at","llmVendor","getVendor","isTextPrompt","type","role","consent","functions","result","api","chat","completions","create","model","messages","content","join","text","tools","function","name","description","parameters","tool_choice","parallel_tool_calls","options","toolCall","choices","message","tool_calls","filter","tc","undefined","input","arguments","reply","props","successInner","call","expected","overrideOperations","operations","strict","take","targetIterator","items","Symbol","iterator","targeted","next","done","length","value","slice","target","operation","some","op","anyOf","callResult","allOf","map","every","r","reduce","acc","Math","max","MathUtil","round","floor","AgenticaBenchmarkUtil","errorToJson","expectedToJson","error","Error","stack","AgenticaPromptReporter","markdown","p","contents","selections","flatMap","s","functionDescriptionCount","controller","reason","executes","e","split","JSON","stringify","AgenticaCallBenchmarkReporter","Object","fromEntries","writeIndex","experiments","exp","scenario","writeExperimentIndex","events","event","i","writeExperimentEvent","flat","average","completed_at","getTime","started_at","a","b","aggregate","usage","toLocaleString","total","cached","output","reasoning","accepted_prediction","rejected_prediction","drawStatus","select","index","prompts","count","Array","from","fill","AgenticaCallBenchmark","constructor","this","agent_","scenarios_","scenarios","config_","repeat","config","simultaneous","result_","execute","listener","Date","semaphore","Semaphore","task","Promise","all","acquire","step","release","cur","AgenticaTokenUsage","plus","zero","report","clone","out","getTokenUsage","conversate","MicroAgenticaCallBenchmark","AgenticaSelectBenchmarkReporter","selected","AgenticaSelectBenchmark","histories_","context","getContext","prompt","factory","createUserInputHistory","executor","TypeError","histories","orchestrate","ready","dispatch","assistantPrompts"],"mappings":";;;;;;;;AAeO,MAAMA,8BAA8B;IACzCC;IACAC;;;AAmCFC,eAAeF,OAAuCG;IACpD,MAAMC,OAA2CD,MAC9CE,eACAC,IAAG;IAQN,MAAMC,YAAYJ,MAAMK;IACxB,MAAMC,eAAeL,MAAMM,SAAS,UAAUN,KAAKO,SAAS;IAC5D,KAAKF,cAAc;QACjB,OAAO;;IAGT,MAAMG,UAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAGrCC,UAAU;IACd,MAAMC,eAAsCP,UAAUQ,IAAIC,KAAKC,YAAYC,OACzE;QACEC,OAAOZ,UAAUY;QACjBC,UAAU,EACR;YACET,MAAM;YACNU,SAAS,EACP,iCACA,IACA,uDACA,gEACA,qDACAC,KAAK;WAET;YACEX,MAAM;YACNU,SAASjB,KAAKmB;;QAGlBC,OAAO,EACL;YACEd,MAAM;YACNe,UAAU;gBACRC,MAAMd,QAAQc;gBACdC,aAAaf,QAAQe;gBACrBC,YAAYhB,QAAQgB;;;QAI1BC,aAAa;QACbC,qBAAqB;OAEvBvB,UAAUwB;IAGZ,MAAMC,YACJlB,OAAOmB,QAAQ,IAAIC,QAAQC,cAAc,IACzCC,QACAC,MAAMA,GAAG3B,SAAS,cAAc2B,GAAGZ,SAASC,SAASd,QAAQc,SAC3D;IAEJ,IAAIM,aAAaM,WAAW;QAC1B,OAAO;;IAGT,MAAMC;;;;;;;SAA0CP,SAASP,SAASe;IAClE,OAAOD,UAAU,OAAOA,MAAME,QAAQ;AACxC;;AAWM,SAAUxC,QAAwCyC;IAoBtD,OAAOC,aAAaD,OAAO5B;AAC7B;;AAEA,SAAS6B,aAA6CD;IAQpD,MAAME,OAAO,CACXC,UACAC,uBAEAH,aAAa;QACXE;QACAE,YAAYD,sBAAsBJ,MAAMK;QACxCC,QAAQN,MAAMM;;IAGlB,QAAQN,MAAMG,SAASnC;MACrB,KAAK;QAAS;YACZ,IAAIuC,OAAO;YACX,MAAMC,iBAAiBR,MAAMG,SAASM,MAAMC,OAAOC;YACnD,IAAIC,WAAWJ,eAAeK;YAE9B,OAAO,MAAM;gBACX,IAAID,SAASE,SAAS,MAAM;oBAC1B,OAAO;wBACL1C,QAAQ;wBACRmC;;;gBAGJ,IAAIA,QAAQP,MAAMK,WAAWU,QAAQ;oBACnC,OAAO;wBAAE3C,QAAQ;;;gBAGnB,MAAMA,SAAS8B,KAAKU,SAASI,OAAOhB,MAAMK,WAAWY,MAAMV;gBAC3D,KAAKnC,OAAOA,QAAQ;oBAClB,IAAI4B,MAAMM,WAAW,MAAM;wBACzB,OAAO;4BAAElC,QAAQ;;;oBAEnBmC,QAAQ;oBACR;;gBAGFA,QAAQnC,OAAOmC;gBACfK,WAAWJ,eAAeK;;;;MAG9B,KAAK;QAAc;YACjB,MAAMK,SAASlB,MAAMG,SAASgB;YAC9B,MAAM/C,SAAS4B,MAAMK,WAAWe,MAAKC,MAAMA,GAAGrC,SAASkC,OAAOlC;YAC9D,IAAIZ,QAAQ;gBACV,OAAO;oBAAEA;oBAAQmC,MAAM;;;YAEzB,OAAO;gBACLnC;;;;MAGJ,KAAK;QACH,KAAK,MAAM+B,YAAYH,MAAMG,SAASmB,OAAO;YAC3C,MAAMC,aAAarB,KAAKC;YACxB,IAAIoB,WAAWnD,QAAQ;gBACrB,OAAOmD;;;QAIX,OAAO;YAAEnD,QAAQ;;;MACnB,KAAK;QAAS;YAQZ,MAAMA,SAAS4B,MAAMG,SAASqB,MAAMC,KAAItB,YAAYD,KAAKC;YACzD,IAAI/B,OAAOsD,OAAMC,KAAKA,EAAEvD,UAAS;gBAC/B,OAAO;oBACLA,QAAQ;oBACRmC,MAAMnC,OAAOwD,QAAO,CAACC,KAAKF,MAAMG,KAAKC,IAAIF,KAAKF,EAAEpB,QAAO;;;YAI3D,OAAO;gBACLnC,QAAQ;;;;AAIhB;;AC3OO,MAAM4D,WAAW;IAOtBC,OAAQjB,SAA0Bc,KAAKI,MAAMlB,QAAQ,OAAO;;;ACJvD,MAAMmB,wBAAwB;IACnCC;IACAC;;;AAGF,SAASD,YAAeE;IAOtB,IAAIA,iBAAiBC,OAAO;QAC1B,OAAO;eACFD;YACHtD,MAAMsD,MAAMtD;YACZQ,SAAS8C,MAAM9C;YACfgD,OAAOF,MAAME;;;IAGjB,OAAOF;AACT;;AAEA,SAASD,eAA+ClC;IACtD,IAAIA,SAASnC,SAAS,cAAc;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfmD,WAAW;gBACTnC,MAAMmB,SAASgB,UAAUnC;gBACzBC,aAAakB,SAASgB,UAAUpC,SAASE;;;WAI1C,IAAIkB,SAASnC,SAAS,SAAS;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfyC,OAAON,SAASM,MAAMgB,IAAIY;;WAGzB,IAAIlC,SAASnC,SAAS,SAAS;QAClC,OAAO;YACLA,MAAMmC,SAASnC;YACfwD,OAAOrB,SAASqB,MAAMC,IAAIY;;WAGzB;QACH,OAAO;YACLrE,MAAMmC,SAASnC;YACfsD,OAAOnB,SAASmB,MAAMG,IAAIY;;;AAGhC;;ACpDO,MAAMI,yBAAyB;cACpCC;;;AAGF,SAASA,WAAyCC;IAChD,IAAIA,EAAE3E,SAAS,cAAc;QAC3B,OAAO,EAAC,kBAAkB2E,EAAEC,UAAU,KAAIhE,KAAK;;IAEjD,IAAI+D,EAAE3E,SAAS,QAAQ;QACrB,OAAO,EAAC,aAAa2E,EAAE1E,SAAS0E,EAAE9D,MAAM,KAAID,KAAK;;IAGnD,IAAI+D,EAAE3E,SAAS,YAAY2E,EAAE3E,SAAS,UAAU;QAC9C,OAAO,EACL,OAAO2E,EAAE3E,SAAS,WAAW,WAAW,eACrC2E,EAAEE,WACFC,SAASC;YACR,MAAMC,2BAA2BD,EAAE5B,UAAUpC,SAASE,aAAa8B,UAAU;YAE7E,OAAO,EACL,QAAQgC,EAAE5B,UAAUnC,QACpB,mBAAmB+D,EAAE5B,UAAU8B,WAAWjE,QAC1C,iBAAiB+D,EAAE5B,UAAUpC,SAASC,QACtC,eAAe+D,EAAEG,UACjB,OACIF,2BAA2B,IAC3B,EAACD,EAAE5B,UAAUpC,SAASE,aAAa,OACnC;AACL,cAELL,KAAK;;IAGT,IAAI+D,EAAE3E,SAAS,YAAY;QACzB,OAAO,EACL,mBACG2E,EAAEQ,SAAS1B,KAAI2B,KAAK,OAAOA,EAAEjC,UAAUnC,UAC1C,OACG2D,EAAE9D,KAAKwE,MAAM,MAAM5B,KAAIsB,KAAK,KAAKA,OACpC,KACAnE,KAAK;;IAGT,IAAI+D,EAAE3E,SAAS,WAAW;QACxB,OAAO,EACL,eACA,aAAa2E,EAAExB,UAAUnC,QACzB,mBAAmB2D,EAAExB,UAAU8B,WAAWjE,QAC1C,iBAAiB2D,EAAExB,UAAUpC,SAASC,QACtC,IACA,WACAsE,KAAKC,UAAUZ,EAAE7C,WAAW,MAAM,IAClC,OACA,KACAlB,KAAK;;IAIT,MAAM,IAAI2D,MAAM;AAClB;;ACnDO,MAAMiB,gCAAgC;cAC3Cd;;;AAGI,SAAUA,WAAyCtE;IACvD,OAAOqF,OAAOC,YAAY,EACxB,EAAC,eAAeC,aAAkBvF,cAC/BA,OAAOwF,YACPnC,KAAwBoC,OAAO,EAC9B,EAAC,KAAKA,IAAIC,SAAS9E,kBAAkB+E,uBAAqBF,WACvDA,IAAIG,OAAOvC,KAAsB,CAACwC,OAAOC,MAAM,EAChD,KAAKL,IAAIC,SAAS9E,QAAQkF,IAAI,KAAKD,MAAMjG,WACzCmG,uBAAqBF,OAAOC,WAG/BE;AAEP;;AAEA,SAAST,aAA2CvF;IAClD,MAAM4F,SAA+C5F,OAAOwF,YACzDnC,KAAIE,KAAKA,EAAEqC,SACXI;IACH,MAAMC,UACFL,OACCvC,KAAI2B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKV,OAAOjD;IACzC,MAAM4D,YAA2CvG,OAAOwG,MAAMD;IAC9D,OAAO,EACL,iCACA,cACA,oBACA,qBAAqBvG,OAAOwF,YAAY7C,OAAO8D,oBAC/C,gBAAgBb,OAAOjD,UACvB,kBAAkBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC3D,kBAAkBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC3D,uBAAuBiB,SAASC,MAAMoC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU9E,MAAMiF,MAAMD,oBACxC,mBAAmBF,UAAU9E,MAAMkF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,qCACA,yCACGzG,OAAOwF,YAAYnC,KAAIoC,OACxB,EACE,IAAIA,IAAIC,SAAS9E,WAAW6E,IAAIC,SAAS9E,mBACzCoG,WACEvB,IAAIG,SACJZ,KAAKA,EAAEpF,SAAS,WAAWoF,EAAEiC,WAAW,QAE1CD,WAAWvB,IAAIG,SAAQZ,KAAKA,EAAEpF,SAAS,WAAWoF,EAAElD,SAAS,QAC7D,GAAG8B,SAASC,MACV4B,IAAIG,OACDvC,KAAI2B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOjD,QAC3C8D,wBACFjG,KAAK,WAETA,KAAK;AACT;;AAEA,SAASmF,uBAAqDF;IAC5D,OAAO,EACL,KAAKA,IAAIC,SAAS9E,QAClB,cACA,mBAAmB6E,IAAIG,OAAOjD,OAAO8D,oBACrC,gBAAgBhB,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC7D,gBAAgB8C,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC7D,qBAAqBiB,SAASC,MAC5B4B,IAAIG,OACDvC,KAAI2B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOjD,QAC3C8D,uBACF,IACA,aACA,uBACA,0BACGhB,IAAIG,OAAOvC,KAAI,CAAC2B,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAEpF,YAC5BoF,EAAEpF,MACF,GAAGgE,SAASC,MAAMmB,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,kBAC1D3F,KAAK,UAET,IACA,eACA,mBACAiF,IAAIC,SAASjF,MACb,IACA,gBACA,WACAyE,KAAKC,UACHpB,sBAAsBE,eAAewB,IAAIC,SAAS3D,WAClD,MACA,IAEF,QACAvB,KAAK;AACT;;AAEA,SAASuF,uBAAqDF,OAA2CqB;IACvG,OAAO,EACL,KAAKA,QAAQ,MAAMrB,MAAMjG,QACzB,cACA,aAAaiG,MAAMH,SAAS9E,QAC5B,aAAaiF,MAAMjG,QACnB,aAAagE,SAASC,MACpBgC,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,WAChDM,0BACEZ,MAAMjG,SAAS,UACf,EACE,eAAeiG,MAAMoB,SAAS,MAAM,OACpC,aAAapB,MAAM/D,OAAO,MAAM,UAElC,IACJ,oBACA,gBAAgBoD,KAAKC,UAAUU,MAAMW,MAAMD,UAAUG,UACrD,eACA,kBAAkBb,MAAMW,MAAMD,UAAU9E,MAAMiF,SAC9C,mBAAmBb,MAAMW,MAAMD,UAAU9E,MAAMkF,UAC/C,iBACA,kBAAkBd,MAAMW,MAAMD,UAAUK,OAAOF,SAC/C,gCAAgCb,MAAMW,MAAMD,UAAUK,OAAOE,uBAC7D,sBAAsBjB,MAAMW,MAAMD,UAAUK,OAAOC,aACnD,gCAAgChB,MAAMW,MAAMD,UAAUK,OAAOG,uBAE7D,IACA,eACA,mBACAlB,MAAMH,SAASjF,MACf,IACA,gBACA,WACAyE,KAAKC,UACHpB,sBAAsBE,eAAe4B,MAAMH,SAAS3D,WACpD,MACA,IAEF,OACA,IACA,0BACG8D,MAAMsB,QAAQ9D,IAAIgB,uBAAuBC,WAC5C,OACIuB,MAAMjG,SAAS,UACf,EACE,YACA,WACAsF,KAAKC,UACHpB,sBAAsBC,YAAY6B,MAAM3B,QACxC,MACA,IAEF,UAEF,KACJ1D,KAAK;AACT;;AAEA,SAASwG,WAA2CpB,QAA8CzG;IAChG,MAAMiI,QAAgB1D,KAAKI,MACxB8B,OAAOtE,OAAOnC,SAASwD,SAASiD,OAAOjD,SAAU;IAGpD,OACE0E,MAAMC,KAAK;QAAE3E,QAAQyE;OAASG,KAAK,KAAK/G,KAAK,MAC3C6G,MAAMC,KAAK;QAAE3E,QAAQ,KAAKyE;OAASG,KAAK,KAAK/G,KAAK;AAExD;;MCxJagH;IAWX,WAAAC,CAAmB7F;QACjB8F,KAAKC,SAAS/F,MAAMvC;QACpBqI,KAAKE,aAAahG,MAAMiG,UAAUhF;QAClC6E,KAAKI,UAAU;YACbC,QAAQnG,MAAMoG,QAAQD,UAAU;YAChCE,cAAcrG,MAAMoG,QAAQC,gBAAgB;YAC5CnI,SAAS8B,MAAMoG,QAAQlI,WAAW;;QAEpC4H,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMhC,aAAmB,IAAIiC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMO,OAAOd,KAAKE,WAAWvE,KAAIjE,MAAOsG;YACtC,MAAME,eACI6C,QAAQC,IACdrB,MAAMC,KAAK;gBAAE3E,QAAQ+E,KAAKI,QAAQC;eAAU1E,KAAIjE;sBACxCkJ,UAAUK;gBAChB,MAAM3D,UACI0C,KAAKkB,KAAKlD;sBACd4C,UAAUO;gBAEhB,IAAIT,aAAa5G,WAAW;oBAC1B4G,SAASpD;;gBAGX,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAY,OAAOZ,OACJtE,QAAO0D,KAAKA,EAAEpF,SAAS,UACvByD,KAAI2B,KAAKA,EAAEwB,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;AAC/E;QAEH,MAAMzD,oBACIiD,QAAQC,IAAIF;QACtB,OAAQd,KAAKQ,UAAU;YACrB1C;YACAY;YACAF,cAAc,IAAImC;YAClB7B,OAAOhB,YACJnC,KAAIkB,KAAKA,EAAEiC,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;;IAsB3E,MAAAC;QACL,IAAIxB,KAAKQ,YAAY,MAAM;YACzB,MAAM,IAAI/D,MAAM;;QAElB,OAAOiB,8BAA8Bd,SAASoD,KAAKQ;;IAG7C,UAAMU,CACZlD;QAEA,MAAMrG,QAAyBqI,KAAKC,OAAOwB;QAC3C,MAAM/C,aAAmB,IAAIiC;QAC7B,MAAMlJ,UAAU,MACdF,4BAA4BE,QAAQ;YAClC4C,UAAU2D,SAAS3D;YACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,YACvByD,KAAIkB,KAAKA,EAAExB;YACdb,QAAQ;;QAEZ,MAAMkH,MAAM;YACV,MAAMnC,SAAShI,4BAA4BE,QAAQ;gBACjD4C,UAAU2D,SAAS3D;gBACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,WACvByD,KAAIkB,KAAKA,EAAEE,aACXuB,OACA3C,KAAIkB,KAAKA,EAAExB;gBACdb,QAAQ;;YAEV,MAAMJ,OAAO3C;YACb,OAAO;gBACLS,MAAOkC,OAAO,YAAY;gBAC1B4D;gBACAuB;gBACAnF;gBACAqF,SAAS9H,MAAME;gBACfiH,OAAOnH,MAAMgK;gBACbjD;gBACAF,cAAc,IAAImC;;AACmC;QAGzD;kBACQhJ,MAAMiK,WAAW5D,SAASjF;YAChC,IAAItB,WAAW;gBACb,OAAOiK;;YAGT,KAAK,IAAItD,IAAY,GAAGA,IAAI4B,KAAKI,QAAQhI,WAAWgG,GAAG;gBACrD,MAAMrD,aACIxD,4BAA4BC,OAAOG;gBAC7C,IAAIoD,SAAS,MAAM;oBACjB;;sBAGIpD,MAAMiK,WAAW7G;gBACvB,IAAItD,WAAW;oBACb,OAAOiK;;;YAGX,OAAOA;UAET,OAAOlF;YACL,OAAO;gBACLtE,MAAM;gBACN8F;gBACAyB,SAAS9H,MAAME;gBACfiH,OAAOnH,MAAMgK;gBACbnF;gBACAkC;gBACAF,cAAc,IAAImC;;;;;;MC3KbkB;IAWX,WAAA9B,CAAmB7F;QACjB8F,KAAKC,SAAS/F,MAAMvC;QACpBqI,KAAKE,aAAahG,MAAMiG,UAAUhF;QAClC6E,KAAKI,UAAU;YACbC,QAAQnG,MAAMoG,QAAQD,UAAU;YAChCE,cAAcrG,MAAMoG,QAAQC,gBAAgB;YAC5CnI,SAAS8B,MAAMoG,QAAQlI,WAAW;;QAEpC4H,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMhC,aAAmB,IAAIiC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMO,OAAOd,KAAKE,WAAWvE,KAAIjE,MAAOsG;YACtC,MAAME,eACI6C,QAAQC,IACdrB,MAAMC,KAAK;gBAAE3E,QAAQ+E,KAAKI,QAAQC;eAAU1E,KAAIjE;sBACxCkJ,UAAUK;gBAChB,MAAM3D,UACI0C,KAAKkB,KAAKlD;sBACd4C,UAAUO;gBAEhB,IAAIT,aAAa5G,WAAW;oBAC1B4G,SAASpD;;gBAGX,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAY,OAAOZ,OACJtE,QAAO0D,KAAKA,EAAEpF,SAAS,UACvByD,KAAI2B,KAAKA,EAAEwB,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;AAC/E;QAEH,MAAMzD,oBACIiD,QAAQC,IAAIF;QACtB,OAAQd,KAAKQ,UAAU;YACrB1C;YACAY;YACAF,cAAc,IAAImC;YAClB7B,OAAOhB,YACJnC,KAAIkB,KAAKA,EAAEiC,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;;IAsB3E,MAAAC;QACL,IAAIxB,KAAKQ,YAAY,MAAM;YACzB,MAAM,IAAI/D,MAAM;;QAElB,OAAOiB,8BAA8Bd,SAASoD,KAAKQ;;IAG7C,UAAMU,CACZlD;QAEA,MAAMrG,QAA8BqI,KAAKC,OAAOwB;QAChD,MAAM/C,aAAmB,IAAIiC;QAC7B,MAAMlJ,UAAU,MACdF,4BAA4BE,QAAQ;YAClC4C,UAAU2D,SAAS3D;YACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,YACvByD,KAAIkB,KAAKA,EAAExB;YACdb,QAAQ;;QAEZ,MAAMkH,MAAM;YACV,MAAMnC,SAAShI,4BAA4BE,QAAQ;gBACjD4C,UAAU2D,SAAS3D;gBACnBE,YAAY5C,MACTE,eACA+B,QAAOiD,KAAKA,EAAE3E,SAAS,YACvByD,KAAIkB,KAAKA,EAAExB;gBACdb,QAAQ;;YAEV,MAAMJ,OAAO3C;YACb,OAAO;gBACLS,MAAOkC,OAAO,YAAY;gBAC1B4D;gBACAuB;gBACAnF;gBACAqF,SAAS9H,MAAME;gBACfiH,OAAOnH,MAAMgK;gBACbjD;gBACAF,cAAc,IAAImC;;AACmC;QAGzD;kBACQhJ,MAAMiK,WAAW5D,SAASjF;YAChC,IAAItB,WAAW;gBACb,OAAOiK;;YAGT,KAAK,IAAItD,IAAY,GAAGA,IAAI4B,KAAKI,QAAQhI,WAAWgG,GAAG;gBACrD,MAAMrD,aACIxD,4BAA4BC,OAAOG;gBAC7C,IAAIoD,SAAS,MAAM;oBACjB;;sBAGIpD,MAAMiK,WAAW7G;gBACvB,IAAItD,WAAW;oBACb,OAAOiK;;;YAGX,OAAOA;UAET,OAAOlF;YACL,OAAO;gBACLtE,MAAM;gBACN8F;gBACAyB,SAAS9H,MAAME;gBACfiH,OAAOnH,MAAMgK;gBACbnF;gBACAkC;gBACAF,cAAc,IAAImC;;;;;;AC9LnB,MAAMmB,kCAAkC;IAC7ClF;;;AAGI,SAAUA,SAAyCtE;IACvD,MAAMuC,WAAW,EACf,EAAC,eAAegD,WAAWvF,cACxBA,OAAOwF,YACPnC,KAAwBoC,OAAO,EAC9B,EAAC,KAAKA,IAAIC,SAAS9E,kBAAkB+E,qBAAqBF,WACvDA,IAAIG,OAAOvC,KAAsB,CAACwC,OAAOC,MAAM,EAChD,KAAKL,IAAIC,SAAS9E,QAAQkF,IAAI,KAAKD,MAAMjG,WACzCmG,qBAAqBF,OAAOC,WAG/BE;IAGL,OAAOX,OAAOC,YAAY/C;AAC5B;;AAEA,SAASgD,WAA2CvF;IAClD,MAAM4F,SAAiD5F,OAAOwF,YAC3DnC,KAAIE,KAAKA,EAAEqC,SACXI;IACH,MAAMC,UACAL,OACCvC,KAAI2B,KAAKA,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,YACjD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKV,OAAOjD;IAC3C,MAAM4D,YAA2CvG,OAAOwG,MAAMD;IAC9D,OAAO,EACL,sCACA,cACA,oBACA,qBAAqBvG,OAAOwF,YAAY7C,OAAO8D,oBAC/C,gBAAgBb,OAAOjD,UACvB,kBAAkBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC3D,kBAAkBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC3D,gBAAgBiD,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,UAAS+C,UACvD,uBAAuBiB,SAASC,MAAMoC,SAASQ,uBAC/C,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU9E,MAAMiF,MAAMD,oBACxC,mBAAmBF,UAAU9E,MAAMkF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,kBACA,+BACA,kCACGzG,OAAOwF,YAAYnC,KAAIoC,OACxB,EACE,IAAIA,IAAIC,SAAS9E,WAAW6E,IAAIC,SAAS9E,mBACzC;QACE,MAAMzB,UAAkBuE,KAAKI,MAC1B2B,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,SAC1C8C,IAAIG,OAAOjD,SACb;QAEJ,OACE0E,MAAMC,KAAK;YAAE3E,QAAQxD;WAAWoI,KAAK,KAAK/G,KAAK,MAC7C6G,MAAMC,KAAK;YAAE3E,QAAQ,KAAKxD;WAAWoI,KAAK,KAAK/G,KAAK;AAEzD,MAVD,IAWA,GAAGoD,SAASC,MACV4B,IAAIG,OACDvC,KACCwC,SACEA,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,YAEnD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOjD,QAC3C8D,wBACFjG,KAAK,WAETA,KAAK;AACT;;AAEA,SAASmF,qBAAqDF;IAC5D,MAAMc,YAA2Cd,IAAIe,MAAMD;IAC3D,OAAO,EACL,KAAKd,IAAIC,SAAS9E,QAClB,cACA,oBACA,gBAAgB6E,IAAIG,OAAOjD,UAC3B,kBAAkB8C,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC/D,kBAAkB8C,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,YAAW+C,UAC/D,gBAAgB8C,IAAIG,OAAOtE,QAAO0D,KAAKA,EAAEpF,SAAS,UAAS+C,UAC3D,uBAAuBiB,SAASC,MAC9B4B,IAAIG,OACDvC,KACCwC,SACEA,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,YAEnD3C,QAAO,CAAC6C,GAAGC,MAAMD,IAAIC,IAAG,KAAKb,IAAIG,OAAOjD,QAC3C8D,uBACF,mBACA,gBAAgBF,UAAUG,MAAMD,oBAChC,eACA,kBAAkBF,UAAU9E,MAAMiF,MAAMD,oBACxC,mBAAmBF,UAAU9E,MAAMkF,OAAOF,oBAC1C,iBACA,kBAAkBF,UAAUK,OAAOF,MAAMD,oBACzC,gCAAgCF,UAAUK,OAAOE,oBAAoBL,oBACrE,sBAAsBF,UAAUK,OAAOC,UAAUJ,oBACjD,gCAAgCF,UAAUK,OAAOG,oBAAoBN,oBACrE,IACA,aACA,qBACA,wBACGhB,IAAIG,OAAOvC,KAAI,CAAC2B,GAAGc,MACpB,EACE,IAAIA,IAAI,SAASA,IAAI,KAAKd,EAAEpF,YAC5BoF,EAAEpF,MACF,GAAGgE,SAASC,MAAMmB,EAAEkB,aAAaC,YAAYnB,EAAEoB,WAAWD,kBAE1D3F,KAAK,UAET,IACA,eACA,mBACAiF,IAAIC,SAASjF,MACb,IACA,gBACA,WACAyE,KAAKC,UACHpB,sBAAsBE,eAAewB,IAAIC,SAAS3D,WAClD,MACA,IAEF,QACAvB,KAAK;AACT;;AAEA,SAASuF,qBAAqDF,OAA6CqB;IACzG,OAAO,EACL,KAAKA,QAAQ,MAAMrB,MAAMjG,QACzB,cACA,aAAaiG,MAAMH,SAAS9E,QAC5B,aAAaiF,MAAMjG,QACnB,cAAciG,MAAMK,aAAaC,YAAYN,MAAMO,WAAWD,WAAWM,0BACrEZ,MAAMjG,SAAS,UACf,EACE,mBACA,gBAAgBiG,MAAMW,MAAMD,UAAUG,MAAMD,oBAC5C,gBACA,kBAAkBZ,MAAMW,MAAMD,UAAU9E,MAAMiF,MAAMD,oBACpD,mBAAmBZ,MAAMW,MAAMD,UAAU9E,MAAMkF,OAAOF,oBACtD,qBACA,kBAAkBZ,MAAMW,MAAMD,UAAUK,OAAOF,MAAMD,oBACrD,sBAAsBZ,MAAMW,MAAMD,UAAUK,OAAOC,UAAUJ,oBAC7D,gCAAgCZ,MAAMW,MAAMD,UAAUK,OAAOE,oBAAoBL,oBACjF,gCAAgCZ,MAAMW,MAAMD,UAAUK,OAAOG,oBAAoBN,uBAEnF,IACJ,IACA,eACA,mBACAZ,MAAMH,SAASjF,MACf,IACA,gBACA,WACAyE,KAAKC,UACHpB,sBAAsBE,eAAe4B,MAAMH,SAAS3D,WACpD,MACA,IAEF,OACA,OACI8D,MAAMjG,SAAS,aAAaiG,MAAMjG,SAAS,YAC3C,EACE,gBACGiG,MAAM4D,SAASpG,KAAIsB,KACpB,EACE,OAAOA,EAAE5B,UAAUnC,QACnB,qBAAqB+D,EAAE5B,UAAU8B,WAAWjE,UAC5C,mBAAmB+D,EAAE5B,UAAUpC,SAASC,UACxC,eAAe+D,EAAEG,UACjB,OACIH,EAAE5B,UAAUpC,SAASE,gBAAgBW,aAAamD,EAAE5B,UAAUpC,SAASE,gBAAgB,KACvF,EAAC8D,EAAE5B,UAAUpC,SAASE,aAAa,OACnC,KACJL,KAAK,YAGX,OACAqF,MAAMjG,SAAS,UACf,EACE,YACA,WACAmE,sBAAsBC,YACpBkB,KAAKC,UAAUU,MAAM3B,OAAO,MAAM,KAEpC,OACA,OAEF,KACJ1D,KAAK;AACT;;MClLakJ;IAYX,WAAAjC,CAAmB7F;QACjB8F,KAAKC,SAAS/F,MAAMvC;QACpBqI,KAAKE,aAAahG,MAAMiG,UAAUhF;QAClC6E,KAAKI,UAAU;YACbC,QAAQnG,MAAMoG,QAAQD,UAAU;YAChCE,cAAcrG,MAAMoG,QAAQC,gBAAgB;;QAE9CP,KAAKiC,aAAa/H,MAAMvC,MAAME,eAAesD;QAC7C6E,KAAKQ,UAAU;;IAmBV,aAAMC,CACXC;QAEA,MAAMhC,aAAmB,IAAIiC;QAC7B,MAAMC,YAAuB,IAAIC,UAAUb,KAAKI,QAAQG;QACxD,MAAMzC,oBACIiD,QAAQC,IACdhB,KAAKE,WAAWvE,KAAIjE,MAAOsG;YACzB,MAAME,eACI6C,QAAQC,IACdrB,MAAMC,KAAK;gBAAE3E,QAAQ+E,KAAKI,QAAQC;eAAU1E,KAAIjE;sBACxCkJ,UAAUK;gBAChB,MAAM3D,UACI0C,KAAKkB,KAAKlD;sBACd4C,UAAUO;gBAChB,IAAIT,aAAa5G,WAAW;oBAC1B4G,SAASpD;;gBAEX,OAAOA;AAAC;YAGd,OAAO;gBACLU;gBACAE;gBACAY,OAAOZ,OACJtE,QAAO0D,KAAKA,EAAEpF,SAAS,UACvByD,KAAI2B,KAAKA,EAAEwB,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;AAC/E;QAGP,OAAQvB,KAAKQ,UAAU;YACrB1C;YACAY;YACAF,cAAc,IAAImC;YAClB7B,OAAOhB,YACJnC,KAAIkB,KAAKA,EAAEiC,QACXhD,QAAO,CAACC,KAAKqF,QAAQC,mBAAmBC,KAAKvF,KAAKqF,OAAMC,mBAAmBE;;;IAuB3E,MAAAC;QACL,IAAIxB,KAAKQ,YAAY,MAAM;YACzB,MAAM,IAAI/D,MAAM;;QAElB,OAAOqF,gCAAgClF,SAASoD,KAAKQ;;IAG/C,UAAMU,CACZlD;QAEA,MAAMU,aAAmB,IAAIiC;QAC7B;YACE,MAAM7B,QAA4BuC,mBAAmBE;YACrD,MAAMW,UAAUlC,KAAKC,OAAOkC,WAAW;gBACrCC,QAAQC,QAAQC,uBAAuB;oBACrCxF,UAAU,EAAC;wBACT5E,MAAM;wBACNa,MAAMiF,SAASjF;;;gBAGnB+F;;YAEF,WAAWoD,QAAQ5B,QAAQiC,aAAa,YAAY;gBAClD,MAAM,IAAIC,UAAU;;YAGtB,MAAMC,mBACKP,QAAQ5B,QAAQiC,UAAUhD,UAAUmD,YAAYnD,QAAQ;mBAC5D2C;gBACHO,WAAWzC,KAAKiC,WAAW9G;gBAC3BuB,OAAO;gBACPiG,OAAO,MAAM;gBACbC,UAAUlL;;YAEd,MAAMqK,WAAgDU,UACnD7I,QAAOiD,KAAKA,EAAE3E,SAAS,WACvByD,KAAIkB,KAAKA,EAAEE,aACXuB;YACH,OAAO;gBACLpG,MAAMX,4BAA4BE,QAAQ;oBACxC4C,UAAU2D,SAAS3D;oBACnBE,YAAYwH,SAASpG,KAAIsB,KAAKA,EAAE5B;qBAE9B,YACA;gBACJ2C;gBACA+D;gBACAjD;gBACA+D,kBAAkBJ,UAEf7I,QAAOiD,KAAKA,EAAE3E,SAAS;gBAC1BwG;gBACAF,cAAc,IAAImC;;UAKtB,OAAOnE;YACL,OAAO;gBACLtE,MAAM;gBACN8F;gBACAxB;gBACAkC;gBACAF,cAAc,IAAImC;;;;;;"}

package/lib/internal/AgenticaBenchmarkPredicator.d.ts

@@ -4,14 +4,14 @@
  *
  * @author Wrtn Technologies
  */
-import type { Agentica, AgenticaOperation } from "@agentica/core";
+import type { Agentica, AgenticaOperation, MicroAgentica } from "@agentica/core";
 import type { ILlmSchema } from "@samchon/openapi";
 import type { IAgenticaBenchmarkExpected } from "../structures/IAgenticaBenchmarkExpected";
 export declare const AgenticaBenchmarkPredicator: {
     isNext: typeof isNext;
     success: typeof success;
 };
-declare function isNext<Model extends ILlmSchema.Model>(agent: Agentica<Model>): Promise<string | null>;
+declare function isNext<Model extends ILlmSchema.Model>(agent: Agentica<Model> | MicroAgentica<Model>): Promise<string | null>;
 /**
  * Check if the called operations match the expected operations.
  *

package/lib/internal/AgenticaBenchmarkPredicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"AgenticaBenchmarkPredicator.js","sourceRoot":"","sources":["../../src/internal/AgenticaBenchmarkPredicator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmIA,0BAqBC;;AA7ID,kDAA0B;AAIb,QAAA,2BAA2B,GAAG;IACzC,MAAM;IACN,OAAO;CACR,CAAC;AAkCF,SAAe,MAAM,CAAiC,KAAsB;;;QAC1E,MAAM,IAAI,GAAuC,KAAK;aACnD,YAAY,EAAE;aACd,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEV;;;;;WAKG;QACH,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QACpC,MAAM,YAAY,GAAG,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,MAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;QACxE,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,OAAO,GAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAGrC,SAAS,CAAC,CAAC,CAAE,CAAC;QAClB,MAAM,MAAM,GAA0B,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAC/E;YACE,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE;wBACP,+BAA+B;wBAC/B,EAAE;wBACF,qDAAqD;wBACrD,8DAA8D;wBAC9D,kDAAkD;qBACnD,CAAC,IAAI,CAAC,IAAI,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,IAAI,CAAC,IAAI;iBACnB;aACF;YACD,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,UAAU;oBAChB,QAAQ,EAAE;wBACR,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,UAAU,EAAE,OAAO,CAAC,UAAiC;qBACtD;iBACF;aACF;YACD,WAAW,EAAE,UAAU;YACvB,mBAAmB,EAAE,KAAK;SAC3B,EACD,SAAS,CAAC,OAAO,CAClB,CAAC;QAEF,MAAM,QAAQ,GAAqD,MAAA,CACjE,MAAA,MAAA,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,OAAO,CAAC,UAAU,mCAAI,EAAE,CAC5C,CAAC,MAAM,CACN,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,UAAU,IAAI,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CAClE,0CAAG,CAAC,CAAC,CAAC;QAEP,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,kRAAqC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7E,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;CAAA;AAED;;;;;;;;GAQG;AACH,SAAgB,OAAO,CAAiC,KAmBvD;IACC,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;AACpC,CAAC;AAED,SAAS,YAAY,CAAiC,KAA2C;IAQ/F,MAAM,IAAI,GAAG,CACX,QAA2C,EAC3C,kBAAoD,EACpD,EAAE,CACF,YAAY,CAAC;QACX,QAAQ;QACR,UAAU,EAAE,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,KAAK,CAAC,UAAU;QAClD,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC,CAAC;IAEL,QAAQ,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC5B,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,IAAI,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;YAErC,OAAO,IAAI,EAAE,CAAC;gBACZ,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC3B,OAAO;wBACL,MAAM,EAAE,IAAI;wBACZ,IAAI;qBACL,CAAC;gBACJ,CAAC;gBACD,IAAI,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;oBACpC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;gBAC3B,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBAClE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACnB,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;wBAC1B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;oBAC3B,CAAC;oBACD,IAAI,IAAI,CAAC,CAAC;oBACV,SAAS;gBACX,CAAC;gBAED,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;gBACpB,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;YACnC,CAAC;QACH,CAAC;QACD,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YACxC,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC7B,CAAC;YACD,OAAO;gBACL,MAAM;aACP,CAAC;QACJ,CAAC;QACD,KAAK,OAAO;YACV,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAClC,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACtB,OAAO,UAAU,CAAC;gBACpB,CAAC;YACH,CAAC;YAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,KAAK,OAAO,CAAC,CAAC,CAAC;YACb;;;;;;eAMG;YACH,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChC,OAAO;oBACL,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;iBAC1D,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,KAAK;aACd,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"AgenticaBenchmarkPredicator.js","sourceRoot":"","sources":["../../src/internal/AgenticaBenchmarkPredicator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmIA,0BAqBC;;AA7ID,kDAA0B;AAIb,QAAA,2BAA2B,GAAG;IACzC,MAAM;IACN,OAAO;CACR,CAAC;AAkCF,SAAe,MAAM,CAAiC,KAA6C;;;QACjG,MAAM,IAAI,GAAuC,KAAK;aACnD,YAAY,EAAE;aACd,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEV;;;;;WAKG;QACH,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QACpC,MAAM,YAAY,GAAG,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,MAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC;QACxE,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,OAAO,GAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAGrC,SAAS,CAAC,CAAC,CAAE,CAAC;QAClB,MAAM,MAAM,GAA0B,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAC/E;YACE,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE;wBACP,+BAA+B;wBAC/B,EAAE;wBACF,qDAAqD;wBACrD,8DAA8D;wBAC9D,kDAAkD;qBACnD,CAAC,IAAI,CAAC,IAAI,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,IAAI,CAAC,IAAI;iBACnB;aACF;YACD,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,UAAU;oBAChB,QAAQ,EAAE;wBACR,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,UAAU,EAAE,OAAO,CAAC,UAAiC;qBACtD;iBACF;aACF;YACD,WAAW,EAAE,UAAU;YACvB,mBAAmB,EAAE,KAAK;SAC3B,EACD,SAAS,CAAC,OAAO,CAClB,CAAC;QAEF,MAAM,QAAQ,GAAqD,MAAA,CACjE,MAAA,MAAA,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,OAAO,CAAC,UAAU,mCAAI,EAAE,CAC5C,CAAC,MAAM,CACN,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,UAAU,IAAI,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CAClE,0CAAG,CAAC,CAAC,CAAC;QAEP,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,kRAAqC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7E,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;CAAA;AAED;;;;;;;;GAQG;AACH,SAAgB,OAAO,CAAiC,KAmBvD;IACC,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;AACpC,CAAC;AAED,SAAS,YAAY,CAAiC,KAA2C;IAQ/F,MAAM,IAAI,GAAG,CACX,QAA2C,EAC3C,kBAAoD,EACpD,EAAE,CACF,YAAY,CAAC;QACX,QAAQ;QACR,UAAU,EAAE,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,KAAK,CAAC,UAAU;QAClD,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC,CAAC;IAEL,QAAQ,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC5B,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,IAAI,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;YAErC,OAAO,IAAI,EAAE,CAAC;gBACZ,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC3B,OAAO;wBACL,MAAM,EAAE,IAAI;wBACZ,IAAI;qBACL,CAAC;gBACJ,CAAC;gBACD,IAAI,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;oBACpC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;gBAC3B,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBAClE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACnB,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;wBAC1B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;oBAC3B,CAAC;oBACD,IAAI,IAAI,CAAC,CAAC;oBACV,SAAS;gBACX,CAAC;gBAED,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;gBACpB,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;YACnC,CAAC;QACH,CAAC;QACD,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YACxC,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC7B,CAAC;YACD,OAAO;gBACL,MAAM;aACP,CAAC;QACJ,CAAC;QACD,KAAK,OAAO;YACV,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAClC,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACtB,OAAO,UAAU,CAAC;gBACpB,CAAC;YACH,CAAC;YAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,KAAK,OAAO,CAAC,CAAC,CAAC;YACb;;;;;;eAMG;YACH,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChC,OAAO;oBACL,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;iBAC1D,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,KAAK;aACd,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentica/benchmark",
-  "version": "0.22.0",
+  "version": "0.23.0",
   "description": "Agentic AI Library specialized in LLM Function Calling",
   "author": "Wrtn Technologies",
   "license": "MIT",
@@ -35,14 +35,14 @@
     "src"
   ],
   "peerDependencies": {
-    "@agentica/core": "^0.22.0"
+    "@agentica/core": "^0.23.0"
   },
   "dependencies": {
     "@samchon/openapi": "^4.2.0",
     "openai": "^4.80.0",
     "tstl": "^3.0.0",
     "typia": "^9.0.1",
-    "@agentica/core": "^0.22.0"
+    "@agentica/core": "^0.23.0"
   },
   "devDependencies": {
     "@rollup/plugin-terser": "^0.4.4",

package/src/MicroAgenticaCallBenchmark.ts

@@ -0,0 +1,282 @@
+import type { MicroAgentica } from "@agentica/core";
+import type { ILlmSchema } from "@samchon/openapi";
+import type { tags } from "typia";
+
+/**
+ * @module
+ * This file contains the implementation of the AgenticaCallBenchmark class.
+ *
+ * @author Wrtn Technologies
+ */
+import { AgenticaTokenUsage } from "@agentica/core";
+import { Semaphore } from "tstl";
+
+import type { IAgenticaCallBenchmarkEvent } from "./structures/IAgenticaCallBenchmarkEvent";
+import type { IAgenticaCallBenchmarkResult } from "./structures/IAgenticaCallBenchmarkResult";
+import type { IAgenticaCallBenchmarkScenario } from "./structures/IAgenticaCallBenchmarkScenario";
+
+import { AgenticaBenchmarkPredicator } from "./internal/AgenticaBenchmarkPredicator";
+import { AgenticaCallBenchmarkReporter } from "./internal/AgenticaCallBenchmarkReporter";
+
+/**
+ * LLM function calling selection benchmark.
+ *
+ * `AgenticaCallBenchmark` is a class for the benchmark of the
+ * LLM (Large Model Language) function calling part. It utilizes both
+ * `selector` and `caller` agents and tests whether the expected
+ * {@link IAgenticaOperation operations} are properly selected and
+ * called from the given
+ * {@link IAgenticaCallBenchmarkScenario scenarios}.
+ *
+ * Note that, this `MicroAgenticaCallBenchmark` consumes a lot of time and
+ * LLM token costs because it needs the whole process of the
+ * {@link MicroAgentica} class with a lot of repetitions. If you don't want
+ * such a heavy benchmark, consider to using
+ * {@link AgenticaSelectBenchmark} instead. In my experience,
+ * {@link MicroAgentica} does not fail to function calling, so the function
+ * selection benchmark is much economical.
+ *
+ * @author Samchon
+ */
+export class MicroAgenticaCallBenchmark<Model extends ILlmSchema.Model> {
+  private agent_: MicroAgentica<Model>;
+  private scenarios_: IAgenticaCallBenchmarkScenario<Model>[];
+  private config_: MicroAgenticaCallBenchmark.IConfig;
+  private result_: IAgenticaCallBenchmarkResult<Model> | null;
+
+  /**
+   * Initializer Constructor.
+   *
+   * @param props Properties of the selection benchmark
+   */
+  public constructor(props: MicroAgenticaCallBenchmark.IProps<Model>) {
+    this.agent_ = props.agent;
+    this.scenarios_ = props.scenarios.slice();
+    this.config_ = {
+      repeat: props.config?.repeat ?? 10,
+      simultaneous: props.config?.simultaneous ?? 10,
+      consent: props.config?.consent ?? 3,
+    };
+    this.result_ = null;
+  }
+
+  /**
+   * Execute the benchmark.
+   *
+   * Execute the benchmark of the LLM function calling, and returns
+   * the result of the benchmark.
+   *
+   * If you wanna see progress of the benchmark, you can pass a callback
+   * function as the argument of the `listener`. The callback function
+   * would be called whenever a benchmark event is occurred.
+   *
+   * Also, you can publish a markdown format report by calling
+   * the {@link report} function after the benchmark execution.
+   *
+   * @param listener Callback function listening the benchmark events
+   * @returns Results of the function calling benchmark
+   */
+  public async execute(
+    listener?: (event: IAgenticaCallBenchmarkEvent<Model>) => void,
+  ): Promise<IAgenticaCallBenchmarkResult<Model>> {
+    const started_at: Date = new Date();
+    const semaphore: Semaphore = new Semaphore(this.config_.simultaneous);
+    const task = this.scenarios_.map(async (scenario) => {
+      const events: IAgenticaCallBenchmarkEvent<Model>[]
+        = await Promise.all(
+          Array.from({ length: this.config_.repeat }).map(async () => {
+            await semaphore.acquire();
+            const e: IAgenticaCallBenchmarkEvent<Model>
+              = await this.step(scenario);
+            await semaphore.release();
+
+            if (listener !== undefined) {
+              listener(e);
+            }
+
+            return e;
+          }),
+        );
+      return {
+        scenario,
+        events,
+        usage: events
+          .filter(e => e.type !== "error")
+          .map(e => e.usage)
+          .reduce((acc, cur) => AgenticaTokenUsage.plus(acc, cur), AgenticaTokenUsage.zero()),
+      };
+    });
+    const experiments: IAgenticaCallBenchmarkResult.IExperiment<Model>[]
+      = await Promise.all(task);
+    return (this.result_ = {
+      experiments,
+      started_at,
+      completed_at: new Date(),
+      usage: experiments
+        .map(p => p.usage)
+        .reduce((acc, cur) => AgenticaTokenUsage.plus(acc, cur), AgenticaTokenUsage.zero()),
+    });
+  }
+
+  /**
+   * Report the benchmark result as markdown files.
+   *
+   * Report the benchmark result {@link execute}d by
+   * `AgenticaCallBenchmark` as markdown files, and returns a dictionary
+   * object of the markdown reporting files. The key of the dictionary
+   * would be file name, and the value would be the markdown content.
+   *
+   * For reference, the markdown files are composed like below:
+   *
+   * - `./README.md`
+   * - `./scenario-1/README.md`
+   * - `./scenario-1/1.success.md`
+   * - `./scenario-1/2.failure.md`
+   * - `./scenario-1/3.error.md`
+   *
+   * @returns Dictionary of markdown files.
+   */
+  public report(): Record<string, string> {
+    if (this.result_ === null) {
+      throw new Error("Benchmark is not executed yet.");
+    }
+    return AgenticaCallBenchmarkReporter.markdown(this.result_);
+  }
+
+  private async step(
+    scenario: IAgenticaCallBenchmarkScenario<Model>,
+  ): Promise<IAgenticaCallBenchmarkEvent<Model>> {
+    const agent: MicroAgentica<Model> = this.agent_.clone();
+    const started_at: Date = new Date();
+    const success = () =>
+      AgenticaBenchmarkPredicator.success({
+        expected: scenario.expected,
+        operations: agent
+          .getHistories()
+          .filter(p => p.type === "execute")
+          .map(p => p.operation),
+        strict: false,
+      });
+    const out = (): IAgenticaCallBenchmarkEvent<Model> => {
+      const select = AgenticaBenchmarkPredicator.success({
+        expected: scenario.expected,
+        operations: agent
+          .getHistories()
+          .filter(p => p.type === "execute")
+          .map(p => p.operation),
+        strict: false,
+      });
+      const call = success();
+      return {
+        type: (call ? "success" : "failure") as "failure",
+        scenario,
+        select,
+        call,
+        prompts: agent.getHistories(),
+        usage: agent.getTokenUsage(),
+        started_at,
+        completed_at: new Date(),
+      } satisfies IAgenticaCallBenchmarkEvent.IFailure<Model>;
+    };
+
+    try {
+      await agent.conversate(scenario.text);
+      if (success()) {
+        return out();
+      }
+
+      for (let i: number = 0; i < this.config_.consent; ++i) {
+        const next: string | null
+          = await AgenticaBenchmarkPredicator.isNext(agent);
+        if (next === null) {
+          break;
+        }
+
+        await agent.conversate(next);
+        if (success()) {
+          return out();
+        }
+      }
+      return out();
+    }
+    catch (error) {
+      return {
+        type: "error",
+        scenario,
+        prompts: agent.getHistories(),
+        usage: agent.getTokenUsage(),
+        error,
+        started_at,
+        completed_at: new Date(),
+      };
+    }
+  }
+}
+export namespace MicroAgenticaCallBenchmark {
+  /**
+   * Properties of the {@link MicroAgenticaCallBenchmark} constructor.
+   */
+  export interface IProps<Model extends ILlmSchema.Model> {
+    /**
+     * AI agent instance.
+     */
+    agent: MicroAgentica<Model>;
+
+    /**
+     * List of scenarios what you expect.
+     */
+    scenarios: IAgenticaCallBenchmarkScenario<Model>[];
+
+    /**
+     * Configuration for the benchmark.
+     */
+    config?: Partial<IConfig>;
+  }
+
+  /**
+   * Configuration for the benchmark.
+   *
+   * `AgenticaSelectBenchmark.IConfig` is a data structure which
+   * represents a configuration for the benchmark, especially the
+   * capacity information of the benchmark execution.
+   */
+  export interface IConfig {
+    /**
+     * Repeat count.
+     *
+     * The number of repeating count for the benchmark execution
+     * for each scenario.
+     *
+     * @default 10
+     */
+    repeat: number & tags.Type<"uint32"> & tags.Minimum<1>;
+
+    /**
+     * Simultaneous count.
+     *
+     * The number of simultaneous count for the parallel benchmark
+     * execution.
+     *
+     * If you configure this property greater than `1`, the benchmark
+     * for each scenario would be executed in parallel in the given
+     * count.
+     *
+     * @default 10
+     */
+    simultaneous: number & tags.Type<"uint32"> & tags.Minimum<1>;
+
+    /**
+     * Number of consents.
+     *
+     * AI agent sometimes asks user to consent to the function
+     * calling, and perform it at the next step.
+     *
+     * This property represents the number of consents to allow.
+     * If the number of consents from the AI agent exceeds the
+     * configured value, the benchmark will be failed.
+     *
+     * @default 3
+     */
+    consent: number;
+  }
+}

package/src/index.ts

@@ -1,3 +1,3 @@
 export * from "./AgenticaCallBenchmark";
-
+export * from "./MicroAgenticaCallBenchmark";
 export * from "./AgenticaSelectBenchmark";

package/src/internal/AgenticaBenchmarkPredicator.ts

@@ -5,7 +5,7 @@
  * @author Wrtn Technologies
  */
 
-import type { Agentica, AgenticaHistory, AgenticaOperation } from "@agentica/core";
+import type { Agentica, AgenticaHistory, AgenticaOperation, MicroAgentica } from "@agentica/core";
 import type { ILlmFunction, ILlmSchema } from "@samchon/openapi";
 import type OpenAI from "openai";
 
@@ -50,7 +50,7 @@ interface IConsentProps {
   reply: string;
 }
 
-async function isNext<Model extends ILlmSchema.Model>(agent: Agentica<Model>): Promise<string | null> {
+async function isNext<Model extends ILlmSchema.Model>(agent: Agentica<Model> | MicroAgentica<Model>): Promise<string | null> {
   const last: AgenticaHistory<Model> | undefined = agent
     .getHistories()
     .at(-1);