@aiscene/core 1.7.0

package/dist/lib/agent/agent.js

@@ -0,0 +1,797 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.n = (module)=>{
+        var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
+        __webpack_require__.d(getter, {
+            a: getter
+        });
+        return getter;
+    };
+})();
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    Agent: ()=>Agent,
+    createAgent: ()=>createAgent
+});
+const external_screenshot_item_js_namespaceObject = require("../screenshot-item.js");
+const index_js_namespaceObject = require("../service/index.js");
+var index_js_default = /*#__PURE__*/ __webpack_require__.n(index_js_namespaceObject);
+const external_types_js_namespaceObject = require("../types.js");
+const util_js_namespaceObject = require("../ai-model/auto-glm/util.js");
+const external_js_yaml_namespaceObject = require("js-yaml");
+var external_js_yaml_default = /*#__PURE__*/ __webpack_require__.n(external_js_yaml_namespaceObject);
+const external_report_generator_js_namespaceObject = require("../report-generator.js");
+const external_utils_js_namespaceObject = require("../utils.js");
+const external_yaml_index_js_namespaceObject = require("../yaml/index.js");
+const external_node_fs_namespaceObject = require("node:fs");
+const external_node_path_namespaceObject = require("node:path");
+const env_namespaceObject = require("@midscene/shared/env");
+const logger_namespaceObject = require("@midscene/shared/logger");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const external_device_index_js_namespaceObject = require("../device/index.js");
+const external_task_cache_js_namespaceObject = require("./task-cache.js");
+const external_tasks_js_namespaceObject = require("./tasks.js");
+const external_ui_utils_js_namespaceObject = require("./ui-utils.js");
+const external_utils_js_namespaceObject_1 = require("./utils.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = (0, logger_namespaceObject.getDebug)('agent');
+const distanceOfTwoPoints = (p1, p2)=>{
+    const [x1, y1] = p1;
+    const [x2, y2] = p2;
+    return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));
+};
+const includedInRect = (point, rect)=>{
+    const [x, y] = point;
+    const { left, top, width, height } = rect;
+    return x >= left && x <= left + width && y >= top && y <= top + height;
+};
+const defaultServiceExtractOption = {
+    domIncluded: false,
+    screenshotIncluded: true
+};
+const CACHE_STRATEGIES = [
+    'read-only',
+    'read-write',
+    'write-only'
+];
+const isValidCacheStrategy = (strategy)=>CACHE_STRATEGIES.some((value)=>value === strategy);
+const CACHE_STRATEGY_VALUES = CACHE_STRATEGIES.map((value)=>`"${value}"`).join(', ');
+const legacyScrollTypeMap = {
+    once: 'singleAction',
+    untilBottom: 'scrollToBottom',
+    untilTop: 'scrollToTop',
+    untilRight: 'scrollToRight',
+    untilLeft: 'scrollToLeft'
+};
+const normalizeScrollType = (scrollType)=>{
+    if (!scrollType) return scrollType;
+    if (scrollType in legacyScrollTypeMap) return legacyScrollTypeMap[scrollType];
+    return scrollType;
+};
+const defaultReplanningCycleLimit = 20;
+const defaultVlmUiTarsReplanningCycleLimit = 40;
+const defaultAutoGlmReplanningCycleLimit = 100;
+class Agent {
+    get onDumpUpdate() {
+        return this.dumpUpdateListeners[0];
+    }
+    set onDumpUpdate(callback) {
+        this.dumpUpdateListeners = [];
+        if (callback) this.dumpUpdateListeners.push(callback);
+    }
+    get aiActContext() {
+        return this.opts.aiActContext ?? this.opts.aiActionContext;
+    }
+    get page() {
+        return this.interface;
+    }
+    ensureVLModelWarning() {
+        if (!this.hasWarnedNonVLModel && 'puppeteer' !== this.interface.interfaceType && 'playwright' !== this.interface.interfaceType && 'static' !== this.interface.interfaceType && 'chrome-extension-proxy' !== this.interface.interfaceType && 'page-over-chrome-extension-bridge' !== this.interface.interfaceType) {
+            this.modelConfigManager.throwErrorIfNonVLModel();
+            this.hasWarnedNonVLModel = true;
+        }
+    }
+    resolveReplanningCycleLimit(modelConfigForPlanning) {
+        if (void 0 !== this.opts.replanningCycleLimit) return this.opts.replanningCycleLimit;
+        return (0, util_js_namespaceObject.isUITars)(modelConfigForPlanning.modelFamily) ? defaultVlmUiTarsReplanningCycleLimit : (0, util_js_namespaceObject.isAutoGLM)(modelConfigForPlanning.modelFamily) ? defaultAutoGlmReplanningCycleLimit : defaultReplanningCycleLimit;
+    }
+    async getActionSpace() {
+        return this.fullActionSpace;
+    }
+    isRetryableContextError(_error) {
+        return false;
+    }
+    async getUIContext(action) {
+        this.ensureVLModelWarning();
+        if (this.frozenUIContext) {
+            debug('Using frozen page context for action:', action);
+            return this.frozenUIContext;
+        }
+        const { modelFamily } = this.modelConfigManager.getModelConfig('default');
+        const maxRetries = Agent.CONTEXT_RETRY_MAX;
+        for(let attempt = 0;; attempt++)try {
+            return await (0, external_utils_js_namespaceObject_1.commonContextParser)(this.interface, {
+                uploadServerUrl: this.modelConfigManager.getUploadTestServerUrl(),
+                screenshotShrinkFactor: this.opts.screenshotShrinkFactor,
+                modelFamily
+            });
+        } catch (error) {
+            if (attempt < maxRetries && this.isRetryableContextError(error)) {
+                debug(`retryable context error (attempt ${attempt + 1}/${maxRetries}), retrying in ${Agent.CONTEXT_RETRY_DELAY_MS}ms: ${error}`);
+                await new Promise((resolve)=>setTimeout(resolve, Agent.CONTEXT_RETRY_DELAY_MS));
+                continue;
+            }
+            throw error;
+        }
+    }
+    async _snapshotContext() {
+        return await this.getUIContext('locate');
+    }
+    async setAIActionContext(prompt) {
+        await this.setAIActContext(prompt);
+    }
+    async setAIActContext(prompt) {
+        if (this.aiActContext) console.warn('aiActContext is already set, and it is called again, will override the previous setting');
+        this.opts.aiActContext = prompt;
+        this.opts.aiActionContext = prompt;
+    }
+    resetDump() {
+        this.dump = new external_types_js_namespaceObject.GroupedActionDump({
+            sdkVersion: (0, external_utils_js_namespaceObject.getVersion)(),
+            groupName: this.opts.groupName,
+            groupDescription: this.opts.groupDescription,
+            executions: [],
+            modelBriefs: [],
+            deviceType: this.interface.interfaceType
+        });
+        this.executionDumpIndexByRunner = new WeakMap();
+        return this.dump;
+    }
+    appendExecutionDump(execution, runner) {
+        const currentDump = this.dump;
+        if (runner) {
+            const existingIndex = this.executionDumpIndexByRunner.get(runner);
+            if (void 0 !== existingIndex) {
+                currentDump.executions[existingIndex] = execution;
+                return;
+            }
+            currentDump.executions.push(execution);
+            this.executionDumpIndexByRunner.set(runner, currentDump.executions.length - 1);
+            return;
+        }
+        currentDump.executions.push(execution);
+    }
+    dumpDataString(opt) {
+        this.dump.groupName = this.opts.groupName;
+        this.dump.groupDescription = this.opts.groupDescription;
+        if (utils_namespaceObject.ifInBrowser || opt?.inlineScreenshots) return this.dump.serializeWithInlineScreenshots();
+        return this.dump.serialize();
+    }
+    reportHTMLString(opt) {
+        return (0, external_utils_js_namespaceObject.reportHTMLContent)(this.dumpDataString(opt));
+    }
+    writeOutActionDumps(executionDump) {
+        const exec = executionDump || this.lastExecutionDump;
+        if (exec) {
+            this.lastExecutionDump = exec;
+            this.reportGenerator.onExecutionUpdate(exec, this.getGroupMeta());
+        }
+        this.reportFile = this.reportGenerator.getReportPath();
+    }
+    getGroupMeta() {
+        return {
+            groupName: this.dump.groupName,
+            groupDescription: this.dump.groupDescription,
+            sdkVersion: this.dump.sdkVersion,
+            modelBriefs: this.dump.modelBriefs,
+            deviceType: this.dump.deviceType
+        };
+    }
+    async callbackOnTaskStartTip(task) {
+        const param = (0, external_ui_utils_js_namespaceObject.paramStr)(task);
+        const tip = param ? `${(0, external_ui_utils_js_namespaceObject.typeStr)(task)} - ${param}` : (0, external_ui_utils_js_namespaceObject.typeStr)(task);
+        if (this.onTaskStartTip) await this.onTaskStartTip(tip);
+    }
+    wrapActionInActionSpace(name) {
+        return async (param)=>await this.callActionInActionSpace(name, param);
+    }
+    async callActionInActionSpace(type, opt) {
+        debug('callActionInActionSpace', type, ',', opt);
+        const actionPlan = {
+            type: type,
+            param: opt || {},
+            thought: ''
+        };
+        debug('actionPlan', actionPlan);
+        const plans = [
+            actionPlan
+        ].filter(Boolean);
+        const title = (0, external_ui_utils_js_namespaceObject.taskTitleStr)(type, (0, external_ui_utils_js_namespaceObject.locateParamStr)(opt?.locate || {}));
+        const defaultIntentModelConfig = this.modelConfigManager.getModelConfig('default');
+        const modelConfigForPlanning = this.modelConfigManager.getModelConfig('planning');
+        const { output } = await this.taskExecutor.runPlans(title, plans, modelConfigForPlanning, defaultIntentModelConfig);
+        return output;
+    }
+    async aiTap(locatePrompt, opt) {
+        (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for tap');
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const fileChooserAccept = opt?.fileChooserAccept ? this.normalizeFileInput(opt.fileChooserAccept) : void 0;
+        return (0, external_tasks_js_namespaceObject.withFileChooser)(this.interface, fileChooserAccept, async ()=>this.callActionInActionSpace('Tap', {
+                locate: detailedLocateParam
+            }));
+    }
+    async aiRightClick(locatePrompt, opt) {
+        (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for right click');
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        return this.callActionInActionSpace('RightClick', {
+            locate: detailedLocateParam
+        });
+    }
+    async aiDoubleClick(locatePrompt, opt) {
+        (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for double click');
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        return this.callActionInActionSpace('DoubleClick', {
+            locate: detailedLocateParam
+        });
+    }
+    async aiHover(locatePrompt, opt) {
+        (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for hover');
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        return this.callActionInActionSpace('Hover', {
+            locate: detailedLocateParam
+        });
+    }
+    async aiInput(locatePromptOrValue, locatePromptOrOpt, optOrUndefined) {
+        let value;
+        let locatePrompt;
+        let opt;
+        if ('object' == typeof locatePromptOrOpt && null !== locatePromptOrOpt && 'value' in locatePromptOrOpt) {
+            locatePrompt = locatePromptOrValue;
+            const optWithValue = locatePromptOrOpt;
+            value = optWithValue.value;
+            opt = optWithValue;
+        } else {
+            value = locatePromptOrValue;
+            locatePrompt = locatePromptOrOpt;
+            opt = {
+                ...optOrUndefined,
+                value
+            };
+        }
+        (0, utils_namespaceObject.assert)('string' == typeof value || 'number' == typeof value, 'input value must be a string or number, use empty string if you want to clear the input');
+        (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for input');
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const stringValue = 'number' == typeof value ? String(value) : value;
+        const mode = opt?.mode === 'append' ? 'typeOnly' : opt?.mode;
+        return this.callActionInActionSpace('Input', {
+            ...opt || {},
+            value: stringValue,
+            locate: detailedLocateParam,
+            mode
+        });
+    }
+    async aiKeyboardPress(locatePromptOrKeyName, locatePromptOrOpt, optOrUndefined) {
+        let keyName;
+        let locatePrompt;
+        let opt;
+        if ('object' == typeof locatePromptOrOpt && null !== locatePromptOrOpt && 'keyName' in locatePromptOrOpt) {
+            locatePrompt = locatePromptOrKeyName;
+            opt = locatePromptOrOpt;
+        } else {
+            keyName = locatePromptOrKeyName;
+            locatePrompt = locatePromptOrOpt;
+            opt = {
+                ...optOrUndefined || {},
+                keyName
+            };
+        }
+        (0, utils_namespaceObject.assert)(opt?.keyName, 'missing keyName for keyboard press');
+        const detailedLocateParam = locatePrompt ? (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt) : void 0;
+        return this.callActionInActionSpace('KeyboardPress', {
+            ...opt || {},
+            locate: detailedLocateParam
+        });
+    }
+    async aiScroll(locatePromptOrScrollParam, locatePromptOrOpt, optOrUndefined) {
+        let scrollParam;
+        let locatePrompt;
+        let opt;
+        const isLocatePromptLike = (value)=>{
+            if ('string' == typeof value || void 0 === value) return true;
+            return 'object' == typeof value && null !== value && 'prompt' in value;
+        };
+        if (isLocatePromptLike(locatePromptOrScrollParam) && 'object' == typeof locatePromptOrOpt && null !== locatePromptOrOpt) {
+            locatePrompt = locatePromptOrScrollParam;
+            opt = locatePromptOrOpt;
+        } else {
+            scrollParam = locatePromptOrScrollParam;
+            locatePrompt = locatePromptOrOpt;
+            opt = {
+                ...optOrUndefined || {},
+                ...scrollParam || {}
+            };
+        }
+        if (opt) {
+            const normalizedScrollType = normalizeScrollType(opt.scrollType);
+            if (normalizedScrollType !== opt.scrollType) opt = {
+                ...opt || {},
+                scrollType: normalizedScrollType
+            };
+        }
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt || '', opt);
+        return this.callActionInActionSpace('Scroll', {
+            ...opt || {},
+            locate: detailedLocateParam
+        });
+    }
+    async aiPinch(locatePrompt, opt) {
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt || '', opt);
+        return this.callActionInActionSpace('Pinch', {
+            ...opt,
+            locate: detailedLocateParam
+        });
+    }
+    async aiAct(taskPrompt, opt) {
+        const fileChooserAccept = opt?.fileChooserAccept ? this.normalizeFileInput(opt.fileChooserAccept) : void 0;
+        const abortSignal = opt?.abortSignal;
+        if (abortSignal?.aborted) throw new Error(`aiAct aborted: ${abortSignal.reason || 'signal already aborted'}`);
+        const runAiAct = async ()=>{
+            const modelConfigForPlanning = this.modelConfigManager.getModelConfig('planning');
+            const defaultIntentModelConfig = this.modelConfigManager.getModelConfig('default');
+            const deepThink = opt?.deepThink === 'unset' ? void 0 : opt?.deepThink;
+            const deepLocate = opt?.deepLocate;
+            const noIndividualLocateModel = modelConfigForPlanning.modelName === defaultIntentModelConfig.modelName && modelConfigForPlanning.openaiBaseURL === defaultIntentModelConfig.openaiBaseURL;
+            const includeBboxInPlanning = !deepThink && noIndividualLocateModel;
+            debug('setting includeBboxInPlanning to', includeBboxInPlanning, {
+                deepThink,
+                noIndividualLocateModel
+            });
+            const cacheable = opt?.cacheable;
+            const replanningCycleLimit = this.resolveReplanningCycleLimit(modelConfigForPlanning);
+            const isVlmUiTars = (0, util_js_namespaceObject.isUITars)(modelConfigForPlanning.modelFamily);
+            const isAutoGlm = (0, util_js_namespaceObject.isAutoGLM)(modelConfigForPlanning.modelFamily);
+            const matchedCache = isVlmUiTars || isAutoGlm || false === cacheable ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
+            if (matchedCache?.cacheUsable && this.taskCache?.isCacheResultUsed && matchedCache.cacheContent?.yamlWorkflow?.trim()) {
+                await this.taskExecutor.loadYamlFlowAsPlanning(taskPrompt, matchedCache.cacheContent.yamlWorkflow);
+                debug('matched cache, will call .runYaml to run the action');
+                const yaml = matchedCache.cacheContent.yamlWorkflow;
+                await this.runYaml(yaml);
+                return;
+            }
+            const imagesIncludeCount = deepThink ? 2 : 1;
+            const { output: actionOutput } = await this.taskExecutor.action(taskPrompt, modelConfigForPlanning, defaultIntentModelConfig, includeBboxInPlanning, this.aiActContext, cacheable, replanningCycleLimit, imagesIncludeCount, deepThink, fileChooserAccept, deepLocate, abortSignal);
+            if (this.taskCache && actionOutput?.yamlFlow?.length && false !== cacheable) {
+                const yamlContent = {
+                    tasks: [
+                        {
+                            name: taskPrompt,
+                            flow: actionOutput.yamlFlow
+                        }
+                    ]
+                };
+                const yamlFlowStr = external_js_yaml_default().dump(yamlContent);
+                this.taskCache.updateOrAppendCacheRecord({
+                    type: 'plan',
+                    prompt: taskPrompt,
+                    yamlWorkflow: yamlFlowStr
+                }, matchedCache);
+            }
+            return actionOutput?.output;
+        };
+        return await runAiAct();
+    }
+    async aiAction(taskPrompt, opt) {
+        return this.aiAct(taskPrompt, opt);
+    }
+    async aiQuery(demand, opt = defaultServiceExtractOption) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
+        const { output } = await this.taskExecutor.createTypeQueryExecution('Query', demand, modelConfig, opt);
+        return output;
+    }
+    async aiBoolean(prompt, opt = defaultServiceExtractOption) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
+        const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
+        const { output } = await this.taskExecutor.createTypeQueryExecution('Boolean', textPrompt, modelConfig, opt, multimodalPrompt);
+        return output;
+    }
+    async aiNumber(prompt, opt = defaultServiceExtractOption) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
+        const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
+        const { output } = await this.taskExecutor.createTypeQueryExecution('Number', textPrompt, modelConfig, opt, multimodalPrompt);
+        return output;
+    }
+    async aiString(prompt, opt = defaultServiceExtractOption) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
+        const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
+        const { output } = await this.taskExecutor.createTypeQueryExecution('String', textPrompt, modelConfig, opt, multimodalPrompt);
+        return output;
+    }
+    async aiAsk(prompt, opt = defaultServiceExtractOption) {
+        return this.aiString(prompt, opt);
+    }
+    async describeElementAtPoint(center, opt) {
+        const { verifyPrompt = true, retryLimit = 3 } = opt || {};
+        let success = false;
+        let retryCount = 0;
+        let resultPrompt = '';
+        let deepLocate = opt?.deepLocate || false;
+        let verifyResult;
+        while(!success && retryCount < retryLimit){
+            if (retryCount >= 2) deepLocate = true;
+            debug('aiDescribe', center, 'verifyPrompt', verifyPrompt, 'retryCount', retryCount, 'deepLocate', deepLocate);
+            const modelConfig = this.modelConfigManager.getModelConfig('insight');
+            const text = await this.service.describe(center, modelConfig, {
+                deepLocate
+            });
+            debug('aiDescribe text', text);
+            (0, utils_namespaceObject.assert)(text.description, `failed to describe element at [${center}]`);
+            resultPrompt = text.description;
+            verifyResult = await this.verifyLocator(resultPrompt, void 0, center, opt);
+            if (verifyResult.pass) success = true;
+            else retryCount++;
+        }
+        return {
+            prompt: resultPrompt,
+            deepLocate,
+            verifyResult
+        };
+    }
+    async verifyLocator(prompt, locateOpt, expectCenter, verifyLocateOption) {
+        debug('verifyLocator', prompt, locateOpt, expectCenter, verifyLocateOption);
+        const { center: verifyCenter, rect: verifyRect } = await this.aiLocate(prompt, locateOpt);
+        const distance = distanceOfTwoPoints(expectCenter, verifyCenter);
+        const included = includedInRect(expectCenter, verifyRect);
+        const pass = distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;
+        const verifyResult = {
+            pass,
+            rect: verifyRect,
+            center: verifyCenter,
+            centerDistance: distance
+        };
+        debug('aiDescribe verifyResult', verifyResult);
+        return verifyResult;
+    }
+    async aiLocate(prompt, opt) {
+        const locateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(prompt, opt);
+        (0, utils_namespaceObject.assert)(locateParam, 'cannot get locate param for aiLocate');
+        const locatePlan = (0, external_tasks_js_namespaceObject.locatePlanForLocate)(locateParam);
+        const plans = [
+            locatePlan
+        ];
+        const defaultIntentModelConfig = this.modelConfigManager.getModelConfig('default');
+        const modelConfigForPlanning = this.modelConfigManager.getModelConfig('planning');
+        const { output } = await this.taskExecutor.runPlans((0, external_ui_utils_js_namespaceObject.taskTitleStr)('Locate', (0, external_ui_utils_js_namespaceObject.locateParamStr)(locateParam)), plans, modelConfigForPlanning, defaultIntentModelConfig);
+        const { element } = output;
+        return {
+            rect: element?.rect,
+            center: element?.center,
+            dpr: element?.dpr
+        };
+    }
+    async aiAssert(assertion, msg, opt) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
+        const serviceOpt = {
+            domIncluded: opt?.domIncluded ?? defaultServiceExtractOption.domIncluded,
+            screenshotIncluded: opt?.screenshotIncluded ?? defaultServiceExtractOption.screenshotIncluded
+        };
+        const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(assertion);
+        const assertionText = 'string' == typeof assertion ? assertion : assertion.prompt;
+        try {
+            const { output, thought } = await this.taskExecutor.createTypeQueryExecution('Assert', textPrompt, modelConfig, serviceOpt, multimodalPrompt);
+            const pass = Boolean(output);
+            const message = pass ? void 0 : `Assertion failed: ${msg || assertionText}\nReason: ${thought || '(no_reason)'}`;
+            if (opt?.keepRawResponse) return {
+                pass,
+                thought,
+                message
+            };
+            if (!pass) throw new Error(message);
+        } catch (error) {
+            if (error instanceof external_tasks_js_namespaceObject.TaskExecutionError) {
+                const errorTask = error.errorTask;
+                const thought = errorTask?.thought;
+                const rawError = errorTask?.error;
+                const rawMessage = errorTask?.errorMessage || (rawError instanceof Error ? rawError.message : rawError ? String(rawError) : void 0);
+                const reason = thought || rawMessage || '(no_reason)';
+                const message = `Assertion failed: ${msg || assertionText}\nReason: ${reason}`;
+                if (opt?.keepRawResponse) return {
+                    pass: false,
+                    thought,
+                    message
+                };
+                throw new Error(message, {
+                    cause: rawError ?? error
+                });
+            }
+            throw error;
+        }
+    }
+    async aiWaitFor(assertion, opt) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
+        await this.taskExecutor.waitFor(assertion, {
+            ...opt,
+            timeoutMs: opt?.timeoutMs || 15000,
+            checkIntervalMs: opt?.checkIntervalMs || 3000
+        }, modelConfig);
+    }
+    async ai(...args) {
+        return this.aiAct(...args);
+    }
+    async runYaml(yamlScriptContent) {
+        const script = (0, external_yaml_index_js_namespaceObject.parseYamlScript)(yamlScriptContent, 'yaml');
+        const player = new external_yaml_index_js_namespaceObject.ScriptPlayer(script, async ()=>({
+                agent: this,
+                freeFn: []
+            }));
+        await player.run();
+        if ('error' === player.status) {
+            const errors = player.taskStatusList.filter((task)=>'error' === task.status).map((task)=>`task - ${task.name}: ${task.error?.message}`).join('\n');
+            throw new Error(`Error(s) occurred in running yaml script:\n${errors}`);
+        }
+        return {
+            result: player.result
+        };
+    }
+    async evaluateJavaScript(script) {
+        (0, utils_namespaceObject.assert)(this.interface.evaluateJavaScript, 'evaluateJavaScript is not supported in current agent');
+        return this.interface.evaluateJavaScript(script);
+    }
+    addDumpUpdateListener(listener) {
+        this.dumpUpdateListeners.push(listener);
+        return ()=>{
+            this.removeDumpUpdateListener(listener);
+        };
+    }
+    removeDumpUpdateListener(listener) {
+        const index = this.dumpUpdateListeners.indexOf(listener);
+        if (index > -1) this.dumpUpdateListeners.splice(index, 1);
+    }
+    clearDumpUpdateListeners() {
+        this.dumpUpdateListeners = [];
+    }
+    async destroy() {
+        if (this.destroyed) return;
+        await this.reportGenerator.flush();
+        const finalPath = await this.reportGenerator.finalize();
+        this.reportFile = finalPath;
+        await this.interface.destroy?.();
+        this.resetDump();
+        this.destroyed = true;
+    }
+    async recordToReport(title, opt) {
+        const base64 = await this.interface.screenshotBase64();
+        const now = Date.now();
+        const screenshot = external_screenshot_item_js_namespaceObject.ScreenshotItem.create(base64, now);
+        const recorder = [
+            {
+                type: 'screenshot',
+                ts: now,
+                screenshot
+            }
+        ];
+        const task = {
+            taskId: (0, utils_namespaceObject.uuid)(),
+            type: 'Log',
+            subType: 'Screenshot',
+            status: 'finished',
+            recorder,
+            timing: {
+                start: now,
+                end: now,
+                cost: 0
+            },
+            param: {
+                content: opt?.content || ''
+            },
+            executor: async ()=>{}
+        };
+        const executionDump = new external_types_js_namespaceObject.ExecutionDump({
+            id: (0, utils_namespaceObject.uuid)(),
+            logTime: now,
+            name: `Log - ${title || 'untitled'}`,
+            description: opt?.content || '',
+            tasks: [
+                task
+            ]
+        });
+        this.appendExecutionDump(executionDump);
+        const dumpString = this.dumpDataString();
+        for (const listener of this.dumpUpdateListeners)try {
+            listener(dumpString);
+        } catch (error) {
+            console.error('Error in onDumpUpdate listener', error);
+        }
+        this.writeOutActionDumps(executionDump);
+        await this.reportGenerator.flush();
+    }
+    async logScreenshot(title, opt) {
+        await this.recordToReport(title, opt);
+    }
+    _unstableLogContent() {
+        const { groupName, groupDescription, executions } = this.dump;
+        return {
+            groupName,
+            groupDescription,
+            executions: executions || []
+        };
+    }
+    async freezePageContext() {
+        debug('Freezing page context');
+        const context = await this._snapshotContext();
+        context._isFrozen = true;
+        this.frozenUIContext = context;
+        debug('Page context frozen successfully');
+    }
+    async unfreezePageContext() {
+        debug('Unfreezing page context');
+        this.frozenUIContext = void 0;
+        debug('Page context unfrozen successfully');
+    }
+    processCacheConfig(opts) {
+        if (true === opts.cache) throw new Error('cache: true requires an explicit cache ID. Please provide:\nExample: cache: { id: "my-cache-id" }');
+        if (opts.cache && 'object' == typeof opts.cache && null !== opts.cache && !opts.cache.id) throw new Error('cache configuration requires an explicit id.\nExample: cache: { id: "my-cache-id" }');
+        const cacheConfig = (0, external_utils_js_namespaceObject.processCacheConfig)(opts.cache, opts.cacheId || opts.testId || 'default');
+        if (!cacheConfig) return null;
+        if ('object' == typeof cacheConfig && null !== cacheConfig) {
+            const id = cacheConfig.id;
+            const rawStrategy = cacheConfig.strategy;
+            let strategyValue;
+            if (void 0 === rawStrategy) strategyValue = 'read-write';
+            else if ('string' == typeof rawStrategy) strategyValue = rawStrategy;
+            else throw new Error(`cache.strategy must be a string when provided, but received type ${typeof rawStrategy}`);
+            if (!isValidCacheStrategy(strategyValue)) throw new Error(`cache.strategy must be one of ${CACHE_STRATEGY_VALUES}, but received "${strategyValue}"`);
+            const isReadOnly = 'read-only' === strategyValue;
+            const isWriteOnly = 'write-only' === strategyValue;
+            return {
+                id,
+                enabled: !isWriteOnly,
+                readOnly: isReadOnly,
+                writeOnly: isWriteOnly
+            };
+        }
+        return null;
+    }
+    normalizeFilePaths(files) {
+        if (utils_namespaceObject.ifInBrowser) throw new Error('File chooser is not supported in browser environment');
+        return files.map((file)=>{
+            const absolutePath = (0, external_node_path_namespaceObject.resolve)(file);
+            if (!(0, external_node_fs_namespaceObject.existsSync)(absolutePath)) throw new Error(`File not found: ${file}`);
+            return absolutePath;
+        });
+    }
+    normalizeFileInput(files) {
+        const filesArray = Array.isArray(files) ? files : [
+            files
+        ];
+        return this.normalizeFilePaths(filesArray);
+    }
+    async flushCache(options) {
+        if (!this.taskCache) throw new Error('Cache is not configured');
+        this.taskCache.flushCacheToFile(options);
+    }
+    constructor(interfaceInstance, opts){
+        _define_property(this, "interface", void 0);
+        _define_property(this, "service", void 0);
+        _define_property(this, "dump", void 0);
+        _define_property(this, "reportFile", void 0);
+        _define_property(this, "reportFileName", void 0);
+        _define_property(this, "taskExecutor", void 0);
+        _define_property(this, "opts", void 0);
+        _define_property(this, "dryMode", false);
+        _define_property(this, "onTaskStartTip", void 0);
+        _define_property(this, "taskCache", void 0);
+        _define_property(this, "dumpUpdateListeners", []);
+        _define_property(this, "destroyed", false);
+        _define_property(this, "modelConfigManager", void 0);
+        _define_property(this, "frozenUIContext", void 0);
+        _define_property(this, "hasWarnedNonVLModel", false);
+        _define_property(this, "executionDumpIndexByRunner", new WeakMap());
+        _define_property(this, "fullActionSpace", void 0);
+        _define_property(this, "reportGenerator", void 0);
+        _define_property(this, "lastExecutionDump", void 0);
+        this.interface = interfaceInstance;
+        const envReplanningCycleLimit = env_namespaceObject.globalConfigManager.getEnvConfigValueAsNumber(env_namespaceObject.MIDSCENE_REPLANNING_CYCLE_LIMIT);
+        this.opts = Object.assign({
+            generateReport: true,
+            autoPrintReportMsg: true,
+            groupName: 'Midscene Report',
+            groupDescription: ''
+        }, opts || {}, opts?.replanningCycleLimit !== void 0 || void 0 === envReplanningCycleLimit || Number.isNaN(envReplanningCycleLimit) ? {} : {
+            replanningCycleLimit: envReplanningCycleLimit
+        });
+        const resolvedAiActContext = this.opts.aiActContext ?? this.opts.aiActionContext;
+        if (void 0 !== resolvedAiActContext) {
+            this.opts.aiActContext = resolvedAiActContext;
+            this.opts.aiActionContext ??= resolvedAiActContext;
+        }
+        if (opts?.modelConfig && ('object' != typeof opts?.modelConfig || Array.isArray(opts.modelConfig))) throw new Error(`opts.modelConfig must be a plain object map of env keys to values, but got ${typeof opts?.modelConfig}`);
+        const hasCustomConfig = opts?.modelConfig || opts?.createOpenAIClient;
+        this.modelConfigManager = hasCustomConfig ? new env_namespaceObject.ModelConfigManager(opts?.modelConfig, opts?.createOpenAIClient) : env_namespaceObject.globalModelConfigManager;
+        this.onTaskStartTip = this.opts.onTaskStartTip;
+        this.service = new (index_js_default())(async ()=>this.getUIContext());
+        const cacheConfigObj = this.processCacheConfig(opts || {});
+        if (cacheConfigObj) this.taskCache = new external_task_cache_js_namespaceObject.TaskCache(cacheConfigObj.id, cacheConfigObj.enabled, void 0, {
+            readOnly: cacheConfigObj.readOnly,
+            writeOnly: cacheConfigObj.writeOnly
+        });
+        const baseActionSpace = this.interface.actionSpace();
+        this.fullActionSpace = [
+            ...baseActionSpace,
+            (0, external_device_index_js_namespaceObject.defineActionSleep)()
+        ];
+        this.taskExecutor = new external_tasks_js_namespaceObject.TaskExecutor(this.interface, this.service, {
+            taskCache: this.taskCache,
+            onTaskStart: this.callbackOnTaskStartTip.bind(this),
+            replanningCycleLimit: this.opts.replanningCycleLimit,
+            waitAfterAction: this.opts.waitAfterAction,
+            useDeviceTimestamp: this.opts.useDeviceTimestamp,
+            actionSpace: this.fullActionSpace,
+            hooks: {
+                onTaskUpdate: (runner)=>{
+                    const executionDump = runner.dump();
+                    this.appendExecutionDump(executionDump, runner);
+                    const dumpString = this.dumpDataString();
+                    for (const listener of this.dumpUpdateListeners)try {
+                        listener(dumpString, executionDump);
+                    } catch (error) {
+                        console.error('Error in onDumpUpdate listener', error);
+                    }
+                    this.writeOutActionDumps(executionDump);
+                }
+            }
+        });
+        this.dump = this.resetDump();
+        this.reportFileName = opts?.reportFileName || (0, external_utils_js_namespaceObject_1.getReportFileName)(opts?.testId || this.interface.interfaceType || 'web');
+        this.reportGenerator = external_report_generator_js_namespaceObject.ReportGenerator.create(this.reportFileName, {
+            generateReport: this.opts.generateReport,
+            outputFormat: this.opts.outputFormat,
+            autoPrintReportMsg: this.opts.autoPrintReportMsg
+        });
+    }
+}
+_define_property(Agent, "CONTEXT_RETRY_MAX", 3);
+_define_property(Agent, "CONTEXT_RETRY_DELAY_MS", 1500);
+const createAgent = (interfaceInstance, opts)=>new Agent(interfaceInstance, opts);
+exports.Agent = __webpack_exports__.Agent;
+exports.createAgent = __webpack_exports__.createAgent;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "Agent",
+    "createAgent"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=agent.js.map