@midscene/core 0.27.1-beta-20250822053848.0 → 0.27.1-beta-20250822103738.0

package/dist/lib/yaml/utils.js

@@ -0,0 +1,128 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.n = (module)=>{
+        var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
+        __webpack_require__.d(getter, {
+            a: getter
+        });
+        return getter;
+    };
+})();
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    interpolateEnvVars: ()=>interpolateEnvVars,
+    parseYamlScript: ()=>parseYamlScript,
+    buildDetailedLocateParam: ()=>buildDetailedLocateParam,
+    buildDetailedLocateParamAndRestParams: ()=>buildDetailedLocateParamAndRestParams
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const external_js_yaml_namespaceObject = require("js-yaml");
+var external_js_yaml_default = /*#__PURE__*/ __webpack_require__.n(external_js_yaml_namespaceObject);
+const debugUtils = (0, logger_namespaceObject.getDebug)('yaml:utils');
+function interpolateEnvVars(content) {
+    return content.replace(/\$\{([^}]+)\}/g, (_, envVar)=>{
+        const value = process.env[envVar.trim()];
+        if (void 0 === value) throw new Error(`Environment variable "${envVar.trim()}" is not defined`);
+        return value;
+    });
+}
+function parseYamlScript(content, filePath, ignoreCheckingTarget) {
+    let processedContent = content;
+    if (-1 !== content.indexOf('android') && content.match(/deviceId:\s*(\d+)/)) {
+        let matchedDeviceId;
+        processedContent = content.replace(/deviceId:\s*(\d+)/g, (match, deviceId)=>{
+            matchedDeviceId = deviceId;
+            return `deviceId: '${deviceId}'`;
+        });
+        console.warn(`please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`);
+    }
+    const interpolatedContent = interpolateEnvVars(processedContent);
+    const obj = external_js_yaml_default().load(interpolatedContent, {
+        schema: external_js_yaml_default().JSON_SCHEMA
+    });
+    const pathTip = filePath ? `, failed to load ${filePath}` : '';
+    const android = void 0 !== obj.android ? Object.assign({}, obj.android || {}) : void 0;
+    const webConfig = obj.web || obj.target;
+    const web = void 0 !== webConfig ? Object.assign({}, webConfig || {}) : void 0;
+    if (!ignoreCheckingTarget) {
+        (0, utils_namespaceObject.assert)(web || android, `at least one of "target", "web", or "android" properties is required in yaml script${pathTip}`);
+        (0, utils_namespaceObject.assert)(web && !android || !web && android, `only one of "target", "web", or "android" properties is allowed in yaml script${pathTip}`);
+        if (web || android) (0, utils_namespaceObject.assert)('object' == typeof web || 'object' == typeof android, `property "target/web/android" must be an object${pathTip}`);
+    }
+    (0, utils_namespaceObject.assert)(obj.tasks, `property "tasks" is required in yaml script ${pathTip}`);
+    (0, utils_namespaceObject.assert)(Array.isArray(obj.tasks), `property "tasks" must be an array in yaml script, but got ${obj.tasks}`);
+    return obj;
+}
+function buildDetailedLocateParam(locatePrompt, opt) {
+    debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);
+    let prompt = locatePrompt || (null == opt ? void 0 : opt.prompt) || (null == opt ? void 0 : opt.locate);
+    let deepThink = false;
+    let cacheable = true;
+    let xpath;
+    if ('object' == typeof opt && null !== opt) {
+        deepThink = opt.deepThink ?? false;
+        cacheable = opt.cacheable ?? true;
+        xpath = opt.xpath;
+        if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) console.warn('conflict prompt for item', locatePrompt, opt, 'maybe you put the prompt in the wrong place');
+        prompt = prompt || opt.prompt;
+    }
+    if (!prompt) return void debugUtils('no prompt, will return undefined in buildDetailedLocateParam', opt);
+    return {
+        prompt,
+        deepThink,
+        cacheable,
+        xpath
+    };
+}
+function buildDetailedLocateParamAndRestParams(locatePrompt, opt, excludeKeys = []) {
+    const locateParam = buildDetailedLocateParam(locatePrompt, opt);
+    const restParams = {};
+    if ('object' == typeof opt && null !== opt) {
+        const allKeys = Object.keys(opt);
+        const locateParamKeys = Object.keys(locateParam || {});
+        for (const key of allKeys)if (!locateParamKeys.includes(key) && !excludeKeys.includes(key) && 'locate' !== key) restParams[key] = opt[key];
+    }
+    return {
+        locateParam,
+        restParams
+    };
+}
+exports.buildDetailedLocateParam = __webpack_exports__.buildDetailedLocateParam;
+exports.buildDetailedLocateParamAndRestParams = __webpack_exports__.buildDetailedLocateParamAndRestParams;
+exports.interpolateEnvVars = __webpack_exports__.interpolateEnvVars;
+exports.parseYamlScript = __webpack_exports__.parseYamlScript;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "buildDetailedLocateParam",
+    "buildDetailedLocateParamAndRestParams",
+    "interpolateEnvVars",
+    "parseYamlScript"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=utils.js.map

package/dist/lib/yaml/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"yaml/utils.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/yaml/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DetailedLocateParam,\n  LocateOption,\n  MidsceneYamlScript,\n  TUserPrompt,\n} from '@/index';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\n\nconst debugUtils = getDebug('yaml:utils');\n\nexport function interpolateEnvVars(content: string): string {\n  return content.replace(/\\$\\{([^}]+)\\}/g, (_, envVar) => {\n    const value = process.env[envVar.trim()];\n    if (value === undefined) {\n      throw new Error(`Environment variable \"${envVar.trim()}\" is not defined`);\n    }\n    return value;\n  });\n}\n\nexport function parseYamlScript(\n  content: string,\n  filePath?: string,\n  ignoreCheckingTarget?: boolean,\n): MidsceneYamlScript {\n  let processedContent = content;\n  if (content.indexOf('android') !== -1 && content.match(/deviceId:\\s*(\\d+)/)) {\n    let matchedDeviceId;\n    processedContent = content.replace(\n      /deviceId:\\s*(\\d+)/g,\n      (match, deviceId) => {\n        matchedDeviceId = deviceId;\n        return `deviceId: '${deviceId}'`;\n      },\n    );\n    console.warn(\n      `please use string-style deviceId in yaml script, for example: deviceId: \"${matchedDeviceId}\"`,\n    );\n  }\n  const interpolatedContent = interpolateEnvVars(processedContent);\n  const obj = yaml.load(interpolatedContent, {\n    schema: yaml.JSON_SCHEMA,\n  }) as MidsceneYamlScript;\n\n  const pathTip = filePath ? `, failed to load ${filePath}` : '';\n  const android =\n    typeof obj.android !== 'undefined'\n      ? Object.assign({}, obj.android || {})\n      : undefined;\n  const webConfig = obj.web || obj.target; // no need to handle null case, because web has required parameters url\n  const web =\n    typeof webConfig !== 'undefined'\n      ? Object.assign({}, webConfig || {})\n      : undefined;\n\n  if (!ignoreCheckingTarget) {\n    // make sure at least one of target/web/android is provided\n    assert(\n      web || android,\n      `at least one of \"target\", \"web\", or \"android\" properties is required in yaml script${pathTip}`,\n    );\n\n    // make sure only one of target/web/android is provided\n    assert(\n      (web && !android) || (!web && android),\n      `only one of \"target\", \"web\", or \"android\" properties is allowed in yaml script${pathTip}`,\n    );\n\n    // make sure the config is valid\n    if (web || android) {\n      assert(\n        typeof web === 'object' || typeof android === 'object',\n        `property \"target/web/android\" must be an object${pathTip}`,\n      );\n    }\n  }\n\n  assert(obj.tasks, `property \"tasks\" is required in yaml script ${pathTip}`);\n  assert(\n    Array.isArray(obj.tasks),\n    `property \"tasks\" must be an array in yaml script, but got ${obj.tasks}`,\n  );\n  return obj;\n}\n\nexport function buildDetailedLocateParam(\n  locatePrompt: TUserPrompt,\n  opt?: LocateOption,\n): DetailedLocateParam | undefined {\n  debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);\n  let prompt = locatePrompt || opt?.prompt || (opt as any)?.locate; // as a shortcut\n  let deepThink = false;\n  let cacheable = true;\n  let xpath = undefined;\n\n  if (typeof opt === 'object' && opt !== null) {\n    deepThink = opt.deepThink ?? false;\n    cacheable = opt.cacheable ?? true;\n    xpath = opt.xpath;\n    if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) {\n      console.warn(\n        'conflict prompt for item',\n        locatePrompt,\n        opt,\n        'maybe you put the prompt in the wrong place',\n      );\n    }\n    prompt = prompt || opt.prompt;\n  }\n\n  if (!prompt) {\n    debugUtils(\n      'no prompt, will return undefined in buildDetailedLocateParam',\n      opt,\n    );\n    return undefined;\n  }\n\n  return {\n    prompt,\n    deepThink,\n    cacheable,\n    xpath,\n  };\n}\n\nexport function buildDetailedLocateParamAndRestParams(\n  locatePrompt: TUserPrompt,\n  opt: LocateOption | undefined,\n  excludeKeys: string[] = [],\n): {\n  locateParam: DetailedLocateParam | undefined;\n  restParams: Record<string, any>;\n} {\n  const locateParam = buildDetailedLocateParam(locatePrompt, opt);\n\n  // Extract all keys from opt except the ones already included in locateParam\n  const restParams: Record<string, any> = {};\n\n  if (typeof opt === 'object' && opt !== null) {\n    // Get all keys from opt\n    const allKeys = Object.keys(opt);\n\n    // Keys already included in locateParam: prompt, deepThink, cacheable, xpath\n    const locateParamKeys = Object.keys(locateParam || {});\n\n    // Extract all other keys\n    for (const key of allKeys) {\n      if (\n        !locateParamKeys.includes(key) &&\n        !excludeKeys.includes(key) &&\n        key !== 'locate'\n      ) {\n        restParams[key] = opt[key as keyof LocateOption];\n      }\n    }\n  }\n\n  return {\n    locateParam,\n    restParams,\n  };\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","debugUtils","getDebug","interpolateEnvVars","content","_","envVar","value","process","undefined","Error","parseYamlScript","filePath","ignoreCheckingTarget","processedContent","matchedDeviceId","match","deviceId","console","interpolatedContent","yaml","pathTip","android","webConfig","web","assert","Array","buildDetailedLocateParam","locatePrompt","opt","prompt","deepThink","cacheable","xpath","buildDetailedLocateParamAndRestParams","excludeKeys","locateParam","restParams","allKeys","locateParamKeys"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACIA,MAAMI,aAAaC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAErB,SAASC,mBAAmBC,OAAe;IAChD,OAAOA,QAAQ,OAAO,CAAC,kBAAkB,CAACC,GAAGC;QAC3C,MAAMC,QAAQC,QAAQ,GAAG,CAACF,OAAO,IAAI,GAAG;QACxC,IAAIC,AAAUE,WAAVF,OACF,MAAM,IAAIG,MAAM,CAAC,sBAAsB,EAAEJ,OAAO,IAAI,GAAG,gBAAgB,CAAC;QAE1E,OAAOC;IACT;AACF;AAEO,SAASI,gBACdP,OAAe,EACfQ,QAAiB,EACjBC,oBAA8B;IAE9B,IAAIC,mBAAmBV;IACvB,IAAIA,AAA+B,OAA/BA,QAAQ,OAAO,CAAC,cAAqBA,QAAQ,KAAK,CAAC,sBAAsB;QAC3E,IAAIW;QACJD,mBAAmBV,QAAQ,OAAO,CAChC,sBACA,CAACY,OAAOC;YACNF,kBAAkBE;YAClB,OAAO,CAAC,WAAW,EAAEA,SAAS,CAAC,CAAC;QAClC;QAEFC,QAAQ,IAAI,CACV,CAAC,yEAAyE,EAAEH,gBAAgB,CAAC,CAAC;IAElG;IACA,MAAMI,sBAAsBhB,mBAAmBW;IAC/C,MAAMhB,MAAMsB,2BAAAA,IAAS,CAACD,qBAAqB;QACzC,QAAQC,AAAAA,2BAAAA,WAAgB;IAC1B;IAEA,MAAMC,UAAUT,WAAW,CAAC,iBAAiB,EAAEA,UAAU,GAAG;IAC5D,MAAMU,UACJ,AAAuB,WAAhBxB,IAAI,OAAO,GACdD,OAAO,MAAM,CAAC,CAAC,GAAGC,IAAI,OAAO,IAAI,CAAC,KAClCW;IACN,MAAMc,YAAYzB,IAAI,GAAG,IAAIA,IAAI,MAAM;IACvC,MAAM0B,MACJ,AAAqB,WAAdD,YACH1B,OAAO,MAAM,CAAC,CAAC,GAAG0B,aAAa,CAAC,KAChCd;IAEN,IAAI,CAACI,sBAAsB;QAEzBY,IAAAA,sBAAAA,MAAAA,AAAAA,EACED,OAAOF,SACP,CAAC,mFAAmF,EAAED,SAAS;QAIjGI,IAAAA,sBAAAA,MAAAA,AAAAA,EACGD,OAAO,CAACF,WAAa,CAACE,OAAOF,SAC9B,CAAC,8EAA8E,EAAED,SAAS;QAI5F,IAAIG,OAAOF,SACTG,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAe,YAAf,OAAOD,OAAoB,AAAmB,YAAnB,OAAOF,SAClC,CAAC,+CAA+C,EAAED,SAAS;IAGjE;IAEAI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO3B,IAAI,KAAK,EAAE,CAAC,4CAA4C,EAAEuB,SAAS;IAC1EI,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,MAAM,OAAO,CAAC5B,IAAI,KAAK,GACvB,CAAC,0DAA0D,EAAEA,IAAI,KAAK,EAAE;IAE1E,OAAOA;AACT;AAEO,SAAS6B,yBACdC,YAAyB,EACzBC,GAAkB;IAElB5B,WAAW,sCAAsC2B,cAAcC;IAC/D,IAAIC,SAASF,gBAAgBC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,MAAM,AAAD,KAAMA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAa,MAAM,AAAD;IAC/D,IAAIE,YAAY;IAChB,IAAIC,YAAY;IAChB,IAAIC;IAEJ,IAAI,AAAe,YAAf,OAAOJ,OAAoBA,AAAQ,SAARA,KAAc;QAC3CE,YAAYF,IAAI,SAAS,IAAI;QAC7BG,YAAYH,IAAI,SAAS,IAAI;QAC7BI,QAAQJ,IAAI,KAAK;QACjB,IAAID,gBAAgBC,IAAI,MAAM,IAAID,iBAAiBC,IAAI,MAAM,EAC3DX,QAAQ,IAAI,CACV,4BACAU,cACAC,KACA;QAGJC,SAASA,UAAUD,IAAI,MAAM;IAC/B;IAEA,IAAI,CAACC,QAAQ,YACX7B,WACE,gEACA4B;IAKJ,OAAO;QACLC;QACAC;QACAC;QACAC;IACF;AACF;AAEO,SAASC,sCACdN,YAAyB,EACzBC,GAA6B,EAC7BM,cAAwB,EAAE;IAK1B,MAAMC,cAAcT,yBAAyBC,cAAcC;IAG3D,MAAMQ,aAAkC,CAAC;IAEzC,IAAI,AAAe,YAAf,OAAOR,OAAoBA,AAAQ,SAARA,KAAc;QAE3C,MAAMS,UAAUzC,OAAO,IAAI,CAACgC;QAG5B,MAAMU,kBAAkB1C,OAAO,IAAI,CAACuC,eAAe,CAAC;QAGpD,KAAK,MAAMxC,OAAO0C,QAChB,IACE,CAACC,gBAAgB,QAAQ,CAAC3C,QAC1B,CAACuC,YAAY,QAAQ,CAACvC,QACtBA,AAAQ,aAARA,KAEAyC,UAAU,CAACzC,IAAI,GAAGiC,GAAG,CAACjC,IAA0B;IAGtD;IAEA,OAAO;QACLwC;QACAC;IACF;AACF"}

package/dist/types/agent/agent.d.ts

@@ -0,0 +1,123 @@
+import { type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentWaitForOpt, type DeviceAction, type ExecutionDump, type GroupedActionDump, Insight, type InsightAction, type InsightExtractOption, type InsightExtractParam, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, type ScrollParam, type TUserPrompt, type UIContext } from '../index';
+import type { AbstractPage } from '../device';
+import { type TModelConfigFn } from '@midscene/shared/env';
+import { TaskCache } from './task-cache';
+import { PageTaskExecutor } from './tasks';
+export interface PageAgentOpt {
+    testId?: string;
+    cacheId?: string;
+    groupName?: string;
+    groupDescription?: string;
+    generateReport?: boolean;
+    autoPrintReportMsg?: boolean;
+    onTaskStartTip?: OnTaskStartTip;
+    aiActionContext?: string;
+    reportFileName?: string;
+    modelConfig?: TModelConfigFn;
+}
+export declare class Agent<PageType extends AbstractPage = AbstractPage> {
+    page: PageType;
+    insight: Insight;
+    dump: GroupedActionDump;
+    reportFile?: string | null;
+    reportFileName?: string;
+    taskExecutor: PageTaskExecutor;
+    opts: PageAgentOpt;
+    /**
+     * If true, the agent will not perform any actions
+     */
+    dryMode: boolean;
+    onTaskStartTip?: OnTaskStartTip;
+    taskCache?: TaskCache;
+    onDumpUpdate?: (dump: string) => void;
+    destroyed: boolean;
+    /**
+     * Frozen page context for consistent AI operations
+     */
+    private frozenPageContext?;
+    constructor(page: PageType, opts?: PageAgentOpt);
+    getActionSpace(): Promise<DeviceAction[]>;
+    getUIContext(action?: InsightAction): Promise<UIContext>;
+    _snapshotContext(): Promise<UIContext>;
+    setAIActionContext(prompt: string): Promise<void>;
+    resetDump(): GroupedActionDump;
+    appendExecutionDump(execution: ExecutionDump): void;
+    dumpDataString(): string;
+    reportHTMLString(): string;
+    writeOutActionDumps(): void;
+    private callbackOnTaskStartTip;
+    private afterTaskRunning;
+    callActionInActionSpace<T = any>(type: string, opt?: T): Promise<any>;
+    aiTap(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiRightClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiHover(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiInput(locatePrompt: TUserPrompt, opt: LocateOption & {
+        value: string;
+    }): Promise<any>;
+    /**
+     * @deprecated Use aiInput(locatePrompt, opt) instead where opt contains the value
+     */
+    aiInput(value: string, locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiKeyboardPress(locatePrompt: TUserPrompt, opt: LocateOption & {
+        keyName: string;
+    }): Promise<any>;
+    /**
+     * @deprecated Use aiKeyboardPress(locatePrompt, opt) instead where opt contains the keyName
+     */
+    aiKeyboardPress(keyName: string, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiScroll(locatePrompt: TUserPrompt | undefined, opt: LocateOption & ScrollParam): Promise<any>;
+    /**
+     * @deprecated Use aiScroll(locatePrompt, opt) instead where opt contains the scroll parameters
+     */
+    aiScroll(scrollParam: ScrollParam, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiAction(taskPrompt: string, opt?: {
+        cacheable?: boolean;
+    }): Promise<{
+        result: Record<string, any>;
+    } | {
+        yamlFlow?: import("../yaml").MidsceneYamlFlowItem[];
+    } | undefined>;
+    aiQuery<ReturnType = any>(demand: InsightExtractParam, opt?: InsightExtractOption): Promise<ReturnType>;
+    aiBoolean(prompt: TUserPrompt, opt?: InsightExtractOption): Promise<boolean>;
+    aiNumber(prompt: TUserPrompt, opt?: InsightExtractOption): Promise<number>;
+    aiString(prompt: TUserPrompt, opt?: InsightExtractOption): Promise<string>;
+    aiAsk(prompt: TUserPrompt, opt?: InsightExtractOption): Promise<string>;
+    describeElementAtPoint(center: [number, number], opt?: {
+        verifyPrompt?: boolean;
+        retryLimit?: number;
+        deepThink?: boolean;
+    } & LocatorValidatorOption): Promise<AgentDescribeElementAtPointResult>;
+    verifyLocator(prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption): Promise<LocateValidatorResult>;
+    aiLocate(prompt: TUserPrompt, opt?: LocateOption): Promise<Pick<LocateResultElement, "center" | "rect"> & {
+        scale: number;
+    }>;
+    aiAssert(assertion: TUserPrompt, msg?: string, opt?: AgentAssertOpt & InsightExtractOption): Promise<{
+        pass: boolean;
+        thought: string | undefined;
+        message: string | undefined;
+    } | undefined>;
+    aiWaitFor(assertion: TUserPrompt, opt?: AgentWaitForOpt): Promise<void>;
+    ai(taskPrompt: string, type?: string): Promise<any>;
+    runYaml(yamlScriptContent: string): Promise<{
+        result: Record<string, any>;
+    }>;
+    evaluateJavaScript(script: string): Promise<any>;
+    destroy(): Promise<void>;
+    logScreenshot(title?: string, opt?: {
+        content: string;
+    }): Promise<void>;
+    _unstableLogContent(): {
+        groupName: string;
+        groupDescription: string | undefined;
+        executions: any[];
+    };
+    /**
+     * Freezes the current page context to be reused in subsequent AI operations
+     * This avoids recalculating page context for each operation
+     */
+    freezePageContext(): Promise<void>;
+    /**
+     * Unfreezes the page context, allowing AI operations to calculate context dynamically
+     */
+    unfreezePageContext(): Promise<void>;
+}

package/dist/types/agent/index.d.ts

@@ -0,0 +1,9 @@
+export { Agent, type PageAgentOpt } from './agent';
+export { commonWebActionsForWebPage, commonContextParser } from './utils';
+export { getReportFileName, printReportMsg, } from './utils';
+export { locateParamStr, paramStr, taskTitleStr, typeStr } from './ui-utils';
+export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
+export { cacheFileExt } from './task-cache';
+export { PageTaskExecutor } from './tasks';
+export { getKeyCommands } from './ui-utils';
+export { getCurrentExecutionFile, trimContextByViewport, } from './utils';

package/dist/types/agent/task-cache.d.ts

@@ -0,0 +1,38 @@
+import type { TUserPrompt } from '../index';
+export declare const debug: import("@midscene/shared/logger").DebugFunction;
+export interface PlanningCache {
+    type: 'plan';
+    prompt: string;
+    yamlWorkflow: string;
+}
+export interface LocateCache {
+    type: 'locate';
+    prompt: TUserPrompt;
+    xpaths: string[];
+}
+export interface MatchCacheResult<T extends PlanningCache | LocateCache> {
+    cacheContent: T;
+    updateFn: (cb: (cache: T) => void) => void;
+}
+export type CacheFileContent = {
+    midsceneVersion: string;
+    cacheId: string;
+    caches: Array<PlanningCache | LocateCache>;
+};
+export declare const cacheFileExt = ".cache.yaml";
+export declare class TaskCache {
+    cacheId: string;
+    cacheFilePath?: string;
+    cache: CacheFileContent;
+    isCacheResultUsed: boolean;
+    cacheOriginalLength: number;
+    private matchedCacheIndices;
+    constructor(cacheId: string, isCacheResultUsed: boolean, cacheFilePath?: string);
+    matchCache(prompt: TUserPrompt, type: 'plan' | 'locate'): MatchCacheResult<PlanningCache | LocateCache> | undefined;
+    matchPlanCache(prompt: string): MatchCacheResult<PlanningCache> | undefined;
+    matchLocateCache(prompt: TUserPrompt): MatchCacheResult<LocateCache> | undefined;
+    appendCache(cache: PlanningCache | LocateCache): void;
+    loadCacheFromFile(): CacheFileContent | undefined;
+    flushCacheToFile(): void;
+    updateOrAppendCacheRecord(newRecord: PlanningCache | LocateCache, cachedRecord?: MatchCacheResult<PlanningCache | LocateCache>): void;
+}

package/dist/types/agent/tasks.d.ts

@@ -0,0 +1,56 @@
+import { type ChatCompletionMessageParam } from '../ai-model';
+import type { AbstractPage } from '../device';
+import { type DetailedLocateParam, type ExecutionTaskApply, type ExecutionTaskProgressOptions, Executor, type Insight, type InsightExtractOption, type InsightExtractParam, type MidsceneYamlFlowItem, type PlanningAction, type PlanningActionParamWaitFor, type PlanningLocateParam, type TMultimodalPrompt, type TUserPrompt } from '../index';
+import type { TaskCache } from './task-cache';
+interface ExecutionResult<OutputType = any> {
+    output: OutputType;
+    thought?: string;
+    executor: Executor;
+}
+export declare function locatePlanForLocate(param: string | DetailedLocateParam): PlanningAction<PlanningLocateParam>;
+export declare class PageTaskExecutor {
+    page: AbstractPage;
+    insight: Insight;
+    taskCache?: TaskCache;
+    conversationHistory: ChatCompletionMessageParam[];
+    onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];
+    constructor(page: AbstractPage, insight: Insight, opts: {
+        taskCache?: TaskCache;
+        onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
+    });
+    private recordScreenshot;
+    private getElementXpath;
+    private prependExecutorWithScreenshot;
+    convertPlanToExecutable(plans: PlanningAction[]): Promise<{
+        tasks: ExecutionTaskApply<any, any, any, any>[];
+    }>;
+    private setupPlanningContext;
+    loadYamlFlowAsPlanning(userInstruction: string, yamlString: string): Promise<{
+        executor: Executor;
+    }>;
+    private planningTaskFromPrompt;
+    private planningTaskToGoal;
+    runPlans(title: string, plans: PlanningAction[]): Promise<ExecutionResult>;
+    action(userPrompt: string, actionContext?: string): Promise<ExecutionResult<{
+        yamlFlow?: MidsceneYamlFlowItem[];
+    } | undefined>>;
+    actionToGoal(userPrompt: string): Promise<ExecutionResult<{
+        yamlFlow?: MidsceneYamlFlowItem[];
+    } | undefined>>;
+    private createTypeQueryTask;
+    createTypeQueryExecution<T>(type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert', demand: InsightExtractParam, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<ExecutionResult<T>>;
+    assert(assertion: TUserPrompt, opt?: InsightExtractOption): Promise<ExecutionResult<boolean>>;
+    /**
+     * Append a message to the conversation history
+     * For user messages with images:
+     * - Keep max 4 user image messages in history
+     * - Remove oldest user image message when limit reached
+     * For assistant messages:
+     * - Simply append to history
+     * @param conversationHistory Message to append
+     */
+    private appendConversationHistory;
+    private appendErrorPlan;
+    waitFor(assertion: TUserPrompt, opt: PlanningActionParamWaitFor): Promise<ExecutionResult<void>>;
+}
+export {};

package/dist/types/agent/ui-utils.d.ts

@@ -0,0 +1,11 @@
+import type { AndroidPullParam, DetailedLocateParam, ExecutionTask, ScrollParam } from '../index';
+export declare function typeStr(task: ExecutionTask): any;
+export declare function getKeyCommands(value: string | string[]): Array<{
+    key: string;
+    command?: string;
+}>;
+export declare function locateParamStr(locate?: DetailedLocateParam): string;
+export declare function scrollParamStr(scrollParam?: ScrollParam): string;
+export declare function pullParamStr(pullParam?: AndroidPullParam): string;
+export declare function taskTitleStr(type: 'Tap' | 'Hover' | 'Input' | 'RightClick' | 'KeyboardPress' | 'Scroll' | 'Action' | 'Query' | 'Assert' | 'WaitFor' | 'Locate' | 'Boolean' | 'Number' | 'String', prompt: string): string;
+export declare function paramStr(task: ExecutionTask): string;

package/dist/types/agent/utils.d.ts

@@ -0,0 +1,49 @@
+import type { AbstractPage } from '../device';
+import type { BaseElement, DeviceAction, ElementTreeNode, ExecutionDump, ExecutorContext, PlanningLocateParam, TMultimodalPrompt, TUserPrompt, UIContext } from '../index';
+import type { PageTaskExecutor } from './tasks';
+export declare function commonContextParser(page: AbstractPage): Promise<UIContext>;
+export declare function getReportFileName(tag?: string): string;
+export declare function printReportMsg(filepath: string): void;
+/**
+ * Get the current execution file name
+ * @returns The name of the current execution file
+ */
+export declare function getCurrentExecutionFile(trace?: string): string | false;
+export declare function generateCacheId(fileName?: string): string;
+export declare function matchElementFromPlan(planLocateParam: PlanningLocateParam, tree: ElementTreeNode<BaseElement>): any;
+export declare function matchElementFromCache(taskExecutor: PageTaskExecutor, xpaths: string[] | undefined, cachePrompt: TUserPrompt, cacheable: boolean | undefined): Promise<any>;
+export declare function trimContextByViewport(execution: ExecutionDump): {
+    tasks: {
+        type: any;
+        subType?: string;
+        param?: any;
+        thought?: string;
+        locate?: PlanningLocateParam | null;
+        pageContext?: UIContext;
+        executor: (param: any, context: ExecutorContext) => void | Promise<void | import("../index").ExecutionTaskReturn<any, any> | undefined> | undefined;
+        output?: any;
+        log?: any;
+        recorder?: import("../index").ExecutionRecorderItem[];
+        hitBy?: import("../index").ExecutionTaskHitBy;
+        status: "pending" | "running" | "finished" | "failed" | "cancelled";
+        error?: Error;
+        errorMessage?: string;
+        errorStack?: string;
+        timing?: {
+            start: number;
+            end?: number;
+            cost?: number;
+        };
+        usage?: import("../index").AIUsageInfo;
+    }[];
+    name: string;
+    description?: string;
+    sdkVersion: string;
+    logTime: number;
+};
+export declare const getMidsceneVersion: () => string;
+export declare const parsePrompt: (prompt: TUserPrompt) => {
+    textPrompt: string;
+    multimodalPrompt?: TMultimodalPrompt;
+};
+export declare const commonWebActionsForWebPage: <T extends AbstractPage>(page: T) => DeviceAction<any>[];

package/dist/types/device/index.d.ts

@@ -0,0 +1,55 @@
+import type { ElementInfo, ElementNode } from '@midscene/shared/extractor';
+import type { DeviceAction, Point, Size, UIContext } from '../types';
+export declare type KeyInput = '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'Power' | 'Eject' | 'Abort' | 'Help' | 'Backspace' | 'Tab' | 'Numpad5' | 'NumpadEnter' | 'Enter' | '\r' | '\n' | 'ShiftLeft' | 'ShiftRight' | 'ControlLeft' | 'ControlRight' | 'AltLeft' | 'AltRight' | 'Pause' | 'CapsLock' | 'Escape' | 'Convert' | 'NonConvert' | 'Space' | 'Numpad9' | 'PageUp' | 'Numpad3' | 'PageDown' | 'End' | 'Numpad1' | 'Home' | 'Numpad7' | 'ArrowLeft' | 'Numpad4' | 'Numpad8' | 'ArrowUp' | 'ArrowRight' | 'Numpad6' | 'Numpad2' | 'ArrowDown' | 'Select' | 'Open' | 'PrintScreen' | 'Insert' | 'Numpad0' | 'Delete' | 'NumpadDecimal' | 'Digit0' | 'Digit1' | 'Digit2' | 'Digit3' | 'Digit4' | 'Digit5' | 'Digit6' | 'Digit7' | 'Digit8' | 'Digit9' | 'KeyA' | 'KeyB' | 'KeyC' | 'KeyD' | 'KeyE' | 'KeyF' | 'KeyG' | 'KeyH' | 'KeyI' | 'KeyJ' | 'KeyK' | 'KeyL' | 'KeyM' | 'KeyN' | 'KeyO' | 'KeyP' | 'KeyQ' | 'KeyR' | 'KeyS' | 'KeyT' | 'KeyU' | 'KeyV' | 'KeyW' | 'KeyX' | 'KeyY' | 'KeyZ' | 'MetaLeft' | 'MetaRight' | 'ContextMenu' | 'NumpadMultiply' | 'NumpadAdd' | 'NumpadSubtract' | 'NumpadDivide' | 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | 'F6' | 'F7' | 'F8' | 'F9' | 'F10' | 'F11' | 'F12' | 'F13' | 'F14' | 'F15' | 'F16' | 'F17' | 'F18' | 'F19' | 'F20' | 'F21' | 'F22' | 'F23' | 'F24' | 'NumLock' | 'ScrollLock' | 'AudioVolumeMute' | 'AudioVolumeDown' | 'AudioVolumeUp' | 'MediaTrackNext' | 'MediaTrackPrevious' | 'MediaStop' | 'MediaPlayPause' | 'Semicolon' | 'Equal' | 'NumpadEqual' | 'Comma' | 'Minus' | 'Period' | 'Slash' | 'Backquote' | 'BracketLeft' | 'Backslash' | 'BracketRight' | 'Quote' | 'AltGraph' | 'Props' | 'Cancel' | 'Clear' | 'Shift' | 'Control' | 'Alt' | 'Accept' | 'ModeChange' | ' ' | 'Print' | 'Execute' | '\u0000' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'Meta' | '*' | '+' | '-' | '/' | ';' | '=' | ',' | '.' | '`' | '[' | '\\' | ']' | "'" | 'Attn' | 'CrSel' | 'ExSel' | 'EraseEof' | 'Play' | 'ZoomOut' | ')' | '!' | '@' | '#' | '$' | '%' | '^' | '&' | '(' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | ':' | '<' | '_' | '>' | '?' | '~' | '{' | '|' | '}' | '"' | 'SoftLeft' | 'SoftRight' | 'Camera' | 'Call' | 'EndCall' | 'VolumeDown' | 'VolumeUp';
+export type MouseButton = 'left' | 'right' | 'middle';
+export interface MouseAction {
+    click: (x: number, y: number, options: {
+        button: MouseButton;
+    }) => Promise<void>;
+    wheel: (deltaX: number, deltaY: number) => Promise<void>;
+    move: (x: number, y: number) => Promise<void>;
+    drag: (from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }) => Promise<void>;
+}
+export interface KeyboardAction {
+    type: (text: string) => Promise<void>;
+    press: (action: {
+        key: KeyInput;
+        command?: string;
+    } | {
+        key: KeyInput;
+        command?: string;
+    }[]) => Promise<void>;
+}
+export interface ChromePageDestroyOptions {
+    closeTab?: boolean;
+}
+export declare abstract class AbstractPage {
+    abstract pageType: string;
+    abstract getElementsInfo(): Promise<ElementInfo[]>;
+    abstract getElementsNodeTree(): Promise<ElementNode>;
+    abstract url(): string | Promise<string>;
+    abstract screenshotBase64(): Promise<string>;
+    abstract size(): Promise<Size>;
+    abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;
+    get mouse(): MouseAction;
+    get keyboard(): KeyboardAction;
+    clearInput(element: ElementInfo): Promise<void>;
+    abstract scrollUntilTop(startingPoint?: Point): Promise<void>;
+    abstract scrollUntilBottom(startingPoint?: Point): Promise<void>;
+    abstract scrollUntilLeft(startingPoint?: Point): Promise<void>;
+    abstract scrollUntilRight(startingPoint?: Point): Promise<void>;
+    abstract scrollUp(distance?: number, startingPoint?: Point): Promise<void>;
+    abstract scrollDown(distance?: number, startingPoint?: Point): Promise<void>;
+    abstract scrollLeft(distance?: number, startingPoint?: Point): Promise<void>;
+    abstract scrollRight(distance?: number, startingPoint?: Point): Promise<void>;
+    abstract beforeAction?(): Promise<void>;
+    abstract destroy(options?: ChromePageDestroyOptions): Promise<void>;
+    abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;
+    abstract getContext?(): Promise<UIContext>;
+}

package/dist/types/index.d.ts

@@ -9,3 +9,4 @@ export { z };
 export default Insight;
 export { Executor, Insight, getVersion };
 export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlFlowItemAIRightClick, MidsceneYamlConfigResult, LocateOption, DetailedLocateParam, } from './yaml';
+export { Agent, type PageAgentOpt } from './agent';

package/dist/types/types.d.ts

@@ -96,6 +96,7 @@ export declare abstract class UIContext<ElementType extends BaseElement = BaseEl
     abstract tree: ElementTreeNode<ElementType>;
     abstract size: Size;
     abstract _isFrozen?: boolean;
+    abstract url?: string;
 }
 /**
  * insight

package/dist/types/yaml/builder.d.ts

@@ -0,0 +1,2 @@
+import type { MidsceneYamlScriptWebEnv, MidsceneYamlTask } from '../types';
+export declare function buildYaml(env: MidsceneYamlScriptWebEnv, tasks: MidsceneYamlTask[]): string;

package/dist/types/yaml/index.d.ts

@@ -0,0 +1,3 @@
+export * from './player';
+export * from './builder';
+export * from './utils';

package/dist/types/yaml/player.d.ts

@@ -0,0 +1,34 @@
+import type { Agent } from '../agent/agent';
+import type { FreeFn, MidsceneYamlScript, MidsceneYamlScriptEnv, ScriptPlayerStatusValue, ScriptPlayerTaskStatus } from '../index';
+export declare class ScriptPlayer<T extends MidsceneYamlScriptEnv> {
+    private script;
+    private setupAgent;
+    onTaskStatusChange?: ((taskStatus: ScriptPlayerTaskStatus) => void) | undefined;
+    currentTaskIndex?: number;
+    taskStatusList: ScriptPlayerTaskStatus[];
+    status: ScriptPlayerStatusValue;
+    reportFile?: string | null;
+    result: Record<string, any>;
+    private unnamedResultIndex;
+    output?: string | null;
+    unstableLogContent?: string | null;
+    errorInSetup?: Error;
+    private pageAgent;
+    agentStatusTip?: string;
+    target?: MidsceneYamlScriptEnv;
+    private actionSpace;
+    private scriptPath?;
+    constructor(script: MidsceneYamlScript, setupAgent: (platform: T) => Promise<{
+        agent: Agent;
+        freeFn: FreeFn[];
+    }>, onTaskStatusChange?: ((taskStatus: ScriptPlayerTaskStatus) => void) | undefined, scriptPath?: string);
+    private setResult;
+    private setPlayerStatus;
+    private notifyCurrentTaskStatusChange;
+    private setTaskStatus;
+    private setTaskIndex;
+    private flushResult;
+    private flushUnstableLogContent;
+    playTask(taskStatus: ScriptPlayerTaskStatus, agent: Agent): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/types/yaml/utils.d.ts

@@ -0,0 +1,8 @@
+import type { DetailedLocateParam, LocateOption, MidsceneYamlScript, TUserPrompt } from '../index';
+export declare function interpolateEnvVars(content: string): string;
+export declare function parseYamlScript(content: string, filePath?: string, ignoreCheckingTarget?: boolean): MidsceneYamlScript;
+export declare function buildDetailedLocateParam(locatePrompt: TUserPrompt, opt?: LocateOption): DetailedLocateParam | undefined;
+export declare function buildDetailedLocateParamAndRestParams(locatePrompt: TUserPrompt, opt: LocateOption | undefined, excludeKeys?: string[]): {
+    locateParam: DetailedLocateParam | undefined;
+    restParams: Record<string, any>;
+};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.27.1-beta-20250822053848.0",
+  "version": "0.27.1-beta-20250822103738.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -31,6 +31,21 @@
       "types": "./dist/types/tree.d.ts",
       "import": "./dist/es/tree.mjs",
       "require": "./dist/lib/tree.js"
+    },
+    "./device": {
+      "types": "./dist/types/device/index.d.ts",
+      "import": "./dist/es/device/index.mjs",
+      "require": "./dist/lib/device/index.js"
+    },
+    "./agent": {
+      "types": "./dist/types/agent/index.d.ts",
+      "import": "./dist/es/agent/index.mjs",
+      "require": "./dist/lib/agent/index.js"
+    },
+    "./yaml": {
+      "types": "./dist/types/yaml/index.d.ts",
+      "import": "./dist/es/yaml/index.mjs",
+      "require": "./dist/lib/yaml/index.js"
     }
   },
   "typesVersions": {
@@ -46,6 +61,15 @@
       ],
       "tree": [
         "./dist/types/tree.d.ts"
+      ],
+      "device": [
+        "./dist/types/device.d.ts"
+      ],
+      "agent": [
+        "./dist/types/agent.d.ts"
+      ],
+      "yaml": [
+        "./dist/types/yaml.d.ts"
       ]
     }
   },
@@ -63,8 +87,8 @@
     "zod": "3.24.3",
     "semver": "7.5.2",
     "js-yaml": "4.1.0",
-    "@midscene/recorder": "0.27.1-beta-20250822053848.0",
-    "@midscene/shared": "0.27.1-beta-20250822053848.0"
+    "@midscene/recorder": "0.27.1-beta-20250822103738.0",
+    "@midscene/shared": "0.27.1-beta-20250822103738.0"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.52.10",