@midscene/core 0.26.5-beta-20250814080504.0 → 0.26.5-beta-20250814125155.0

package/dist/lib/insight/index.js

@@ -0,0 +1,295 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    default: ()=>Insight
+});
+const common_js_namespaceObject = require("../ai-model/common.js");
+const index_js_namespaceObject = require("../ai-model/index.js");
+const inspect_js_namespaceObject = require("../ai-model/inspect.js");
+const describe_js_namespaceObject = require("../ai-model/prompt/describe.js");
+const env_namespaceObject = require("@midscene/shared/env");
+const img_namespaceObject = require("@midscene/shared/img");
+const logger_namespaceObject = require("@midscene/shared/logger");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const external_utils_js_namespaceObject = require("./utils.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = (0, logger_namespaceObject.getDebug)('ai:insight');
+class Insight {
+    async locate(query, opt) {
+        var _parseResult_errors;
+        const { callAI } = opt || {};
+        const queryPrompt = 'string' == typeof query ? query : query.prompt;
+        (0, utils_namespaceObject.assert)(queryPrompt, 'query is required for locate');
+        const dumpSubscriber = this.onceDumpUpdatedFn;
+        this.onceDumpUpdatedFn = void 0;
+        (0, utils_namespaceObject.assert)('object' == typeof query, 'query should be an object for locate');
+        const globalDeepThinkSwitch = (0, env_namespaceObject.getAIConfigInBoolean)(env_namespaceObject.MIDSCENE_FORCE_DEEP_THINK);
+        if (globalDeepThinkSwitch) debug('globalDeepThinkSwitch', globalDeepThinkSwitch);
+        let searchAreaPrompt;
+        if (query.deepThink || globalDeepThinkSwitch) searchAreaPrompt = query.prompt;
+        if (searchAreaPrompt && !(0, env_namespaceObject.vlLocateMode)()) {
+            console.warn('The "deepThink" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model');
+            searchAreaPrompt = void 0;
+        }
+        const context = (null == opt ? void 0 : opt.context) || await this.contextRetrieverFn('locate');
+        let searchArea;
+        let searchAreaRawResponse;
+        let searchAreaUsage;
+        let searchAreaResponse;
+        if (searchAreaPrompt) {
+            searchAreaResponse = await (0, inspect_js_namespaceObject.AiLocateSection)({
+                context,
+                sectionDescription: searchAreaPrompt
+            });
+            (0, utils_namespaceObject.assert)(searchAreaResponse.rect, `cannot find search area for "${searchAreaPrompt}"${searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''}`);
+            searchAreaRawResponse = searchAreaResponse.rawResponse;
+            searchAreaUsage = searchAreaResponse.usage;
+            searchArea = searchAreaResponse.rect;
+        }
+        const startTime = Date.now();
+        const { parseResult, rect, elementById, rawResponse, usage, isOrderSensitive } = await (0, index_js_namespaceObject.AiLocateElement)({
+            callAI: callAI || this.aiVendorFn,
+            context,
+            targetElementDescription: queryPrompt,
+            searchConfig: searchAreaResponse
+        });
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse: JSON.stringify(rawResponse),
+            formatResponse: JSON.stringify(parseResult),
+            usage,
+            searchArea,
+            searchAreaRawResponse,
+            searchAreaUsage
+        };
+        let errorLog;
+        if (null == (_parseResult_errors = parseResult.errors) ? void 0 : _parseResult_errors.length) errorLog = `AI model failed to locate: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'locate',
+            userQuery: {
+                element: queryPrompt
+            },
+            matchedElement: [],
+            matchedRect: rect,
+            data: null,
+            taskInfo,
+            deepThink: !!searchArea,
+            error: errorLog
+        };
+        const elements = [];
+        (parseResult.elements || []).forEach((item)=>{
+            if ('id' in item) {
+                const element = elementById(null == item ? void 0 : item.id);
+                if (!element) return void console.warn(`locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`);
+                elements.push(element);
+            }
+        });
+        (0, external_utils_js_namespaceObject.emitInsightDump)({
+            ...dumpData,
+            matchedElement: elements
+        }, dumpSubscriber);
+        if (errorLog) throw new Error(errorLog);
+        (0, utils_namespaceObject.assert)(elements.length <= 1, `locate: multiple elements found, length = ${elements.length}`);
+        if (1 === elements.length) return {
+            element: {
+                id: elements[0].id,
+                indexId: elements[0].indexId,
+                center: elements[0].center,
+                rect: elements[0].rect,
+                xpaths: elements[0].xpaths || [],
+                attributes: elements[0].attributes,
+                isOrderSensitive
+            },
+            rect
+        };
+        return {
+            element: null,
+            rect
+        };
+    }
+    async extract(dataDemand, opt, multimodalPrompt) {
+        var _parseResult_errors;
+        (0, utils_namespaceObject.assert)('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
+        const dumpSubscriber = this.onceDumpUpdatedFn;
+        this.onceDumpUpdatedFn = void 0;
+        const context = await this.contextRetrieverFn('extract');
+        const startTime = Date.now();
+        const { parseResult, usage } = await (0, index_js_namespaceObject.AiExtractElementInfo)({
+            context,
+            dataQuery: dataDemand,
+            multimodalPrompt,
+            extractOption: opt
+        });
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse: JSON.stringify(parseResult)
+        };
+        let errorLog;
+        if (null == (_parseResult_errors = parseResult.errors) ? void 0 : _parseResult_errors.length) errorLog = `AI response error: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'extract',
+            userQuery: {
+                dataDemand
+            },
+            matchedElement: [],
+            data: null,
+            taskInfo,
+            error: errorLog
+        };
+        const { data, thought } = parseResult || {};
+        (0, external_utils_js_namespaceObject.emitInsightDump)({
+            ...dumpData,
+            data
+        }, dumpSubscriber);
+        if (errorLog && !data) throw new Error(errorLog);
+        return {
+            data,
+            thought,
+            usage
+        };
+    }
+    async assert(assertion) {
+        const dumpSubscriber = this.onceDumpUpdatedFn;
+        this.onceDumpUpdatedFn = void 0;
+        const context = await this.contextRetrieverFn('assert');
+        const startTime = Date.now();
+        const assertResult = await (0, inspect_js_namespaceObject.AiAssert)({
+            assertion,
+            context
+        });
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse: JSON.stringify(assertResult.content)
+        };
+        const { thought, pass } = assertResult.content;
+        const dumpData = {
+            type: 'assert',
+            userQuery: {
+                assertion
+            },
+            matchedElement: [],
+            data: null,
+            taskInfo,
+            assertionPass: pass,
+            assertionThought: thought,
+            error: pass ? void 0 : thought
+        };
+        (0, external_utils_js_namespaceObject.emitInsightDump)(dumpData, dumpSubscriber);
+        return {
+            pass,
+            thought,
+            usage: assertResult.usage
+        };
+    }
+    async describe(target, opt) {
+        (0, utils_namespaceObject.assert)(target, 'target is required for insight.describe');
+        const context = await this.contextRetrieverFn('describe');
+        const { screenshotBase64, size } = context;
+        (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshot is required for insight.describe');
+        const systemPrompt = (0, describe_js_namespaceObject.elementDescriberInstruction)();
+        const defaultRectSize = 30;
+        const targetRect = Array.isArray(target) ? {
+            left: Math.floor(target[0] - defaultRectSize / 2),
+            top: Math.floor(target[1] - defaultRectSize / 2),
+            width: defaultRectSize,
+            height: defaultRectSize
+        } : target;
+        let imagePayload = await (0, img_namespaceObject.compositeElementInfoImg)({
+            inputImgBase64: screenshotBase64,
+            size,
+            elementsPositionInfo: [
+                {
+                    rect: targetRect
+                }
+            ],
+            borderThickness: 3
+        });
+        if (null == opt ? void 0 : opt.deepThink) {
+            const searchArea = (0, common_js_namespaceObject.expandSearchArea)(targetRect, context.size);
+            debug('describe: set searchArea', searchArea);
+            imagePayload = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea, (0, env_namespaceObject.getAIConfigInBoolean)(env_namespaceObject.MIDSCENE_USE_QWEN_VL));
+        }
+        const msgs = [
+            {
+                role: 'system',
+                content: systemPrompt
+            },
+            {
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: imagePayload,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            }
+        ];
+        const callAIFn = this.aiVendorFn || index_js_namespaceObject.callToGetJSONObject;
+        const res = await callAIFn(msgs, common_js_namespaceObject.AIActionType.DESCRIBE_ELEMENT);
+        const { content } = res;
+        (0, utils_namespaceObject.assert)(!content.error, `describe failed: ${content.error}`);
+        (0, utils_namespaceObject.assert)(content.description, 'failed to describe the element');
+        return content;
+    }
+    constructor(context, opt){
+        _define_property(this, "contextRetrieverFn", void 0);
+        _define_property(this, "aiVendorFn", common_js_namespaceObject.callAiFn);
+        _define_property(this, "onceDumpUpdatedFn", void 0);
+        _define_property(this, "taskInfo", void 0);
+        (0, utils_namespaceObject.assert)(context, 'context is required for Insight');
+        if ('function' == typeof context) this.contextRetrieverFn = context;
+        else this.contextRetrieverFn = ()=>Promise.resolve(context);
+        if (void 0 !== (null == opt ? void 0 : opt.aiVendorFn)) this.aiVendorFn = opt.aiVendorFn;
+        if (void 0 !== (null == opt ? void 0 : opt.taskInfo)) this.taskInfo = opt.taskInfo;
+    }
+}
+exports["default"] = __webpack_exports__["default"];
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "default"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=index.js.map

package/dist/lib/insight/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"insight/index.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/insight/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  AIActionType,\n  type AIArgs,\n  callAiFn,\n  expandSearchArea,\n} from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callToGetJSONObject,\n} from '@/ai-model/index';\nimport { AiAssert, AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  TMultimodalPrompt,\n  TUserPrompt,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_FORCE_DEEP_THINK,\n  MIDSCENE_USE_QWEN_VL,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n  callAI?: typeof callAiFn<AIElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt?: LocateOpts,\n  ): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = getAIConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    if (searchAreaPrompt && !vlLocateMode()) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn('locate'));\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      elementById,\n      rawResponse,\n      usage,\n      isOrderSensitive,\n    } = await AiLocateElement({\n      callAI: callAI || this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n          isOrderSensitive,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<{\n    data: T;\n    thought?: string;\n    usage?: AIUsageInfo;\n  }> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n    };\n  }\n\n  async assert(assertion: TUserPrompt): Promise<InsightAssertionResponse> {\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      userQuery: {\n        assertion,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n  async describe(\n    target: Rect | [number, number],\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn('describe');\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size);\n      debug('describe: set searchArea', searchArea);\n      imagePayload = await cropByRect(\n        imagePayload,\n        searchArea,\n        getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL),\n      );\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn =\n      this.aiVendorFn || callToGetJSONObject<AIDescribeElementResponse>;\n\n    const res = await callAIFn(msgs, AIActionType.DESCRIBE_ELEMENT);\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Insight","query","opt","_parseResult_errors","callAI","queryPrompt","assert","dumpSubscriber","undefined","globalDeepThinkSwitch","getAIConfigInBoolean","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","vlLocateMode","console","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","elementById","rawResponse","usage","isOrderSensitive","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","item","element","emitInsightDump","Error","dataDemand","multimodalPrompt","AiExtractElementInfo","data","thought","assertion","assertResult","AiAssert","pass","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","cropByRect","MIDSCENE_USE_QWEN_VL","msgs","callAIFn","callToGetJSONObject","res","AIActionType","content","callAiFn","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;AC+CA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAmCnB,MAAM,OACJC,KAA0B,EAC1BC,GAAgB,EACO;YA+EnBC;QA9EJ,MAAM,EAAEC,MAAM,EAAE,GAAGF,OAAO,CAAC;QAC3B,MAAMG,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QACpB,MAAME,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzBF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMQ,wBAAwBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA,EAC5BC,oBAAAA,yBAAyBA;QAE3B,IAAIF,uBACFX,MAAM,yBAAyBW;QAEjC,IAAIG;QACJ,IAAIX,MAAM,SAAS,IAAIQ,uBACrBG,mBAAmBX,MAAM,MAAM;QAGjC,IAAIW,oBAAoB,CAACC,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,KAAgB;YACvCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBJ;QACrB;QAEA,MAAMO,UAAUb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE/D,IAAIc;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIP,kBAAkB;YACpBO,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBACzCL;gBACA,oBAAoBH;YACtB;YACAN,IAAAA,sBAAAA,MAAAA,AAAAA,EACEa,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAEP,iBAAiB,CAAC,EAChDO,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,WAAW,EACXC,KAAK,EACLC,gBAAgB,EACjB,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgB;YACxB,QAAQzB,UAAU,IAAI,CAAC,UAAU;YACjCW;YACA,0BAA0BV;YAC1B,cAAcc;QAChB;QAEA,MAAMW,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAX;YACAC;YACAC;QACF;QAEA,IAAIe;QACJ,IAAI,QAAA9B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B8B,WAAW,CAAC,6BAA6B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS7B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAamB;YACb,MAAM;YACNO;YACA,WAAW,CAAC,CAACf;YACb,OAAOiB;QACT;QAEA,MAAME,WAA0B,EAAE;QACjCZ,CAAAA,YAAY,QAAQ,IAAI,EAAC,EAAG,OAAO,CAAC,CAACa;YACpC,IAAI,QAAQA,MAAM;gBAChB,MAAMC,UAAUZ,YAAYW,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE;gBAEpC,IAAI,CAACC,SAAS,YACZvB,QAAQ,IAAI,CACV,CAAC,+BAA+B,EAAEsB,KAAK,EAAE,CAAC,0CAA0C,CAAC;gBAIzFD,SAAS,IAAI,CAACE;YAChB;QACF;QAEAC,IAAAA,kCAAAA,eAAAA,AAAAA,EACE;YACE,GAAGJ,QAAQ;YACX,gBAAgBC;QAClB,GACA5B;QAGF,IAAI0B,UACF,MAAM,IAAIM,MAAMN;QAGlB3B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE6B,SAAS,MAAM,IAAI,GACnB,CAAC,0CAA0C,EAAEA,SAAS,MAAM,EAAE;QAGhE,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,SAASA,QAAQ,CAAC,EAAE,CAAE,OAAO;gBAC7B,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM,IAAI,EAAE;gBACjC,YAAYA,QAAQ,CAAC,EAAE,CAAE,UAAU;gBACnCP;YACF;YACAJ;QACF;QAEF,OAAO;YACL,SAAS;YACTA;QACF;IACF;IAEA,MAAM,QACJgB,UAA+B,EAC/BtC,GAA0B,EAC1BuC,gBAAoC,EAKnC;YA0BGtC;QAzBJG,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAOkC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAMjC,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzB,MAAMO,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE9C,MAAMM,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEI,KAAK,EAAE,GAAG,MAAMe,AAAAA,IAAAA,yBAAAA,oBAAAA,AAAAA,EAAwB;YAC3D3B;YACA,WAAWyB;YACXC;YACA,eAAevC;QACjB;QAEA,MAAM4B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACT;QAC9B;QAEA,IAAIU;QACJ,IAAI,QAAA9B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B8B,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTM;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNT;YACA,OAAOE;QACT;QAEA,MAAM,EAAEU,IAAI,EAAEC,OAAO,EAAE,GAAGrB,eAAe,CAAC;QAG1Ce,IAAAA,kCAAAA,eAAAA,AAAAA,EACE;YACE,GAAGJ,QAAQ;YACXS;QACF,GACApC;QAGF,IAAI0B,YAAY,CAACU,MACf,MAAM,IAAIJ,MAAMN;QAGlB,OAAO;YACLU;YACAC;YACAjB;QACF;IACF;IAEA,MAAM,OAAOkB,SAAsB,EAAqC;QACtE,MAAMtC,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzB,MAAMO,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAC9C,MAAMM,YAAYC,KAAK,GAAG;QAC1B,MAAMwB,eAAe,MAAMC,AAAAA,IAAAA,2BAAAA,QAAAA,AAAAA,EAAS;YAClCF;YACA9B;QACF;QAEA,MAAMe,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACc,aAAa,OAAO;QAClD;QAEA,MAAM,EAAEF,OAAO,EAAEI,IAAI,EAAE,GAAGF,aAAa,OAAO;QAC9C,MAAMZ,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTW;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNd;YACA,eAAeiB;YACf,kBAAkBJ;YAClB,OAAOI,OAAOxC,SAAYoC;QAC5B;QACAN,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBJ,UAAU3B;QAE1B,OAAO;YACLyC;YACAJ;YACA,OAAOE,aAAa,KAAK;QAC3B;IACF;IACA,MAAM,SACJG,MAA+B,EAC/B/C,GAEC,EACwD;QACzDI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2C,QAAQ;QACf,MAAMlC,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAC9C,MAAM,EAAEmC,gBAAgB,EAAEC,IAAI,EAAE,GAAGpC;QACnCT,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO4C,kBAAkB;QAEzB,MAAME,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIrD,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMc,aAAa4C,AAAAA,IAAAA,0BAAAA,gBAAAA,AAAAA,EAAiBL,YAAYxC,QAAQ,IAAI;YAC5DjB,MAAM,4BAA4BkB;YAClC0C,eAAe,MAAMG,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EACnBH,cACA1C,YACAN,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA,EAAqBoD,oBAAAA,oBAAoBA;QAE7C;QAEA,MAAMC,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WACJ,IAAI,CAAC,UAAU,IAAIC,yBAAAA,mBAAmBA;QAExC,MAAMC,MAAM,MAAMF,SAASD,MAAMI,0BAAAA,YAAAA,CAAAA,gBAA6B;QAE9D,MAAM,EAAEC,OAAO,EAAE,GAAGF;QACpB5D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC8D,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1D9D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO8D,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAhWA,YACErD,OAEmE,EACnEb,GAAoB,CACpB;QAfF;QAIA,qCAAoDmE,0BAAAA,QAAQA;QAE5D;QAEA;QAQE/D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOS,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMuD,QAAQ,OAAO,CAACvD;QAGlD,IAAI,AAA2B,WAApBb,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA8UF"}

package/dist/lib/insight/utils.js

@@ -0,0 +1,53 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    emitInsightDump: ()=>emitInsightDump
+});
+const external_utils_js_namespaceObject = require("../utils.js");
+const env_namespaceObject = require("@midscene/shared/env");
+const utils_namespaceObject = require("@midscene/shared/utils");
+function emitInsightDump(data, dumpSubscriber) {
+    const baseData = {
+        sdkVersion: (0, external_utils_js_namespaceObject.getVersion)(),
+        logTime: Date.now(),
+        model_name: (0, env_namespaceObject.getAIConfig)(env_namespaceObject.MIDSCENE_MODEL_NAME) || ''
+    };
+    const finalData = {
+        logId: (0, utils_namespaceObject.uuid)(),
+        ...baseData,
+        ...data
+    };
+    null == dumpSubscriber || dumpSubscriber(finalData);
+}
+exports.emitInsightDump = __webpack_exports__.emitInsightDump;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "emitInsightDump"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=utils.js.map

package/dist/lib/insight/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"insight/utils.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/insight/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","emitInsightDump","data","dumpSubscriber","baseData","getVersion","Date","getAIConfig","MIDSCENE_MODEL_NAME","finalData","uuid"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACIO,SAASI,gBACdC,IAA+B,EAC/BC,cAA+B;IAE/B,MAAMC,WAAqB;QACzB,YAAYC,AAAAA,IAAAA,kCAAAA,UAAAA,AAAAA;QACZ,SAASC,KAAK,GAAG;QACjB,YAAYC,AAAAA,IAAAA,oBAAAA,WAAAA,AAAAA,EAAYC,oBAAAA,mBAAmBA,KAAK;IAClD;IACA,MAAMC,YAAyB;QAC7B,OAAOC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA;QACP,GAAGN,QAAQ;QACX,GAAGF,IAAI;IACT;IAEAC,QAAAA,kBAAAA,eAAiBM;AACnB"}

package/dist/lib/types.js

@@ -0,0 +1,82 @@
+"use strict";
+var __webpack_modules__ = {
+    "./yaml": function(module) {
+        module.exports = require("./yaml.js");
+    }
+};
+var __webpack_module_cache__ = {};
+function __webpack_require__(moduleId) {
+    var cachedModule = __webpack_module_cache__[moduleId];
+    if (void 0 !== cachedModule) return cachedModule.exports;
+    var module = __webpack_module_cache__[moduleId] = {
+        exports: {}
+    };
+    __webpack_modules__[moduleId](module, module.exports, __webpack_require__);
+    return module.exports;
+}
+(()=>{
+    __webpack_require__.n = (module)=>{
+        var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
+        __webpack_require__.d(getter, {
+            a: getter
+        });
+        return getter;
+    };
+})();
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+(()=>{
+    __webpack_require__.r(__webpack_exports__);
+    __webpack_require__.d(__webpack_exports__, {
+        AIResponseFormat: ()=>AIResponseFormat,
+        UIContext: ()=>UIContext
+    });
+    var _yaml__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__("./yaml");
+    var __WEBPACK_REEXPORT_OBJECT__ = {};
+    for(var __WEBPACK_IMPORT_KEY__ in _yaml__WEBPACK_IMPORTED_MODULE_0__)if ([
+        "default",
+        "AIResponseFormat",
+        "UIContext"
+    ].indexOf(__WEBPACK_IMPORT_KEY__) < 0) __WEBPACK_REEXPORT_OBJECT__[__WEBPACK_IMPORT_KEY__] = (function(key) {
+        return _yaml__WEBPACK_IMPORTED_MODULE_0__[key];
+    }).bind(0, __WEBPACK_IMPORT_KEY__);
+    __webpack_require__.d(__webpack_exports__, __WEBPACK_REEXPORT_OBJECT__);
+    var AIResponseFormat = /*#__PURE__*/ function(AIResponseFormat) {
+        AIResponseFormat["JSON"] = "json_object";
+        AIResponseFormat["TEXT"] = "text";
+        return AIResponseFormat;
+    }({});
+    class UIContext {
+    }
+})();
+exports.AIResponseFormat = __webpack_exports__.AIResponseFormat;
+exports.UIContext = __webpack_exports__.UIContext;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "AIResponseFormat",
+    "UIContext"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=types.js.map

package/dist/lib/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/types.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n};\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\nexport interface AIElementLocatorResponse {\n  elements: {\n    id: string;\n    reason?: string;\n    text?: string;\n    xpaths?: string[];\n  }[];\n  bbox?: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse =\n  | AIElementLocatorResponse\n  | AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext<ElementType extends BaseElement = BaseElement> {\n  abstract screenshotBase64: string;\n\n  abstract tree: ElementTreeNode<ElementType>;\n\n  abstract size: Size;\n}\n\n/**\n * insight\n */\n\nexport type CallAIFn = <T>(\n  messages: ChatCompletionMessageParam[],\n) => Promise<T>;\n\nexport interface InsightOptions {\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n  aiVendorFn?: CallAIFn;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type InsightExtractParam = string | Record<string, string>;\n\nexport type LocateResultElement = {\n  id: string;\n  indexId?: number;\n  center: [number, number];\n  rect: Rect;\n  xpaths: string[];\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n  isOrderSensitive?: boolean;\n};\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface InsightTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  sdkVersion: string;\n  logTime: number;\n  model_name: string;\n  model_description?: string;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface InsightDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: InsightExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: BaseElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: InsightTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialInsightDumpFromSDK = Omit<\n  InsightDump,\n  'sdkVersion' | 'logTime' | 'logId' | 'model_name'\n>;\n\nexport type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type InsightAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  id?: string;\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type:\n    | 'Locate'\n    | 'Tap'\n    | 'RightClick'\n    | 'Hover'\n    | 'Drag'\n    | 'Input'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Error'\n    | 'Assert'\n    | 'AssertWithoutThrow'\n    | 'Sleep'\n    | 'Finished'\n    | 'AndroidBackButton'\n    | 'AndroidHomeButton'\n    | 'AndroidRecentAppsButton'\n    | 'AndroidLongPress'\n    | 'AndroidPull';\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface PlanningAIResponse {\n  action?: PlanningAction; // this is the qwen mode\n  actions?: PlanningAction[];\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport type PlanningActionParamTap = null;\nexport type PlanningActionParamHover = null;\nexport type PlanningActionParamRightClick = null;\n\nexport interface PlanningActionParamInputOrKeyPress {\n  value: string;\n  autoDismissKeyboard?: boolean;\n}\n\nexport interface PlanningActionParamAssert {\n  assertion: TUserPrompt;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {\n  assertion: string;\n};\n\nexport interface AndroidLongPressParam {\n  duration?: number;\n}\n\nexport interface AndroidPullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n  ignoreMarker?: boolean;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  pageContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n}\n\n/*\ntask - insight-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskInsightDumpLog {\n  dump?: InsightDump;\n}\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - insight-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: InsightExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  InsightAssertionResponse,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    log?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  groupName: string;\n  groupDescription?: string;\n  executions: ExecutionDump[];\n}\n\nexport type PageType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android';\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport type TMultimodalPrompt = {\n  /**\n   * Support use image to inspect elements.\n   * The \"images\" field is an object that uses image name as key and image url as value.\n   * The image url can be a local path, a http link , or a base64 string.\n   */\n  images?: {\n    name: string;\n    url: string;\n  }[];\n  /**\n   * By default, the image url in the \"images\" filed starts with `https://` or `http://` will be directly sent to the LLM.\n   * In case the images are not accessible to the LLM (One common case is that image url is internal network only.), you can enable this option.\n   * Then image will be download and convert to base64 format.\n   */\n  convertHttpImage2Base64?: boolean;\n};\n\nexport type TUserPrompt =\n  | string\n  | ({\n      prompt: string;\n    } & Partial<TMultimodalPrompt>);\n\nexport interface DeviceAction<ParamType = any> {\n  name: string;\n  description?: string;\n  paramSchema?: string;\n  paramDescription?: string;\n  location?: 'required' | 'optional' | false;\n  whatToLocate?: string; // what to locate if location is required or optional\n  call: (context: ExecutorContext, param: ParamType) => Promise<void> | void;\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","AIResponseFormat","UIContext"],"mappings":";;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;IC0BO,IAAKI,mBAAAA,WAAAA,GAAAA,SAAAA,gBAAgB;;;eAAhBA;;IAwFL,MAAeC;IAMtB"}