@midscene/core 0.30.6-beta-20251022093704.0 → 1.0.1-beta-20251022061922.0

package/dist/es/insight/index.mjs

@@ -2,11 +2,12 @@ import { AIActionType, expandSearchArea } from "../ai-model/common.mjs";
 import { AiExtractElementInfo, AiLocateElement, callAIWithObjectResponse } from "../ai-model/index.mjs";
 import { AiLocateSection } from "../ai-model/inspect.mjs";
 import { elementDescriberInstruction } from "../ai-model/prompt/describe.mjs";
+import { InsightError } from "../types.mjs";
 import { MIDSCENE_FORCE_DEEP_THINK, globalConfigManager } from "@midscene/shared/env";
 import { compositeElementInfoImg, cropByRect } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
-import { emitInsightDump } from "./utils.mjs";
+import { createInsightDump } from "./utils.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -23,8 +24,6 @@ class Insight {
         var _parseResult_errors;
         const queryPrompt = 'string' == typeof query ? query : query.prompt;
         assert(queryPrompt, 'query is required for locate');
-        const dumpSubscriber = this.onceDumpUpdatedFn;
-        this.onceDumpUpdatedFn = void 0;
         assert('object' == typeof query, 'query should be an object for locate');
         const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(MIDSCENE_FORCE_DEEP_THINK);
         if (globalDeepThinkSwitch) debug('globalDeepThinkSwitch', globalDeepThinkSwitch);
@@ -35,7 +34,7 @@ class Insight {
             console.warn('The "deepThink" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model');
             searchAreaPrompt = void 0;
         }
-        const context = (null == opt ? void 0 : opt.context) || await this.contextRetrieverFn('locate');
+        const context = (null == opt ? void 0 : opt.context) || await this.contextRetrieverFn();
         let searchArea;
         let searchAreaRawResponse;
         let searchAreaUsage;
@@ -92,12 +91,12 @@ class Insight {
                 elements.push(element);
             }
         });
-        emitInsightDump({
+        const dump = createInsightDump({
             ...dumpData,
             matchedElement: elements
-        }, dumpSubscriber);
-        if (errorLog) throw new Error(errorLog);
-        assert(elements.length <= 1, `locate: multiple elements found, length = ${elements.length}`);
+        });
+        if (errorLog) throw new InsightError(errorLog, dump);
+        if (elements.length > 1) throw new InsightError(`locate: multiple elements found, length = ${elements.length}`, dump);
         if (1 === elements.length) return {
             element: {
                 id: elements[0].id,
@@ -108,19 +107,19 @@ class Insight {
                 attributes: elements[0].attributes,
                 isOrderSensitive
             },
-            rect
+            rect,
+            dump
         };
         return {
             element: null,
-            rect
+            rect,
+            dump
         };
     }
     async extract(dataDemand, modelConfig, opt, multimodalPrompt) {
         var _parseResult_errors;
         assert('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
-        const dumpSubscriber = this.onceDumpUpdatedFn;
-        this.onceDumpUpdatedFn = void 0;
-        const context = await this.contextRetrieverFn('extract');
+        const context = await this.contextRetrieverFn();
         const startTime = Date.now();
         const { parseResult, usage } = await AiExtractElementInfo({
             context,
@@ -148,20 +147,21 @@ class Insight {
             error: errorLog
         };
         const { data, thought } = parseResult || {};
-        emitInsightDump({
+        const dump = createInsightDump({
             ...dumpData,
             data
-        }, dumpSubscriber);
-        if (errorLog && !data && !(null == opt ? void 0 : opt.doNotThrowError)) throw new Error(errorLog);
+        });
+        if (errorLog && !data && !(null == opt ? void 0 : opt.doNotThrowError)) throw new InsightError(errorLog, dump);
         return {
             data,
             thought,
-            usage
+            usage,
+            dump
         };
     }
     async describe(target, modelConfig, opt) {
         assert(target, 'target is required for insight.describe');
-        const context = await this.contextRetrieverFn('describe');
+        const context = await this.contextRetrieverFn();
         const { screenshotBase64, size } = context;
         assert(screenshotBase64, 'screenshot is required for insight.describe');
         const { vlMode } = modelConfig;
@@ -217,7 +217,6 @@ class Insight {
     constructor(context, opt){
         _define_property(this, "contextRetrieverFn", void 0);
         _define_property(this, "aiVendorFn", callAIWithObjectResponse);
-        _define_property(this, "onceDumpUpdatedFn", void 0);
         _define_property(this, "taskInfo", void 0);
         assert(context, 'context is required for Insight');
         if ('function' == typeof context) this.contextRetrieverFn = context;

package/dist/es/insight/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"insight/index.mjs","sources":["webpack://@midscene/core/./src/insight/index.ts"],"sourcesContent":["import { AIActionType, type AIArgs, expandSearchArea } from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_FORCE_DEEP_THINK,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../ai-model/common';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface InsightOptions {\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n  aiVendorFn?: typeof callAIWithObjectResponse;\n}\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined> =\n    callAIWithObjectResponse;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    // just for unit test, aiVendorFn is callAIWithObjectResponse by default\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n  ): Promise<LocateResult> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { vlMode } = modelConfig;\n\n    if (searchAreaPrompt && !vlMode) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn('locate'));\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      elementById,\n      rawResponse,\n      usage,\n      isOrderSensitive,\n    } = await AiLocateElement({\n      callAIFn: this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n          isOrderSensitive,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<{\n    data: T;\n    thought?: string;\n    usage?: AIUsageInfo;\n  }> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data && !opt?.doNotThrowError) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn('describe');\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { vlMode } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size, vlMode);\n      debug('describe: set searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        vlMode === 'qwen-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn = this\n      .aiVendorFn as typeof callAIWithObjectResponse<AIDescribeElementResponse>;\n\n    const res = await callAIFn(\n      msgs,\n      AIActionType.DESCRIBE_ELEMENT,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Insight","query","opt","modelConfig","_parseResult_errors","queryPrompt","assert","dumpSubscriber","undefined","globalDeepThinkSwitch","globalConfigManager","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","vlMode","console","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","elementById","rawResponse","usage","isOrderSensitive","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","item","element","emitInsightDump","Error","dataDemand","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","callAIFn","res","AIActionType","content","callAIWithObjectResponse","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;AA+CA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IAqCnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACF;YAkFnBC;QAjFJ,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QACpB,MAAME,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzBF,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMQ,wBAAwBC,oBAAoB,qBAAqB,CACrEC;QAEF,IAAIF,uBACFX,MAAM,yBAAyBW;QAEjC,IAAIG;QACJ,IAAIX,MAAM,SAAS,IAAIQ,uBACrBG,mBAAmBX,MAAM,MAAM;QAGjC,MAAM,EAAEY,MAAM,EAAE,GAAGV;QAEnB,IAAIS,oBAAoB,CAACC,QAAQ;YAC/BC,QAAQ,IAAI,CACV;YAEFF,mBAAmBJ;QACrB;QAEA,MAAMO,UAAUb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE/D,IAAIc;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIP,kBAAkB;YACpBO,qBAAqB,MAAMC,gBAAgB;gBACzCL;gBACA,oBAAoBH;gBACpBT;YACF;YACAG,OACEa,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAEP,iBAAiB,CAAC,EAChDO,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,WAAW,EACXC,KAAK,EACLC,gBAAgB,EACjB,GAAG,MAAMC,gBAAgB;YACxB,UAAU,IAAI,CAAC,UAAU;YACzBd;YACA,0BAA0BV;YAC1B,cAAcc;YACdhB;QACF;QAEA,MAAM2B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAX;YACAC;YACAC;QACF;QAEA,IAAIe;QACJ,IAAI,QAAA7B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B6B,WAAW,CAAC,6BAA6B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS7B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAamB;YACb,MAAM;YACNO;YACA,WAAW,CAAC,CAACf;YACb,OAAOiB;QACT;QAEA,MAAME,WAA0B,EAAE;QACjCZ,CAAAA,YAAY,QAAQ,IAAI,EAAC,EAAG,OAAO,CAAC,CAACa;YACpC,IAAI,QAAQA,MAAM;gBAChB,MAAMC,UAAUZ,YAAYW,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE;gBAEpC,IAAI,CAACC,SAAS,YACZvB,QAAQ,IAAI,CACV,CAAC,+BAA+B,EAAEsB,KAAK,EAAE,CAAC,0CAA0C,CAAC;gBAIzFD,SAAS,IAAI,CAACE;YAChB;QACF;QAEAC,gBACE;YACE,GAAGJ,QAAQ;YACX,gBAAgBC;QAClB,GACA5B;QAGF,IAAI0B,UACF,MAAM,IAAIM,MAAMN;QAGlB3B,OACE6B,SAAS,MAAM,IAAI,GACnB,CAAC,0CAA0C,EAAEA,SAAS,MAAM,EAAE;QAGhE,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,SAASA,QAAQ,CAAC,EAAE,CAAE,OAAO;gBAC7B,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM,IAAI,EAAE;gBACjC,YAAYA,QAAQ,CAAC,EAAE,CAAE,UAAU;gBACnCP;YACF;YACAJ;QACF;QAEF,OAAO;YACL,SAAS;YACTA;QACF;IACF;IAEA,MAAM,QACJgB,UAA+B,EAC/BrC,WAAyB,EACzBD,GAA0B,EAC1BuC,gBAAoC,EAKnC;YA4BGrC;QA3BJE,OACE,AAAsB,YAAtB,OAAOkC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAMjC,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzB,MAAMO,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE9C,MAAMM,YAAYC,KAAK,GAAG;QAE1B,MAAM,EAAEC,WAAW,EAAEI,KAAK,EAAE,GAAG,MAAMe,qBAAwB;YAC3D3B;YACA,WAAWyB;YACXC;YACA,eAAevC;YACfC;QACF;QAEA,MAAM2B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACT;QAC9B;QAEA,IAAIU;QACJ,IAAI,QAAA7B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B6B,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTM;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNT;YACA,OAAOE;QACT;QAEA,MAAM,EAAEU,IAAI,EAAEC,OAAO,EAAE,GAAGrB,eAAe,CAAC;QAG1Ce,gBACE;YACE,GAAGJ,QAAQ;YACXS;QACF,GACApC;QAGF,IAAI0B,YAAY,CAACU,QAAQ,CAACzC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,GAC3C,MAAM,IAAIqC,MAAMN;QAGlB,OAAO;YACLU;YACAC;YACAjB;QACF;IACF;IAEA,MAAM,SACJkB,MAA+B,EAC/B1C,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOuC,QAAQ;QACf,MAAM9B,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAC9C,MAAM,EAAE+B,gBAAgB,EAAEC,IAAI,EAAE,GAAGhC;QACnCT,OAAOwC,kBAAkB;QAEzB,MAAM,EAAEjC,MAAM,EAAE,GAAGV;QACnB,MAAM6C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIjD,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMc,aAAawC,iBAAiBL,YAAYpC,QAAQ,IAAI,EAAEF;YAC9Df,MAAM,4BAA4BkB;YAClC,MAAMyC,gBAAgB,MAAMC,WAC1BJ,cACAtC,YACAH,AAAW,cAAXA;YAEFyC,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WAAW,IAAI,CAClB,UAAU;QAEb,MAAMC,MAAM,MAAMD,SAChBD,MACAG,aAAa,gBAAgB,EAC7B3D;QAGF,MAAM,EAAE4D,OAAO,EAAE,GAAGF;QACpBvD,OAAO,CAACyD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DzD,OAAOyD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAxUA,YACEhD,OAEmE,EACnEb,GAAoB,CACpB;QAhBF;QAIA,qCACE8D;QAEF;QAEA;QAQE1D,OAAOS,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMkD,QAAQ,OAAO,CAAClD;QAIlD,IAAI,AAA2B,WAApBb,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAqTF"}
+{"version":3,"file":"insight/index.mjs","sources":["webpack://@midscene/core/./src/insight/index.ts"],"sourcesContent":["import { AIActionType, type AIArgs, expandSearchArea } from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  InsightAction,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightExtractResult,\n  InsightTaskInfo,\n  LocateResultWithDump,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { InsightError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_FORCE_DEEP_THINK,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../ai-model/common';\nimport { createInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface InsightOptions {\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n  aiVendorFn?: typeof callAIWithObjectResponse;\n}\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: () => Promise<ContextType> | ContextType;\n\n  aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined> =\n    callAIWithObjectResponse;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: ContextType | (() => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    // just for unit test, aiVendorFn is callAIWithObjectResponse by default\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { vlMode } = modelConfig;\n\n    if (searchAreaPrompt && !vlMode) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      elementById,\n      rawResponse,\n      usage,\n      isOrderSensitive,\n    } = await AiLocateElement({\n      callAIFn: this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    const dump = createInsightDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new InsightError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new InsightError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n          isOrderSensitive,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<InsightExtractResult<T>> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const context = await this.contextRetrieverFn();\n\n    const startTime = Date.now();\n\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    const dump = createInsightDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data && !opt?.doNotThrowError) {\n      throw new InsightError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn();\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { vlMode } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size, vlMode);\n      debug('describe: set searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        vlMode === 'qwen-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn = this\n      .aiVendorFn as typeof callAIWithObjectResponse<AIDescribeElementResponse>;\n\n    const res = await callAIFn(\n      msgs,\n      AIActionType.DESCRIBE_ELEMENT,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Insight","query","opt","modelConfig","_parseResult_errors","queryPrompt","assert","globalDeepThinkSwitch","globalConfigManager","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","vlMode","console","undefined","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","elementById","rawResponse","usage","isOrderSensitive","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","item","element","dump","createInsightDump","InsightError","dataDemand","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","callAIFn","res","AIActionType","content","callAIWithObjectResponse","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;AAgDA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IA+BnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACM;YAgF3BC;QA/EJ,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMM,wBAAwBC,oBAAoB,qBAAqB,CACrEC;QAEF,IAAIF,uBACFT,MAAM,yBAAyBS;QAEjC,IAAIG;QACJ,IAAIT,MAAM,SAAS,IAAIM,uBACrBG,mBAAmBT,MAAM,MAAM;QAGjC,MAAM,EAAEU,MAAM,EAAE,GAAGR;QAEnB,IAAIO,oBAAoB,CAACC,QAAQ;YAC/BC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,MAAMC,UAAUZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIa;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,gBAAgB;gBACzCL;gBACA,oBAAoBJ;gBACpBP;YACF;YACAG,OACEY,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,WAAW,EACXC,KAAK,EACLC,gBAAgB,EACjB,GAAG,MAAMC,gBAAgB;YACxB,UAAU,IAAI,CAAC,UAAU;YACzBd;YACA,0BAA0BT;YAC1B,cAAca;YACdf;QACF;QAEA,MAAM0B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAX;YACAC;YACAC;QACF;QAEA,IAAIe;QACJ,IAAI,QAAA5B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B4B,WAAW,CAAC,6BAA6B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS5B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAakB;YACb,MAAM;YACNO;YACA,WAAW,CAAC,CAACf;YACb,OAAOiB;QACT;QAEA,MAAME,WAA0B,EAAE;QACjCZ,CAAAA,YAAY,QAAQ,IAAI,EAAC,EAAG,OAAO,CAAC,CAACa;YACpC,IAAI,QAAQA,MAAM;gBAChB,MAAMC,UAAUZ,YAAYW,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE;gBAEpC,IAAI,CAACC,SAAS,YACZxB,QAAQ,IAAI,CACV,CAAC,+BAA+B,EAAEuB,KAAK,EAAE,CAAC,0CAA0C,CAAC;gBAIzFD,SAAS,IAAI,CAACE;YAChB;QACF;QAEA,MAAMC,OAAOC,kBAAkB;YAC7B,GAAGL,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIO,aAAaP,UAAUK;QAGnC,IAAIH,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIK,aACR,CAAC,0CAA0C,EAAEL,SAAS,MAAM,EAAE,EAC9DG;QAIJ,IAAIH,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,SAASA,QAAQ,CAAC,EAAE,CAAE,OAAO;gBAC7B,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM,IAAI,EAAE;gBACjC,YAAYA,QAAQ,CAAC,EAAE,CAAE,UAAU;gBACnCP;YACF;YACAJ;YACAc;QACF;QAGF,OAAO;YACL,SAAS;YACTd;YACAc;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BrC,WAAyB,EACzBD,GAA0B,EAC1BuC,gBAAoC,EACF;YAyB9BrC;QAxBJE,OACE,AAAsB,YAAtB,OAAOkC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAM1B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAE7C,MAAMM,YAAYC,KAAK,GAAG;QAE1B,MAAM,EAAEC,WAAW,EAAEI,KAAK,EAAE,GAAG,MAAMgB,qBAAwB;YAC3D5B;YACA,WAAW0B;YACXC;YACA,eAAevC;YACfC;QACF;QAEA,MAAM0B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACT;QAC9B;QAEA,IAAIU;QACJ,IAAI,QAAA5B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B4B,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTO;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNV;YACA,OAAOE;QACT;QAEA,MAAM,EAAEW,IAAI,EAAEC,OAAO,EAAE,GAAGtB,eAAe,CAAC;QAG1C,MAAMe,OAAOC,kBAAkB;YAC7B,GAAGL,QAAQ;YACXU;QACF;QAEA,IAAIX,YAAY,CAACW,QAAQ,CAACzC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,GAC3C,MAAM,IAAIqC,aAAaP,UAAUK;QAGnC,OAAO;YACLM;YACAC;YACAlB;YACAW;QACF;IACF;IAEA,MAAM,SACJQ,MAA+B,EAC/B1C,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOuC,QAAQ;QACf,MAAM/B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEgC,gBAAgB,EAAEC,IAAI,EAAE,GAAGjC;QACnCR,OAAOwC,kBAAkB;QAEzB,MAAM,EAAEnC,MAAM,EAAE,GAAGR;QACnB,MAAM6C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIjD,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMa,aAAayC,iBAAiBL,YAAYrC,QAAQ,IAAI,EAAEH;YAC9Db,MAAM,4BAA4BiB;YAClC,MAAM0C,gBAAgB,MAAMC,WAC1BJ,cACAvC,YACAJ,AAAW,cAAXA;YAEF2C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WAAW,IAAI,CAClB,UAAU;QAEb,MAAMC,MAAM,MAAMD,SAChBD,MACAG,aAAa,gBAAgB,EAC7B3D;QAGF,MAAM,EAAE4D,OAAO,EAAE,GAAGF;QACpBvD,OAAO,CAACyD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DzD,OAAOyD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IA7TA,YACEjD,OAAiE,EACjEZ,GAAoB,CACpB;QAVF;QAEA,qCACE8D;QAEF;QAME1D,OAAOQ,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMmD,QAAQ,OAAO,CAACnD;QAIlD,IAAI,AAA2B,WAApBZ,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA4SF"}

package/dist/es/insight/utils.mjs

@@ -1,5 +1,5 @@
 import { uuid } from "@midscene/shared/utils";
-function emitInsightDump(data, dumpSubscriber) {
+function createInsightDump(data) {
     const baseData = {
         logTime: Date.now()
     };
@@ -8,8 +8,8 @@ function emitInsightDump(data, dumpSubscriber) {
         ...baseData,
         ...data
     };
-    null == dumpSubscriber || dumpSubscriber(finalData);
+    return finalData;
 }
-export { emitInsightDump };
+export { createInsightDump };
 
 //# sourceMappingURL=utils.mjs.map

package/dist/es/insight/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"insight/utils.mjs","sources":["webpack://@midscene/core/./src/insight/utils.ts"],"sourcesContent":["import type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n"],"names":["emitInsightDump","data","dumpSubscriber","baseData","Date","finalData","uuid"],"mappings":";AAQO,SAASA,gBACdC,IAA+B,EAC/BC,cAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC;QACP,GAAGH,QAAQ;QACX,GAAGF,IAAI;IACT;IAEAC,QAAAA,kBAAAA,eAAiBG;AACnB"}
+{"version":3,"file":"insight/utils.mjs","sources":["webpack://@midscene/core/./src/insight/utils.ts"],"sourcesContent":["import type { DumpMeta, InsightDump, PartialInsightDumpFromSDK } from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function createInsightDump(\n  data: PartialInsightDumpFromSDK,\n): InsightDump {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  return finalData;\n}\n"],"names":["createInsightDump","data","baseData","Date","finalData","uuid"],"mappings":";AAGO,SAASA,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT"}

package/dist/es/report.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"report.mjs","sources":["webpack://@midscene/core/./src/report.ts"],"sourcesContent":["import * as fs from 'node:fs';\nimport * as path from 'node:path';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { getReportFileName } from './agent';\nimport type { ReportFileWithAttributes } from './types';\nimport { getReportTpl, reportHTMLContent } from './utils';\n\nexport class ReportMergingTool {\n  private reportInfos: ReportFileWithAttributes[] = [];\n  public append(reportInfo: ReportFileWithAttributes) {\n    this.reportInfos.push(reportInfo);\n  }\n  public clear() {\n    this.reportInfos = [];\n  }\n  private extractScriptContent(filePath: string): string {\n    // Regular expression to match content between script tags\n    // Requires newline before <script and </script>\n    const scriptRegex =\n      /\\n<script type=\"midscene_web_dump\" type=\"application\\/json\"[^>]*>([\\s\\S]*?)\\n<\\/script>/;\n\n    const fileContent = fs.readFileSync(filePath, 'utf-8');\n    const match = scriptRegex.exec(fileContent);\n\n    return match ? match[1].trim() : '';\n  }\n\n  public mergeReports(\n    reportFileName: 'AUTO' | string = 'AUTO', // user custom report filename, save into midscene report dir if undefined\n    opts?: {\n      rmOriginalReports?: boolean; // whether to remove origin report files\n      overwrite?: boolean; // if outfilepath specified, throw an error when overwrite = true, otherwise overwrite the file\n    },\n  ): string | null {\n    if (this.reportInfos.length <= 1) {\n      console.log('Not enough report to merge');\n      return null;\n    }\n    opts = Object.assign(\n      {\n        rmOriginalReports: false,\n        overwrite: false,\n      },\n      opts || {},\n    );\n    const { rmOriginalReports, overwrite } = opts;\n    let outputFilePath;\n    const targetDir = `${getMidsceneRunSubDir('report')}`;\n    if (reportFileName === 'AUTO') {\n      outputFilePath = path.resolve(\n        targetDir,\n        `${getReportFileName('merged-report')}.html`,\n      );\n    } else {\n      // user specified a outfilepath\n      outputFilePath = path.resolve(targetDir, `${reportFileName}.html`);\n      if (fs.existsSync(outputFilePath) && !overwrite) {\n        throw Error(\n          `report file already existed: ${outputFilePath}\\nset override to true to overwrite this file.`,\n        );\n      } else if (fs.existsSync(outputFilePath) && overwrite) {\n        fs.unlinkSync(outputFilePath);\n      }\n    }\n\n    console.log(\n      `Start merging ${this.reportInfos.length} reports...\\nCreating template file...`,\n    );\n\n    try {\n      // Write template\n      fs.appendFileSync(outputFilePath, getReportTpl());\n\n      // Process all reports one by one\n      for (let i = 0; i < this.reportInfos.length; i++) {\n        const reportInfo = this.reportInfos[i];\n        console.log(`Processing report ${i + 1}/${this.reportInfos.length}`);\n\n        const dumpString = this.extractScriptContent(reportInfo.reportFilePath);\n        const reportAttributes = reportInfo.reportAttributes;\n\n        const reportHtmlStr = `${reportHTMLContent(\n          {\n            dumpString,\n            attributes: {\n              playwright_test_duration: reportAttributes.testDuration,\n              playwright_test_status: reportAttributes.testStatus,\n              playwright_test_title: reportAttributes.testTitle,\n              playwright_test_id: reportAttributes.testId,\n              playwright_test_description: reportAttributes.testDescription,\n            },\n          },\n          undefined,\n          undefined,\n          false,\n        )}\\n`; // use existed function to achieve report script content\n\n        fs.appendFileSync(outputFilePath, reportHtmlStr);\n      }\n\n      console.log(`Successfully merged new report: ${outputFilePath}`);\n\n      // Remove original reports if needed\n      if (rmOriginalReports) {\n        for (const info of this.reportInfos) {\n          try {\n            fs.unlinkSync(info.reportFilePath);\n          } catch (error) {\n            console.error(\n              `Error deleting report ${info.reportFilePath}:`,\n              error,\n            );\n          }\n        }\n        console.log(`Removed ${this.reportInfos.length} original reports`);\n      }\n      return outputFilePath;\n    } catch (error) {\n      console.error('Error in mergeReports:', error);\n      throw error;\n    }\n  }\n}\n"],"names":["ReportMergingTool","reportInfo","filePath","scriptRegex","fileContent","fs","match","reportFileName","opts","console","Object","rmOriginalReports","overwrite","outputFilePath","targetDir","getMidsceneRunSubDir","path","getReportFileName","Error","getReportTpl","i","dumpString","reportAttributes","reportHtmlStr","reportHTMLContent","undefined","info","error"],"mappings":";;;;;;;;;;;;;;;AAOO,MAAMA;IAEJ,OAAOC,UAAoC,EAAE;QAClD,IAAI,CAAC,WAAW,CAAC,IAAI,CAACA;IACxB;IACO,QAAQ;QACb,IAAI,CAAC,WAAW,GAAG,EAAE;IACvB;IACQ,qBAAqBC,QAAgB,EAAU;QAGrD,MAAMC,cACJ;QAEF,MAAMC,cAAcC,aAAgBH,UAAU;QAC9C,MAAMI,QAAQH,YAAY,IAAI,CAACC;QAE/B,OAAOE,QAAQA,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK;IACnC;IAEO,aACLC,iBAAkC,MAAM,EACxCC,IAGC,EACc;QACf,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,GAAG;YAChCC,QAAQ,GAAG,CAAC;YACZ,OAAO;QACT;QACAD,OAAOE,OAAO,MAAM,CAClB;YACE,mBAAmB;YACnB,WAAW;QACb,GACAF,QAAQ,CAAC;QAEX,MAAM,EAAEG,iBAAiB,EAAEC,SAAS,EAAE,GAAGJ;QACzC,IAAIK;QACJ,MAAMC,YAAY,GAAGC,qBAAqB,WAAW;QACrD,IAAIR,AAAmB,WAAnBA,gBACFM,iBAAiBG,QACfF,WACA,GAAGG,kBAAkB,iBAAiB,KAAK,CAAC;aAEzC;YAELJ,iBAAiBG,QAAaF,WAAW,GAAGP,eAAe,KAAK,CAAC;YACjE,IAAIF,WAAcQ,mBAAmB,CAACD,WACpC,MAAMM,MACJ,CAAC,6BAA6B,EAAEL,eAAe,8CAA8C,CAAC;YAE3F,IAAIR,WAAcQ,mBAAmBD,WAC1CP,WAAcQ;QAElB;QAEAJ,QAAQ,GAAG,CACT,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,sCAAsC,CAAC;QAGlF,IAAI;YAEFJ,eAAkBQ,gBAAgBM;YAGlC,IAAK,IAAIC,IAAI,GAAGA,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAEA,IAAK;gBAChD,MAAMnB,aAAa,IAAI,CAAC,WAAW,CAACmB,EAAE;gBACtCX,QAAQ,GAAG,CAAC,CAAC,kBAAkB,EAAEW,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;gBAEnE,MAAMC,aAAa,IAAI,CAAC,oBAAoB,CAACpB,WAAW,cAAc;gBACtE,MAAMqB,mBAAmBrB,WAAW,gBAAgB;gBAEpD,MAAMsB,gBAAgB,GAAGC,kBACvB;oBACEH;oBACA,YAAY;wBACV,0BAA0BC,iBAAiB,YAAY;wBACvD,wBAAwBA,iBAAiB,UAAU;wBACnD,uBAAuBA,iBAAiB,SAAS;wBACjD,oBAAoBA,iBAAiB,MAAM;wBAC3C,6BAA6BA,iBAAiB,eAAe;oBAC/D;gBACF,GACAG,QACAA,QACA,OACA,EAAE,CAAC;gBAELpB,eAAkBQ,gBAAgBU;YACpC;YAEAd,QAAQ,GAAG,CAAC,CAAC,gCAAgC,EAAEI,gBAAgB;YAG/D,IAAIF,mBAAmB;gBACrB,KAAK,MAAMe,QAAQ,IAAI,CAAC,WAAW,CACjC,IAAI;oBACFrB,WAAcqB,KAAK,cAAc;gBACnC,EAAE,OAAOC,OAAO;oBACdlB,QAAQ,KAAK,CACX,CAAC,sBAAsB,EAAEiB,KAAK,cAAc,CAAC,CAAC,CAAC,EAC/CC;gBAEJ;gBAEFlB,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;YACnE;YACA,OAAOI;QACT,EAAE,OAAOc,OAAO;YACdlB,QAAQ,KAAK,CAAC,0BAA0BkB;YACxC,MAAMA;QACR;IACF;;QAjHA,uBAAQ,eAA0C,EAAE;;AAkHtD"}
+{"version":3,"file":"report.mjs","sources":["webpack://@midscene/core/./src/report.ts"],"sourcesContent":["import * as fs from 'node:fs';\nimport * as path from 'node:path';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { getReportFileName } from './agent';\nimport type { ReportFileWithAttributes } from './types';\nimport { getReportTpl, reportHTMLContent } from './utils';\n\nexport class ReportMergingTool {\n  private reportInfos: ReportFileWithAttributes[] = [];\n  public append(reportInfo: ReportFileWithAttributes) {\n    this.reportInfos.push(reportInfo);\n  }\n  public clear() {\n    this.reportInfos = [];\n  }\n  private extractScriptContent(filePath: string): string {\n    // Regular expression to match content between script tags\n    // Requires newline before <script and </script>\n    const scriptRegex =\n      /\\n<script type=\"midscene_web_dump\" type=\"application\\/json\"[^>]*>([\\s\\S]*?)\\n<\\/script>/;\n\n    const fileContent = fs.readFileSync(filePath, 'utf-8');\n    const match = scriptRegex.exec(fileContent);\n\n    return match ? match[1].trim() : '';\n  }\n\n  public mergeReports(\n    reportFileName: 'AUTO' | string = 'AUTO', // user custom report filename, save into midscene report dir if undefined\n    opts?: {\n      rmOriginalReports?: boolean; // whether to remove origin report files\n      overwrite?: boolean; // if output filepath specified, throw an error when overwrite = true, otherwise overwrite the file\n    },\n  ): string | null {\n    if (this.reportInfos.length <= 1) {\n      console.log('Not enough report to merge');\n      return null;\n    }\n    opts = Object.assign(\n      {\n        rmOriginalReports: false,\n        overwrite: false,\n      },\n      opts || {},\n    );\n    const { rmOriginalReports, overwrite } = opts;\n    let outputFilePath;\n    const targetDir = `${getMidsceneRunSubDir('report')}`;\n    if (reportFileName === 'AUTO') {\n      outputFilePath = path.resolve(\n        targetDir,\n        `${getReportFileName('merged-report')}.html`,\n      );\n    } else {\n      // user specified a output filepath\n      outputFilePath = path.resolve(targetDir, `${reportFileName}.html`);\n      if (fs.existsSync(outputFilePath) && !overwrite) {\n        throw Error(\n          `report file already existed: ${outputFilePath}\\nset override to true to overwrite this file.`,\n        );\n      } else if (fs.existsSync(outputFilePath) && overwrite) {\n        fs.unlinkSync(outputFilePath);\n      }\n    }\n\n    console.log(\n      `Start merging ${this.reportInfos.length} reports...\\nCreating template file...`,\n    );\n\n    try {\n      // Write template\n      fs.appendFileSync(outputFilePath, getReportTpl());\n\n      // Process all reports one by one\n      for (let i = 0; i < this.reportInfos.length; i++) {\n        const reportInfo = this.reportInfos[i];\n        console.log(`Processing report ${i + 1}/${this.reportInfos.length}`);\n\n        const dumpString = this.extractScriptContent(reportInfo.reportFilePath);\n        const reportAttributes = reportInfo.reportAttributes;\n\n        const reportHtmlStr = `${reportHTMLContent(\n          {\n            dumpString,\n            attributes: {\n              playwright_test_duration: reportAttributes.testDuration,\n              playwright_test_status: reportAttributes.testStatus,\n              playwright_test_title: reportAttributes.testTitle,\n              playwright_test_id: reportAttributes.testId,\n              playwright_test_description: reportAttributes.testDescription,\n            },\n          },\n          undefined,\n          undefined,\n          false,\n        )}\\n`; // use existed function to achieve report script content\n\n        fs.appendFileSync(outputFilePath, reportHtmlStr);\n      }\n\n      console.log(`Successfully merged new report: ${outputFilePath}`);\n\n      // Remove original reports if needed\n      if (rmOriginalReports) {\n        for (const info of this.reportInfos) {\n          try {\n            fs.unlinkSync(info.reportFilePath);\n          } catch (error) {\n            console.error(\n              `Error deleting report ${info.reportFilePath}:`,\n              error,\n            );\n          }\n        }\n        console.log(`Removed ${this.reportInfos.length} original reports`);\n      }\n      return outputFilePath;\n    } catch (error) {\n      console.error('Error in mergeReports:', error);\n      throw error;\n    }\n  }\n}\n"],"names":["ReportMergingTool","reportInfo","filePath","scriptRegex","fileContent","fs","match","reportFileName","opts","console","Object","rmOriginalReports","overwrite","outputFilePath","targetDir","getMidsceneRunSubDir","path","getReportFileName","Error","getReportTpl","i","dumpString","reportAttributes","reportHtmlStr","reportHTMLContent","undefined","info","error"],"mappings":";;;;;;;;;;;;;;;AAOO,MAAMA;IAEJ,OAAOC,UAAoC,EAAE;QAClD,IAAI,CAAC,WAAW,CAAC,IAAI,CAACA;IACxB;IACO,QAAQ;QACb,IAAI,CAAC,WAAW,GAAG,EAAE;IACvB;IACQ,qBAAqBC,QAAgB,EAAU;QAGrD,MAAMC,cACJ;QAEF,MAAMC,cAAcC,aAAgBH,UAAU;QAC9C,MAAMI,QAAQH,YAAY,IAAI,CAACC;QAE/B,OAAOE,QAAQA,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK;IACnC;IAEO,aACLC,iBAAkC,MAAM,EACxCC,IAGC,EACc;QACf,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,GAAG;YAChCC,QAAQ,GAAG,CAAC;YACZ,OAAO;QACT;QACAD,OAAOE,OAAO,MAAM,CAClB;YACE,mBAAmB;YACnB,WAAW;QACb,GACAF,QAAQ,CAAC;QAEX,MAAM,EAAEG,iBAAiB,EAAEC,SAAS,EAAE,GAAGJ;QACzC,IAAIK;QACJ,MAAMC,YAAY,GAAGC,qBAAqB,WAAW;QACrD,IAAIR,AAAmB,WAAnBA,gBACFM,iBAAiBG,QACfF,WACA,GAAGG,kBAAkB,iBAAiB,KAAK,CAAC;aAEzC;YAELJ,iBAAiBG,QAAaF,WAAW,GAAGP,eAAe,KAAK,CAAC;YACjE,IAAIF,WAAcQ,mBAAmB,CAACD,WACpC,MAAMM,MACJ,CAAC,6BAA6B,EAAEL,eAAe,8CAA8C,CAAC;YAE3F,IAAIR,WAAcQ,mBAAmBD,WAC1CP,WAAcQ;QAElB;QAEAJ,QAAQ,GAAG,CACT,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,sCAAsC,CAAC;QAGlF,IAAI;YAEFJ,eAAkBQ,gBAAgBM;YAGlC,IAAK,IAAIC,IAAI,GAAGA,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAEA,IAAK;gBAChD,MAAMnB,aAAa,IAAI,CAAC,WAAW,CAACmB,EAAE;gBACtCX,QAAQ,GAAG,CAAC,CAAC,kBAAkB,EAAEW,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;gBAEnE,MAAMC,aAAa,IAAI,CAAC,oBAAoB,CAACpB,WAAW,cAAc;gBACtE,MAAMqB,mBAAmBrB,WAAW,gBAAgB;gBAEpD,MAAMsB,gBAAgB,GAAGC,kBACvB;oBACEH;oBACA,YAAY;wBACV,0BAA0BC,iBAAiB,YAAY;wBACvD,wBAAwBA,iBAAiB,UAAU;wBACnD,uBAAuBA,iBAAiB,SAAS;wBACjD,oBAAoBA,iBAAiB,MAAM;wBAC3C,6BAA6BA,iBAAiB,eAAe;oBAC/D;gBACF,GACAG,QACAA,QACA,OACA,EAAE,CAAC;gBAELpB,eAAkBQ,gBAAgBU;YACpC;YAEAd,QAAQ,GAAG,CAAC,CAAC,gCAAgC,EAAEI,gBAAgB;YAG/D,IAAIF,mBAAmB;gBACrB,KAAK,MAAMe,QAAQ,IAAI,CAAC,WAAW,CACjC,IAAI;oBACFrB,WAAcqB,KAAK,cAAc;gBACnC,EAAE,OAAOC,OAAO;oBACdlB,QAAQ,KAAK,CACX,CAAC,sBAAsB,EAAEiB,KAAK,cAAc,CAAC,CAAC,CAAC,EAC/CC;gBAEJ;gBAEFlB,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;YACnE;YACA,OAAOI;QACT,EAAE,OAAOc,OAAO;YACdlB,QAAQ,KAAK,CAAC,0BAA0BkB;YACxC,MAAMA;QACR;IACF;;QAjHA,uBAAQ,eAA0C,EAAE;;AAkHtD"}

package/dist/es/{ai-model/action-executor.mjs → task-runner.mjs}

@@ -9,25 +9,63 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-class Executor {
+class TaskRunner {
+    async captureScreenshot() {
+        try {
+            const uiContext = await this.uiContextBuilder();
+            return null == uiContext ? void 0 : uiContext.screenshotBase64;
+        } catch (error) {
+            console.error('error while capturing screenshot', error);
+        }
+    }
+    attachRecorderItem(task, contextOrScreenshot, phase) {
+        const timing = phase;
+        const screenshot = 'string' == typeof contextOrScreenshot ? contextOrScreenshot : null == contextOrScreenshot ? void 0 : contextOrScreenshot.screenshotBase64;
+        if (!timing || !screenshot) return;
+        const recorderItem = {
+            type: 'screenshot',
+            ts: Date.now(),
+            screenshot,
+            timing
+        };
+        if (!task.recorder) {
+            task.recorder = [
+                recorderItem
+            ];
+            return;
+        }
+        task.recorder.push(recorderItem);
+    }
     markTaskAsPending(task) {
         return {
             status: 'pending',
             ...task
         };
     }
+    findPreviousNonSubTaskUIContext(currentIndex) {
+        for(let i = currentIndex - 1; i >= 0; i--){
+            const candidate = this.tasks[i];
+            if (candidate && !candidate.subTask) {
+                if (candidate.uiContext) return candidate.uiContext;
+            }
+        }
+    }
     async append(task) {
         var _this_latestErrorTask, _this_latestErrorTask1;
-        assert('error' !== this.status, `executor is in error state, cannot append task\nerror=${null == (_this_latestErrorTask = this.latestErrorTask()) ? void 0 : _this_latestErrorTask.error}\n${null == (_this_latestErrorTask1 = this.latestErrorTask()) ? void 0 : _this_latestErrorTask1.errorStack}`);
+        assert('error' !== this.status, `task runner is in error state, cannot append task\nerror=${null == (_this_latestErrorTask = this.latestErrorTask()) ? void 0 : _this_latestErrorTask.error}\n${null == (_this_latestErrorTask1 = this.latestErrorTask()) ? void 0 : _this_latestErrorTask1.errorStack}`);
         if (Array.isArray(task)) this.tasks.push(...task.map((item)=>this.markTaskAsPending(item)));
         else this.tasks.push(this.markTaskAsPending(task));
         if ('running' !== this.status) this.status = 'pending';
     }
+    async appendAndFlush(task) {
+        await this.append(task);
+        return this.flush();
+    }
     async flush() {
-        if ('init' === this.status && this.tasks.length > 0) console.warn('illegal state for executor, status is init but tasks are not empty');
-        assert('running' !== this.status, 'executor is already running');
-        assert('completed' !== this.status, 'executor is already completed');
-        assert('error' !== this.status, 'executor is in error state');
+        if ('init' === this.status && this.tasks.length > 0) console.warn('illegal state for task runner, status is init but tasks are not empty');
+        assert('running' !== this.status, 'task runner is already running');
+        assert('completed' !== this.status, 'task runner is already completed');
+        assert('error' !== this.status, 'task runner is in error state');
         const nextPendingIndex = this.tasks.findIndex((task)=>'pending' === task.status);
         if (nextPendingIndex < 0) return;
         this.status = 'running';
@@ -55,9 +93,16 @@ class Executor {
                 const { executor, param } = task;
                 assert(executor, `executor is required for task type: ${task.type}`);
                 let returnValue;
+                let uiContext;
+                if (task.subTask) {
+                    uiContext = this.findPreviousNonSubTaskUIContext(taskIndex);
+                    assert(uiContext, 'subTask requires uiContext from previous non-subTask task');
+                } else uiContext = await this.uiContextBuilder();
+                task.uiContext = uiContext;
                 const executorContext = {
                     task,
-                    element: null == previousFindOutput ? void 0 : previousFindOutput.element
+                    element: null == previousFindOutput ? void 0 : previousFindOutput.element,
+                    uiContext
                 };
                 if ('Insight' === task.type) {
                     assert('Locate' === task.subType || 'Query' === task.subType || 'Assert' === task.subType || 'WaitFor' === task.subType || 'Boolean' === task.subType || 'Number' === task.subType || 'String' === task.subType, `unsupported insight subType: ${task.subType}`);
@@ -68,6 +113,11 @@ class Executor {
                     console.warn(`unsupported task type: ${task.type}, will try to execute it directly`);
                     returnValue = await task.executor(param, executorContext);
                 }
+                const isLastTask = taskIndex === this.tasks.length - 1;
+                if (isLastTask) {
+                    const screenshot = await this.captureScreenshot();
+                    this.attachRecorderItem(task, screenshot, 'after');
+                }
                 Object.assign(task, returnValue);
                 task.status = 'finished';
                 task.timing.end = Date.now();
@@ -113,17 +163,38 @@ class Executor {
         };
         return dumpData;
     }
-    constructor(name, options){
+    async appendErrorPlan(errorMsg) {
+        const errorTask = {
+            type: 'Action',
+            subType: 'Error',
+            param: {
+                thought: errorMsg
+            },
+            thought: errorMsg,
+            locate: null,
+            executor: async ()=>{
+                throw new Error(errorMsg || 'error without thought');
+            }
+        };
+        await this.appendAndFlush(errorTask);
+        return {
+            output: void 0,
+            runner: this
+        };
+    }
+    constructor(name, uiContextBuilder, options){
         _define_property(this, "name", void 0);
         _define_property(this, "tasks", void 0);
         _define_property(this, "status", void 0);
         _define_property(this, "onTaskStart", void 0);
+        _define_property(this, "uiContextBuilder", void 0);
         this.status = (null == options ? void 0 : options.tasks) && options.tasks.length > 0 ? 'pending' : 'init';
         this.name = name;
         this.tasks = ((null == options ? void 0 : options.tasks) || []).map((item)=>this.markTaskAsPending(item));
         this.onTaskStart = null == options ? void 0 : options.onTaskStart;
+        this.uiContextBuilder = uiContextBuilder;
     }
 }
-export { Executor };
+export { TaskRunner };
 
-//# sourceMappingURL=action-executor.mjs.map
+//# sourceMappingURL=task-runner.mjs.map

package/dist/es/task-runner.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-runner.mjs","sources":["webpack://@midscene/core/./src/task-runner.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionRecorderItem,\n  ExecutionTask,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n  PlanningActionParamError,\n  UIContext,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nexport class TaskRunner {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n  }\n\n  private async captureScreenshot(): Promise<string | undefined> {\n    try {\n      const uiContext = await this.uiContextBuilder();\n      return uiContext?.screenshotBase64;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    contextOrScreenshot: UIContext | string | undefined,\n    phase: 'before' | 'after',\n  ): void {\n    const timing = phase;\n    const screenshot =\n      typeof contextOrScreenshot === 'string'\n        ? contextOrScreenshot\n        : contextOrScreenshot?.screenshotBase64;\n    if (!timing || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private findPreviousNonSubTaskUIContext(\n    currentIndex: number,\n  ): UIContext | undefined {\n    for (let i = currentIndex - 1; i >= 0; i--) {\n      const candidate = this.tasks[i];\n      if (!candidate || candidate.subTask) {\n        continue;\n      }\n      if (candidate.uiContext) {\n        return candidate.uiContext;\n      }\n    }\n    return undefined;\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task);\n    return this.flush();\n  }\n\n  async flush(): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n    assert(this.status !== 'error', 'task runner is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        let uiContext: UIContext | undefined;\n        if (task.subTask) {\n          uiContext = this.findPreviousNonSubTaskUIContext(taskIndex);\n          assert(\n            uiContext,\n            'subTask requires uiContext from previous non-subTask task',\n          );\n        } else {\n          uiContext = await this.uiContextBuilder();\n        }\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      locate: null,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n"],"names":["TaskRunner","uiContext","error","console","task","contextOrScreenshot","phase","timing","screenshot","recorderItem","Date","currentIndex","i","candidate","_this_latestErrorTask","_this_latestErrorTask1","assert","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","executorContext","isLastTask","Object","outputIndex","Math","thought","output","errorTaskIndex","dumpData","errorMsg","errorTask","Error","undefined","name","uiContextBuilder","options"],"mappings":";;;;;;;;;;;AAeO,MAAMA;IA6BX,MAAc,oBAAiD;QAC7D,IAAI;YACF,MAAMC,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,OAAOA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,gBAAgB;QACpC,EAAE,OAAOC,OAAO;YACdC,QAAQ,KAAK,CAAC,oCAAoCD;QACpD;IAEF;IAEQ,mBACNE,IAAmB,EACnBC,mBAAmD,EACnDC,KAAyB,EACnB;QACN,MAAMC,SAASD;QACf,MAAME,aACJ,AAA+B,YAA/B,OAAOH,sBACHA,sBACAA,QAAAA,sBAAAA,KAAAA,IAAAA,oBAAqB,gBAAgB;QAC3C,IAAI,CAACE,UAAU,CAACC,YACd;QAGF,MAAMC,eAAsC;YAC1C,MAAM;YACN,IAAIC,KAAK,GAAG;YACZF;YACAD;QACF;QAEA,IAAI,CAACH,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACK;aAAa;YAC9B;QACF;QACAL,KAAK,QAAQ,CAAC,IAAI,CAACK;IACrB;IAEQ,kBAAkBL,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQ;YACR,GAAGA,IAAI;QACT;IACF;IAEQ,gCACNO,YAAoB,EACG;QACvB,IAAK,IAAIC,IAAID,eAAe,GAAGC,KAAK,GAAGA,IAAK;YAC1C,MAAMC,YAAY,IAAI,CAAC,KAAK,CAACD,EAAE;YAC/B,IAAI,AAACC,cAAaA,UAAU,OAAO,EAGnC;gBAAA,IAAIA,UAAU,SAAS,EACrB,OAAOA,UAAU,SAAS;YAC5B;QACF;IAEF;IAEA,MAAM,OAAOT,IAA+C,EAAiB;YAGbU,uBAAkCC;QAFhGC,OACE,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACX,CAAC,yDAAyD,EAAE,QAAAF,CAAAA,wBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,sBAAwB,KAAK,CAAC,EAAE,EAAE,QAAAC,CAAAA,yBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,uBAAwB,UAAU,EAAE;QAEpI,IAAIE,MAAM,OAAO,CAACb,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACc,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACd;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;IAElB;IAEA,MAAM,eACJA,IAA+C,EACS;QACxD,MAAM,IAAI,CAAC,MAAM,CAACA;QAClB,OAAO,IAAI,CAAC,KAAK;IACnB;IAEA,MAAM,QAAgE;QACpE,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDD,QAAQ,IAAI,CACV;QAIJa,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QACpCA,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM,EAAc;QAEhC,MAAMG,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACf,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIe,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMhB,OAAO,IAAI,CAAC,KAAK,CAACgB,UAAU;YAClCJ,OACEZ,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOM,KAAK,GAAG;YACjB;YACA,IAAI;gBACFN,KAAK,MAAM,GAAG;gBACd,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOmB,GAAG;oBACVpB,QAAQ,KAAK,CAAC,wBAAwBoB;gBACxC;gBACAP,OACE;oBAAC;oBAAW;oBAAU;iBAAW,CAAC,OAAO,CAACZ,KAAK,IAAI,KAAK,GACxD,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEoB,QAAQ,EAAEC,KAAK,EAAE,GAAGrB;gBAC5BY,OAAOQ,UAAU,CAAC,oCAAoC,EAAEpB,KAAK,IAAI,EAAE;gBAEnE,IAAIsB;gBACJ,IAAIzB;gBACJ,IAAIG,KAAK,OAAO,EAAE;oBAChBH,YAAY,IAAI,CAAC,+BAA+B,CAACmB;oBACjDJ,OACEf,WACA;gBAEJ,OACEA,YAAY,MAAM,IAAI,CAAC,gBAAgB;gBAEzCG,KAAK,SAAS,GAAGH;gBACjB,MAAM0B,kBAAmC;oBACvCvB;oBACA,SAASkB,QAAAA,qBAAAA,KAAAA,IAAAA,mBAAoB,OAAO;oBACpCrB;gBACF;gBAEA,IAAIG,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BY,OACEZ,AAAiB,aAAjBA,KAAK,OAAO,IACVA,AAAiB,YAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;oBACzC,IAAIvB,AAAiB,aAAjBA,KAAK,OAAO,EACdkB,qBACEI,QAAAA,cAAAA,KAAAA,IAAAA,YACC,MAAM;gBAEb,OAAO,IAAItB,AAAc,aAAdA,KAAK,IAAI,IAAiBA,AAAc,eAAdA,KAAK,IAAI,EAC5CsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;qBACpC;oBACLxB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;gBAC3C;gBAEA,MAAMC,aAAaR,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIQ,YAAY;oBACd,MAAMpB,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACJ,MAAMI,YAAY;gBAC5C;gBAEAqB,OAAO,MAAM,CAACzB,MAAMsB;gBACpBtB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGM,KAAK,GAAG;gBAC1BN,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtDgB;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBjB,KAAK,KAAK,GAAGmB;gBACbnB,KAAK,YAAY,GACfmB,AAAAA,CAAAA,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO,AAAD,KAAM,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEnB,KAAK,UAAU,GAAGmB,EAAE,KAAK;gBAEzBnB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGM,KAAK,GAAG;gBAC1BN,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD;YACF;QACF;QAGA,IAAK,IAAIQ,IAAIQ,YAAY,GAAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAGzB,IAAIS,uBACF,IAAI,CAAC,MAAM,GAAG;aAEd,IAAI,CAAC,MAAM,GAAG;QAGhB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMS,cAAcC,KAAK,GAAG,CAACX,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEY,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAET,MAAMC,iBAAiB,IAAI,CAAC,KAAK,CAAC,SAAS,CACzC,CAAC9B,OAASA,AAAgB,aAAhBA,KAAK,MAAM;QAEvB,IAAI8B,kBAAkB,GACpB,OAAO,IAAI,CAAC,KAAK,CAACA,eAAe;QAEnC,OAAO;IACT;IAEA,OAAsB;QACpB,MAAMC,WAA0B;YAC9B,SAASzB,KAAK,GAAG;YACjB,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;QACA,OAAOyB;IACT;IAEA,MAAM,gBAAgBC,QAAgB,EAGnC;QACD,MAAMC,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASD;YACX;YACA,SAASA;YACT,QAAQ;YACR,UAAU;gBACR,MAAM,IAAIE,MAAMF,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACC;QAE1B,OAAO;YACL,QAAQE;YACR,QAAQ,IAAI;QACd;IACF;IAlSA,YACEC,IAAY,EACZC,gBAA0C,EAC1CC,OAEC,CACD;QAjBF;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QASE,IAAI,CAAC,MAAM,GACTA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAKA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAGF;QACZ,IAAI,CAAC,KAAK,GAAIE,AAAAA,CAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAK,EAAC,EAAG,GAAG,CAAC,CAACxB,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGwB,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,WAAW;QACvC,IAAI,CAAC,gBAAgB,GAAGD;IAC1B;AAoRF"}

package/dist/es/types.mjs

@@ -1,4 +1,14 @@
 export * from "./yaml.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
 var types_AIResponseFormat = /*#__PURE__*/ function(AIResponseFormat) {
     AIResponseFormat["JSON"] = "json_object";
     AIResponseFormat["TEXT"] = "text";
@@ -6,6 +16,13 @@ var types_AIResponseFormat = /*#__PURE__*/ function(AIResponseFormat) {
 }({});
 class UIContext {
 }
-export { types_AIResponseFormat as AIResponseFormat, UIContext };
+class InsightError extends Error {
+    constructor(message, dump){
+        super(message), _define_property(this, "dump", void 0);
+        this.name = 'InsightError';
+        this.dump = dump;
+    }
+}
+export { types_AIResponseFormat as AIResponseFormat, InsightError, UIContext };
 
 //# sourceMappingURL=types.mjs.map

package/dist/es/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"types.mjs","sources":["webpack://@midscene/core/./src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { TModelConfigFn } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\nexport interface AIElementLocatorResponse {\n  elements: {\n    id: string;\n    reason?: string;\n    text?: string;\n    xpaths?: string[];\n  }[];\n  bbox?: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse =\n  | AIElementLocatorResponse\n  | AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext<ElementType extends BaseElement = BaseElement> {\n  abstract screenshotBase64: string;\n\n  abstract tree: ElementTreeNode<ElementType>;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type InsightExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport type LocateResultElement = {\n  center: [number, number];\n  rect: Rect;\n  id: string;\n  indexId?: number;\n  xpaths: string[];\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n  isOrderSensitive?: boolean;\n};\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface InsightTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface InsightDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: InsightExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: BaseElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: InsightTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialInsightDumpFromSDK = Omit<\n  InsightDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type InsightAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  id?: string;\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface PlanningAIResponse {\n  action?: PlanningAction; // this is the qwen mode\n  actions?: PlanningAction[];\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport type PlanningActionParamTap = null;\nexport type PlanningActionParamHover = null;\nexport type PlanningActionParamRightClick = null;\n\nexport interface PlanningActionParamInputOrKeyPress {\n  value: string;\n  autoDismissKeyboard?: boolean;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface AndroidLongPressParam {\n  duration?: number;\n}\n\nexport interface AndroidPullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActionContext?: string;\n}\n\n/*\ntask - insight-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = InsightDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - insight-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: InsightExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  InsightAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext<WebElementInfo>;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n  cache?: Cache;\n  replanningCycleLimit?: number;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["AIResponseFormat","UIContext"],"mappings":";AAqCO,IAAKA,yBAAgBA,WAAAA,GAAAA,SAAhBA,gBAAgB;;;WAAhBA;;AAwFL,MAAeC;AAQtB"}
+{"version":3,"file":"types.mjs","sources":["webpack://@midscene/core/./src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type {\n  CreateOpenAIClientFn,\n  TModelConfigFn,\n} from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\nexport interface AIElementLocatorResponse {\n  elements: {\n    id: string;\n    reason?: string;\n    text?: string;\n    xpaths?: string[];\n  }[];\n  bbox?: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse =\n  | AIElementLocatorResponse\n  | AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext<ElementType extends BaseElement = BaseElement> {\n  abstract screenshotBase64: string;\n\n  abstract tree: ElementTreeNode<ElementType>;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type InsightExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport type LocateResultElement = {\n  center: [number, number];\n  rect: Rect;\n  id: string;\n  indexId?: number;\n  xpaths: string[];\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n  isOrderSensitive?: boolean;\n};\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface InsightTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface InsightDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: InsightExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: BaseElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: InsightTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialInsightDumpFromSDK = Omit<\n  InsightDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface InsightResultBase {\n  dump: InsightDump;\n}\n\nexport type LocateResultWithDump = LocateResult & InsightResultBase;\n\nexport interface InsightExtractResult<T> extends InsightResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n}\n\nexport class InsightError extends Error {\n  dump: InsightDump;\n\n  constructor(message: string, dump: InsightDump) {\n    super(message);\n    this.name = 'InsightError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type InsightAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  id?: string;\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  subTask?: boolean;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActionContext?: string;\n}\n\n/*\ntask - insight-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = InsightDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - insight-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: InsightExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  InsightAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext<WebElementInfo>;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n  cache?: Cache;\n  replanningCycleLimit?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: (config) => {\n   *   const openai = new OpenAI({\n   *     apiKey: config.openaiApiKey,\n   *     baseURL: config.openaiBaseURL,\n   *   });\n   *\n   *   // Wrap with langsmith for planning tasks\n   *   if (config.intent === 'planning') {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["AIResponseFormat","UIContext","InsightError","Error","message","dump"],"mappings":";AAAqD;;;;;;;;;;AAwC9C,IAAKA,yBAAgBA,WAAAA,GAAAA,SAAhBA,gBAAgB;;;WAAhBA;;AAwFL,MAAeC;AAQtB;AAmFO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF"}