@midscene/core 1.15.5 → 1.15.6

package/dist/lib/ai-model/workflows/insight/boolean.js

@@ -0,0 +1,54 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    extractBoolean: ()=>extractBoolean
+});
+const utils_namespaceObject = require("@midscene/shared/utils");
+async function extractBoolean(options) {
+    const { booleanFn } = options.modelRuntime.adapter.insight;
+    (0, utils_namespaceObject.assert)(booleanFn, 'A boolean adapter is required');
+    const keys = 'object' == typeof options.dataQuery ? Object.keys(options.dataQuery) : [];
+    (0, utils_namespaceObject.assert)(1 === keys.length && ('Boolean' === keys[0] || 'StatementIsTruthy' === keys[0]), 'This model only supports boolean insight tasks. Use the default model for extraction.');
+    const result = await booleanFn(options);
+    return {
+        parseResult: {
+            data: {
+                [keys[0]]: result.value
+            }
+        },
+        rawResponse: result.rawResponse,
+        usage: result.usage
+    };
+}
+exports.extractBoolean = __webpack_exports__.extractBoolean;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "extractBoolean"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=boolean.js.map

package/dist/lib/ai-model/workflows/insight/boolean.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/insight/boolean.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/boolean.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIDataExtractionResponse,\n  AIUsageInfo,\n  RawAssistantOutput,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport type { ExtractOptions } from './extraction';\n\nexport type BooleanFn = (options: ExtractOptions) => Promise<{\n  value: boolean;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}>;\n\nexport async function extractBoolean<T>(options: ExtractOptions): Promise<{\n  parseResult: AIDataExtractionResponse<T>;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  rawAssistantOutput?: RawAssistantOutput;\n  reasoning_content?: string;\n}> {\n  const { booleanFn } = options.modelRuntime.adapter.insight;\n  assert(booleanFn, 'A boolean adapter is required');\n  const keys =\n    typeof options.dataQuery === 'object' ? Object.keys(options.dataQuery) : [];\n  assert(\n    keys.length === 1 &&\n      (keys[0] === 'Boolean' || keys[0] === 'StatementIsTruthy'),\n    'This model only supports boolean insight tasks. Use the default model for extraction.',\n  );\n  const result = await booleanFn(options);\n  return {\n    parseResult: {\n      data: { [keys[0]]: result.value } as T,\n    },\n    rawResponse: result.rawResponse,\n    usage: result.usage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","extractBoolean","options","booleanFn","assert","keys","result"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACQO,eAAeI,eAAkBC,OAAuB;IAO7D,MAAM,EAAEC,SAAS,EAAE,GAAGD,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO;IAC1DE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;IAClB,MAAME,OACJ,AAA6B,YAA7B,OAAOH,QAAQ,SAAS,GAAgBL,OAAO,IAAI,CAACK,QAAQ,SAAS,IAAI,EAAE;IAC7EE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,AAAgB,MAAhBA,KAAK,MAAM,IACRA,CAAAA,AAAY,cAAZA,IAAI,CAAC,EAAE,IAAkBA,AAAY,wBAAZA,IAAI,CAAC,EAAE,AAAuB,GAC1D;IAEF,MAAMC,SAAS,MAAMH,UAAUD;IAC/B,OAAO;QACL,aAAa;YACX,MAAM;gBAAE,CAACG,IAAI,CAAC,EAAE,CAAC,EAAEC,OAAO,KAAK;YAAC;QAClC;QACA,aAAaA,OAAO,WAAW;QAC/B,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/lib/ai-model/workflows/insight/extraction.js

@@ -32,9 +32,11 @@ const index_js_namespaceObject = require("../../prompt/insight/index.js");
 const external_service_caller_index_js_namespaceObject = require("../../service-caller/index.js");
 const semantic_retry_js_namespaceObject = require("../../service-caller/semantic-retry.js");
 const multimodal_prompt_js_namespaceObject = require("../../shared/multimodal-prompt.js");
+const external_boolean_js_namespaceObject = require("./boolean.js");
 const external_insight_response_parser_js_namespaceObject = require("./insight-response-parser.js");
 const debugInsight = (0, logger_namespaceObject.getDebug)('ai:insight');
 async function AiExtractElementInfo(options) {
+    if (options.modelRuntime.adapter.insight.booleanFn) return (0, external_boolean_js_namespaceObject.extractBoolean)(options);
     const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } = options;
     const insightProtocol = modelRuntime.adapter.insight.protocol;
     const systemPrompt = (0, index_js_namespaceObject.buildInsightSystemPrompt)({

package/dist/lib/ai-model/workflows/insight/extraction.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInsight","getDebug","AiExtractElementInfo","options","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACiBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEvB,eAAeC,qBAAwBC,OAQ7C;IACC,MAAM,EAAEC,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEL;IACF,MAAMM,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAC5Bb,QAAQ,eAAe,IAAI,IAC3BC,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,AAAAA,IAAAA,qCAAAA,0BAAAA,AAAAA,EAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaL,QAAQ,WAAW;gBAChC,sBAAsBsB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,AAAAA,IAAAA,oDAAAA,oBAAAA,AAAAA,EAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaL,QAAQ,WAAW;QAChC,cAAc,CAACsC;YACbzC,aACE,iDACAyC,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { extractBoolean } from './boolean';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport interface ExtractOptions {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}\n\nexport async function AiExtractElementInfo<T>(options: ExtractOptions) {\n  if (options.modelRuntime.adapter.insight.booleanFn) {\n    return extractBoolean<T>(options);\n  }\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInsight","getDebug","AiExtractElementInfo","options","extractBoolean","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACkBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAYvB,eAAeC,qBAAwBC,OAAuB;IACnE,IAAIA,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAChD,OAAOC,AAAAA,IAAAA,oCAAAA,cAAAA,AAAAA,EAAkBD;IAE3B,MAAM,EAAEE,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEN;IACF,MAAMO,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAC5Bd,QAAQ,eAAe,IAAI,IAC3BE,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,AAAAA,IAAAA,qCAAAA,0BAAAA,AAAAA,EAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaN,QAAQ,WAAW;gBAChC,sBAAsBuB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,AAAAA,IAAAA,oDAAAA,oBAAAA,AAAAA,EAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaN,QAAQ,WAAW;QAChC,cAAc,CAACuC;YACb1C,aACE,iDACA0C,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}