@midscene/core 1.10.1 → 1.10.2

package/dist/lib/element-describer.js.map

@@ -1 +1 @@
-{"version":3,"file":"element-describer.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/element-describer.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ModelRuntime } from '@/ai-model/models';\nimport { assert } from '@midscene/shared/utils';\nimport { createScreenshotBoundUIContext } from './agent/utils';\nimport type Service from './service';\nimport type {\n  AgentDescribeElementAtPointResult,\n  LocateOption,\n  LocateResultElement,\n  LocateValidatorResult,\n  LocatorValidatorOption,\n  Rect,\n  Size,\n  UIContext,\n} from './types';\nimport { buildDetailedLocateParam } from './yaml/utils';\n\nexport type DescribeElementCoordinateSpace = 'screenshot' | 'logical';\n\nexport type LocatorVerifyFn = (input: {\n  prompt: string;\n  expectCenter: [number, number];\n  deepLocate: boolean;\n  retryCount: number;\n  verifyResult: LocateValidatorResult;\n}) => LocateValidatorResult | boolean;\n\nexport type DescribeElementAtPointOptions = {\n  verifyPrompt?: boolean;\n  retryLimit?: number;\n  deepLocate?: boolean;\n  locatorVerifyFn?: LocatorVerifyFn;\n  screenshotBase64?: string;\n  screenshotSize?: Size;\n  coordinateSpace?: DescribeElementCoordinateSpace;\n  logicalSize?: Size;\n  onProgress?: (progress: {\n    prompt?: string;\n    deepLocate?: boolean;\n    verifyResult?: LocateValidatorResult;\n  }) => void;\n} & LocatorValidatorOption;\n\ntype ScreenshotBoundContextOptions = {\n  screenshotBase64?: string;\n  screenshotSize?: Size;\n  coordinateSpace?: DescribeElementCoordinateSpace;\n  logicalSize?: Size;\n};\n\nexport type VerifyElementDescriptionAtPointOptions =\n  ScreenshotBoundContextOptions & LocatorValidatorOption;\n\nexport type VerifyElementByServiceLocateOptions =\n  VerifyElementDescriptionAtPointOptions &\n    Pick<LocateOption, 'cacheable' | 'deepLocate' | 'xpath'> & {\n      abortSignal?: AbortSignal;\n    };\n\nexport type ElementDescriberRuntime = {\n  service: Pick<Service, 'describe' | 'locate'>;\n  describeModelRuntime: ModelRuntime;\n  locateModelRuntime: ModelRuntime;\n};\n\nconst distanceOfTwoPoints = (p1: [number, number], p2: [number, number]) => {\n  const [x1, y1] = p1;\n  const [x2, y2] = p2;\n  return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));\n};\n\nconst includedInRect = (point: [number, number], rect: Rect) => {\n  const [x, y] = point;\n  const { left, top, width, height } = rect;\n  return x >= left && x <= left + width && y >= top && y <= top + height;\n};\n\nconst buildLocateValidatorResult = (\n  expectCenter: [number, number],\n  located: Pick<LocateResultElement, 'center' | 'rect'>,\n  verifyLocateOption?: LocatorValidatorOption,\n): LocateValidatorResult => {\n  const distance = distanceOfTwoPoints(expectCenter, located.center);\n  const included = includedInRect(expectCenter, located.rect);\n  const pass =\n    distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;\n  return {\n    pass,\n    rect: located.rect,\n    center: located.center,\n    centerDistance: distance,\n    includedInRect: included,\n  };\n};\n\nfunction assertPositiveSize(\n  size: Size | undefined,\n  label: string,\n): asserts size is Size {\n  assert(\n    size &&\n      Number.isFinite(size.width) &&\n      Number.isFinite(size.height) &&\n      size.width > 0 &&\n      size.height > 0,\n    `${label} must include positive width and height`,\n  );\n}\n\nconst mapPointToScreenshotSpace = (\n  center: [number, number],\n  screenshotSize: Size,\n  opt: ScreenshotBoundContextOptions,\n): [number, number] => {\n  const coordinateSpace = opt.coordinateSpace || 'screenshot';\n  if (coordinateSpace === 'screenshot') {\n    return center;\n  }\n\n  assertPositiveSize(\n    opt.logicalSize,\n    'logicalSize is required when coordinateSpace is logical',\n  );\n  return [\n    (center[0] * screenshotSize.width) / opt.logicalSize.width,\n    (center[1] * screenshotSize.height) / opt.logicalSize.height,\n  ];\n};\n\nconst createScreenshotBoundLocatorContext = async (\n  center: [number, number],\n  opt?: ScreenshotBoundContextOptions,\n): Promise<{\n  screenshotContext?: UIContext;\n  locateOpt?: LocateOption;\n  targetCenter: [number, number];\n}> => {\n  const screenshotContext = opt?.screenshotBase64\n    ? await createScreenshotBoundUIContext(opt.screenshotBase64, opt)\n    : undefined;\n  const targetCenter = screenshotContext\n    ? mapPointToScreenshotSpace(center, screenshotContext.shotSize, opt || {})\n    : center;\n  return {\n    screenshotContext,\n    locateOpt: screenshotContext ? { uiContext: screenshotContext } : undefined,\n    targetCenter,\n  };\n};\n\nexport async function verifyLocator(\n  runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>,\n  prompt: string,\n  locateOpt: LocateOption | undefined,\n  expectCenter: [number, number],\n  verifyLocateOption?: LocatorValidatorOption,\n): Promise<LocateValidatorResult> {\n  return locateAndVerify(runtime, prompt, expectCenter, {\n    centerDistanceThreshold: verifyLocateOption?.centerDistanceThreshold,\n    uiContext: locateOpt?.uiContext,\n  });\n}\n\nfunction applyLocatorVerifyFn(\n  locatorVerifyFn: LocatorVerifyFn | undefined,\n  input: Parameters<LocatorVerifyFn>[0],\n): LocateValidatorResult {\n  if (!locatorVerifyFn) {\n    return input.verifyResult;\n  }\n  const customResult = locatorVerifyFn(input);\n  if (typeof customResult === 'boolean') {\n    return {\n      ...input.verifyResult,\n      pass: customResult,\n    };\n  }\n  return customResult;\n}\n\nfunction errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nexport async function describeElementAtPoint(\n  runtime: ElementDescriberRuntime,\n  center: [number, number],\n  opt?: DescribeElementAtPointOptions,\n): Promise<AgentDescribeElementAtPointResult> {\n  const { verifyPrompt = true, retryLimit = 3 } = opt || {};\n  const { screenshotContext, locateOpt, targetCenter } =\n    await createScreenshotBoundLocatorContext(center, opt);\n\n  let success = false;\n  let retryCount = 0;\n  let resultPrompt = '';\n  let deepLocate = opt?.deepLocate || false;\n  let verifyResult: LocateValidatorResult | undefined;\n  let lastError: string | undefined;\n  let failureStage: AgentDescribeElementAtPointResult['failureStage'];\n\n  while (!success && retryCount < retryLimit) {\n    if (retryCount >= 2) {\n      deepLocate = true;\n    }\n    const describeOpt = screenshotContext\n      ? { deepLocate, context: screenshotContext }\n      : { deepLocate };\n\n    let text: Awaited<\n      ReturnType<ElementDescriberRuntime['service']['describe']>\n    >;\n    try {\n      text = await runtime.service.describe(\n        targetCenter,\n        runtime.describeModelRuntime,\n        describeOpt,\n      );\n    } catch (error) {\n      return {\n        prompt: resultPrompt,\n        deepLocate,\n        verifyResult,\n        success: false,\n        error: errorMessage(error),\n        failureStage: 'describe',\n      };\n    }\n    if (!text.description) {\n      return {\n        prompt: resultPrompt,\n        deepLocate,\n        verifyResult,\n        success: false,\n        error: `failed to describe element at [${targetCenter}]`,\n        failureStage: 'describe',\n      };\n    }\n    resultPrompt = text.description;\n\n    if (!verifyPrompt) {\n      opt?.onProgress?.({ prompt: resultPrompt, deepLocate });\n      success = true;\n      break;\n    }\n\n    try {\n      verifyResult = await verifyLocator(\n        runtime,\n        resultPrompt,\n        locateOpt,\n        targetCenter,\n        opt,\n      );\n      verifyResult = applyLocatorVerifyFn(opt?.locatorVerifyFn, {\n        prompt: resultPrompt,\n        expectCenter: targetCenter,\n        deepLocate,\n        retryCount,\n        verifyResult,\n      });\n      opt?.onProgress?.({ prompt: resultPrompt, deepLocate, verifyResult });\n      if (verifyResult.pass) {\n        success = true;\n      } else {\n        lastError = undefined;\n        failureStage = 'verify';\n        retryCount++;\n      }\n    } catch (error) {\n      lastError = errorMessage(error);\n      failureStage = 'verify';\n      opt?.onProgress?.({ prompt: resultPrompt, deepLocate });\n      retryCount++;\n    }\n  }\n\n  return {\n    prompt: resultPrompt,\n    deepLocate,\n    verifyResult,\n    success,\n    error:\n      success || !verifyPrompt\n        ? undefined\n        : lastError || 'describeElementAtPoint verify failed',\n    failureStage: success ? undefined : failureStage,\n  };\n}\n\nexport async function verifyElementDescriptionAtPoint(\n  runtime: ElementDescriberRuntime,\n  description: string,\n  center: [number, number],\n  opt?: VerifyElementDescriptionAtPointOptions,\n): Promise<LocateValidatorResult> {\n  assert(description?.trim(), 'description must not be empty');\n  const { locateOpt, targetCenter } = await createScreenshotBoundLocatorContext(\n    center,\n    opt,\n  );\n\n  return verifyLocator(runtime, description, locateOpt, targetCenter, opt);\n}\n\nexport async function verifyElementByServiceLocate(\n  runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>,\n  description: string,\n  center: [number, number],\n  opt?: VerifyElementByServiceLocateOptions,\n): Promise<LocateValidatorResult> {\n  return locateAndVerify(runtime, description, center, opt);\n}\n\nasync function locateAndVerify(\n  runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>,\n  description: string,\n  center: [number, number],\n  opt?: VerifyElementByServiceLocateOptions & { uiContext?: UIContext },\n): Promise<LocateValidatorResult> {\n  assert(description?.trim(), 'description must not be empty');\n  const { screenshotContext, targetCenter } =\n    await createScreenshotBoundLocatorContext(center, opt);\n  const context = opt?.uiContext || screenshotContext;\n  const locateParam = buildDetailedLocateParam(description, {\n    cacheable: opt?.cacheable,\n    deepLocate: opt?.deepLocate,\n    xpath: opt?.xpath,\n  });\n  assert(locateParam, 'cannot get locate param for service locate');\n\n  const locateResult = await runtime.service.locate(\n    locateParam,\n    context ? { context } : {},\n    runtime.locateModelRuntime,\n    opt?.abortSignal,\n  );\n  assert(locateResult.element, `Element not found: ${description}`);\n  return buildLocateValidatorResult(targetCenter, locateResult.element, opt);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","distanceOfTwoPoints","p1","p2","x1","y1","x2","y2","Math","includedInRect","point","rect","x","y","left","top","width","height","buildLocateValidatorResult","expectCenter","located","verifyLocateOption","distance","included","pass","assertPositiveSize","size","label","assert","Number","mapPointToScreenshotSpace","center","screenshotSize","opt","coordinateSpace","createScreenshotBoundLocatorContext","screenshotContext","createScreenshotBoundUIContext","undefined","targetCenter","verifyLocator","runtime","prompt","locateOpt","locateAndVerify","applyLocatorVerifyFn","locatorVerifyFn","input","customResult","errorMessage","error","Error","String","describeElementAtPoint","verifyPrompt","retryLimit","success","retryCount","resultPrompt","deepLocate","verifyResult","lastError","failureStage","describeOpt","text","verifyElementDescriptionAtPoint","description","verifyElementByServiceLocate","context","locateParam","buildDetailedLocateParam","locateResult"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;AC0DA,MAAMI,sBAAsB,CAACC,IAAsBC;IACjD,MAAM,CAACC,IAAIC,GAAG,GAAGH;IACjB,MAAM,CAACI,IAAIC,GAAG,GAAGJ;IACjB,OAAOK,KAAK,KAAK,CAACA,KAAK,IAAI,CAAEJ,AAAAA,CAAAA,KAAKE,EAAC,KAAM,IAAKD,AAAAA,CAAAA,KAAKE,EAAC,KAAM;AAC5D;AAEA,MAAME,iBAAiB,CAACC,OAAyBC;IAC/C,MAAM,CAACC,GAAGC,EAAE,GAAGH;IACf,MAAM,EAAEI,IAAI,EAAEC,GAAG,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGN;IACrC,OAAOC,KAAKE,QAAQF,KAAKE,OAAOE,SAASH,KAAKE,OAAOF,KAAKE,MAAME;AAClE;AAEA,MAAMC,6BAA6B,CACjCC,cACAC,SACAC;IAEA,MAAMC,WAAWrB,oBAAoBkB,cAAcC,QAAQ,MAAM;IACjE,MAAMG,WAAWd,eAAeU,cAAcC,QAAQ,IAAI;IAC1D,MAAMI,OACJF,YAAaD,CAAAA,oBAAoB,2BAA2B,EAAC,KAAME;IACrE,OAAO;QACLC;QACA,MAAMJ,QAAQ,IAAI;QAClB,QAAQA,QAAQ,MAAM;QACtB,gBAAgBE;QAChB,gBAAgBC;IAClB;AACF;AAEA,SAASE,mBACPC,IAAsB,EACtBC,KAAa;IAEbC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEF,QACEG,OAAO,QAAQ,CAACH,KAAK,KAAK,KAC1BG,OAAO,QAAQ,CAACH,KAAK,MAAM,KAC3BA,KAAK,KAAK,GAAG,KACbA,KAAK,MAAM,GAAG,GAChB,GAAGC,MAAM,uCAAuC,CAAC;AAErD;AAEA,MAAMG,4BAA4B,CAChCC,QACAC,gBACAC;IAEA,MAAMC,kBAAkBD,IAAI,eAAe,IAAI;IAC/C,IAAIC,AAAoB,iBAApBA,iBACF,OAAOH;IAGTN,mBACEQ,IAAI,WAAW,EACf;IAEF,OAAO;QACJF,MAAM,CAAC,EAAE,GAAGC,eAAe,KAAK,GAAIC,IAAI,WAAW,CAAC,KAAK;QACzDF,MAAM,CAAC,EAAE,GAAGC,eAAe,MAAM,GAAIC,IAAI,WAAW,CAAC,MAAM;KAC7D;AACH;AAEA,MAAME,sCAAsC,OAC1CJ,QACAE;IAMA,MAAMG,oBAAoBH,KAAK,mBAC3B,MAAMI,AAAAA,IAAAA,yBAAAA,8BAAAA,AAAAA,EAA+BJ,IAAI,gBAAgB,EAAEA,OAC3DK;IACJ,MAAMC,eAAeH,oBACjBN,0BAA0BC,QAAQK,kBAAkB,QAAQ,EAAEH,OAAO,CAAC,KACtEF;IACJ,OAAO;QACLK;QACA,WAAWA,oBAAoB;YAAE,WAAWA;QAAkB,IAAIE;QAClEC;IACF;AACF;AAEO,eAAeC,cACpBC,OAAwE,EACxEC,MAAc,EACdC,SAAmC,EACnCxB,YAA8B,EAC9BE,kBAA2C;IAE3C,OAAOuB,gBAAgBH,SAASC,QAAQvB,cAAc;QACpD,yBAAyBE,oBAAoB;QAC7C,WAAWsB,WAAW;IACxB;AACF;AAEA,SAASE,qBACPC,eAA4C,EAC5CC,KAAqC;IAErC,IAAI,CAACD,iBACH,OAAOC,MAAM,YAAY;IAE3B,MAAMC,eAAeF,gBAAgBC;IACrC,IAAI,AAAwB,aAAxB,OAAOC,cACT,OAAO;QACL,GAAGD,MAAM,YAAY;QACrB,MAAMC;IACR;IAEF,OAAOA;AACT;AAEA,SAASC,aAAaC,KAAc;IAClC,OAAOA,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;AACzD;AAEO,eAAeG,uBACpBZ,OAAgC,EAChCV,MAAwB,EACxBE,GAAmC;IAEnC,MAAM,EAAEqB,eAAe,IAAI,EAAEC,aAAa,CAAC,EAAE,GAAGtB,OAAO,CAAC;IACxD,MAAM,EAAEG,iBAAiB,EAAEO,SAAS,EAAEJ,YAAY,EAAE,GAClD,MAAMJ,oCAAoCJ,QAAQE;IAEpD,IAAIuB,UAAU;IACd,IAAIC,aAAa;IACjB,IAAIC,eAAe;IACnB,IAAIC,aAAa1B,KAAK,cAAc;IACpC,IAAI2B;IACJ,IAAIC;IACJ,IAAIC;IAEJ,MAAO,CAACN,WAAWC,aAAaF,WAAY;QAC1C,IAAIE,cAAc,GAChBE,aAAa;QAEf,MAAMI,cAAc3B,oBAChB;YAAEuB;YAAY,SAASvB;QAAkB,IACzC;YAAEuB;QAAW;QAEjB,IAAIK;QAGJ,IAAI;YACFA,OAAO,MAAMvB,QAAQ,OAAO,CAAC,QAAQ,CACnCF,cACAE,QAAQ,oBAAoB,EAC5BsB;QAEJ,EAAE,OAAOb,OAAO;YACd,OAAO;gBACL,QAAQQ;gBACRC;gBACAC;gBACA,SAAS;gBACT,OAAOX,aAAaC;gBACpB,cAAc;YAChB;QACF;QACA,IAAI,CAACc,KAAK,WAAW,EACnB,OAAO;YACL,QAAQN;YACRC;YACAC;YACA,SAAS;YACT,OAAO,CAAC,+BAA+B,EAAErB,aAAa,CAAC,CAAC;YACxD,cAAc;QAChB;QAEFmB,eAAeM,KAAK,WAAW;QAE/B,IAAI,CAACV,cAAc;YACjBrB,KAAK,aAAa;gBAAE,QAAQyB;gBAAcC;YAAW;YACrDH,UAAU;YACV;QACF;QAEA,IAAI;YACFI,eAAe,MAAMpB,cACnBC,SACAiB,cACAf,WACAJ,cACAN;YAEF2B,eAAef,qBAAqBZ,KAAK,iBAAiB;gBACxD,QAAQyB;gBACR,cAAcnB;gBACdoB;gBACAF;gBACAG;YACF;YACA3B,KAAK,aAAa;gBAAE,QAAQyB;gBAAcC;gBAAYC;YAAa;YACnE,IAAIA,aAAa,IAAI,EACnBJ,UAAU;iBACL;gBACLK,YAAYvB;gBACZwB,eAAe;gBACfL;YACF;QACF,EAAE,OAAOP,OAAO;YACdW,YAAYZ,aAAaC;YACzBY,eAAe;YACf7B,KAAK,aAAa;gBAAE,QAAQyB;gBAAcC;YAAW;YACrDF;QACF;IACF;IAEA,OAAO;QACL,QAAQC;QACRC;QACAC;QACAJ;QACA,OACEA,WAAW,CAACF,eACRhB,SACAuB,aAAa;QACnB,cAAcL,UAAUlB,SAAYwB;IACtC;AACF;AAEO,eAAeG,gCACpBxB,OAAgC,EAChCyB,WAAmB,EACnBnC,MAAwB,EACxBE,GAA4C;IAE5CL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsC,aAAa,QAAQ;IAC5B,MAAM,EAAEvB,SAAS,EAAEJ,YAAY,EAAE,GAAG,MAAMJ,oCACxCJ,QACAE;IAGF,OAAOO,cAAcC,SAASyB,aAAavB,WAAWJ,cAAcN;AACtE;AAEO,eAAekC,6BACpB1B,OAAwE,EACxEyB,WAAmB,EACnBnC,MAAwB,EACxBE,GAAyC;IAEzC,OAAOW,gBAAgBH,SAASyB,aAAanC,QAAQE;AACvD;AAEA,eAAeW,gBACbH,OAAwE,EACxEyB,WAAmB,EACnBnC,MAAwB,EACxBE,GAAqE;IAErEL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsC,aAAa,QAAQ;IAC5B,MAAM,EAAE9B,iBAAiB,EAAEG,YAAY,EAAE,GACvC,MAAMJ,oCAAoCJ,QAAQE;IACpD,MAAMmC,UAAUnC,KAAK,aAAaG;IAClC,MAAMiC,cAAcC,AAAAA,IAAAA,uCAAAA,wBAAAA,AAAAA,EAAyBJ,aAAa;QACxD,WAAWjC,KAAK;QAChB,YAAYA,KAAK;QACjB,OAAOA,KAAK;IACd;IACAL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOyC,aAAa;IAEpB,MAAME,eAAe,MAAM9B,QAAQ,OAAO,CAAC,MAAM,CAC/C4B,aACAD,UAAU;QAAEA;IAAQ,IAAI,CAAC,GACzB3B,QAAQ,kBAAkB,EAC1BR,KAAK;IAEPL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2C,aAAa,OAAO,EAAE,CAAC,mBAAmB,EAAEL,aAAa;IAChE,OAAOhD,2BAA2BqB,cAAcgC,aAAa,OAAO,EAAEtC;AACxE"}
+{"version":3,"file":"element-describer.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/element-describer.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ModelRuntime } from '@/ai-model/models';\nimport { assert } from '@midscene/shared/utils';\nimport { createScreenshotBoundUIContext } from './agent/utils';\nimport type Service from './service';\nimport type {\n  AgentDescribeElementAtPointResult,\n  LocateOption,\n  LocateResultElement,\n  LocateValidatorResult,\n  LocatorValidatorOption,\n  Rect,\n  Size,\n  UIContext,\n} from './types';\nimport { buildDetailedLocateParam } from './yaml/utils';\n\nexport type DescribeElementCoordinateSpace = 'screenshot' | 'logical';\n\nexport type LocatorVerifyFn = (input: {\n  prompt: string;\n  expectCenter: [number, number];\n  retryCount: number;\n  verifyResult: LocateValidatorResult;\n}) => LocateValidatorResult | boolean;\n\nexport type DescribeElementAtPointOptions = {\n  verifyPrompt?: boolean;\n  retryLimit?: number;\n  deepDescribe?: boolean;\n  deepLocate?: boolean;\n  locatorVerifyFn?: LocatorVerifyFn;\n  screenshotBase64?: string;\n  screenshotSize?: Size;\n  coordinateSpace?: DescribeElementCoordinateSpace;\n  logicalSize?: Size;\n  onProgress?: (progress: {\n    prompt?: string;\n    deepDescribe?: boolean;\n    deepLocate?: boolean;\n    verifyResult?: LocateValidatorResult;\n  }) => void;\n} & LocatorValidatorOption;\n\ntype ScreenshotBoundContextOptions = {\n  screenshotBase64?: string;\n  screenshotSize?: Size;\n  coordinateSpace?: DescribeElementCoordinateSpace;\n  logicalSize?: Size;\n};\n\nexport type VerifyElementDescriptionAtPointOptions =\n  ScreenshotBoundContextOptions & LocatorValidatorOption;\n\ntype LocateAndVerifyOptions = VerifyElementDescriptionAtPointOptions &\n  Pick<LocateOption, 'cacheable' | 'deepLocate' | 'xpath'> & {\n    abortSignal?: AbortSignal;\n  };\n\nexport type ElementDescriberRuntime = {\n  service: Pick<Service, 'describe' | 'locate'>;\n  describeModelRuntime: ModelRuntime;\n  locateModelRuntime: ModelRuntime;\n};\n\ntype ServiceDescribeOptions = NonNullable<\n  Parameters<ElementDescriberRuntime['service']['describe']>[2]\n>;\n\nconst distanceOfTwoPoints = (p1: [number, number], p2: [number, number]) => {\n  const [x1, y1] = p1;\n  const [x2, y2] = p2;\n  return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));\n};\n\nconst includedInRect = (point: [number, number], rect: Rect) => {\n  const [x, y] = point;\n  const { left, top, width, height } = rect;\n  return x >= left && x <= left + width && y >= top && y <= top + height;\n};\n\nconst buildLocateValidatorResult = (\n  expectCenter: [number, number],\n  located: Pick<LocateResultElement, 'center' | 'rect'>,\n  verifyLocateOption?: LocatorValidatorOption,\n): LocateValidatorResult => {\n  const distance = distanceOfTwoPoints(expectCenter, located.center);\n  const included = includedInRect(expectCenter, located.rect);\n  const pass =\n    distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;\n  return {\n    pass,\n    rect: located.rect,\n    center: located.center,\n    centerDistance: distance,\n    includedInRect: included,\n  };\n};\n\nfunction assertPositiveSize(\n  size: Size | undefined,\n  label: string,\n): asserts size is Size {\n  assert(\n    size &&\n      Number.isFinite(size.width) &&\n      Number.isFinite(size.height) &&\n      size.width > 0 &&\n      size.height > 0,\n    `${label} must include positive width and height`,\n  );\n}\n\nconst mapPointToScreenshotSpace = (\n  center: [number, number],\n  screenshotSize: Size,\n  opt: ScreenshotBoundContextOptions,\n): [number, number] => {\n  const coordinateSpace = opt.coordinateSpace || 'screenshot';\n  if (coordinateSpace === 'screenshot') {\n    return center;\n  }\n\n  assertPositiveSize(\n    opt.logicalSize,\n    'logicalSize is required when coordinateSpace is logical',\n  );\n  return [\n    (center[0] * screenshotSize.width) / opt.logicalSize.width,\n    (center[1] * screenshotSize.height) / opt.logicalSize.height,\n  ];\n};\n\nconst createScreenshotBoundLocatorContext = async (\n  center: [number, number],\n  opt?: ScreenshotBoundContextOptions,\n): Promise<{\n  screenshotContext?: UIContext;\n  locateOpt?: LocateOption;\n  targetCenter: [number, number];\n}> => {\n  const screenshotContext = opt?.screenshotBase64\n    ? await createScreenshotBoundUIContext(opt.screenshotBase64, opt)\n    : undefined;\n  const targetCenter = screenshotContext\n    ? mapPointToScreenshotSpace(center, screenshotContext.shotSize, opt || {})\n    : center;\n  return {\n    screenshotContext,\n    locateOpt: screenshotContext ? { uiContext: screenshotContext } : undefined,\n    targetCenter,\n  };\n};\n\nexport async function verifyLocator(\n  runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>,\n  prompt: string,\n  locateOpt: LocateOption | undefined,\n  expectCenter: [number, number],\n  verifyLocateOption?: LocatorValidatorOption &\n    Pick<LocateOption, 'deepLocate'>,\n): Promise<LocateValidatorResult> {\n  return locateAndVerify(runtime, prompt, expectCenter, {\n    centerDistanceThreshold: verifyLocateOption?.centerDistanceThreshold,\n    deepLocate: verifyLocateOption?.deepLocate,\n    uiContext: locateOpt?.uiContext,\n  });\n}\n\nfunction applyLocatorVerifyFn(\n  locatorVerifyFn: LocatorVerifyFn | undefined,\n  input: Parameters<LocatorVerifyFn>[0],\n): LocateValidatorResult {\n  if (!locatorVerifyFn) {\n    return input.verifyResult;\n  }\n  const customResult = locatorVerifyFn(input);\n  if (typeof customResult === 'boolean') {\n    return {\n      ...input.verifyResult,\n      pass: customResult,\n    };\n  }\n  return customResult;\n}\n\nfunction errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nexport async function describeElementAtPoint(\n  runtime: ElementDescriberRuntime,\n  center: [number, number],\n  opt?: DescribeElementAtPointOptions,\n): Promise<AgentDescribeElementAtPointResult> {\n  const { verifyPrompt = true, retryLimit = 3 } = opt || {};\n  const { screenshotContext, locateOpt, targetCenter } =\n    await createScreenshotBoundLocatorContext(center, opt);\n\n  let success = false;\n  let retryCount = 0;\n  let resultPrompt = '';\n  const autoRetryDeepDescribe = opt?.deepDescribe === undefined;\n  const autoRetryDeepLocate = opt?.deepLocate === undefined;\n  let deepDescribe = opt?.deepDescribe || false;\n  let deepLocate = opt?.deepLocate || false;\n  let verifyResult: LocateValidatorResult | undefined;\n  let lastError: string | undefined;\n  let failureStage: AgentDescribeElementAtPointResult['failureStage'];\n\n  while (!success && retryCount < retryLimit) {\n    if (retryCount >= 1 && autoRetryDeepDescribe) {\n      deepDescribe = true;\n    }\n    if (retryCount >= 1 && autoRetryDeepLocate) {\n      deepLocate = true;\n    }\n    const describeModelRuntime = runtime.describeModelRuntime;\n    const locateModelRuntime = runtime.locateModelRuntime;\n    const retryRuntime: ElementDescriberRuntime = {\n      ...runtime,\n      describeModelRuntime,\n      locateModelRuntime,\n    };\n    const describeOpt: ServiceDescribeOptions = screenshotContext\n      ? {\n          deepDescribe,\n          context: screenshotContext,\n        }\n      : {\n          deepDescribe,\n        };\n    let text: Awaited<\n      ReturnType<ElementDescriberRuntime['service']['describe']>\n    >;\n    try {\n      text = await retryRuntime.service.describe(\n        targetCenter,\n        retryRuntime.describeModelRuntime,\n        describeOpt,\n      );\n    } catch (error) {\n      return {\n        prompt: resultPrompt,\n        deepLocate,\n        deepDescribe,\n        verifyResult,\n        success: false,\n        error: errorMessage(error),\n        failureStage: 'describe',\n      };\n    }\n    if (!text.description) {\n      return {\n        prompt: resultPrompt,\n        deepLocate,\n        deepDescribe,\n        verifyResult,\n        success: false,\n        error: `failed to describe element at [${targetCenter}]`,\n        failureStage: 'describe',\n      };\n    }\n    resultPrompt = text.description;\n    if (!verifyPrompt) {\n      opt?.onProgress?.({\n        prompt: resultPrompt,\n        deepDescribe,\n        deepLocate,\n      });\n      success = true;\n      break;\n    }\n\n    try {\n      const candidateVerifyResult = await verifyLocator(\n        retryRuntime,\n        resultPrompt,\n        locateOpt,\n        targetCenter,\n        {\n          ...opt,\n          deepLocate,\n        },\n      );\n      verifyResult = applyLocatorVerifyFn(opt?.locatorVerifyFn, {\n        prompt: resultPrompt,\n        expectCenter: targetCenter,\n        retryCount,\n        verifyResult: candidateVerifyResult,\n      });\n      opt?.onProgress?.({\n        prompt: resultPrompt,\n        deepDescribe,\n        deepLocate,\n        verifyResult,\n      });\n      if (verifyResult.pass) {\n        success = true;\n        break;\n      }\n      lastError = undefined;\n      failureStage = 'verify';\n    } catch (error) {\n      lastError = errorMessage(error);\n      failureStage = 'verify';\n      opt?.onProgress?.({\n        prompt: resultPrompt,\n        deepDescribe,\n        deepLocate,\n      });\n    }\n    retryCount++;\n  }\n\n  return {\n    prompt: resultPrompt,\n    deepLocate,\n    deepDescribe,\n    verifyResult,\n    success,\n    error:\n      success || !verifyPrompt\n        ? undefined\n        : lastError || 'describeElementAtPoint verify failed',\n    failureStage: success ? undefined : failureStage,\n  };\n}\n\nexport async function verifyElementDescriptionAtPoint(\n  runtime: ElementDescriberRuntime,\n  description: string,\n  center: [number, number],\n  opt?: VerifyElementDescriptionAtPointOptions,\n): Promise<LocateValidatorResult> {\n  assert(description?.trim(), 'description must not be empty');\n  const { locateOpt, targetCenter } = await createScreenshotBoundLocatorContext(\n    center,\n    opt,\n  );\n\n  return verifyLocator(runtime, description, locateOpt, targetCenter, opt);\n}\n\nasync function locateAndVerify(\n  runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>,\n  description: string,\n  center: [number, number],\n  opt?: LocateAndVerifyOptions & {\n    uiContext?: UIContext;\n  },\n): Promise<LocateValidatorResult> {\n  assert(description?.trim(), 'description must not be empty');\n  const { screenshotContext, targetCenter } =\n    await createScreenshotBoundLocatorContext(center, opt);\n  const context = opt?.uiContext || screenshotContext;\n  const locateParam = buildDetailedLocateParam(description, {\n    cacheable: opt?.cacheable,\n    deepLocate: opt?.deepLocate,\n    xpath: opt?.xpath,\n  });\n  assert(locateParam, 'cannot get locate param for service locate');\n\n  const locateResult = await runtime.service.locate(\n    locateParam,\n    context ? { context } : {},\n    runtime.locateModelRuntime,\n    opt?.abortSignal,\n  );\n  assert(locateResult.element, `Element not found: ${description}`);\n  const verifyResult = buildLocateValidatorResult(\n    targetCenter,\n    locateResult.element,\n    opt,\n  );\n  return verifyResult;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","distanceOfTwoPoints","p1","p2","x1","y1","x2","y2","Math","includedInRect","point","rect","x","y","left","top","width","height","buildLocateValidatorResult","expectCenter","located","verifyLocateOption","distance","included","pass","assertPositiveSize","size","label","assert","Number","mapPointToScreenshotSpace","center","screenshotSize","opt","coordinateSpace","createScreenshotBoundLocatorContext","screenshotContext","createScreenshotBoundUIContext","undefined","targetCenter","verifyLocator","runtime","prompt","locateOpt","locateAndVerify","applyLocatorVerifyFn","locatorVerifyFn","input","customResult","errorMessage","error","Error","String","describeElementAtPoint","verifyPrompt","retryLimit","success","retryCount","resultPrompt","autoRetryDeepDescribe","autoRetryDeepLocate","deepDescribe","deepLocate","verifyResult","lastError","failureStage","describeModelRuntime","locateModelRuntime","retryRuntime","describeOpt","text","candidateVerifyResult","verifyElementDescriptionAtPoint","description","context","locateParam","buildDetailedLocateParam","locateResult"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;AC8DA,MAAMI,sBAAsB,CAACC,IAAsBC;IACjD,MAAM,CAACC,IAAIC,GAAG,GAAGH;IACjB,MAAM,CAACI,IAAIC,GAAG,GAAGJ;IACjB,OAAOK,KAAK,KAAK,CAACA,KAAK,IAAI,CAAEJ,AAAAA,CAAAA,KAAKE,EAAC,KAAM,IAAKD,AAAAA,CAAAA,KAAKE,EAAC,KAAM;AAC5D;AAEA,MAAME,iBAAiB,CAACC,OAAyBC;IAC/C,MAAM,CAACC,GAAGC,EAAE,GAAGH;IACf,MAAM,EAAEI,IAAI,EAAEC,GAAG,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGN;IACrC,OAAOC,KAAKE,QAAQF,KAAKE,OAAOE,SAASH,KAAKE,OAAOF,KAAKE,MAAME;AAClE;AAEA,MAAMC,6BAA6B,CACjCC,cACAC,SACAC;IAEA,MAAMC,WAAWrB,oBAAoBkB,cAAcC,QAAQ,MAAM;IACjE,MAAMG,WAAWd,eAAeU,cAAcC,QAAQ,IAAI;IAC1D,MAAMI,OACJF,YAAaD,CAAAA,oBAAoB,2BAA2B,EAAC,KAAME;IACrE,OAAO;QACLC;QACA,MAAMJ,QAAQ,IAAI;QAClB,QAAQA,QAAQ,MAAM;QACtB,gBAAgBE;QAChB,gBAAgBC;IAClB;AACF;AAEA,SAASE,mBACPC,IAAsB,EACtBC,KAAa;IAEbC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEF,QACEG,OAAO,QAAQ,CAACH,KAAK,KAAK,KAC1BG,OAAO,QAAQ,CAACH,KAAK,MAAM,KAC3BA,KAAK,KAAK,GAAG,KACbA,KAAK,MAAM,GAAG,GAChB,GAAGC,MAAM,uCAAuC,CAAC;AAErD;AAEA,MAAMG,4BAA4B,CAChCC,QACAC,gBACAC;IAEA,MAAMC,kBAAkBD,IAAI,eAAe,IAAI;IAC/C,IAAIC,AAAoB,iBAApBA,iBACF,OAAOH;IAGTN,mBACEQ,IAAI,WAAW,EACf;IAEF,OAAO;QACJF,MAAM,CAAC,EAAE,GAAGC,eAAe,KAAK,GAAIC,IAAI,WAAW,CAAC,KAAK;QACzDF,MAAM,CAAC,EAAE,GAAGC,eAAe,MAAM,GAAIC,IAAI,WAAW,CAAC,MAAM;KAC7D;AACH;AAEA,MAAME,sCAAsC,OAC1CJ,QACAE;IAMA,MAAMG,oBAAoBH,KAAK,mBAC3B,MAAMI,AAAAA,IAAAA,yBAAAA,8BAAAA,AAAAA,EAA+BJ,IAAI,gBAAgB,EAAEA,OAC3DK;IACJ,MAAMC,eAAeH,oBACjBN,0BAA0BC,QAAQK,kBAAkB,QAAQ,EAAEH,OAAO,CAAC,KACtEF;IACJ,OAAO;QACLK;QACA,WAAWA,oBAAoB;YAAE,WAAWA;QAAkB,IAAIE;QAClEC;IACF;AACF;AAEO,eAAeC,cACpBC,OAAwE,EACxEC,MAAc,EACdC,SAAmC,EACnCxB,YAA8B,EAC9BE,kBACkC;IAElC,OAAOuB,gBAAgBH,SAASC,QAAQvB,cAAc;QACpD,yBAAyBE,oBAAoB;QAC7C,YAAYA,oBAAoB;QAChC,WAAWsB,WAAW;IACxB;AACF;AAEA,SAASE,qBACPC,eAA4C,EAC5CC,KAAqC;IAErC,IAAI,CAACD,iBACH,OAAOC,MAAM,YAAY;IAE3B,MAAMC,eAAeF,gBAAgBC;IACrC,IAAI,AAAwB,aAAxB,OAAOC,cACT,OAAO;QACL,GAAGD,MAAM,YAAY;QACrB,MAAMC;IACR;IAEF,OAAOA;AACT;AAEA,SAASC,aAAaC,KAAc;IAClC,OAAOA,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;AACzD;AAEO,eAAeG,uBACpBZ,OAAgC,EAChCV,MAAwB,EACxBE,GAAmC;IAEnC,MAAM,EAAEqB,eAAe,IAAI,EAAEC,aAAa,CAAC,EAAE,GAAGtB,OAAO,CAAC;IACxD,MAAM,EAAEG,iBAAiB,EAAEO,SAAS,EAAEJ,YAAY,EAAE,GAClD,MAAMJ,oCAAoCJ,QAAQE;IAEpD,IAAIuB,UAAU;IACd,IAAIC,aAAa;IACjB,IAAIC,eAAe;IACnB,MAAMC,wBAAwB1B,KAAK,iBAAiBK;IACpD,MAAMsB,sBAAsB3B,KAAK,eAAeK;IAChD,IAAIuB,eAAe5B,KAAK,gBAAgB;IACxC,IAAI6B,aAAa7B,KAAK,cAAc;IACpC,IAAI8B;IACJ,IAAIC;IACJ,IAAIC;IAEJ,MAAO,CAACT,WAAWC,aAAaF,WAAY;QAC1C,IAAIE,cAAc,KAAKE,uBACrBE,eAAe;QAEjB,IAAIJ,cAAc,KAAKG,qBACrBE,aAAa;QAEf,MAAMI,uBAAuBzB,QAAQ,oBAAoB;QACzD,MAAM0B,qBAAqB1B,QAAQ,kBAAkB;QACrD,MAAM2B,eAAwC;YAC5C,GAAG3B,OAAO;YACVyB;YACAC;QACF;QACA,MAAME,cAAsCjC,oBACxC;YACEyB;YACA,SAASzB;QACX,IACA;YACEyB;QACF;QACJ,IAAIS;QAGJ,IAAI;YACFA,OAAO,MAAMF,aAAa,OAAO,CAAC,QAAQ,CACxC7B,cACA6B,aAAa,oBAAoB,EACjCC;QAEJ,EAAE,OAAOnB,OAAO;YACd,OAAO;gBACL,QAAQQ;gBACRI;gBACAD;gBACAE;gBACA,SAAS;gBACT,OAAOd,aAAaC;gBACpB,cAAc;YAChB;QACF;QACA,IAAI,CAACoB,KAAK,WAAW,EACnB,OAAO;YACL,QAAQZ;YACRI;YACAD;YACAE;YACA,SAAS;YACT,OAAO,CAAC,+BAA+B,EAAExB,aAAa,CAAC,CAAC;YACxD,cAAc;QAChB;QAEFmB,eAAeY,KAAK,WAAW;QAC/B,IAAI,CAAChB,cAAc;YACjBrB,KAAK,aAAa;gBAChB,QAAQyB;gBACRG;gBACAC;YACF;YACAN,UAAU;YACV;QACF;QAEA,IAAI;YACF,MAAMe,wBAAwB,MAAM/B,cAClC4B,cACAV,cACAf,WACAJ,cACA;gBACE,GAAGN,GAAG;gBACN6B;YACF;YAEFC,eAAelB,qBAAqBZ,KAAK,iBAAiB;gBACxD,QAAQyB;gBACR,cAAcnB;gBACdkB;gBACA,cAAcc;YAChB;YACAtC,KAAK,aAAa;gBAChB,QAAQyB;gBACRG;gBACAC;gBACAC;YACF;YACA,IAAIA,aAAa,IAAI,EAAE;gBACrBP,UAAU;gBACV;YACF;YACAQ,YAAY1B;YACZ2B,eAAe;QACjB,EAAE,OAAOf,OAAO;YACdc,YAAYf,aAAaC;YACzBe,eAAe;YACfhC,KAAK,aAAa;gBAChB,QAAQyB;gBACRG;gBACAC;YACF;QACF;QACAL;IACF;IAEA,OAAO;QACL,QAAQC;QACRI;QACAD;QACAE;QACAP;QACA,OACEA,WAAW,CAACF,eACRhB,SACA0B,aAAa;QACnB,cAAcR,UAAUlB,SAAY2B;IACtC;AACF;AAEO,eAAeO,gCACpB/B,OAAgC,EAChCgC,WAAmB,EACnB1C,MAAwB,EACxBE,GAA4C;IAE5CL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO6C,aAAa,QAAQ;IAC5B,MAAM,EAAE9B,SAAS,EAAEJ,YAAY,EAAE,GAAG,MAAMJ,oCACxCJ,QACAE;IAGF,OAAOO,cAAcC,SAASgC,aAAa9B,WAAWJ,cAAcN;AACtE;AAEA,eAAeW,gBACbH,OAAwE,EACxEgC,WAAmB,EACnB1C,MAAwB,EACxBE,GAEC;IAEDL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO6C,aAAa,QAAQ;IAC5B,MAAM,EAAErC,iBAAiB,EAAEG,YAAY,EAAE,GACvC,MAAMJ,oCAAoCJ,QAAQE;IACpD,MAAMyC,UAAUzC,KAAK,aAAaG;IAClC,MAAMuC,cAAcC,AAAAA,IAAAA,uCAAAA,wBAAAA,AAAAA,EAAyBH,aAAa;QACxD,WAAWxC,KAAK;QAChB,YAAYA,KAAK;QACjB,OAAOA,KAAK;IACd;IACAL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO+C,aAAa;IAEpB,MAAME,eAAe,MAAMpC,QAAQ,OAAO,CAAC,MAAM,CAC/CkC,aACAD,UAAU;QAAEA;IAAQ,IAAI,CAAC,GACzBjC,QAAQ,kBAAkB,EAC1BR,KAAK;IAEPL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOiD,aAAa,OAAO,EAAE,CAAC,mBAAmB,EAAEJ,aAAa;IAChE,MAAMV,eAAe7C,2BACnBqB,cACAsC,aAAa,OAAO,EACpB5C;IAEF,OAAO8B;AACT"}

package/dist/lib/index.js

@@ -46,10 +46,9 @@ __webpack_require__.d(__webpack_exports__, {
     getTaskSearchArea: ()=>task_service_dump_js_namespaceObject.getTaskSearchArea,
     parseDumpScriptAttributes: ()=>external_dump_index_js_namespaceObject.parseDumpScriptAttributes,
     splitReportHtmlByExecution: ()=>external_report_js_namespaceObject.splitReportHtmlByExecution,
-    verifyElementByServiceLocate: ()=>external_element_describer_js_namespaceObject.verifyElementByServiceLocate,
+    z: ()=>external_zod_namespaceObject.z,
     generateDumpScriptTag: ()=>external_dump_index_js_namespaceObject.generateDumpScriptTag,
     SizeSchema: ()=>external_ai_model_index_js_namespaceObject.SizeSchema,
-    z: ()=>external_zod_namespaceObject.z,
     default: ()=>src,
     getVersion: ()=>external_utils_js_namespaceObject.getVersion,
     getMidsceneLocationSchema: ()=>external_ai_model_index_js_namespaceObject.getMidsceneLocationSchema,
@@ -151,7 +150,6 @@ exports.runConnectivityTest = __webpack_exports__.runConnectivityTest;
 exports.splitReportFile = __webpack_exports__.splitReportFile;
 exports.splitReportHtmlByExecution = __webpack_exports__.splitReportHtmlByExecution;
 exports.unescapeContent = __webpack_exports__.unescapeContent;
-exports.verifyElementByServiceLocate = __webpack_exports__.verifyElementByServiceLocate;
 exports.verifyElementDescriptionAtPoint = __webpack_exports__.verifyElementDescriptionAtPoint;
 exports.verifyLocator = __webpack_exports__.verifyLocator;
 exports.z = __webpack_exports__.z;
@@ -203,7 +201,6 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "splitReportFile",
     "splitReportHtmlByExecution",
     "unescapeContent",
-    "verifyElementByServiceLocate",
     "verifyElementDescriptionAtPoint",
     "verifyLocator",
     "z"

package/dist/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/index.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  AiLocateElement,\n  runConnectivityTest,\n  getMidsceneLocationSchema,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  type ConnectivityTestConfig,\n  type ConnectivityTestResult,\n} from './ai-model/index';\n\nexport {\n  MIDSCENE_MODEL_NAME,\n  type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport {\n  ServiceError,\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n  type IExecutionDump,\n  type IReportActionDump,\n  type IGroupedActionDump,\n  type ReportMeta,\n  type GroupMeta,\n} from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  MidsceneYamlConfig,\n  MidsceneYamlScriptWebEnv,\n  MidsceneYamlScriptAndroidEnv,\n  MidsceneYamlScriptIOSEnv,\n  MidsceneYamlScriptEnv,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport {\n  Agent,\n  type AgentOpt,\n  type AiActOptions,\n  type GherkinStepKeyword,\n  type RunGherkinScenarioOptions,\n  createAgent,\n} from './agent';\nexport {\n  describeElementAtPoint,\n  verifyElementDescriptionAtPoint,\n  verifyElementByServiceLocate,\n  verifyLocator,\n  type DescribeElementAtPointOptions,\n  type DescribeElementCoordinateSpace,\n  type ElementDescriberRuntime,\n  type VerifyElementDescriptionAtPointOptions,\n  type VerifyElementByServiceLocateOptions,\n} from './element-describer';\n\n// Dump utilities\nexport {\n  restoreImageReferences,\n  escapeContent,\n  unescapeContent,\n  parseImageScripts,\n  parseDumpScript,\n  parseDumpScriptAttributes,\n  generateImageScriptTag,\n  generateDumpScriptTag,\n  deriveTaskStatus,\n  deriveCaseStatus,\n} from './dump';\nexport type { TaskStatusFields, DerivedTaskStatus } from './dump';\nexport {\n  getTaskSearchArea,\n  getTaskServiceDump,\n} from './dump/task-service-dump';\n\n// Report generator\nexport type { IReportGenerator } from './report-generator';\nexport { ReportGenerator, nullReportGenerator } from './report-generator';\nexport {\n  collectDedupedExecutions,\n  ReportMergingTool,\n  dedupeExecutionsKeepLatest,\n  splitReportHtmlByExecution,\n} from './report';\nexport {\n  createReportCliCommands,\n  reportFileToMarkdown,\n  splitReportFile,\n  mergeReportFiles,\n  type ConsumeReportFileAction,\n  type ReportFileToMarkdownOptions,\n  type ReportCliCommandDefinition,\n  type ReportCliCommandEntry,\n  type SplitReportFileOptions,\n  type MergeReportFilesOptions,\n  type MergeReportFilesResult,\n} from './report-cli';\n\n// ScreenshotItem\nexport { ScreenshotItem } from './screenshot-item';\nexport { ScreenshotStore, type ScreenshotRef } from './dump/screenshot-store';\n\nexport {\n  executionToMarkdown,\n  reportToMarkdown,\n  type ExecutionMarkdownOptions,\n  type ExecutionMarkdownResult,\n  type ReportMarkdownResult,\n  type MarkdownAttachment,\n} from './report-markdown';\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","Service"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACmCA,YAAeI"}
+{"version":3,"file":"index.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/index.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  AiLocateElement,\n  runConnectivityTest,\n  getMidsceneLocationSchema,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  type ConnectivityTestConfig,\n  type ConnectivityTestResult,\n} from './ai-model/index';\n\nexport {\n  MIDSCENE_MODEL_NAME,\n  type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport {\n  ServiceError,\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n  type IExecutionDump,\n  type IReportActionDump,\n  type IGroupedActionDump,\n  type ReportMeta,\n  type GroupMeta,\n} from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  MidsceneYamlConfig,\n  MidsceneYamlScriptWebEnv,\n  MidsceneYamlScriptAndroidEnv,\n  MidsceneYamlScriptIOSEnv,\n  MidsceneYamlScriptEnv,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport {\n  Agent,\n  type AgentOpt,\n  type AiActOptions,\n  type GherkinStepKeyword,\n  type RunGherkinScenarioOptions,\n  createAgent,\n} from './agent';\nexport {\n  describeElementAtPoint,\n  verifyElementDescriptionAtPoint,\n  verifyLocator,\n  type DescribeElementAtPointOptions,\n  type DescribeElementCoordinateSpace,\n  type ElementDescriberRuntime,\n  type VerifyElementDescriptionAtPointOptions,\n} from './element-describer';\n\n// Dump utilities\nexport {\n  restoreImageReferences,\n  escapeContent,\n  unescapeContent,\n  parseImageScripts,\n  parseDumpScript,\n  parseDumpScriptAttributes,\n  generateImageScriptTag,\n  generateDumpScriptTag,\n  deriveTaskStatus,\n  deriveCaseStatus,\n} from './dump';\nexport type { TaskStatusFields, DerivedTaskStatus } from './dump';\nexport {\n  getTaskSearchArea,\n  getTaskServiceDump,\n} from './dump/task-service-dump';\n\n// Report generator\nexport type { IReportGenerator } from './report-generator';\nexport { ReportGenerator, nullReportGenerator } from './report-generator';\nexport {\n  collectDedupedExecutions,\n  ReportMergingTool,\n  dedupeExecutionsKeepLatest,\n  splitReportHtmlByExecution,\n} from './report';\nexport {\n  createReportCliCommands,\n  reportFileToMarkdown,\n  splitReportFile,\n  mergeReportFiles,\n  type ConsumeReportFileAction,\n  type ReportFileToMarkdownOptions,\n  type ReportCliCommandDefinition,\n  type ReportCliCommandEntry,\n  type SplitReportFileOptions,\n  type MergeReportFilesOptions,\n  type MergeReportFilesResult,\n} from './report-cli';\n\n// ScreenshotItem\nexport { ScreenshotItem } from './screenshot-item';\nexport { ScreenshotStore, type ScreenshotRef } from './dump/screenshot-store';\n\nexport {\n  executionToMarkdown,\n  reportToMarkdown,\n  type ExecutionMarkdownOptions,\n  type ExecutionMarkdownResult,\n  type ReportMarkdownResult,\n  type MarkdownAttachment,\n} from './report-markdown';\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","Service"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACmCA,YAAeI"}

package/dist/lib/service/index.js

@@ -30,7 +30,6 @@ const errors_js_namespaceObject = require("../ai-model/errors.js");
 const inspect_js_namespaceObject = require("../ai-model/inspect.js");
 const describe_js_namespaceObject = require("../ai-model/prompt/describe.js");
 const index_js_namespaceObject = require("../ai-model/service-caller/index.js");
-const external_common_js_namespaceObject = require("../common.js");
 const external_types_js_namespaceObject = require("../types.js");
 const img_namespaceObject = require("@midscene/shared/img");
 const logger_namespaceObject = require("@midscene/shared/logger");
@@ -278,28 +277,113 @@ class Service {
         (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshot is required for service.describe');
         const systemPrompt = (0, describe_js_namespaceObject.elementDescriberInstruction)();
         const defaultRectSize = 30;
-        const targetRect = Array.isArray(target) ? {
+        const targetFromPoint = Array.isArray(target);
+        const targetRect = targetFromPoint ? {
             left: Math.floor(target[0] - defaultRectSize / 2),
             top: Math.floor(target[1] - defaultRectSize / 2),
             width: defaultRectSize,
             height: defaultRectSize
         } : target;
-        let imagePayload = await (0, img_namespaceObject.compositeElementInfoImg)({
+        const targetPoint = targetFromPoint ? {
+            x: target[0],
+            y: target[1]
+        } : void 0;
+        const usePointMarker = targetFromPoint;
+        const imagePayload = usePointMarker ? await (0, img_namespaceObject.compositePointMarkerImg)({
+            inputImgBase64: screenshotBase64,
+            size: shotSize,
+            point: targetPoint
+        }) : await (0, img_namespaceObject.compositeElementInfoImg)({
             inputImgBase64: screenshotBase64,
             size: shotSize,
             elementsPositionInfo: [
                 {
-                    rect: targetRect
+                    rect: (0, external_utils_js_namespaceObject.getDescribeMarkerRect)(targetRect)
                 }
             ],
-            borderThickness: 3
+            borderThickness: (0, external_utils_js_namespaceObject.getDescribeMarkerBorderThickness)(targetRect),
+            centerPoint: true
         });
-        if (opt?.deepLocate) {
-            const searchArea = (0, external_common_js_namespaceObject.expandSearchArea)(targetRect, shotSize);
-            debug('describe: cropping to searchArea', searchArea);
-            const croppedResult = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea);
-            imagePayload = croppedResult.imageBase64;
-        }
+        const shouldDeepDescribe = opt?.deepDescribe;
+        let imageContent;
+        if (shouldDeepDescribe) {
+            const contextAreas = (0, external_utils_js_namespaceObject.getDescribeDeepContextAreas)(targetRect, shotSize);
+            const contextImages = await Promise.all(contextAreas.map(async (area)=>{
+                debug('describe: cropping deep context area', area);
+                const croppedResult = await (0, img_namespaceObject.cropByRect)(screenshotBase64, area.rect);
+                const cropSize = {
+                    width: croppedResult.width,
+                    height: croppedResult.height
+                };
+                const targetInCrop = (0, external_utils_js_namespaceObject.getRectInCrop)(targetRect, area.rect, cropSize);
+                const markedCropPayload = targetFromPoint ? await (0, img_namespaceObject.compositePointMarkerImg)({
+                    inputImgBase64: croppedResult.imageBase64,
+                    size: cropSize,
+                    point: {
+                        x: targetPoint.x - area.rect.left,
+                        y: targetPoint.y - area.rect.top
+                    }
+                }) : await (0, img_namespaceObject.compositeElementInfoImg)({
+                    inputImgBase64: croppedResult.imageBase64,
+                    size: cropSize,
+                    elementsPositionInfo: [
+                        {
+                            rect: (0, external_utils_js_namespaceObject.getDescribeMarkerRect)(targetInCrop)
+                        }
+                    ],
+                    borderThickness: (0, external_utils_js_namespaceObject.getDescribeMarkerBorderThickness)(targetInCrop),
+                    centerPoint: true
+                });
+                const resizeSize = (0, external_utils_js_namespaceObject.getDescribeDeepLocateResizeSize)(croppedResult);
+                return {
+                    kind: area.kind,
+                    imageBase64: resizeSize ? await (0, img_namespaceObject.resizeImgBase64)(markedCropPayload, resizeSize) : markedCropPayload
+                };
+            }));
+            const contextImageContent = contextImages.flatMap((item, index)=>[
+                    {
+                        type: 'text',
+                        text: `Image ${index + 2}: focused detail crop around the target, for reading text, icon shape, and exact local boundaries.`
+                    },
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: item.imageBase64,
+                            detail: 'high'
+                        }
+                    }
+                ]);
+            imageContent = [
+                {
+                    type: 'text',
+                    text: 'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.'
+                },
+                {
+                    type: 'text',
+                    text: 'Image 1: full screenshot overview with the target marker, for page position and ownership context.'
+                },
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                },
+                ...contextImageContent
+            ];
+        } else imageContent = [
+            {
+                type: 'text',
+                text: 'Full screenshot with a temporary callout marking the target:'
+            },
+            {
+                type: 'image_url',
+                image_url: {
+                    url: imagePayload,
+                    detail: 'high'
+                }
+            }
+        ];
         const msgs = [
             {
                 role: 'system',
@@ -307,15 +391,7 @@ class Service {
             },
             {
                 role: 'user',
-                content: [
-                    {
-                        type: 'image_url',
-                        image_url: {
-                            url: imagePayload,
-                            detail: 'high'
-                        }
-                    }
-                ]
+                content: imageContent
             }
         ];
         let res;

package/dist/lib/service/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { defaultModelFamilyRequiredForLocateMessage } from '@/ai-model/errors';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/inspect';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport type { AIArgs } from '@/ai-model/types';\nimport type { SearchAreaConfig } from '@/ai-model/workflows/inspect/types';\nimport { expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  createServiceDump,\n  recoverDescribeResponseFromParseError,\n} from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\ninterface LocateSearchAreaResult {\n  config?: SearchAreaConfig;\n  trace: {\n    sourceRect?: Rect;\n    rawResponse?: string;\n    rawChoiceMessage?: unknown;\n    usage?: AIUsageInfo;\n  };\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelRuntime: ModelRuntime,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const { config: modelConfig } = modelRuntime;\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    if (!modelConfig.modelFamily) {\n      throw new Error(defaultModelFamilyRequiredForLocateMessage);\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    const searchArea = await this.resolveLocateSearchArea({\n      query,\n      queryPrompt,\n      opt,\n      context,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n    } = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchArea.config,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea: searchArea.trace.sourceRect,\n      searchAreaRawResponse: searchArea.trace.rawResponse,\n      searchAreaRawChoiceMessage: searchArea.trace.rawChoiceMessage,\n      searchAreaUsage: searchArea.trace.usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea.trace.sourceRect,\n      error: errorLog,\n    };\n\n    const element = parseResult.element;\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: element ? [element] : [],\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (element) {\n      return {\n        element: {\n          center: element.center,\n          rect: element.rect,\n          description: element.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  private async resolveLocateSearchArea(options: {\n    query: PlanningLocateParam;\n    queryPrompt: TUserPrompt;\n    opt: LocateOpts;\n    context: UIContext;\n    modelRuntime: ModelRuntime;\n    abortSignal?: AbortSignal;\n  }): Promise<LocateSearchAreaResult> {\n    const { query, queryPrompt, opt, context, modelRuntime, abortSignal } =\n      options;\n    const { adapter } = modelRuntime;\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    if (!query.deepLocate) {\n      return { trace: {} };\n    }\n\n    if (hasPlanLocatedElement) {\n      const config = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n      });\n\n      return {\n        config,\n        trace: {\n          sourceRect: config.sourceRect,\n          rawResponse: JSON.stringify({\n            source: 'plan-located-element',\n            rect: opt.planLocatedElement!.rect,\n          }),\n        },\n      };\n    }\n\n    if (adapter.locate.supportsSearchArea) {\n      const searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: queryPrompt,\n        modelRuntime,\n        abortSignal,\n      });\n      const { searchAreaConfig } = searchAreaResponse;\n      assert(\n        searchAreaConfig,\n        `cannot find search area for \"${queryPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n\n      return {\n        config: searchAreaConfig,\n        trace: {\n          sourceRect: searchAreaConfig.sourceRect,\n          rawResponse: searchAreaResponse.rawResponse,\n          rawChoiceMessage: searchAreaResponse.rawChoiceMessage,\n          usage: searchAreaResponse.usage,\n        },\n      };\n    }\n\n    const firstPassLocateResult = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      modelRuntime,\n      abortSignal,\n    });\n    assert(\n      firstPassLocateResult.rect,\n      `cannot find search area for \"${queryPrompt}\"${\n        firstPassLocateResult.parseResult.errors?.length\n          ? `: ${firstPassLocateResult.parseResult.errors.join('\\n')}`\n          : ''\n      }`,\n    );\n\n    const config = await buildSearchAreaConfig({\n      context,\n      baseRect: firstPassLocateResult.rect,\n    });\n\n    return {\n      config,\n      trace: {\n        sourceRect: config.sourceRect,\n        rawResponse: JSON.stringify({\n          source: 'deep-locate-first-pass',\n          rect: firstPassLocateResult.rect,\n          rawResponse: firstPassLocateResult.rawResponse,\n        }),\n        rawChoiceMessage: firstPassLocateResult.rawChoiceMessage,\n        usage: firstPassLocateResult.usage,\n      },\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let rawChoiceMessage: unknown;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelRuntime,\n        pageDescription,\n        abortSignal: executionOptions?.abortSignal,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      rawChoiceMessage = result.rawChoiceMessage;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          rawChoiceMessage: error.rawChoiceMessage,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelRuntime: ModelRuntime,\n    opt?: {\n      deepLocate?: boolean;\n      context?: UIContext;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = opt?.context || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      // Describe is not a coordinate-parsing flow, so it does not need image\n      // padding for bbox normalization.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(imagePayload, searchArea);\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    let res: Awaited<\n      ReturnType<typeof callAIWithObjectResponse<AIDescribeElementResponse>>\n    >;\n    try {\n      res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n        msgs,\n        modelRuntime,\n      );\n    } catch (error) {\n      const recoveredResponse = recoverDescribeResponseFromParseError(error);\n      if (!recoveredResponse) {\n        throw error;\n      }\n      debug('describe: recovered malformed description JSON response');\n      return recoveredResponse;\n    }\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelRuntime","abortSignal","modelConfig","queryPrompt","assert","Error","defaultModelFamilyRequiredForLocateMessage","context","searchArea","startTime","Date","parseResult","rect","rawResponse","rawChoiceMessage","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","element","dump","createServiceDump","ServiceError","options","adapter","hasPlanLocatedElement","config","buildSearchAreaConfig","searchAreaResponse","AiLocateSection","searchAreaConfig","firstPassLocateResult","dataDemand","pageDescription","multimodalPrompt","executionOptions","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","recoveredResponse","recoverDescribeResponseFromParseError","content","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;AC0DA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,YAA0B,EAC1BC,WAAyB,EACM;QAC/B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGF;QAChC,MAAMG,cAAc,AAAiB,YAAjB,OAAOL,QAAqBA,QAAQA,MAAM,MAAM;QACpEM,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAON,OAAoB;QAElC,IAAI,CAACI,YAAY,WAAW,EAC1B,MAAM,IAAIG,MAAMC,0BAAAA,0CAA0CA;QAG5D,MAAMC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,MAAMS,aAAa,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACpDV;YACAK;YACAJ;YACAQ;YACAP;YACAC;QACF;QAEA,MAAMQ,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YACxBV;YACA,0BAA0BJ;YAC1B,cAAcK,WAAW,MAAM;YAC/BR;YACAC;QACF;QAEA,MAAMiB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACP;YAC5BC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACA,YAAYP,WAAW,KAAK,CAAC,UAAU;YACvC,uBAAuBA,WAAW,KAAK,CAAC,WAAW;YACnD,4BAA4BA,WAAW,KAAK,CAAC,gBAAgB;YAC7D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACvCQ;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,4BAA4B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASnB;YACX;YACA,aAAaS;YACb,MAAM;YACNO;YACA,YAAY,CAAC,CAACX,WAAW,KAAK,CAAC,UAAU;YACzC,OAAOa;QACT;QAEA,MAAME,UAAUZ,YAAY,OAAO;QAEnC,MAAMa,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC,UAAU;gBAACA;aAAQ,GAAG,EAAE;QAC1C;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SACF,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,MAAM;gBACtB,MAAMA,QAAQ,IAAI;gBAClB,aAAaA,QAAQ,WAAW;YAClC;YACAX;YACAY;QACF;QAGF,OAAO;YACL,SAAS;YACTZ;YACAY;QACF;IACF;IAEA,MAAc,wBAAwBG,OAOrC,EAAmC;QAClC,MAAM,EAAE7B,KAAK,EAAEK,WAAW,EAAEJ,GAAG,EAAEQ,OAAO,EAAEP,YAAY,EAAEC,WAAW,EAAE,GACnE0B;QACF,MAAM,EAAEC,OAAO,EAAE,GAAG5B;QACpB,MAAM6B,wBAAwB,CAAC,CAAC9B,KAAK,oBAAoB;QAEzD,IAAI,CAACD,MAAM,UAAU,EACnB,OAAO;YAAE,OAAO,CAAC;QAAE;QAGrB,IAAI+B,uBAAuB;YACzB,MAAMC,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;gBACzCxB;gBACA,UAAUR,IAAI,kBAAkB,CAAE,IAAI;YACxC;YAEA,OAAO;gBACL+B;gBACA,OAAO;oBACL,YAAYA,OAAO,UAAU;oBAC7B,aAAaV,KAAK,SAAS,CAAC;wBAC1B,QAAQ;wBACR,MAAMrB,IAAI,kBAAkB,CAAE,IAAI;oBACpC;gBACF;YACF;QACF;QAEA,IAAI6B,QAAQ,MAAM,CAAC,kBAAkB,EAAE;YACrC,MAAMI,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBAC/C1B;gBACA,oBAAoBJ;gBACpBH;gBACAC;YACF;YACA,MAAM,EAAEiC,gBAAgB,EAAE,GAAGF;YAC7B5B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8B,kBACA,CAAC,6BAA6B,EAAE/B,YAAY,CAAC,EAC3C6B,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAGJ,OAAO;gBACL,QAAQE;gBACR,OAAO;oBACL,YAAYA,iBAAiB,UAAU;oBACvC,aAAaF,mBAAmB,WAAW;oBAC3C,kBAAkBA,mBAAmB,gBAAgB;oBACrD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;QACF;QAEA,MAAMG,wBAAwB,MAAMlB,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YAClDV;YACA,0BAA0BJ;YAC1BH;YACAC;QACF;QACAG,IAAAA,sBAAAA,MAAAA,AAAAA,EACE+B,sBAAsB,IAAI,EAC1B,CAAC,6BAA6B,EAAEhC,YAAY,CAAC,EAC3CgC,sBAAsB,WAAW,CAAC,MAAM,EAAE,SACtC,CAAC,EAAE,EAAEA,sBAAsB,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAC1D,IACJ;QAGJ,MAAML,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;YACzCxB;YACA,UAAU4B,sBAAsB,IAAI;QACtC;QAEA,OAAO;YACLL;YACA,OAAO;gBACL,YAAYA,OAAO,UAAU;gBAC7B,aAAaV,KAAK,SAAS,CAAC;oBAC1B,QAAQ;oBACR,MAAMe,sBAAsB,IAAI;oBAChC,aAAaA,sBAAsB,WAAW;gBAChD;gBACA,kBAAkBA,sBAAsB,gBAAgB;gBACxD,OAAOA,sBAAsB,KAAK;YACpC;QACF;IACF;IAEA,MAAM,QACJC,UAA+B,EAC/BpC,YAA0B,EAC1BD,GAA0B,EAC1BsC,eAAwB,EACxBC,gBAAoC,EACpC/B,OAAmB,EACnBgC,gBAEC,EACiC;QAClCnC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChBH,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAM3B,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMwB,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,oBAAAA,AAAAA,EAAwB;gBAC3ClC;gBACA,WAAW6B;gBACXE;gBACA,eAAevC;gBACfC;gBACAqC;gBACA,aAAaE,kBAAkB;YACjC;YACA5B,cAAc6B,OAAO,WAAW;YAChC3B,cAAc2B,OAAO,WAAW;YAChC1B,mBAAmB0B,OAAO,gBAAgB;YAC1CzB,QAAQyB,OAAO,KAAK;YACpBxB,oBAAoBwB,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,yBAAAA,oBAAoBA,EAAE;gBAEzC,MAAMzB,WAAWR,KAAK,GAAG,KAAKD;gBAC9B,MAAMU,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAawB,MAAM,WAAW;oBAC9B,kBAAkBA,MAAM,gBAAgB;oBACxC,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMlB,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEW;oBAAW;oBACxB,MAAM;oBACNjB;oBACA,OAAOuB,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIhB,kCAAAA,YAAYA,CAACgB,MAAM,OAAO,EAAElB;YACxC;YACA,MAAMkB;QACR;QAEA,MAAMxB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZL;YACAC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTc;YACF;YACA,MAAM;YACNjB;YACA,OAAOE;QACT;QAEA,MAAM,EAAEuB,IAAI,EAAEC,OAAO,EAAE,GAAGlC,eAAe,CAAC;QAE1C,MAAMa,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXsB;QACF;QAEA,IAAIvB,YAAY,CAACuB,MACf,MAAM,IAAIlB,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLoB;YACAC;YACA9B;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJsB,MAA+B,EAC/B9C,YAA0B,EAC1BD,GAGC,EACwD;QACzDK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO0C,QAAQ;QACf,MAAMvC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAC9D,MAAM,EAAEgD,QAAQ,EAAE,GAAGxC;QACrB,MAAMyC,mBAAmBzC,QAAQ,UAAU,CAAC,MAAM;QAClDH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO4C,kBAAkB;QACzB,MAAMC,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIrD,KAAK,YAAY;YACnB,MAAMS,aAAaiD,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;YAQhDpD,MAAM,oCAAoCa;YAC1C,MAAMkD,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAAWJ,cAAc/C;YACrD+C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,IAAIM;QAGJ,IAAI;YACFA,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACVF,MACA5D;QAEJ,EAAE,OAAO0C,OAAO;YACd,MAAMqB,oBAAoBC,AAAAA,IAAAA,kCAAAA,qCAAAA,AAAAA,EAAsCtB;YAChE,IAAI,CAACqB,mBACH,MAAMrB;YAER/C,MAAM;YACN,OAAOoE;QACT;QAEA,MAAM,EAAEE,OAAO,EAAE,GAAGJ;QACpBzD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC6D,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1D7D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO6D,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAhaA,YACE1D,OAA2D,EAC3DR,GAAoB,CACpB;QAPF;QAEA;QAMEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAM2D,QAAQ,OAAO,CAAC3D;QAGlD,IAAI,AAAyB,WAAlBR,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAmZF"}
+{"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { defaultModelFamilyRequiredForLocateMessage } from '@/ai-model/errors';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/inspect';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport type { AIArgs } from '@/ai-model/types';\nimport type { SearchAreaConfig } from '@/ai-model/workflows/inspect/types';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n  compositeElementInfoImg,\n  compositePointMarkerImg,\n  cropByRect,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionContentPart } from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  createServiceDump,\n  getDescribeDeepContextAreas,\n  getDescribeDeepLocateResizeSize,\n  getDescribeMarkerBorderThickness,\n  getDescribeMarkerRect,\n  getRectInCrop,\n  recoverDescribeResponseFromParseError,\n} from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\ninterface LocateSearchAreaResult {\n  config?: SearchAreaConfig;\n  trace: {\n    sourceRect?: Rect;\n    rawResponse?: string;\n    rawChoiceMessage?: unknown;\n    usage?: AIUsageInfo;\n  };\n}\n\nconst debug = getDebug('ai:service');\n\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelRuntime: ModelRuntime,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const { config: modelConfig } = modelRuntime;\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    if (!modelConfig.modelFamily) {\n      throw new Error(defaultModelFamilyRequiredForLocateMessage);\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    const searchArea = await this.resolveLocateSearchArea({\n      query,\n      queryPrompt,\n      opt,\n      context,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n    } = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchArea.config,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea: searchArea.trace.sourceRect,\n      searchAreaRawResponse: searchArea.trace.rawResponse,\n      searchAreaRawChoiceMessage: searchArea.trace.rawChoiceMessage,\n      searchAreaUsage: searchArea.trace.usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea.trace.sourceRect,\n      error: errorLog,\n    };\n\n    const element = parseResult.element;\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: element ? [element] : [],\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (element) {\n      return {\n        element: {\n          center: element.center,\n          rect: element.rect,\n          description: element.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  private async resolveLocateSearchArea(options: {\n    query: PlanningLocateParam;\n    queryPrompt: TUserPrompt;\n    opt: LocateOpts;\n    context: UIContext;\n    modelRuntime: ModelRuntime;\n    abortSignal?: AbortSignal;\n  }): Promise<LocateSearchAreaResult> {\n    const { query, queryPrompt, opt, context, modelRuntime, abortSignal } =\n      options;\n    const { adapter } = modelRuntime;\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    if (!query.deepLocate) {\n      return { trace: {} };\n    }\n\n    if (hasPlanLocatedElement) {\n      const config = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n      });\n\n      return {\n        config,\n        trace: {\n          sourceRect: config.sourceRect,\n          rawResponse: JSON.stringify({\n            source: 'plan-located-element',\n            rect: opt.planLocatedElement!.rect,\n          }),\n        },\n      };\n    }\n\n    if (adapter.locate.supportsSearchArea) {\n      const searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: queryPrompt,\n        modelRuntime,\n        abortSignal,\n      });\n      const { searchAreaConfig } = searchAreaResponse;\n      assert(\n        searchAreaConfig,\n        `cannot find search area for \"${queryPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n\n      return {\n        config: searchAreaConfig,\n        trace: {\n          sourceRect: searchAreaConfig.sourceRect,\n          rawResponse: searchAreaResponse.rawResponse,\n          rawChoiceMessage: searchAreaResponse.rawChoiceMessage,\n          usage: searchAreaResponse.usage,\n        },\n      };\n    }\n\n    const firstPassLocateResult = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      modelRuntime,\n      abortSignal,\n    });\n    assert(\n      firstPassLocateResult.rect,\n      `cannot find search area for \"${queryPrompt}\"${\n        firstPassLocateResult.parseResult.errors?.length\n          ? `: ${firstPassLocateResult.parseResult.errors.join('\\n')}`\n          : ''\n      }`,\n    );\n\n    const config = await buildSearchAreaConfig({\n      context,\n      baseRect: firstPassLocateResult.rect,\n    });\n\n    return {\n      config,\n      trace: {\n        sourceRect: config.sourceRect,\n        rawResponse: JSON.stringify({\n          source: 'deep-locate-first-pass',\n          rect: firstPassLocateResult.rect,\n          rawResponse: firstPassLocateResult.rawResponse,\n        }),\n        rawChoiceMessage: firstPassLocateResult.rawChoiceMessage,\n        usage: firstPassLocateResult.usage,\n      },\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let rawChoiceMessage: unknown;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelRuntime,\n        pageDescription,\n        abortSignal: executionOptions?.abortSignal,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      rawChoiceMessage = result.rawChoiceMessage;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          rawChoiceMessage: error.rawChoiceMessage,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelRuntime: ModelRuntime,\n    opt?: {\n      deepDescribe?: boolean;\n      context?: UIContext;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = opt?.context || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetFromPoint = Array.isArray(target);\n    const targetRect: Rect = targetFromPoint\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n    const targetPoint = targetFromPoint\n      ? {\n          x: target[0],\n          y: target[1],\n        }\n      : undefined;\n\n    const usePointMarker = targetFromPoint;\n    const imagePayload = usePointMarker\n      ? await compositePointMarkerImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          point: targetPoint!,\n        })\n      : await compositeElementInfoImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          elementsPositionInfo: [\n            {\n              rect: getDescribeMarkerRect(targetRect),\n            },\n          ],\n          borderThickness: getDescribeMarkerBorderThickness(targetRect),\n          centerPoint: true,\n        });\n\n    const shouldDeepDescribe = opt?.deepDescribe;\n    let imageContent: ChatCompletionContentPart[];\n    if (shouldDeepDescribe) {\n      const contextAreas = getDescribeDeepContextAreas(targetRect, shotSize);\n      const contextImages = await Promise.all(\n        contextAreas.map(async (area) => {\n          debug('describe: cropping deep context area', area);\n          const croppedResult = await cropByRect(screenshotBase64, area.rect);\n          const cropSize = {\n            width: croppedResult.width,\n            height: croppedResult.height,\n          };\n          const targetInCrop = getRectInCrop(targetRect, area.rect, cropSize);\n          const markedCropPayload = targetFromPoint\n            ? await compositePointMarkerImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                point: {\n                  x: targetPoint!.x - area.rect.left,\n                  y: targetPoint!.y - area.rect.top,\n                },\n              })\n            : await compositeElementInfoImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                elementsPositionInfo: [\n                  {\n                    rect: getDescribeMarkerRect(targetInCrop),\n                  },\n                ],\n                borderThickness: getDescribeMarkerBorderThickness(targetInCrop),\n                centerPoint: true,\n              });\n          const resizeSize = getDescribeDeepLocateResizeSize(croppedResult);\n          return {\n            kind: area.kind,\n            imageBase64: resizeSize\n              ? await resizeImgBase64(markedCropPayload, resizeSize)\n              : markedCropPayload,\n          };\n        }),\n      );\n      const contextImageContent =\n        contextImages.flatMap<ChatCompletionContentPart>((item, index) => [\n          {\n            type: 'text',\n            text: `Image ${index + 2}: focused detail crop around the target, for reading text, icon shape, and exact local boundaries.`,\n          },\n          {\n            type: 'image_url',\n            image_url: {\n              url: item.imageBase64,\n              detail: 'high',\n            },\n          },\n        ]);\n\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.',\n        },\n        {\n          type: 'text' as const,\n          text: 'Image 1: full screenshot overview with the target marker, for page position and ownership context.',\n        },\n        {\n          type: 'image_url' as const,\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        ...contextImageContent,\n      ];\n    } else {\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Full screenshot with a temporary callout marking the target:',\n        },\n        {\n          type: 'image_url' as const,\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ];\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: imageContent,\n      },\n    ];\n\n    let res: Awaited<\n      ReturnType<typeof callAIWithObjectResponse<AIDescribeElementResponse>>\n    >;\n    try {\n      res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n        msgs,\n        modelRuntime,\n      );\n    } catch (error) {\n      const recoveredResponse = recoverDescribeResponseFromParseError(error);\n      if (!recoveredResponse) {\n        throw error;\n      }\n      debug('describe: recovered malformed description JSON response');\n      return recoveredResponse;\n    }\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelRuntime","abortSignal","modelConfig","queryPrompt","assert","Error","defaultModelFamilyRequiredForLocateMessage","context","searchArea","startTime","Date","parseResult","rect","rawResponse","rawChoiceMessage","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","element","dump","createServiceDump","ServiceError","options","adapter","hasPlanLocatedElement","config","buildSearchAreaConfig","searchAreaResponse","AiLocateSection","searchAreaConfig","firstPassLocateResult","dataDemand","pageDescription","multimodalPrompt","executionOptions","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetFromPoint","Array","targetRect","Math","targetPoint","undefined","usePointMarker","imagePayload","compositePointMarkerImg","compositeElementInfoImg","getDescribeMarkerRect","getDescribeMarkerBorderThickness","shouldDeepDescribe","imageContent","contextAreas","getDescribeDeepContextAreas","contextImages","Promise","area","croppedResult","cropByRect","cropSize","targetInCrop","getRectInCrop","markedCropPayload","resizeSize","getDescribeDeepLocateResizeSize","resizeImgBase64","contextImageContent","item","index","msgs","res","callAIWithObjectResponse","recoveredResponse","recoverDescribeResponseFromParseError","content"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;ACoEA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAER,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,YAA0B,EAC1BC,WAAyB,EACM;QAC/B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGF;QAChC,MAAMG,cAAc,AAAiB,YAAjB,OAAOL,QAAqBA,QAAQA,MAAM,MAAM;QACpEM,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAON,OAAoB;QAElC,IAAI,CAACI,YAAY,WAAW,EAC1B,MAAM,IAAIG,MAAMC,0BAAAA,0CAA0CA;QAG5D,MAAMC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,MAAMS,aAAa,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACpDV;YACAK;YACAJ;YACAQ;YACAP;YACAC;QACF;QAEA,MAAMQ,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YACxBV;YACA,0BAA0BJ;YAC1B,cAAcK,WAAW,MAAM;YAC/BR;YACAC;QACF;QAEA,MAAMiB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACP;YAC5BC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACA,YAAYP,WAAW,KAAK,CAAC,UAAU;YACvC,uBAAuBA,WAAW,KAAK,CAAC,WAAW;YACnD,4BAA4BA,WAAW,KAAK,CAAC,gBAAgB;YAC7D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACvCQ;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,4BAA4B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASnB;YACX;YACA,aAAaS;YACb,MAAM;YACNO;YACA,YAAY,CAAC,CAACX,WAAW,KAAK,CAAC,UAAU;YACzC,OAAOa;QACT;QAEA,MAAME,UAAUZ,YAAY,OAAO;QAEnC,MAAMa,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC,UAAU;gBAACA;aAAQ,GAAG,EAAE;QAC1C;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SACF,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,MAAM;gBACtB,MAAMA,QAAQ,IAAI;gBAClB,aAAaA,QAAQ,WAAW;YAClC;YACAX;YACAY;QACF;QAGF,OAAO;YACL,SAAS;YACTZ;YACAY;QACF;IACF;IAEA,MAAc,wBAAwBG,OAOrC,EAAmC;QAClC,MAAM,EAAE7B,KAAK,EAAEK,WAAW,EAAEJ,GAAG,EAAEQ,OAAO,EAAEP,YAAY,EAAEC,WAAW,EAAE,GACnE0B;QACF,MAAM,EAAEC,OAAO,EAAE,GAAG5B;QACpB,MAAM6B,wBAAwB,CAAC,CAAC9B,KAAK,oBAAoB;QAEzD,IAAI,CAACD,MAAM,UAAU,EACnB,OAAO;YAAE,OAAO,CAAC;QAAE;QAGrB,IAAI+B,uBAAuB;YACzB,MAAMC,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;gBACzCxB;gBACA,UAAUR,IAAI,kBAAkB,CAAE,IAAI;YACxC;YAEA,OAAO;gBACL+B;gBACA,OAAO;oBACL,YAAYA,OAAO,UAAU;oBAC7B,aAAaV,KAAK,SAAS,CAAC;wBAC1B,QAAQ;wBACR,MAAMrB,IAAI,kBAAkB,CAAE,IAAI;oBACpC;gBACF;YACF;QACF;QAEA,IAAI6B,QAAQ,MAAM,CAAC,kBAAkB,EAAE;YACrC,MAAMI,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBAC/C1B;gBACA,oBAAoBJ;gBACpBH;gBACAC;YACF;YACA,MAAM,EAAEiC,gBAAgB,EAAE,GAAGF;YAC7B5B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8B,kBACA,CAAC,6BAA6B,EAAE/B,YAAY,CAAC,EAC3C6B,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAGJ,OAAO;gBACL,QAAQE;gBACR,OAAO;oBACL,YAAYA,iBAAiB,UAAU;oBACvC,aAAaF,mBAAmB,WAAW;oBAC3C,kBAAkBA,mBAAmB,gBAAgB;oBACrD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;QACF;QAEA,MAAMG,wBAAwB,MAAMlB,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YAClDV;YACA,0BAA0BJ;YAC1BH;YACAC;QACF;QACAG,IAAAA,sBAAAA,MAAAA,AAAAA,EACE+B,sBAAsB,IAAI,EAC1B,CAAC,6BAA6B,EAAEhC,YAAY,CAAC,EAC3CgC,sBAAsB,WAAW,CAAC,MAAM,EAAE,SACtC,CAAC,EAAE,EAAEA,sBAAsB,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAC1D,IACJ;QAGJ,MAAML,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;YACzCxB;YACA,UAAU4B,sBAAsB,IAAI;QACtC;QAEA,OAAO;YACLL;YACA,OAAO;gBACL,YAAYA,OAAO,UAAU;gBAC7B,aAAaV,KAAK,SAAS,CAAC;oBAC1B,QAAQ;oBACR,MAAMe,sBAAsB,IAAI;oBAChC,aAAaA,sBAAsB,WAAW;gBAChD;gBACA,kBAAkBA,sBAAsB,gBAAgB;gBACxD,OAAOA,sBAAsB,KAAK;YACpC;QACF;IACF;IAEA,MAAM,QACJC,UAA+B,EAC/BpC,YAA0B,EAC1BD,GAA0B,EAC1BsC,eAAwB,EACxBC,gBAAoC,EACpC/B,OAAmB,EACnBgC,gBAEC,EACiC;QAClCnC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChBH,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAM3B,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMwB,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,oBAAAA,AAAAA,EAAwB;gBAC3ClC;gBACA,WAAW6B;gBACXE;gBACA,eAAevC;gBACfC;gBACAqC;gBACA,aAAaE,kBAAkB;YACjC;YACA5B,cAAc6B,OAAO,WAAW;YAChC3B,cAAc2B,OAAO,WAAW;YAChC1B,mBAAmB0B,OAAO,gBAAgB;YAC1CzB,QAAQyB,OAAO,KAAK;YACpBxB,oBAAoBwB,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,yBAAAA,oBAAoBA,EAAE;gBAEzC,MAAMzB,WAAWR,KAAK,GAAG,KAAKD;gBAC9B,MAAMU,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAawB,MAAM,WAAW;oBAC9B,kBAAkBA,MAAM,gBAAgB;oBACxC,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMlB,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEW;oBAAW;oBACxB,MAAM;oBACNjB;oBACA,OAAOuB,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIhB,kCAAAA,YAAYA,CAACgB,MAAM,OAAO,EAAElB;YACxC;YACA,MAAMkB;QACR;QAEA,MAAMxB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZL;YACAC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTc;YACF;YACA,MAAM;YACNjB;YACA,OAAOE;QACT;QAEA,MAAM,EAAEuB,IAAI,EAAEC,OAAO,EAAE,GAAGlC,eAAe,CAAC;QAE1C,MAAMa,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXsB;QACF;QAEA,IAAIvB,YAAY,CAACuB,MACf,MAAM,IAAIlB,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLoB;YACAC;YACA9B;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJsB,MAA+B,EAC/B9C,YAA0B,EAC1BD,GAGC,EACwD;QACzDK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO0C,QAAQ;QACf,MAAMvC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAC9D,MAAM,EAAEgD,QAAQ,EAAE,GAAGxC;QACrB,MAAMyC,mBAAmBzC,QAAQ,UAAU,CAAC,MAAM;QAClDH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO4C,kBAAkB;QACzB,MAAMC,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,kBAAkBC,MAAM,OAAO,CAACP;QACtC,MAAMQ,aAAmBF,kBACrB;YACE,MAAMG,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKI,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QACJ,MAAMU,cAAcJ,kBAChB;YACE,GAAGN,MAAM,CAAC,EAAE;YACZ,GAAGA,MAAM,CAAC,EAAE;QACd,IACAW;QAEJ,MAAMC,iBAAiBN;QACvB,MAAMO,eAAeD,iBACjB,MAAME,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC5B,gBAAgBZ;YAChB,MAAMD;YACN,OAAOS;QACT,KACA,MAAMK,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC5B,gBAAgBb;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMe,AAAAA,IAAAA,kCAAAA,qBAAAA,AAAAA,EAAsBR;gBAC9B;aACD;YACD,iBAAiBS,AAAAA,IAAAA,kCAAAA,gCAAAA,AAAAA,EAAiCT;YAClD,aAAa;QACf;QAEJ,MAAMU,qBAAqBjE,KAAK;QAChC,IAAIkE;QACJ,IAAID,oBAAoB;YACtB,MAAME,eAAeC,AAAAA,IAAAA,kCAAAA,2BAAAA,AAAAA,EAA4Bb,YAAYP;YAC7D,MAAMqB,gBAAgB,MAAMC,QAAQ,GAAG,CACrCH,aAAa,GAAG,CAAC,OAAOI;gBACtB3E,MAAM,wCAAwC2E;gBAC9C,MAAMC,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAAWxB,kBAAkBsB,KAAK,IAAI;gBAClE,MAAMG,WAAW;oBACf,OAAOF,cAAc,KAAK;oBAC1B,QAAQA,cAAc,MAAM;gBAC9B;gBACA,MAAMG,eAAeC,AAAAA,IAAAA,kCAAAA,aAAAA,AAAAA,EAAcrB,YAAYgB,KAAK,IAAI,EAAEG;gBAC1D,MAAMG,oBAAoBxB,kBACtB,MAAMQ,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;oBAC5B,gBAAgBW,cAAc,WAAW;oBACzC,MAAME;oBACN,OAAO;wBACL,GAAGjB,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,IAAI;wBAClC,GAAGd,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,GAAG;oBACnC;gBACF,KACA,MAAMT,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;oBAC5B,gBAAgBU,cAAc,WAAW;oBACzC,MAAME;oBACN,sBAAsB;wBACpB;4BACE,MAAMX,AAAAA,IAAAA,kCAAAA,qBAAAA,AAAAA,EAAsBY;wBAC9B;qBACD;oBACD,iBAAiBX,AAAAA,IAAAA,kCAAAA,gCAAAA,AAAAA,EAAiCW;oBAClD,aAAa;gBACf;gBACJ,MAAMG,aAAaC,AAAAA,IAAAA,kCAAAA,+BAAAA,AAAAA,EAAgCP;gBACnD,OAAO;oBACL,MAAMD,KAAK,IAAI;oBACf,aAAaO,aACT,MAAME,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBH,mBAAmBC,cACzCD;gBACN;YACF;YAEF,MAAMI,sBACJZ,cAAc,OAAO,CAA4B,CAACa,MAAMC,QAAU;oBAChE;wBACE,MAAM;wBACN,MAAM,CAAC,MAAM,EAAEA,QAAQ,EAAE,kGAAkG,CAAC;oBAC9H;oBACA;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKD,KAAK,WAAW;4BACrB,QAAQ;wBACV;oBACF;iBACD;YAEHhB,eAAe;gBACb;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKN;wBACL,QAAQ;oBACV;gBACF;mBACGqB;aACJ;QACH,OACEf,eAAe;YACb;gBACE,MAAM;gBACN,MAAM;YACR;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKN;oBACL,QAAQ;gBACV;YACF;SACD;QAGH,MAAMwB,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASlC;YAAa;YACxC;gBACE,MAAM;gBACN,SAASgB;YACX;SACD;QAED,IAAImB;QAGJ,IAAI;YACFA,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACVF,MACAnF;QAEJ,EAAE,OAAO0C,OAAO;YACd,MAAM4C,oBAAoBC,AAAAA,IAAAA,kCAAAA,qCAAAA,AAAAA,EAAsC7C;YAChE,IAAI,CAAC4C,mBACH,MAAM5C;YAER/C,MAAM;YACN,OAAO2F;QACT;QAEA,MAAM,EAAEE,OAAO,EAAE,GAAGJ;QACpBhF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACoF,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DpF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOoF,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IApfA,YACEjF,OAA2D,EAC3DR,GAAoB,CACpB;QAPF;QAEA;QAMEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAM8D,QAAQ,OAAO,CAAC9D;QAGlD,IAAI,AAAyB,WAAlBR,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAueF"}

package/dist/lib/service/utils.js

@@ -24,11 +24,86 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
+    DESCRIBE_DEEP_CONTEXT_CONFIG: ()=>DESCRIBE_DEEP_CONTEXT_CONFIG,
+    getDescribeDeepContextAreas: ()=>getDescribeDeepContextAreas,
+    DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS: ()=>DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS,
+    getDescribeDeepLocateResizeSize: ()=>getDescribeDeepLocateResizeSize,
     recoverDescribeResponseFromParseError: ()=>recoverDescribeResponseFromParseError,
-    createServiceDump: ()=>createServiceDump
+    clampRect: ()=>clampRect,
+    getRectInCrop: ()=>getRectInCrop,
+    DESCRIBE_POINT_MARKER_MAX_SIZE: ()=>DESCRIBE_POINT_MARKER_MAX_SIZE,
+    DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH: ()=>DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH,
+    getDescribeMarkerBorderThickness: ()=>getDescribeMarkerBorderThickness,
+    createServiceDump: ()=>createServiceDump,
+    getDescribeMarkerRect: ()=>getDescribeMarkerRect,
+    DESCRIBE_RECT_MARKER_BORDER_THICKNESS: ()=>DESCRIBE_RECT_MARKER_BORDER_THICKNESS
 });
 const index_js_namespaceObject = require("../ai-model/service-caller/index.js");
+const external_common_js_namespaceObject = require("../common.js");
 const utils_namespaceObject = require("@midscene/shared/utils");
+const DESCRIBE_POINT_MARKER_MAX_SIZE = 40;
+const DESCRIBE_RECT_MARKER_BORDER_THICKNESS = 1;
+const DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS = 2;
+const DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH = 100;
+const DESCRIBE_WIDE_MARKER_HORIZONTAL_INSET_RATIO = 0.15;
+const DESCRIBE_WIDE_MARKER_VERTICAL_INSET_RATIO = 0.1;
+const DESCRIBE_DEEP_CONTEXT_CONFIG = {
+    resize: {
+        cropMaxLongEdge: 1000,
+        cropUpscaleMaxRatio: 2
+    }
+};
+function clampRect(rect, size) {
+    const width = Math.min(rect.width, size.width);
+    const height = Math.min(rect.height, size.height);
+    return {
+        left: Math.max(0, Math.min(rect.left, size.width - width)),
+        top: Math.max(0, Math.min(rect.top, size.height - height)),
+        width,
+        height
+    };
+}
+function getDescribeDeepContextAreas(rect, screenSize) {
+    return [
+        {
+            kind: 'focused',
+            rect: (0, external_common_js_namespaceObject.expandSearchArea)(rect, screenSize)
+        }
+    ];
+}
+function getRectInCrop(rect, cropRect, cropSize) {
+    return clampRect({
+        left: rect.left - cropRect.left,
+        top: rect.top - cropRect.top,
+        width: rect.width,
+        height: rect.height
+    }, cropSize);
+}
+function getDescribeMarkerRect(rect) {
+    if (rect.width < DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH) return rect;
+    const horizontalInset = Math.round(rect.width * DESCRIBE_WIDE_MARKER_HORIZONTAL_INSET_RATIO);
+    const verticalInset = Math.round(rect.height * DESCRIBE_WIDE_MARKER_VERTICAL_INSET_RATIO);
+    return {
+        left: rect.left + horizontalInset,
+        top: rect.top + verticalInset,
+        width: Math.max(rect.width - 2 * horizontalInset, 1),
+        height: Math.max(rect.height - 2 * verticalInset, 1)
+    };
+}
+function getDescribeMarkerBorderThickness(rect) {
+    return rect.width <= DESCRIBE_POINT_MARKER_MAX_SIZE && rect.height <= DESCRIBE_POINT_MARKER_MAX_SIZE ? DESCRIBE_RECT_MARKER_BORDER_THICKNESS : DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS;
+}
+function getDescribeDeepLocateResizeSize(size) {
+    const maxEdge = Math.max(size.width, size.height);
+    if (!maxEdge) return;
+    const { resize } = DESCRIBE_DEEP_CONTEXT_CONFIG;
+    const scale = Math.min(resize.cropUpscaleMaxRatio, resize.cropMaxLongEdge / maxEdge);
+    if (scale <= 1.05) return;
+    return {
+        width: Math.round(size.width * scale),
+        height: Math.round(size.height * scale)
+    };
+}
 function createServiceDump(data) {
     const baseData = {
         logTime: Date.now()
@@ -88,10 +163,32 @@ function recoverDescribeResponseFromParseError(error) {
         description
     };
 }
+exports.DESCRIBE_DEEP_CONTEXT_CONFIG = __webpack_exports__.DESCRIBE_DEEP_CONTEXT_CONFIG;
+exports.DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS = __webpack_exports__.DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS;
+exports.DESCRIBE_POINT_MARKER_MAX_SIZE = __webpack_exports__.DESCRIBE_POINT_MARKER_MAX_SIZE;
+exports.DESCRIBE_RECT_MARKER_BORDER_THICKNESS = __webpack_exports__.DESCRIBE_RECT_MARKER_BORDER_THICKNESS;
+exports.DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH = __webpack_exports__.DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH;
+exports.clampRect = __webpack_exports__.clampRect;
 exports.createServiceDump = __webpack_exports__.createServiceDump;
+exports.getDescribeDeepContextAreas = __webpack_exports__.getDescribeDeepContextAreas;
+exports.getDescribeDeepLocateResizeSize = __webpack_exports__.getDescribeDeepLocateResizeSize;
+exports.getDescribeMarkerBorderThickness = __webpack_exports__.getDescribeMarkerBorderThickness;
+exports.getDescribeMarkerRect = __webpack_exports__.getDescribeMarkerRect;
+exports.getRectInCrop = __webpack_exports__.getRectInCrop;
 exports.recoverDescribeResponseFromParseError = __webpack_exports__.recoverDescribeResponseFromParseError;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "DESCRIBE_DEEP_CONTEXT_CONFIG",
+    "DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS",
+    "DESCRIBE_POINT_MARKER_MAX_SIZE",
+    "DESCRIBE_RECT_MARKER_BORDER_THICKNESS",
+    "DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH",
+    "clampRect",
     "createServiceDump",
+    "getDescribeDeepContextAreas",
+    "getDescribeDeepLocateResizeSize",
+    "getDescribeMarkerBorderThickness",
+    "getDescribeMarkerRect",
+    "getRectInCrop",
     "recoverDescribeResponseFromParseError"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {

package/dist/lib/service/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"service/utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  AIResponseParseError,\n  extractJSONFromCodeBlock,\n} from '@/ai-model/service-caller';\nimport type { AIDescribeElementResponse } from '@/types';\nimport type { DumpMeta, PartialServiceDumpFromSDK, ServiceDump } from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function createServiceDump(\n  data: PartialServiceDumpFromSDK,\n): ServiceDump {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: ServiceDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  return finalData;\n}\n\nfunction readNextSignificantChar(input: string, startIndex: number) {\n  let index = startIndex;\n  while (index < input.length && /\\s/.test(input[index])) {\n    index += 1;\n  }\n  return input[index];\n}\n\nfunction extractPossiblyMalformedStringField(input: string, fieldName: string) {\n  const escapedFieldName = fieldName.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n  const fieldStart = new RegExp(`\"${escapedFieldName}\"\\\\s*:\\\\s*\"`).exec(input);\n  if (!fieldStart) {\n    return undefined;\n  }\n\n  let index = fieldStart.index + fieldStart[0].length;\n  let escaped = false;\n  let valueForJsonParse = '';\n\n  for (; index < input.length; index += 1) {\n    const char = input[index];\n\n    if (escaped) {\n      valueForJsonParse += char;\n      escaped = false;\n      continue;\n    }\n\n    if (char === '\\\\') {\n      valueForJsonParse += char;\n      escaped = true;\n      continue;\n    }\n\n    if (char !== '\"') {\n      valueForJsonParse += char;\n      continue;\n    }\n\n    const nextSignificantChar = readNextSignificantChar(input, index + 1);\n    if (\n      nextSignificantChar === ',' ||\n      nextSignificantChar === '}' ||\n      nextSignificantChar === ']' ||\n      nextSignificantChar === undefined\n    ) {\n      try {\n        return JSON.parse(`\"${valueForJsonParse}\"`);\n      } catch {\n        return valueForJsonParse;\n      }\n    }\n\n    valueForJsonParse += '\\\\\"';\n  }\n\n  return undefined;\n}\n\nexport function recoverDescribeResponseFromParseError(\n  error: unknown,\n): Pick<AIDescribeElementResponse, 'description'> | undefined {\n  const message = error instanceof Error ? error.message : String(error);\n  const rawResponse =\n    error instanceof AIResponseParseError\n      ? error.rawResponse\n      : message.match(/Response -\\s*\\n\\s*([\\s\\S]*)$/)?.[1];\n\n  if (\n    !rawResponse ||\n    (!message.includes('failed to parse LLM response into JSON') &&\n      !(error instanceof AIResponseParseError))\n  ) {\n    return undefined;\n  }\n\n  const jsonLikeResponse = extractJSONFromCodeBlock(rawResponse);\n  const description = extractPossiblyMalformedStringField(\n    jsonLikeResponse,\n    'description',\n  )?.trim();\n\n  if (!description) {\n    return undefined;\n  }\n\n  return { description };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createServiceDump","data","baseData","Date","finalData","uuid","readNextSignificantChar","input","startIndex","index","extractPossiblyMalformedStringField","fieldName","escapedFieldName","fieldStart","RegExp","escaped","valueForJsonParse","char","nextSignificantChar","undefined","JSON","recoverDescribeResponseFromParseError","error","message","Error","String","rawResponse","AIResponseParseError","jsonLikeResponse","extractJSONFromCodeBlock","description"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACEO,SAASI,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT;AAEA,SAASE,wBAAwBC,KAAa,EAAEC,UAAkB;IAChE,IAAIC,QAAQD;IACZ,MAAOC,QAAQF,MAAM,MAAM,IAAI,KAAK,IAAI,CAACA,KAAK,CAACE,MAAM,EACnDA,SAAS;IAEX,OAAOF,KAAK,CAACE,MAAM;AACrB;AAEA,SAASC,oCAAoCH,KAAa,EAAEI,SAAiB;IAC3E,MAAMC,mBAAmBD,UAAU,OAAO,CAAC,uBAAuB;IAClE,MAAME,aAAa,IAAIC,OAAO,CAAC,CAAC,EAAEF,iBAAiB,WAAW,CAAC,EAAE,IAAI,CAACL;IACtE,IAAI,CAACM,YACH;IAGF,IAAIJ,QAAQI,WAAW,KAAK,GAAGA,UAAU,CAAC,EAAE,CAAC,MAAM;IACnD,IAAIE,UAAU;IACd,IAAIC,oBAAoB;IAExB,MAAOP,QAAQF,MAAM,MAAM,EAAEE,SAAS,EAAG;QACvC,MAAMQ,OAAOV,KAAK,CAACE,MAAM;QAEzB,IAAIM,SAAS;YACXC,qBAAqBC;YACrBF,UAAU;YACV;QACF;QAEA,IAAIE,AAAS,SAATA,MAAe;YACjBD,qBAAqBC;YACrBF,UAAU;YACV;QACF;QAEA,IAAIE,AAAS,QAATA,MAAc;YAChBD,qBAAqBC;YACrB;QACF;QAEA,MAAMC,sBAAsBZ,wBAAwBC,OAAOE,QAAQ;QACnE,IACES,AAAwB,QAAxBA,uBACAA,AAAwB,QAAxBA,uBACAA,AAAwB,QAAxBA,uBACAA,AAAwBC,WAAxBD,qBAEA,IAAI;YACF,OAAOE,KAAK,KAAK,CAAC,CAAC,CAAC,EAAEJ,kBAAkB,CAAC,CAAC;QAC5C,EAAE,OAAM;YACN,OAAOA;QACT;QAGFA,qBAAqB;IACvB;AAGF;AAEO,SAASK,sCACdC,KAAc;IAEd,MAAMC,UAAUD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;IAChE,MAAMI,cACJJ,iBAAiBK,yBAAAA,oBAAoBA,GACjCL,MAAM,WAAW,GACjBC,QAAQ,KAAK,CAAC,iCAAiC,CAAC,EAAE;IAExD,IACE,CAACG,eACA,CAACH,QAAQ,QAAQ,CAAC,6CACjB,CAAED,CAAAA,iBAAiBK,yBAAAA,oBAAmB,AAAnBA,GAErB;IAGF,MAAMC,mBAAmBC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyBH;IAClD,MAAMI,cAAcpB,oCAClBkB,kBACA,gBACC;IAEH,IAAI,CAACE,aACH;IAGF,OAAO;QAAEA;IAAY;AACvB"}
+{"version":3,"file":"service/utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  AIResponseParseError,\n  extractJSONFromCodeBlock,\n} from '@/ai-model/service-caller';\nimport { expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  DumpMeta,\n  PartialServiceDumpFromSDK,\n  Rect,\n  ServiceDump,\n  Size,\n} from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport const DESCRIBE_POINT_MARKER_MAX_SIZE = 40;\nexport const DESCRIBE_RECT_MARKER_BORDER_THICKNESS = 1;\nexport const DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS = 2;\nexport const DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH = 100;\nconst DESCRIBE_WIDE_MARKER_HORIZONTAL_INSET_RATIO = 0.15;\nconst DESCRIBE_WIDE_MARKER_VERTICAL_INSET_RATIO = 0.1;\n\n// Deep describe uses two images: an overview for page-level ownership and a\n// focused crop for local detail. The overview keeps the original screenshot\n// size; only focused crops may be upscaled for small local details.\nexport const DESCRIBE_DEEP_CONTEXT_CONFIG = {\n  resize: {\n    cropMaxLongEdge: 1000,\n    cropUpscaleMaxRatio: 2,\n  },\n} as const;\n\nexport type DescribeDeepContextArea = {\n  kind: 'focused';\n  rect: Rect;\n};\n\nexport function clampRect(rect: Rect, size: Size): Rect {\n  const width = Math.min(rect.width, size.width);\n  const height = Math.min(rect.height, size.height);\n  return {\n    left: Math.max(0, Math.min(rect.left, size.width - width)),\n    top: Math.max(0, Math.min(rect.top, size.height - height)),\n    width,\n    height,\n  };\n}\n\nexport function getDescribeDeepContextAreas(\n  rect: Rect,\n  screenSize: Size,\n): DescribeDeepContextArea[] {\n  return [{ kind: 'focused', rect: expandSearchArea(rect, screenSize) }];\n}\n\nexport function getRectInCrop(\n  rect: Rect,\n  cropRect: Rect,\n  cropSize: Size,\n): Rect {\n  return clampRect(\n    {\n      left: rect.left - cropRect.left,\n      top: rect.top - cropRect.top,\n      width: rect.width,\n      height: rect.height,\n    },\n    cropSize,\n  );\n}\n\nexport function getDescribeMarkerRect(rect: Rect): Rect {\n  if (rect.width < DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH) {\n    return rect;\n  }\n\n  const horizontalInset = Math.round(\n    rect.width * DESCRIBE_WIDE_MARKER_HORIZONTAL_INSET_RATIO,\n  );\n  const verticalInset = Math.round(\n    rect.height * DESCRIBE_WIDE_MARKER_VERTICAL_INSET_RATIO,\n  );\n\n  return {\n    left: rect.left + horizontalInset,\n    top: rect.top + verticalInset,\n    width: Math.max(rect.width - horizontalInset * 2, 1),\n    height: Math.max(rect.height - verticalInset * 2, 1),\n  };\n}\n\nexport function getDescribeMarkerBorderThickness(rect: Rect): number {\n  return rect.width <= DESCRIBE_POINT_MARKER_MAX_SIZE &&\n    rect.height <= DESCRIBE_POINT_MARKER_MAX_SIZE\n    ? DESCRIBE_RECT_MARKER_BORDER_THICKNESS\n    : DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS;\n}\n\nexport function getDescribeDeepLocateResizeSize(size: Size): Size | undefined {\n  const maxEdge = Math.max(size.width, size.height);\n  if (!maxEdge) {\n    return undefined;\n  }\n  const { resize } = DESCRIBE_DEEP_CONTEXT_CONFIG;\n  const scale = Math.min(\n    resize.cropUpscaleMaxRatio,\n    resize.cropMaxLongEdge / maxEdge,\n  );\n  if (scale <= 1.05) {\n    return undefined;\n  }\n  return {\n    width: Math.round(size.width * scale),\n    height: Math.round(size.height * scale),\n  };\n}\n\nexport function createServiceDump(\n  data: PartialServiceDumpFromSDK,\n): ServiceDump {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: ServiceDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  return finalData;\n}\n\nfunction readNextSignificantChar(input: string, startIndex: number) {\n  let index = startIndex;\n  while (index < input.length && /\\s/.test(input[index])) {\n    index += 1;\n  }\n  return input[index];\n}\n\nfunction extractPossiblyMalformedStringField(input: string, fieldName: string) {\n  const escapedFieldName = fieldName.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n  const fieldStart = new RegExp(`\"${escapedFieldName}\"\\\\s*:\\\\s*\"`).exec(input);\n  if (!fieldStart) {\n    return undefined;\n  }\n\n  let index = fieldStart.index + fieldStart[0].length;\n  let escaped = false;\n  let valueForJsonParse = '';\n\n  for (; index < input.length; index += 1) {\n    const char = input[index];\n\n    if (escaped) {\n      valueForJsonParse += char;\n      escaped = false;\n      continue;\n    }\n\n    if (char === '\\\\') {\n      valueForJsonParse += char;\n      escaped = true;\n      continue;\n    }\n\n    if (char !== '\"') {\n      valueForJsonParse += char;\n      continue;\n    }\n\n    const nextSignificantChar = readNextSignificantChar(input, index + 1);\n    if (\n      nextSignificantChar === ',' ||\n      nextSignificantChar === '}' ||\n      nextSignificantChar === ']' ||\n      nextSignificantChar === undefined\n    ) {\n      try {\n        return JSON.parse(`\"${valueForJsonParse}\"`);\n      } catch {\n        return valueForJsonParse;\n      }\n    }\n\n    valueForJsonParse += '\\\\\"';\n  }\n\n  return undefined;\n}\n\nexport function recoverDescribeResponseFromParseError(\n  error: unknown,\n): Pick<AIDescribeElementResponse, 'description'> | undefined {\n  const message = error instanceof Error ? error.message : String(error);\n  const rawResponse =\n    error instanceof AIResponseParseError\n      ? error.rawResponse\n      : message.match(/Response -\\s*\\n\\s*([\\s\\S]*)$/)?.[1];\n\n  if (\n    !rawResponse ||\n    (!message.includes('failed to parse LLM response into JSON') &&\n      !(error instanceof AIResponseParseError))\n  ) {\n    return undefined;\n  }\n\n  const jsonLikeResponse = extractJSONFromCodeBlock(rawResponse);\n  const description = extractPossiblyMalformedStringField(\n    jsonLikeResponse,\n    'description',\n  )?.trim();\n\n  if (!description) {\n    return undefined;\n  }\n\n  return { description };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","DESCRIBE_POINT_MARKER_MAX_SIZE","DESCRIBE_RECT_MARKER_BORDER_THICKNESS","DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS","DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH","DESCRIBE_WIDE_MARKER_HORIZONTAL_INSET_RATIO","DESCRIBE_WIDE_MARKER_VERTICAL_INSET_RATIO","DESCRIBE_DEEP_CONTEXT_CONFIG","clampRect","rect","size","width","Math","height","getDescribeDeepContextAreas","screenSize","expandSearchArea","getRectInCrop","cropRect","cropSize","getDescribeMarkerRect","horizontalInset","verticalInset","getDescribeMarkerBorderThickness","getDescribeDeepLocateResizeSize","maxEdge","resize","scale","createServiceDump","data","baseData","Date","finalData","uuid","readNextSignificantChar","input","startIndex","index","extractPossiblyMalformedStringField","fieldName","escapedFieldName","fieldStart","RegExp","escaped","valueForJsonParse","char","nextSignificantChar","undefined","JSON","recoverDescribeResponseFromParseError","error","message","Error","String","rawResponse","AIResponseParseError","jsonLikeResponse","extractJSONFromCodeBlock","description"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;ACSO,MAAMI,iCAAiC;AACvC,MAAMC,wCAAwC;AAC9C,MAAMC,8CAA8C;AACpD,MAAMC,uCAAuC;AACpD,MAAMC,8CAA8C;AACpD,MAAMC,4CAA4C;AAK3C,MAAMC,+BAA+B;IAC1C,QAAQ;QACN,iBAAiB;QACjB,qBAAqB;IACvB;AACF;AAOO,SAASC,UAAUC,IAAU,EAAEC,IAAU;IAC9C,MAAMC,QAAQC,KAAK,GAAG,CAACH,KAAK,KAAK,EAAEC,KAAK,KAAK;IAC7C,MAAMG,SAASD,KAAK,GAAG,CAACH,KAAK,MAAM,EAAEC,KAAK,MAAM;IAChD,OAAO;QACL,MAAME,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,KAAK,IAAI,EAAEC,KAAK,KAAK,GAAGC;QACnD,KAAKC,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,KAAK,GAAG,EAAEC,KAAK,MAAM,GAAGG;QAClDF;QACAE;IACF;AACF;AAEO,SAASC,4BACdL,IAAU,EACVM,UAAgB;IAEhB,OAAO;QAAC;YAAE,MAAM;YAAW,MAAMC,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBP,MAAMM;QAAY;KAAE;AACxE;AAEO,SAASE,cACdR,IAAU,EACVS,QAAc,EACdC,QAAc;IAEd,OAAOX,UACL;QACE,MAAMC,KAAK,IAAI,GAAGS,SAAS,IAAI;QAC/B,KAAKT,KAAK,GAAG,GAAGS,SAAS,GAAG;QAC5B,OAAOT,KAAK,KAAK;QACjB,QAAQA,KAAK,MAAM;IACrB,GACAU;AAEJ;AAEO,SAASC,sBAAsBX,IAAU;IAC9C,IAAIA,KAAK,KAAK,GAAGL,sCACf,OAAOK;IAGT,MAAMY,kBAAkBT,KAAK,KAAK,CAChCH,KAAK,KAAK,GAAGJ;IAEf,MAAMiB,gBAAgBV,KAAK,KAAK,CAC9BH,KAAK,MAAM,GAAGH;IAGhB,OAAO;QACL,MAAMG,KAAK,IAAI,GAAGY;QAClB,KAAKZ,KAAK,GAAG,GAAGa;QAChB,OAAOV,KAAK,GAAG,CAACH,KAAK,KAAK,GAAGY,AAAkB,IAAlBA,iBAAqB;QAClD,QAAQT,KAAK,GAAG,CAACH,KAAK,MAAM,GAAGa,AAAgB,IAAhBA,eAAmB;IACpD;AACF;AAEO,SAASC,iCAAiCd,IAAU;IACzD,OAAOA,KAAK,KAAK,IAAIR,kCACnBQ,KAAK,MAAM,IAAIR,iCACbC,wCACAC;AACN;AAEO,SAASqB,gCAAgCd,IAAU;IACxD,MAAMe,UAAUb,KAAK,GAAG,CAACF,KAAK,KAAK,EAAEA,KAAK,MAAM;IAChD,IAAI,CAACe,SACH;IAEF,MAAM,EAAEC,MAAM,EAAE,GAAGnB;IACnB,MAAMoB,QAAQf,KAAK,GAAG,CACpBc,OAAO,mBAAmB,EAC1BA,OAAO,eAAe,GAAGD;IAE3B,IAAIE,SAAS,MACX;IAEF,OAAO;QACL,OAAOf,KAAK,KAAK,CAACF,KAAK,KAAK,GAAGiB;QAC/B,QAAQf,KAAK,KAAK,CAACF,KAAK,MAAM,GAAGiB;IACnC;AACF;AAEO,SAASC,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT;AAEA,SAASE,wBAAwBC,KAAa,EAAEC,UAAkB;IAChE,IAAIC,QAAQD;IACZ,MAAOC,QAAQF,MAAM,MAAM,IAAI,KAAK,IAAI,CAACA,KAAK,CAACE,MAAM,EACnDA,SAAS;IAEX,OAAOF,KAAK,CAACE,MAAM;AACrB;AAEA,SAASC,oCAAoCH,KAAa,EAAEI,SAAiB;IAC3E,MAAMC,mBAAmBD,UAAU,OAAO,CAAC,uBAAuB;IAClE,MAAME,aAAa,IAAIC,OAAO,CAAC,CAAC,EAAEF,iBAAiB,WAAW,CAAC,EAAE,IAAI,CAACL;IACtE,IAAI,CAACM,YACH;IAGF,IAAIJ,QAAQI,WAAW,KAAK,GAAGA,UAAU,CAAC,EAAE,CAAC,MAAM;IACnD,IAAIE,UAAU;IACd,IAAIC,oBAAoB;IAExB,MAAOP,QAAQF,MAAM,MAAM,EAAEE,SAAS,EAAG;QACvC,MAAMQ,OAAOV,KAAK,CAACE,MAAM;QAEzB,IAAIM,SAAS;YACXC,qBAAqBC;YACrBF,UAAU;YACV;QACF;QAEA,IAAIE,AAAS,SAATA,MAAe;YACjBD,qBAAqBC;YACrBF,UAAU;YACV;QACF;QAEA,IAAIE,AAAS,QAATA,MAAc;YAChBD,qBAAqBC;YACrB;QACF;QAEA,MAAMC,sBAAsBZ,wBAAwBC,OAAOE,QAAQ;QACnE,IACES,AAAwB,QAAxBA,uBACAA,AAAwB,QAAxBA,uBACAA,AAAwB,QAAxBA,uBACAA,AAAwBC,WAAxBD,qBAEA,IAAI;YACF,OAAOE,KAAK,KAAK,CAAC,CAAC,CAAC,EAAEJ,kBAAkB,CAAC,CAAC;QAC5C,EAAE,OAAM;YACN,OAAOA;QACT;QAGFA,qBAAqB;IACvB;AAGF;AAEO,SAASK,sCACdC,KAAc;IAEd,MAAMC,UAAUD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;IAChE,MAAMI,cACJJ,iBAAiBK,yBAAAA,oBAAoBA,GACjCL,MAAM,WAAW,GACjBC,QAAQ,KAAK,CAAC,iCAAiC,CAAC,EAAE;IAExD,IACE,CAACG,eACA,CAACH,QAAQ,QAAQ,CAAC,6CACjB,CAAED,CAAAA,iBAAiBK,yBAAAA,oBAAmB,AAAnBA,GAErB;IAGF,MAAMC,mBAAmBC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyBH;IAClD,MAAMI,cAAcpB,oCAClBkB,kBACA,gBACC;IAEH,IAAI,CAACE,aACH;IAGF,OAAO;QAAEA;IAAY;AACvB"}