@midscene/core 1.10.5-beta-20260716114658.0 → 1.10.5

package/dist/es/agent/utils.mjs

@@ -5,7 +5,7 @@ import { ScreenshotItem } from "../screenshot-item.mjs";
 import { uploadTestInfoToServer } from "../utils.mjs";
 import { MIDSCENE_REPORT_QUIET, MIDSCENE_REPORT_TAG_NAME, globalConfigManager } from "@midscene/shared/env";
 import { generateElementByRect } from "@midscene/shared/extractor";
-import { createImgBase64ByFormat, imageInfoOfBase64, resizeImgBase64 } from "@midscene/shared/img";
+import { convertImgBufferToJpeg, createImgBase64ByFormat, imageInfoOfBase64, parseBase64, resizeImgBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert, ifInBrowser, logMsg, uuid } from "@midscene/shared/utils";
 import dayjs from "dayjs";
@@ -93,15 +93,23 @@ async function commonContextParser(interfaceInstance, _opt) {
             shrunkShotToLogicalRatio
         };
     }
-    return {
-        shotSize: {
-            width: imgWidth,
-            height: imgHeight
-        },
-        deprecatedDpr: dpr,
-        screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),
-        shrunkShotToLogicalRatio
-    };
+    {
+        let outputScreenshotBase64 = screenshotBase64;
+        const { mimeType, body } = parseBase64(screenshotBase64);
+        if ('image/png' === mimeType.toLowerCase()) {
+            const jpegBuffer = await convertImgBufferToJpeg(Buffer.from(body, 'base64'), 90);
+            outputScreenshotBase64 = createImgBase64ByFormat('jpeg', jpegBuffer.toString('base64'));
+        }
+        return {
+            shotSize: {
+                width: imgWidth,
+                height: imgHeight
+            },
+            deprecatedDpr: dpr,
+            screenshot: ScreenshotItem.create(outputScreenshotBase64, screenshotCapturedAt),
+            shrunkShotToLogicalRatio
+        };
+    }
 }
 async function createScreenshotBoundUIContext(screenshotBase64, opt) {
     const normalizedScreenshotBase64 = normalizeScreenshotBase64(screenshotBase64);
@@ -173,7 +181,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.10.5-beta-20260716114658.0";
+const getMidsceneVersion = ()=>"1.10.5";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/agent/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport { pixelBboxToRect } from '@/ai-model/workflows/inspect/locate-result-rect';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PixelBbox,\n  PlanningLocateParam,\n  PlanningLocateParamWithLocatedPixelBbox,\n  Rect,\n  ScrollParam,\n  Size,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport {\n  createImgBase64ByFormat,\n  imageInfoOfBase64,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst agentDebug = getDebug('agent');\nconst screenshotDataUrlPattern = /^data:image\\/[a-zA-Z0-9.+-]+;base64,/i;\n\nconst inferBase64ImageFormat = (base64Body: string) => {\n  if (base64Body.startsWith('iVBORw0KGgo')) {\n    return 'png';\n  }\n  return 'jpeg';\n};\n\nconst normalizeScreenshotBase64 = (screenshotBase64: string) => {\n  const trimmedBase64 = screenshotBase64.trim();\n  if (screenshotDataUrlPattern.test(trimmedBase64)) {\n    return trimmedBase64;\n  }\n\n  const base64Body = trimmedBase64.replace(/\\s/g, '');\n  assert(base64Body, 'screenshotBase64 must include image data');\n  return createImgBase64ByFormat(\n    inferBase64ImageFormat(base64Body),\n    base64Body,\n  );\n};\n\nconst legacyScrollTypeMap = {\n  once: 'singleAction',\n  untilBottom: 'scrollToBottom',\n  untilTop: 'scrollToTop',\n  untilRight: 'scrollToRight',\n  untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n  scrollType: string | undefined,\n): ScrollParam['scrollType'] | undefined => {\n  if (!scrollType) {\n    return undefined;\n  }\n\n  if (scrollType in legacyScrollTypeMap) {\n    return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n  }\n\n  return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport async function createScreenshotBoundUIContext(\n  screenshotBase64: string,\n  opt: {\n    screenshotSize?: Size;\n  },\n): Promise<UIContext> {\n  const normalizedScreenshotBase64 =\n    normalizeScreenshotBase64(screenshotBase64);\n  const actualScreenshotSize = await imageInfoOfBase64(\n    normalizedScreenshotBase64,\n  );\n  if (\n    opt.screenshotSize &&\n    (opt.screenshotSize.width !== actualScreenshotSize.width ||\n      opt.screenshotSize.height !== actualScreenshotSize.height)\n  ) {\n    agentDebug(\n      'describeElementAtPoint screenshotSize mismatch, use actual size',\n      {\n        provided: opt.screenshotSize,\n        actual: actualScreenshotSize,\n      },\n    );\n  }\n\n  return {\n    screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n    shotSize: actualScreenshotSize,\n    shrunkShotToLogicalRatio: 1,\n    _isFrozen: true,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n  fileExists?: (path: string) => boolean;\n  isInBrowser?: boolean;\n  resolvePath?: (path: string) => string;\n  wslDistroName?: string;\n  cwd?: string;\n};\n\nexport function normalizeFilePaths(\n  files: string[],\n  options: NormalizeFilePathsOptions = {},\n): string[] {\n  const {\n    fileExists = existsSync,\n    isInBrowser = ifInBrowser,\n    resolvePath = resolve,\n    wslDistroName = process.env.WSL_DISTRO_NAME,\n    cwd = process.cwd(),\n  } = options;\n\n  if (isInBrowser) {\n    throw new Error('File chooser is not supported in browser environment');\n  }\n\n  return files.map((file) => {\n    const absolutePath = resolvePath(file);\n    if (!fileExists(absolutePath)) {\n      throw new Error(\n        `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n      );\n    }\n\n    if (!wslDistroName) {\n      return absolutePath;\n    }\n\n    const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n    if (wslMount) {\n      return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n    }\n\n    return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n  });\n}\n\nexport function isPixelBbox(value: unknown): value is PixelBbox {\n  return (\n    Array.isArray(value) &&\n    value.length === 4 &&\n    value.every((item) => typeof item === 'number' && Number.isFinite(item))\n  );\n}\n\ntype PlanningLocateParamWithMaybeLocatedPixelBbox = PlanningLocateParam & {\n  locatedPixelBbox?: unknown;\n};\n\nexport function ifPlanLocateParamHasLocatedPixelBbox(\n  planLocateParam: PlanningLocateParamWithMaybeLocatedPixelBbox,\n): planLocateParam is PlanningLocateParamWithLocatedPixelBbox {\n  return isPixelBbox(planLocateParam.locatedPixelBbox);\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParamWithLocatedPixelBbox,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);\n\n  const element = generateElementByRect(\n    rect,\n    typeof planLocateParam.prompt === 'string'\n      ? planLocateParam.prompt\n      : planLocateParam.prompt?.prompt || '',\n  );\n  return element;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["agentDebug","getDebug","screenshotDataUrlPattern","inferBase64ImageFormat","base64Body","normalizeScreenshotBase64","screenshotBase64","trimmedBase64","assert","createImgBase64ByFormat","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","actualScreenshotSize","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","isPixelBbox","value","Array","item","ifPlanLocateParamHasLocatedPixelBbox","planLocateParam","matchElementFromPlan","rect","pixelBboxToRect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;;;;AAoCA,MAAMA,aAAaC,SAAS;AAC5B,MAAMC,2BAA2B;AAEjC,MAAMC,yBAAyB,CAACC;IAC9B,IAAIA,WAAW,UAAU,CAAC,gBACxB,OAAO;IAET,OAAO;AACT;AAEA,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,gBAAgBD,iBAAiB,IAAI;IAC3C,IAAIJ,yBAAyB,IAAI,CAACK,gBAChC,OAAOA;IAGT,MAAMH,aAAaG,cAAc,OAAO,CAAC,OAAO;IAChDC,OAAOJ,YAAY;IACnB,OAAOK,wBACLN,uBAAuBC,aACvBA;AAEJ;AAEA,MAAMM,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQf,SAAS;IAEvBO,OAAOM,mBAAmB;IAE1BE,MAAM;IACN,MAAMC,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNE,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMG,gBAAgB,MAAML,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOM,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIL,MAAM,CAAC,MAAM,EAAEI,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMf,mBAAmB,MAAMQ,kBAAkB,gBAAgB;IACjE,MAAMW,uBAAuBC,KAAK,GAAG;IACrClB,OAAOF,kBAAmB;IAG1BU,MAAM;IACN,MAAM,EAAE,OAAOW,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,kBAAkBvB;IAE1B,IAAI,CAACiB,OAAO,QAAQ,CAACI,aAAa,CAACJ,OAAO,QAAQ,CAACK,YACjD,MAAM,IAAIJ,MACR,CAAC,wFAAwF,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIJ,MACR,CAAC,0FAA0F,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAGjIZ,MAAM,yBAAyBW,UAAU,KAAKC;IAM9C,MAAME,oBAAoBV,eAAeC;IACzC,MAAMU,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBZ;IAExB,IAAIU,sBAAsBC,sBAAsB;QAC9Cf,MACE,CAAC,4CAA4C,EAAEI,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAES,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBX;IAEtB;IAEA,MAAMY,mBAAmBlB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACQ,OAAO,QAAQ,CAACU,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIT,MACR,CAAC,wEAAwE,EAAES,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBhB,MAAM,mBAAmBkB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCjB,MAAM,4BAA4BmB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CjB,MACE,CAAC,mCAAmC,EAAEiB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,gBAAgBlC,kBAAkB;YAC5D,OAAO8B;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,eAAe,MAAM,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,eAAe,MAAM,CAACnC,kBAAkBmB;QACpDU;IACF;AACF;AAEO,eAAeO,+BACpBpC,gBAAwB,EACxBqC,GAEC;IAED,MAAMC,6BACJvC,0BAA0BC;IAC5B,MAAMuC,uBAAuB,MAAMhB,kBACjCe;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKE,qBAAqB,KAAK,IACtDF,IAAI,cAAc,CAAC,MAAM,KAAKE,qBAAqB,MAAK,GAE1D7C,WACE,mEACA;QACE,UAAU2C,IAAI,cAAc;QAC5B,QAAQE;IACV;IAIJ,OAAO;QACL,YAAYJ,eAAe,MAAM,CAACG,4BAA4BlB,KAAK,GAAG;QACtE,UAAUmB;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASC,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAUO,SAASG,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAIxC,MAAM;IAGlB,OAAOoC,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAIhD,MACR,CAAC,gBAAgB,EAAE+C,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAEO,SAASE,YAAYC,KAAc;IACxC,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,OAAS,AAAgB,YAAhB,OAAOA,QAAqBtD,OAAO,QAAQ,CAACsD;AAEtE;AAMO,SAASC,qCACdC,eAA6D;IAE7D,OAAOL,YAAYK,gBAAgB,gBAAgB;AACrD;AAEO,SAASC,qBACdD,eAAwD;IAExD,IAAI,CAACA,iBACH;IAGF,MAAME,OAAOC,gBAAgBH,gBAAgB,gBAAgB;IAE7D,MAAMI,UAAUC,sBACdH,MACA,AAAkC,YAAlC,OAAOF,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;IAExC,OAAOI;AACT;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMT,OACJ,MAAMK,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACN9C,KAAK,KAAK,CAAC4C,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpC5C,KAAK,KAAK,CAAC4C,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOO,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,8BAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAhD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOgD;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACN9C,KAAK,KAAK,CAAC8C,QAAQ,MAAM,CAAC,EAAE,GAAGhD;YAC/BE,KAAK,KAAK,CAAC8C,QAAQ,MAAM,CAAC,EAAE,GAAGhD;SAChC;QACD,MAAM;YACJ,GAAGgD,QAAQ,IAAI;YACf,MAAM9C,KAAK,KAAK,CAAC8C,QAAQ,IAAI,CAAC,IAAI,GAAGhD;YACrC,KAAKE,KAAK,KAAK,CAAC8C,QAAQ,IAAI,CAAC,GAAG,GAAGhD;YACnC,OAAOE,KAAK,KAAK,CAAC8C,QAAQ,IAAI,CAAC,KAAK,GAAGhD;YACvC,QAAQE,KAAK,KAAK,CAAC8C,QAAQ,IAAI,CAAC,MAAM,GAAGhD;QAC3C;IACF;AACF;AAEO,MAAM+D,uCAAuC,CAClDjB,MACA9C;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO8C;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAM5C,KAAK,KAAK,CAAC4C,KAAK,IAAI,GAAG9C;QAC7B,KAAKE,KAAK,KAAK,CAAC4C,KAAK,GAAG,GAAG9C;QAC3B,OAAOE,KAAK,KAAK,CAAC4C,KAAK,KAAK,GAAG9C;QAC/B,QAAQE,KAAK,KAAK,CAAC4C,KAAK,MAAM,GAAG9C;IACnC;AACF"}
+{"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport { pixelBboxToRect } from '@/ai-model/workflows/inspect/locate-result-rect';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PixelBbox,\n  PlanningLocateParam,\n  PlanningLocateParamWithLocatedPixelBbox,\n  Rect,\n  ScrollParam,\n  Size,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport {\n  convertImgBufferToJpeg,\n  createImgBase64ByFormat,\n  imageInfoOfBase64,\n  parseBase64,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst agentDebug = getDebug('agent');\nconst screenshotDataUrlPattern = /^data:image\\/[a-zA-Z0-9.+-]+;base64,/i;\n\nconst inferBase64ImageFormat = (base64Body: string) => {\n  if (base64Body.startsWith('iVBORw0KGgo')) {\n    return 'png';\n  }\n  return 'jpeg';\n};\n\nconst normalizeScreenshotBase64 = (screenshotBase64: string) => {\n  const trimmedBase64 = screenshotBase64.trim();\n  if (screenshotDataUrlPattern.test(trimmedBase64)) {\n    return trimmedBase64;\n  }\n\n  const base64Body = trimmedBase64.replace(/\\s/g, '');\n  assert(base64Body, 'screenshotBase64 must include image data');\n  return createImgBase64ByFormat(\n    inferBase64ImageFormat(base64Body),\n    base64Body,\n  );\n};\n\nconst legacyScrollTypeMap = {\n  once: 'singleAction',\n  untilBottom: 'scrollToBottom',\n  untilTop: 'scrollToTop',\n  untilRight: 'scrollToRight',\n  untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n  scrollType: string | undefined,\n): ScrollParam['scrollType'] | undefined => {\n  if (!scrollType) {\n    return undefined;\n  }\n\n  if (scrollType in legacyScrollTypeMap) {\n    return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n  }\n\n  return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  } else {\n    // For screenshots that do not need shrinking, convert PNG to JPEG to reduce the image payload in model requests and reports. (Shrunk images are already JPEG.)\n    // This mainly covers Android's default screenshot path, which produces PNG screenshots.\n    // Compared with conversion on Android, centralizing it here means each platform does not need to handle screenshot formats itself, and allows future output formats such as WebP.\n    // Built-in paths that already output JPEG are unaffected, and custom devices that output JPEG will not be compressed again.\n    // The Web platform already outputs JPEG, so it does not enter this branch. Other built-in device platforms run in Node, where Sharp conversion is fast enough that its extra cost is negligible.\n    let outputScreenshotBase64 = screenshotBase64;\n    const { mimeType, body } = parseBase64(screenshotBase64);\n    if (mimeType.toLowerCase() === 'image/png') {\n      const jpegBuffer = await convertImgBufferToJpeg(\n        Buffer.from(body, 'base64'),\n        90,\n      );\n      outputScreenshotBase64 = createImgBase64ByFormat(\n        'jpeg',\n        jpegBuffer.toString('base64'),\n      );\n    }\n\n    return {\n      shotSize: {\n        width: imgWidth,\n        height: imgHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(\n        outputScreenshotBase64,\n        screenshotCapturedAt,\n      ),\n      shrunkShotToLogicalRatio,\n    };\n  }\n}\n\nexport async function createScreenshotBoundUIContext(\n  screenshotBase64: string,\n  opt: {\n    screenshotSize?: Size;\n  },\n): Promise<UIContext> {\n  const normalizedScreenshotBase64 =\n    normalizeScreenshotBase64(screenshotBase64);\n  const actualScreenshotSize = await imageInfoOfBase64(\n    normalizedScreenshotBase64,\n  );\n  if (\n    opt.screenshotSize &&\n    (opt.screenshotSize.width !== actualScreenshotSize.width ||\n      opt.screenshotSize.height !== actualScreenshotSize.height)\n  ) {\n    agentDebug(\n      'describeElementAtPoint screenshotSize mismatch, use actual size',\n      {\n        provided: opt.screenshotSize,\n        actual: actualScreenshotSize,\n      },\n    );\n  }\n\n  return {\n    screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n    shotSize: actualScreenshotSize,\n    shrunkShotToLogicalRatio: 1,\n    _isFrozen: true,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n  fileExists?: (path: string) => boolean;\n  isInBrowser?: boolean;\n  resolvePath?: (path: string) => string;\n  wslDistroName?: string;\n  cwd?: string;\n};\n\nexport function normalizeFilePaths(\n  files: string[],\n  options: NormalizeFilePathsOptions = {},\n): string[] {\n  const {\n    fileExists = existsSync,\n    isInBrowser = ifInBrowser,\n    resolvePath = resolve,\n    wslDistroName = process.env.WSL_DISTRO_NAME,\n    cwd = process.cwd(),\n  } = options;\n\n  if (isInBrowser) {\n    throw new Error('File chooser is not supported in browser environment');\n  }\n\n  return files.map((file) => {\n    const absolutePath = resolvePath(file);\n    if (!fileExists(absolutePath)) {\n      throw new Error(\n        `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n      );\n    }\n\n    if (!wslDistroName) {\n      return absolutePath;\n    }\n\n    const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n    if (wslMount) {\n      return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n    }\n\n    return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n  });\n}\n\nexport function isPixelBbox(value: unknown): value is PixelBbox {\n  return (\n    Array.isArray(value) &&\n    value.length === 4 &&\n    value.every((item) => typeof item === 'number' && Number.isFinite(item))\n  );\n}\n\ntype PlanningLocateParamWithMaybeLocatedPixelBbox = PlanningLocateParam & {\n  locatedPixelBbox?: unknown;\n};\n\nexport function ifPlanLocateParamHasLocatedPixelBbox(\n  planLocateParam: PlanningLocateParamWithMaybeLocatedPixelBbox,\n): planLocateParam is PlanningLocateParamWithLocatedPixelBbox {\n  return isPixelBbox(planLocateParam.locatedPixelBbox);\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParamWithLocatedPixelBbox,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);\n\n  const element = generateElementByRect(\n    rect,\n    typeof planLocateParam.prompt === 'string'\n      ? planLocateParam.prompt\n      : planLocateParam.prompt?.prompt || '',\n  );\n  return element;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["agentDebug","getDebug","screenshotDataUrlPattern","inferBase64ImageFormat","base64Body","normalizeScreenshotBase64","screenshotBase64","trimmedBase64","assert","createImgBase64ByFormat","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","outputScreenshotBase64","mimeType","body","parseBase64","jpegBuffer","convertImgBufferToJpeg","Buffer","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","actualScreenshotSize","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","isPixelBbox","value","Array","item","ifPlanLocateParamHasLocatedPixelBbox","planLocateParam","matchElementFromPlan","rect","pixelBboxToRect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;;;;AAsCA,MAAMA,aAAaC,SAAS;AAC5B,MAAMC,2BAA2B;AAEjC,MAAMC,yBAAyB,CAACC;IAC9B,IAAIA,WAAW,UAAU,CAAC,gBACxB,OAAO;IAET,OAAO;AACT;AAEA,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,gBAAgBD,iBAAiB,IAAI;IAC3C,IAAIJ,yBAAyB,IAAI,CAACK,gBAChC,OAAOA;IAGT,MAAMH,aAAaG,cAAc,OAAO,CAAC,OAAO;IAChDC,OAAOJ,YAAY;IACnB,OAAOK,wBACLN,uBAAuBC,aACvBA;AAEJ;AAEA,MAAMM,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQf,SAAS;IAEvBO,OAAOM,mBAAmB;IAE1BE,MAAM;IACN,MAAMC,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNE,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMG,gBAAgB,MAAML,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOM,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIL,MAAM,CAAC,MAAM,EAAEI,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMf,mBAAmB,MAAMQ,kBAAkB,gBAAgB;IACjE,MAAMW,uBAAuBC,KAAK,GAAG;IACrClB,OAAOF,kBAAmB;IAG1BU,MAAM;IACN,MAAM,EAAE,OAAOW,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,kBAAkBvB;IAE1B,IAAI,CAACiB,OAAO,QAAQ,CAACI,aAAa,CAACJ,OAAO,QAAQ,CAACK,YACjD,MAAM,IAAIJ,MACR,CAAC,wFAAwF,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIJ,MACR,CAAC,0FAA0F,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAGjIZ,MAAM,yBAAyBW,UAAU,KAAKC;IAM9C,MAAME,oBAAoBV,eAAeC;IACzC,MAAMU,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBZ;IAExB,IAAIU,sBAAsBC,sBAAsB;QAC9Cf,MACE,CAAC,4CAA4C,EAAEI,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAES,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBX;IAEtB;IAEA,MAAMY,mBAAmBlB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACQ,OAAO,QAAQ,CAACU,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIT,MACR,CAAC,wEAAwE,EAAES,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBhB,MAAM,mBAAmBkB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCjB,MAAM,4BAA4BmB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CjB,MACE,CAAC,mCAAmC,EAAEiB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,gBAAgBlC,kBAAkB;YAC5D,OAAO8B;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,eAAe,MAAM,CAACF,eAAed;YACjDU;QACF;IACF;IAAO;QAML,IAAIO,yBAAyBpC;QAC7B,MAAM,EAAEqC,QAAQ,EAAEC,IAAI,EAAE,GAAGC,YAAYvC;QACvC,IAAIqC,AAA2B,gBAA3BA,SAAS,WAAW,IAAoB;YAC1C,MAAMG,aAAa,MAAMC,uBACvBC,OAAO,IAAI,CAACJ,MAAM,WAClB;YAEFF,yBAAyBjC,wBACvB,QACAqC,WAAW,QAAQ,CAAC;QAExB;QAEA,OAAO;YACL,UAAU;gBACR,OAAOnB;gBACP,QAAQC;YACV;YACA,eAAeM;YACf,YAAYO,eAAe,MAAM,CAC/BC,wBACAjB;YAEFU;QACF;IACF;AACF;AAEO,eAAec,+BACpB3C,gBAAwB,EACxB4C,GAEC;IAED,MAAMC,6BACJ9C,0BAA0BC;IAC5B,MAAM8C,uBAAuB,MAAMvB,kBACjCsB;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKE,qBAAqB,KAAK,IACtDF,IAAI,cAAc,CAAC,MAAM,KAAKE,qBAAqB,MAAK,GAE1DpD,WACE,mEACA;QACE,UAAUkD,IAAI,cAAc;QAC5B,QAAQE;IACV;IAIJ,OAAO;QACL,YAAYX,eAAe,MAAM,CAACU,4BAA4BzB,KAAK,GAAG;QACtE,UAAU0B;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASC,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAUO,SAASG,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAI/C,MAAM;IAGlB,OAAO2C,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAIvD,MACR,CAAC,gBAAgB,EAAEsD,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAEO,SAASE,YAAYC,KAAc;IACxC,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,OAAS,AAAgB,YAAhB,OAAOA,QAAqB7D,OAAO,QAAQ,CAAC6D;AAEtE;AAMO,SAASC,qCACdC,eAA6D;IAE7D,OAAOL,YAAYK,gBAAgB,gBAAgB;AACrD;AAEO,SAASC,qBACdD,eAAwD;IAExD,IAAI,CAACA,iBACH;IAGF,MAAME,OAAOC,gBAAgBH,gBAAgB,gBAAgB;IAE7D,MAAMI,UAAUC,sBACdH,MACA,AAAkC,YAAlC,OAAOF,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;IAExC,OAAOI;AACT;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMT,OACJ,MAAMK,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNrD,KAAK,KAAK,CAACmD,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCnD,KAAK,KAAK,CAACmD,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOO,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,8BAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAvD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOuD;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNrD,KAAK,KAAK,CAACqD,QAAQ,MAAM,CAAC,EAAE,GAAGvD;YAC/BE,KAAK,KAAK,CAACqD,QAAQ,MAAM,CAAC,EAAE,GAAGvD;SAChC;QACD,MAAM;YACJ,GAAGuD,QAAQ,IAAI;YACf,MAAMrD,KAAK,KAAK,CAACqD,QAAQ,IAAI,CAAC,IAAI,GAAGvD;YACrC,KAAKE,KAAK,KAAK,CAACqD,QAAQ,IAAI,CAAC,GAAG,GAAGvD;YACnC,OAAOE,KAAK,KAAK,CAACqD,QAAQ,IAAI,CAAC,KAAK,GAAGvD;YACvC,QAAQE,KAAK,KAAK,CAACqD,QAAQ,IAAI,CAAC,MAAM,GAAGvD;QAC3C;IACF;AACF;AAEO,MAAMsE,uCAAuC,CAClDjB,MACArD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOqD;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMnD,KAAK,KAAK,CAACmD,KAAK,IAAI,GAAGrD;QAC7B,KAAKE,KAAK,KAAK,CAACmD,KAAK,GAAG,GAAGrD;QAC3B,OAAOE,KAAK,KAAK,CAACmD,KAAK,KAAK,GAAGrD;QAC/B,QAAQE,KAAK,KAAK,CAACmD,KAAK,MAAM,GAAGrD;IACnC;AACF"}

package/dist/es/ai-model/models/doubao.mjs

@@ -10,48 +10,22 @@ const doubaoPointCoordinatesMeta = {
     order: 'xy',
     normalizedBy: 1000
 };
-function parseNumbersFromBboxString(input) {
-    return (input.match(/\d+/g) ?? []).map(Number).filter(Number.isFinite);
+const coordinateSequencePattern = /(?:^|[^a-zA-Z0-9])(\d+(?:[^a-zA-Z0-9]+\d+)+)(?=$|[^a-zA-Z0-9])/g;
+function isFourFiniteNumberArray(input) {
+    return Array.isArray(input) && 4 === input.length && input.every((value)=>'number' == typeof value && Number.isFinite(value));
 }
-function parseLeadingNumberFromString(input) {
-    const match = input.match(/^\s*(\d+)/);
-    return match ? Number(match[1]) : void 0;
-}
-function isCleanCoordinateString(input) {
-    return /^\s*\d+(?:[\s,;]+\d+)*\s*[,;]?\s*$/.test(input);
-}
-function parseNumbersFromBboxArray(input) {
-    const numbers = [];
-    for (const item of input){
-        if ('number' == typeof item) {
-            numbers.push(item);
-            continue;
-        }
-        if ('string' == typeof item) if (isCleanCoordinateString(item)) {
-            numbers.push(...parseNumbersFromBboxString(item));
-            continue;
-        } else {
-            const leadingNumber = parseLeadingNumberFromString(item);
-            if (void 0 !== leadingNumber) numbers.push(leadingNumber);
-        }
-        break;
-    }
-    return numbers.filter(Number.isFinite);
-}
-function parseNumbersFromTaggedBboxArray(input) {
-    if (input.length < 2) return null;
-    const taggedBbox = input[1];
-    if ('string' == typeof taggedBbox) return parseNumbersFromBboxString(taggedBbox);
-    if (Array.isArray(taggedBbox)) return parseNumbersFromBboxArray(taggedBbox);
-    return null;
+function parseNumbersFromUnexpectedBboxStructure(input) {
+    const serialized = JSON.stringify(input);
+    if (!serialized) return [];
+    const sequences = Array.from(serialized.matchAll(coordinateSequencePattern), (match)=>match[1].match(/\d+/g)?.map(Number) ?? []);
+    const longestLength = Math.max(0, ...sequences.map((sequence)=>sequence.length));
+    const longestSequences = sequences.filter((sequence)=>sequence.length === longestLength);
+    if (1 !== longestSequences.length) return [];
+    return longestSequences[0];
 }
 function parseDoubaoRawLocateValue(input) {
     const bbox = unwrapCoordinateListLikeInput(input);
-    let bboxList = [];
-    if ('string' == typeof bbox) {
-        bboxList = parseNumbersFromBboxString(bbox);
-        if (4 !== bboxList.length) throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);
-    } else bboxList = Array.isArray(bbox) ? 'string' == typeof bbox[0] && 'bbox' === bbox[0].trim() ? parseNumbersFromTaggedBboxArray(bbox) ?? [] : parseNumbersFromBboxArray(bbox) : bbox;
+    const bboxList = isFourFiniteNumberArray(bbox) ? bbox : parseNumbersFromUnexpectedBboxStructure(bbox);
     if (4 === bboxList.length || 5 === bboxList.length) return createLocateResultValue(doubaoBboxCoordinatesMeta, [
         bboxList[0],
         bboxList[1],

package/dist/es/ai-model/models/doubao.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/doubao.mjs","sources":["../../../../src/ai-model/models/doubao.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { parseModelResponseJson } from '../service-caller/json';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst doubaoBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst doubaoPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\nfunction parseNumbersFromBboxString(input: string): number[] {\n  return (input.match(/\\d+/g) ?? []).map(Number).filter(Number.isFinite);\n}\n\nfunction parseLeadingNumberFromString(input: string): number | undefined {\n  const match = input.match(/^\\s*(\\d+)/);\n  return match ? Number(match[1]) : undefined;\n}\n\n/**\n * Clean coordinate strings can contain multiple positive integers, e.g.\n * - \"123 100\"\n * - \"123,100\"\n * - \"277; 664 291;\"\n * Dirty strings like \"345<\" are handled by taking the leading number and\n * dropping the remaining array items.\n */\nfunction isCleanCoordinateString(input: string): boolean {\n  return /^\\s*\\d+(?:[\\s,;]+\\d+)*\\s*[,;]?\\s*$/.test(input);\n}\n\nfunction parseNumbersFromBboxArray(input: unknown[]): number[] {\n  const numbers: number[] = [];\n\n  for (const item of input) {\n    if (typeof item === 'number') {\n      numbers.push(item);\n      continue;\n    }\n\n    if (typeof item === 'string') {\n      if (isCleanCoordinateString(item)) {\n        numbers.push(...parseNumbersFromBboxString(item));\n        continue;\n      } else {\n        const leadingNumber = parseLeadingNumberFromString(item);\n        if (leadingNumber !== undefined) {\n          numbers.push(leadingNumber);\n        }\n        // Once a dirty string appears, the remaining array items usually belong\n        // to broken JSON repair output, e.g. [410, 295, 885, \"345<\", \"/bbox>...\"].\n        break;\n      }\n    }\n\n    break;\n  }\n\n  return numbers.filter(Number.isFinite);\n}\n\n/**\n * Some models return a tagged bbox tuple, e.g.\n * - [\"bbox\", [782, 541, 815, 559]]\n * - [\"bbox\", \"782, 541, 815, 559\"]\n */\nfunction parseNumbersFromTaggedBboxArray(input: unknown[]): number[] | null {\n  if (input.length < 2) {\n    return null;\n  }\n\n  const taggedBbox = input[1];\n  if (typeof taggedBbox === 'string') {\n    return parseNumbersFromBboxString(taggedBbox);\n  }\n\n  if (Array.isArray(taggedBbox)) {\n    return parseNumbersFromBboxArray(taggedBbox);\n  }\n\n  return null;\n}\n\nexport function parseDoubaoRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  let bboxList: number[] = [];\n\n  if (typeof bbox === 'string') {\n    /**\n     * Some models return bbox as a string, e.g.\n     * - { \"bbox\": \"[336, 163, 717, 200]\" }.\n     * - { \"bbox\": \"336, 163, 717, 200\" }.\n     * - { \"bbox\": \"336 163 717 200\" }.\n     */\n    bboxList = parseNumbersFromBboxString(bbox);\n    if (bboxList.length !== 4) {\n      throw new Error(\n        `invalid bbox data string for doubao-vision mode: ${bbox}`,\n      );\n    }\n  } else if (Array.isArray(bbox)) {\n    if (typeof bbox[0] === 'string' && bbox[0].trim() === 'bbox') {\n      bboxList = parseNumbersFromTaggedBboxArray(bbox) ?? [];\n    } else {\n      bboxList = parseNumbersFromBboxArray(bbox);\n    }\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(doubaoPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bboxList.length === 8) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nconst buildDoubaoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n    if (reasoningEffort) {\n      modelSpecificConfig.reasoning_effort = reasoningEffort;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst doubaoVisionAdapter: ModelAdapterDefinition = {\n  jsonParser: parseModelResponseJson,\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningBudget'],\n    buildChatCompletionParams: buildDoubaoChatCompletionParams,\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    resultAdapter: {\n      coordinates: doubaoBboxCoordinatesMeta,\n      parseRawLocateValue: parseDoubaoRawLocateValue,\n    },\n  },\n};\n\nexport const doubaoAdapters = {\n  'doubao-vision': doubaoVisionAdapter,\n  'doubao-seed': doubaoVisionAdapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'doubao-vision' | 'doubao-seed'\n>;\n"],"names":["doubaoBboxCoordinatesMeta","doubaoPointCoordinatesMeta","parseNumbersFromBboxString","input","Number","parseLeadingNumberFromString","match","undefined","isCleanCoordinateString","parseNumbersFromBboxArray","numbers","item","leadingNumber","parseNumbersFromTaggedBboxArray","taggedBbox","Array","parseDoubaoRawLocateValue","bbox","unwrapCoordinateListLikeInput","bboxList","Error","createLocateResultValue","msg","JSON","buildDoubaoChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","modelSpecificConfig","doubaoVisionAdapter","parseModelResponseJson","doubaoAdapters"],"mappings":";;AAaA,MAAMA,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAEA,SAASC,2BAA2BC,KAAa;IAC/C,OAAQA,AAAAA,CAAAA,MAAM,KAAK,CAAC,WAAW,EAAC,EAAG,GAAG,CAACC,QAAQ,MAAM,CAACA,OAAO,QAAQ;AACvE;AAEA,SAASC,6BAA6BF,KAAa;IACjD,MAAMG,QAAQH,MAAM,KAAK,CAAC;IAC1B,OAAOG,QAAQF,OAAOE,KAAK,CAAC,EAAE,IAAIC;AACpC;AAUA,SAASC,wBAAwBL,KAAa;IAC5C,OAAO,qCAAqC,IAAI,CAACA;AACnD;AAEA,SAASM,0BAA0BN,KAAgB;IACjD,MAAMO,UAAoB,EAAE;IAE5B,KAAK,MAAMC,QAAQR,MAAO;QACxB,IAAI,AAAgB,YAAhB,OAAOQ,MAAmB;YAC5BD,QAAQ,IAAI,CAACC;YACb;QACF;QAEA,IAAI,AAAgB,YAAhB,OAAOA,MACT,IAAIH,wBAAwBG,OAAO;YACjCD,QAAQ,IAAI,IAAIR,2BAA2BS;YAC3C;QACF,OAAO;YACL,MAAMC,gBAAgBP,6BAA6BM;YACnD,IAAIC,AAAkBL,WAAlBK,eACFF,QAAQ,IAAI,CAACE;QAKjB;QAGF;IACF;IAEA,OAAOF,QAAQ,MAAM,CAACN,OAAO,QAAQ;AACvC;AAOA,SAASS,gCAAgCV,KAAgB;IACvD,IAAIA,MAAM,MAAM,GAAG,GACjB,OAAO;IAGT,MAAMW,aAAaX,KAAK,CAAC,EAAE;IAC3B,IAAI,AAAsB,YAAtB,OAAOW,YACT,OAAOZ,2BAA2BY;IAGpC,IAAIC,MAAM,OAAO,CAACD,aAChB,OAAOL,0BAA0BK;IAGnC,OAAO;AACT;AAEO,SAASE,0BAA0Bb,KAAc;IACtD,MAAMc,OAAOC,8BAA8Bf;IAC3C,IAAIgB,WAAqB,EAAE;IAE3B,IAAI,AAAgB,YAAhB,OAAOF,MAAmB;QAO5BE,WAAWjB,2BAA2Be;QACtC,IAAIE,AAAoB,MAApBA,SAAS,MAAM,EACjB,MAAM,IAAIC,MACR,CAAC,iDAAiD,EAAEH,MAAM;IAGhE,OAEIE,WAFOJ,MAAM,OAAO,CAACE,QACnB,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,IAAiBA,AAAmB,WAAnBA,IAAI,CAAC,EAAE,CAAC,IAAI,KAClCJ,gCAAgCI,SAAS,EAAE,GAE3CR,0BAA0BQ,QAG5BA;IAGb,IAAIE,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOE,wBAAwBrB,2BAA2B;QACxDmB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOE,wBAAwBpB,4BAA4B;QACzDkB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOE,wBAAwBrB,2BAA2B;QACxDmB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAMG,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACN,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAME;AAClB;AAEA,MAAME,kCAAkC,CACtCrB;IAEA,MAAM,EAAEsB,gBAAgB,EAAEC,UAAU,EAAE,GAAGvB;IACzC,MAAM,EAAEwB,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BnB,WAA3BmB,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,MAAMI,sBAA+C,CAAC;IAEtD,IAAIH,AAAqB,cAArBA,kBAAgC;QAClCG,oBAAoB,QAAQ,GAAG;YAC7B,MAAOH,oBAAoB,QAAS,YAAY;QAClD;QACA,IAAIC,iBACFE,oBAAoB,gBAAgB,GAAGF;IAE3C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGC,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,sBAA8C;IAClD,YAAYC;IACZ,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BR;QAC3B,+BAA+B;IACjC;IACA,QAAQ;QACN,eAAe;YACb,aAAaxB;YACb,qBAAqBgB;QACvB;IACF;AACF;AAEO,MAAMiB,iBAAiB;IAC5B,iBAAiBF;IACjB,eAAeA;AACjB"}
+{"version":3,"file":"ai-model/models/doubao.mjs","sources":["../../../../src/ai-model/models/doubao.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { parseModelResponseJson } from '../service-caller/json';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst doubaoBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst doubaoPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\n/**\n * Finds a sequence of numbers separated by punctuation or whitespace in a\n * string. Separators cannot be letters, which prevents extracting the `2` in\n * a mixed alphanumeric token such as `bbox_2d` as a coordinate.\n */\nconst coordinateSequencePattern =\n  /(?:^|[^a-zA-Z0-9])(\\d+(?:[^a-zA-Z0-9]+\\d+)+)(?=$|[^a-zA-Z0-9])/g;\n\nfunction isFourFiniteNumberArray(input: unknown): input is number[] {\n  return (\n    Array.isArray(input) &&\n    input.length === 4 &&\n    input.every((value) => typeof value === 'number' && Number.isFinite(value))\n  );\n}\n\nfunction parseNumbersFromUnexpectedBboxStructure(input: unknown): number[] {\n  const serialized = JSON.stringify(input);\n  if (!serialized) {\n    return [];\n  }\n\n  const sequences = Array.from(\n    serialized.matchAll(coordinateSequencePattern),\n    (match) => match[1].match(/\\d+/g)?.map(Number) ?? [],\n  );\n  const longestLength = Math.max(\n    0,\n    ...sequences.map((sequence) => sequence.length),\n  );\n  const longestSequences = sequences.filter(\n    (sequence) => sequence.length === longestLength,\n  );\n\n  if (longestSequences.length !== 1) {\n    return [];\n  }\n\n  return longestSequences[0];\n}\n\nexport function parseDoubaoRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  const bboxList = isFourFiniteNumberArray(bbox)\n    ? bbox\n    : parseNumbersFromUnexpectedBboxStructure(bbox);\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(doubaoPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bboxList.length === 8) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nconst buildDoubaoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n    if (reasoningEffort) {\n      modelSpecificConfig.reasoning_effort = reasoningEffort;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst doubaoVisionAdapter: ModelAdapterDefinition = {\n  jsonParser: parseModelResponseJson,\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningBudget'],\n    buildChatCompletionParams: buildDoubaoChatCompletionParams,\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    resultAdapter: {\n      coordinates: doubaoBboxCoordinatesMeta,\n      parseRawLocateValue: parseDoubaoRawLocateValue,\n    },\n  },\n};\n\nexport const doubaoAdapters = {\n  'doubao-vision': doubaoVisionAdapter,\n  'doubao-seed': doubaoVisionAdapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'doubao-vision' | 'doubao-seed'\n>;\n"],"names":["doubaoBboxCoordinatesMeta","doubaoPointCoordinatesMeta","coordinateSequencePattern","isFourFiniteNumberArray","input","Array","value","Number","parseNumbersFromUnexpectedBboxStructure","serialized","JSON","sequences","match","longestLength","Math","sequence","longestSequences","parseDoubaoRawLocateValue","bbox","unwrapCoordinateListLikeInput","bboxList","createLocateResultValue","msg","Error","buildDoubaoChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","undefined","modelSpecificConfig","doubaoVisionAdapter","parseModelResponseJson","doubaoAdapters"],"mappings":";;AAaA,MAAMA,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAOA,MAAMC,4BACJ;AAEF,SAASC,wBAAwBC,KAAc;IAC7C,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AAExE;AAEA,SAASE,wCAAwCJ,KAAc;IAC7D,MAAMK,aAAaC,KAAK,SAAS,CAACN;IAClC,IAAI,CAACK,YACH,OAAO,EAAE;IAGX,MAAME,YAAYN,MAAM,IAAI,CAC1BI,WAAW,QAAQ,CAACP,4BACpB,CAACU,QAAUA,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,IAAIL,WAAW,EAAE;IAEtD,MAAMM,gBAAgBC,KAAK,GAAG,CAC5B,MACGH,UAAU,GAAG,CAAC,CAACI,WAAaA,SAAS,MAAM;IAEhD,MAAMC,mBAAmBL,UAAU,MAAM,CACvC,CAACI,WAAaA,SAAS,MAAM,KAAKF;IAGpC,IAAIG,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,OAAO,EAAE;IAGX,OAAOA,gBAAgB,CAAC,EAAE;AAC5B;AAEO,SAASC,0BAA0Bb,KAAc;IACtD,MAAMc,OAAOC,8BAA8Bf;IAC3C,MAAMgB,WAAWjB,wBAAwBe,QACrCA,OACAV,wCAAwCU;IAE5C,IAAIE,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOC,wBAAwBrB,2BAA2B;QACxDoB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOC,wBAAwBpB,4BAA4B;QACzDmB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOC,wBAAwBrB,2BAA2B;QACxDoB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAME,MAAM,CAAC,0CAA0C,EAAEZ,KAAK,SAAS,CAACQ,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIK,MAAMD;AAClB;AAEA,MAAME,kCAAkC,CACtCpB;IAEA,MAAM,EAAEqB,gBAAgB,EAAEC,UAAU,EAAE,GAAGtB;IACzC,MAAM,EAAEuB,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,MAAMK,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,QAAQ,GAAG;YAC7B,MAAOJ,oBAAoB,QAAS,YAAY;QAClD;QACA,IAAIC,iBACFG,oBAAoB,gBAAgB,GAAGH;IAE3C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,sBAA8C;IAClD,YAAYC;IACZ,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BT;QAC3B,+BAA+B;IACjC;IACA,QAAQ;QACN,eAAe;YACb,aAAaxB;YACb,qBAAqBiB;QACvB;IACF;AACF;AAEO,MAAMiB,iBAAiB;IAC5B,iBAAiBF;IACjB,eAAeA;AACjB"}

package/dist/es/ai-model/service-caller/json.mjs

@@ -26,52 +26,8 @@ function trimParsedJsonStrings(obj, context = {}) {
     if ('string' == typeof obj) return obj.trim();
     return obj;
 }
-function findNextNonWhitespaceChar(text, startIndex) {
-    for(let index = startIndex; index < text.length; index++){
-        const char = text[index];
-        if (!/\s/.test(char)) return char;
-    }
-}
-function escapeBareQuotesInsideJsonStrings(jsonBlock) {
-    let repaired = '';
-    let inString = false;
-    let escaped = false;
-    let changed = false;
-    for(let index = 0; index < jsonBlock.length; index++){
-        const char = jsonBlock[index];
-        if (!inString) {
-            if ('"' === char) inString = true;
-            repaired += char;
-            continue;
-        }
-        if (escaped) {
-            repaired += char;
-            escaped = false;
-            continue;
-        }
-        if ('\\' === char) {
-            repaired += char;
-            escaped = true;
-            continue;
-        }
-        if ('"' === char) {
-            const nextNonWhitespaceChar = findNextNonWhitespaceChar(jsonBlock, index + 1);
-            const isStringTerminator = void 0 === nextNonWhitespaceChar || ':' === nextNonWhitespaceChar || ',' === nextNonWhitespaceChar || '}' === nextNonWhitespaceChar || ']' === nextNonWhitespaceChar;
-            if (isStringTerminator) {
-                inString = false;
-                repaired += char;
-            } else {
-                repaired += '\\"';
-                changed = true;
-            }
-            continue;
-        }
-        repaired += char;
-    }
-    return changed ? repaired : jsonBlock;
-}
-function repairKnownJsonIssues(jsonBlock, _rawResponse, _context) {
-    return escapeBareQuotesInsideJsonStrings(jsonBlock);
+function repairKnownJsonIssues(jsonBlock, _rawResponse) {
+    return jsonBlock;
 }
 function assertJsonObject(parsed) {
     if (!parsed || 'object' != typeof parsed || Array.isArray(parsed)) throw new Error(`expected parsed LLM response to be a JSON object, got ${JSON.stringify(parsed)}`);
@@ -91,7 +47,7 @@ function parseModelResponseJson(raw, context) {
         parsedObj = parseJsonWithRepair(cleanJsonString);
         if (requireObject) assertJsonObject(parsedObj);
     } catch (e1) {
-        const code = repairKnownJsonIssues(cleanJsonString, raw, context);
+        const code = repairKnownJsonIssues(cleanJsonString, raw);
         if (code === cleanJsonString) throw new Error(`failed to parse LLM response into JSON. Error - ${String(e1)}. Response - \n ${raw}`);
         try {
             parsedObj = parseJsonWithRepair(code);

package/dist/es/ai-model/service-caller/json.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/json.mjs","sources":["../../../../src/ai-model/service-caller/json.ts"],"sourcesContent":["import { jsonrepair } from 'jsonrepair';\n\n/**\n * Extract the JSON portion from a model response.\n *\n * This mainly handles fenced JSON like ```json { ... } ``` by removing the\n * fence. If that does not produce a JSON object, it falls back to a greedy regex\n * that extracts from the first \"{\" to the last \"}\". If that still fails, the\n * original response is returned.\n *\n * Expected model responses are JSON objects, possibly wrapped in markdown\n * fences. Natural language mixed with JSON, or arrays like\n * [{\"type\":\"Tap\"}, {\"type\":\"Hover\"}], are outside the supported contract and\n * are not reliably recoverable.\n *\n * This legacy extractor is also used by extraction responses that can be any\n * JSON value. In those cases, arrays/strings/numbers have no object braces and\n * pass through unchanged.\n */\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport type JsonParserSource =\n  | 'generic-object'\n  | 'planning-action-param'\n  | 'locate'\n  | 'section-locator';\n\nexport interface JsonParserContext {\n  source: JsonParserSource;\n  preserveStringValueKeys?: string[];\n  requireObject?: boolean;\n}\n\nexport type JsonParser = (raw: string, context?: JsonParserContext) => unknown;\n\n/**\n * Trim whitespace in parsed JSON by normalizing:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction trimParsedJsonStrings(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => trimParsedJsonStrings(item, context));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : trimParsedJsonStrings(value, context);\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nfunction findNextNonWhitespaceChar(text: string, startIndex: number) {\n  for (let index = startIndex; index < text.length; index++) {\n    const char = text[index];\n    if (!/\\s/.test(char)) {\n      return char;\n    }\n  }\n  return undefined;\n}\n\nfunction escapeBareQuotesInsideJsonStrings(jsonBlock: string) {\n  let repaired = '';\n  let inString = false;\n  let escaped = false;\n  let changed = false;\n\n  for (let index = 0; index < jsonBlock.length; index++) {\n    const char = jsonBlock[index];\n\n    if (!inString) {\n      if (char === '\"') {\n        inString = true;\n      }\n      repaired += char;\n      continue;\n    }\n\n    if (escaped) {\n      repaired += char;\n      escaped = false;\n      continue;\n    }\n\n    if (char === '\\\\') {\n      repaired += char;\n      escaped = true;\n      continue;\n    }\n\n    if (char === '\"') {\n      const nextNonWhitespaceChar = findNextNonWhitespaceChar(\n        jsonBlock,\n        index + 1,\n      );\n      const isStringTerminator =\n        nextNonWhitespaceChar === undefined ||\n        nextNonWhitespaceChar === ':' ||\n        nextNonWhitespaceChar === ',' ||\n        nextNonWhitespaceChar === '}' ||\n        nextNonWhitespaceChar === ']';\n\n      if (isStringTerminator) {\n        inString = false;\n        repaired += char;\n      } else {\n        repaired += '\\\\\"';\n        changed = true;\n      }\n      continue;\n    }\n\n    repaired += char;\n  }\n\n  return changed ? repaired : jsonBlock;\n}\n\nfunction repairKnownJsonIssues(\n  jsonBlock: string,\n  _rawResponse: string,\n  _context?: JsonParserContext,\n): string {\n  return escapeBareQuotesInsideJsonStrings(jsonBlock);\n}\n\nfunction assertJsonObject(\n  parsed: unknown,\n): asserts parsed is Record<string, unknown> {\n  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {\n    throw new Error(\n      `expected parsed LLM response to be a JSON object, got ${JSON.stringify(\n        parsed,\n      )}`,\n    );\n  }\n}\n\nfunction parseJsonWithRepair(jsonStr: string) {\n  try {\n    return JSON.parse(jsonStr);\n  } catch {\n    return JSON.parse(jsonrepair(jsonStr));\n  }\n}\n\nexport function parseModelResponseJson(\n  raw: string,\n  context?: JsonParserContext,\n) {\n  const cleanJsonString = extractJSONFromCodeBlock(raw);\n  const requireObject = context?.requireObject ?? true;\n\n  let parsedObj: unknown;\n\n  try {\n    parsedObj = parseJsonWithRepair(cleanJsonString);\n    if (requireObject) {\n      assertJsonObject(parsedObj);\n    }\n  } catch (e1) {\n    const code = repairKnownJsonIssues(cleanJsonString, raw, context);\n    if (code === cleanJsonString) {\n      throw new Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          e1,\n        )}. Response - \\n ${raw}`,\n      );\n    }\n\n    try {\n      parsedObj = parseJsonWithRepair(code);\n      if (requireObject) {\n        assertJsonObject(parsedObj);\n      }\n    } catch (e2) {\n      throw new Error(\n        `failed to parse LLM response into JSON. First error - ${String(\n          e1,\n        )}. Second error - ${String(e2)}. Response - \\n ${raw}`,\n      );\n    }\n  }\n\n  return trimParsedJsonStrings(parsedObj, context);\n}\n"],"names":["extractJSONFromCodeBlock","response","jsonMatch","codeBlockMatch","jsonLikeMatch","trimParsedJsonStrings","obj","context","Array","item","normalized","key","value","Object","trimmedKey","preserveStringValue","normalizedValue","findNextNonWhitespaceChar","text","startIndex","index","char","escapeBareQuotesInsideJsonStrings","jsonBlock","repaired","inString","escaped","changed","nextNonWhitespaceChar","isStringTerminator","undefined","repairKnownJsonIssues","_rawResponse","_context","assertJsonObject","parsed","Error","JSON","parseJsonWithRepair","jsonStr","jsonrepair","parseModelResponseJson","raw","cleanJsonString","requireObject","parsedObj","e1","code","String","e2"],"mappings":";AAmBO,SAASA,yBAAyBC,QAAgB;IACvD,IAAI;QAEF,MAAMC,YAAYD,SAAS,KAAK,CAAC;QACjC,IAAIC,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBF,SAAS,KAAK,CACnC;QAEF,IAAIE,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBH,SAAS,KAAK,CAAC;QACrC,IAAIG,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOH;AACT;AAsBA,SAASI,sBACPC,GAAQ,EACRC,UAA8D,CAAC,CAAC;IAGhE,IAAID,QAAAA,KACF,OAAOA;IAIT,IAAIE,MAAM,OAAO,CAACF,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACG,OAASJ,sBAAsBI,MAAMF;IAIvD,IAAI,AAAe,YAAf,OAAOD,KAAkB;QAC3B,MAAMI,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACP,KAAM;YAE9C,MAAMQ,aAAaH,IAAI,IAAI;YAC3B,MAAMI,sBACJR,QAAQ,uBAAuB,EAAE,SAASO,eAAe;YAE3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOJ,QACHG,sBACEH,QACAA,MAAM,IAAI,KACZP,sBAAsBO,OAAOL;YAEnCG,UAAU,CAACI,WAAW,GAAGE;QAC3B;QAEA,OAAON;IACT;IAGA,IAAI,AAAe,YAAf,OAAOJ,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEA,SAASW,0BAA0BC,IAAY,EAAEC,UAAkB;IACjE,IAAK,IAAIC,QAAQD,YAAYC,QAAQF,KAAK,MAAM,EAAEE,QAAS;QACzD,MAAMC,OAAOH,IAAI,CAACE,MAAM;QACxB,IAAI,CAAC,KAAK,IAAI,CAACC,OACb,OAAOA;IAEX;AAEF;AAEA,SAASC,kCAAkCC,SAAiB;IAC1D,IAAIC,WAAW;IACf,IAAIC,WAAW;IACf,IAAIC,UAAU;IACd,IAAIC,UAAU;IAEd,IAAK,IAAIP,QAAQ,GAAGA,QAAQG,UAAU,MAAM,EAAEH,QAAS;QACrD,MAAMC,OAAOE,SAAS,CAACH,MAAM;QAE7B,IAAI,CAACK,UAAU;YACb,IAAIJ,AAAS,QAATA,MACFI,WAAW;YAEbD,YAAYH;YACZ;QACF;QAEA,IAAIK,SAAS;YACXF,YAAYH;YACZK,UAAU;YACV;QACF;QAEA,IAAIL,AAAS,SAATA,MAAe;YACjBG,YAAYH;YACZK,UAAU;YACV;QACF;QAEA,IAAIL,AAAS,QAATA,MAAc;YAChB,MAAMO,wBAAwBX,0BAC5BM,WACAH,QAAQ;YAEV,MAAMS,qBACJD,AAA0BE,WAA1BF,yBACAA,AAA0B,QAA1BA,yBACAA,AAA0B,QAA1BA,yBACAA,AAA0B,QAA1BA,yBACAA,AAA0B,QAA1BA;YAEF,IAAIC,oBAAoB;gBACtBJ,WAAW;gBACXD,YAAYH;YACd,OAAO;gBACLG,YAAY;gBACZG,UAAU;YACZ;YACA;QACF;QAEAH,YAAYH;IACd;IAEA,OAAOM,UAAUH,WAAWD;AAC9B;AAEA,SAASQ,sBACPR,SAAiB,EACjBS,YAAoB,EACpBC,QAA4B;IAE5B,OAAOX,kCAAkCC;AAC3C;AAEA,SAASW,iBACPC,MAAe;IAEf,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,UAAuB3B,MAAM,OAAO,CAAC2B,SACzD,MAAM,IAAIC,MACR,CAAC,sDAAsD,EAAEC,KAAK,SAAS,CACrEF,SACC;AAGT;AAEA,SAASG,oBAAoBC,OAAe;IAC1C,IAAI;QACF,OAAOF,KAAK,KAAK,CAACE;IACpB,EAAE,OAAM;QACN,OAAOF,KAAK,KAAK,CAACG,WAAWD;IAC/B;AACF;AAEO,SAASE,uBACdC,GAAW,EACXnC,OAA2B;IAE3B,MAAMoC,kBAAkB3C,yBAAyB0C;IACjD,MAAME,gBAAgBrC,SAAS,iBAAiB;IAEhD,IAAIsC;IAEJ,IAAI;QACFA,YAAYP,oBAAoBK;QAChC,IAAIC,eACFV,iBAAiBW;IAErB,EAAE,OAAOC,IAAI;QACX,MAAMC,OAAOhB,sBAAsBY,iBAAiBD,KAAKnC;QACzD,IAAIwC,SAASJ,iBACX,MAAM,IAAIP,MACR,CAAC,gDAAgD,EAAEY,OACjDF,IACA,gBAAgB,EAAEJ,KAAK;QAI7B,IAAI;YACFG,YAAYP,oBAAoBS;YAChC,IAAIH,eACFV,iBAAiBW;QAErB,EAAE,OAAOI,IAAI;YACX,MAAM,IAAIb,MACR,CAAC,sDAAsD,EAAEY,OACvDF,IACA,iBAAiB,EAAEE,OAAOC,IAAI,gBAAgB,EAAEP,KAAK;QAE3D;IACF;IAEA,OAAOrC,sBAAsBwC,WAAWtC;AAC1C"}
+{"version":3,"file":"ai-model/service-caller/json.mjs","sources":["../../../../src/ai-model/service-caller/json.ts"],"sourcesContent":["import { jsonrepair } from 'jsonrepair';\n\n/**\n * Extract the JSON portion from a model response.\n *\n * This mainly handles fenced JSON like ```json { ... } ``` by removing the\n * fence. If that does not produce a JSON object, it falls back to a greedy regex\n * that extracts from the first \"{\" to the last \"}\". If that still fails, the\n * original response is returned.\n *\n * Expected model responses are JSON objects, possibly wrapped in markdown\n * fences. Natural language mixed with JSON, or arrays like\n * [{\"type\":\"Tap\"}, {\"type\":\"Hover\"}], are outside the supported contract and\n * are not reliably recoverable.\n *\n * This legacy extractor is also used by extraction responses that can be any\n * JSON value. In those cases, arrays/strings/numbers have no object braces and\n * pass through unchanged.\n */\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport type JsonParserSource =\n  | 'generic-object'\n  | 'planning-action-param'\n  | 'locate'\n  | 'section-locator';\n\nexport interface JsonParserContext {\n  source: JsonParserSource;\n  preserveStringValueKeys?: string[];\n  requireObject?: boolean;\n}\n\nexport type JsonParser = (raw: string, context?: JsonParserContext) => unknown;\n\n/**\n * Trim whitespace in parsed JSON by normalizing:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction trimParsedJsonStrings(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => trimParsedJsonStrings(item, context));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : trimParsedJsonStrings(value, context);\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nfunction repairKnownJsonIssues(\n  jsonBlock: string,\n  _rawResponse: string,\n): string {\n  // TODO: Add project-specific repairs that jsonrepair cannot handle.\n  return jsonBlock;\n}\n\nfunction assertJsonObject(\n  parsed: unknown,\n): asserts parsed is Record<string, unknown> {\n  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {\n    throw new Error(\n      `expected parsed LLM response to be a JSON object, got ${JSON.stringify(\n        parsed,\n      )}`,\n    );\n  }\n}\n\nfunction parseJsonWithRepair(jsonStr: string) {\n  try {\n    return JSON.parse(jsonStr);\n  } catch {\n    return JSON.parse(jsonrepair(jsonStr));\n  }\n}\n\nexport function parseModelResponseJson(\n  raw: string,\n  context?: JsonParserContext,\n) {\n  const cleanJsonString = extractJSONFromCodeBlock(raw);\n  const requireObject = context?.requireObject ?? true;\n\n  let parsedObj: unknown;\n\n  try {\n    parsedObj = parseJsonWithRepair(cleanJsonString);\n    if (requireObject) {\n      assertJsonObject(parsedObj);\n    }\n  } catch (e1) {\n    const code = repairKnownJsonIssues(cleanJsonString, raw);\n    if (code === cleanJsonString) {\n      throw new Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          e1,\n        )}. Response - \\n ${raw}`,\n      );\n    }\n\n    try {\n      parsedObj = parseJsonWithRepair(code);\n      if (requireObject) {\n        assertJsonObject(parsedObj);\n      }\n    } catch (e2) {\n      throw new Error(\n        `failed to parse LLM response into JSON. First error - ${String(\n          e1,\n        )}. Second error - ${String(e2)}. Response - \\n ${raw}`,\n      );\n    }\n  }\n\n  return trimParsedJsonStrings(parsedObj, context);\n}\n"],"names":["extractJSONFromCodeBlock","response","jsonMatch","codeBlockMatch","jsonLikeMatch","trimParsedJsonStrings","obj","context","Array","item","normalized","key","value","Object","trimmedKey","preserveStringValue","normalizedValue","repairKnownJsonIssues","jsonBlock","_rawResponse","assertJsonObject","parsed","Error","JSON","parseJsonWithRepair","jsonStr","jsonrepair","parseModelResponseJson","raw","cleanJsonString","requireObject","parsedObj","e1","code","String","e2"],"mappings":";AAmBO,SAASA,yBAAyBC,QAAgB;IACvD,IAAI;QAEF,MAAMC,YAAYD,SAAS,KAAK,CAAC;QACjC,IAAIC,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBF,SAAS,KAAK,CACnC;QAEF,IAAIE,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBH,SAAS,KAAK,CAAC;QACrC,IAAIG,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOH;AACT;AAsBA,SAASI,sBACPC,GAAQ,EACRC,UAA8D,CAAC,CAAC;IAGhE,IAAID,QAAAA,KACF,OAAOA;IAIT,IAAIE,MAAM,OAAO,CAACF,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACG,OAASJ,sBAAsBI,MAAMF;IAIvD,IAAI,AAAe,YAAf,OAAOD,KAAkB;QAC3B,MAAMI,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACP,KAAM;YAE9C,MAAMQ,aAAaH,IAAI,IAAI;YAC3B,MAAMI,sBACJR,QAAQ,uBAAuB,EAAE,SAASO,eAAe;YAE3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOJ,QACHG,sBACEH,QACAA,MAAM,IAAI,KACZP,sBAAsBO,OAAOL;YAEnCG,UAAU,CAACI,WAAW,GAAGE;QAC3B;QAEA,OAAON;IACT;IAGA,IAAI,AAAe,YAAf,OAAOJ,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEA,SAASW,sBACPC,SAAiB,EACjBC,YAAoB;IAGpB,OAAOD;AACT;AAEA,SAASE,iBACPC,MAAe;IAEf,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,UAAuBb,MAAM,OAAO,CAACa,SACzD,MAAM,IAAIC,MACR,CAAC,sDAAsD,EAAEC,KAAK,SAAS,CACrEF,SACC;AAGT;AAEA,SAASG,oBAAoBC,OAAe;IAC1C,IAAI;QACF,OAAOF,KAAK,KAAK,CAACE;IACpB,EAAE,OAAM;QACN,OAAOF,KAAK,KAAK,CAACG,WAAWD;IAC/B;AACF;AAEO,SAASE,uBACdC,GAAW,EACXrB,OAA2B;IAE3B,MAAMsB,kBAAkB7B,yBAAyB4B;IACjD,MAAME,gBAAgBvB,SAAS,iBAAiB;IAEhD,IAAIwB;IAEJ,IAAI;QACFA,YAAYP,oBAAoBK;QAChC,IAAIC,eACFV,iBAAiBW;IAErB,EAAE,OAAOC,IAAI;QACX,MAAMC,OAAOhB,sBAAsBY,iBAAiBD;QACpD,IAAIK,SAASJ,iBACX,MAAM,IAAIP,MACR,CAAC,gDAAgD,EAAEY,OACjDF,IACA,gBAAgB,EAAEJ,KAAK;QAI7B,IAAI;YACFG,YAAYP,oBAAoBS;YAChC,IAAIH,eACFV,iBAAiBW;QAErB,EAAE,OAAOI,IAAI;YACX,MAAM,IAAIb,MACR,CAAC,sDAAsD,EAAEY,OACvDF,IACA,iBAAiB,EAAEE,OAAOC,IAAI,gBAAgB,EAAEP,KAAK;QAE3D;IACF;IAEA,OAAOvB,sBAAsB0B,WAAWxB;AAC1C"}