@midscene/web 0.18.1 → 0.18.2

package/dist/es/utils.js

@@ -195,7 +195,10 @@ function trimContextByViewport(execution) {
       if (task.pageContext?.tree) {
         newTask.pageContext = {
           ...task.pageContext,
-          tree: filterVisibleTree(task.pageContext.tree)
+          tree: filterVisibleTree(task.pageContext.tree) || {
+            node: null,
+            children: []
+          }
         };
       }
       return newTask;

package/dist/es/utils.js.map

@@ -1 +1 @@
-{"version":3,"mappings":";AAUA,SAAS,wCAAwC;AACjD,SAAS,8BAA8B;AACvC,SAAS,0BAA0B,mBAAmB;AAEtD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AAEhC,SAAS,QAAQ,QAAQ,YAAY;AACrC,OAAO,WAAW;;;ACZX,IAAM,iBAAN,MAA4C;AAAA,EAoBjD,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAWG;AACD,SAAK,UAAU;AACf,SAAK,OAAO;AACZ,SAAK,SAAS;AAAA,MACZ,KAAK,MAAM,KAAK,OAAO,KAAK,QAAQ,CAAC;AAAA,MACrC,KAAK,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC;AAAA,IACvC;AACA,SAAK,KAAK;AACV,SAAK,aAAa;AAClB,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,YAAY;AAAA,EACnB;AACF;;;AD9BA,eAAsB,wBACpB,MACA,MACuB;AACvB,SAAO,MAAM,kBAAkB;AAC/B,MAAK,KAAoB,sBAAsB;AAC7C,WAAO,MAAO,KAAa,qBAAqB;AAAA,EAClD;AACA,QAAM,MAAM,MAAM,KAAK,IAAI;AAC3B,yBAAuB,EAAE,SAAS,IAAI,CAAC;AAEvC,MAAI;AACJ,MAAI;AAEJ,QAAM,QAAQ,IAAI;AAAA,IAChB,KAAK,iBAAiB,EAAE,KAAK,CAAC,WAAW;AACvC,yBAAmB;AAAA,IACrB,CAAC;AAAA,IACD,KAAK,oBAAoB,EAAE,KAAK,OAAO,aAAa;AAClD,aAAO;AAAA,IACT,CAAC;AAAA,EACH,CAAC;AAED,QAAM,UAAU,aAAa,MAAO,CAAC,gBAAgB;AACnD,UAAM,EAAE,MAAM,IAAI,SAAS,YAAY,SAAS,UAAU,IAAI;AAC9D,WAAO,IAAI,eAAe;AAAA,MACxB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,SAAO,kBAAmB,8BAA8B;AAExD,QAAM,OAAO,MAAM,KAAK,KAAK;AAE7B,MAAI,KAAK,OAAO,KAAK,MAAM,GAAG;AAE5B,uBAAmB,MAAM,gBAAgB,kBAAkB;AAAA,MACzD,OAAO,KAAK;AAAA,MACZ,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EAEH;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,eAAe,MAAM,OAAO;AAC1C,QAAM,gBAAgB,YAAY,wBAAwB;AAC1D,QAAM,qBAAqB,MAAM,EAAE,OAAO,qBAAqB;AAE/D,QAAM,WAAW,KAAK,EAAE,UAAU,GAAG,CAAC;AACtC,SAAO,GAAG,iBAAiB,GAAG,IAAI,kBAAkB,IAAI,QAAQ;AAClE;AAEO,SAAS,eAAe,UAAkB;AAC/C,SAAO,mCAAmC,QAAQ,EAAE;AACtD;AAMO,SAAS,wBAAwB,OAAgC;AACtE,QAAM,QAAQ,IAAI,MAAM;AACxB,QAAM,aAAa,SAAS,MAAM;AAClC,QAAM,SAAS,QAAQ,IAAI,KAAK;AAChC,MAAI,YAAY;AACd,UAAM,aAAa,WAAW,MAAM,IAAI;AACxC,eAAW,QAAQ,YAAY;AAC7B,UACE,KAAK,SAAS,QAAQ,KACtB,KAAK,SAAS,QAAQ,KACtB,KAAK,SAAS,KAAK,KACnB,KAAK,SAAS,KAAK,GACnB;AACA,cAAM,QAAQ,KAAK,MAAM,uCAAuC;AAChE,YAAI,QAAQ,CAAC,GAAG;AACd,gBAAM,iBAAiB,MAAM,CAAC,EAC3B,QAAQ,QAAQ,EAAE,EAClB,KAAK,EACL,QAAQ,OAAO,EAAE;AACpB,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,gBAAgB,oBAAI,IAAoB;AAEvC,SAAS,gBAAgB,UAA2B;AACzD,MAAI,WAAW,YAAY,wBAAwB;AACnD,MAAI,CAAC,UAAU;AACb,eAAW,KAAK;AAChB,YAAQ;AAAA,MACN;AAAA,IACF;AAAA,EACF;AAEA,MAAI,cAAc,IAAI,QAAQ,GAAG;AAC/B,UAAM,eAAe,cAAc,IAAI,QAAQ;AAC/C,QAAI,iBAAiB,QAAW;AAC9B,oBAAc,IAAI,UAAU,eAAe,CAAC;AAAA,IAC9C;AAAA,EACF,OAAO;AACL,kBAAc,IAAI,UAAU,CAAC;AAAA,EAC/B;AACA,SAAO,GAAG,QAAQ,IAAI,cAAc,IAAI,QAAQ,CAAC;AACnD;AAEO,IAAM,yCACX;AAEK,SAAS,gCAAgC,KAAa;AAE3D,SAAO,IAAI,QAAQ,eAAe,GAAG;AACvC;AAEO,SAAS,gBACd,MACA,OACA;AACA,OAAK,GAAG,SAAS,OAAO,UAAU;AAChC,QAAI,CAAC,OAAO;AACV,cAAQ,KAAK,yDAAyD;AACtE;AAAA,IACF;AACA,UAAM,MAAM,MAAO,MAAwB,IAAI;AAC/C,YAAQ,IAAI,iBAAiB,GAAG,EAAE;AAClC,QAAI,CAAE,MAAwB,SAAS,GAAG;AACxC,UAAI;AACF,cAAO,MAAwB,MAAM;AAAA,MACvC,SAAS,OAAO;AACd,cAAM,yBAAyB,GAAG,YAAY,KAAK,EAAE;AAAA,MACvD;AAAA,IACF,OAAO;AACL,YAAM,uCAAuC,GAAG,EAAE;AAAA,IACpD;AAEA,QAAI,CAAC,KAAK,SAAS,GAAG;AACpB,UAAI;AACF,cAAM,KAAK,KAAK,GAAG;AAAA,MACrB,SAAS,OAAO;AACd,cAAM,kBAAkB,GAAG,YAAY,KAAK,EAAE;AAAA,MAChD;AAAA,IACF,OAAO;AACL,YAAM,qCAAqC,GAAG,EAAE;AAAA,IAClD;AAAA,EACF,CAAC;AACH;AAEO,SAAS,qBACd,iBACA,MACA;AACA,MAAI,CAAC,iBAAiB;AACpB,WAAO;AAAA,EACT;AACA,MAAI,gBAAgB,IAAI;AACtB,WAAO,qBAAqB,gBAAgB,EAAE;AAAA,EAChD;AAEA,MAAI,gBAAgB,MAAM;AACxB,UAAM,iBAAiB;AAAA,MACrB,GAAG,KAAK,OAAO,gBAAgB,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,KAAK,CAAC;AAAA,MACrE,GAAG,KAAK,OAAO,gBAAgB,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,KAAK,CAAC;AAAA,IACvE;AACA,QAAI,UAAU,iCAAiC,MAAM,cAAc;AAEnE,QAAI,CAAC,SAAS;AACZ,gBAAU,0BAA0B,cAAc;AAAA,IACpD;AAEA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAEO,SAAS,sBAAsB,WAA0B;AAC9D,WAAS,kBACP,MACqC;AACrC,QAAI,CAAC,QAAQ,CAAC,KAAK,QAAQ,KAAK,KAAK,cAAc;AAAM,aAAO;AAChE,UAAM,mBAAmB,MAAM,QAAQ,KAAK,QAAQ,IAC/C,KAAK,SACH,IAAI,iBAAiB,EACrB,OAAO,CAAC,UAAU,UAAU,IAAI,IACnC;AACJ,WAAO;AAAA,MACL,GAAG;AAAA,MACH,GAAI,mBAAmB,EAAE,UAAU,iBAAiB,IAAI,CAAC;AAAA,IAC3D;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAG;AAAA,IACH,OAAO,MAAM,QAAQ,UAAU,KAAK,IAChC,UAAU,MAAM,IAAI,CAAC,SAAwB;AAC3C,YAAM,UAAU,EAAE,GAAG,KAAK;AAC1B,UAAI,KAAK,aAAa,MAAM;AAC1B,gBAAQ,cAAc;AAAA,UACpB,GAAG,KAAK;AAAA,UACR,MAAM,kBAAkB,KAAK,YAAY,IAAI;AAAA,QAC/C;AAAA,MACF;AACA,aAAO;AAAA,IACT,CAAC,IACD,UAAU;AAAA,EAChB;AACF","names":[],"ignoreList":[],"sources":["../../src/common/utils.ts","../../src/web-element.ts"],"sourcesContent":["import type { StaticPage } from '@/playground';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  ExecutionDump,\n  ExecutionTask,\n  PlanningLocateParam,\n  PlaywrightParserOpt,\n  UIContext,\n} from '@midscene/core';\nimport { elementByPositionWithElementInfo } from '@midscene/core/ai-model';\nimport { uploadTestInfoToServer } from '@midscene/core/utils';\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from '@midscene/shared/env';\nimport type { ElementInfo } from '@midscene/shared/extractor';\nimport {\n  generateElementByPosition,\n  getNodeFromCacheList,\n  traverseTree,\n} from '@midscene/shared/extractor';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport type { DebugFunction } from '@midscene/shared/logger';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { Page as PlaywrightPage } from 'playwright';\nimport type { Page as PuppeteerPage } from 'puppeteer';\nimport { WebElementInfo } from '../web-element';\nimport type { WebPage } from './page';\n\nexport type WebUIContext = UIContext<WebElementInfo> & {\n  url: string;\n};\n\nexport async function parseContextFromWebPage(\n  page: WebPage,\n  _opt?: PlaywrightParserOpt,\n): Promise<WebUIContext> {\n  assert(page, 'page is required');\n  if ((page as StaticPage)._forceUsePageContext) {\n    return await (page as any)._forceUsePageContext();\n  }\n  const url = await page.url();\n  uploadTestInfoToServer({ testUrl: url });\n\n  let screenshotBase64: string;\n  let tree: ElementTreeNode<ElementInfo>;\n\n  await Promise.all([\n    page.screenshotBase64().then((base64) => {\n      screenshotBase64 = base64;\n    }),\n    page.getElementsNodeTree().then(async (treeRoot) => {\n      tree = treeRoot;\n    }),\n  ]);\n\n  const webTree = traverseTree(tree!, (elementInfo) => {\n    const { rect, id, content, attributes, indexId, isVisible } = elementInfo;\n    return new WebElementInfo({\n      rect,\n      id,\n      content,\n      attributes,\n      indexId,\n      isVisible,\n    });\n  });\n\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await page.size();\n\n  if (size.dpr && size.dpr > 1) {\n    // console.time('resizeImgBase64');\n    screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n      width: size.width,\n      height: size.height,\n    });\n    // console.timeEnd('resizeImgBase64');\n  }\n\n  return {\n    tree: webTree,\n    size,\n    screenshotBase64: screenshotBase64!,\n    url,\n  };\n}\n\nexport function reportFileName(tag = 'web') {\n  const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\n  'NOT_IMPLEMENTED_AS_DESIGNED';\n\nexport function replaceIllegalPathCharsAndSpace(str: string) {\n  // Only replace characters that are illegal in filenames, but preserve path separators\n  return str.replace(/[:*?\"<>| ]/g, '-');\n}\n\nexport function forceClosePopup(\n  page: PuppeteerPage | PlaywrightPage,\n  debug: DebugFunction,\n) {\n  page.on('popup', async (popup) => {\n    if (!popup) {\n      console.warn('got a popup event, but the popup is not ready yet, skip');\n      return;\n    }\n    const url = await (popup as PuppeteerPage).url();\n    console.log(`Popup opened: ${url}`);\n    if (!(popup as PuppeteerPage).isClosed()) {\n      try {\n        await (popup as PuppeteerPage).close(); // Close the newly opened TAB\n      } catch (error) {\n        debug(`failed to close popup ${url}, error: ${error}`);\n      }\n    } else {\n      debug(`popup is already closed, skip close ${url}`);\n    }\n\n    if (!page.isClosed()) {\n      try {\n        await page.goto(url);\n      } catch (error) {\n        debug(`failed to goto ${url}, error: ${error}`);\n      }\n    } else {\n      debug(`page is already closed, skip goto ${url}`);\n    }\n  });\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n  tree: ElementTreeNode<BaseElement>,\n) {\n  if (!planLocateParam) {\n    return undefined;\n  }\n  if (planLocateParam.id) {\n    return getNodeFromCacheList(planLocateParam.id);\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n    let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n    if (!element) {\n      element = generateElementByPosition(centerPosition) as BaseElement;\n    }\n\n    return element;\n  }\n\n  return undefined;\n}\n\nexport function trimContextByViewport(execution: ExecutionDump) {\n  function filterVisibleTree(\n    node: ElementTreeNode<BaseElement>,\n  ): ElementTreeNode<BaseElement> | null {\n    if (!node || !node.node || node.node.isVisible !== true) return null;\n    const filteredChildren = Array.isArray(node.children)\n      ? (node.children\n          .map(filterVisibleTree)\n          .filter((child) => child !== null) as ElementTreeNode<BaseElement>[])\n      : undefined;\n    return {\n      ...node,\n      ...(filteredChildren ? { children: filteredChildren } : {}),\n    };\n  }\n\n  return {\n    ...execution,\n    tasks: Array.isArray(execution.tasks)\n      ? execution.tasks.map((task: ExecutionTask) => {\n          const newTask = { ...task };\n          if (task.pageContext?.tree) {\n            newTask.pageContext = {\n              ...task.pageContext,\n              tree: filterVisibleTree(task.pageContext.tree)!,\n            };\n          }\n          return newTask;\n        })\n      : execution.tasks,\n  };\n}\n","import type { BaseElement, Rect } from '@midscene/core';\nimport type { NodeType } from '@midscene/shared/constants';\nexport interface WebElementInfoType extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport class WebElementInfo implements BaseElement {\n  content: string;\n\n  rect: Rect;\n\n  center: [number, number];\n\n  id: string;\n\n  indexId: number;\n\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n\n  xpaths?: string[];\n\n  isVisible: boolean;\n\n  constructor({\n    content,\n    rect,\n    id,\n    attributes,\n    indexId,\n    xpaths,\n    isVisible,\n  }: {\n    content: string;\n    rect: Rect;\n    id: string;\n    attributes: {\n      nodeType: NodeType;\n      [key: string]: string;\n    };\n    indexId: number;\n    xpaths?: string[];\n    isVisible: boolean;\n  }) {\n    this.content = content;\n    this.rect = rect;\n    this.center = [\n      Math.floor(rect.left + rect.width / 2),\n      Math.floor(rect.top + rect.height / 2),\n    ];\n    this.id = id;\n    this.attributes = attributes;\n    this.indexId = indexId;\n    this.xpaths = xpaths;\n    this.isVisible = isVisible;\n  }\n}\n"]}
+{"version":3,"mappings":";AAUA,SAAS,wCAAwC;AACjD,SAAS,8BAA8B;AACvC,SAAS,0BAA0B,mBAAmB;AAEtD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AAEhC,SAAS,QAAQ,QAAQ,YAAY;AACrC,OAAO,WAAW;;;ACZX,IAAM,iBAAN,MAA4C;AAAA,EAoBjD,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAWG;AACD,SAAK,UAAU;AACf,SAAK,OAAO;AACZ,SAAK,SAAS;AAAA,MACZ,KAAK,MAAM,KAAK,OAAO,KAAK,QAAQ,CAAC;AAAA,MACrC,KAAK,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC;AAAA,IACvC;AACA,SAAK,KAAK;AACV,SAAK,aAAa;AAClB,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,YAAY;AAAA,EACnB;AACF;;;AD9BA,eAAsB,wBACpB,MACA,MACuB;AACvB,SAAO,MAAM,kBAAkB;AAC/B,MAAK,KAAoB,sBAAsB;AAC7C,WAAO,MAAO,KAAa,qBAAqB;AAAA,EAClD;AACA,QAAM,MAAM,MAAM,KAAK,IAAI;AAC3B,yBAAuB,EAAE,SAAS,IAAI,CAAC;AAEvC,MAAI;AACJ,MAAI;AAEJ,QAAM,QAAQ,IAAI;AAAA,IAChB,KAAK,iBAAiB,EAAE,KAAK,CAAC,WAAW;AACvC,yBAAmB;AAAA,IACrB,CAAC;AAAA,IACD,KAAK,oBAAoB,EAAE,KAAK,OAAO,aAAa;AAClD,aAAO;AAAA,IACT,CAAC;AAAA,EACH,CAAC;AAED,QAAM,UAAU,aAAa,MAAO,CAAC,gBAAgB;AACnD,UAAM,EAAE,MAAM,IAAI,SAAS,YAAY,SAAS,UAAU,IAAI;AAC9D,WAAO,IAAI,eAAe;AAAA,MACxB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,SAAO,kBAAmB,8BAA8B;AAExD,QAAM,OAAO,MAAM,KAAK,KAAK;AAE7B,MAAI,KAAK,OAAO,KAAK,MAAM,GAAG;AAE5B,uBAAmB,MAAM,gBAAgB,kBAAkB;AAAA,MACzD,OAAO,KAAK;AAAA,MACZ,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EAEH;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,eAAe,MAAM,OAAO;AAC1C,QAAM,gBAAgB,YAAY,wBAAwB;AAC1D,QAAM,qBAAqB,MAAM,EAAE,OAAO,qBAAqB;AAE/D,QAAM,WAAW,KAAK,EAAE,UAAU,GAAG,CAAC;AACtC,SAAO,GAAG,iBAAiB,GAAG,IAAI,kBAAkB,IAAI,QAAQ;AAClE;AAEO,SAAS,eAAe,UAAkB;AAC/C,SAAO,mCAAmC,QAAQ,EAAE;AACtD;AAMO,SAAS,wBAAwB,OAAgC;AACtE,QAAM,QAAQ,IAAI,MAAM;AACxB,QAAM,aAAa,SAAS,MAAM;AAClC,QAAM,SAAS,QAAQ,IAAI,KAAK;AAChC,MAAI,YAAY;AACd,UAAM,aAAa,WAAW,MAAM,IAAI;AACxC,eAAW,QAAQ,YAAY;AAC7B,UACE,KAAK,SAAS,QAAQ,KACtB,KAAK,SAAS,QAAQ,KACtB,KAAK,SAAS,KAAK,KACnB,KAAK,SAAS,KAAK,GACnB;AACA,cAAM,QAAQ,KAAK,MAAM,uCAAuC;AAChE,YAAI,QAAQ,CAAC,GAAG;AACd,gBAAM,iBAAiB,MAAM,CAAC,EAC3B,QAAQ,QAAQ,EAAE,EAClB,KAAK,EACL,QAAQ,OAAO,EAAE;AACpB,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,gBAAgB,oBAAI,IAAoB;AAEvC,SAAS,gBAAgB,UAA2B;AACzD,MAAI,WAAW,YAAY,wBAAwB;AACnD,MAAI,CAAC,UAAU;AACb,eAAW,KAAK;AAChB,YAAQ;AAAA,MACN;AAAA,IACF;AAAA,EACF;AAEA,MAAI,cAAc,IAAI,QAAQ,GAAG;AAC/B,UAAM,eAAe,cAAc,IAAI,QAAQ;AAC/C,QAAI,iBAAiB,QAAW;AAC9B,oBAAc,IAAI,UAAU,eAAe,CAAC;AAAA,IAC9C;AAAA,EACF,OAAO;AACL,kBAAc,IAAI,UAAU,CAAC;AAAA,EAC/B;AACA,SAAO,GAAG,QAAQ,IAAI,cAAc,IAAI,QAAQ,CAAC;AACnD;AAEO,IAAM,yCACX;AAEK,SAAS,gCAAgC,KAAa;AAE3D,SAAO,IAAI,QAAQ,eAAe,GAAG;AACvC;AAEO,SAAS,gBACd,MACA,OACA;AACA,OAAK,GAAG,SAAS,OAAO,UAAU;AAChC,QAAI,CAAC,OAAO;AACV,cAAQ,KAAK,yDAAyD;AACtE;AAAA,IACF;AACA,UAAM,MAAM,MAAO,MAAwB,IAAI;AAC/C,YAAQ,IAAI,iBAAiB,GAAG,EAAE;AAClC,QAAI,CAAE,MAAwB,SAAS,GAAG;AACxC,UAAI;AACF,cAAO,MAAwB,MAAM;AAAA,MACvC,SAAS,OAAO;AACd,cAAM,yBAAyB,GAAG,YAAY,KAAK,EAAE;AAAA,MACvD;AAAA,IACF,OAAO;AACL,YAAM,uCAAuC,GAAG,EAAE;AAAA,IACpD;AAEA,QAAI,CAAC,KAAK,SAAS,GAAG;AACpB,UAAI;AACF,cAAM,KAAK,KAAK,GAAG;AAAA,MACrB,SAAS,OAAO;AACd,cAAM,kBAAkB,GAAG,YAAY,KAAK,EAAE;AAAA,MAChD;AAAA,IACF,OAAO;AACL,YAAM,qCAAqC,GAAG,EAAE;AAAA,IAClD;AAAA,EACF,CAAC;AACH;AAEO,SAAS,qBACd,iBACA,MACA;AACA,MAAI,CAAC,iBAAiB;AACpB,WAAO;AAAA,EACT;AACA,MAAI,gBAAgB,IAAI;AACtB,WAAO,qBAAqB,gBAAgB,EAAE;AAAA,EAChD;AAEA,MAAI,gBAAgB,MAAM;AACxB,UAAM,iBAAiB;AAAA,MACrB,GAAG,KAAK,OAAO,gBAAgB,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,KAAK,CAAC;AAAA,MACrE,GAAG,KAAK,OAAO,gBAAgB,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,KAAK,CAAC;AAAA,IACvE;AACA,QAAI,UAAU,iCAAiC,MAAM,cAAc;AAEnE,QAAI,CAAC,SAAS;AACZ,gBAAU,0BAA0B,cAAc;AAAA,IACpD;AAEA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAEO,SAAS,sBAAsB,WAA0B;AAC9D,WAAS,kBACP,MACqC;AACrC,QAAI,CAAC,QAAQ,CAAC,KAAK,QAAQ,KAAK,KAAK,cAAc;AAAM,aAAO;AAChE,UAAM,mBAAmB,MAAM,QAAQ,KAAK,QAAQ,IAC/C,KAAK,SACH,IAAI,iBAAiB,EACrB,OAAO,CAAC,UAAU,UAAU,IAAI,IACnC;AACJ,WAAO;AAAA,MACL,GAAG;AAAA,MACH,GAAI,mBAAmB,EAAE,UAAU,iBAAiB,IAAI,CAAC;AAAA,IAC3D;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAG;AAAA,IACH,OAAO,MAAM,QAAQ,UAAU,KAAK,IAChC,UAAU,MAAM,IAAI,CAAC,SAAwB;AAC3C,YAAM,UAAU,EAAE,GAAG,KAAK;AAC1B,UAAI,KAAK,aAAa,MAAM;AAC1B,gBAAQ,cAAc;AAAA,UACpB,GAAG,KAAK;AAAA,UACR,MAAM,kBAAkB,KAAK,YAAY,IAAI,KAAK;AAAA,YAChD,MAAM;AAAA,YACN,UAAU,CAAC;AAAA,UACb;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT,CAAC,IACD,UAAU;AAAA,EAChB;AACF","names":[],"ignoreList":[],"sources":["../../src/common/utils.ts","../../src/web-element.ts"],"sourcesContent":["import type { StaticPage } from '@/playground';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  ExecutionDump,\n  ExecutionTask,\n  PlanningLocateParam,\n  PlaywrightParserOpt,\n  UIContext,\n} from '@midscene/core';\nimport { elementByPositionWithElementInfo } from '@midscene/core/ai-model';\nimport { uploadTestInfoToServer } from '@midscene/core/utils';\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from '@midscene/shared/env';\nimport type { ElementInfo } from '@midscene/shared/extractor';\nimport {\n  generateElementByPosition,\n  getNodeFromCacheList,\n  traverseTree,\n} from '@midscene/shared/extractor';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport type { DebugFunction } from '@midscene/shared/logger';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { Page as PlaywrightPage } from 'playwright';\nimport type { Page as PuppeteerPage } from 'puppeteer';\nimport { WebElementInfo } from '../web-element';\nimport type { WebPage } from './page';\n\nexport type WebUIContext = UIContext<WebElementInfo> & {\n  url: string;\n};\n\nexport async function parseContextFromWebPage(\n  page: WebPage,\n  _opt?: PlaywrightParserOpt,\n): Promise<WebUIContext> {\n  assert(page, 'page is required');\n  if ((page as StaticPage)._forceUsePageContext) {\n    return await (page as any)._forceUsePageContext();\n  }\n  const url = await page.url();\n  uploadTestInfoToServer({ testUrl: url });\n\n  let screenshotBase64: string;\n  let tree: ElementTreeNode<ElementInfo>;\n\n  await Promise.all([\n    page.screenshotBase64().then((base64) => {\n      screenshotBase64 = base64;\n    }),\n    page.getElementsNodeTree().then(async (treeRoot) => {\n      tree = treeRoot;\n    }),\n  ]);\n\n  const webTree = traverseTree(tree!, (elementInfo) => {\n    const { rect, id, content, attributes, indexId, isVisible } = elementInfo;\n    return new WebElementInfo({\n      rect,\n      id,\n      content,\n      attributes,\n      indexId,\n      isVisible,\n    });\n  });\n\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await page.size();\n\n  if (size.dpr && size.dpr > 1) {\n    // console.time('resizeImgBase64');\n    screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n      width: size.width,\n      height: size.height,\n    });\n    // console.timeEnd('resizeImgBase64');\n  }\n\n  return {\n    tree: webTree,\n    size,\n    screenshotBase64: screenshotBase64!,\n    url,\n  };\n}\n\nexport function reportFileName(tag = 'web') {\n  const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\n  'NOT_IMPLEMENTED_AS_DESIGNED';\n\nexport function replaceIllegalPathCharsAndSpace(str: string) {\n  // Only replace characters that are illegal in filenames, but preserve path separators\n  return str.replace(/[:*?\"<>| ]/g, '-');\n}\n\nexport function forceClosePopup(\n  page: PuppeteerPage | PlaywrightPage,\n  debug: DebugFunction,\n) {\n  page.on('popup', async (popup) => {\n    if (!popup) {\n      console.warn('got a popup event, but the popup is not ready yet, skip');\n      return;\n    }\n    const url = await (popup as PuppeteerPage).url();\n    console.log(`Popup opened: ${url}`);\n    if (!(popup as PuppeteerPage).isClosed()) {\n      try {\n        await (popup as PuppeteerPage).close(); // Close the newly opened TAB\n      } catch (error) {\n        debug(`failed to close popup ${url}, error: ${error}`);\n      }\n    } else {\n      debug(`popup is already closed, skip close ${url}`);\n    }\n\n    if (!page.isClosed()) {\n      try {\n        await page.goto(url);\n      } catch (error) {\n        debug(`failed to goto ${url}, error: ${error}`);\n      }\n    } else {\n      debug(`page is already closed, skip goto ${url}`);\n    }\n  });\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n  tree: ElementTreeNode<BaseElement>,\n) {\n  if (!planLocateParam) {\n    return undefined;\n  }\n  if (planLocateParam.id) {\n    return getNodeFromCacheList(planLocateParam.id);\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n    let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n    if (!element) {\n      element = generateElementByPosition(centerPosition) as BaseElement;\n    }\n\n    return element;\n  }\n\n  return undefined;\n}\n\nexport function trimContextByViewport(execution: ExecutionDump) {\n  function filterVisibleTree(\n    node: ElementTreeNode<BaseElement>,\n  ): ElementTreeNode<BaseElement> | null {\n    if (!node || !node.node || node.node.isVisible !== true) return null;\n    const filteredChildren = Array.isArray(node.children)\n      ? (node.children\n          .map(filterVisibleTree)\n          .filter((child) => child !== null) as ElementTreeNode<BaseElement>[])\n      : undefined;\n    return {\n      ...node,\n      ...(filteredChildren ? { children: filteredChildren } : {}),\n    };\n  }\n\n  return {\n    ...execution,\n    tasks: Array.isArray(execution.tasks)\n      ? execution.tasks.map((task: ExecutionTask) => {\n          const newTask = { ...task };\n          if (task.pageContext?.tree) {\n            newTask.pageContext = {\n              ...task.pageContext,\n              tree: filterVisibleTree(task.pageContext.tree) || {\n                node: null,\n                children: [],\n              },\n            };\n          }\n          return newTask;\n        })\n      : execution.tasks,\n  };\n}\n","import type { BaseElement, Rect } from '@midscene/core';\nimport type { NodeType } from '@midscene/shared/constants';\nexport interface WebElementInfoType extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport class WebElementInfo implements BaseElement {\n  content: string;\n\n  rect: Rect;\n\n  center: [number, number];\n\n  id: string;\n\n  indexId: number;\n\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n\n  xpaths?: string[];\n\n  isVisible: boolean;\n\n  constructor({\n    content,\n    rect,\n    id,\n    attributes,\n    indexId,\n    xpaths,\n    isVisible,\n  }: {\n    content: string;\n    rect: Rect;\n    id: string;\n    attributes: {\n      nodeType: NodeType;\n      [key: string]: string;\n    };\n    indexId: number;\n    xpaths?: string[];\n    isVisible: boolean;\n  }) {\n    this.content = content;\n    this.rect = rect;\n    this.center = [\n      Math.floor(rect.left + rect.width / 2),\n      Math.floor(rect.top + rect.height / 2),\n    ];\n    this.id = id;\n    this.attributes = attributes;\n    this.indexId = indexId;\n    this.xpaths = xpaths;\n    this.isVisible = isVisible;\n  }\n}\n"]}

package/dist/lib/agent.js

@@ -638,7 +638,10 @@ function trimContextByViewport(execution) {
       if (task.pageContext?.tree) {
         newTask.pageContext = {
           ...task.pageContext,
-          tree: filterVisibleTree(task.pageContext.tree)
+          tree: filterVisibleTree(task.pageContext.tree) || {
+            node: null,
+            children: []
+          }
         };
       }
       return newTask;
@@ -1762,7 +1765,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
 var import_semver = __toESM(require("semver"));
 
 // package.json
-var version = "0.18.1";
+var version = "0.18.2";
 
 // src/common/task-cache.ts
 var debug3 = (0, import_logger3.getDebug)("cache");