@midscene/web 0.15.0 → 0.15.1

package/dist/lib/playwright-report.js

@@ -48,7 +48,7 @@ function reportFileName(tag = "web") {
   return `${reportTagName || tag}-${dateTimeInFileName}`;
 }
 function printReportMsg(filepath) {
-  console.log("Midscene - report file updated:", filepath);
+  (0, import_utils2.logMsg)(`Midscene - report file updated: ${filepath}`);
 }
 
 // src/playwright/reporter/index.ts

package/dist/lib/playwright-report.js.map

@@ -1 +1 @@
-{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACMA,iBAKO;AACP,mBAAuC;AACvC,uBAAyB;AAEzB,uBAAyC;AACzC,iBAAyD;AACzD,IAAAA,gBAA6B;AAC7B,mBAAkB;AAyFX,SAAS,eAAe,MAAM,OAAO;AAC1C,QAAM,oBAAgB,wBAAY,mCAAwB;AAC1D,QAAM,yBAAqB,aAAAC,SAAM,EAAE,OAAO,yBAAyB;AACnE,SAAO,GAAG,iBAAiB,GAAG,IAAI,kBAAkB;AACtD;AAEO,SAAS,eAAe,UAAkB;AAE/C,UAAQ,IAAI,mCAAmC,QAAQ;AAGzD;;;ADpHA,IAAAD,gBAAgC;AAUhC,SAAS,UAAU,SAAgB;AACjC,MAAI,QAAQ,IAAI,UAAU,QAAQ;AAChC,YAAQ,IAAI,wBAAwB,GAAG,OAAO;AAAA,EAChD;AACF;AAEA,IAAM,eAAgD,CAAC;AACvD,IAAI;AACJ,SAAS,eAAe;AACtB,QAAM,iBAAa,+BAAgB,UAAU,YAAY;AACzD,gBAAc,eAAe,UAAU;AACzC;AAEA,IAAM,mBAAN,MAA2C;AAAA,EACzC,MAAM,QAAQ,QAAoB,OAAc;AAC9C,QAAI,CAAC,UAAU;AACb,iBAAW,eAAe,mBAAmB;AAAA,IAC/C;AAAA,EAGF;AAAA,EAEA,YAAY,MAAgB,SAAqB;AAAA,EAEjD;AAAA,EAEA,UAAU,MAAgB,QAAoB;AAC5C,UAAM,iBAAiB,KAAK,YAAY,KAAK,CAAC,eAAe;AAC3D,aAAO,WAAW,SAAS;AAAA,IAC7B,CAAC;AACD,QAAI,CAAC,gBAAgB;AAAa;AAClC,iBAAa,KAAK;AAAA,MAChB,YAAY,eAAe;AAAA,MAC3B,YAAY;AAAA,QACV,oBAAoB,KAAK;AAAA,QACzB,uBAAuB,KAAK;AAAA,QAC5B,wBAAwB,OAAO;AAAA,QAC/B,0BAA0B,OAAO;AAAA,MACnC;AAAA,IACF,CAAC;AAED,SAAK,cAAc,KAAK,YAAY;AAAA,MAClC,CAAC,eAAe,WAAW,SAAS;AAAA,IACtC;AAEA,iBAAa;AAAA,EACf;AAAA,EAEA,MAAM,QAAoB;AACxB,iBAAa;AAEb,WAAO,qBAAqB,OAAO,MAAM,EAAE;AAAA,EAC7C;AACF;AAEA,IAAO,mBAAQ","names":["import_utils","dayjs"],"ignoreList":[],"sources":["../../src/playwright/reporter/index.ts","../../src/common/utils.ts"],"sourcesContent":["import { printReportMsg, reportFileName } from '@/common/utils';\nimport type { ReportDumpWithAttributes } from '@midscene/core';\nimport { writeDumpReport } from '@midscene/core/utils';\nimport type {\n  FullConfig,\n  FullResult,\n  Reporter,\n  Suite,\n  TestCase,\n  TestResult,\n} from '@playwright/test/reporter';\n\nfunction logger(...message: any[]) {\n  if (process.env.DEBUG === 'true') {\n    console.log('Midscene e2e report:', ...message);\n  }\n}\n\nconst testDataList: Array<ReportDumpWithAttributes> = [];\nlet filename: string;\nfunction updateReport() {\n  const reportPath = writeDumpReport(filename, testDataList);\n  reportPath && printReportMsg(reportPath);\n}\n\nclass MidsceneReporter implements Reporter {\n  async onBegin(config: FullConfig, suite: Suite) {\n    if (!filename) {\n      filename = reportFileName('playwright-merged');\n    }\n    // const suites = suite.allTests();\n    // logger(`Starting the run with ${suites.length} tests`);\n  }\n\n  onTestBegin(test: TestCase, _result: TestResult) {\n    // logger(`Starting test ${test.title}`);\n  }\n\n  onTestEnd(test: TestCase, result: TestResult) {\n    const dumpAnnotation = test.annotations.find((annotation) => {\n      return annotation.type === 'MIDSCENE_DUMP_ANNOTATION';\n    });\n    if (!dumpAnnotation?.description) return;\n    testDataList.push({\n      dumpString: dumpAnnotation.description,\n      attributes: {\n        playwright_test_id: test.id,\n        playwright_test_title: test.title,\n        playwright_test_status: result.status,\n        playwright_test_duration: result.duration,\n      },\n    });\n\n    test.annotations = test.annotations.filter(\n      (annotation) => annotation.type !== 'MIDSCENE_DUMP_ANNOTATION',\n    );\n\n    updateReport();\n  }\n\n  onEnd(result: FullResult) {\n    updateReport();\n\n    logger(`Finished the run: ${result.status}`);\n  }\n}\n\nexport default MidsceneReporter;\n","import type { StaticPage } from '@/playground';\nimport type {\n  ElementTreeNode,\n  PlaywrightParserOpt,\n  UIContext,\n} from '@midscene/core';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  MIDSCENE_USE_VLM_UI_TARS,\n  getAIConfig,\n  getAIConfigInBoolean,\n} from '@midscene/core/env';\nimport { uploadTestInfoToServer } from '@midscene/core/utils';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { ElementInfo } from '@midscene/shared/extractor';\nimport { traverseTree, treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg, resizeImgBase64 } from '@midscene/shared/img';\nimport { assert, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport { WebElementInfo } from '../web-element';\nimport type { WebPage } from './page';\nexport type WebUIContext = UIContext<WebElementInfo> & {\n  url: string;\n};\n\nexport async function parseContextFromWebPage(\n  page: WebPage,\n  _opt?: PlaywrightParserOpt,\n): Promise<WebUIContext> {\n  assert(page, 'page is required');\n  if ((page as StaticPage)._forceUsePageContext) {\n    return await (page as any)._forceUsePageContext();\n  }\n  const url = await page.url();\n  uploadTestInfoToServer({ testUrl: url });\n\n  let screenshotBase64: string;\n  let tree: ElementTreeNode<ElementInfo>;\n\n  await Promise.all([\n    page.screenshotBase64().then((base64) => {\n      screenshotBase64 = base64;\n    }),\n    page.getElementsNodeTree().then(async (treeRoot) => {\n      tree = treeRoot;\n    }),\n  ]);\n\n  const webTree = traverseTree(tree!, (elementInfo) => {\n    const { rect, id, content, attributes, locator, indexId } = elementInfo;\n    return new WebElementInfo({\n      rect,\n      locator,\n      id,\n      content,\n      attributes,\n      indexId,\n    });\n  });\n\n  const elementsInfo = treeToList(webTree);\n\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const size = await page.size();\n\n  if (size.dpr && size.dpr > 1) {\n    // console.time('resizeImgBase64');\n    screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n      width: size.width,\n      height: size.height,\n    });\n    // console.timeEnd('resizeImgBase64');\n  }\n\n  let screenshotBase64WithElementMarker = screenshotBase64;\n  if (!getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS)) {\n    if (_opt?.ignoreMarker) {\n      screenshotBase64WithElementMarker = screenshotBase64;\n    } else {\n      screenshotBase64WithElementMarker = await compositeElementInfoImg({\n        inputImgBase64: screenshotBase64,\n        elementsPositionInfo: elementsPositionInfoWithoutText,\n        size,\n      });\n    }\n  }\n\n  return {\n    content: elementsInfo!,\n    tree: webTree,\n    size,\n    screenshotBase64: screenshotBase64!,\n    screenshotBase64WithElementMarker: screenshotBase64WithElementMarker,\n    url,\n  };\n}\n\nexport function reportFileName(tag = 'web') {\n  const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss-SSS');\n  return `${reportTagName || tag}-${dateTimeInFileName}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  //mcp need use obj format to console msg: https://github.com/modelcontextprotocol/typescript-sdk/issues/244\n  console.log('Midscene - report file updated:', filepath);\n  // For MCP\n  // console.log(`{\"midscene-report-file-updated\": \"${filepath}\"}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\n  'NOT_IMPLEMENTED_AS_DESIGNED';\n"]}
+{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACMA,iBAKO;AACP,mBAAuC;AACvC,uBAAyB;AAEzB,uBAAyC;AACzC,iBAAyD;AACzD,IAAAA,gBAAqC;AACrC,mBAAkB;AAyFX,SAAS,eAAe,MAAM,OAAO;AAC1C,QAAM,oBAAgB,wBAAY,mCAAwB;AAC1D,QAAM,yBAAqB,aAAAC,SAAM,EAAE,OAAO,yBAAyB;AACnE,SAAO,GAAG,iBAAiB,GAAG,IAAI,kBAAkB;AACtD;AAEO,SAAS,eAAe,UAAkB;AAC/C,4BAAO,mCAAmC,QAAQ,EAAE;AACtD;;;ADjHA,IAAAD,gBAAgC;AAUhC,SAAS,UAAU,SAAgB;AACjC,MAAI,QAAQ,IAAI,UAAU,QAAQ;AAChC,YAAQ,IAAI,wBAAwB,GAAG,OAAO;AAAA,EAChD;AACF;AAEA,IAAM,eAAgD,CAAC;AACvD,IAAI;AACJ,SAAS,eAAe;AACtB,QAAM,iBAAa,+BAAgB,UAAU,YAAY;AACzD,gBAAc,eAAe,UAAU;AACzC;AAEA,IAAM,mBAAN,MAA2C;AAAA,EACzC,MAAM,QAAQ,QAAoB,OAAc;AAC9C,QAAI,CAAC,UAAU;AACb,iBAAW,eAAe,mBAAmB;AAAA,IAC/C;AAAA,EAGF;AAAA,EAEA,YAAY,MAAgB,SAAqB;AAAA,EAEjD;AAAA,EAEA,UAAU,MAAgB,QAAoB;AAC5C,UAAM,iBAAiB,KAAK,YAAY,KAAK,CAAC,eAAe;AAC3D,aAAO,WAAW,SAAS;AAAA,IAC7B,CAAC;AACD,QAAI,CAAC,gBAAgB;AAAa;AAClC,iBAAa,KAAK;AAAA,MAChB,YAAY,eAAe;AAAA,MAC3B,YAAY;AAAA,QACV,oBAAoB,KAAK;AAAA,QACzB,uBAAuB,KAAK;AAAA,QAC5B,wBAAwB,OAAO;AAAA,QAC/B,0BAA0B,OAAO;AAAA,MACnC;AAAA,IACF,CAAC;AAED,SAAK,cAAc,KAAK,YAAY;AAAA,MAClC,CAAC,eAAe,WAAW,SAAS;AAAA,IACtC;AAEA,iBAAa;AAAA,EACf;AAAA,EAEA,MAAM,QAAoB;AACxB,iBAAa;AAEb,WAAO,qBAAqB,OAAO,MAAM,EAAE;AAAA,EAC7C;AACF;AAEA,IAAO,mBAAQ","names":["import_utils","dayjs"],"ignoreList":[],"sources":["../../src/playwright/reporter/index.ts","../../src/common/utils.ts"],"sourcesContent":["import { printReportMsg, reportFileName } from '@/common/utils';\nimport type { ReportDumpWithAttributes } from '@midscene/core';\nimport { writeDumpReport } from '@midscene/core/utils';\nimport type {\n  FullConfig,\n  FullResult,\n  Reporter,\n  Suite,\n  TestCase,\n  TestResult,\n} from '@playwright/test/reporter';\n\nfunction logger(...message: any[]) {\n  if (process.env.DEBUG === 'true') {\n    console.log('Midscene e2e report:', ...message);\n  }\n}\n\nconst testDataList: Array<ReportDumpWithAttributes> = [];\nlet filename: string;\nfunction updateReport() {\n  const reportPath = writeDumpReport(filename, testDataList);\n  reportPath && printReportMsg(reportPath);\n}\n\nclass MidsceneReporter implements Reporter {\n  async onBegin(config: FullConfig, suite: Suite) {\n    if (!filename) {\n      filename = reportFileName('playwright-merged');\n    }\n    // const suites = suite.allTests();\n    // logger(`Starting the run with ${suites.length} tests`);\n  }\n\n  onTestBegin(test: TestCase, _result: TestResult) {\n    // logger(`Starting test ${test.title}`);\n  }\n\n  onTestEnd(test: TestCase, result: TestResult) {\n    const dumpAnnotation = test.annotations.find((annotation) => {\n      return annotation.type === 'MIDSCENE_DUMP_ANNOTATION';\n    });\n    if (!dumpAnnotation?.description) return;\n    testDataList.push({\n      dumpString: dumpAnnotation.description,\n      attributes: {\n        playwright_test_id: test.id,\n        playwright_test_title: test.title,\n        playwright_test_status: result.status,\n        playwright_test_duration: result.duration,\n      },\n    });\n\n    test.annotations = test.annotations.filter(\n      (annotation) => annotation.type !== 'MIDSCENE_DUMP_ANNOTATION',\n    );\n\n    updateReport();\n  }\n\n  onEnd(result: FullResult) {\n    updateReport();\n\n    logger(`Finished the run: ${result.status}`);\n  }\n}\n\nexport default MidsceneReporter;\n","import type { StaticPage } from '@/playground';\nimport type {\n  ElementTreeNode,\n  PlaywrightParserOpt,\n  UIContext,\n} from '@midscene/core';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  MIDSCENE_USE_VLM_UI_TARS,\n  getAIConfig,\n  getAIConfigInBoolean,\n} from '@midscene/core/env';\nimport { uploadTestInfoToServer } from '@midscene/core/utils';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { ElementInfo } from '@midscene/shared/extractor';\nimport { traverseTree, treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg, resizeImgBase64 } from '@midscene/shared/img';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport { WebElementInfo } from '../web-element';\nimport type { WebPage } from './page';\nexport type WebUIContext = UIContext<WebElementInfo> & {\n  url: string;\n};\n\nexport async function parseContextFromWebPage(\n  page: WebPage,\n  _opt?: PlaywrightParserOpt,\n): Promise<WebUIContext> {\n  assert(page, 'page is required');\n  if ((page as StaticPage)._forceUsePageContext) {\n    return await (page as any)._forceUsePageContext();\n  }\n  const url = await page.url();\n  uploadTestInfoToServer({ testUrl: url });\n\n  let screenshotBase64: string;\n  let tree: ElementTreeNode<ElementInfo>;\n\n  await Promise.all([\n    page.screenshotBase64().then((base64) => {\n      screenshotBase64 = base64;\n    }),\n    page.getElementsNodeTree().then(async (treeRoot) => {\n      tree = treeRoot;\n    }),\n  ]);\n\n  const webTree = traverseTree(tree!, (elementInfo) => {\n    const { rect, id, content, attributes, locator, indexId } = elementInfo;\n    return new WebElementInfo({\n      rect,\n      locator,\n      id,\n      content,\n      attributes,\n      indexId,\n    });\n  });\n\n  const elementsInfo = treeToList(webTree);\n\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const size = await page.size();\n\n  if (size.dpr && size.dpr > 1) {\n    // console.time('resizeImgBase64');\n    screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n      width: size.width,\n      height: size.height,\n    });\n    // console.timeEnd('resizeImgBase64');\n  }\n\n  let screenshotBase64WithElementMarker = screenshotBase64;\n  if (!getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS)) {\n    if (_opt?.ignoreMarker) {\n      screenshotBase64WithElementMarker = screenshotBase64;\n    } else {\n      screenshotBase64WithElementMarker = await compositeElementInfoImg({\n        inputImgBase64: screenshotBase64,\n        elementsPositionInfo: elementsPositionInfoWithoutText,\n        size,\n      });\n    }\n  }\n\n  return {\n    content: elementsInfo!,\n    tree: webTree,\n    size,\n    screenshotBase64: screenshotBase64!,\n    screenshotBase64WithElementMarker: screenshotBase64WithElementMarker,\n    url,\n  };\n}\n\nexport function reportFileName(tag = 'web') {\n  const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss-SSS');\n  return `${reportTagName || tag}-${dateTimeInFileName}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\n  'NOT_IMPLEMENTED_AS_DESIGNED';\n"]}

package/dist/lib/playwright.js

@@ -683,6 +683,10 @@ var PageTaskExecutor = class {
             const dumpCollector = (dump) => {
               insightDump = dump;
               usage = dump?.taskInfo?.usage;
+              task.log = {
+                dump: insightDump
+              };
+              task.usage = usage;
             };
             this.insight.onceDumpUpdatedFn = dumpCollector;
             const shotTime = Date.now();
@@ -693,6 +697,7 @@ var PageTaskExecutor = class {
               screenshot: pageContext.screenshotBase64,
               timing: "before locate"
             };
+            task.recorder = [recordItem];
             const cachePrompt = param.prompt;
             const locateCache = cacheGroup?.matchCache(
               pageContext,
@@ -736,9 +741,6 @@ var PageTaskExecutor = class {
               });
             }
             if (!element) {
-              task.log = {
-                dump: insightDump
-              };
               throw new Error(`Element not found: ${param.prompt}`);
             }
             return {
@@ -746,15 +748,10 @@ var PageTaskExecutor = class {
                 element
               },
               pageContext,
-              log: {
-                dump: insightDump
-              },
               cache: {
                 hit: cacheHitFlag
               },
-              recorder: [recordItem],
-              aiCost,
-              usage
+              aiCost
             };
           }
         };
@@ -1669,7 +1666,7 @@ function reportFileName(tag = "web") {
   return `${reportTagName || tag}-${dateTimeInFileName}`;
 }
 function printReportMsg(filepath) {
-  console.log("Midscene - report file updated:", filepath);
+  (0, import_utils9.logMsg)(`Midscene - report file updated: ${filepath}`);
 }
 
 // src/common/agent.ts