misoai-web 1.0.3 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +349 -5
  2. package/dist/es/agent.js +2676 -0
  3. package/dist/es/agent.js.map +1 -0
  4. package/dist/es/bridge-mode-browser.js +955 -0
  5. package/dist/es/bridge-mode-browser.js.map +1 -0
  6. package/dist/es/bridge-mode.js +3037 -0
  7. package/dist/es/bridge-mode.js.map +1 -0
  8. package/dist/es/chrome-extension.js +3424 -0
  9. package/dist/es/chrome-extension.js.map +1 -0
  10. package/dist/es/index.js +3292 -0
  11. package/dist/es/index.js.map +1 -0
  12. package/dist/es/midscene-playground.js +3012 -0
  13. package/dist/es/midscene-playground.js.map +1 -0
  14. package/dist/es/midscene-server.js +247 -0
  15. package/dist/es/midscene-server.js.map +1 -0
  16. package/dist/es/playground.js +2783 -0
  17. package/dist/es/playground.js.map +1 -0
  18. package/dist/es/playwright-report.js +120 -0
  19. package/dist/es/playwright-report.js.map +1 -0
  20. package/dist/es/playwright.js +3237 -0
  21. package/dist/es/playwright.js.map +1 -0
  22. package/dist/es/puppeteer-agent-launcher.js +3187 -0
  23. package/dist/es/puppeteer-agent-launcher.js.map +1 -0
  24. package/dist/es/puppeteer.js +3034 -0
  25. package/dist/es/puppeteer.js.map +1 -0
  26. package/dist/es/ui-utils.js +106 -0
  27. package/dist/es/ui-utils.js.map +1 -0
  28. package/dist/es/utils.js +197 -0
  29. package/dist/es/utils.js.map +1 -0
  30. package/dist/es/yaml.js +351 -0
  31. package/dist/es/yaml.js.map +1 -0
  32. package/dist/lib/agent.js +2691 -0
  33. package/dist/lib/agent.js.map +1 -0
  34. package/dist/lib/bridge-mode-browser.js +989 -0
  35. package/dist/lib/bridge-mode-browser.js.map +1 -0
  36. package/dist/lib/bridge-mode.js +3057 -0
  37. package/dist/lib/bridge-mode.js.map +1 -0
  38. package/dist/lib/chrome-extension.js +3441 -0
  39. package/dist/lib/chrome-extension.js.map +1 -0
  40. package/dist/lib/index.js +3308 -0
  41. package/dist/lib/index.js.map +1 -0
  42. package/dist/lib/midscene-playground.js +3016 -0
  43. package/dist/lib/midscene-playground.js.map +1 -0
  44. package/dist/lib/midscene-server.js +273 -0
  45. package/dist/lib/midscene-server.js.map +1 -0
  46. package/dist/lib/playground.js +2802 -0
  47. package/dist/lib/playground.js.map +1 -0
  48. package/dist/lib/playwright-report.js +148 -0
  49. package/dist/lib/playwright-report.js.map +1 -0
  50. package/dist/lib/playwright.js +3254 -0
  51. package/dist/lib/playwright.js.map +1 -0
  52. package/dist/lib/puppeteer-agent-launcher.js +3200 -0
  53. package/dist/lib/puppeteer-agent-launcher.js.map +1 -0
  54. package/dist/lib/puppeteer.js +3045 -0
  55. package/dist/lib/puppeteer.js.map +1 -0
  56. package/dist/lib/ui-utils.js +137 -0
  57. package/dist/lib/ui-utils.js.map +1 -0
  58. package/dist/lib/utils.js +235 -0
  59. package/dist/lib/utils.js.map +1 -0
  60. package/dist/lib/yaml.js +390 -0
  61. package/dist/lib/yaml.js.map +1 -0
  62. package/dist/types/agent.d.ts +290 -0
  63. package/dist/types/bridge-mode-browser.d.ts +9 -0
  64. package/dist/types/bridge-mode.d.ts +40 -0
  65. package/dist/types/browser-a1877d18.d.ts +37 -0
  66. package/dist/types/chrome-extension.d.ts +18 -0
  67. package/dist/types/index.d.ts +16 -0
  68. package/dist/types/midscene-playground.d.ts +2 -0
  69. package/dist/types/midscene-server.d.ts +31 -0
  70. package/dist/types/page-663ece08.d.ts +333 -0
  71. package/dist/types/playground.d.ts +17 -0
  72. package/dist/types/playwright-report.d.ts +11 -0
  73. package/dist/types/playwright.d.ts +87 -0
  74. package/dist/types/puppeteer-agent-launcher.d.ts +40 -0
  75. package/dist/types/puppeteer.d.ts +17 -0
  76. package/dist/types/ui-utils.d.ts +14 -0
  77. package/dist/types/utils-badc824e.d.ts +34 -0
  78. package/dist/types/utils.d.ts +8 -0
  79. package/dist/types/yaml.d.ts +15 -0
  80. package/iife-script/htmlElement.js +99 -37
  81. package/iife-script/htmlElementDebug.js +92 -9
  82. package/package.json +2 -2
@@ -0,0 +1 @@
1
+ {"version":3,"mappings":";AAQA,SAAS,wCAAwC;AACjD,SAAS,8BAA8B;AACvC,SAAS,0BAA0B,mBAAmB;AAEtD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AAEhC,SAAS,QAAQ,QAAQ,YAAY;AACrC,OAAO,WAAW;AAoEX,SAAS,eAAe,MAAM,OAAO;AAC1C,QAAM,gBAAgB,YAAY,wBAAwB;AAC1D,QAAM,qBAAqB,MAAM,EAAE,OAAO,qBAAqB;AAE/D,QAAM,WAAW,KAAK,EAAE,UAAU,GAAG,CAAC;AACtC,SAAO,GAAG,iBAAiB,GAAG,IAAI,kBAAkB,IAAI,QAAQ;AAClE;AAEO,SAAS,eAAe,UAAkB;AAC/C,SAAO,mCAAmC,QAAQ,EAAE;AACtD;AA0DO,SAAS,gCAAgC,KAAa;AAC3D,SAAO,IAAI,QAAQ,kBAAkB,GAAG;AAC1C;;;ACzJA,SAAS,uBAAuB;AAUhC,SAAS,UAAU,SAAgB;AACjC,MAAI,QAAQ,IAAI,UAAU,QAAQ;AAChC,YAAQ,IAAI,wBAAwB,GAAG,OAAO;AAAA,EAChD;AACF;AAEA,IAAM,eAAgD,CAAC;AACvD,IAAI;AACJ,IAAM,sBAA2C,oBAAI,IAAI;AAEzD,SAAS,kBAAkB,WAA2B;AACpD,MAAI,CAAC,oBAAoB,IAAI,SAAS,GAAG;AAGvC,UAAM,UAAU,cAAc,gCAAgC,SAAS,CAAC;AACxE,UAAM,oBAAoB,eAAe,OAAO;AAChD,wBAAoB,IAAI,WAAW,iBAAiB;AAAA,EACtD;AACA,SAAO,oBAAoB,IAAI,SAAS;AAC1C;AAEA,SAAS,aAAa,MAA6B,QAAiB;AAClE,MAAI,SAAS,YAAY;AAEvB,UAAM,WAAW,aAAa;AAAA,MAC5B,CAAC,SAAS,KAAK,YAAY,uBAAuB;AAAA,IACpD;AAEA,QAAI,UAAU;AAEZ,YAAM,iBAAiB;AAAA,QACrB,SAAS,YAAY;AAAA,MACvB;AAEA,YAAM,aAAa,gBAAgB,gBAAgB,CAAC,QAAQ,CAAC;AAC7D,oBAAc,eAAe,UAAU;AAAA,IACzC;AAAA,EACF,WAAW,SAAS,UAAU;AAE5B,QAAI,CAAC,gBAAgB;AACnB,uBAAiB,eAAe,mBAAmB;AAAA,IACrD;AAEA,UAAM,aAAa,gBAAgB,gBAAgB,YAAY;AAC/D,kBAAc,eAAe,UAAU;AAAA,EACzC,OAAO;AACL,UAAM,IAAI;AAAA,MACR,+CAA+C,IAAI;AAAA,IACrD;AAAA,EACF;AACF;AAEA,SAAS,QAAQ,cAAsB;AACrC,MAAI,CAAC,cAAc;AACjB,WAAO;AAAA,EACT;AAEA,MAAI,iBAAiB,YAAY,iBAAiB,YAAY;AAC5D,UAAM,IAAI;AAAA,MACR,+CAA+C,YAAY;AAAA,IAC7D;AAAA,EACF;AAEA,SAAO;AACT;AAEA,IAAM,mBAAN,MAA2C;AAAA,EAGzC,MAAM,QAAQ,QAAoB,OAAc;AAC9C,UAAM,eAAe,OAAO,WAAW,CAAC,IAAI,CAAC,GAAG;AAEhD,SAAK,OAAO,QAAQ,YAAY;AAAA,EAIlC;AAAA,EAEA,YAAY,MAAgB,SAAqB;AAAA,EAEjD;AAAA,EAEA,UAAU,MAAgB,QAAoB;AAC5C,UAAM,iBAAiB,KAAK,YAAY,KAAK,CAAC,eAAe;AAC3D,aAAO,WAAW,SAAS;AAAA,IAC7B,CAAC;AACD,QAAI,CAAC,gBAAgB;AAAa;AAClC,UAAM,QAAQ,OAAO,QAAQ,WAAW,OAAO,KAAK,MAAM;AAC1D,UAAM,SAAS,GAAG,KAAK,EAAE,GAAG,KAAK;AACjC,UAAM,WAAqC;AAAA,MACzC,YAAY,eAAe;AAAA,MAC3B,YAAY;AAAA,QACV,oBAAoB;AAAA,QACpB,uBAAuB,GAAG,KAAK,KAAK,GAAG,KAAK;AAAA,QAC5C,wBAAwB,OAAO;AAAA,QAC/B,0BAA0B,OAAO;AAAA,MACnC;AAAA,IACF;AAEA,iBAAa,KAAK,QAAQ;AAE1B,iBAAa,KAAK,MAAO,MAAM;AAE/B,SAAK,cAAc,KAAK,YAAY;AAAA,MAClC,CAAC,eAAe,WAAW,SAAS;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,MAAM,QAAoB;AACxB,iBAAa,KAAK,IAAK;AAEvB,WAAO,qBAAqB,OAAO,MAAM,EAAE;AAAA,EAC7C;AACF;AAEA,IAAO,mBAAQ","names":[],"ignoreList":[],"sources":["../../src/common/utils.ts","../../src/playwright/reporter/index.ts"],"sourcesContent":["import type { StaticPage } from '@/playground';\r\nimport type {\r\n BaseElement,\r\n ElementTreeNode,\r\n PlanningLocateParam,\r\n PlaywrightParserOpt,\r\n UIContext,\r\n} from 'misoai-core';\r\nimport { elementByPositionWithElementInfo } from 'misoai-core/ai-model';\r\nimport { uploadTestInfoToServer } from 'misoai-core/utils';\r\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from 'misoai-shared/env';\r\nimport type { ElementInfo } from 'misoai-shared/extractor';\r\nimport {\r\n generateElementByPosition,\r\n getNodeFromCacheList,\r\n traverseTree,\r\n treeToList,\r\n} from 'misoai-shared/extractor';\r\nimport { resizeImgBase64 } from 'misoai-shared/img';\r\nimport type { DebugFunction } from 'misoai-shared/logger';\r\nimport { assert, logMsg, uuid } from 'misoai-shared/utils';\r\nimport dayjs from 'dayjs';\r\nimport type { Page as PlaywrightPage } from 'playwright';\r\nimport type { Page as PuppeteerPage } from 'puppeteer';\r\nimport { WebElementInfo } from '../web-element';\r\nimport type { WebPage } from './page';\r\n\r\nexport type WebUIContext = UIContext<WebElementInfo> & {\r\n url: string;\r\n};\r\n\r\nexport async function parseContextFromWebPage(\r\n page: WebPage,\r\n _opt?: PlaywrightParserOpt,\r\n): Promise<WebUIContext> {\r\n assert(page, 'page is required');\r\n if ((page as StaticPage)._forceUsePageContext) {\r\n return await (page as any)._forceUsePageContext();\r\n }\r\n const url = await page.url();\r\n uploadTestInfoToServer({ testUrl: url });\r\n\r\n let screenshotBase64: string;\r\n let tree: ElementTreeNode<ElementInfo>;\r\n\r\n await Promise.all([\r\n page.screenshotBase64().then((base64) => {\r\n screenshotBase64 = base64;\r\n }),\r\n page.getElementsNodeTree().then(async (treeRoot) => {\r\n tree = treeRoot;\r\n }),\r\n ]);\r\n\r\n const webTree = traverseTree(tree!, (elementInfo) => {\r\n const { rect, id, content, attributes, locator, indexId } = elementInfo;\r\n return new WebElementInfo({\r\n rect,\r\n locator,\r\n id,\r\n content,\r\n attributes,\r\n indexId,\r\n });\r\n });\r\n\r\n assert(screenshotBase64!, 'screenshotBase64 is required');\r\n\r\n const elementsInfo = treeToList(webTree);\r\n const size = await page.size();\r\n\r\n if (size.dpr && size.dpr > 1) {\r\n // console.time('resizeImgBase64');\r\n screenshotBase64 = await resizeImgBase64(screenshotBase64, {\r\n width: size.width,\r\n height: size.height,\r\n });\r\n // console.timeEnd('resizeImgBase64');\r\n }\r\n\r\n return {\r\n content: elementsInfo!,\r\n tree: webTree,\r\n size,\r\n screenshotBase64: screenshotBase64!,\r\n url,\r\n };\r\n}\r\n\r\nexport function reportFileName(tag = 'web') {\r\n const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\r\n const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\r\n // ensure uniqueness at the same time\r\n const uniqueId = uuid().substring(0, 8);\r\n return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\r\n}\r\n\r\nexport function printReportMsg(filepath: string) {\r\n logMsg(`Midscene - report file updated: ${filepath}`);\r\n}\r\n\r\n/**\r\n * Get the current execution file name\r\n * @returns The name of the current execution file\r\n */\r\nexport function getCurrentExecutionFile(trace?: string): string | false {\r\n const error = new Error();\r\n const stackTrace = trace || error.stack;\r\n const pkgDir = process.cwd() || '';\r\n if (stackTrace) {\r\n const stackLines = stackTrace.split('\\n');\r\n for (const line of stackLines) {\r\n if (\r\n line.includes('.spec.') ||\r\n line.includes('.test.') ||\r\n line.includes('.ts') ||\r\n line.includes('.js')\r\n ) {\r\n const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\r\n if (match?.[1]) {\r\n const targetFileName = match[1]\r\n .replace(pkgDir, '')\r\n .trim()\r\n .replace('at ', '');\r\n return targetFileName;\r\n }\r\n }\r\n }\r\n }\r\n return false;\r\n}\r\n\r\nconst testFileIndex = new Map<string, number>();\r\n\r\nexport function generateCacheId(fileName?: string): string {\r\n let taskFile = fileName || getCurrentExecutionFile();\r\n if (!taskFile) {\r\n taskFile = uuid();\r\n console.warn(\r\n 'Midscene - using random UUID for cache id. Cache may be invalid.',\r\n );\r\n }\r\n\r\n if (testFileIndex.has(taskFile)) {\r\n const currentIndex = testFileIndex.get(taskFile);\r\n if (currentIndex !== undefined) {\r\n testFileIndex.set(taskFile, currentIndex + 1);\r\n }\r\n } else {\r\n testFileIndex.set(taskFile, 1);\r\n }\r\n return `${taskFile}-${testFileIndex.get(taskFile)}`;\r\n}\r\n\r\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\r\n 'NOT_IMPLEMENTED_AS_DESIGNED';\r\n\r\nexport function replaceIllegalPathCharsAndSpace(str: string) {\r\n return str.replace(/[/\\\\:*?\"<>| ]/g, '-');\r\n}\r\n\r\nexport function forceClosePopup(\r\n page: PuppeteerPage | PlaywrightPage,\r\n debug: DebugFunction,\r\n) {\r\n page.on('popup', async (popup) => {\r\n if (!popup) {\r\n console.warn('got a popup event, but the popup is not ready yet, skip');\r\n return;\r\n }\r\n const url = await (popup as PuppeteerPage).url();\r\n console.log(`Popup opened: ${url}`);\r\n if (!(popup as PuppeteerPage).isClosed()) {\r\n try {\r\n await (popup as PuppeteerPage).close(); // Close the newly opened TAB\r\n } catch (error) {\r\n debug(`failed to close popup ${url}, error: ${error}`);\r\n }\r\n } else {\r\n debug(`popup is already closed, skip close ${url}`);\r\n }\r\n\r\n if (!page.isClosed()) {\r\n try {\r\n await page.goto(url);\r\n } catch (error) {\r\n debug(`failed to goto ${url}, error: ${error}`);\r\n }\r\n } else {\r\n debug(`page is already closed, skip goto ${url}`);\r\n }\r\n });\r\n}\r\n\r\nexport function matchElementFromPlan(\r\n planLocateParam: PlanningLocateParam,\r\n tree: ElementTreeNode<BaseElement>,\r\n) {\r\n if (!planLocateParam) {\r\n return undefined;\r\n }\r\n if (planLocateParam.id) {\r\n return getNodeFromCacheList(planLocateParam.id);\r\n }\r\n\r\n if (planLocateParam.bbox) {\r\n const centerPosition = {\r\n x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\r\n y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\r\n };\r\n let element = elementByPositionWithElementInfo(tree, centerPosition);\r\n\r\n if (!element) {\r\n element = generateElementByPosition(centerPosition) as BaseElement;\r\n }\r\n\r\n return element;\r\n }\r\n\r\n return undefined;\r\n}\r\n","import {\r\n printReportMsg,\r\n replaceIllegalPathCharsAndSpace,\r\n reportFileName,\r\n} from '@/common/utils';\r\nimport type { ReportDumpWithAttributes } from 'misoai-core';\r\nimport { writeDumpReport } from 'misoai-core/utils';\r\nimport type {\r\n FullConfig,\r\n FullResult,\r\n Reporter,\r\n Suite,\r\n TestCase,\r\n TestResult,\r\n} from '@playwright/test/reporter';\r\n\r\nfunction logger(...message: any[]) {\r\n if (process.env.DEBUG === 'true') {\r\n console.log('Midscene e2e report:', ...message);\r\n }\r\n}\r\n\r\nconst testDataList: Array<ReportDumpWithAttributes> = [];\r\nlet mergedFilename: string;\r\nconst testTitleToFilename: Map<string, string> = new Map();\r\n\r\nfunction getStableFilename(testTitle: string): string {\r\n if (!testTitleToFilename.has(testTitle)) {\r\n // use reportFileName to generate the base filename\r\n // only replace the illegal characters in the file system: /, \\, :, *, ?, \", <, >, |\r\n const baseTag = `playwright-${replaceIllegalPathCharsAndSpace(testTitle)}`;\r\n const generatedFilename = reportFileName(baseTag);\r\n testTitleToFilename.set(testTitle, generatedFilename);\r\n }\r\n return testTitleToFilename.get(testTitle)!;\r\n}\r\n\r\nfunction updateReport(mode: 'merged' | 'separate', testId?: string) {\r\n if (mode === 'separate') {\r\n // in separate mode, find the data for the corresponding testID and generate a separate report\r\n const testData = testDataList.find(\r\n (data) => data.attributes?.playwright_test_id === testId,\r\n );\r\n\r\n if (testData) {\r\n // use the stable filename\r\n const stableFilename = getStableFilename(\r\n testData.attributes?.playwright_test_title,\r\n );\r\n\r\n const reportPath = writeDumpReport(stableFilename, [testData]);\r\n reportPath && printReportMsg(reportPath);\r\n }\r\n } else if (mode === 'merged') {\r\n // in merged mode, write all test data into one file\r\n if (!mergedFilename) {\r\n mergedFilename = reportFileName('playwright-merged');\r\n }\r\n\r\n const reportPath = writeDumpReport(mergedFilename, testDataList);\r\n reportPath && printReportMsg(reportPath);\r\n } else {\r\n throw new Error(\r\n `Unknown reporter type in playwright config: ${mode}, only support 'merged' or 'separate'`,\r\n );\r\n }\r\n}\r\n\r\nfunction getMode(reporterType: string) {\r\n if (!reporterType) {\r\n return 'merged';\r\n }\r\n\r\n if (reporterType !== 'merged' && reporterType !== 'separate') {\r\n throw new Error(\r\n `Unknown reporter type in playwright config: ${reporterType}, only support 'merged' or 'separate'`,\r\n );\r\n }\r\n\r\n return reporterType;\r\n}\r\n\r\nclass MidsceneReporter implements Reporter {\r\n mode?: 'merged' | 'separate';\r\n\r\n async onBegin(config: FullConfig, suite: Suite) {\r\n const reporterType = config.reporter?.[1]?.[1]?.type;\r\n\r\n this.mode = getMode(reporterType);\r\n\r\n // const suites = suite.allTests();\r\n // logger(`Starting the run with ${suites.length} tests`);\r\n }\r\n\r\n onTestBegin(test: TestCase, _result: TestResult) {\r\n // logger(`Starting test ${test.title}`);\r\n }\r\n\r\n onTestEnd(test: TestCase, result: TestResult) {\r\n const dumpAnnotation = test.annotations.find((annotation) => {\r\n return annotation.type === 'MIDSCENE_DUMP_ANNOTATION';\r\n });\r\n if (!dumpAnnotation?.description) return;\r\n const retry = result.retry ? `(retry #${result.retry})` : '';\r\n const testId = `${test.id}${retry}`;\r\n const testData: ReportDumpWithAttributes = {\r\n dumpString: dumpAnnotation.description,\r\n attributes: {\r\n playwright_test_id: testId,\r\n playwright_test_title: `${test.title}${retry}`,\r\n playwright_test_status: result.status,\r\n playwright_test_duration: result.duration,\r\n },\r\n };\r\n\r\n testDataList.push(testData);\r\n\r\n updateReport(this.mode!, testId);\r\n\r\n test.annotations = test.annotations.filter(\r\n (annotation) => annotation.type !== 'MIDSCENE_DUMP_ANNOTATION',\r\n );\r\n }\r\n\r\n onEnd(result: FullResult) {\r\n updateReport(this.mode!);\r\n\r\n logger(`Finished the run: ${result.status}`);\r\n }\r\n}\r\n\r\nexport default MidsceneReporter;\r\n"]}