misoai-web 1.0.2 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +8 -8
  3. package/bin/midscene-playground +2 -2
  4. package/dist/es/agent.js +167 -44
  5. package/dist/es/agent.js.map +1 -1
  6. package/dist/es/bridge-mode-browser.js +64 -17
  7. package/dist/es/bridge-mode-browser.js.map +1 -1
  8. package/dist/es/bridge-mode.js +169 -46
  9. package/dist/es/bridge-mode.js.map +1 -1
  10. package/dist/es/chrome-extension.js +229 -59
  11. package/dist/es/chrome-extension.js.map +1 -1
  12. package/dist/es/index.js +183 -45
  13. package/dist/es/index.js.map +1 -1
  14. package/dist/es/midscene-playground.js +173 -44
  15. package/dist/es/midscene-playground.js.map +1 -1
  16. package/dist/es/midscene-server.js.map +1 -1
  17. package/dist/es/playground.js +173 -44
  18. package/dist/es/playground.js.map +1 -1
  19. package/dist/es/playwright-report.js.map +1 -1
  20. package/dist/es/playwright.js +183 -45
  21. package/dist/es/playwright.js.map +1 -1
  22. package/dist/es/puppeteer-agent-launcher.js +183 -45
  23. package/dist/es/puppeteer-agent-launcher.js.map +1 -1
  24. package/dist/es/puppeteer.js +183 -45
  25. package/dist/es/puppeteer.js.map +1 -1
  26. package/dist/es/ui-utils.js.map +1 -1
  27. package/dist/es/utils.js.map +1 -1
  28. package/dist/es/yaml.js +21 -3
  29. package/dist/es/yaml.js.map +1 -1
  30. package/dist/lib/agent.js +167 -44
  31. package/dist/lib/agent.js.map +1 -1
  32. package/dist/lib/bridge-mode-browser.js +64 -17
  33. package/dist/lib/bridge-mode-browser.js.map +1 -1
  34. package/dist/lib/bridge-mode.js +169 -46
  35. package/dist/lib/bridge-mode.js.map +1 -1
  36. package/dist/lib/chrome-extension.js +229 -59
  37. package/dist/lib/chrome-extension.js.map +1 -1
  38. package/dist/lib/index.js +181 -46
  39. package/dist/lib/index.js.map +1 -1
  40. package/dist/lib/midscene-playground.js +173 -44
  41. package/dist/lib/midscene-playground.js.map +1 -1
  42. package/dist/lib/midscene-server.js.map +1 -1
  43. package/dist/lib/playground.js +173 -44
  44. package/dist/lib/playground.js.map +1 -1
  45. package/dist/lib/playwright-report.js.map +1 -1
  46. package/dist/lib/playwright.js +181 -46
  47. package/dist/lib/playwright.js.map +1 -1
  48. package/dist/lib/puppeteer-agent-launcher.js +181 -46
  49. package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
  50. package/dist/lib/puppeteer.js +181 -46
  51. package/dist/lib/puppeteer.js.map +1 -1
  52. package/dist/lib/ui-utils.js.map +1 -1
  53. package/dist/lib/utils.js.map +1 -1
  54. package/dist/lib/yaml.js +21 -3
  55. package/dist/lib/yaml.js.map +1 -1
  56. package/dist/types/agent.d.ts +16 -6
  57. package/dist/types/bridge-mode-browser.d.ts +2 -2
  58. package/dist/types/bridge-mode.d.ts +2 -2
  59. package/dist/types/{browser-d447695b.d.ts → browser-a1877d18.d.ts} +1 -1
  60. package/dist/types/chrome-extension.d.ts +2 -2
  61. package/dist/types/index.d.ts +1 -1
  62. package/dist/types/midscene-server.d.ts +1 -1
  63. package/dist/types/{page-b8ada1f3.d.ts → page-663ece08.d.ts} +41 -30
  64. package/dist/types/playground.d.ts +2 -2
  65. package/dist/types/playwright.d.ts +1 -1
  66. package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
  67. package/dist/types/puppeteer.d.ts +1 -1
  68. package/dist/types/utils.d.ts +1 -1
  69. package/dist/types/yaml.d.ts +1 -1
  70. package/iife-script/htmlElement.js +99 -37
  71. package/iife-script/htmlElementDebug.js +92 -9
  72. package/package.json +23 -24
@@ -1 +1 @@
1
- {"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,qBAAwD;AAExD,uBAAqB;;;ACKrB,sBAAiD;AACjD,mBAAuC;AACvC,iBAAsD;AAEtD,uBAKO;AACP,iBAAgC;AAEhC,IAAAA,gBAAqC;AACrC,mBAAkB;AAqIX,IAAM,yCACX;;;ADtJF,IAAAA,gBAA0B;AAC1B,uBAAuC;AACvC,IAAAC,cAAiC;AACjC,IAAAD,gBAA4B;AAC5B,kBAAiB;AACjB,oBAAmB;AACnB,qBAAoB;AAIpB,IAAM,cAAc;AAGpB,IAAM,eAAe,CAAC,KAAU,KAAU,KAAU,SAAc;AAChE,UAAQ,MAAM,GAAG;AACjB,MAAI,OAAO,GAAG,EAAE,KAAK;AAAA,IACnB,OAAO,IAAI;AAAA,EACb,CAAC;AACH;AAEA,IAAM,QAAQ,YAAY;AACxB,MAAI,CAAC,2BAAa;AAChB,UAAM,EAAE,OAAO,IAAI,cAAAE,QAAO,OAAO;AAEjC,QAAI,QAAQ;AACV,wCAAiB,MAAM;AAAA,IACzB;AAAA,EACF;AACF;AAEA,IAAqB,mBAArB,MAAsC;AAAA,EAcpC,YACE,WACA,YACA,YACA;AACA,SAAK,UAAM,eAAAC,SAAQ;AACnB,SAAK,aAAS,yBAAU;AACxB,SAAK,YAAY;AACjB,SAAK,aAAa;AAClB,SAAK,aAAa;AAClB,SAAK,mBAAmB,CAAC;AACzB,UAAM;AAAA,EACR;AAAA,EAEA,gBAAgBC,OAAc;AAC5B,eAAO,uBAAK,KAAK,QAAQ,GAAGA,KAAI,OAAO;AAAA,EACzC;AAAA,EAEA,gBAAgBA,OAAc,SAAiB;AAC7C,UAAM,UAAU,KAAK,gBAAgBA,KAAI;AACzC,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,sCAAc,SAAS,OAAO;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,MAAe;AAC1B,SAAK,OAAO,QAAQ;AACpB,SAAK,IAAI,IAAI,YAAY;AAEzB,SAAK,IAAI;AAAA,UACP,YAAAC,SAAK;AAAA,QACH,QAAQ;AAAA,QACR,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,SAAK,IAAI,IAAI,eAAW,YAAAA,SAAK,GAAG,OAAO,KAAK,QAAQ;AAElD,UAAI,KAAK;AAAA,QACP,QAAQ;AAAA,MACV,CAAC;AAAA,IACH,CAAC;AAMD,SAAK,IAAI,IAAI,kBAAkB,OAAO,KAAK,QAAQ;AACjD,YAAM,EAAE,MAAAD,MAAK,IAAI,IAAI;AACrB,YAAM,cAAc,KAAK,gBAAgBA,KAAI;AAE7C,UAAI,KAAC,2BAAW,WAAW,GAAG;AAC5B,eAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,UAC1B,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAEA,YAAM,cAAU,6BAAa,aAAa,MAAM;AAChD,UAAI,KAAK;AAAA,QACP;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAED,SAAK,IAAI,IAAI,iCAA6B,YAAAC,SAAK,GAAG,OAAO,KAAK,QAAQ;AACpE,YAAM,EAAE,UAAU,IAAI,IAAI;AAC1B,UAAI,KAAK;AAAA,QACP,KAAK,KAAK,iBAAiB,SAAS,KAAK;AAAA,MAC3C,CAAC;AAAA,IACH,CAAC;AAID,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAF,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,UAAU,IAAI,KAAK;AAEzB,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,cAAMC,YAAO,+BAAW;AACxB,aAAK,gBAAgBA,OAAM,OAAO;AAClC,eAAO,IAAI,KAAK;AAAA,UACd,UAAU,eAAeA,KAAI;AAAA,UAC7B,MAAAA;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAD,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,MAAM,QAAQ,WAAW,UAAU,IAAI,IAAI;AAE5D,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM;AACT,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,QAAQ;AACX,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAGA,cAAM,OAAO,IAAI,KAAK,UAAU,OAAO;AACvC,cAAM,QAAQ,IAAI,KAAK,WAAW,IAAI;AAEtC,YAAI,WAAW;AACb,eAAK,iBAAiB,SAAS,IAAI;AAEnC,gBAAM,iBAAiB,CAAC,QAAgB;AACtC,iBAAK,iBAAiB,SAAS,IAAI;AAAA,UACrC;AAAA,QACF;AAEA,cAAM,WAMF;AAAA,UACF,QAAQ;AAAA,UACR,MAAM;AAAA,UACN,OAAO;AAAA,UACP,YAAY;AAAA,UACZ;AAAA,QACF;AAEA,cAAM,YAAY,KAAK,IAAI;AAC3B,YAAI;AACF,cAAI,SAAS,WAAW;AACtB,qBAAS,SAAS,MAAM,MAAM,QAAQ,MAAM;AAAA,UAC9C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,MAAM;AAAA,UAC/C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,QAAQ,QAAW;AAAA,cACxD,iBAAiB;AAAA,YACnB,CAAC;AAAA,UACH,WAAW,SAAS,SAAS;AAC3B,qBAAS,SAAS,MAAM,MAAM,MAAM,QAAQ;AAAA,cAC1C;AAAA,YACF,CAAC;AAAA,UACH,OAAO;AACL,qBAAS,QAAQ,iBAAiB,IAAI;AAAA,UACxC;AAAA,QACF,SAAS,OAAY;AACnB,cAAI,CAAC,MAAM,QAAQ,SAAS,sCAAsC,GAAG;AACnE,qBAAS,QAAQ,MAAM;AAAA,UACzB;AAAA,QACF;AAEA,YAAI;AACF,mBAAS,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AACjD,mBAAS,aAAa,MAAM,iBAAiB,KAAK;AAElD,gBAAM,oBAAoB;AAAA,QAC5B,SAAS,OAAY;AACnB,kBAAQ;AAAA,YACN,qCAAqC,SAAS,KAAK,MAAM,OAAO;AAAA,UAClE;AAAA,QACF;AAEA,YAAI,KAAK,QAAQ;AACjB,cAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,YAAI,SAAS,OAAO;AAClB,kBAAQ;AAAA,YACN,+BAA+B,QAAQ,kBAAkB,SAAS,KAAK,SAAS,KAAK;AAAA,UACvF;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,6BAA6B,QAAQ,kBAAkB,SAAS;AAAA,UAClE;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAA,QAAQ,KAAK,EAAE,OAAO,MAAM,CAAC;AAAA,MAC7B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,IAAI,IAAI;AAEzB,YAAI,CAAC,YAAY,OAAO,aAAa,UAAU;AAC7C,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI;AACF,4CAAiB,QAAQ;AAEzB,iBAAO,IAAI,KAAK;AAAA,YACd,QAAQ;AAAA,YACR,SAAS;AAAA,UACX,CAAC;AAAA,QACH,SAAS,OAAY;AACnB,kBAAQ,MAAM,+BAA+B,MAAM,OAAO,EAAE;AAC5D,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO,+BAA+B,MAAM,OAAO;AAAA,UACrD,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,QAAI,KAAK,YAAY;AACnB,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAE9B,YAAI,SAAS,aAAa;AAAA,MAC5B,CAAC;AAED,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAC9B,cAAM,oBAAgB,uBAAK,KAAK,YAAa,IAAI,IAAI;AACrD,gBAAI,2BAAW,aAAa,GAAG;AAC7B,cAAI,SAAS,aAAa;AAAA,QAC5B,OAAO;AACL,cAAI,aAAS,uBAAK,KAAK,YAAa,YAAY,CAAC;AAAA,QACnD;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAMG,QAAO,KAAK;AAClB,WAAK,SAAS,KAAK,IAAI,OAAOA,OAAM,MAAM;AACxC,gBAAQ,IAAI;AAAA,MACd,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AAEN,QAAI,KAAK,QAAQ;AACf,aAAO,KAAK,OAAO,MAAM;AAAA,IAC3B;AAAA,EACF;AACF","names":["import_utils","import_env","dotenv","express","uuid","cors","port"],"ignoreList":[],"sources":["../../src/playground/server.ts","../../src/common/utils.ts"],"sourcesContent":["import { randomUUID } from 'node:crypto';\nimport { existsSync, readFileSync, writeFileSync } from 'node:fs';\nimport type { Server } from 'node:http';\nimport { join } from 'node:path';\nimport { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '@/common/utils';\nimport { getTmpDir } from 'misoai-core/utils';\nimport { PLAYGROUND_SERVER_PORT } from 'misoai-shared/constants';\nimport { overrideAIConfig } from 'misoai-shared/env';\nimport { ifInBrowser } from 'misoai-shared/utils';\nimport cors from 'cors';\nimport dotenv from 'dotenv';\nimport express from 'express';\nimport type { PageAgent } from '../common/agent';\nimport type { AbstractPage } from '../page';\n\nconst defaultPort = PLAYGROUND_SERVER_PORT;\n// const staticPath = join(__dirname, '../../static');\n\nconst errorHandler = (err: any, req: any, res: any, next: any) => {\n console.error(err);\n res.status(500).json({\n error: err.message,\n });\n};\n\nconst setup = async () => {\n if (!ifInBrowser) {\n const { parsed } = dotenv.config();\n\n if (parsed) {\n overrideAIConfig(parsed);\n }\n }\n};\n\nexport default class PlaygroundServer {\n app: express.Application;\n tmpDir: string;\n server?: Server;\n port?: number | null;\n pageClass: new (\n ...args: any[]\n ) => AbstractPage;\n agentClass: new (\n ...args: any[]\n ) => PageAgent;\n staticPath?: string;\n taskProgressTips: Record<string, string>;\n\n constructor(\n pageClass: new (...args: any[]) => AbstractPage,\n agentClass: new (...args: any[]) => PageAgent,\n staticPath?: string,\n ) {\n this.app = express();\n this.tmpDir = getTmpDir()!;\n this.pageClass = pageClass;\n this.agentClass = agentClass;\n this.staticPath = staticPath;\n this.taskProgressTips = {};\n setup();\n }\n\n filePathForUuid(uuid: string) {\n return join(this.tmpDir, `${uuid}.json`);\n }\n\n saveContextFile(uuid: string, context: string) {\n const tmpFile = this.filePathForUuid(uuid);\n console.log(`save context file: ${tmpFile}`);\n writeFileSync(tmpFile, context);\n return tmpFile;\n }\n\n async launch(port?: number) {\n this.port = port || defaultPort;\n this.app.use(errorHandler);\n\n this.app.use(\n cors({\n origin: '*',\n credentials: true,\n }),\n );\n\n this.app.get('/status', cors(), async (req, res) => {\n // const modelName = g\n res.send({\n status: 'ok',\n });\n });\n\n // this.app.get('/playground/:uuid', async (req, res) => {\n // res.sendFile(join(staticPath, 'index.html'));\n // });\n\n this.app.get('/context/:uuid', async (req, res) => {\n const { uuid } = req.params;\n const contextFile = this.filePathForUuid(uuid);\n\n if (!existsSync(contextFile)) {\n return res.status(404).json({\n error: 'Context not found',\n });\n }\n\n const context = readFileSync(contextFile, 'utf8');\n res.json({\n context,\n });\n });\n\n this.app.get('/task-progress/:requestId', cors(), async (req, res) => {\n const { requestId } = req.params;\n res.json({\n tip: this.taskProgressTips[requestId] || '',\n });\n });\n\n // -------------------------\n // actions from report file\n this.app.post(\n '/playground-with-context',\n express.json({ limit: '50mb' }),\n async (req, res) => {\n const context = req.body.context;\n\n if (!context) {\n return res.status(400).json({\n error: 'context is required',\n });\n }\n\n const uuid = randomUUID();\n this.saveContextFile(uuid, context);\n return res.json({\n location: `/playground/${uuid}`,\n uuid,\n });\n },\n );\n\n this.app.post(\n '/execute',\n express.json({ limit: '30mb' }),\n async (req, res) => {\n const { context, type, prompt, requestId, deepThink } = req.body;\n\n if (!context) {\n return res.status(400).json({\n error: 'context is required',\n });\n }\n\n if (!type) {\n return res.status(400).json({\n error: 'type is required',\n });\n }\n\n if (!prompt) {\n return res.status(400).json({\n error: 'prompt is required',\n });\n }\n\n // build an agent with context\n const page = new this.pageClass(context);\n const agent = new this.agentClass(page);\n\n if (requestId) {\n this.taskProgressTips[requestId] = '';\n\n agent.onTaskStartTip = (tip: string) => {\n this.taskProgressTips[requestId] = tip;\n };\n }\n\n const response: {\n result: any;\n dump: string | null;\n error: string | null;\n reportHTML: string | null;\n requestId?: string;\n } = {\n result: null,\n dump: null,\n error: null,\n reportHTML: null,\n requestId,\n };\n\n const startTime = Date.now();\n try {\n if (type === 'aiQuery') {\n response.result = await agent.aiQuery(prompt);\n } else if (type === 'aiAction') {\n response.result = await agent.aiAction(prompt);\n } else if (type === 'aiAssert') {\n response.result = await agent.aiAssert(prompt, undefined, {\n keepRawResponse: true,\n });\n } else if (type === 'aiTap') {\n response.result = await agent.aiTap(prompt, {\n deepThink,\n });\n } else {\n response.error = `Unknown type: ${type}`;\n }\n } catch (error: any) {\n if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {\n response.error = error.message;\n }\n }\n\n try {\n response.dump = JSON.parse(agent.dumpDataString());\n response.reportHTML = agent.reportHTMLString() || null;\n\n agent.writeOutActionDumps();\n } catch (error: any) {\n console.error(\n `write out dump failed: requestId: ${requestId}, ${error.message}`,\n );\n }\n\n res.send(response);\n const timeCost = Date.now() - startTime;\n\n if (response.error) {\n console.error(\n `handle request failed after ${timeCost}ms: requestId: ${requestId}, ${response.error}`,\n );\n } else {\n console.log(\n `handle request done after ${timeCost}ms: requestId: ${requestId}`,\n );\n }\n },\n );\n\n this.app.post(\n '/config',\n express.json({ limit: '1mb' }),\n async (req, res) => {\n const { aiConfig } = req.body;\n\n if (!aiConfig || typeof aiConfig !== 'object') {\n return res.status(400).json({\n error: 'aiConfig is required and must be an object',\n });\n }\n\n try {\n overrideAIConfig(aiConfig);\n\n return res.json({\n status: 'ok',\n message: 'AI config updated successfully',\n });\n } catch (error: any) {\n console.error(`Failed to update AI config: ${error.message}`);\n return res.status(500).json({\n error: `Failed to update AI config: ${error.message}`,\n });\n }\n },\n );\n\n // Set up static file serving after all API routes are defined\n if (this.staticPath) {\n this.app.get('/', (req, res) => {\n // compatible with windows\n res.redirect('/index.html');\n });\n\n this.app.get('*', (req, res) => {\n const requestedPath = join(this.staticPath!, req.path);\n if (existsSync(requestedPath)) {\n res.sendFile(requestedPath);\n } else {\n res.sendFile(join(this.staticPath!, 'index.html'));\n }\n });\n }\n\n return new Promise((resolve, reject) => {\n const port = this.port;\n this.server = this.app.listen(port, () => {\n resolve(this);\n });\n });\n }\n\n close() {\n // close the server\n if (this.server) {\n return this.server.close();\n }\n }\n}\n","import type { StaticPage } from '@/playground';\nimport type {\n BaseElement,\n ElementTreeNode,\n PlanningLocateParam,\n PlaywrightParserOpt,\n UIContext,\n} from 'misoai-core';\nimport { elementByPositionWithElementInfo } from 'misoai-core/ai-model';\nimport { uploadTestInfoToServer } from 'misoai-core/utils';\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from 'misoai-shared/env';\nimport type { ElementInfo } from 'misoai-shared/extractor';\nimport {\n generateElementByPosition,\n getNodeFromCacheList,\n traverseTree,\n treeToList,\n} from 'misoai-shared/extractor';\nimport { resizeImgBase64 } from 'misoai-shared/img';\nimport type { DebugFunction } from 'misoai-shared/logger';\nimport { assert, logMsg, uuid } from 'misoai-shared/utils';\nimport dayjs from 'dayjs';\nimport type { Page as PlaywrightPage } from 'playwright';\nimport type { Page as PuppeteerPage } from 'puppeteer';\nimport { WebElementInfo } from '../web-element';\nimport type { WebPage } from './page';\n\nexport type WebUIContext = UIContext<WebElementInfo> & {\n url: string;\n};\n\nexport async function parseContextFromWebPage(\n page: WebPage,\n _opt?: PlaywrightParserOpt,\n): Promise<WebUIContext> {\n assert(page, 'page is required');\n if ((page as StaticPage)._forceUsePageContext) {\n return await (page as any)._forceUsePageContext();\n }\n const url = await page.url();\n uploadTestInfoToServer({ testUrl: url });\n\n let screenshotBase64: string;\n let tree: ElementTreeNode<ElementInfo>;\n\n await Promise.all([\n page.screenshotBase64().then((base64) => {\n screenshotBase64 = base64;\n }),\n page.getElementsNodeTree().then(async (treeRoot) => {\n tree = treeRoot;\n }),\n ]);\n\n const webTree = traverseTree(tree!, (elementInfo) => {\n const { rect, id, content, attributes, locator, indexId } = elementInfo;\n return new WebElementInfo({\n rect,\n locator,\n id,\n content,\n attributes,\n indexId,\n });\n });\n\n assert(screenshotBase64!, 'screenshotBase64 is required');\n\n const elementsInfo = treeToList(webTree);\n const size = await page.size();\n\n if (size.dpr && size.dpr > 1) {\n // console.time('resizeImgBase64');\n screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n width: size.width,\n height: size.height,\n });\n // console.timeEnd('resizeImgBase64');\n }\n\n return {\n content: elementsInfo!,\n tree: webTree,\n size,\n screenshotBase64: screenshotBase64!,\n url,\n };\n}\n\nexport function reportFileName(tag = 'web') {\n const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\n const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n // ensure uniqueness at the same time\n const uniqueId = uuid().substring(0, 8);\n return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n const error = new Error();\n const stackTrace = trace || error.stack;\n const pkgDir = process.cwd() || '';\n if (stackTrace) {\n const stackLines = stackTrace.split('\\n');\n for (const line of stackLines) {\n if (\n line.includes('.spec.') ||\n line.includes('.test.') ||\n line.includes('.ts') ||\n line.includes('.js')\n ) {\n const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n if (match?.[1]) {\n const targetFileName = match[1]\n .replace(pkgDir, '')\n .trim()\n .replace('at ', '');\n return targetFileName;\n }\n }\n }\n }\n return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n let taskFile = fileName || getCurrentExecutionFile();\n if (!taskFile) {\n taskFile = uuid();\n console.warn(\n 'Midscene - using random UUID for cache id. Cache may be invalid.',\n );\n }\n\n if (testFileIndex.has(taskFile)) {\n const currentIndex = testFileIndex.get(taskFile);\n if (currentIndex !== undefined) {\n testFileIndex.set(taskFile, currentIndex + 1);\n }\n } else {\n testFileIndex.set(taskFile, 1);\n }\n return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\n 'NOT_IMPLEMENTED_AS_DESIGNED';\n\nexport function replaceIllegalPathCharsAndSpace(str: string) {\n return str.replace(/[/\\\\:*?\"<>| ]/g, '-');\n}\n\nexport function forceClosePopup(\n page: PuppeteerPage | PlaywrightPage,\n debug: DebugFunction,\n) {\n page.on('popup', async (popup) => {\n if (!popup) {\n console.warn('got a popup event, but the popup is not ready yet, skip');\n return;\n }\n const url = await (popup as PuppeteerPage).url();\n console.log(`Popup opened: ${url}`);\n if (!(popup as PuppeteerPage).isClosed()) {\n try {\n await (popup as PuppeteerPage).close(); // Close the newly opened TAB\n } catch (error) {\n debug(`failed to close popup ${url}, error: ${error}`);\n }\n } else {\n debug(`popup is already closed, skip close ${url}`);\n }\n\n if (!page.isClosed()) {\n try {\n await page.goto(url);\n } catch (error) {\n debug(`failed to goto ${url}, error: ${error}`);\n }\n } else {\n debug(`page is already closed, skip goto ${url}`);\n }\n });\n}\n\nexport function matchElementFromPlan(\n planLocateParam: PlanningLocateParam,\n tree: ElementTreeNode<BaseElement>,\n) {\n if (!planLocateParam) {\n return undefined;\n }\n if (planLocateParam.id) {\n return getNodeFromCacheList(planLocateParam.id);\n }\n\n if (planLocateParam.bbox) {\n const centerPosition = {\n x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n };\n let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n if (!element) {\n element = generateElementByPosition(centerPosition) as BaseElement;\n }\n\n return element;\n }\n\n return undefined;\n}\n"]}
1
+ {"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,qBAAwD;AAExD,uBAAqB;;;ACKrB,sBAAiD;AACjD,mBAAuC;AACvC,iBAAsD;AAEtD,uBAKO;AACP,iBAAgC;AAEhC,IAAAA,gBAAqC;AACrC,mBAAkB;AAqIX,IAAM,yCACX;;;ADtJF,IAAAA,gBAA0B;AAC1B,uBAAuC;AACvC,IAAAC,cAAiC;AACjC,IAAAD,gBAA4B;AAC5B,kBAAiB;AACjB,oBAAmB;AACnB,qBAAoB;AAIpB,IAAM,cAAc;AAGpB,IAAM,eAAe,CAAC,KAAU,KAAU,KAAU,SAAc;AAChE,UAAQ,MAAM,GAAG;AACjB,MAAI,OAAO,GAAG,EAAE,KAAK;AAAA,IACnB,OAAO,IAAI;AAAA,EACb,CAAC;AACH;AAEA,IAAM,QAAQ,YAAY;AACxB,MAAI,CAAC,2BAAa;AAChB,UAAM,EAAE,OAAO,IAAI,cAAAE,QAAO,OAAO;AAEjC,QAAI,QAAQ;AACV,wCAAiB,MAAM;AAAA,IACzB;AAAA,EACF;AACF;AAEA,IAAqB,mBAArB,MAAsC;AAAA,EAcpC,YACE,WACA,YACA,YACA;AACA,SAAK,UAAM,eAAAC,SAAQ;AACnB,SAAK,aAAS,yBAAU;AACxB,SAAK,YAAY;AACjB,SAAK,aAAa;AAClB,SAAK,aAAa;AAClB,SAAK,mBAAmB,CAAC;AACzB,UAAM;AAAA,EACR;AAAA,EAEA,gBAAgBC,OAAc;AAC5B,eAAO,uBAAK,KAAK,QAAQ,GAAGA,KAAI,OAAO;AAAA,EACzC;AAAA,EAEA,gBAAgBA,OAAc,SAAiB;AAC7C,UAAM,UAAU,KAAK,gBAAgBA,KAAI;AACzC,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,sCAAc,SAAS,OAAO;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,MAAe;AAC1B,SAAK,OAAO,QAAQ;AACpB,SAAK,IAAI,IAAI,YAAY;AAEzB,SAAK,IAAI;AAAA,UACP,YAAAC,SAAK;AAAA,QACH,QAAQ;AAAA,QACR,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,SAAK,IAAI,IAAI,eAAW,YAAAA,SAAK,GAAG,OAAO,KAAK,QAAQ;AAElD,UAAI,KAAK;AAAA,QACP,QAAQ;AAAA,MACV,CAAC;AAAA,IACH,CAAC;AAMD,SAAK,IAAI,IAAI,kBAAkB,OAAO,KAAK,QAAQ;AACjD,YAAM,EAAE,MAAAD,MAAK,IAAI,IAAI;AACrB,YAAM,cAAc,KAAK,gBAAgBA,KAAI;AAE7C,UAAI,KAAC,2BAAW,WAAW,GAAG;AAC5B,eAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,UAC1B,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAEA,YAAM,cAAU,6BAAa,aAAa,MAAM;AAChD,UAAI,KAAK;AAAA,QACP;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAED,SAAK,IAAI,IAAI,iCAA6B,YAAAC,SAAK,GAAG,OAAO,KAAK,QAAQ;AACpE,YAAM,EAAE,UAAU,IAAI,IAAI;AAC1B,UAAI,KAAK;AAAA,QACP,KAAK,KAAK,iBAAiB,SAAS,KAAK;AAAA,MAC3C,CAAC;AAAA,IACH,CAAC;AAID,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAF,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,UAAU,IAAI,KAAK;AAEzB,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,cAAMC,YAAO,+BAAW;AACxB,aAAK,gBAAgBA,OAAM,OAAO;AAClC,eAAO,IAAI,KAAK;AAAA,UACd,UAAU,eAAeA,KAAI;AAAA,UAC7B,MAAAA;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAD,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,MAAM,QAAQ,WAAW,UAAU,IAAI,IAAI;AAE5D,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM;AACT,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,QAAQ;AACX,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAGA,cAAM,OAAO,IAAI,KAAK,UAAU,OAAO;AACvC,cAAM,QAAQ,IAAI,KAAK,WAAW,IAAI;AAEtC,YAAI,WAAW;AACb,eAAK,iBAAiB,SAAS,IAAI;AAEnC,gBAAM,iBAAiB,CAAC,QAAgB;AACtC,iBAAK,iBAAiB,SAAS,IAAI;AAAA,UACrC;AAAA,QACF;AAEA,cAAM,WAMF;AAAA,UACF,QAAQ;AAAA,UACR,MAAM;AAAA,UACN,OAAO;AAAA,UACP,YAAY;AAAA,UACZ;AAAA,QACF;AAEA,cAAM,YAAY,KAAK,IAAI;AAC3B,YAAI;AACF,cAAI,SAAS,WAAW;AACtB,qBAAS,SAAS,MAAM,MAAM,QAAQ,MAAM;AAAA,UAC9C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,MAAM;AAAA,UAC/C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,QAAQ,QAAW;AAAA,cACxD,iBAAiB;AAAA,YACnB,CAAC;AAAA,UACH,WAAW,SAAS,SAAS;AAC3B,qBAAS,SAAS,MAAM,MAAM,MAAM,QAAQ;AAAA,cAC1C;AAAA,YACF,CAAC;AAAA,UACH,OAAO;AACL,qBAAS,QAAQ,iBAAiB,IAAI;AAAA,UACxC;AAAA,QACF,SAAS,OAAY;AACnB,cAAI,CAAC,MAAM,QAAQ,SAAS,sCAAsC,GAAG;AACnE,qBAAS,QAAQ,MAAM;AAAA,UACzB;AAAA,QACF;AAEA,YAAI;AACF,mBAAS,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AACjD,mBAAS,aAAa,MAAM,iBAAiB,KAAK;AAElD,gBAAM,oBAAoB;AAAA,QAC5B,SAAS,OAAY;AACnB,kBAAQ;AAAA,YACN,qCAAqC,SAAS,KAAK,MAAM,OAAO;AAAA,UAClE;AAAA,QACF;AAEA,YAAI,KAAK,QAAQ;AACjB,cAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,YAAI,SAAS,OAAO;AAClB,kBAAQ;AAAA,YACN,+BAA+B,QAAQ,kBAAkB,SAAS,KAAK,SAAS,KAAK;AAAA,UACvF;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,6BAA6B,QAAQ,kBAAkB,SAAS;AAAA,UAClE;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAA,QAAQ,KAAK,EAAE,OAAO,MAAM,CAAC;AAAA,MAC7B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,IAAI,IAAI;AAEzB,YAAI,CAAC,YAAY,OAAO,aAAa,UAAU;AAC7C,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI;AACF,4CAAiB,QAAQ;AAEzB,iBAAO,IAAI,KAAK;AAAA,YACd,QAAQ;AAAA,YACR,SAAS;AAAA,UACX,CAAC;AAAA,QACH,SAAS,OAAY;AACnB,kBAAQ,MAAM,+BAA+B,MAAM,OAAO,EAAE;AAC5D,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO,+BAA+B,MAAM,OAAO;AAAA,UACrD,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,QAAI,KAAK,YAAY;AACnB,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAE9B,YAAI,SAAS,aAAa;AAAA,MAC5B,CAAC;AAED,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAC9B,cAAM,oBAAgB,uBAAK,KAAK,YAAa,IAAI,IAAI;AACrD,gBAAI,2BAAW,aAAa,GAAG;AAC7B,cAAI,SAAS,aAAa;AAAA,QAC5B,OAAO;AACL,cAAI,aAAS,uBAAK,KAAK,YAAa,YAAY,CAAC;AAAA,QACnD;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAMG,QAAO,KAAK;AAClB,WAAK,SAAS,KAAK,IAAI,OAAOA,OAAM,MAAM;AACxC,gBAAQ,IAAI;AAAA,MACd,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AAEN,QAAI,KAAK,QAAQ;AACf,aAAO,KAAK,OAAO,MAAM;AAAA,IAC3B;AAAA,EACF;AACF","names":["import_utils","import_env","dotenv","express","uuid","cors","port"],"ignoreList":[],"sources":["../../src/playground/server.ts","../../src/common/utils.ts"],"sourcesContent":["import { randomUUID } from 'node:crypto';\r\nimport { existsSync, readFileSync, writeFileSync } from 'node:fs';\r\nimport type { Server } from 'node:http';\r\nimport { join } from 'node:path';\r\nimport { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '@/common/utils';\r\nimport { getTmpDir } from 'misoai-core/utils';\r\nimport { PLAYGROUND_SERVER_PORT } from 'misoai-shared/constants';\r\nimport { overrideAIConfig } from 'misoai-shared/env';\r\nimport { ifInBrowser } from 'misoai-shared/utils';\r\nimport cors from 'cors';\r\nimport dotenv from 'dotenv';\r\nimport express from 'express';\r\nimport type { PageAgent } from '../common/agent';\r\nimport type { AbstractPage } from '../page';\r\n\r\nconst defaultPort = PLAYGROUND_SERVER_PORT;\r\n// const staticPath = join(__dirname, '../../static');\r\n\r\nconst errorHandler = (err: any, req: any, res: any, next: any) => {\r\n console.error(err);\r\n res.status(500).json({\r\n error: err.message,\r\n });\r\n};\r\n\r\nconst setup = async () => {\r\n if (!ifInBrowser) {\r\n const { parsed } = dotenv.config();\r\n\r\n if (parsed) {\r\n overrideAIConfig(parsed);\r\n }\r\n }\r\n};\r\n\r\nexport default class PlaygroundServer {\r\n app: express.Application;\r\n tmpDir: string;\r\n server?: Server;\r\n port?: number | null;\r\n pageClass: new (\r\n ...args: any[]\r\n ) => AbstractPage;\r\n agentClass: new (\r\n ...args: any[]\r\n ) => PageAgent;\r\n staticPath?: string;\r\n taskProgressTips: Record<string, string>;\r\n\r\n constructor(\r\n pageClass: new (...args: any[]) => AbstractPage,\r\n agentClass: new (...args: any[]) => PageAgent,\r\n staticPath?: string,\r\n ) {\r\n this.app = express();\r\n this.tmpDir = getTmpDir()!;\r\n this.pageClass = pageClass;\r\n this.agentClass = agentClass;\r\n this.staticPath = staticPath;\r\n this.taskProgressTips = {};\r\n setup();\r\n }\r\n\r\n filePathForUuid(uuid: string) {\r\n return join(this.tmpDir, `${uuid}.json`);\r\n }\r\n\r\n saveContextFile(uuid: string, context: string) {\r\n const tmpFile = this.filePathForUuid(uuid);\r\n console.log(`save context file: ${tmpFile}`);\r\n writeFileSync(tmpFile, context);\r\n return tmpFile;\r\n }\r\n\r\n async launch(port?: number) {\r\n this.port = port || defaultPort;\r\n this.app.use(errorHandler);\r\n\r\n this.app.use(\r\n cors({\r\n origin: '*',\r\n credentials: true,\r\n }),\r\n );\r\n\r\n this.app.get('/status', cors(), async (req, res) => {\r\n // const modelName = g\r\n res.send({\r\n status: 'ok',\r\n });\r\n });\r\n\r\n // this.app.get('/playground/:uuid', async (req, res) => {\r\n // res.sendFile(join(staticPath, 'index.html'));\r\n // });\r\n\r\n this.app.get('/context/:uuid', async (req, res) => {\r\n const { uuid } = req.params;\r\n const contextFile = this.filePathForUuid(uuid);\r\n\r\n if (!existsSync(contextFile)) {\r\n return res.status(404).json({\r\n error: 'Context not found',\r\n });\r\n }\r\n\r\n const context = readFileSync(contextFile, 'utf8');\r\n res.json({\r\n context,\r\n });\r\n });\r\n\r\n this.app.get('/task-progress/:requestId', cors(), async (req, res) => {\r\n const { requestId } = req.params;\r\n res.json({\r\n tip: this.taskProgressTips[requestId] || '',\r\n });\r\n });\r\n\r\n // -------------------------\r\n // actions from report file\r\n this.app.post(\r\n '/playground-with-context',\r\n express.json({ limit: '50mb' }),\r\n async (req, res) => {\r\n const context = req.body.context;\r\n\r\n if (!context) {\r\n return res.status(400).json({\r\n error: 'context is required',\r\n });\r\n }\r\n\r\n const uuid = randomUUID();\r\n this.saveContextFile(uuid, context);\r\n return res.json({\r\n location: `/playground/${uuid}`,\r\n uuid,\r\n });\r\n },\r\n );\r\n\r\n this.app.post(\r\n '/execute',\r\n express.json({ limit: '30mb' }),\r\n async (req, res) => {\r\n const { context, type, prompt, requestId, deepThink } = req.body;\r\n\r\n if (!context) {\r\n return res.status(400).json({\r\n error: 'context is required',\r\n });\r\n }\r\n\r\n if (!type) {\r\n return res.status(400).json({\r\n error: 'type is required',\r\n });\r\n }\r\n\r\n if (!prompt) {\r\n return res.status(400).json({\r\n error: 'prompt is required',\r\n });\r\n }\r\n\r\n // build an agent with context\r\n const page = new this.pageClass(context);\r\n const agent = new this.agentClass(page);\r\n\r\n if (requestId) {\r\n this.taskProgressTips[requestId] = '';\r\n\r\n agent.onTaskStartTip = (tip: string) => {\r\n this.taskProgressTips[requestId] = tip;\r\n };\r\n }\r\n\r\n const response: {\r\n result: any;\r\n dump: string | null;\r\n error: string | null;\r\n reportHTML: string | null;\r\n requestId?: string;\r\n } = {\r\n result: null,\r\n dump: null,\r\n error: null,\r\n reportHTML: null,\r\n requestId,\r\n };\r\n\r\n const startTime = Date.now();\r\n try {\r\n if (type === 'aiQuery') {\r\n response.result = await agent.aiQuery(prompt);\r\n } else if (type === 'aiAction') {\r\n response.result = await agent.aiAction(prompt);\r\n } else if (type === 'aiAssert') {\r\n response.result = await agent.aiAssert(prompt, undefined, {\r\n keepRawResponse: true,\r\n });\r\n } else if (type === 'aiTap') {\r\n response.result = await agent.aiTap(prompt, {\r\n deepThink,\r\n });\r\n } else {\r\n response.error = `Unknown type: ${type}`;\r\n }\r\n } catch (error: any) {\r\n if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {\r\n response.error = error.message;\r\n }\r\n }\r\n\r\n try {\r\n response.dump = JSON.parse(agent.dumpDataString());\r\n response.reportHTML = agent.reportHTMLString() || null;\r\n\r\n agent.writeOutActionDumps();\r\n } catch (error: any) {\r\n console.error(\r\n `write out dump failed: requestId: ${requestId}, ${error.message}`,\r\n );\r\n }\r\n\r\n res.send(response);\r\n const timeCost = Date.now() - startTime;\r\n\r\n if (response.error) {\r\n console.error(\r\n `handle request failed after ${timeCost}ms: requestId: ${requestId}, ${response.error}`,\r\n );\r\n } else {\r\n console.log(\r\n `handle request done after ${timeCost}ms: requestId: ${requestId}`,\r\n );\r\n }\r\n },\r\n );\r\n\r\n this.app.post(\r\n '/config',\r\n express.json({ limit: '1mb' }),\r\n async (req, res) => {\r\n const { aiConfig } = req.body;\r\n\r\n if (!aiConfig || typeof aiConfig !== 'object') {\r\n return res.status(400).json({\r\n error: 'aiConfig is required and must be an object',\r\n });\r\n }\r\n\r\n try {\r\n overrideAIConfig(aiConfig);\r\n\r\n return res.json({\r\n status: 'ok',\r\n message: 'AI config updated successfully',\r\n });\r\n } catch (error: any) {\r\n console.error(`Failed to update AI config: ${error.message}`);\r\n return res.status(500).json({\r\n error: `Failed to update AI config: ${error.message}`,\r\n });\r\n }\r\n },\r\n );\r\n\r\n // Set up static file serving after all API routes are defined\r\n if (this.staticPath) {\r\n this.app.get('/', (req, res) => {\r\n // compatible with windows\r\n res.redirect('/index.html');\r\n });\r\n\r\n this.app.get('*', (req, res) => {\r\n const requestedPath = join(this.staticPath!, req.path);\r\n if (existsSync(requestedPath)) {\r\n res.sendFile(requestedPath);\r\n } else {\r\n res.sendFile(join(this.staticPath!, 'index.html'));\r\n }\r\n });\r\n }\r\n\r\n return new Promise((resolve, reject) => {\r\n const port = this.port;\r\n this.server = this.app.listen(port, () => {\r\n resolve(this);\r\n });\r\n });\r\n }\r\n\r\n close() {\r\n // close the server\r\n if (this.server) {\r\n return this.server.close();\r\n }\r\n }\r\n}\r\n","import type { StaticPage } from '@/playground';\r\nimport type {\r\n BaseElement,\r\n ElementTreeNode,\r\n PlanningLocateParam,\r\n PlaywrightParserOpt,\r\n UIContext,\r\n} from 'misoai-core';\r\nimport { elementByPositionWithElementInfo } from 'misoai-core/ai-model';\r\nimport { uploadTestInfoToServer } from 'misoai-core/utils';\r\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from 'misoai-shared/env';\r\nimport type { ElementInfo } from 'misoai-shared/extractor';\r\nimport {\r\n generateElementByPosition,\r\n getNodeFromCacheList,\r\n traverseTree,\r\n treeToList,\r\n} from 'misoai-shared/extractor';\r\nimport { resizeImgBase64 } from 'misoai-shared/img';\r\nimport type { DebugFunction } from 'misoai-shared/logger';\r\nimport { assert, logMsg, uuid } from 'misoai-shared/utils';\r\nimport dayjs from 'dayjs';\r\nimport type { Page as PlaywrightPage } from 'playwright';\r\nimport type { Page as PuppeteerPage } from 'puppeteer';\r\nimport { WebElementInfo } from '../web-element';\r\nimport type { WebPage } from './page';\r\n\r\nexport type WebUIContext = UIContext<WebElementInfo> & {\r\n url: string;\r\n};\r\n\r\nexport async function parseContextFromWebPage(\r\n page: WebPage,\r\n _opt?: PlaywrightParserOpt,\r\n): Promise<WebUIContext> {\r\n assert(page, 'page is required');\r\n if ((page as StaticPage)._forceUsePageContext) {\r\n return await (page as any)._forceUsePageContext();\r\n }\r\n const url = await page.url();\r\n uploadTestInfoToServer({ testUrl: url });\r\n\r\n let screenshotBase64: string;\r\n let tree: ElementTreeNode<ElementInfo>;\r\n\r\n await Promise.all([\r\n page.screenshotBase64().then((base64) => {\r\n screenshotBase64 = base64;\r\n }),\r\n page.getElementsNodeTree().then(async (treeRoot) => {\r\n tree = treeRoot;\r\n }),\r\n ]);\r\n\r\n const webTree = traverseTree(tree!, (elementInfo) => {\r\n const { rect, id, content, attributes, locator, indexId } = elementInfo;\r\n return new WebElementInfo({\r\n rect,\r\n locator,\r\n id,\r\n content,\r\n attributes,\r\n indexId,\r\n });\r\n });\r\n\r\n assert(screenshotBase64!, 'screenshotBase64 is required');\r\n\r\n const elementsInfo = treeToList(webTree);\r\n const size = await page.size();\r\n\r\n if (size.dpr && size.dpr > 1) {\r\n // console.time('resizeImgBase64');\r\n screenshotBase64 = await resizeImgBase64(screenshotBase64, {\r\n width: size.width,\r\n height: size.height,\r\n });\r\n // console.timeEnd('resizeImgBase64');\r\n }\r\n\r\n return {\r\n content: elementsInfo!,\r\n tree: webTree,\r\n size,\r\n screenshotBase64: screenshotBase64!,\r\n url,\r\n };\r\n}\r\n\r\nexport function reportFileName(tag = 'web') {\r\n const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\r\n const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\r\n // ensure uniqueness at the same time\r\n const uniqueId = uuid().substring(0, 8);\r\n return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\r\n}\r\n\r\nexport function printReportMsg(filepath: string) {\r\n logMsg(`Midscene - report file updated: ${filepath}`);\r\n}\r\n\r\n/**\r\n * Get the current execution file name\r\n * @returns The name of the current execution file\r\n */\r\nexport function getCurrentExecutionFile(trace?: string): string | false {\r\n const error = new Error();\r\n const stackTrace = trace || error.stack;\r\n const pkgDir = process.cwd() || '';\r\n if (stackTrace) {\r\n const stackLines = stackTrace.split('\\n');\r\n for (const line of stackLines) {\r\n if (\r\n line.includes('.spec.') ||\r\n line.includes('.test.') ||\r\n line.includes('.ts') ||\r\n line.includes('.js')\r\n ) {\r\n const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\r\n if (match?.[1]) {\r\n const targetFileName = match[1]\r\n .replace(pkgDir, '')\r\n .trim()\r\n .replace('at ', '');\r\n return targetFileName;\r\n }\r\n }\r\n }\r\n }\r\n return false;\r\n}\r\n\r\nconst testFileIndex = new Map<string, number>();\r\n\r\nexport function generateCacheId(fileName?: string): string {\r\n let taskFile = fileName || getCurrentExecutionFile();\r\n if (!taskFile) {\r\n taskFile = uuid();\r\n console.warn(\r\n 'Midscene - using random UUID for cache id. Cache may be invalid.',\r\n );\r\n }\r\n\r\n if (testFileIndex.has(taskFile)) {\r\n const currentIndex = testFileIndex.get(taskFile);\r\n if (currentIndex !== undefined) {\r\n testFileIndex.set(taskFile, currentIndex + 1);\r\n }\r\n } else {\r\n testFileIndex.set(taskFile, 1);\r\n }\r\n return `${taskFile}-${testFileIndex.get(taskFile)}`;\r\n}\r\n\r\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\r\n 'NOT_IMPLEMENTED_AS_DESIGNED';\r\n\r\nexport function replaceIllegalPathCharsAndSpace(str: string) {\r\n return str.replace(/[/\\\\:*?\"<>| ]/g, '-');\r\n}\r\n\r\nexport function forceClosePopup(\r\n page: PuppeteerPage | PlaywrightPage,\r\n debug: DebugFunction,\r\n) {\r\n page.on('popup', async (popup) => {\r\n if (!popup) {\r\n console.warn('got a popup event, but the popup is not ready yet, skip');\r\n return;\r\n }\r\n const url = await (popup as PuppeteerPage).url();\r\n console.log(`Popup opened: ${url}`);\r\n if (!(popup as PuppeteerPage).isClosed()) {\r\n try {\r\n await (popup as PuppeteerPage).close(); // Close the newly opened TAB\r\n } catch (error) {\r\n debug(`failed to close popup ${url}, error: ${error}`);\r\n }\r\n } else {\r\n debug(`popup is already closed, skip close ${url}`);\r\n }\r\n\r\n if (!page.isClosed()) {\r\n try {\r\n await page.goto(url);\r\n } catch (error) {\r\n debug(`failed to goto ${url}, error: ${error}`);\r\n }\r\n } else {\r\n debug(`page is already closed, skip goto ${url}`);\r\n }\r\n });\r\n}\r\n\r\nexport function matchElementFromPlan(\r\n planLocateParam: PlanningLocateParam,\r\n tree: ElementTreeNode<BaseElement>,\r\n) {\r\n if (!planLocateParam) {\r\n return undefined;\r\n }\r\n if (planLocateParam.id) {\r\n return getNodeFromCacheList(planLocateParam.id);\r\n }\r\n\r\n if (planLocateParam.bbox) {\r\n const centerPosition = {\r\n x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\r\n y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\r\n };\r\n let element = elementByPositionWithElementInfo(tree, centerPosition);\r\n\r\n if (!element) {\r\n element = generateElementByPosition(centerPosition) as BaseElement;\r\n }\r\n\r\n return element;\r\n }\r\n\r\n return undefined;\r\n}\r\n"]}
@@ -241,7 +241,9 @@ var ScriptPlayer = class {
241
241
  typeof prompt === "string",
242
242
  "prompt for aiAction must be a string"
243
243
  );
244
- await agent.aiAction(prompt);
244
+ await agent.aiAction(prompt, {
245
+ cacheable: actionTask.cacheable
246
+ });
245
247
  } else if ("aiAssert" in flowItem) {
246
248
  const assertTask = flowItem;
247
249
  const prompt = assertTask.aiAssert;
@@ -443,8 +445,24 @@ function interpolateEnvVars(content) {
443
445
  });
444
446
  }
445
447
  function parseYamlScript(content, filePath, ignoreCheckingTarget) {
446
- const interpolatedContent = interpolateEnvVars(content);
447
- const obj = import_js_yaml2.default.load(interpolatedContent);
448
+ let processedContent = content;
449
+ if (content.indexOf("android") !== -1 && content.match(/deviceId:\s*(\d+)/)) {
450
+ let matchedDeviceId;
451
+ processedContent = content.replace(
452
+ /deviceId:\s*(\d+)/g,
453
+ (match, deviceId) => {
454
+ matchedDeviceId = deviceId;
455
+ return `deviceId: '${deviceId}'`;
456
+ }
457
+ );
458
+ console.warn(
459
+ `please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`
460
+ );
461
+ }
462
+ const interpolatedContent = interpolateEnvVars(processedContent);
463
+ const obj = import_js_yaml2.default.load(interpolatedContent, {
464
+ schema: import_js_yaml2.default.JSON_SCHEMA
465
+ });
448
466
  const pathTip = filePath ? `, failed to load ${filePath}` : "";
449
467
  const android = typeof obj.android !== "undefined" ? Object.assign({}, obj.android || {}) : void 0;
450
468
  const webConfig = obj.web || obj.target;
@@ -485,7 +503,6 @@ var import_misoai_core = require("misoai-core");
485
503
  var import_ai_model2 = require("misoai-core/ai-model");
486
504
  var import_utils5 = require("misoai-core/utils");
487
505
  var import_constants = require("misoai-shared/constants");
488
- var import_fs = require("misoai-shared/fs");
489
506
  var import_logger = require("misoai-shared/logger");
490
507
  var import_utils6 = require("misoai-shared/utils");
491
508
 
@@ -603,16 +620,18 @@ var PageTaskExecutor = class {
603
620
  );
604
621
  if (info?.id) {
605
622
  elementId = info.id;
623
+ } else {
624
+ debug(
625
+ "no element id found for position node, will not update cache",
626
+ element
627
+ );
606
628
  }
607
629
  }
608
630
  if (!elementId) {
609
631
  return void 0;
610
632
  }
611
633
  try {
612
- const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
613
- const result = await this.page.evaluateJavaScript?.(
614
- `${elementInfosScriptContent}midscene_element_inspector.getXpathsById('${elementId}')`
615
- );
634
+ const result = await this.page.getXpathsById(elementId);
616
635
  return result;
617
636
  } catch (error) {
618
637
  debug("getXpathsById error: ", error);
@@ -651,7 +670,7 @@ var PageTaskExecutor = class {
651
670
  };
652
671
  return taskWithScreenshot;
653
672
  }
654
- async convertPlanToExecutable(plans) {
673
+ async convertPlanToExecutable(plans, opts) {
655
674
  const tasks = [];
656
675
  plans.forEach((plan2) => {
657
676
  if (plan2.type === "Locate") {
@@ -661,7 +680,10 @@ var PageTaskExecutor = class {
661
680
  const taskFind = {
662
681
  type: "Insight",
663
682
  subType: "Locate",
664
- param: plan2.locate || void 0,
683
+ param: plan2.locate ? {
684
+ ...plan2.locate,
685
+ cacheable: opts?.cacheable
686
+ } : void 0,
665
687
  thought: plan2.thought,
666
688
  locate: plan2.locate,
667
689
  executor: async (param, taskContext) => {
@@ -698,19 +720,21 @@ var PageTaskExecutor = class {
698
720
  let elementFromCache = null;
699
721
  try {
700
722
  if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
701
- const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
702
- const element2 = await this.page.evaluateJavaScript?.(
703
- `${elementInfosScriptContent}midscene_element_inspector.getElementInfoByXpath('${xpaths[0]}')`
704
- );
705
- if (element2?.id) {
706
- elementFromCache = element2;
707
- debug("cache hit, prompt: %s", cachePrompt);
708
- cacheHitFlag = true;
709
- debug(
710
- "found a new new element with same xpath, xpath: %s, id: %s",
711
- xpaths[0],
712
- element2?.id
723
+ for (let i = 0; i < xpaths.length; i++) {
724
+ const element2 = await this.page.getElementInfoByXpath(
725
+ xpaths[i]
713
726
  );
727
+ if (element2?.id) {
728
+ elementFromCache = element2;
729
+ debug("cache hit, prompt: %s", cachePrompt);
730
+ cacheHitFlag = true;
731
+ debug(
732
+ "found a new new element with same xpath, xpath: %s, id: %s",
733
+ xpaths[i],
734
+ element2?.id
735
+ );
736
+ break;
737
+ }
714
738
  }
715
739
  }
716
740
  } catch (error) {
@@ -723,12 +747,14 @@ var PageTaskExecutor = class {
723
747
  context: pageContext
724
748
  })).element;
725
749
  const aiCost = Date.now() - startTime;
750
+ let currentXpaths;
726
751
  if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
727
752
  const elementXpaths = await this.getElementXpath(
728
753
  pageContext,
729
754
  element
730
755
  );
731
- if (elementXpaths) {
756
+ if (elementXpaths?.length) {
757
+ currentXpaths = elementXpaths;
732
758
  this.taskCache.updateOrAppendCacheRecord(
733
759
  {
734
760
  type: "locate",
@@ -738,7 +764,11 @@ var PageTaskExecutor = class {
738
764
  locateCacheRecord
739
765
  );
740
766
  } else {
741
- debug("no xpaths found, will not update cache", cachePrompt);
767
+ debug(
768
+ "no xpaths found, will not update cache",
769
+ cachePrompt,
770
+ elementXpaths
771
+ );
742
772
  }
743
773
  }
744
774
  if (!element) {
@@ -750,7 +780,9 @@ var PageTaskExecutor = class {
750
780
  },
751
781
  pageContext,
752
782
  cache: {
753
- hit: cacheHitFlag
783
+ hit: cacheHitFlag,
784
+ originalXpaths: xpaths,
785
+ currentXpaths
754
786
  },
755
787
  aiCost
756
788
  };
@@ -1118,6 +1150,7 @@ var PageTaskExecutor = class {
1118
1150
  sleep: sleep2
1119
1151
  } = planResult;
1120
1152
  executorContext.task.log = {
1153
+ ...executorContext.task.log || {},
1121
1154
  rawResponse
1122
1155
  };
1123
1156
  executorContext.task.usage = usage;
@@ -1242,11 +1275,11 @@ var PageTaskExecutor = class {
1242
1275
  };
1243
1276
  return task;
1244
1277
  }
1245
- async runPlans(title, plans) {
1278
+ async runPlans(title, plans, opts) {
1246
1279
  const taskExecutor = new import_misoai_core.Executor(title, {
1247
1280
  onTaskStart: this.onTaskStartCallback
1248
1281
  });
1249
- const { tasks } = await this.convertPlanToExecutable(plans);
1282
+ const { tasks } = await this.convertPlanToExecutable(plans, opts);
1250
1283
  await taskExecutor.append(tasks);
1251
1284
  const result = await taskExecutor.flush();
1252
1285
  return {
@@ -1254,7 +1287,7 @@ var PageTaskExecutor = class {
1254
1287
  executor: taskExecutor
1255
1288
  };
1256
1289
  }
1257
- async action(userPrompt, actionContext) {
1290
+ async action(userPrompt, actionContext, opts) {
1258
1291
  const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
1259
1292
  onTaskStart: this.onTaskStartCallback
1260
1293
  });
@@ -1279,7 +1312,7 @@ var PageTaskExecutor = class {
1279
1312
  yamlFlow.push(...planResult.yamlFlow || []);
1280
1313
  let executables;
1281
1314
  try {
1282
- executables = await this.convertPlanToExecutable(plans);
1315
+ executables = await this.convertPlanToExecutable(plans, opts);
1283
1316
  taskExecutor.append(executables.tasks);
1284
1317
  } catch (error) {
1285
1318
  return this.appendErrorPlan(
@@ -1317,7 +1350,7 @@ var PageTaskExecutor = class {
1317
1350
  executor: taskExecutor
1318
1351
  };
1319
1352
  }
1320
- async actionToGoal(userPrompt) {
1353
+ async actionToGoal(userPrompt, opts) {
1321
1354
  const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
1322
1355
  onTaskStart: this.onTaskStartCallback
1323
1356
  });
@@ -1341,7 +1374,7 @@ var PageTaskExecutor = class {
1341
1374
  yamlFlow.push(...output.yamlFlow || []);
1342
1375
  let executables;
1343
1376
  try {
1344
- executables = await this.convertPlanToExecutable(plans);
1377
+ executables = await this.convertPlanToExecutable(plans, opts);
1345
1378
  taskExecutor.append(executables.tasks);
1346
1379
  } catch (error) {
1347
1380
  return this.appendErrorPlan(
@@ -1646,7 +1679,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
1646
1679
  var import_semver = __toESM(require("semver"));
1647
1680
 
1648
1681
  // package.json
1649
- var version = "1.0.2";
1682
+ var version = "1.0.4";
1650
1683
 
1651
1684
  // src/common/task-cache.ts
1652
1685
  var debug3 = (0, import_logger3.getDebug)("cache");
@@ -1989,9 +2022,9 @@ var PageAgent = class {
1989
2022
  buildDetailedLocateParam(locatePrompt, opt) {
1990
2023
  (0, import_utils12.assert)(locatePrompt, "missing locate prompt");
1991
2024
  if (typeof opt === "object") {
1992
- const prompt = opt.prompt || locatePrompt;
1993
- const deepThink = opt.deepThink || false;
1994
- const cacheable = opt.cacheable || true;
2025
+ const prompt = opt.prompt ?? locatePrompt;
2026
+ const deepThink = opt.deepThink ?? false;
2027
+ const cacheable = opt.cacheable ?? true;
1995
2028
  return {
1996
2029
  prompt,
1997
2030
  deepThink,
@@ -2010,7 +2043,8 @@ var PageAgent = class {
2010
2043
  const plans = buildPlans("Tap", detailedLocateParam);
2011
2044
  const { executor, output } = await this.taskExecutor.runPlans(
2012
2045
  taskTitleStr("Tap", locateParamStr(detailedLocateParam)),
2013
- plans
2046
+ plans,
2047
+ { cacheable: opt?.cacheable }
2014
2048
  );
2015
2049
  const metadata = this.afterTaskRunning(executor);
2016
2050
  return {
@@ -2026,7 +2060,8 @@ var PageAgent = class {
2026
2060
  const plans = buildPlans("Hover", detailedLocateParam);
2027
2061
  const { executor, output } = await this.taskExecutor.runPlans(
2028
2062
  taskTitleStr("Hover", locateParamStr(detailedLocateParam)),
2029
- plans
2063
+ plans,
2064
+ { cacheable: opt?.cacheable }
2030
2065
  );
2031
2066
  const metadata = this.afterTaskRunning(executor);
2032
2067
  return {
@@ -2049,7 +2084,8 @@ var PageAgent = class {
2049
2084
  });
2050
2085
  const { executor, output } = await this.taskExecutor.runPlans(
2051
2086
  taskTitleStr("Input", locateParamStr(detailedLocateParam)),
2052
- plans
2087
+ plans,
2088
+ { cacheable: opt?.cacheable }
2053
2089
  );
2054
2090
  const metadata = this.afterTaskRunning(executor);
2055
2091
  return {
@@ -2065,7 +2101,8 @@ var PageAgent = class {
2065
2101
  });
2066
2102
  const { executor, output } = await this.taskExecutor.runPlans(
2067
2103
  taskTitleStr("KeyboardPress", locateParamStr(detailedLocateParam)),
2068
- plans
2104
+ plans,
2105
+ { cacheable: opt?.cacheable }
2069
2106
  );
2070
2107
  const metadata = this.afterTaskRunning(executor);
2071
2108
  return {
@@ -2079,7 +2116,8 @@ var PageAgent = class {
2079
2116
  const paramInTitle = locatePrompt ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
2080
2117
  const { executor, output } = await this.taskExecutor.runPlans(
2081
2118
  taskTitleStr("Scroll", paramInTitle),
2082
- plans
2119
+ plans,
2120
+ { cacheable: opt?.cacheable }
2083
2121
  );
2084
2122
  const metadata = this.afterTaskRunning(executor);
2085
2123
  return {
@@ -2088,6 +2126,19 @@ var PageAgent = class {
2088
2126
  };
2089
2127
  }
2090
2128
  async aiAction(taskPrompt, opt) {
2129
+ try {
2130
+ const aiModel = await import("misoai-core/ai-model");
2131
+ const contextStore = aiModel.getContextStore();
2132
+ const processedPrompt = contextStore.replaceAllReferences(taskPrompt, "action");
2133
+ contextStore.addStep({
2134
+ type: "action",
2135
+ summary: `Action: ${processedPrompt}`,
2136
+ prompt: processedPrompt
2137
+ });
2138
+ taskPrompt = processedPrompt;
2139
+ } catch (error) {
2140
+ debug4("Context store not available:", error);
2141
+ }
2091
2142
  const cacheable = opt?.cacheable;
2092
2143
  const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
2093
2144
  const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
@@ -2105,7 +2156,9 @@ var PageAgent = class {
2105
2156
  metadata: metadata2
2106
2157
  };
2107
2158
  }
2108
- const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext));
2159
+ const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
2160
+ cacheable
2161
+ }));
2109
2162
  if (this.taskCache && output?.yamlFlow && cacheable !== false) {
2110
2163
  const yamlContent = {
2111
2164
  tasks: [
@@ -2132,7 +2185,63 @@ var PageAgent = class {
2132
2185
  };
2133
2186
  }
2134
2187
  async aiQuery(demand) {
2135
- const { output, executor } = await this.taskExecutor.query(demand);
2188
+ let processedDemand = demand;
2189
+ let storageKey;
2190
+ try {
2191
+ const aiModel = await import("misoai-core/ai-model");
2192
+ const contextStore = aiModel.getContextStore();
2193
+ if (typeof demand === "string") {
2194
+ const storageInstruction = contextStore.parseStorageInstruction(demand);
2195
+ if (storageInstruction) {
2196
+ storageKey = storageInstruction.key;
2197
+ processedDemand = storageInstruction.cleanText;
2198
+ contextStore._pendingAliases = storageInstruction.aliases;
2199
+ } else {
2200
+ const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
2201
+ if (storageMatch) {
2202
+ storageKey = storageMatch[1];
2203
+ processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
2204
+ }
2205
+ }
2206
+ }
2207
+ } catch (error) {
2208
+ debug4("Context store not available:", error);
2209
+ }
2210
+ const { output, executor } = await this.taskExecutor.query(processedDemand);
2211
+ if (storageKey && output) {
2212
+ try {
2213
+ const aiModel = await import("misoai-core/ai-model");
2214
+ const contextStore = aiModel.getContextStore();
2215
+ const pendingAliases = contextStore._pendingAliases;
2216
+ if (pendingAliases) {
2217
+ contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
2218
+ delete contextStore._pendingAliases;
2219
+ } else {
2220
+ contextStore.storeData(storageKey, output);
2221
+ }
2222
+ contextStore.addStep({
2223
+ type: "query",
2224
+ summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
2225
+ data: output,
2226
+ prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2227
+ });
2228
+ } catch (error) {
2229
+ debug4("Failed to store query result:", error);
2230
+ }
2231
+ } else {
2232
+ try {
2233
+ const aiModel = await import("misoai-core/ai-model");
2234
+ const contextStore = aiModel.getContextStore();
2235
+ contextStore.addStep({
2236
+ type: "query",
2237
+ summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
2238
+ data: output,
2239
+ prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2240
+ });
2241
+ } catch (error) {
2242
+ debug4("Failed to add query step:", error);
2243
+ }
2244
+ }
2136
2245
  const metadata = this.afterTaskRunning(executor);
2137
2246
  return {
2138
2247
  result: output,
@@ -2227,7 +2336,8 @@ var PageAgent = class {
2227
2336
  const plans = buildPlans("Locate", detailedLocateParam);
2228
2337
  const { executor, output } = await this.taskExecutor.runPlans(
2229
2338
  taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
2230
- plans
2339
+ plans,
2340
+ { cacheable: opt?.cacheable }
2231
2341
  );
2232
2342
  const metadata = this.afterTaskRunning(executor);
2233
2343
  const { element } = output;
@@ -2241,6 +2351,19 @@ var PageAgent = class {
2241
2351
  };
2242
2352
  }
2243
2353
  async aiAssert(assertion, msg, opt) {
2354
+ let processedAssertion = assertion;
2355
+ try {
2356
+ const aiModel = await import("misoai-core/ai-model");
2357
+ const contextStore = aiModel.getContextStore();
2358
+ processedAssertion = contextStore.replaceAllReferences(assertion, "assertion");
2359
+ contextStore.addStep({
2360
+ type: "assertion",
2361
+ summary: `Assertion: ${processedAssertion}`,
2362
+ prompt: processedAssertion
2363
+ });
2364
+ } catch (error) {
2365
+ debug4("Context store not available:", error);
2366
+ }
2244
2367
  let currentUrl = "";
2245
2368
  if (this.page.url) {
2246
2369
  try {
@@ -2248,7 +2371,7 @@ var PageAgent = class {
2248
2371
  } catch (e) {
2249
2372
  }
2250
2373
  }
2251
- const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
2374
+ const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
2252
2375
  const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
2253
2376
  const metadata = this.afterTaskRunning(executor, true);
2254
2377
  if (output && opt?.keepRawResponse) {
@@ -2515,6 +2638,12 @@ var StaticPage = class {
2515
2638
  async getElementsNodeTree() {
2516
2639
  return ThrowNotImplemented("getElementsNodeTree");
2517
2640
  }
2641
+ async getXpathsById(id) {
2642
+ return ThrowNotImplemented("getXpathsById");
2643
+ }
2644
+ async getElementInfoByXpath(xpath) {
2645
+ return ThrowNotImplemented("getElementInfoByXpath");
2646
+ }
2518
2647
  async size() {
2519
2648
  return this.uiContext.size;
2520
2649
  }