misoai-web 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -8
- package/bin/midscene-playground +2 -2
- package/dist/es/agent.js +167 -44
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +64 -17
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +169 -46
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +229 -59
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +183 -45
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +173 -44
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +173 -44
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +183 -45
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +183 -45
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +183 -45
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +21 -3
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +167 -44
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +64 -17
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +169 -46
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +229 -59
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +181 -46
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +173 -44
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +173 -44
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +181 -46
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +181 -46
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +181 -46
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +21 -3
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +16 -6
- package/dist/types/bridge-mode-browser.d.ts +2 -2
- package/dist/types/bridge-mode.d.ts +2 -2
- package/dist/types/{browser-d447695b.d.ts → browser-a1877d18.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -2
- package/dist/types/index.d.ts +1 -1
- package/dist/types/midscene-server.d.ts +1 -1
- package/dist/types/{page-b8ada1f3.d.ts → page-663ece08.d.ts} +41 -30
- package/dist/types/playground.d.ts +2 -2
- package/dist/types/playwright.d.ts +1 -1
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
- package/dist/types/puppeteer.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +1 -1
- package/iife-script/htmlElement.js +99 -37
- package/iife-script/htmlElementDebug.js +92 -9
- package/package.json +23 -24
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,qBAAwD;AAExD,uBAAqB;;;ACKrB,sBAAiD;AACjD,mBAAuC;AACvC,iBAAsD;AAEtD,uBAKO;AACP,iBAAgC;AAEhC,IAAAA,gBAAqC;AACrC,mBAAkB;AAqIX,IAAM,yCACX;;;ADtJF,IAAAA,gBAA0B;AAC1B,uBAAuC;AACvC,IAAAC,cAAiC;AACjC,IAAAD,gBAA4B;AAC5B,kBAAiB;AACjB,oBAAmB;AACnB,qBAAoB;AAIpB,IAAM,cAAc;AAGpB,IAAM,eAAe,CAAC,KAAU,KAAU,KAAU,SAAc;AAChE,UAAQ,MAAM,GAAG;AACjB,MAAI,OAAO,GAAG,EAAE,KAAK;AAAA,IACnB,OAAO,IAAI;AAAA,EACb,CAAC;AACH;AAEA,IAAM,QAAQ,YAAY;AACxB,MAAI,CAAC,2BAAa;AAChB,UAAM,EAAE,OAAO,IAAI,cAAAE,QAAO,OAAO;AAEjC,QAAI,QAAQ;AACV,wCAAiB,MAAM;AAAA,IACzB;AAAA,EACF;AACF;AAEA,IAAqB,mBAArB,MAAsC;AAAA,EAcpC,YACE,WACA,YACA,YACA;AACA,SAAK,UAAM,eAAAC,SAAQ;AACnB,SAAK,aAAS,yBAAU;AACxB,SAAK,YAAY;AACjB,SAAK,aAAa;AAClB,SAAK,aAAa;AAClB,SAAK,mBAAmB,CAAC;AACzB,UAAM;AAAA,EACR;AAAA,EAEA,gBAAgBC,OAAc;AAC5B,eAAO,uBAAK,KAAK,QAAQ,GAAGA,KAAI,OAAO;AAAA,EACzC;AAAA,EAEA,gBAAgBA,OAAc,SAAiB;AAC7C,UAAM,UAAU,KAAK,gBAAgBA,KAAI;AACzC,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,sCAAc,SAAS,OAAO;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,MAAe;AAC1B,SAAK,OAAO,QAAQ;AACpB,SAAK,IAAI,IAAI,YAAY;AAEzB,SAAK,IAAI;AAAA,UACP,YAAAC,SAAK;AAAA,QACH,QAAQ;AAAA,QACR,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,SAAK,IAAI,IAAI,eAAW,YAAAA,SAAK,GAAG,OAAO,KAAK,QAAQ;AAElD,UAAI,KAAK;AAAA,QACP,QAAQ;AAAA,MACV,CAAC;AAAA,IACH,CAAC;AAMD,SAAK,IAAI,IAAI,kBAAkB,OAAO,KAAK,QAAQ;AACjD,YAAM,EAAE,MAAAD,MAAK,IAAI,IAAI;AACrB,YAAM,cAAc,KAAK,gBAAgBA,KAAI;AAE7C,UAAI,KAAC,2BAAW,WAAW,GAAG;AAC5B,eAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,UAC1B,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAEA,YAAM,cAAU,6BAAa,aAAa,MAAM;AAChD,UAAI,KAAK;AAAA,QACP;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAED,SAAK,IAAI,IAAI,iCAA6B,YAAAC,SAAK,GAAG,OAAO,KAAK,QAAQ;AACpE,YAAM,EAAE,UAAU,IAAI,IAAI;AAC1B,UAAI,KAAK;AAAA,QACP,KAAK,KAAK,iBAAiB,SAAS,KAAK;AAAA,MAC3C,CAAC;AAAA,IACH,CAAC;AAID,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAF,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,UAAU,IAAI,KAAK;AAEzB,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,cAAMC,YAAO,+BAAW;AACxB,aAAK,gBAAgBA,OAAM,OAAO;AAClC,eAAO,IAAI,KAAK;AAAA,UACd,UAAU,eAAeA,KAAI;AAAA,UAC7B,MAAAA;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAD,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,MAAM,QAAQ,WAAW,UAAU,IAAI,IAAI;AAE5D,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM;AACT,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,QAAQ;AACX,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAGA,cAAM,OAAO,IAAI,KAAK,UAAU,OAAO;AACvC,cAAM,QAAQ,IAAI,KAAK,WAAW,IAAI;AAEtC,YAAI,WAAW;AACb,eAAK,iBAAiB,SAAS,IAAI;AAEnC,gBAAM,iBAAiB,CAAC,QAAgB;AACtC,iBAAK,iBAAiB,SAAS,IAAI;AAAA,UACrC;AAAA,QACF;AAEA,cAAM,WAMF;AAAA,UACF,QAAQ;AAAA,UACR,MAAM;AAAA,UACN,OAAO;AAAA,UACP,YAAY;AAAA,UACZ;AAAA,QACF;AAEA,cAAM,YAAY,KAAK,IAAI;AAC3B,YAAI;AACF,cAAI,SAAS,WAAW;AACtB,qBAAS,SAAS,MAAM,MAAM,QAAQ,MAAM;AAAA,UAC9C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,MAAM;AAAA,UAC/C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,QAAQ,QAAW;AAAA,cACxD,iBAAiB;AAAA,YACnB,CAAC;AAAA,UACH,WAAW,SAAS,SAAS;AAC3B,qBAAS,SAAS,MAAM,MAAM,MAAM,QAAQ;AAAA,cAC1C;AAAA,YACF,CAAC;AAAA,UACH,OAAO;AACL,qBAAS,QAAQ,iBAAiB,IAAI;AAAA,UACxC;AAAA,QACF,SAAS,OAAY;AACnB,cAAI,CAAC,MAAM,QAAQ,SAAS,sCAAsC,GAAG;AACnE,qBAAS,QAAQ,MAAM;AAAA,UACzB;AAAA,QACF;AAEA,YAAI;AACF,mBAAS,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AACjD,mBAAS,aAAa,MAAM,iBAAiB,KAAK;AAElD,gBAAM,oBAAoB;AAAA,QAC5B,SAAS,OAAY;AACnB,kBAAQ;AAAA,YACN,qCAAqC,SAAS,KAAK,MAAM,OAAO;AAAA,UAClE;AAAA,QACF;AAEA,YAAI,KAAK,QAAQ;AACjB,cAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,YAAI,SAAS,OAAO;AAClB,kBAAQ;AAAA,YACN,+BAA+B,QAAQ,kBAAkB,SAAS,KAAK,SAAS,KAAK;AAAA,UACvF;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,6BAA6B,QAAQ,kBAAkB,SAAS;AAAA,UAClE;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAA,QAAQ,KAAK,EAAE,OAAO,MAAM,CAAC;AAAA,MAC7B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,IAAI,IAAI;AAEzB,YAAI,CAAC,YAAY,OAAO,aAAa,UAAU;AAC7C,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI;AACF,4CAAiB,QAAQ;AAEzB,iBAAO,IAAI,KAAK;AAAA,YACd,QAAQ;AAAA,YACR,SAAS;AAAA,UACX,CAAC;AAAA,QACH,SAAS,OAAY;AACnB,kBAAQ,MAAM,+BAA+B,MAAM,OAAO,EAAE;AAC5D,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO,+BAA+B,MAAM,OAAO;AAAA,UACrD,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,QAAI,KAAK,YAAY;AACnB,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAE9B,YAAI,SAAS,aAAa;AAAA,MAC5B,CAAC;AAED,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAC9B,cAAM,oBAAgB,uBAAK,KAAK,YAAa,IAAI,IAAI;AACrD,gBAAI,2BAAW,aAAa,GAAG;AAC7B,cAAI,SAAS,aAAa;AAAA,QAC5B,OAAO;AACL,cAAI,aAAS,uBAAK,KAAK,YAAa,YAAY,CAAC;AAAA,QACnD;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAMG,QAAO,KAAK;AAClB,WAAK,SAAS,KAAK,IAAI,OAAOA,OAAM,MAAM;AACxC,gBAAQ,IAAI;AAAA,MACd,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AAEN,QAAI,KAAK,QAAQ;AACf,aAAO,KAAK,OAAO,MAAM;AAAA,IAC3B;AAAA,EACF;AACF","names":["import_utils","import_env","dotenv","express","uuid","cors","port"],"ignoreList":[],"sources":["../../src/playground/server.ts","../../src/common/utils.ts"],"sourcesContent":["import { randomUUID } from 'node:crypto';\nimport { existsSync, readFileSync, writeFileSync } from 'node:fs';\nimport type { Server } from 'node:http';\nimport { join } from 'node:path';\nimport { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '@/common/utils';\nimport { getTmpDir } from 'misoai-core/utils';\nimport { PLAYGROUND_SERVER_PORT } from 'misoai-shared/constants';\nimport { overrideAIConfig } from 'misoai-shared/env';\nimport { ifInBrowser } from 'misoai-shared/utils';\nimport cors from 'cors';\nimport dotenv from 'dotenv';\nimport express from 'express';\nimport type { PageAgent } from '../common/agent';\nimport type { AbstractPage } from '../page';\n\nconst defaultPort = PLAYGROUND_SERVER_PORT;\n// const staticPath = join(__dirname, '../../static');\n\nconst errorHandler = (err: any, req: any, res: any, next: any) => {\n console.error(err);\n res.status(500).json({\n error: err.message,\n });\n};\n\nconst setup = async () => {\n if (!ifInBrowser) {\n const { parsed } = dotenv.config();\n\n if (parsed) {\n overrideAIConfig(parsed);\n }\n }\n};\n\nexport default class PlaygroundServer {\n app: express.Application;\n tmpDir: string;\n server?: Server;\n port?: number | null;\n pageClass: new (\n ...args: any[]\n ) => AbstractPage;\n agentClass: new (\n ...args: any[]\n ) => PageAgent;\n staticPath?: string;\n taskProgressTips: Record<string, string>;\n\n constructor(\n pageClass: new (...args: any[]) => AbstractPage,\n agentClass: new (...args: any[]) => PageAgent,\n staticPath?: string,\n ) {\n this.app = express();\n this.tmpDir = getTmpDir()!;\n this.pageClass = pageClass;\n this.agentClass = agentClass;\n this.staticPath = staticPath;\n this.taskProgressTips = {};\n setup();\n }\n\n filePathForUuid(uuid: string) {\n return join(this.tmpDir, `${uuid}.json`);\n }\n\n saveContextFile(uuid: string, context: string) {\n const tmpFile = this.filePathForUuid(uuid);\n console.log(`save context file: ${tmpFile}`);\n writeFileSync(tmpFile, context);\n return tmpFile;\n }\n\n async launch(port?: number) {\n this.port = port || defaultPort;\n this.app.use(errorHandler);\n\n this.app.use(\n cors({\n origin: '*',\n credentials: true,\n }),\n );\n\n this.app.get('/status', cors(), async (req, res) => {\n // const modelName = g\n res.send({\n status: 'ok',\n });\n });\n\n // this.app.get('/playground/:uuid', async (req, res) => {\n // res.sendFile(join(staticPath, 'index.html'));\n // });\n\n this.app.get('/context/:uuid', async (req, res) => {\n const { uuid } = req.params;\n const contextFile = this.filePathForUuid(uuid);\n\n if (!existsSync(contextFile)) {\n return res.status(404).json({\n error: 'Context not found',\n });\n }\n\n const context = readFileSync(contextFile, 'utf8');\n res.json({\n context,\n });\n });\n\n this.app.get('/task-progress/:requestId', cors(), async (req, res) => {\n const { requestId } = req.params;\n res.json({\n tip: this.taskProgressTips[requestId] || '',\n });\n });\n\n // -------------------------\n // actions from report file\n this.app.post(\n '/playground-with-context',\n express.json({ limit: '50mb' }),\n async (req, res) => {\n const context = req.body.context;\n\n if (!context) {\n return res.status(400).json({\n error: 'context is required',\n });\n }\n\n const uuid = randomUUID();\n this.saveContextFile(uuid, context);\n return res.json({\n location: `/playground/${uuid}`,\n uuid,\n });\n },\n );\n\n this.app.post(\n '/execute',\n express.json({ limit: '30mb' }),\n async (req, res) => {\n const { context, type, prompt, requestId, deepThink } = req.body;\n\n if (!context) {\n return res.status(400).json({\n error: 'context is required',\n });\n }\n\n if (!type) {\n return res.status(400).json({\n error: 'type is required',\n });\n }\n\n if (!prompt) {\n return res.status(400).json({\n error: 'prompt is required',\n });\n }\n\n // build an agent with context\n const page = new this.pageClass(context);\n const agent = new this.agentClass(page);\n\n if (requestId) {\n this.taskProgressTips[requestId] = '';\n\n agent.onTaskStartTip = (tip: string) => {\n this.taskProgressTips[requestId] = tip;\n };\n }\n\n const response: {\n result: any;\n dump: string | null;\n error: string | null;\n reportHTML: string | null;\n requestId?: string;\n } = {\n result: null,\n dump: null,\n error: null,\n reportHTML: null,\n requestId,\n };\n\n const startTime = Date.now();\n try {\n if (type === 'aiQuery') {\n response.result = await agent.aiQuery(prompt);\n } else if (type === 'aiAction') {\n response.result = await agent.aiAction(prompt);\n } else if (type === 'aiAssert') {\n response.result = await agent.aiAssert(prompt, undefined, {\n keepRawResponse: true,\n });\n } else if (type === 'aiTap') {\n response.result = await agent.aiTap(prompt, {\n deepThink,\n });\n } else {\n response.error = `Unknown type: ${type}`;\n }\n } catch (error: any) {\n if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {\n response.error = error.message;\n }\n }\n\n try {\n response.dump = JSON.parse(agent.dumpDataString());\n response.reportHTML = agent.reportHTMLString() || null;\n\n agent.writeOutActionDumps();\n } catch (error: any) {\n console.error(\n `write out dump failed: requestId: ${requestId}, ${error.message}`,\n );\n }\n\n res.send(response);\n const timeCost = Date.now() - startTime;\n\n if (response.error) {\n console.error(\n `handle request failed after ${timeCost}ms: requestId: ${requestId}, ${response.error}`,\n );\n } else {\n console.log(\n `handle request done after ${timeCost}ms: requestId: ${requestId}`,\n );\n }\n },\n );\n\n this.app.post(\n '/config',\n express.json({ limit: '1mb' }),\n async (req, res) => {\n const { aiConfig } = req.body;\n\n if (!aiConfig || typeof aiConfig !== 'object') {\n return res.status(400).json({\n error: 'aiConfig is required and must be an object',\n });\n }\n\n try {\n overrideAIConfig(aiConfig);\n\n return res.json({\n status: 'ok',\n message: 'AI config updated successfully',\n });\n } catch (error: any) {\n console.error(`Failed to update AI config: ${error.message}`);\n return res.status(500).json({\n error: `Failed to update AI config: ${error.message}`,\n });\n }\n },\n );\n\n // Set up static file serving after all API routes are defined\n if (this.staticPath) {\n this.app.get('/', (req, res) => {\n // compatible with windows\n res.redirect('/index.html');\n });\n\n this.app.get('*', (req, res) => {\n const requestedPath = join(this.staticPath!, req.path);\n if (existsSync(requestedPath)) {\n res.sendFile(requestedPath);\n } else {\n res.sendFile(join(this.staticPath!, 'index.html'));\n }\n });\n }\n\n return new Promise((resolve, reject) => {\n const port = this.port;\n this.server = this.app.listen(port, () => {\n resolve(this);\n });\n });\n }\n\n close() {\n // close the server\n if (this.server) {\n return this.server.close();\n }\n }\n}\n","import type { StaticPage } from '@/playground';\nimport type {\n BaseElement,\n ElementTreeNode,\n PlanningLocateParam,\n PlaywrightParserOpt,\n UIContext,\n} from 'misoai-core';\nimport { elementByPositionWithElementInfo } from 'misoai-core/ai-model';\nimport { uploadTestInfoToServer } from 'misoai-core/utils';\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from 'misoai-shared/env';\nimport type { ElementInfo } from 'misoai-shared/extractor';\nimport {\n generateElementByPosition,\n getNodeFromCacheList,\n traverseTree,\n treeToList,\n} from 'misoai-shared/extractor';\nimport { resizeImgBase64 } from 'misoai-shared/img';\nimport type { DebugFunction } from 'misoai-shared/logger';\nimport { assert, logMsg, uuid } from 'misoai-shared/utils';\nimport dayjs from 'dayjs';\nimport type { Page as PlaywrightPage } from 'playwright';\nimport type { Page as PuppeteerPage } from 'puppeteer';\nimport { WebElementInfo } from '../web-element';\nimport type { WebPage } from './page';\n\nexport type WebUIContext = UIContext<WebElementInfo> & {\n url: string;\n};\n\nexport async function parseContextFromWebPage(\n page: WebPage,\n _opt?: PlaywrightParserOpt,\n): Promise<WebUIContext> {\n assert(page, 'page is required');\n if ((page as StaticPage)._forceUsePageContext) {\n return await (page as any)._forceUsePageContext();\n }\n const url = await page.url();\n uploadTestInfoToServer({ testUrl: url });\n\n let screenshotBase64: string;\n let tree: ElementTreeNode<ElementInfo>;\n\n await Promise.all([\n page.screenshotBase64().then((base64) => {\n screenshotBase64 = base64;\n }),\n page.getElementsNodeTree().then(async (treeRoot) => {\n tree = treeRoot;\n }),\n ]);\n\n const webTree = traverseTree(tree!, (elementInfo) => {\n const { rect, id, content, attributes, locator, indexId } = elementInfo;\n return new WebElementInfo({\n rect,\n locator,\n id,\n content,\n attributes,\n indexId,\n });\n });\n\n assert(screenshotBase64!, 'screenshotBase64 is required');\n\n const elementsInfo = treeToList(webTree);\n const size = await page.size();\n\n if (size.dpr && size.dpr > 1) {\n // console.time('resizeImgBase64');\n screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n width: size.width,\n height: size.height,\n });\n // console.timeEnd('resizeImgBase64');\n }\n\n return {\n content: elementsInfo!,\n tree: webTree,\n size,\n screenshotBase64: screenshotBase64!,\n url,\n };\n}\n\nexport function reportFileName(tag = 'web') {\n const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\n const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n // ensure uniqueness at the same time\n const uniqueId = uuid().substring(0, 8);\n return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n const error = new Error();\n const stackTrace = trace || error.stack;\n const pkgDir = process.cwd() || '';\n if (stackTrace) {\n const stackLines = stackTrace.split('\\n');\n for (const line of stackLines) {\n if (\n line.includes('.spec.') ||\n line.includes('.test.') ||\n line.includes('.ts') ||\n line.includes('.js')\n ) {\n const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n if (match?.[1]) {\n const targetFileName = match[1]\n .replace(pkgDir, '')\n .trim()\n .replace('at ', '');\n return targetFileName;\n }\n }\n }\n }\n return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n let taskFile = fileName || getCurrentExecutionFile();\n if (!taskFile) {\n taskFile = uuid();\n console.warn(\n 'Midscene - using random UUID for cache id. Cache may be invalid.',\n );\n }\n\n if (testFileIndex.has(taskFile)) {\n const currentIndex = testFileIndex.get(taskFile);\n if (currentIndex !== undefined) {\n testFileIndex.set(taskFile, currentIndex + 1);\n }\n } else {\n testFileIndex.set(taskFile, 1);\n }\n return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\n 'NOT_IMPLEMENTED_AS_DESIGNED';\n\nexport function replaceIllegalPathCharsAndSpace(str: string) {\n return str.replace(/[/\\\\:*?\"<>| ]/g, '-');\n}\n\nexport function forceClosePopup(\n page: PuppeteerPage | PlaywrightPage,\n debug: DebugFunction,\n) {\n page.on('popup', async (popup) => {\n if (!popup) {\n console.warn('got a popup event, but the popup is not ready yet, skip');\n return;\n }\n const url = await (popup as PuppeteerPage).url();\n console.log(`Popup opened: ${url}`);\n if (!(popup as PuppeteerPage).isClosed()) {\n try {\n await (popup as PuppeteerPage).close(); // Close the newly opened TAB\n } catch (error) {\n debug(`failed to close popup ${url}, error: ${error}`);\n }\n } else {\n debug(`popup is already closed, skip close ${url}`);\n }\n\n if (!page.isClosed()) {\n try {\n await page.goto(url);\n } catch (error) {\n debug(`failed to goto ${url}, error: ${error}`);\n }\n } else {\n debug(`page is already closed, skip goto ${url}`);\n }\n });\n}\n\nexport function matchElementFromPlan(\n planLocateParam: PlanningLocateParam,\n tree: ElementTreeNode<BaseElement>,\n) {\n if (!planLocateParam) {\n return undefined;\n }\n if (planLocateParam.id) {\n return getNodeFromCacheList(planLocateParam.id);\n }\n\n if (planLocateParam.bbox) {\n const centerPosition = {\n x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n };\n let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n if (!element) {\n element = generateElementByPosition(centerPosition) as BaseElement;\n }\n\n return element;\n }\n\n return undefined;\n}\n"]}
|
1
|
+
{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,qBAAwD;AAExD,uBAAqB;;;ACKrB,sBAAiD;AACjD,mBAAuC;AACvC,iBAAsD;AAEtD,uBAKO;AACP,iBAAgC;AAEhC,IAAAA,gBAAqC;AACrC,mBAAkB;AAqIX,IAAM,yCACX;;;ADtJF,IAAAA,gBAA0B;AAC1B,uBAAuC;AACvC,IAAAC,cAAiC;AACjC,IAAAD,gBAA4B;AAC5B,kBAAiB;AACjB,oBAAmB;AACnB,qBAAoB;AAIpB,IAAM,cAAc;AAGpB,IAAM,eAAe,CAAC,KAAU,KAAU,KAAU,SAAc;AAChE,UAAQ,MAAM,GAAG;AACjB,MAAI,OAAO,GAAG,EAAE,KAAK;AAAA,IACnB,OAAO,IAAI;AAAA,EACb,CAAC;AACH;AAEA,IAAM,QAAQ,YAAY;AACxB,MAAI,CAAC,2BAAa;AAChB,UAAM,EAAE,OAAO,IAAI,cAAAE,QAAO,OAAO;AAEjC,QAAI,QAAQ;AACV,wCAAiB,MAAM;AAAA,IACzB;AAAA,EACF;AACF;AAEA,IAAqB,mBAArB,MAAsC;AAAA,EAcpC,YACE,WACA,YACA,YACA;AACA,SAAK,UAAM,eAAAC,SAAQ;AACnB,SAAK,aAAS,yBAAU;AACxB,SAAK,YAAY;AACjB,SAAK,aAAa;AAClB,SAAK,aAAa;AAClB,SAAK,mBAAmB,CAAC;AACzB,UAAM;AAAA,EACR;AAAA,EAEA,gBAAgBC,OAAc;AAC5B,eAAO,uBAAK,KAAK,QAAQ,GAAGA,KAAI,OAAO;AAAA,EACzC;AAAA,EAEA,gBAAgBA,OAAc,SAAiB;AAC7C,UAAM,UAAU,KAAK,gBAAgBA,KAAI;AACzC,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,sCAAc,SAAS,OAAO;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,MAAe;AAC1B,SAAK,OAAO,QAAQ;AACpB,SAAK,IAAI,IAAI,YAAY;AAEzB,SAAK,IAAI;AAAA,UACP,YAAAC,SAAK;AAAA,QACH,QAAQ;AAAA,QACR,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,SAAK,IAAI,IAAI,eAAW,YAAAA,SAAK,GAAG,OAAO,KAAK,QAAQ;AAElD,UAAI,KAAK;AAAA,QACP,QAAQ;AAAA,MACV,CAAC;AAAA,IACH,CAAC;AAMD,SAAK,IAAI,IAAI,kBAAkB,OAAO,KAAK,QAAQ;AACjD,YAAM,EAAE,MAAAD,MAAK,IAAI,IAAI;AACrB,YAAM,cAAc,KAAK,gBAAgBA,KAAI;AAE7C,UAAI,KAAC,2BAAW,WAAW,GAAG;AAC5B,eAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,UAC1B,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAEA,YAAM,cAAU,6BAAa,aAAa,MAAM;AAChD,UAAI,KAAK;AAAA,QACP;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAED,SAAK,IAAI,IAAI,iCAA6B,YAAAC,SAAK,GAAG,OAAO,KAAK,QAAQ;AACpE,YAAM,EAAE,UAAU,IAAI,IAAI;AAC1B,UAAI,KAAK;AAAA,QACP,KAAK,KAAK,iBAAiB,SAAS,KAAK;AAAA,MAC3C,CAAC;AAAA,IACH,CAAC;AAID,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAF,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,UAAU,IAAI,KAAK;AAEzB,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,cAAMC,YAAO,+BAAW;AACxB,aAAK,gBAAgBA,OAAM,OAAO;AAClC,eAAO,IAAI,KAAK;AAAA,UACd,UAAU,eAAeA,KAAI;AAAA,UAC7B,MAAAA;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAD,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,MAAM,QAAQ,WAAW,UAAU,IAAI,IAAI;AAE5D,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM;AACT,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,QAAQ;AACX,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAGA,cAAM,OAAO,IAAI,KAAK,UAAU,OAAO;AACvC,cAAM,QAAQ,IAAI,KAAK,WAAW,IAAI;AAEtC,YAAI,WAAW;AACb,eAAK,iBAAiB,SAAS,IAAI;AAEnC,gBAAM,iBAAiB,CAAC,QAAgB;AACtC,iBAAK,iBAAiB,SAAS,IAAI;AAAA,UACrC;AAAA,QACF;AAEA,cAAM,WAMF;AAAA,UACF,QAAQ;AAAA,UACR,MAAM;AAAA,UACN,OAAO;AAAA,UACP,YAAY;AAAA,UACZ;AAAA,QACF;AAEA,cAAM,YAAY,KAAK,IAAI;AAC3B,YAAI;AACF,cAAI,SAAS,WAAW;AACtB,qBAAS,SAAS,MAAM,MAAM,QAAQ,MAAM;AAAA,UAC9C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,MAAM;AAAA,UAC/C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,QAAQ,QAAW;AAAA,cACxD,iBAAiB;AAAA,YACnB,CAAC;AAAA,UACH,WAAW,SAAS,SAAS;AAC3B,qBAAS,SAAS,MAAM,MAAM,MAAM,QAAQ;AAAA,cAC1C;AAAA,YACF,CAAC;AAAA,UACH,OAAO;AACL,qBAAS,QAAQ,iBAAiB,IAAI;AAAA,UACxC;AAAA,QACF,SAAS,OAAY;AACnB,cAAI,CAAC,MAAM,QAAQ,SAAS,sCAAsC,GAAG;AACnE,qBAAS,QAAQ,MAAM;AAAA,UACzB;AAAA,QACF;AAEA,YAAI;AACF,mBAAS,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AACjD,mBAAS,aAAa,MAAM,iBAAiB,KAAK;AAElD,gBAAM,oBAAoB;AAAA,QAC5B,SAAS,OAAY;AACnB,kBAAQ;AAAA,YACN,qCAAqC,SAAS,KAAK,MAAM,OAAO;AAAA,UAClE;AAAA,QACF;AAEA,YAAI,KAAK,QAAQ;AACjB,cAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,YAAI,SAAS,OAAO;AAClB,kBAAQ;AAAA,YACN,+BAA+B,QAAQ,kBAAkB,SAAS,KAAK,SAAS,KAAK;AAAA,UACvF;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,6BAA6B,QAAQ,kBAAkB,SAAS;AAAA,UAClE;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAA,QAAQ,KAAK,EAAE,OAAO,MAAM,CAAC;AAAA,MAC7B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,IAAI,IAAI;AAEzB,YAAI,CAAC,YAAY,OAAO,aAAa,UAAU;AAC7C,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI;AACF,4CAAiB,QAAQ;AAEzB,iBAAO,IAAI,KAAK;AAAA,YACd,QAAQ;AAAA,YACR,SAAS;AAAA,UACX,CAAC;AAAA,QACH,SAAS,OAAY;AACnB,kBAAQ,MAAM,+BAA+B,MAAM,OAAO,EAAE;AAC5D,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO,+BAA+B,MAAM,OAAO;AAAA,UACrD,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,QAAI,KAAK,YAAY;AACnB,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAE9B,YAAI,SAAS,aAAa;AAAA,MAC5B,CAAC;AAED,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAC9B,cAAM,oBAAgB,uBAAK,KAAK,YAAa,IAAI,IAAI;AACrD,gBAAI,2BAAW,aAAa,GAAG;AAC7B,cAAI,SAAS,aAAa;AAAA,QAC5B,OAAO;AACL,cAAI,aAAS,uBAAK,KAAK,YAAa,YAAY,CAAC;AAAA,QACnD;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAMG,QAAO,KAAK;AAClB,WAAK,SAAS,KAAK,IAAI,OAAOA,OAAM,MAAM;AACxC,gBAAQ,IAAI;AAAA,MACd,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AAEN,QAAI,KAAK,QAAQ;AACf,aAAO,KAAK,OAAO,MAAM;AAAA,IAC3B;AAAA,EACF;AACF","names":["import_utils","import_env","dotenv","express","uuid","cors","port"],"ignoreList":[],"sources":["../../src/playground/server.ts","../../src/common/utils.ts"],"sourcesContent":["import { randomUUID } from 'node:crypto';\r\nimport { existsSync, readFileSync, writeFileSync } from 'node:fs';\r\nimport type { Server } from 'node:http';\r\nimport { join } from 'node:path';\r\nimport { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '@/common/utils';\r\nimport { getTmpDir } from 'misoai-core/utils';\r\nimport { PLAYGROUND_SERVER_PORT } from 'misoai-shared/constants';\r\nimport { overrideAIConfig } from 'misoai-shared/env';\r\nimport { ifInBrowser } from 'misoai-shared/utils';\r\nimport cors from 'cors';\r\nimport dotenv from 'dotenv';\r\nimport express from 'express';\r\nimport type { PageAgent } from '../common/agent';\r\nimport type { AbstractPage } from '../page';\r\n\r\nconst defaultPort = PLAYGROUND_SERVER_PORT;\r\n// const staticPath = join(__dirname, '../../static');\r\n\r\nconst errorHandler = (err: any, req: any, res: any, next: any) => {\r\n console.error(err);\r\n res.status(500).json({\r\n error: err.message,\r\n });\r\n};\r\n\r\nconst setup = async () => {\r\n if (!ifInBrowser) {\r\n const { parsed } = dotenv.config();\r\n\r\n if (parsed) {\r\n overrideAIConfig(parsed);\r\n }\r\n }\r\n};\r\n\r\nexport default class PlaygroundServer {\r\n app: express.Application;\r\n tmpDir: string;\r\n server?: Server;\r\n port?: number | null;\r\n pageClass: new (\r\n ...args: any[]\r\n ) => AbstractPage;\r\n agentClass: new (\r\n ...args: any[]\r\n ) => PageAgent;\r\n staticPath?: string;\r\n taskProgressTips: Record<string, string>;\r\n\r\n constructor(\r\n pageClass: new (...args: any[]) => AbstractPage,\r\n agentClass: new (...args: any[]) => PageAgent,\r\n staticPath?: string,\r\n ) {\r\n this.app = express();\r\n this.tmpDir = getTmpDir()!;\r\n this.pageClass = pageClass;\r\n this.agentClass = agentClass;\r\n this.staticPath = staticPath;\r\n this.taskProgressTips = {};\r\n setup();\r\n }\r\n\r\n filePathForUuid(uuid: string) {\r\n return join(this.tmpDir, `${uuid}.json`);\r\n }\r\n\r\n saveContextFile(uuid: string, context: string) {\r\n const tmpFile = this.filePathForUuid(uuid);\r\n console.log(`save context file: ${tmpFile}`);\r\n writeFileSync(tmpFile, context);\r\n return tmpFile;\r\n }\r\n\r\n async launch(port?: number) {\r\n this.port = port || defaultPort;\r\n this.app.use(errorHandler);\r\n\r\n this.app.use(\r\n cors({\r\n origin: '*',\r\n credentials: true,\r\n }),\r\n );\r\n\r\n this.app.get('/status', cors(), async (req, res) => {\r\n // const modelName = g\r\n res.send({\r\n status: 'ok',\r\n });\r\n });\r\n\r\n // this.app.get('/playground/:uuid', async (req, res) => {\r\n // res.sendFile(join(staticPath, 'index.html'));\r\n // });\r\n\r\n this.app.get('/context/:uuid', async (req, res) => {\r\n const { uuid } = req.params;\r\n const contextFile = this.filePathForUuid(uuid);\r\n\r\n if (!existsSync(contextFile)) {\r\n return res.status(404).json({\r\n error: 'Context not found',\r\n });\r\n }\r\n\r\n const context = readFileSync(contextFile, 'utf8');\r\n res.json({\r\n context,\r\n });\r\n });\r\n\r\n this.app.get('/task-progress/:requestId', cors(), async (req, res) => {\r\n const { requestId } = req.params;\r\n res.json({\r\n tip: this.taskProgressTips[requestId] || '',\r\n });\r\n });\r\n\r\n // -------------------------\r\n // actions from report file\r\n this.app.post(\r\n '/playground-with-context',\r\n express.json({ limit: '50mb' }),\r\n async (req, res) => {\r\n const context = req.body.context;\r\n\r\n if (!context) {\r\n return res.status(400).json({\r\n error: 'context is required',\r\n });\r\n }\r\n\r\n const uuid = randomUUID();\r\n this.saveContextFile(uuid, context);\r\n return res.json({\r\n location: `/playground/${uuid}`,\r\n uuid,\r\n });\r\n },\r\n );\r\n\r\n this.app.post(\r\n '/execute',\r\n express.json({ limit: '30mb' }),\r\n async (req, res) => {\r\n const { context, type, prompt, requestId, deepThink } = req.body;\r\n\r\n if (!context) {\r\n return res.status(400).json({\r\n error: 'context is required',\r\n });\r\n }\r\n\r\n if (!type) {\r\n return res.status(400).json({\r\n error: 'type is required',\r\n });\r\n }\r\n\r\n if (!prompt) {\r\n return res.status(400).json({\r\n error: 'prompt is required',\r\n });\r\n }\r\n\r\n // build an agent with context\r\n const page = new this.pageClass(context);\r\n const agent = new this.agentClass(page);\r\n\r\n if (requestId) {\r\n this.taskProgressTips[requestId] = '';\r\n\r\n agent.onTaskStartTip = (tip: string) => {\r\n this.taskProgressTips[requestId] = tip;\r\n };\r\n }\r\n\r\n const response: {\r\n result: any;\r\n dump: string | null;\r\n error: string | null;\r\n reportHTML: string | null;\r\n requestId?: string;\r\n } = {\r\n result: null,\r\n dump: null,\r\n error: null,\r\n reportHTML: null,\r\n requestId,\r\n };\r\n\r\n const startTime = Date.now();\r\n try {\r\n if (type === 'aiQuery') {\r\n response.result = await agent.aiQuery(prompt);\r\n } else if (type === 'aiAction') {\r\n response.result = await agent.aiAction(prompt);\r\n } else if (type === 'aiAssert') {\r\n response.result = await agent.aiAssert(prompt, undefined, {\r\n keepRawResponse: true,\r\n });\r\n } else if (type === 'aiTap') {\r\n response.result = await agent.aiTap(prompt, {\r\n deepThink,\r\n });\r\n } else {\r\n response.error = `Unknown type: ${type}`;\r\n }\r\n } catch (error: any) {\r\n if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {\r\n response.error = error.message;\r\n }\r\n }\r\n\r\n try {\r\n response.dump = JSON.parse(agent.dumpDataString());\r\n response.reportHTML = agent.reportHTMLString() || null;\r\n\r\n agent.writeOutActionDumps();\r\n } catch (error: any) {\r\n console.error(\r\n `write out dump failed: requestId: ${requestId}, ${error.message}`,\r\n );\r\n }\r\n\r\n res.send(response);\r\n const timeCost = Date.now() - startTime;\r\n\r\n if (response.error) {\r\n console.error(\r\n `handle request failed after ${timeCost}ms: requestId: ${requestId}, ${response.error}`,\r\n );\r\n } else {\r\n console.log(\r\n `handle request done after ${timeCost}ms: requestId: ${requestId}`,\r\n );\r\n }\r\n },\r\n );\r\n\r\n this.app.post(\r\n '/config',\r\n express.json({ limit: '1mb' }),\r\n async (req, res) => {\r\n const { aiConfig } = req.body;\r\n\r\n if (!aiConfig || typeof aiConfig !== 'object') {\r\n return res.status(400).json({\r\n error: 'aiConfig is required and must be an object',\r\n });\r\n }\r\n\r\n try {\r\n overrideAIConfig(aiConfig);\r\n\r\n return res.json({\r\n status: 'ok',\r\n message: 'AI config updated successfully',\r\n });\r\n } catch (error: any) {\r\n console.error(`Failed to update AI config: ${error.message}`);\r\n return res.status(500).json({\r\n error: `Failed to update AI config: ${error.message}`,\r\n });\r\n }\r\n },\r\n );\r\n\r\n // Set up static file serving after all API routes are defined\r\n if (this.staticPath) {\r\n this.app.get('/', (req, res) => {\r\n // compatible with windows\r\n res.redirect('/index.html');\r\n });\r\n\r\n this.app.get('*', (req, res) => {\r\n const requestedPath = join(this.staticPath!, req.path);\r\n if (existsSync(requestedPath)) {\r\n res.sendFile(requestedPath);\r\n } else {\r\n res.sendFile(join(this.staticPath!, 'index.html'));\r\n }\r\n });\r\n }\r\n\r\n return new Promise((resolve, reject) => {\r\n const port = this.port;\r\n this.server = this.app.listen(port, () => {\r\n resolve(this);\r\n });\r\n });\r\n }\r\n\r\n close() {\r\n // close the server\r\n if (this.server) {\r\n return this.server.close();\r\n }\r\n }\r\n}\r\n","import type { StaticPage } from '@/playground';\r\nimport type {\r\n BaseElement,\r\n ElementTreeNode,\r\n PlanningLocateParam,\r\n PlaywrightParserOpt,\r\n UIContext,\r\n} from 'misoai-core';\r\nimport { elementByPositionWithElementInfo } from 'misoai-core/ai-model';\r\nimport { uploadTestInfoToServer } from 'misoai-core/utils';\r\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from 'misoai-shared/env';\r\nimport type { ElementInfo } from 'misoai-shared/extractor';\r\nimport {\r\n generateElementByPosition,\r\n getNodeFromCacheList,\r\n traverseTree,\r\n treeToList,\r\n} from 'misoai-shared/extractor';\r\nimport { resizeImgBase64 } from 'misoai-shared/img';\r\nimport type { DebugFunction } from 'misoai-shared/logger';\r\nimport { assert, logMsg, uuid } from 'misoai-shared/utils';\r\nimport dayjs from 'dayjs';\r\nimport type { Page as PlaywrightPage } from 'playwright';\r\nimport type { Page as PuppeteerPage } from 'puppeteer';\r\nimport { WebElementInfo } from '../web-element';\r\nimport type { WebPage } from './page';\r\n\r\nexport type WebUIContext = UIContext<WebElementInfo> & {\r\n url: string;\r\n};\r\n\r\nexport async function parseContextFromWebPage(\r\n page: WebPage,\r\n _opt?: PlaywrightParserOpt,\r\n): Promise<WebUIContext> {\r\n assert(page, 'page is required');\r\n if ((page as StaticPage)._forceUsePageContext) {\r\n return await (page as any)._forceUsePageContext();\r\n }\r\n const url = await page.url();\r\n uploadTestInfoToServer({ testUrl: url });\r\n\r\n let screenshotBase64: string;\r\n let tree: ElementTreeNode<ElementInfo>;\r\n\r\n await Promise.all([\r\n page.screenshotBase64().then((base64) => {\r\n screenshotBase64 = base64;\r\n }),\r\n page.getElementsNodeTree().then(async (treeRoot) => {\r\n tree = treeRoot;\r\n }),\r\n ]);\r\n\r\n const webTree = traverseTree(tree!, (elementInfo) => {\r\n const { rect, id, content, attributes, locator, indexId } = elementInfo;\r\n return new WebElementInfo({\r\n rect,\r\n locator,\r\n id,\r\n content,\r\n attributes,\r\n indexId,\r\n });\r\n });\r\n\r\n assert(screenshotBase64!, 'screenshotBase64 is required');\r\n\r\n const elementsInfo = treeToList(webTree);\r\n const size = await page.size();\r\n\r\n if (size.dpr && size.dpr > 1) {\r\n // console.time('resizeImgBase64');\r\n screenshotBase64 = await resizeImgBase64(screenshotBase64, {\r\n width: size.width,\r\n height: size.height,\r\n });\r\n // console.timeEnd('resizeImgBase64');\r\n }\r\n\r\n return {\r\n content: elementsInfo!,\r\n tree: webTree,\r\n size,\r\n screenshotBase64: screenshotBase64!,\r\n url,\r\n };\r\n}\r\n\r\nexport function reportFileName(tag = 'web') {\r\n const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\r\n const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\r\n // ensure uniqueness at the same time\r\n const uniqueId = uuid().substring(0, 8);\r\n return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\r\n}\r\n\r\nexport function printReportMsg(filepath: string) {\r\n logMsg(`Midscene - report file updated: ${filepath}`);\r\n}\r\n\r\n/**\r\n * Get the current execution file name\r\n * @returns The name of the current execution file\r\n */\r\nexport function getCurrentExecutionFile(trace?: string): string | false {\r\n const error = new Error();\r\n const stackTrace = trace || error.stack;\r\n const pkgDir = process.cwd() || '';\r\n if (stackTrace) {\r\n const stackLines = stackTrace.split('\\n');\r\n for (const line of stackLines) {\r\n if (\r\n line.includes('.spec.') ||\r\n line.includes('.test.') ||\r\n line.includes('.ts') ||\r\n line.includes('.js')\r\n ) {\r\n const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\r\n if (match?.[1]) {\r\n const targetFileName = match[1]\r\n .replace(pkgDir, '')\r\n .trim()\r\n .replace('at ', '');\r\n return targetFileName;\r\n }\r\n }\r\n }\r\n }\r\n return false;\r\n}\r\n\r\nconst testFileIndex = new Map<string, number>();\r\n\r\nexport function generateCacheId(fileName?: string): string {\r\n let taskFile = fileName || getCurrentExecutionFile();\r\n if (!taskFile) {\r\n taskFile = uuid();\r\n console.warn(\r\n 'Midscene - using random UUID for cache id. Cache may be invalid.',\r\n );\r\n }\r\n\r\n if (testFileIndex.has(taskFile)) {\r\n const currentIndex = testFileIndex.get(taskFile);\r\n if (currentIndex !== undefined) {\r\n testFileIndex.set(taskFile, currentIndex + 1);\r\n }\r\n } else {\r\n testFileIndex.set(taskFile, 1);\r\n }\r\n return `${taskFile}-${testFileIndex.get(taskFile)}`;\r\n}\r\n\r\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\r\n 'NOT_IMPLEMENTED_AS_DESIGNED';\r\n\r\nexport function replaceIllegalPathCharsAndSpace(str: string) {\r\n return str.replace(/[/\\\\:*?\"<>| ]/g, '-');\r\n}\r\n\r\nexport function forceClosePopup(\r\n page: PuppeteerPage | PlaywrightPage,\r\n debug: DebugFunction,\r\n) {\r\n page.on('popup', async (popup) => {\r\n if (!popup) {\r\n console.warn('got a popup event, but the popup is not ready yet, skip');\r\n return;\r\n }\r\n const url = await (popup as PuppeteerPage).url();\r\n console.log(`Popup opened: ${url}`);\r\n if (!(popup as PuppeteerPage).isClosed()) {\r\n try {\r\n await (popup as PuppeteerPage).close(); // Close the newly opened TAB\r\n } catch (error) {\r\n debug(`failed to close popup ${url}, error: ${error}`);\r\n }\r\n } else {\r\n debug(`popup is already closed, skip close ${url}`);\r\n }\r\n\r\n if (!page.isClosed()) {\r\n try {\r\n await page.goto(url);\r\n } catch (error) {\r\n debug(`failed to goto ${url}, error: ${error}`);\r\n }\r\n } else {\r\n debug(`page is already closed, skip goto ${url}`);\r\n }\r\n });\r\n}\r\n\r\nexport function matchElementFromPlan(\r\n planLocateParam: PlanningLocateParam,\r\n tree: ElementTreeNode<BaseElement>,\r\n) {\r\n if (!planLocateParam) {\r\n return undefined;\r\n }\r\n if (planLocateParam.id) {\r\n return getNodeFromCacheList(planLocateParam.id);\r\n }\r\n\r\n if (planLocateParam.bbox) {\r\n const centerPosition = {\r\n x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\r\n y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\r\n };\r\n let element = elementByPositionWithElementInfo(tree, centerPosition);\r\n\r\n if (!element) {\r\n element = generateElementByPosition(centerPosition) as BaseElement;\r\n }\r\n\r\n return element;\r\n }\r\n\r\n return undefined;\r\n}\r\n"]}
|
package/dist/lib/playground.js
CHANGED
@@ -241,7 +241,9 @@ var ScriptPlayer = class {
|
|
241
241
|
typeof prompt === "string",
|
242
242
|
"prompt for aiAction must be a string"
|
243
243
|
);
|
244
|
-
await agent.aiAction(prompt
|
244
|
+
await agent.aiAction(prompt, {
|
245
|
+
cacheable: actionTask.cacheable
|
246
|
+
});
|
245
247
|
} else if ("aiAssert" in flowItem) {
|
246
248
|
const assertTask = flowItem;
|
247
249
|
const prompt = assertTask.aiAssert;
|
@@ -443,8 +445,24 @@ function interpolateEnvVars(content) {
|
|
443
445
|
});
|
444
446
|
}
|
445
447
|
function parseYamlScript(content, filePath, ignoreCheckingTarget) {
|
446
|
-
|
447
|
-
|
448
|
+
let processedContent = content;
|
449
|
+
if (content.indexOf("android") !== -1 && content.match(/deviceId:\s*(\d+)/)) {
|
450
|
+
let matchedDeviceId;
|
451
|
+
processedContent = content.replace(
|
452
|
+
/deviceId:\s*(\d+)/g,
|
453
|
+
(match, deviceId) => {
|
454
|
+
matchedDeviceId = deviceId;
|
455
|
+
return `deviceId: '${deviceId}'`;
|
456
|
+
}
|
457
|
+
);
|
458
|
+
console.warn(
|
459
|
+
`please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`
|
460
|
+
);
|
461
|
+
}
|
462
|
+
const interpolatedContent = interpolateEnvVars(processedContent);
|
463
|
+
const obj = import_js_yaml2.default.load(interpolatedContent, {
|
464
|
+
schema: import_js_yaml2.default.JSON_SCHEMA
|
465
|
+
});
|
448
466
|
const pathTip = filePath ? `, failed to load ${filePath}` : "";
|
449
467
|
const android = typeof obj.android !== "undefined" ? Object.assign({}, obj.android || {}) : void 0;
|
450
468
|
const webConfig = obj.web || obj.target;
|
@@ -485,7 +503,6 @@ var import_misoai_core = require("misoai-core");
|
|
485
503
|
var import_ai_model2 = require("misoai-core/ai-model");
|
486
504
|
var import_utils5 = require("misoai-core/utils");
|
487
505
|
var import_constants = require("misoai-shared/constants");
|
488
|
-
var import_fs = require("misoai-shared/fs");
|
489
506
|
var import_logger = require("misoai-shared/logger");
|
490
507
|
var import_utils6 = require("misoai-shared/utils");
|
491
508
|
|
@@ -603,16 +620,18 @@ var PageTaskExecutor = class {
|
|
603
620
|
);
|
604
621
|
if (info?.id) {
|
605
622
|
elementId = info.id;
|
623
|
+
} else {
|
624
|
+
debug(
|
625
|
+
"no element id found for position node, will not update cache",
|
626
|
+
element
|
627
|
+
);
|
606
628
|
}
|
607
629
|
}
|
608
630
|
if (!elementId) {
|
609
631
|
return void 0;
|
610
632
|
}
|
611
633
|
try {
|
612
|
-
const
|
613
|
-
const result = await this.page.evaluateJavaScript?.(
|
614
|
-
`${elementInfosScriptContent}midscene_element_inspector.getXpathsById('${elementId}')`
|
615
|
-
);
|
634
|
+
const result = await this.page.getXpathsById(elementId);
|
616
635
|
return result;
|
617
636
|
} catch (error) {
|
618
637
|
debug("getXpathsById error: ", error);
|
@@ -651,7 +670,7 @@ var PageTaskExecutor = class {
|
|
651
670
|
};
|
652
671
|
return taskWithScreenshot;
|
653
672
|
}
|
654
|
-
async convertPlanToExecutable(plans) {
|
673
|
+
async convertPlanToExecutable(plans, opts) {
|
655
674
|
const tasks = [];
|
656
675
|
plans.forEach((plan2) => {
|
657
676
|
if (plan2.type === "Locate") {
|
@@ -661,7 +680,10 @@ var PageTaskExecutor = class {
|
|
661
680
|
const taskFind = {
|
662
681
|
type: "Insight",
|
663
682
|
subType: "Locate",
|
664
|
-
param: plan2.locate
|
683
|
+
param: plan2.locate ? {
|
684
|
+
...plan2.locate,
|
685
|
+
cacheable: opts?.cacheable
|
686
|
+
} : void 0,
|
665
687
|
thought: plan2.thought,
|
666
688
|
locate: plan2.locate,
|
667
689
|
executor: async (param, taskContext) => {
|
@@ -698,19 +720,21 @@ var PageTaskExecutor = class {
|
|
698
720
|
let elementFromCache = null;
|
699
721
|
try {
|
700
722
|
if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
);
|
705
|
-
if (element2?.id) {
|
706
|
-
elementFromCache = element2;
|
707
|
-
debug("cache hit, prompt: %s", cachePrompt);
|
708
|
-
cacheHitFlag = true;
|
709
|
-
debug(
|
710
|
-
"found a new new element with same xpath, xpath: %s, id: %s",
|
711
|
-
xpaths[0],
|
712
|
-
element2?.id
|
723
|
+
for (let i = 0; i < xpaths.length; i++) {
|
724
|
+
const element2 = await this.page.getElementInfoByXpath(
|
725
|
+
xpaths[i]
|
713
726
|
);
|
727
|
+
if (element2?.id) {
|
728
|
+
elementFromCache = element2;
|
729
|
+
debug("cache hit, prompt: %s", cachePrompt);
|
730
|
+
cacheHitFlag = true;
|
731
|
+
debug(
|
732
|
+
"found a new new element with same xpath, xpath: %s, id: %s",
|
733
|
+
xpaths[i],
|
734
|
+
element2?.id
|
735
|
+
);
|
736
|
+
break;
|
737
|
+
}
|
714
738
|
}
|
715
739
|
}
|
716
740
|
} catch (error) {
|
@@ -723,12 +747,14 @@ var PageTaskExecutor = class {
|
|
723
747
|
context: pageContext
|
724
748
|
})).element;
|
725
749
|
const aiCost = Date.now() - startTime;
|
750
|
+
let currentXpaths;
|
726
751
|
if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
|
727
752
|
const elementXpaths = await this.getElementXpath(
|
728
753
|
pageContext,
|
729
754
|
element
|
730
755
|
);
|
731
|
-
if (elementXpaths) {
|
756
|
+
if (elementXpaths?.length) {
|
757
|
+
currentXpaths = elementXpaths;
|
732
758
|
this.taskCache.updateOrAppendCacheRecord(
|
733
759
|
{
|
734
760
|
type: "locate",
|
@@ -738,7 +764,11 @@ var PageTaskExecutor = class {
|
|
738
764
|
locateCacheRecord
|
739
765
|
);
|
740
766
|
} else {
|
741
|
-
debug(
|
767
|
+
debug(
|
768
|
+
"no xpaths found, will not update cache",
|
769
|
+
cachePrompt,
|
770
|
+
elementXpaths
|
771
|
+
);
|
742
772
|
}
|
743
773
|
}
|
744
774
|
if (!element) {
|
@@ -750,7 +780,9 @@ var PageTaskExecutor = class {
|
|
750
780
|
},
|
751
781
|
pageContext,
|
752
782
|
cache: {
|
753
|
-
hit: cacheHitFlag
|
783
|
+
hit: cacheHitFlag,
|
784
|
+
originalXpaths: xpaths,
|
785
|
+
currentXpaths
|
754
786
|
},
|
755
787
|
aiCost
|
756
788
|
};
|
@@ -1118,6 +1150,7 @@ var PageTaskExecutor = class {
|
|
1118
1150
|
sleep: sleep2
|
1119
1151
|
} = planResult;
|
1120
1152
|
executorContext.task.log = {
|
1153
|
+
...executorContext.task.log || {},
|
1121
1154
|
rawResponse
|
1122
1155
|
};
|
1123
1156
|
executorContext.task.usage = usage;
|
@@ -1242,11 +1275,11 @@ var PageTaskExecutor = class {
|
|
1242
1275
|
};
|
1243
1276
|
return task;
|
1244
1277
|
}
|
1245
|
-
async runPlans(title, plans) {
|
1278
|
+
async runPlans(title, plans, opts) {
|
1246
1279
|
const taskExecutor = new import_misoai_core.Executor(title, {
|
1247
1280
|
onTaskStart: this.onTaskStartCallback
|
1248
1281
|
});
|
1249
|
-
const { tasks } = await this.convertPlanToExecutable(plans);
|
1282
|
+
const { tasks } = await this.convertPlanToExecutable(plans, opts);
|
1250
1283
|
await taskExecutor.append(tasks);
|
1251
1284
|
const result = await taskExecutor.flush();
|
1252
1285
|
return {
|
@@ -1254,7 +1287,7 @@ var PageTaskExecutor = class {
|
|
1254
1287
|
executor: taskExecutor
|
1255
1288
|
};
|
1256
1289
|
}
|
1257
|
-
async action(userPrompt, actionContext) {
|
1290
|
+
async action(userPrompt, actionContext, opts) {
|
1258
1291
|
const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
|
1259
1292
|
onTaskStart: this.onTaskStartCallback
|
1260
1293
|
});
|
@@ -1279,7 +1312,7 @@ var PageTaskExecutor = class {
|
|
1279
1312
|
yamlFlow.push(...planResult.yamlFlow || []);
|
1280
1313
|
let executables;
|
1281
1314
|
try {
|
1282
|
-
executables = await this.convertPlanToExecutable(plans);
|
1315
|
+
executables = await this.convertPlanToExecutable(plans, opts);
|
1283
1316
|
taskExecutor.append(executables.tasks);
|
1284
1317
|
} catch (error) {
|
1285
1318
|
return this.appendErrorPlan(
|
@@ -1317,7 +1350,7 @@ var PageTaskExecutor = class {
|
|
1317
1350
|
executor: taskExecutor
|
1318
1351
|
};
|
1319
1352
|
}
|
1320
|
-
async actionToGoal(userPrompt) {
|
1353
|
+
async actionToGoal(userPrompt, opts) {
|
1321
1354
|
const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
|
1322
1355
|
onTaskStart: this.onTaskStartCallback
|
1323
1356
|
});
|
@@ -1341,7 +1374,7 @@ var PageTaskExecutor = class {
|
|
1341
1374
|
yamlFlow.push(...output.yamlFlow || []);
|
1342
1375
|
let executables;
|
1343
1376
|
try {
|
1344
|
-
executables = await this.convertPlanToExecutable(plans);
|
1377
|
+
executables = await this.convertPlanToExecutable(plans, opts);
|
1345
1378
|
taskExecutor.append(executables.tasks);
|
1346
1379
|
} catch (error) {
|
1347
1380
|
return this.appendErrorPlan(
|
@@ -1646,7 +1679,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
|
|
1646
1679
|
var import_semver = __toESM(require("semver"));
|
1647
1680
|
|
1648
1681
|
// package.json
|
1649
|
-
var version = "1.0.
|
1682
|
+
var version = "1.0.4";
|
1650
1683
|
|
1651
1684
|
// src/common/task-cache.ts
|
1652
1685
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -1989,9 +2022,9 @@ var PageAgent = class {
|
|
1989
2022
|
buildDetailedLocateParam(locatePrompt, opt) {
|
1990
2023
|
(0, import_utils12.assert)(locatePrompt, "missing locate prompt");
|
1991
2024
|
if (typeof opt === "object") {
|
1992
|
-
const prompt = opt.prompt
|
1993
|
-
const deepThink = opt.deepThink
|
1994
|
-
const cacheable = opt.cacheable
|
2025
|
+
const prompt = opt.prompt ?? locatePrompt;
|
2026
|
+
const deepThink = opt.deepThink ?? false;
|
2027
|
+
const cacheable = opt.cacheable ?? true;
|
1995
2028
|
return {
|
1996
2029
|
prompt,
|
1997
2030
|
deepThink,
|
@@ -2010,7 +2043,8 @@ var PageAgent = class {
|
|
2010
2043
|
const plans = buildPlans("Tap", detailedLocateParam);
|
2011
2044
|
const { executor, output } = await this.taskExecutor.runPlans(
|
2012
2045
|
taskTitleStr("Tap", locateParamStr(detailedLocateParam)),
|
2013
|
-
plans
|
2046
|
+
plans,
|
2047
|
+
{ cacheable: opt?.cacheable }
|
2014
2048
|
);
|
2015
2049
|
const metadata = this.afterTaskRunning(executor);
|
2016
2050
|
return {
|
@@ -2026,7 +2060,8 @@ var PageAgent = class {
|
|
2026
2060
|
const plans = buildPlans("Hover", detailedLocateParam);
|
2027
2061
|
const { executor, output } = await this.taskExecutor.runPlans(
|
2028
2062
|
taskTitleStr("Hover", locateParamStr(detailedLocateParam)),
|
2029
|
-
plans
|
2063
|
+
plans,
|
2064
|
+
{ cacheable: opt?.cacheable }
|
2030
2065
|
);
|
2031
2066
|
const metadata = this.afterTaskRunning(executor);
|
2032
2067
|
return {
|
@@ -2049,7 +2084,8 @@ var PageAgent = class {
|
|
2049
2084
|
});
|
2050
2085
|
const { executor, output } = await this.taskExecutor.runPlans(
|
2051
2086
|
taskTitleStr("Input", locateParamStr(detailedLocateParam)),
|
2052
|
-
plans
|
2087
|
+
plans,
|
2088
|
+
{ cacheable: opt?.cacheable }
|
2053
2089
|
);
|
2054
2090
|
const metadata = this.afterTaskRunning(executor);
|
2055
2091
|
return {
|
@@ -2065,7 +2101,8 @@ var PageAgent = class {
|
|
2065
2101
|
});
|
2066
2102
|
const { executor, output } = await this.taskExecutor.runPlans(
|
2067
2103
|
taskTitleStr("KeyboardPress", locateParamStr(detailedLocateParam)),
|
2068
|
-
plans
|
2104
|
+
plans,
|
2105
|
+
{ cacheable: opt?.cacheable }
|
2069
2106
|
);
|
2070
2107
|
const metadata = this.afterTaskRunning(executor);
|
2071
2108
|
return {
|
@@ -2079,7 +2116,8 @@ var PageAgent = class {
|
|
2079
2116
|
const paramInTitle = locatePrompt ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
|
2080
2117
|
const { executor, output } = await this.taskExecutor.runPlans(
|
2081
2118
|
taskTitleStr("Scroll", paramInTitle),
|
2082
|
-
plans
|
2119
|
+
plans,
|
2120
|
+
{ cacheable: opt?.cacheable }
|
2083
2121
|
);
|
2084
2122
|
const metadata = this.afterTaskRunning(executor);
|
2085
2123
|
return {
|
@@ -2088,6 +2126,19 @@ var PageAgent = class {
|
|
2088
2126
|
};
|
2089
2127
|
}
|
2090
2128
|
async aiAction(taskPrompt, opt) {
|
2129
|
+
try {
|
2130
|
+
const aiModel = await import("misoai-core/ai-model");
|
2131
|
+
const contextStore = aiModel.getContextStore();
|
2132
|
+
const processedPrompt = contextStore.replaceAllReferences(taskPrompt, "action");
|
2133
|
+
contextStore.addStep({
|
2134
|
+
type: "action",
|
2135
|
+
summary: `Action: ${processedPrompt}`,
|
2136
|
+
prompt: processedPrompt
|
2137
|
+
});
|
2138
|
+
taskPrompt = processedPrompt;
|
2139
|
+
} catch (error) {
|
2140
|
+
debug4("Context store not available:", error);
|
2141
|
+
}
|
2091
2142
|
const cacheable = opt?.cacheable;
|
2092
2143
|
const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
|
2093
2144
|
const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
|
@@ -2105,7 +2156,9 @@ var PageAgent = class {
|
|
2105
2156
|
metadata: metadata2
|
2106
2157
|
};
|
2107
2158
|
}
|
2108
|
-
const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext
|
2159
|
+
const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
|
2160
|
+
cacheable
|
2161
|
+
}));
|
2109
2162
|
if (this.taskCache && output?.yamlFlow && cacheable !== false) {
|
2110
2163
|
const yamlContent = {
|
2111
2164
|
tasks: [
|
@@ -2132,7 +2185,63 @@ var PageAgent = class {
|
|
2132
2185
|
};
|
2133
2186
|
}
|
2134
2187
|
async aiQuery(demand) {
|
2135
|
-
|
2188
|
+
let processedDemand = demand;
|
2189
|
+
let storageKey;
|
2190
|
+
try {
|
2191
|
+
const aiModel = await import("misoai-core/ai-model");
|
2192
|
+
const contextStore = aiModel.getContextStore();
|
2193
|
+
if (typeof demand === "string") {
|
2194
|
+
const storageInstruction = contextStore.parseStorageInstruction(demand);
|
2195
|
+
if (storageInstruction) {
|
2196
|
+
storageKey = storageInstruction.key;
|
2197
|
+
processedDemand = storageInstruction.cleanText;
|
2198
|
+
contextStore._pendingAliases = storageInstruction.aliases;
|
2199
|
+
} else {
|
2200
|
+
const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
|
2201
|
+
if (storageMatch) {
|
2202
|
+
storageKey = storageMatch[1];
|
2203
|
+
processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
|
2204
|
+
}
|
2205
|
+
}
|
2206
|
+
}
|
2207
|
+
} catch (error) {
|
2208
|
+
debug4("Context store not available:", error);
|
2209
|
+
}
|
2210
|
+
const { output, executor } = await this.taskExecutor.query(processedDemand);
|
2211
|
+
if (storageKey && output) {
|
2212
|
+
try {
|
2213
|
+
const aiModel = await import("misoai-core/ai-model");
|
2214
|
+
const contextStore = aiModel.getContextStore();
|
2215
|
+
const pendingAliases = contextStore._pendingAliases;
|
2216
|
+
if (pendingAliases) {
|
2217
|
+
contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
|
2218
|
+
delete contextStore._pendingAliases;
|
2219
|
+
} else {
|
2220
|
+
contextStore.storeData(storageKey, output);
|
2221
|
+
}
|
2222
|
+
contextStore.addStep({
|
2223
|
+
type: "query",
|
2224
|
+
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
|
2225
|
+
data: output,
|
2226
|
+
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2227
|
+
});
|
2228
|
+
} catch (error) {
|
2229
|
+
debug4("Failed to store query result:", error);
|
2230
|
+
}
|
2231
|
+
} else {
|
2232
|
+
try {
|
2233
|
+
const aiModel = await import("misoai-core/ai-model");
|
2234
|
+
const contextStore = aiModel.getContextStore();
|
2235
|
+
contextStore.addStep({
|
2236
|
+
type: "query",
|
2237
|
+
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
|
2238
|
+
data: output,
|
2239
|
+
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2240
|
+
});
|
2241
|
+
} catch (error) {
|
2242
|
+
debug4("Failed to add query step:", error);
|
2243
|
+
}
|
2244
|
+
}
|
2136
2245
|
const metadata = this.afterTaskRunning(executor);
|
2137
2246
|
return {
|
2138
2247
|
result: output,
|
@@ -2227,7 +2336,8 @@ var PageAgent = class {
|
|
2227
2336
|
const plans = buildPlans("Locate", detailedLocateParam);
|
2228
2337
|
const { executor, output } = await this.taskExecutor.runPlans(
|
2229
2338
|
taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
|
2230
|
-
plans
|
2339
|
+
plans,
|
2340
|
+
{ cacheable: opt?.cacheable }
|
2231
2341
|
);
|
2232
2342
|
const metadata = this.afterTaskRunning(executor);
|
2233
2343
|
const { element } = output;
|
@@ -2241,6 +2351,19 @@ var PageAgent = class {
|
|
2241
2351
|
};
|
2242
2352
|
}
|
2243
2353
|
async aiAssert(assertion, msg, opt) {
|
2354
|
+
let processedAssertion = assertion;
|
2355
|
+
try {
|
2356
|
+
const aiModel = await import("misoai-core/ai-model");
|
2357
|
+
const contextStore = aiModel.getContextStore();
|
2358
|
+
processedAssertion = contextStore.replaceAllReferences(assertion, "assertion");
|
2359
|
+
contextStore.addStep({
|
2360
|
+
type: "assertion",
|
2361
|
+
summary: `Assertion: ${processedAssertion}`,
|
2362
|
+
prompt: processedAssertion
|
2363
|
+
});
|
2364
|
+
} catch (error) {
|
2365
|
+
debug4("Context store not available:", error);
|
2366
|
+
}
|
2244
2367
|
let currentUrl = "";
|
2245
2368
|
if (this.page.url) {
|
2246
2369
|
try {
|
@@ -2248,7 +2371,7 @@ var PageAgent = class {
|
|
2248
2371
|
} catch (e) {
|
2249
2372
|
}
|
2250
2373
|
}
|
2251
|
-
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${
|
2374
|
+
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
|
2252
2375
|
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2253
2376
|
const metadata = this.afterTaskRunning(executor, true);
|
2254
2377
|
if (output && opt?.keepRawResponse) {
|
@@ -2515,6 +2638,12 @@ var StaticPage = class {
|
|
2515
2638
|
async getElementsNodeTree() {
|
2516
2639
|
return ThrowNotImplemented("getElementsNodeTree");
|
2517
2640
|
}
|
2641
|
+
async getXpathsById(id) {
|
2642
|
+
return ThrowNotImplemented("getXpathsById");
|
2643
|
+
}
|
2644
|
+
async getElementInfoByXpath(xpath) {
|
2645
|
+
return ThrowNotImplemented("getElementInfoByXpath");
|
2646
|
+
}
|
2518
2647
|
async size() {
|
2519
2648
|
return this.uiContext.size;
|
2520
2649
|
}
|