misoai-web 1.5.9 → 1.6.1

package/dist/lib/midscene-server.js

@@ -38,23 +38,23 @@ var import_node_fs = require("fs");
 var import_node_path = require("path");
 
 // src/common/utils.ts
+var import_dayjs = __toESM(require("dayjs"));
 var import_ai_model = require("misoai-core/ai-model");
 var import_utils = require("misoai-core/utils");
 var import_env = require("misoai-shared/env");
 var import_extractor = require("misoai-shared/extractor");
 var import_img = require("misoai-shared/img");
 var import_utils2 = require("misoai-shared/utils");
-var import_dayjs = __toESM(require("dayjs"));
 var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
 
 // src/playground/server.ts
+var import_cors = __toESM(require("cors"));
+var import_dotenv = __toESM(require("dotenv"));
+var import_express = __toESM(require("express"));
 var import_utils4 = require("misoai-core/utils");
 var import_constants = require("misoai-shared/constants");
 var import_env2 = require("misoai-shared/env");
 var import_utils5 = require("misoai-shared/utils");
-var import_cors = __toESM(require("cors"));
-var import_dotenv = __toESM(require("dotenv"));
-var import_express = __toESM(require("express"));
 var defaultPort = import_constants.PLAYGROUND_SERVER_PORT;
 var errorHandler = (err, req, res, next) => {
   console.error(err);

package/dist/lib/midscene-server.js.map

@@ -1 +1 @@
-{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,qBAAwD;AAExD,uBAAqB;;;ACKrB,sBAAiD;AACjD,mBAAuC;AACvC,iBAAsD;AAEtD,uBAKO;AACP,iBAAgC;AAEhC,IAAAA,gBAAqC;AACrC,mBAAkB;AAuIX,IAAM,yCACX;;;ADxJF,IAAAA,gBAA0B;AAC1B,uBAAuC;AACvC,IAAAC,cAAiC;AACjC,IAAAD,gBAA4B;AAC5B,kBAAiB;AACjB,oBAAmB;AACnB,qBAAoB;AAIpB,IAAM,cAAc;AAGpB,IAAM,eAAe,CAAC,KAAU,KAAU,KAAU,SAAc;AAChE,UAAQ,MAAM,GAAG;AACjB,MAAI,OAAO,GAAG,EAAE,KAAK;AAAA,IACnB,OAAO,IAAI;AAAA,EACb,CAAC;AACH;AAEA,IAAM,QAAQ,YAAY;AACxB,MAAI,CAAC,2BAAa;AAChB,UAAM,EAAE,OAAO,IAAI,cAAAE,QAAO,OAAO;AAEjC,QAAI,QAAQ;AACV,wCAAiB,MAAM;AAAA,IACzB;AAAA,EACF;AACF;AAEA,IAAqB,mBAArB,MAAsC;AAAA,EAcpC,YACE,WACA,YACA,YACA;AACA,SAAK,UAAM,eAAAC,SAAQ;AACnB,SAAK,aAAS,yBAAU;AACxB,SAAK,YAAY;AACjB,SAAK,aAAa;AAClB,SAAK,aAAa;AAClB,SAAK,mBAAmB,CAAC;AACzB,UAAM;AAAA,EACR;AAAA,EAEA,gBAAgBC,OAAc;AAC5B,eAAO,uBAAK,KAAK,QAAQ,GAAGA,KAAI,OAAO;AAAA,EACzC;AAAA,EAEA,gBAAgBA,OAAc,SAAiB;AAC7C,UAAM,UAAU,KAAK,gBAAgBA,KAAI;AACzC,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,sCAAc,SAAS,OAAO;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,MAAe;AAC1B,SAAK,OAAO,QAAQ;AACpB,SAAK,IAAI,IAAI,YAAY;AAEzB,SAAK,IAAI;AAAA,UACP,YAAAC,SAAK;AAAA,QACH,QAAQ;AAAA,QACR,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,SAAK,IAAI,IAAI,eAAW,YAAAA,SAAK,GAAG,OAAO,KAAK,QAAQ;AAElD,UAAI,KAAK;AAAA,QACP,QAAQ;AAAA,MACV,CAAC;AAAA,IACH,CAAC;AAMD,SAAK,IAAI,IAAI,kBAAkB,OAAO,KAAK,QAAQ;AACjD,YAAM,EAAE,MAAAD,MAAK,IAAI,IAAI;AACrB,YAAM,cAAc,KAAK,gBAAgBA,KAAI;AAE7C,UAAI,KAAC,2BAAW,WAAW,GAAG;AAC5B,eAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,UAC1B,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAEA,YAAM,cAAU,6BAAa,aAAa,MAAM;AAChD,UAAI,KAAK;AAAA,QACP;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAED,SAAK,IAAI,IAAI,iCAA6B,YAAAC,SAAK,GAAG,OAAO,KAAK,QAAQ;AACpE,YAAM,EAAE,UAAU,IAAI,IAAI;AAC1B,UAAI,KAAK;AAAA,QACP,KAAK,KAAK,iBAAiB,SAAS,KAAK;AAAA,MAC3C,CAAC;AAAA,IACH,CAAC;AAID,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAF,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,UAAU,IAAI,KAAK;AAEzB,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,cAAMC,YAAO,+BAAW;AACxB,aAAK,gBAAgBA,OAAM,OAAO;AAClC,eAAO,IAAI,KAAK;AAAA,UACd,UAAU,eAAeA,KAAI;AAAA,UAC7B,MAAAA;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAD,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,MAAM,QAAQ,WAAW,UAAU,IAAI,IAAI;AAE5D,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM;AACT,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,QAAQ;AACX,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAGA,cAAM,OAAO,IAAI,KAAK,UAAU,OAAO;AACvC,cAAM,QAAQ,IAAI,KAAK,WAAW,IAAI;AAEtC,YAAI,WAAW;AACb,eAAK,iBAAiB,SAAS,IAAI;AAEnC,gBAAM,iBAAiB,CAAC,QAAgB;AACtC,iBAAK,iBAAiB,SAAS,IAAI;AAAA,UACrC;AAAA,QACF;AAEA,cAAM,WAMF;AAAA,UACF,QAAQ;AAAA,UACR,MAAM;AAAA,UACN,OAAO;AAAA,UACP,YAAY;AAAA,UACZ;AAAA,QACF;AAEA,cAAM,YAAY,KAAK,IAAI;AAC3B,YAAI;AACF,cAAI,SAAS,WAAW;AACtB,qBAAS,SAAS,MAAM,MAAM,QAAQ,MAAM;AAAA,UAC9C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,MAAM;AAAA,UAC/C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,QAAQ,QAAW;AAAA,cACxD,iBAAiB;AAAA,YACnB,CAAC;AAAA,UACH,WAAW,SAAS,SAAS;AAC3B,qBAAS,SAAS,MAAM,MAAM,MAAM,QAAQ;AAAA,cAC1C;AAAA,YACF,CAAC;AAAA,UACH,OAAO;AACL,qBAAS,QAAQ,iBAAiB,IAAI;AAAA,UACxC;AAAA,QACF,SAAS,OAAY;AACnB,cAAI,CAAC,MAAM,QAAQ,SAAS,sCAAsC,GAAG;AACnE,qBAAS,QAAQ,MAAM;AAAA,UACzB;AAAA,QACF;AAEA,YAAI;AACF,mBAAS,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AACjD,mBAAS,aAAa,MAAM,iBAAiB,KAAK;AAElD,gBAAM,oBAAoB;AAAA,QAC5B,SAAS,OAAY;AACnB,kBAAQ;AAAA,YACN,qCAAqC,SAAS,KAAK,MAAM,OAAO;AAAA,UAClE;AAAA,QACF;AAEA,YAAI,KAAK,QAAQ;AACjB,cAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,YAAI,SAAS,OAAO;AAClB,kBAAQ;AAAA,YACN,+BAA+B,QAAQ,kBAAkB,SAAS,KAAK,SAAS,KAAK;AAAA,UACvF;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,6BAA6B,QAAQ,kBAAkB,SAAS;AAAA,UAClE;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAA,QAAQ,KAAK,EAAE,OAAO,MAAM,CAAC;AAAA,MAC7B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,IAAI,IAAI;AAEzB,YAAI,CAAC,YAAY,OAAO,aAAa,UAAU;AAC7C,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI;AACF,4CAAiB,QAAQ;AAEzB,iBAAO,IAAI,KAAK;AAAA,YACd,QAAQ;AAAA,YACR,SAAS;AAAA,UACX,CAAC;AAAA,QACH,SAAS,OAAY;AACnB,kBAAQ,MAAM,+BAA+B,MAAM,OAAO,EAAE;AAC5D,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO,+BAA+B,MAAM,OAAO;AAAA,UACrD,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,QAAI,KAAK,YAAY;AACnB,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAE9B,YAAI,SAAS,aAAa;AAAA,MAC5B,CAAC;AAED,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAC9B,cAAM,oBAAgB,uBAAK,KAAK,YAAa,IAAI,IAAI;AACrD,gBAAI,2BAAW,aAAa,GAAG;AAC7B,cAAI,SAAS,aAAa;AAAA,QAC5B,OAAO;AACL,cAAI,aAAS,uBAAK,KAAK,YAAa,YAAY,CAAC;AAAA,QACnD;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAMG,QAAO,KAAK;AAClB,WAAK,SAAS,KAAK,IAAI,OAAOA,OAAM,MAAM;AACxC,gBAAQ,IAAI;AAAA,MACd,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AAEN,QAAI,KAAK,QAAQ;AACf,aAAO,KAAK,OAAO,MAAM;AAAA,IAC3B;AAAA,EACF;AACF","names":["import_utils","import_env","dotenv","express","uuid","cors","port"],"ignoreList":[],"sources":["../../src/playground/server.ts","../../src/common/utils.ts"],"sourcesContent":["import { randomUUID } from 'node:crypto';\nimport { existsSync, readFileSync, writeFileSync } from 'node:fs';\nimport type { Server } from 'node:http';\nimport { join } from 'node:path';\nimport { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '@/common/utils';\nimport { getTmpDir } from 'misoai-core/utils';\nimport { PLAYGROUND_SERVER_PORT } from 'misoai-shared/constants';\nimport { overrideAIConfig } from 'misoai-shared/env';\nimport { ifInBrowser } from 'misoai-shared/utils';\nimport cors from 'cors';\nimport dotenv from 'dotenv';\nimport express from 'express';\nimport type { PageAgent } from '../common/agent';\nimport type { AbstractPage } from '../page';\n\nconst defaultPort = PLAYGROUND_SERVER_PORT;\n// const staticPath = join(__dirname, '../../static');\n\nconst errorHandler = (err: any, req: any, res: any, next: any) => {\n  console.error(err);\n  res.status(500).json({\n    error: err.message,\n  });\n};\n\nconst setup = async () => {\n  if (!ifInBrowser) {\n    const { parsed } = dotenv.config();\n\n    if (parsed) {\n      overrideAIConfig(parsed);\n    }\n  }\n};\n\nexport default class PlaygroundServer {\n  app: express.Application;\n  tmpDir: string;\n  server?: Server;\n  port?: number | null;\n  pageClass: new (\n    ...args: any[]\n  ) => AbstractPage;\n  agentClass: new (\n    ...args: any[]\n  ) => PageAgent;\n  staticPath?: string;\n  taskProgressTips: Record<string, string>;\n\n  constructor(\n    pageClass: new (...args: any[]) => AbstractPage,\n    agentClass: new (...args: any[]) => PageAgent,\n    staticPath?: string,\n  ) {\n    this.app = express();\n    this.tmpDir = getTmpDir()!;\n    this.pageClass = pageClass;\n    this.agentClass = agentClass;\n    this.staticPath = staticPath;\n    this.taskProgressTips = {};\n    setup();\n  }\n\n  filePathForUuid(uuid: string) {\n    return join(this.tmpDir, `${uuid}.json`);\n  }\n\n  saveContextFile(uuid: string, context: string) {\n    const tmpFile = this.filePathForUuid(uuid);\n    console.log(`save context file: ${tmpFile}`);\n    writeFileSync(tmpFile, context);\n    return tmpFile;\n  }\n\n  async launch(port?: number) {\n    this.port = port || defaultPort;\n    this.app.use(errorHandler);\n\n    this.app.use(\n      cors({\n        origin: '*',\n        credentials: true,\n      }),\n    );\n\n    this.app.get('/status', cors(), async (req, res) => {\n      // const modelName = g\n      res.send({\n        status: 'ok',\n      });\n    });\n\n    // this.app.get('/playground/:uuid', async (req, res) => {\n    //   res.sendFile(join(staticPath, 'index.html'));\n    // });\n\n    this.app.get('/context/:uuid', async (req, res) => {\n      const { uuid } = req.params;\n      const contextFile = this.filePathForUuid(uuid);\n\n      if (!existsSync(contextFile)) {\n        return res.status(404).json({\n          error: 'Context not found',\n        });\n      }\n\n      const context = readFileSync(contextFile, 'utf8');\n      res.json({\n        context,\n      });\n    });\n\n    this.app.get('/task-progress/:requestId', cors(), async (req, res) => {\n      const { requestId } = req.params;\n      res.json({\n        tip: this.taskProgressTips[requestId] || '',\n      });\n    });\n\n    // -------------------------\n    // actions from report file\n    this.app.post(\n      '/playground-with-context',\n      express.json({ limit: '50mb' }),\n      async (req, res) => {\n        const context = req.body.context;\n\n        if (!context) {\n          return res.status(400).json({\n            error: 'context is required',\n          });\n        }\n\n        const uuid = randomUUID();\n        this.saveContextFile(uuid, context);\n        return res.json({\n          location: `/playground/${uuid}`,\n          uuid,\n        });\n      },\n    );\n\n    this.app.post(\n      '/execute',\n      express.json({ limit: '30mb' }),\n      async (req, res) => {\n        const { context, type, prompt, requestId, deepThink } = req.body;\n\n        if (!context) {\n          return res.status(400).json({\n            error: 'context is required',\n          });\n        }\n\n        if (!type) {\n          return res.status(400).json({\n            error: 'type is required',\n          });\n        }\n\n        if (!prompt) {\n          return res.status(400).json({\n            error: 'prompt is required',\n          });\n        }\n\n        // build an agent with context\n        const page = new this.pageClass(context);\n        const agent = new this.agentClass(page);\n\n        if (requestId) {\n          this.taskProgressTips[requestId] = '';\n\n          agent.onTaskStartTip = (tip: string) => {\n            this.taskProgressTips[requestId] = tip;\n          };\n        }\n\n        const response: {\n          result: any;\n          dump: string | null;\n          error: string | null;\n          reportHTML: string | null;\n          requestId?: string;\n        } = {\n          result: null,\n          dump: null,\n          error: null,\n          reportHTML: null,\n          requestId,\n        };\n\n        const startTime = Date.now();\n        try {\n          if (type === 'aiQuery') {\n            response.result = await agent.aiQuery(prompt);\n          } else if (type === 'aiAction') {\n            response.result = await agent.aiAction(prompt);\n          } else if (type === 'aiAssert') {\n            response.result = await agent.aiAssert(prompt, undefined, {\n              keepRawResponse: true,\n            });\n          } else if (type === 'aiTap') {\n            response.result = await agent.aiTap(prompt, {\n              deepThink,\n            });\n          } else {\n            response.error = `Unknown type: ${type}`;\n          }\n        } catch (error: any) {\n          if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {\n            response.error = error.message;\n          }\n        }\n\n        try {\n          response.dump = JSON.parse(agent.dumpDataString());\n          response.reportHTML = agent.reportHTMLString() || null;\n\n          agent.writeOutActionDumps();\n        } catch (error: any) {\n          console.error(\n            `write out dump failed: requestId: ${requestId}, ${error.message}`,\n          );\n        }\n\n        res.send(response);\n        const timeCost = Date.now() - startTime;\n\n        if (response.error) {\n          console.error(\n            `handle request failed after ${timeCost}ms: requestId: ${requestId}, ${response.error}`,\n          );\n        } else {\n          console.log(\n            `handle request done after ${timeCost}ms: requestId: ${requestId}`,\n          );\n        }\n      },\n    );\n\n    this.app.post(\n      '/config',\n      express.json({ limit: '1mb' }),\n      async (req, res) => {\n        const { aiConfig } = req.body;\n\n        if (!aiConfig || typeof aiConfig !== 'object') {\n          return res.status(400).json({\n            error: 'aiConfig is required and must be an object',\n          });\n        }\n\n        try {\n          overrideAIConfig(aiConfig);\n\n          return res.json({\n            status: 'ok',\n            message: 'AI config updated successfully',\n          });\n        } catch (error: any) {\n          console.error(`Failed to update AI config: ${error.message}`);\n          return res.status(500).json({\n            error: `Failed to update AI config: ${error.message}`,\n          });\n        }\n      },\n    );\n\n    // Set up static file serving after all API routes are defined\n    if (this.staticPath) {\n      this.app.get('/', (req, res) => {\n        // compatible with windows\n        res.redirect('/index.html');\n      });\n\n      this.app.get('*', (req, res) => {\n        const requestedPath = join(this.staticPath!, req.path);\n        if (existsSync(requestedPath)) {\n          res.sendFile(requestedPath);\n        } else {\n          res.sendFile(join(this.staticPath!, 'index.html'));\n        }\n      });\n    }\n\n    return new Promise((resolve, reject) => {\n      const port = this.port;\n      this.server = this.app.listen(port, () => {\n        resolve(this);\n      });\n    });\n  }\n\n  close() {\n    // close the server\n    if (this.server) {\n      return this.server.close();\n    }\n  }\n}\n","import type { StaticPage } from '@/playground';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  PlanningLocateParam,\n  PlaywrightParserOpt,\n  UIContext,\n} from 'misoai-core';\nimport { elementByPositionWithElementInfo } from 'misoai-core/ai-model';\nimport { uploadTestInfoToServer } from 'misoai-core/utils';\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from 'misoai-shared/env';\nimport type { ElementInfo } from 'misoai-shared/extractor';\nimport {\n  generateElementByPosition,\n  getNodeFromCacheList,\n  traverseTree,\n  treeToList,\n} from 'misoai-shared/extractor';\nimport { resizeImgBase64 } from 'misoai-shared/img';\nimport type { DebugFunction } from 'misoai-shared/logger';\nimport { assert, logMsg, uuid } from 'misoai-shared/utils';\nimport dayjs from 'dayjs';\nimport type { Page as PlaywrightPage } from 'playwright';\nimport type { Page as PuppeteerPage } from 'puppeteer';\nimport { WebElementInfo } from '../web-element';\nimport type { WebPage } from './page';\n\nexport type WebUIContext = UIContext<WebElementInfo> & {\n  url: string;\n};\n\nexport async function parseContextFromWebPage(\n  page: WebPage,\n  _opt?: PlaywrightParserOpt,\n): Promise<WebUIContext> {\n  assert(page, 'page is required');\n  if ((page as StaticPage)._forceUsePageContext) {\n    return await (page as any)._forceUsePageContext();\n  }\n  const url = await page.url();\n  uploadTestInfoToServer({ testUrl: url });\n\n  let screenshotBase64: string;\n  let tree: ElementTreeNode<ElementInfo>;\n\n  await Promise.all([\n    page.screenshotBase64().then((base64) => {\n      screenshotBase64 = base64;\n    }),\n    page.getElementsNodeTree().then(async (treeRoot) => {\n      tree = treeRoot;\n    }),\n  ]);\n\n  const webTree = traverseTree(tree!, (elementInfo) => {\n    const { rect, id, content, attributes, locator, indexId, isVisible } =\n      elementInfo;\n    return new WebElementInfo({\n      rect,\n      locator,\n      id,\n      content,\n      attributes,\n      indexId,\n      isVisible,\n    });\n  });\n\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const elementsInfo = treeToList(webTree);\n  const size = await page.size();\n\n  if (size.dpr && size.dpr > 1) {\n    // console.time('resizeImgBase64');\n    screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n      width: size.width,\n      height: size.height,\n    });\n    // console.timeEnd('resizeImgBase64');\n  }\n\n  return {\n    content: elementsInfo!,\n    tree: webTree,\n    size,\n    screenshotBase64: screenshotBase64!,\n    url,\n  };\n}\n\nexport function reportFileName(tag = 'web') {\n  const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\n  'NOT_IMPLEMENTED_AS_DESIGNED';\n\nexport function replaceIllegalPathCharsAndSpace(str: string) {\n  // Only replace characters that are illegal in filenames, but preserve path separators\n  return str.replace(/[:*?\"<>| ]/g, '-');\n}\n\nexport function forceClosePopup(\n  page: PuppeteerPage | PlaywrightPage,\n  debug: DebugFunction,\n) {\n  page.on('popup', async (popup) => {\n    if (!popup) {\n      console.warn('got a popup event, but the popup is not ready yet, skip');\n      return;\n    }\n    const url = await (popup as PuppeteerPage).url();\n    console.log(`Popup opened: ${url}`);\n    if (!(popup as PuppeteerPage).isClosed()) {\n      try {\n        await (popup as PuppeteerPage).close(); // Close the newly opened TAB\n      } catch (error) {\n        debug(`failed to close popup ${url}, error: ${error}`);\n      }\n    } else {\n      debug(`popup is already closed, skip close ${url}`);\n    }\n\n    if (!page.isClosed()) {\n      try {\n        await page.goto(url);\n      } catch (error) {\n        debug(`failed to goto ${url}, error: ${error}`);\n      }\n    } else {\n      debug(`page is already closed, skip goto ${url}`);\n    }\n  });\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n  tree: ElementTreeNode<BaseElement>,\n) {\n  if (!planLocateParam) {\n    return undefined;\n  }\n  if (planLocateParam.id) {\n    return getNodeFromCacheList(planLocateParam.id);\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n    let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n    if (!element) {\n      element = generateElementByPosition(centerPosition) as BaseElement;\n    }\n\n    return element;\n  }\n\n  return undefined;\n}\n"]}
+{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,qBAAwD;AAExD,uBAAqB;;;ACFrB,mBAAkB;AAQlB,sBAAiD;AACjD,mBAAuC;AACvC,iBAAsD;AAEtD,uBAKO;AACP,iBAAgC;AAEhC,IAAAA,gBAAqC;AAuI9B,IAAM,yCACX;;;ADxJF,kBAAiB;AACjB,oBAAmB;AACnB,qBAAoB;AACpB,IAAAA,gBAA0B;AAC1B,uBAAuC;AACvC,IAAAC,cAAiC;AACjC,IAAAD,gBAA4B;AAI5B,IAAM,cAAc;AAGpB,IAAM,eAAe,CAAC,KAAU,KAAU,KAAU,SAAc;AAChE,UAAQ,MAAM,GAAG;AACjB,MAAI,OAAO,GAAG,EAAE,KAAK;AAAA,IACnB,OAAO,IAAI;AAAA,EACb,CAAC;AACH;AAEA,IAAM,QAAQ,YAAY;AACxB,MAAI,CAAC,2BAAa;AAChB,UAAM,EAAE,OAAO,IAAI,cAAAE,QAAO,OAAO;AAEjC,QAAI,QAAQ;AACV,wCAAiB,MAAM;AAAA,IACzB;AAAA,EACF;AACF;AAEA,IAAqB,mBAArB,MAAsC;AAAA,EAcpC,YACE,WACA,YACA,YACA;AACA,SAAK,UAAM,eAAAC,SAAQ;AACnB,SAAK,aAAS,yBAAU;AACxB,SAAK,YAAY;AACjB,SAAK,aAAa;AAClB,SAAK,aAAa;AAClB,SAAK,mBAAmB,CAAC;AACzB,UAAM;AAAA,EACR;AAAA,EAEA,gBAAgBC,OAAc;AAC5B,eAAO,uBAAK,KAAK,QAAQ,GAAGA,KAAI,OAAO;AAAA,EACzC;AAAA,EAEA,gBAAgBA,OAAc,SAAiB;AAC7C,UAAM,UAAU,KAAK,gBAAgBA,KAAI;AACzC,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,sCAAc,SAAS,OAAO;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,MAAe;AAC1B,SAAK,OAAO,QAAQ;AACpB,SAAK,IAAI,IAAI,YAAY;AAEzB,SAAK,IAAI;AAAA,UACP,YAAAC,SAAK;AAAA,QACH,QAAQ;AAAA,QACR,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,SAAK,IAAI,IAAI,eAAW,YAAAA,SAAK,GAAG,OAAO,KAAK,QAAQ;AAElD,UAAI,KAAK;AAAA,QACP,QAAQ;AAAA,MACV,CAAC;AAAA,IACH,CAAC;AAMD,SAAK,IAAI,IAAI,kBAAkB,OAAO,KAAK,QAAQ;AACjD,YAAM,EAAE,MAAAD,MAAK,IAAI,IAAI;AACrB,YAAM,cAAc,KAAK,gBAAgBA,KAAI;AAE7C,UAAI,KAAC,2BAAW,WAAW,GAAG;AAC5B,eAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,UAC1B,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAEA,YAAM,cAAU,6BAAa,aAAa,MAAM;AAChD,UAAI,KAAK;AAAA,QACP;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAED,SAAK,IAAI,IAAI,iCAA6B,YAAAC,SAAK,GAAG,OAAO,KAAK,QAAQ;AACpE,YAAM,EAAE,UAAU,IAAI,IAAI;AAC1B,UAAI,KAAK;AAAA,QACP,KAAK,KAAK,iBAAiB,SAAS,KAAK;AAAA,MAC3C,CAAC;AAAA,IACH,CAAC;AAID,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAF,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,UAAU,IAAI,KAAK;AAEzB,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,cAAMC,YAAO,+BAAW;AACxB,aAAK,gBAAgBA,OAAM,OAAO;AAClC,eAAO,IAAI,KAAK;AAAA,UACd,UAAU,eAAeA,KAAI;AAAA,UAC7B,MAAAA;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAD,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;AAAA,MAC9B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,MAAM,QAAQ,WAAW,UAAU,IAAI,IAAI;AAE5D,YAAI,CAAC,SAAS;AACZ,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM;AACT,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,QAAQ;AACX,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAGA,cAAM,OAAO,IAAI,KAAK,UAAU,OAAO;AACvC,cAAM,QAAQ,IAAI,KAAK,WAAW,IAAI;AAEtC,YAAI,WAAW;AACb,eAAK,iBAAiB,SAAS,IAAI;AAEnC,gBAAM,iBAAiB,CAAC,QAAgB;AACtC,iBAAK,iBAAiB,SAAS,IAAI;AAAA,UACrC;AAAA,QACF;AAEA,cAAM,WAMF;AAAA,UACF,QAAQ;AAAA,UACR,MAAM;AAAA,UACN,OAAO;AAAA,UACP,YAAY;AAAA,UACZ;AAAA,QACF;AAEA,cAAM,YAAY,KAAK,IAAI;AAC3B,YAAI;AACF,cAAI,SAAS,WAAW;AACtB,qBAAS,SAAS,MAAM,MAAM,QAAQ,MAAM;AAAA,UAC9C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,MAAM;AAAA,UAC/C,WAAW,SAAS,YAAY;AAC9B,qBAAS,SAAS,MAAM,MAAM,SAAS,QAAQ,QAAW;AAAA,cACxD,iBAAiB;AAAA,YACnB,CAAC;AAAA,UACH,WAAW,SAAS,SAAS;AAC3B,qBAAS,SAAS,MAAM,MAAM,MAAM,QAAQ;AAAA,cAC1C;AAAA,YACF,CAAC;AAAA,UACH,OAAO;AACL,qBAAS,QAAQ,iBAAiB,IAAI;AAAA,UACxC;AAAA,QACF,SAAS,OAAY;AACnB,cAAI,CAAC,MAAM,QAAQ,SAAS,sCAAsC,GAAG;AACnE,qBAAS,QAAQ,MAAM;AAAA,UACzB;AAAA,QACF;AAEA,YAAI;AACF,mBAAS,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AACjD,mBAAS,aAAa,MAAM,iBAAiB,KAAK;AAElD,gBAAM,oBAAoB;AAAA,QAC5B,SAAS,OAAY;AACnB,kBAAQ;AAAA,YACN,qCAAqC,SAAS,KAAK,MAAM,OAAO;AAAA,UAClE;AAAA,QACF;AAEA,YAAI,KAAK,QAAQ;AACjB,cAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,YAAI,SAAS,OAAO;AAClB,kBAAQ;AAAA,YACN,+BAA+B,QAAQ,kBAAkB,SAAS,KAAK,SAAS,KAAK;AAAA,UACvF;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,6BAA6B,QAAQ,kBAAkB,SAAS;AAAA,UAClE;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,SAAK,IAAI;AAAA,MACP;AAAA,MACA,eAAAA,QAAQ,KAAK,EAAE,OAAO,MAAM,CAAC;AAAA,MAC7B,OAAO,KAAK,QAAQ;AAClB,cAAM,EAAE,SAAS,IAAI,IAAI;AAEzB,YAAI,CAAC,YAAY,OAAO,aAAa,UAAU;AAC7C,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI;AACF,4CAAiB,QAAQ;AAEzB,iBAAO,IAAI,KAAK;AAAA,YACd,QAAQ;AAAA,YACR,SAAS;AAAA,UACX,CAAC;AAAA,QACH,SAAS,OAAY;AACnB,kBAAQ,MAAM,+BAA+B,MAAM,OAAO,EAAE;AAC5D,iBAAO,IAAI,OAAO,GAAG,EAAE,KAAK;AAAA,YAC1B,OAAO,+BAA+B,MAAM,OAAO;AAAA,UACrD,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,QAAI,KAAK,YAAY;AACnB,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAE9B,YAAI,SAAS,aAAa;AAAA,MAC5B,CAAC;AAED,WAAK,IAAI,IAAI,KAAK,CAAC,KAAK,QAAQ;AAC9B,cAAM,oBAAgB,uBAAK,KAAK,YAAa,IAAI,IAAI;AACrD,gBAAI,2BAAW,aAAa,GAAG;AAC7B,cAAI,SAAS,aAAa;AAAA,QAC5B,OAAO;AACL,cAAI,aAAS,uBAAK,KAAK,YAAa,YAAY,CAAC;AAAA,QACnD;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAMG,QAAO,KAAK;AAClB,WAAK,SAAS,KAAK,IAAI,OAAOA,OAAM,MAAM;AACxC,gBAAQ,IAAI;AAAA,MACd,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AAEN,QAAI,KAAK,QAAQ;AACf,aAAO,KAAK,OAAO,MAAM;AAAA,IAC3B;AAAA,EACF;AACF","names":["import_utils","import_env","dotenv","express","uuid","cors","port"],"ignoreList":[],"sources":["../../src/playground/server.ts","../../src/common/utils.ts"],"sourcesContent":["import { randomUUID } from 'node:crypto';\nimport { existsSync, readFileSync, writeFileSync } from 'node:fs';\nimport type { Server } from 'node:http';\nimport { join } from 'node:path';\nimport { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '@/common/utils';\nimport cors from 'cors';\nimport dotenv from 'dotenv';\nimport express from 'express';\nimport { getTmpDir } from 'misoai-core/utils';\nimport { PLAYGROUND_SERVER_PORT } from 'misoai-shared/constants';\nimport { overrideAIConfig } from 'misoai-shared/env';\nimport { ifInBrowser } from 'misoai-shared/utils';\nimport type { PageAgent } from '../common/agent';\nimport type { AbstractPage } from '../page';\n\nconst defaultPort = PLAYGROUND_SERVER_PORT;\n// const staticPath = join(__dirname, '../../static');\n\nconst errorHandler = (err: any, req: any, res: any, next: any) => {\n  console.error(err);\n  res.status(500).json({\n    error: err.message,\n  });\n};\n\nconst setup = async () => {\n  if (!ifInBrowser) {\n    const { parsed } = dotenv.config();\n\n    if (parsed) {\n      overrideAIConfig(parsed);\n    }\n  }\n};\n\nexport default class PlaygroundServer {\n  app: express.Application;\n  tmpDir: string;\n  server?: Server;\n  port?: number | null;\n  pageClass: new (\n    ...args: any[]\n  ) => AbstractPage;\n  agentClass: new (\n    ...args: any[]\n  ) => PageAgent;\n  staticPath?: string;\n  taskProgressTips: Record<string, string>;\n\n  constructor(\n    pageClass: new (...args: any[]) => AbstractPage,\n    agentClass: new (...args: any[]) => PageAgent,\n    staticPath?: string,\n  ) {\n    this.app = express();\n    this.tmpDir = getTmpDir()!;\n    this.pageClass = pageClass;\n    this.agentClass = agentClass;\n    this.staticPath = staticPath;\n    this.taskProgressTips = {};\n    setup();\n  }\n\n  filePathForUuid(uuid: string) {\n    return join(this.tmpDir, `${uuid}.json`);\n  }\n\n  saveContextFile(uuid: string, context: string) {\n    const tmpFile = this.filePathForUuid(uuid);\n    console.log(`save context file: ${tmpFile}`);\n    writeFileSync(tmpFile, context);\n    return tmpFile;\n  }\n\n  async launch(port?: number) {\n    this.port = port || defaultPort;\n    this.app.use(errorHandler);\n\n    this.app.use(\n      cors({\n        origin: '*',\n        credentials: true,\n      }),\n    );\n\n    this.app.get('/status', cors(), async (req, res) => {\n      // const modelName = g\n      res.send({\n        status: 'ok',\n      });\n    });\n\n    // this.app.get('/playground/:uuid', async (req, res) => {\n    //   res.sendFile(join(staticPath, 'index.html'));\n    // });\n\n    this.app.get('/context/:uuid', async (req, res) => {\n      const { uuid } = req.params;\n      const contextFile = this.filePathForUuid(uuid);\n\n      if (!existsSync(contextFile)) {\n        return res.status(404).json({\n          error: 'Context not found',\n        });\n      }\n\n      const context = readFileSync(contextFile, 'utf8');\n      res.json({\n        context,\n      });\n    });\n\n    this.app.get('/task-progress/:requestId', cors(), async (req, res) => {\n      const { requestId } = req.params;\n      res.json({\n        tip: this.taskProgressTips[requestId] || '',\n      });\n    });\n\n    // -------------------------\n    // actions from report file\n    this.app.post(\n      '/playground-with-context',\n      express.json({ limit: '50mb' }),\n      async (req, res) => {\n        const context = req.body.context;\n\n        if (!context) {\n          return res.status(400).json({\n            error: 'context is required',\n          });\n        }\n\n        const uuid = randomUUID();\n        this.saveContextFile(uuid, context);\n        return res.json({\n          location: `/playground/${uuid}`,\n          uuid,\n        });\n      },\n    );\n\n    this.app.post(\n      '/execute',\n      express.json({ limit: '30mb' }),\n      async (req, res) => {\n        const { context, type, prompt, requestId, deepThink } = req.body;\n\n        if (!context) {\n          return res.status(400).json({\n            error: 'context is required',\n          });\n        }\n\n        if (!type) {\n          return res.status(400).json({\n            error: 'type is required',\n          });\n        }\n\n        if (!prompt) {\n          return res.status(400).json({\n            error: 'prompt is required',\n          });\n        }\n\n        // build an agent with context\n        const page = new this.pageClass(context);\n        const agent = new this.agentClass(page);\n\n        if (requestId) {\n          this.taskProgressTips[requestId] = '';\n\n          agent.onTaskStartTip = (tip: string) => {\n            this.taskProgressTips[requestId] = tip;\n          };\n        }\n\n        const response: {\n          result: any;\n          dump: string | null;\n          error: string | null;\n          reportHTML: string | null;\n          requestId?: string;\n        } = {\n          result: null,\n          dump: null,\n          error: null,\n          reportHTML: null,\n          requestId,\n        };\n\n        const startTime = Date.now();\n        try {\n          if (type === 'aiQuery') {\n            response.result = await agent.aiQuery(prompt);\n          } else if (type === 'aiAction') {\n            response.result = await agent.aiAction(prompt);\n          } else if (type === 'aiAssert') {\n            response.result = await agent.aiAssert(prompt, undefined, {\n              keepRawResponse: true,\n            });\n          } else if (type === 'aiTap') {\n            response.result = await agent.aiTap(prompt, {\n              deepThink,\n            });\n          } else {\n            response.error = `Unknown type: ${type}`;\n          }\n        } catch (error: any) {\n          if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {\n            response.error = error.message;\n          }\n        }\n\n        try {\n          response.dump = JSON.parse(agent.dumpDataString());\n          response.reportHTML = agent.reportHTMLString() || null;\n\n          agent.writeOutActionDumps();\n        } catch (error: any) {\n          console.error(\n            `write out dump failed: requestId: ${requestId}, ${error.message}`,\n          );\n        }\n\n        res.send(response);\n        const timeCost = Date.now() - startTime;\n\n        if (response.error) {\n          console.error(\n            `handle request failed after ${timeCost}ms: requestId: ${requestId}, ${response.error}`,\n          );\n        } else {\n          console.log(\n            `handle request done after ${timeCost}ms: requestId: ${requestId}`,\n          );\n        }\n      },\n    );\n\n    this.app.post(\n      '/config',\n      express.json({ limit: '1mb' }),\n      async (req, res) => {\n        const { aiConfig } = req.body;\n\n        if (!aiConfig || typeof aiConfig !== 'object') {\n          return res.status(400).json({\n            error: 'aiConfig is required and must be an object',\n          });\n        }\n\n        try {\n          overrideAIConfig(aiConfig);\n\n          return res.json({\n            status: 'ok',\n            message: 'AI config updated successfully',\n          });\n        } catch (error: any) {\n          console.error(`Failed to update AI config: ${error.message}`);\n          return res.status(500).json({\n            error: `Failed to update AI config: ${error.message}`,\n          });\n        }\n      },\n    );\n\n    // Set up static file serving after all API routes are defined\n    if (this.staticPath) {\n      this.app.get('/', (req, res) => {\n        // compatible with windows\n        res.redirect('/index.html');\n      });\n\n      this.app.get('*', (req, res) => {\n        const requestedPath = join(this.staticPath!, req.path);\n        if (existsSync(requestedPath)) {\n          res.sendFile(requestedPath);\n        } else {\n          res.sendFile(join(this.staticPath!, 'index.html'));\n        }\n      });\n    }\n\n    return new Promise((resolve, reject) => {\n      const port = this.port;\n      this.server = this.app.listen(port, () => {\n        resolve(this);\n      });\n    });\n  }\n\n  close() {\n    // close the server\n    if (this.server) {\n      return this.server.close();\n    }\n  }\n}\n","import type { StaticPage } from '@/playground';\nimport dayjs from 'dayjs';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  PlanningLocateParam,\n  PlaywrightParserOpt,\n  UIContext,\n} from 'misoai-core';\nimport { elementByPositionWithElementInfo } from 'misoai-core/ai-model';\nimport { uploadTestInfoToServer } from 'misoai-core/utils';\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from 'misoai-shared/env';\nimport type { ElementInfo } from 'misoai-shared/extractor';\nimport {\n  generateElementByPosition,\n  getNodeFromCacheList,\n  traverseTree,\n  treeToList,\n} from 'misoai-shared/extractor';\nimport { resizeImgBase64 } from 'misoai-shared/img';\nimport type { DebugFunction } from 'misoai-shared/logger';\nimport { assert, logMsg, uuid } from 'misoai-shared/utils';\nimport type { Page as PlaywrightPage } from 'playwright';\nimport type { Page as PuppeteerPage } from 'puppeteer';\nimport { WebElementInfo } from '../web-element';\nimport type { WebPage } from './page';\n\nexport type WebUIContext = UIContext<WebElementInfo> & {\n  url: string;\n};\n\nexport async function parseContextFromWebPage(\n  page: WebPage,\n  _opt?: PlaywrightParserOpt,\n): Promise<WebUIContext> {\n  assert(page, 'page is required');\n  if ((page as StaticPage)._forceUsePageContext) {\n    return await (page as any)._forceUsePageContext();\n  }\n  const url = await page.url();\n  uploadTestInfoToServer({ testUrl: url });\n\n  let screenshotBase64: string;\n  let tree: ElementTreeNode<ElementInfo>;\n\n  await Promise.all([\n    page.screenshotBase64().then((base64) => {\n      screenshotBase64 = base64;\n    }),\n    page.getElementsNodeTree().then(async (treeRoot) => {\n      tree = treeRoot;\n    }),\n  ]);\n\n  const webTree = traverseTree(tree!, (elementInfo) => {\n    const { rect, id, content, attributes, locator, indexId, isVisible } =\n      elementInfo;\n    return new WebElementInfo({\n      rect,\n      locator,\n      id,\n      content,\n      attributes,\n      indexId,\n      isVisible,\n    });\n  });\n\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const elementsInfo = treeToList(webTree);\n  const size = await page.size();\n\n  if (size.dpr && size.dpr > 1) {\n    // console.time('resizeImgBase64');\n    screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n      width: size.width,\n      height: size.height,\n    });\n    // console.timeEnd('resizeImgBase64');\n  }\n\n  return {\n    content: elementsInfo!,\n    tree: webTree,\n    size,\n    screenshotBase64: screenshotBase64!,\n    url,\n  };\n}\n\nexport function reportFileName(tag = 'web') {\n  const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\n  'NOT_IMPLEMENTED_AS_DESIGNED';\n\nexport function replaceIllegalPathCharsAndSpace(str: string) {\n  // Only replace characters that are illegal in filenames, but preserve path separators\n  return str.replace(/[:*?\"<>| ]/g, '-');\n}\n\nexport function forceClosePopup(\n  page: PuppeteerPage | PlaywrightPage,\n  debug: DebugFunction,\n) {\n  page.on('popup', async (popup) => {\n    if (!popup) {\n      console.warn('got a popup event, but the popup is not ready yet, skip');\n      return;\n    }\n    const url = await (popup as PuppeteerPage).url();\n    console.log(`Popup opened: ${url}`);\n    if (!(popup as PuppeteerPage).isClosed()) {\n      try {\n        await (popup as PuppeteerPage).close(); // Close the newly opened TAB\n      } catch (error) {\n        debug(`failed to close popup ${url}, error: ${error}`);\n      }\n    } else {\n      debug(`popup is already closed, skip close ${url}`);\n    }\n\n    if (!page.isClosed()) {\n      try {\n        await page.goto(url);\n      } catch (error) {\n        debug(`failed to goto ${url}, error: ${error}`);\n      }\n    } else {\n      debug(`page is already closed, skip goto ${url}`);\n    }\n  });\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n  tree: ElementTreeNode<BaseElement>,\n) {\n  if (!planLocateParam) {\n    return undefined;\n  }\n  if (planLocateParam.id) {\n    return getNodeFromCacheList(planLocateParam.id);\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n    let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n    if (!element) {\n      element = generateElementByPosition(centerPosition) as BaseElement;\n    }\n\n    return element;\n  }\n\n  return undefined;\n}\n"]}

package/dist/lib/playground.js

@@ -37,13 +37,13 @@ __export(playground_exports, {
 module.exports = __toCommonJS(playground_exports);
 
 // src/common/utils.ts
+var import_dayjs = __toESM(require("dayjs"));
 var import_ai_model = require("misoai-core/ai-model");
 var import_utils = require("misoai-core/utils");
 var import_env = require("misoai-shared/env");
 var import_extractor = require("misoai-shared/extractor");
 var import_img = require("misoai-shared/img");
 var import_utils2 = require("misoai-shared/utils");
-var import_dayjs = __toESM(require("dayjs"));
 
 // src/web-element.ts
 var WebElementInfo = class {
@@ -462,8 +462,8 @@ var ScriptPlayer = class {
 var import_js_yaml = __toESM(require("js-yaml"));
 
 // src/yaml/utils.ts
-var import_utils4 = require("misoai-shared/utils");
 var import_js_yaml2 = __toESM(require("js-yaml"));
+var import_utils4 = require("misoai-shared/utils");
 function interpolateEnvVars(content) {
   return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
     const value = process.env[envVar.trim()];
@@ -639,8 +639,12 @@ var WorkflowMemory = class {
     const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
     workflow.memory = [...memory];
     workflow.metadata.totalSteps = workflow.steps.length;
-    workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
-    workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
+    workflow.metadata.completedSteps = workflow.steps.filter(
+      (s) => s.status === "completed"
+    ).length;
+    workflow.metadata.failedSteps = workflow.steps.filter(
+      (s) => s.status === "failed"
+    ).length;
     this.workflows.set(workflowId, workflow);
     this.enforceRetentionPolicy();
   }
@@ -651,7 +655,9 @@ var WorkflowMemory = class {
     const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
     workflow.context = { ...workflow.context, ...context };
     if (context.currentStep) {
-      const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
+      const existingStep = workflow.steps.find(
+        (s) => s.stepName === context.currentStep
+      );
       if (!existingStep) {
         workflow.steps.push({
           stepId: `step_${workflow.steps.length + 1}`,
@@ -696,7 +702,9 @@ var WorkflowMemory = class {
   enforceRetentionPolicy() {
     const maxWorkflows = 10;
     if (this.workflows.size > maxWorkflows) {
-      const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
+      const sortedWorkflows = Array.from(this.workflows.entries()).sort(
+        ([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime)
+      );
       const toDelete = sortedWorkflows.slice(maxWorkflows);
       toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
     }
@@ -1435,7 +1443,9 @@ var PageTaskExecutor = class {
    */
   getPersistentExecutor() {
     if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
-      const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
+      const previousMemory = this.workflowMemory.getWorkflowMemory(
+        this.sessionContext.workflowId
+      );
       this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
         onTaskStart: this.onTaskStartCallback,
         initialMemory: previousMemory
@@ -1464,7 +1474,9 @@ var PageTaskExecutor = class {
     if (this.persistentExecutor) {
       this.persistentExecutor.clearMemory();
     }
-    this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
+    this.workflowMemory.clearWorkflow(
+      this.sessionContext.workflowId || "default"
+    );
   }
   /**
    * Mevcut hafızayı döndürür
@@ -1476,7 +1488,9 @@ var PageTaskExecutor = class {
    * İş akışı hafızasını döndürür
    */
   getWorkflowMemory() {
-    return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
+    return this.workflowMemory.getWorkflowData(
+      this.sessionContext.workflowId || "default"
+    );
   }
   /**
    * Hafıza istatistiklerini döndürür
@@ -1484,7 +1498,13 @@ var PageTaskExecutor = class {
   getMemoryStats() {
     return this.persistentExecutor?.getMemoryStats() || {
       totalItems: 0,
-      analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
+      analytics: {
+        totalTasks: 0,
+        memoryHits: 0,
+        memoryMisses: 0,
+        averageMemorySize: 0,
+        memoryEffectiveness: 0
+      },
       config: this.memoryConfig
     };
   }
@@ -1520,11 +1540,14 @@ var PageTaskExecutor = class {
     let taskExecutor;
     if (useMemory) {
       taskExecutor = this.getPersistentExecutor();
-      this.workflowMemory.updateWorkflowContext({
-        currentStep: title,
-        pageInfo: this.sessionContext.pageInfo,
-        timestamp: Date.now()
-      }, this.sessionContext.workflowId || "default");
+      this.workflowMemory.updateWorkflowContext(
+        {
+          currentStep: title,
+          pageInfo: this.sessionContext.pageInfo,
+          timestamp: Date.now()
+        },
+        this.sessionContext.workflowId || "default"
+      );
     } else {
       taskExecutor = new import_misoai_core.Executor(title, {
         onTaskStart: this.onTaskStartCallback
@@ -1822,9 +1845,15 @@ var PageTaskExecutor = class {
   }
   async waitFor(assertion, opt) {
     const description = `waitFor: ${assertion}`;
-    const taskExecutor = new import_misoai_core.Executor(taskTitleStr("WaitFor", description), {
-      onTaskStart: this.onTaskStartCallback
-    });
+    const useMemory = true;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+    } else {
+      taskExecutor = new import_misoai_core.Executor(taskTitleStr("WaitFor", description), {
+        onTaskStart: this.onTaskStartCallback
+      });
+    }
     const { timeoutMs, checkIntervalMs } = opt;
     (0, import_utils6.assert)(assertion, "No assertion for waitFor");
     (0, import_utils6.assert)(timeoutMs, "No timeoutMs for waitFor");
@@ -1879,6 +1908,25 @@ var PageTaskExecutor = class {
       `waitFor timeout: ${errorThought}`
     );
   }
+  /**
+   * Hafızaya yeni bir öğe ekler
+   */
+  addToMemory(memoryItem) {
+    if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
+      const previousMemory = this.workflowMemory.getWorkflowMemory(
+        this.sessionContext.workflowId
+      );
+      this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
+        onTaskStart: this.onTaskStartCallback,
+        initialMemory: previousMemory
+      });
+    }
+    this.persistentExecutor.memoryStore?.add(memoryItem);
+    this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
+      "add",
+      memoryItem
+    );
+  }
 };
 
 // src/common/plan-builder.ts
@@ -1966,14 +2014,14 @@ function buildPlans(type, locateParam, param) {
 var import_node_assert = __toESM(require("assert"));
 var import_node_fs2 = require("fs");
 var import_node_path2 = require("path");
+var import_js_yaml3 = __toESM(require("js-yaml"));
 var import_common2 = require("misoai-shared/common");
 var import_logger3 = require("misoai-shared/logger");
 var import_utils9 = require("misoai-shared/utils");
-var import_js_yaml3 = __toESM(require("js-yaml"));
 var import_semver = __toESM(require("semver"));
 
 // package.json
-var version = "1.5.8";
+var version = "1.6.1";
 
 // src/common/task-cache.ts
 var debug3 = (0, import_logger3.getDebug)("cache");
@@ -2184,7 +2232,10 @@ var PageAgent = class {
     }
     this.taskExecutor = new PageTaskExecutor(this.page, this.insight, {
       taskCache: this.taskCache,
-      onTaskStart: this.callbackOnTaskStartTip.bind(this)
+      onTaskStart: this.callbackOnTaskStartTip.bind(this),
+      memoryConfig: opts?.memoryConfig,
+      sessionId: opts?.sessionId,
+      workflowId: opts?.workflowId
     });
     this.dump = this.resetDump();
     this.reportFileName = reportFileName(
@@ -2265,22 +2316,28 @@ var PageAgent = class {
     const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
     const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
     const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
-    const planningTasks = executor.tasks.filter((task) => task.type === "Planning");
-    const insightTasks = executor.tasks.filter((task) => task.type === "Insight");
+    const planningTasks = executor.tasks.filter(
+      (task) => task.type === "Planning"
+    );
+    const insightTasks = executor.tasks.filter(
+      (task) => task.type === "Insight"
+    );
     const actionTasks = executor.tasks.filter((task) => task.type === "Action");
     const planning = planningTasks.length > 0 ? {
       type: "Planning",
-      description: `Planning for task execution`,
+      description: "Planning for task execution",
       steps: planningTasks.map((task) => task.thought || "Planning step")
     } : void 0;
     const insight = insightTasks.length > 0 ? {
       type: "Insight",
-      description: `Insight for task execution`,
-      elements: insightTasks.map((task) => task.thought || "Insight element")
+      description: "Insight for task execution",
+      elements: insightTasks.map(
+        (task) => task.thought || "Insight element"
+      )
     } : void 0;
     const action = actionTasks.length > 0 ? {
       type: "Action",
-      description: `Action for task execution`,
+      description: "Action for task execution",
       result: lastTask?.output
     } : void 0;
     const actionDetails = executor.tasks.map((task) => ({
@@ -2614,7 +2671,10 @@ ${memoryContext}` : void 0;
     }
     const memoryContext = this.getMemoryAsContext();
     const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
-    const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
+    const { output, executor } = await this.taskExecutor.assert(
+      assertionWithContext,
+      memoryContext
+    );
     const metadata = this.afterTaskRunning(executor, true);
     if (output && opt?.keepRawResponse) {
       return {
@@ -2635,6 +2695,7 @@ ${reasonMsg}`);
   }
   async aiCaptcha(options) {
     const { deepThink = false, autoDetectComplexity = true } = options || {};
+    const memoryContext = this.getMemoryAsContext();
     let shouldUseDeepThink = deepThink;
     if (autoDetectComplexity && !deepThink) {
       const context = await this.getUIContext();
@@ -2652,7 +2713,10 @@ A complex CAPTCHA typically has one or more of these characteristics:
 Return only "complex" or "simple" based on your analysis.
 `;
         const complexityMsgs = [
-          { role: "system", content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity." },
+          {
+            role: "system",
+            content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity."
+          },
           {
             role: "user",
             content: [
@@ -2676,7 +2740,12 @@ Return only "complex" or "simple" based on your analysis.
         );
         const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
         shouldUseDeepThink = responseText.includes("complex");
-        debug4("CAPTCHA complexity analysis:", responseText, "Using deep think:", shouldUseDeepThink);
+        debug4(
+          "CAPTCHA complexity analysis:",
+          responseText,
+          "Using deep think:",
+          shouldUseDeepThink
+        );
       } catch (error) {
         debug4("Failed to analyze CAPTCHA complexity:", error);
       }
@@ -2693,7 +2762,9 @@ Return only "complex" or "simple" based on your analysis.
           await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
         } else if (action.type === "input" && action.value) {
           if (action.target) {
-            await this.aiInput(action.value, action.target, { deepThink: shouldUseDeepThink });
+            await this.aiInput(action.value, action.target, {
+              deepThink: shouldUseDeepThink
+            });
           }
         } else if (action.type === "verify" && action.target) {
           await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
@@ -2705,7 +2776,9 @@ Return only "complex" or "simple" based on your analysis.
           if (action.coordinates) {
             const x = action.coordinates[0];
             const y = action.coordinates[1];
-            await this.aiTap(`element at coordinates (${x}, ${y})`, { deepThink: shouldUseDeepThink });
+            await this.aiTap(`element at coordinates (${x}, ${y})`, {
+              deepThink: shouldUseDeepThink
+            });
           } else if (action.target) {
             await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
           }
@@ -2715,6 +2788,26 @@ Return only "complex" or "simple" based on your analysis.
       }
     }
     await new Promise((resolve2) => setTimeout(resolve2, 3e3));
+    const captchaMemoryItem = {
+      id: `captcha_${Date.now()}`,
+      timestamp: Date.now(),
+      taskType: "Action",
+      summary: `Solved ${captchaResult.captchaType} CAPTCHA: ${captchaResult.thought}`,
+      context: {
+        url: await this.page.url?.() || "",
+        captchaType: captchaResult.captchaType,
+        actions: captchaResult.actions,
+        deepThink: actualDeepThink
+      },
+      metadata: {
+        executionTime: Date.now() - Date.now(),
+        // Will be updated
+        success: true,
+        confidence: 0.9
+      },
+      tags: ["captcha", "action", captchaResult.captchaType]
+    };
+    this.taskExecutor.addToMemory(captchaMemoryItem);
     const metadata = {
       status: "finished",
       usage,
@@ -2731,10 +2824,15 @@ Return only "complex" or "simple" based on your analysis.
   }
   async aiWaitFor(assertion, opt) {
     const startTime = Date.now();
-    const { executor } = await this.taskExecutor.waitFor(assertion, {
+    const memoryContext = this.getMemoryAsContext();
+    const assertionWithContext = memoryContext ? `${assertion}
+
+Previous workflow steps:
+${memoryContext}` : assertion;
+    const { executor } = await this.taskExecutor.waitFor(assertionWithContext, {
       timeoutMs: opt?.timeoutMs || 15 * 1e3,
       checkIntervalMs: opt?.checkIntervalMs || 3 * 1e3,
-      assertion
+      assertion: assertionWithContext
     });
     const metadata = {
       status: executor.isInErrorState() ? "failed" : "finished",
@@ -2832,25 +2930,27 @@ ${errors}`);
       const executionDump = {
         name: screenshotTitle,
         description: content,
-        tasks: [{
-          type: "Screenshot",
-          subType: "log",
-          status: "finished",
-          executor: null,
-          param: {
-            title: screenshotTitle,
-            content
-          },
-          output: {
-            screenshot
-          },
-          thought: `Logged screenshot: ${screenshotTitle}`,
-          timing: {
-            start: Date.now(),
-            end: Date.now(),
-            cost: 0
+        tasks: [
+          {
+            type: "Screenshot",
+            subType: "log",
+            status: "finished",
+            executor: null,
+            param: {
+              title: screenshotTitle,
+              content
+            },
+            output: {
+              screenshot
+            },
+            thought: `Logged screenshot: ${screenshotTitle}`,
+            timing: {
+              start: Date.now(),
+              end: Date.now(),
+              cost: 0
+            }
           }
-        }],
+        ],
         sdkVersion: "1.0.0",
         logTime: Date.now(),
         model_name: "screenshot"
@@ -2902,7 +3002,9 @@ ${errors}`);
         totalTasks: stats.analytics.totalTasks,
         memoryHits: stats.analytics.memoryHits,
         memoryMisses: stats.analytics.memoryMisses,
-        memoryEffectiveness: Math.round(stats.analytics.memoryEffectiveness * 100),
+        memoryEffectiveness: Math.round(
+          stats.analytics.memoryEffectiveness * 100
+        ),
         averageMemorySize: Math.round(stats.analytics.averageMemorySize * 100) / 100
       },
       config: stats.config,
@@ -2966,7 +3068,9 @@ ${errors}`);
   calculateSuccessRate(memory) {
     if (memory.length === 0)
       return 0;
-    const successCount = memory.filter((item) => item.metadata?.success !== false).length;
+    const successCount = memory.filter(
+      (item) => item.metadata?.success !== false
+    ).length;
     return Math.round(successCount / memory.length * 100);
   }
   calculateAverageExecutionTime(memory) {