@midscene/playground 1.9.8 → 1.10.0

package/dist/es/index.mjs

@@ -1,5 +1,6 @@
 import { dataExtractionAPIs, executeAction, formatErrorMessage, noReplayAPIs, validateStructuredParams, validationAPIs } from "./common.mjs";
-import { PlaygroundServer } from "./server.mjs";
+import { PlaygroundServer, buildPlaygroundBrowserUrl } from "./server.mjs";
+import { describeRecorderUIEvent, describeRecorderUIEvents, getRecorderUIEventTargetRect } from "./recorder-ui-describer.mjs";
 import { playgroundForAgent, playgroundForAgentFactory, playgroundForSessionManager } from "./launcher.mjs";
 import { createMjpegPreviewDescriptor, createScrcpyPreviewDescriptor, createScreenshotPreviewDescriptor, definePlaygroundPlatform, resolvePreparedLaunchOptions } from "./platform.mjs";
 import { launchPreparedPlaygroundPlatform } from "./platform-launcher.mjs";
@@ -8,4 +9,4 @@ import { PlaygroundSDK } from "./sdk/index.mjs";
 import { BasePlaygroundAdapter } from "./adapters/base.mjs";
 import { LocalExecutionAdapter } from "./adapters/local-execution.mjs";
 import { RemoteExecutionAdapter } from "./adapters/remote-execution.mjs";
-export { BasePlaygroundAdapter, LocalExecutionAdapter, PlaygroundSDK, PlaygroundServer, RemoteExecutionAdapter, createMjpegPreviewDescriptor, createScrcpyPreviewDescriptor, createScreenshotPreviewDescriptor, dataExtractionAPIs, definePlaygroundPlatform, executeAction, formatErrorMessage, launchPreparedPlaygroundPlatform, noReplayAPIs, playgroundForAgent, playgroundForAgentFactory, playgroundForPlatforms, playgroundForSessionManager, prepareMultiPlatformPlayground, resolvePreparedLaunchOptions, validateStructuredParams, validationAPIs };
+export { BasePlaygroundAdapter, LocalExecutionAdapter, PlaygroundSDK, PlaygroundServer, RemoteExecutionAdapter, buildPlaygroundBrowserUrl, createMjpegPreviewDescriptor, createScrcpyPreviewDescriptor, createScreenshotPreviewDescriptor, dataExtractionAPIs, definePlaygroundPlatform, describeRecorderUIEvent, describeRecorderUIEvents, executeAction, formatErrorMessage, getRecorderUIEventTargetRect, launchPreparedPlaygroundPlatform, noReplayAPIs, playgroundForAgent, playgroundForAgentFactory, playgroundForPlatforms, playgroundForSessionManager, prepareMultiPlatformPlayground, resolvePreparedLaunchOptions, validateStructuredParams, validationAPIs };

package/dist/es/launcher.mjs

@@ -1,7 +1,7 @@
 import { spawn } from "node:child_process";
 import { PLAYGROUND_SERVER_PORT } from "@midscene/shared/constants";
 import cors from "cors";
-import server_0 from "./server.mjs";
+import server_0, { buildPlaygroundBrowserUrl, resolvePlaygroundBrowserHost } from "./server.mjs";
 const LOOPBACK_HOSTS = new Set([
     'localhost',
     '127.0.0.1',
@@ -47,12 +47,13 @@ function createPlaygroundLauncher(agentOrFactory) {
             if (configureServer) await configureServer(server);
             const launchedServer = await server.launch(port);
             if (verbose) console.log(`✅ Playground server started on port ${port}`);
-            const url = `http://127.0.0.1:${port}`;
+            const host = resolvePlaygroundBrowserHost();
+            const url = buildPlaygroundBrowserUrl(host, port);
             if (openBrowser) await openInBrowser(url, browserCommand, verbose);
             return {
                 server: launchedServer,
                 port,
-                host: '127.0.0.1',
+                host,
                 close: async ()=>{
                     if (verbose) console.log('🛑 Shutting down Midscene Playground...');
                     try {

package/dist/es/launcher.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"launcher.mjs","sources":["../../src/launcher.ts"],"sourcesContent":["import { spawn } from 'node:child_process';\nimport type { Agent } from '@midscene/core/agent';\nimport { PLAYGROUND_SERVER_PORT } from '@midscene/shared/constants';\nimport cors, { type CorsOptions } from 'cors';\nimport PlaygroundServer from './server';\nimport type { AgentFactory } from './types';\n\nexport interface LaunchPlaygroundOptions {\n  /**\n   * Port to start the playground server on\n   * @default 5800\n   */\n  port?: number;\n\n  /**\n   * Whether to automatically open the playground in browser\n   * @default true\n   */\n  openBrowser?: boolean;\n\n  /**\n   * Custom browser command to open playground\n   * @default 'open' on macOS, 'start' on Windows, 'xdg-open' on Linux\n   */\n  browserCommand?: string;\n\n  /**\n   * Whether to show server logs\n   * @default true\n   */\n  verbose?: boolean;\n\n  /**\n   * Fixed ID for the playground server instance\n   * If provided, the same ID will be used across restarts,\n   * allowing chat history to persist\n   * @default undefined (generates random UUID)\n   */\n  id?: string;\n\n  /**\n   * Whether to enable CORS (Cross-Origin Resource Sharing)\n   * @default false\n   */\n  enableCors?: boolean;\n\n  /**\n   * Custom static assets directory for the playground frontend\n   * @default bundled static assets from @midscene/playground\n   */\n  staticPath?: string;\n\n  /**\n   * Hook for configuring the PlaygroundServer before launch\n   * Useful for adding custom middleware beyond the built-in CORS option\n   */\n  configureServer?: (server: PlaygroundServer) => void | Promise<void>;\n\n  /**\n   * CORS configuration options\n   * @default only allows loopback browser origins when enableCors is true\n   */\n  corsOptions?: CorsOptions;\n}\n\nexport interface LaunchPlaygroundResult {\n  /**\n   * The playground server instance\n   */\n  server: PlaygroundServer;\n\n  /**\n   * The server port\n   */\n  port: number;\n\n  /**\n   * The server host\n   */\n  host: string;\n\n  /**\n   * Function to gracefully shutdown the playground\n   */\n  close: () => Promise<void>;\n}\n\ntype LaunchableAgentSource = Agent | AgentFactory | undefined;\n\nconst LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]']);\n\nfunction isLoopbackOrigin(origin?: string) {\n  if (!origin) {\n    return true;\n  }\n\n  try {\n    const url = new URL(origin);\n    return LOOPBACK_HOSTS.has(url.hostname);\n  } catch {\n    return false;\n  }\n}\n\nfunction createDefaultCorsOptions(): CorsOptions {\n  return {\n    origin(origin, callback) {\n      callback(null, isLoopbackOrigin(origin));\n    },\n    credentials: true,\n  };\n}\n\n/**\n * Create a playground launcher for a specific agent\n *\n * @example\n * ```typescript\n * import { playgroundForAgent } from '@midscene/playground';\n * import { SampleDevice, Agent } from '@midscene/core';\n *\n * const device = new SampleDevice();\n * const agent = new Agent(device);\n *\n * // Launch playground for the agent\n * const server = await playgroundForAgent(agent).launch();\n *\n * // Launch with CORS enabled\n * const serverWithCors = await playgroundForAgent(agent).launch({\n *   enableCors: true,\n *   corsOptions: {\n *     origin: ['http://localhost:3000', 'http://localhost:8080'],\n *     credentials: true\n *   }\n * });\n *\n * // Later, when you want to shutdown:\n * server.close();\n * ```\n */\nfunction createPlaygroundLauncher(agentOrFactory: LaunchableAgentSource) {\n  return {\n    /**\n     * Launch the playground server with optional configuration\n     */\n    async launch(\n      options: LaunchPlaygroundOptions = {},\n    ): Promise<LaunchPlaygroundResult> {\n      const {\n        port = PLAYGROUND_SERVER_PORT,\n        openBrowser = true,\n        browserCommand,\n        verbose = true,\n        id,\n        enableCors = false,\n        staticPath,\n        configureServer,\n        corsOptions = createDefaultCorsOptions(),\n      } = options;\n\n      if (\n        agentOrFactory &&\n        typeof agentOrFactory !== 'function' &&\n        !agentOrFactory.interface\n      ) {\n        throw new Error('Agent must have an interface property');\n      }\n\n      if (verbose) {\n        console.log('🚀 Starting Midscene Playground...');\n        if (!agentOrFactory) {\n          console.log('📱 Agent: session-managed');\n        } else if (typeof agentOrFactory === 'function') {\n          console.log('📱 Agent: factory');\n        } else {\n          console.log(`📱 Agent: ${agentOrFactory.constructor.name}`);\n          console.log(`🖥️ Page: ${agentOrFactory.interface.constructor.name}`);\n        }\n        console.log(`🌐 Port: ${port}`);\n        if (staticPath) {\n          console.log(`📁 Static path: ${staticPath}`);\n        }\n        if (enableCors) {\n          console.log('🔓 CORS enabled');\n        }\n      }\n\n      const server = new PlaygroundServer(agentOrFactory, staticPath, id);\n\n      if (enableCors) {\n        server.app.use(cors(corsOptions));\n      }\n\n      if (configureServer) {\n        await configureServer(server);\n      }\n\n      const launchedServer = await server.launch(port);\n\n      if (verbose) {\n        console.log(`✅ Playground server started on port ${port}`);\n      }\n\n      const url = `http://127.0.0.1:${port}`;\n\n      // Open browser if requested\n      if (openBrowser) {\n        await openInBrowser(url, browserCommand, verbose);\n      }\n\n      return {\n        server: launchedServer,\n        port,\n        host: '127.0.0.1',\n        close: async () => {\n          if (verbose) {\n            console.log('🛑 Shutting down Midscene Playground...');\n          }\n\n          try {\n            await launchedServer.close();\n            if (verbose) {\n              console.log('✅ Playground shutdown complete');\n            }\n          } catch (error) {\n            if (verbose) {\n              console.error('❌ Error during playground shutdown:', error);\n            }\n            throw error;\n          }\n        },\n      };\n    },\n  };\n}\n\n/**\n * Create a playground launcher from an already initialized agent instance\n */\nexport function playgroundForAgent(agent: Agent) {\n  return createPlaygroundLauncher(agent);\n}\n\n/**\n * Create a playground launcher from an agent factory\n * Useful for device-backed agents that need to be recreated after cancellation\n */\nexport function playgroundForAgentFactory(agentFactory: AgentFactory) {\n  return createPlaygroundLauncher(agentFactory);\n}\n\nexport function playgroundForSessionManager() {\n  return createPlaygroundLauncher(undefined);\n}\n\n/**\n * Open URL in browser using platform-appropriate command\n */\nasync function openInBrowser(\n  url: string,\n  customCommand?: string,\n  verbose = true,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    let command: string;\n    let args: string[];\n\n    if (customCommand) {\n      command = customCommand;\n      args = [url];\n    } else {\n      // Detect platform and use appropriate command\n      switch (process.platform) {\n        case 'darwin':\n          command = 'open';\n          args = [url];\n          break;\n        case 'win32':\n          command = 'start';\n          args = ['', url]; // Empty string for title\n          break;\n        default:\n          command = 'xdg-open';\n          args = [url];\n          break;\n      }\n    }\n\n    if (verbose) {\n      console.log(`🌐 Opening browser: ${command} ${args.join(' ')}`);\n    }\n\n    const child = spawn(command, args, {\n      detached: true,\n      stdio: 'ignore',\n    });\n\n    child.on('error', (error) => {\n      if (verbose) {\n        console.warn('⚠️  Failed to open browser automatically:', error.message);\n        console.log(`🌐 Please open manually: ${url}`);\n      }\n      // Don't reject, just continue - browser opening is optional\n      resolve();\n    });\n\n    child.on('close', () => {\n      resolve();\n    });\n\n    // Don't wait for the browser process\n    child.unref();\n  });\n}\n"],"names":["LOOPBACK_HOSTS","Set","isLoopbackOrigin","origin","url","URL","createDefaultCorsOptions","callback","createPlaygroundLauncher","agentOrFactory","options","port","PLAYGROUND_SERVER_PORT","openBrowser","browserCommand","verbose","id","enableCors","staticPath","configureServer","corsOptions","Error","console","server","PlaygroundServer","cors","launchedServer","openInBrowser","error","playgroundForAgent","agent","playgroundForAgentFactory","agentFactory","playgroundForSessionManager","undefined","customCommand","Promise","resolve","reject","command","args","process","child","spawn"],"mappings":";;;;AAyFA,MAAMA,iBAAiB,IAAIC,IAAI;IAAC;IAAa;IAAa;IAAO;CAAQ;AAEzE,SAASC,iBAAiBC,MAAe;IACvC,IAAI,CAACA,QACH,OAAO;IAGT,IAAI;QACF,MAAMC,MAAM,IAAIC,IAAIF;QACpB,OAAOH,eAAe,GAAG,CAACI,IAAI,QAAQ;IACxC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,SAASE;IACP,OAAO;QACL,QAAOH,MAAM,EAAEI,QAAQ;YACrBA,SAAS,MAAML,iBAAiBC;QAClC;QACA,aAAa;IACf;AACF;AA6BA,SAASK,yBAAyBC,cAAqC;IACrE,OAAO;QAIL,MAAM,QACJC,UAAmC,CAAC,CAAC;YAErC,MAAM,EACJC,OAAOC,sBAAsB,EAC7BC,cAAc,IAAI,EAClBC,cAAc,EACdC,UAAU,IAAI,EACdC,EAAE,EACFC,aAAa,KAAK,EAClBC,UAAU,EACVC,eAAe,EACfC,cAAcd,0BAA0B,EACzC,GAAGI;YAEJ,IACED,kBACA,AAA0B,cAA1B,OAAOA,kBACP,CAACA,eAAe,SAAS,EAEzB,MAAM,IAAIY,MAAM;YAGlB,IAAIN,SAAS;gBACXO,QAAQ,GAAG,CAAC;gBACZ,IAAKb,gBAEE,IAAI,AAA0B,cAA1B,OAAOA,gBAChBa,QAAQ,GAAG,CAAC;qBACP;oBACLA,QAAQ,GAAG,CAAC,CAAC,UAAU,EAAEb,eAAe,WAAW,CAAC,IAAI,EAAE;oBAC1Da,QAAQ,GAAG,CAAC,CAAC,UAAU,EAAEb,eAAe,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE;gBACtE;qBANEa,QAAQ,GAAG,CAAC;gBAOdA,QAAQ,GAAG,CAAC,CAAC,SAAS,EAAEX,MAAM;gBAC9B,IAAIO,YACFI,QAAQ,GAAG,CAAC,CAAC,gBAAgB,EAAEJ,YAAY;gBAE7C,IAAID,YACFK,QAAQ,GAAG,CAAC;YAEhB;YAEA,MAAMC,SAAS,IAAIC,SAAiBf,gBAAgBS,YAAYF;YAEhE,IAAIC,YACFM,OAAO,GAAG,CAAC,GAAG,CAACE,KAAKL;YAGtB,IAAID,iBACF,MAAMA,gBAAgBI;YAGxB,MAAMG,iBAAiB,MAAMH,OAAO,MAAM,CAACZ;YAE3C,IAAII,SACFO,QAAQ,GAAG,CAAC,CAAC,oCAAoC,EAAEX,MAAM;YAG3D,MAAMP,MAAM,CAAC,iBAAiB,EAAEO,MAAM;YAGtC,IAAIE,aACF,MAAMc,cAAcvB,KAAKU,gBAAgBC;YAG3C,OAAO;gBACL,QAAQW;gBACRf;gBACA,MAAM;gBACN,OAAO;oBACL,IAAII,SACFO,QAAQ,GAAG,CAAC;oBAGd,IAAI;wBACF,MAAMI,eAAe,KAAK;wBAC1B,IAAIX,SACFO,QAAQ,GAAG,CAAC;oBAEhB,EAAE,OAAOM,OAAO;wBACd,IAAIb,SACFO,QAAQ,KAAK,CAAC,uCAAuCM;wBAEvD,MAAMA;oBACR;gBACF;YACF;QACF;IACF;AACF;AAKO,SAASC,mBAAmBC,KAAY;IAC7C,OAAOtB,yBAAyBsB;AAClC;AAMO,SAASC,0BAA0BC,YAA0B;IAClE,OAAOxB,yBAAyBwB;AAClC;AAEO,SAASC;IACd,OAAOzB,yBAAyB0B;AAClC;AAKA,eAAeP,cACbvB,GAAW,EACX+B,aAAsB,EACtBpB,UAAU,IAAI;IAEd,OAAO,IAAIqB,QAAQ,CAACC,SAASC;QAC3B,IAAIC;QACJ,IAAIC;QAEJ,IAAIL,eAAe;YACjBI,UAAUJ;YACVK,OAAO;gBAACpC;aAAI;QACd,OAEE,OAAQqC,QAAQ,QAAQ;YACtB,KAAK;gBACHF,UAAU;gBACVC,OAAO;oBAACpC;iBAAI;gBACZ;YACF,KAAK;gBACHmC,UAAU;gBACVC,OAAO;oBAAC;oBAAIpC;iBAAI;gBAChB;YACF;gBACEmC,UAAU;gBACVC,OAAO;oBAACpC;iBAAI;gBACZ;QACJ;QAGF,IAAIW,SACFO,QAAQ,GAAG,CAAC,CAAC,oBAAoB,EAAEiB,QAAQ,CAAC,EAAEC,KAAK,IAAI,CAAC,MAAM;QAGhE,MAAME,QAAQC,MAAMJ,SAASC,MAAM;YACjC,UAAU;YACV,OAAO;QACT;QAEAE,MAAM,EAAE,CAAC,SAAS,CAACd;YACjB,IAAIb,SAAS;gBACXO,QAAQ,IAAI,CAAC,6CAA6CM,MAAM,OAAO;gBACvEN,QAAQ,GAAG,CAAC,CAAC,yBAAyB,EAAElB,KAAK;YAC/C;YAEAiC;QACF;QAEAK,MAAM,EAAE,CAAC,SAAS;YAChBL;QACF;QAGAK,MAAM,KAAK;IACb;AACF"}
+{"version":3,"file":"launcher.mjs","sources":["../../src/launcher.ts"],"sourcesContent":["import { spawn } from 'node:child_process';\nimport type { Agent } from '@midscene/core/agent';\nimport { PLAYGROUND_SERVER_PORT } from '@midscene/shared/constants';\nimport cors, { type CorsOptions } from 'cors';\nimport PlaygroundServer, {\n  buildPlaygroundBrowserUrl,\n  resolvePlaygroundBrowserHost,\n} from './server';\nimport type { AgentFactory } from './types';\n\nexport interface LaunchPlaygroundOptions {\n  /**\n   * Port to start the playground server on\n   * @default 5800\n   */\n  port?: number;\n\n  /**\n   * Whether to automatically open the playground in browser\n   * @default true\n   */\n  openBrowser?: boolean;\n\n  /**\n   * Custom browser command to open playground\n   * @default 'open' on macOS, 'start' on Windows, 'xdg-open' on Linux\n   */\n  browserCommand?: string;\n\n  /**\n   * Whether to show server logs\n   * @default true\n   */\n  verbose?: boolean;\n\n  /**\n   * Fixed ID for the playground server instance\n   * If provided, the same ID will be used across restarts,\n   * allowing chat history to persist\n   * @default undefined (generates random UUID)\n   */\n  id?: string;\n\n  /**\n   * Whether to enable CORS (Cross-Origin Resource Sharing)\n   * @default false\n   */\n  enableCors?: boolean;\n\n  /**\n   * Custom static assets directory for the playground frontend\n   * @default bundled static assets from @midscene/playground\n   */\n  staticPath?: string;\n\n  /**\n   * Hook for configuring the PlaygroundServer before launch\n   * Useful for adding custom middleware beyond the built-in CORS option\n   */\n  configureServer?: (server: PlaygroundServer) => void | Promise<void>;\n\n  /**\n   * CORS configuration options\n   * @default only allows loopback browser origins when enableCors is true\n   */\n  corsOptions?: CorsOptions;\n}\n\nexport interface LaunchPlaygroundResult {\n  /**\n   * The playground server instance\n   */\n  server: PlaygroundServer;\n\n  /**\n   * The server port\n   */\n  port: number;\n\n  /**\n   * The server host\n   */\n  host: string;\n\n  /**\n   * Function to gracefully shutdown the playground\n   */\n  close: () => Promise<void>;\n}\n\ntype LaunchableAgentSource = Agent | AgentFactory | undefined;\n\nconst LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]']);\n\nfunction isLoopbackOrigin(origin?: string) {\n  if (!origin) {\n    return true;\n  }\n\n  try {\n    const url = new URL(origin);\n    return LOOPBACK_HOSTS.has(url.hostname);\n  } catch {\n    return false;\n  }\n}\n\nfunction createDefaultCorsOptions(): CorsOptions {\n  return {\n    origin(origin, callback) {\n      callback(null, isLoopbackOrigin(origin));\n    },\n    credentials: true,\n  };\n}\n\n/**\n * Create a playground launcher for a specific agent\n *\n * @example\n * ```typescript\n * import { playgroundForAgent } from '@midscene/playground';\n * import { SampleDevice, Agent } from '@midscene/core';\n *\n * const device = new SampleDevice();\n * const agent = new Agent(device);\n *\n * // Launch playground for the agent\n * const server = await playgroundForAgent(agent).launch();\n *\n * // Launch with CORS enabled\n * const serverWithCors = await playgroundForAgent(agent).launch({\n *   enableCors: true,\n *   corsOptions: {\n *     origin: ['http://localhost:3000', 'http://localhost:8080'],\n *     credentials: true\n *   }\n * });\n *\n * // Later, when you want to shutdown:\n * server.close();\n * ```\n */\nfunction createPlaygroundLauncher(agentOrFactory: LaunchableAgentSource) {\n  return {\n    /**\n     * Launch the playground server with optional configuration\n     */\n    async launch(\n      options: LaunchPlaygroundOptions = {},\n    ): Promise<LaunchPlaygroundResult> {\n      const {\n        port = PLAYGROUND_SERVER_PORT,\n        openBrowser = true,\n        browserCommand,\n        verbose = true,\n        id,\n        enableCors = false,\n        staticPath,\n        configureServer,\n        corsOptions = createDefaultCorsOptions(),\n      } = options;\n\n      if (\n        agentOrFactory &&\n        typeof agentOrFactory !== 'function' &&\n        !agentOrFactory.interface\n      ) {\n        throw new Error('Agent must have an interface property');\n      }\n\n      if (verbose) {\n        console.log('🚀 Starting Midscene Playground...');\n        if (!agentOrFactory) {\n          console.log('📱 Agent: session-managed');\n        } else if (typeof agentOrFactory === 'function') {\n          console.log('📱 Agent: factory');\n        } else {\n          console.log(`📱 Agent: ${agentOrFactory.constructor.name}`);\n          console.log(`🖥️ Page: ${agentOrFactory.interface.constructor.name}`);\n        }\n        console.log(`🌐 Port: ${port}`);\n        if (staticPath) {\n          console.log(`📁 Static path: ${staticPath}`);\n        }\n        if (enableCors) {\n          console.log('🔓 CORS enabled');\n        }\n      }\n\n      const server = new PlaygroundServer(agentOrFactory, staticPath, id);\n\n      if (enableCors) {\n        server.app.use(cors(corsOptions));\n      }\n\n      if (configureServer) {\n        await configureServer(server);\n      }\n\n      const launchedServer = await server.launch(port);\n\n      if (verbose) {\n        console.log(`✅ Playground server started on port ${port}`);\n      }\n\n      const host = resolvePlaygroundBrowserHost();\n      const url = buildPlaygroundBrowserUrl(host, port);\n\n      // Open browser if requested\n      if (openBrowser) {\n        await openInBrowser(url, browserCommand, verbose);\n      }\n\n      return {\n        server: launchedServer,\n        port,\n        host,\n        close: async () => {\n          if (verbose) {\n            console.log('🛑 Shutting down Midscene Playground...');\n          }\n\n          try {\n            await launchedServer.close();\n            if (verbose) {\n              console.log('✅ Playground shutdown complete');\n            }\n          } catch (error) {\n            if (verbose) {\n              console.error('❌ Error during playground shutdown:', error);\n            }\n            throw error;\n          }\n        },\n      };\n    },\n  };\n}\n\n/**\n * Create a playground launcher from an already initialized agent instance\n */\nexport function playgroundForAgent(agent: Agent) {\n  return createPlaygroundLauncher(agent);\n}\n\n/**\n * Create a playground launcher from an agent factory\n * Useful for device-backed agents that need to be recreated after cancellation\n */\nexport function playgroundForAgentFactory(agentFactory: AgentFactory) {\n  return createPlaygroundLauncher(agentFactory);\n}\n\nexport function playgroundForSessionManager() {\n  return createPlaygroundLauncher(undefined);\n}\n\n/**\n * Open URL in browser using platform-appropriate command\n */\nasync function openInBrowser(\n  url: string,\n  customCommand?: string,\n  verbose = true,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    let command: string;\n    let args: string[];\n\n    if (customCommand) {\n      command = customCommand;\n      args = [url];\n    } else {\n      // Detect platform and use appropriate command\n      switch (process.platform) {\n        case 'darwin':\n          command = 'open';\n          args = [url];\n          break;\n        case 'win32':\n          command = 'start';\n          args = ['', url]; // Empty string for title\n          break;\n        default:\n          command = 'xdg-open';\n          args = [url];\n          break;\n      }\n    }\n\n    if (verbose) {\n      console.log(`🌐 Opening browser: ${command} ${args.join(' ')}`);\n    }\n\n    const child = spawn(command, args, {\n      detached: true,\n      stdio: 'ignore',\n    });\n\n    child.on('error', (error) => {\n      if (verbose) {\n        console.warn('⚠️  Failed to open browser automatically:', error.message);\n        console.log(`🌐 Please open manually: ${url}`);\n      }\n      // Don't reject, just continue - browser opening is optional\n      resolve();\n    });\n\n    child.on('close', () => {\n      resolve();\n    });\n\n    // Don't wait for the browser process\n    child.unref();\n  });\n}\n"],"names":["LOOPBACK_HOSTS","Set","isLoopbackOrigin","origin","url","URL","createDefaultCorsOptions","callback","createPlaygroundLauncher","agentOrFactory","options","port","PLAYGROUND_SERVER_PORT","openBrowser","browserCommand","verbose","id","enableCors","staticPath","configureServer","corsOptions","Error","console","server","PlaygroundServer","cors","launchedServer","host","resolvePlaygroundBrowserHost","buildPlaygroundBrowserUrl","openInBrowser","error","playgroundForAgent","agent","playgroundForAgentFactory","agentFactory","playgroundForSessionManager","undefined","customCommand","Promise","resolve","reject","command","args","process","child","spawn"],"mappings":";;;;AA4FA,MAAMA,iBAAiB,IAAIC,IAAI;IAAC;IAAa;IAAa;IAAO;CAAQ;AAEzE,SAASC,iBAAiBC,MAAe;IACvC,IAAI,CAACA,QACH,OAAO;IAGT,IAAI;QACF,MAAMC,MAAM,IAAIC,IAAIF;QACpB,OAAOH,eAAe,GAAG,CAACI,IAAI,QAAQ;IACxC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,SAASE;IACP,OAAO;QACL,QAAOH,MAAM,EAAEI,QAAQ;YACrBA,SAAS,MAAML,iBAAiBC;QAClC;QACA,aAAa;IACf;AACF;AA6BA,SAASK,yBAAyBC,cAAqC;IACrE,OAAO;QAIL,MAAM,QACJC,UAAmC,CAAC,CAAC;YAErC,MAAM,EACJC,OAAOC,sBAAsB,EAC7BC,cAAc,IAAI,EAClBC,cAAc,EACdC,UAAU,IAAI,EACdC,EAAE,EACFC,aAAa,KAAK,EAClBC,UAAU,EACVC,eAAe,EACfC,cAAcd,0BAA0B,EACzC,GAAGI;YAEJ,IACED,kBACA,AAA0B,cAA1B,OAAOA,kBACP,CAACA,eAAe,SAAS,EAEzB,MAAM,IAAIY,MAAM;YAGlB,IAAIN,SAAS;gBACXO,QAAQ,GAAG,CAAC;gBACZ,IAAKb,gBAEE,IAAI,AAA0B,cAA1B,OAAOA,gBAChBa,QAAQ,GAAG,CAAC;qBACP;oBACLA,QAAQ,GAAG,CAAC,CAAC,UAAU,EAAEb,eAAe,WAAW,CAAC,IAAI,EAAE;oBAC1Da,QAAQ,GAAG,CAAC,CAAC,UAAU,EAAEb,eAAe,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE;gBACtE;qBANEa,QAAQ,GAAG,CAAC;gBAOdA,QAAQ,GAAG,CAAC,CAAC,SAAS,EAAEX,MAAM;gBAC9B,IAAIO,YACFI,QAAQ,GAAG,CAAC,CAAC,gBAAgB,EAAEJ,YAAY;gBAE7C,IAAID,YACFK,QAAQ,GAAG,CAAC;YAEhB;YAEA,MAAMC,SAAS,IAAIC,SAAiBf,gBAAgBS,YAAYF;YAEhE,IAAIC,YACFM,OAAO,GAAG,CAAC,GAAG,CAACE,KAAKL;YAGtB,IAAID,iBACF,MAAMA,gBAAgBI;YAGxB,MAAMG,iBAAiB,MAAMH,OAAO,MAAM,CAACZ;YAE3C,IAAII,SACFO,QAAQ,GAAG,CAAC,CAAC,oCAAoC,EAAEX,MAAM;YAG3D,MAAMgB,OAAOC;YACb,MAAMxB,MAAMyB,0BAA0BF,MAAMhB;YAG5C,IAAIE,aACF,MAAMiB,cAAc1B,KAAKU,gBAAgBC;YAG3C,OAAO;gBACL,QAAQW;gBACRf;gBACAgB;gBACA,OAAO;oBACL,IAAIZ,SACFO,QAAQ,GAAG,CAAC;oBAGd,IAAI;wBACF,MAAMI,eAAe,KAAK;wBAC1B,IAAIX,SACFO,QAAQ,GAAG,CAAC;oBAEhB,EAAE,OAAOS,OAAO;wBACd,IAAIhB,SACFO,QAAQ,KAAK,CAAC,uCAAuCS;wBAEvD,MAAMA;oBACR;gBACF;YACF;QACF;IACF;AACF;AAKO,SAASC,mBAAmBC,KAAY;IAC7C,OAAOzB,yBAAyByB;AAClC;AAMO,SAASC,0BAA0BC,YAA0B;IAClE,OAAO3B,yBAAyB2B;AAClC;AAEO,SAASC;IACd,OAAO5B,yBAAyB6B;AAClC;AAKA,eAAeP,cACb1B,GAAW,EACXkC,aAAsB,EACtBvB,UAAU,IAAI;IAEd,OAAO,IAAIwB,QAAQ,CAACC,SAASC;QAC3B,IAAIC;QACJ,IAAIC;QAEJ,IAAIL,eAAe;YACjBI,UAAUJ;YACVK,OAAO;gBAACvC;aAAI;QACd,OAEE,OAAQwC,QAAQ,QAAQ;YACtB,KAAK;gBACHF,UAAU;gBACVC,OAAO;oBAACvC;iBAAI;gBACZ;YACF,KAAK;gBACHsC,UAAU;gBACVC,OAAO;oBAAC;oBAAIvC;iBAAI;gBAChB;YACF;gBACEsC,UAAU;gBACVC,OAAO;oBAACvC;iBAAI;gBACZ;QACJ;QAGF,IAAIW,SACFO,QAAQ,GAAG,CAAC,CAAC,oBAAoB,EAAEoB,QAAQ,CAAC,EAAEC,KAAK,IAAI,CAAC,MAAM;QAGhE,MAAME,QAAQC,MAAMJ,SAASC,MAAM;YACjC,UAAU;YACV,OAAO;QACT;QAEAE,MAAM,EAAE,CAAC,SAAS,CAACd;YACjB,IAAIhB,SAAS;gBACXO,QAAQ,IAAI,CAAC,6CAA6CS,MAAM,OAAO;gBACvET,QAAQ,GAAG,CAAC,CAAC,yBAAyB,EAAElB,KAAK;YAC/C;YAEAoC;QACF;QAEAK,MAAM,EAAE,CAAC,SAAS;YAChBL;QACF;QAGAK,MAAM,KAAK;IACb;AACF"}

package/dist/es/recorder-ai-service.mjs

@@ -0,0 +1,2 @@
+import { callAIWithObjectResponse } from "@midscene/core/ai-model";
+export { callAIWithObjectResponse };

package/dist/es/recorder-ui-describer-prompt.mjs

@@ -0,0 +1,56 @@
+const RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = `You convert Studio preview recorder UI events into semantic replay instructions.
+
+The recorder works from screenshots and mapped real-device coordinates only. Infer stable UI intent from the highlighted BEFORE screenshot. The AFTER screenshot is contextual evidence for state changes and scroll destinations.
+
+Output JSON only:
+{
+  "elementDescription": "short stable target/region description",
+  "replayInstruction": "one executable natural-language replay step",
+  "actionSummary": "short timeline summary",
+  "scrollDestinationDescription": "for scroll only: concrete newly visible destination content or goal",
+  "confidence": "high" | "medium" | "low",
+  "error"?: "only if no useful visual description can be inferred"
+}
+
+Rules:
+- Do NOT output coordinates as the main description.
+- Do NOT mention "near coordinates", "nearby element", "near point", "red marker", highlighted box, highlighted element, or screenshot.
+- Prefer stable target descriptions in this order: exact stable control text > stable label > role + stable section/context > icon purpose > visual position.
+- Treat placeholder or hint text that can change by user, time, data, or context as dynamic. Do not use dynamic hint values as the primary target description; prefer role + stable region + intent.
+- For repeated collections, treat item identity text as dynamic unless the user is clearly verifying that exact item. This includes any list/grid/table/feed/menu or repeated record surface. Do not output descriptions like "<role> titled/named '<content>'"; prefer stable role + region + selection intent.
+- Keep quoted UI text in the original UI language when it is a stable control label.
+- Apply the platform guidance from the user event:
+  - Web: button, input, link, menu item, tab, dialog, aria-label, placeholder, form section.
+  - Mobile: tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, screen section.
+  - Desktop/computer: menu item, toolbar button, dialog field, sidebar item, window control, file row, application region.
+- Pointer action rules:
+  - Preserve event.actionType semantics. Tap, DoubleClick, LongPress, and RightClick must not all become Click.
+  - Tap replayInstruction format: Tap on the element described as "<elementDescription>".
+  - DoubleClick replayInstruction format: Double click on the element described as "<elementDescription>".
+  - LongPress replayInstruction format: Long press the element described as "<elementDescription>".
+  - RightClick replayInstruction format: Right click on the element described as "<elementDescription>".
+  - Click replayInstruction format: Click on the element described as "<elementDescription>".
+- Input-specific rules:
+  - The highlighted BEFORE screenshot marks the field that receives the text.
+  - The screenshot after the action may show the typed value; use it only to confirm the field, never as the field description.
+  - elementDescription must identify the field itself by stable field role, nearby label, region, section, or sequence intent.
+  - Never use "AI is analyzing element", the typed value, the page title alone, or a generic "input field" as elementDescription.
+  - For consecutive input events, distinguish fields by stable role, section, order, current focus, filled/empty state, and neighboring actions instead of reusing the same generic field description.
+  - Input replayInstruction format: Input "<value>" into the element described as "<elementDescription>".
+- Scroll target quality bar:
+  - elementDescription describes the scrollable page, panel, list, table, or section at the highlighted scroll point.
+  - When multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu. Do not generalize a panel/list scroll into the whole page.
+  - scrollDestinationDescription is required and describes what the scroll is trying to reveal or reach, using newly visible headings, section titles, list items, or stable content from the AFTER screenshot.
+  - Prefer descriptions like "Playwright integration documentation page, scrolling toward the API reference section" or "Android API documentation page, scrolling to the installation steps section".
+  - Do NOT write generic phrases like "more content", "the page", "current screen", or "main scrollable area".
+- Scroll replayInstruction format: Scroll the page/region with description "<elementDescription>" by value "<recorded value>" until "<scrollDestinationDescription>" is visible.
+- Scroll actionSummary format: Scroll <elementDescription> toward <scrollDestinationDescription>.
+- Drag/Swipe rules:
+  - Drag replayInstruction format: Drag through the area described as "<elementDescription>".
+  - Swipe replayInstruction format: Swipe through the area described as "<elementDescription>".
+  - Describe start/end regions or the dragged UI control; do not describe only the gesture path.
+- KeyboardPress replayInstruction format: Press "<value>" on the element described as "<elementDescription>".
+- If uncertain, provide the best concrete visible text/role/context description. Set confidence to "low"; do not fall back to coordinates.`;
+export { RECORDER_UI_DESCRIBER_SYSTEM_PROMPT };
+
+//# sourceMappingURL=recorder-ui-describer-prompt.mjs.map

package/dist/es/recorder-ui-describer-prompt.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"recorder-ui-describer-prompt.mjs","sources":["../../src/recorder-ui-describer-prompt.ts"],"sourcesContent":["export const RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = `You convert Studio preview recorder UI events into semantic replay instructions.\n\nThe recorder works from screenshots and mapped real-device coordinates only. Infer stable UI intent from the highlighted BEFORE screenshot. The AFTER screenshot is contextual evidence for state changes and scroll destinations.\n\nOutput JSON only:\n{\n  \"elementDescription\": \"short stable target/region description\",\n  \"replayInstruction\": \"one executable natural-language replay step\",\n  \"actionSummary\": \"short timeline summary\",\n  \"scrollDestinationDescription\": \"for scroll only: concrete newly visible destination content or goal\",\n  \"confidence\": \"high\" | \"medium\" | \"low\",\n  \"error\"?: \"only if no useful visual description can be inferred\"\n}\n\nRules:\n- Do NOT output coordinates as the main description.\n- Do NOT mention \"near coordinates\", \"nearby element\", \"near point\", \"red marker\", highlighted box, highlighted element, or screenshot.\n- Prefer stable target descriptions in this order: exact stable control text > stable label > role + stable section/context > icon purpose > visual position.\n- Treat placeholder or hint text that can change by user, time, data, or context as dynamic. Do not use dynamic hint values as the primary target description; prefer role + stable region + intent.\n- For repeated collections, treat item identity text as dynamic unless the user is clearly verifying that exact item. This includes any list/grid/table/feed/menu or repeated record surface. Do not output descriptions like \"<role> titled/named '<content>'\"; prefer stable role + region + selection intent.\n- Keep quoted UI text in the original UI language when it is a stable control label.\n- Apply the platform guidance from the user event:\n  - Web: button, input, link, menu item, tab, dialog, aria-label, placeholder, form section.\n  - Mobile: tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, screen section.\n  - Desktop/computer: menu item, toolbar button, dialog field, sidebar item, window control, file row, application region.\n- Pointer action rules:\n  - Preserve event.actionType semantics. Tap, DoubleClick, LongPress, and RightClick must not all become Click.\n  - Tap replayInstruction format: Tap on the element described as \"<elementDescription>\".\n  - DoubleClick replayInstruction format: Double click on the element described as \"<elementDescription>\".\n  - LongPress replayInstruction format: Long press the element described as \"<elementDescription>\".\n  - RightClick replayInstruction format: Right click on the element described as \"<elementDescription>\".\n  - Click replayInstruction format: Click on the element described as \"<elementDescription>\".\n- Input-specific rules:\n  - The highlighted BEFORE screenshot marks the field that receives the text.\n  - The screenshot after the action may show the typed value; use it only to confirm the field, never as the field description.\n  - elementDescription must identify the field itself by stable field role, nearby label, region, section, or sequence intent.\n  - Never use \"AI is analyzing element\", the typed value, the page title alone, or a generic \"input field\" as elementDescription.\n  - For consecutive input events, distinguish fields by stable role, section, order, current focus, filled/empty state, and neighboring actions instead of reusing the same generic field description.\n  - Input replayInstruction format: Input \"<value>\" into the element described as \"<elementDescription>\".\n- Scroll target quality bar:\n  - elementDescription describes the scrollable page, panel, list, table, or section at the highlighted scroll point.\n  - When multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu. Do not generalize a panel/list scroll into the whole page.\n  - scrollDestinationDescription is required and describes what the scroll is trying to reveal or reach, using newly visible headings, section titles, list items, or stable content from the AFTER screenshot.\n  - Prefer descriptions like \"Playwright integration documentation page, scrolling toward the API reference section\" or \"Android API documentation page, scrolling to the installation steps section\".\n  - Do NOT write generic phrases like \"more content\", \"the page\", \"current screen\", or \"main scrollable area\".\n- Scroll replayInstruction format: Scroll the page/region with description \"<elementDescription>\" by value \"<recorded value>\" until \"<scrollDestinationDescription>\" is visible.\n- Scroll actionSummary format: Scroll <elementDescription> toward <scrollDestinationDescription>.\n- Drag/Swipe rules:\n  - Drag replayInstruction format: Drag through the area described as \"<elementDescription>\".\n  - Swipe replayInstruction format: Swipe through the area described as \"<elementDescription>\".\n  - Describe start/end regions or the dragged UI control; do not describe only the gesture path.\n- KeyboardPress replayInstruction format: Press \"<value>\" on the element described as \"<elementDescription>\".\n- If uncertain, provide the best concrete visible text/role/context description. Set confidence to \"low\"; do not fall back to coordinates.`;\n"],"names":["RECORDER_UI_DESCRIBER_SYSTEM_PROMPT"],"mappings":"AAAO,MAAMA,sCAAsC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;0IAoDsF,CAAC"}

package/dist/es/recorder-ui-describer.mjs

@@ -0,0 +1,392 @@
+import { ModelConfigManager } from "@midscene/shared/env";
+import { buildMidsceneRecorderActionSummary, buildMidsceneRecorderReplayInstruction, getMidsceneRecorderSemantic } from "@midscene/shared/recorder";
+import { callAIWithObjectResponse } from "./recorder-ai-service.mjs";
+import { RECORDER_UI_DESCRIBER_SYSTEM_PROMPT } from "./recorder-ui-describer-prompt.mjs";
+const RECORDER_UI_DESCRIBER_DEFAULT_RETRIES = 2;
+const RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS = 200;
+const RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY = 2;
+function delay(ms) {
+    return new Promise((resolve)=>setTimeout(resolve, ms));
+}
+function isResolvedModelConfig(modelConfig) {
+    return 'string' == typeof modelConfig.modelName && ('string' == typeof modelConfig.modelDescription || 'string' == typeof modelConfig.intent || 'string' == typeof modelConfig.slot || 'string' == typeof modelConfig.openaiApiKey);
+}
+function resolveRecorderModelConfig(modelConfig) {
+    if (isResolvedModelConfig(modelConfig)) return modelConfig;
+    const rawModelConfig = Object.entries(modelConfig).reduce((result, [key, value])=>{
+        if ('string' == typeof value || 'number' == typeof value) result[key] = value;
+        return result;
+    }, {});
+    return new ModelConfigManager(rawModelConfig).getModelConfig('default');
+}
+function clamp(value, min, max) {
+    return Math.min(Math.max(value, min), max);
+}
+function isFiniteNumber(value) {
+    return 'number' == typeof value && Number.isFinite(value);
+}
+function isPendingDescription(value) {
+    return value?.trim() === 'AI is analyzing element...';
+}
+function getRecorderEventScreenshot(event) {
+    return event.screenshotWithBox || event.screenshotBefore || event.screenshotAfter;
+}
+function getRecorderEventAfterScreenshot(event) {
+    return event.screenshotAfter || event.screenshotWithBox;
+}
+function normalizeActionType(event) {
+    return event.actionType?.trim();
+}
+function getPlatformId(target) {
+    return target?.platformId?.toLowerCase();
+}
+function getPlatformSurface(target) {
+    switch(getPlatformId(target)){
+        case 'web':
+            return 'current web page';
+        case 'android':
+        case 'ios':
+        case 'harmony':
+            return 'current mobile screen';
+        case 'computer':
+            return 'current desktop screen';
+        default:
+            return 'current UI';
+    }
+}
+function getPlatformGuidance(target) {
+    switch(getPlatformId(target)){
+        case 'web':
+            return 'For web targets, use web UI terms such as button, input, link, menu item, tab, dialog, aria-label, placeholder, and form section when visible or inferable.';
+        case 'android':
+        case 'ios':
+        case 'harmony':
+            return 'For mobile targets, use mobile UI terms such as tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, and screen section.';
+        case 'computer':
+            return 'For desktop/computer targets, use desktop UI terms such as menu item, toolbar button, dialog field, sidebar item, window control, file row, and application region.';
+        default:
+            return 'Use platform-neutral UI terms such as control, field, item, icon button, list item, region, panel, and page section.';
+    }
+}
+function getPointerActionVerb(event) {
+    switch(normalizeActionType(event)){
+        case 'Tap':
+            return 'Tap';
+        case 'DoubleClick':
+            return 'Double click';
+        case 'LongPress':
+            return 'Long press';
+        case 'RightClick':
+            return 'Right click';
+        default:
+            return 'Click';
+    }
+}
+function getDragActionVerb(event) {
+    switch(normalizeActionType(event)){
+        case 'Swipe':
+            return 'Swipe';
+        case 'DragAndDrop':
+            return 'Drag';
+        default:
+            return 'Drag';
+    }
+}
+function pointToRect(x, y, size, pageInfo) {
+    const width = pageInfo.width || size;
+    const height = pageInfo.height || size;
+    const left = clamp(Math.floor(x - size / 2), 0, Math.max(width - 1, 0));
+    const top = clamp(Math.floor(y - size / 2), 0, Math.max(height - 1, 0));
+    return {
+        left,
+        top,
+        width: Math.min(size, Math.max(width - left, 1)),
+        height: Math.min(size, Math.max(height - top, 1))
+    };
+}
+function getPointRectSize(event) {
+    switch(event.type){
+        case 'scroll':
+            return 96;
+        case 'drag':
+            return 64;
+        default:
+            return 36;
+    }
+}
+function getRecorderUIEventTargetRect(event) {
+    const rect = event.elementRect;
+    if (!rect) return null;
+    if (isFiniteNumber(rect.width) && rect.width > 0 && isFiniteNumber(rect.height) && rect.height > 0 && (isFiniteNumber(rect.left) || isFiniteNumber(rect.top))) return {
+        left: rect.left || 0,
+        top: rect.top || 0,
+        width: rect.width,
+        height: rect.height
+    };
+    if (isFiniteNumber(rect.x) && isFiniteNumber(rect.y)) return pointToRect(rect.x, rect.y, getPointRectSize(event), event.pageInfo);
+    return null;
+}
+function getFallbackDescription(event, target) {
+    const pageContext = getPageSemanticContext(event);
+    const surface = getPlatformSurface(target);
+    switch(event.type){
+        case 'navigation':
+            return event.url || event.value || event.actionType || 'navigation';
+        case 'scroll':
+            return pageContext ? `${pageContext} scrollable content` : `scrollable content on the ${surface}`;
+        case 'drag':
+            return pageContext ? `gesture area in ${pageContext}` : `gesture area on the ${surface}`;
+        case 'input':
+            return `unresolved input field on the ${surface}`;
+        case 'keydown':
+            return pageContext ? `focused control in ${pageContext}` : `focused control on the ${surface}`;
+        default:
+            return pageContext ? `control in ${pageContext}` : `control on the ${surface}`;
+    }
+}
+function buildSemanticAction(event, scrollDestinationDescription) {
+    return {
+        type: event.type,
+        actionType: event.actionType,
+        value: event.value,
+        url: event.url,
+        scrollDestinationDescription
+    };
+}
+function getFallbackReplayInstruction(event, elementDescription) {
+    return buildMidsceneRecorderReplayInstruction(buildSemanticAction(event), elementDescription);
+}
+function getFallbackActionSummary(event, elementDescription) {
+    return buildMidsceneRecorderActionSummary(buildSemanticAction(event), elementDescription);
+}
+function getActionGuidance(event, target) {
+    const platformGuidance = getPlatformGuidance(target);
+    switch(event.type){
+        case 'click':
+            return `${platformGuidance} Identify the ${getPointerActionVerb(event).toLowerCase()} target by exact visible text first, then label/placeholder, then role plus stable surrounding context, then icon purpose, then visual position. Never describe it by coordinates, marker location, or as a nearby element.`;
+        case 'input':
+            return `${platformGuidance} Identify the exact input field at the marker before text entry. Use stable visible label, field role, field name, surrounding section, or sequence intent. Treat hint text that can change by user, time, data, or context as secondary evidence. Preserve the recorded input value only in replayInstruction; never describe the typed value or page title alone as the field.`;
+        case 'scroll':
+            return `${platformGuidance} Identify the scrollable page/region at the highlighted scroll point and concrete destination content revealed after scrolling. If multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu; do not generalize a panel/list scroll into the whole page. Use newly visible headings, section titles, list/table names, list items, or stable region labels; never say only "more content" or "current page".`;
+        case 'drag':
+            return `${platformGuidance} Identify the ${getDragActionVerb(event).toLowerCase()} start/end regions or the dragged UI control. Do not describe only the gesture path or coordinates.`;
+        case 'keydown':
+            return `${platformGuidance} Identify the focused element or keyboard target if visible, and preserve the recorded key in the replay instruction.`;
+        default:
+            return `${platformGuidance} Identify the UI target involved in this event using the most stable visible text or surrounding context.`;
+    }
+}
+function getEventRawCoordinates(event) {
+    const x = event.elementRect?.x;
+    const y = event.elementRect?.y;
+    if (isFiniteNumber(x) && isFiniteNumber(y)) return {
+        x,
+        y
+    };
+}
+function getPageSemanticContext(event) {
+    const candidates = [
+        event.title,
+        event.url
+    ].map((item)=>item?.trim()).filter(Boolean);
+    return candidates[0];
+}
+function isWeakDescription(value) {
+    if (!value) return true;
+    if (isPendingDescription(value)) return true;
+    const normalized = value.trim().toLowerCase();
+    const compact = normalized.replace(/\s+/g, '');
+    return 0 === normalized.length || /^\(?\d+(?:\.\d+)?,\s*\d+(?:\.\d+)?\)?$/.test(normalized) || 'target' === normalized || 'element' === normalized || 'target element' === normalized || 'the element' === normalized || 'page element' === normalized || 'input field' === normalized || 'text input' === normalized || 'text field' === normalized || 'search box' === normalized || 'more content' === normalized || 'the page' === normalized || 'current page' === normalized || 'current screen' === normalized || 'the screen' === normalized || 'current ui' === normalized || 'current visible ui' === normalized || 'current visible page' === normalized || 'current visible screen' === normalized || 'main area' === normalized || 'main scrollable area' === normalized || 'scrollable area' === normalized || 'highlighted element' === normalized || 'highlighted item' === normalized || 'marked element' === normalized || 'marked item' === normalized || normalized.includes('ai is analyzing element') || compact.includes('坐标') || compact.includes('附近') || compact.includes('附近的元素') || normalized.includes('coordinate') || normalized.includes('near the coordinate') || normalized.includes('near coordinates') || normalized.includes('nearby element') || normalized.includes('nearby item') || normalized.includes('near the marker') || normalized.includes('near marker') || normalized.includes('near the point') || normalized.includes('near point') || normalized.includes('at the point') || normalized.includes('button near point') || normalized.includes('shown in the screenshot') || normalized.includes('red rectangle') || normalized.includes('red marker') || normalized.includes('red box') || normalized.includes('highlighted element') || normalized.includes('highlighted item') || normalized.includes('highlighted screenshot');
+}
+function isWeakReplayInstruction(value) {
+    if (!value) return true;
+    if (isPendingDescription(value)) return true;
+    const normalized = value.trim().toLowerCase();
+    const compact = normalized.replace(/\s+/g, '');
+    return compact.includes('坐标') || compact.includes('附近') || normalized.includes('coordinate') || normalized.includes('near the coordinate') || normalized.includes('nearby element') || normalized.includes('nearby item') || normalized.includes('near the marker') || normalized.includes('near marker') || normalized.includes('near the point') || normalized.includes('near point') || normalized.includes('at the point') || normalized.includes('ai is analyzing element') || normalized.includes('more content') || normalized.includes('current page') || normalized.includes('current screen') || normalized.includes('highlighted element') || normalized.includes('highlighted item') || normalized.includes('red marker') || normalized.includes('red box') || normalized.includes('shown in the screenshot') || normalized.includes('highlighted screenshot');
+}
+function normalizeForComparison(value) {
+    return value.trim().toLowerCase().replace(/["'`]/g, '').replace(/\s+/g, ' ');
+}
+function isInputValueUsedAsFieldDescription(event, elementDescription) {
+    if ('input' !== event.type || !event.value || !elementDescription) return false;
+    const typedValue = normalizeForComparison(event.value);
+    if (!typedValue) return false;
+    const description = normalizeForComparison(elementDescription);
+    return description === typedValue || description === `${typedValue} input` || description === `${typedValue} field` || description === `${typedValue} text field` || description === `input ${typedValue}` || description === `field ${typedValue}` || description.includes(`typed value ${typedValue}`) || description.includes(`value ${typedValue}`);
+}
+function hasScrollDestination(replayInstruction, scrollDestinationDescription) {
+    if (scrollDestinationDescription && !isWeakDescription(scrollDestinationDescription)) return true;
+    const normalized = replayInstruction.toLowerCase();
+    return normalized.includes(' until ') || normalized.includes(' visible') || normalized.includes(' reveal') || normalized.includes(' to the ') || normalized.includes(' toward ');
+}
+async function describeWithRetry(event, target, highlightedScreenshot, modelConfig, options) {
+    let lastError;
+    for(let attempt = 1; attempt <= options.maxRetries; attempt += 1)try {
+        const afterScreenshot = getRecorderEventAfterScreenshot(event);
+        const pageContext = getPageSemanticContext(event);
+        const platformGuidance = getPlatformGuidance(target);
+        const userContent = [
+            {
+                type: 'text',
+                text: `Recorder event:
+${JSON.stringify({
+                    type: event.type,
+                    actionType: event.actionType,
+                    value: event.value,
+                    rawCoordinates: getEventRawCoordinates(event),
+                    url: event.url,
+                    title: event.title,
+                    pageContext,
+                    pageInfo: event.pageInfo,
+                    target,
+                    platformGuidance,
+                    guidance: getActionGuidance(event, target)
+                }, null, 2)}
+
+The target or region is highlighted in the screenshot below. Convert this event into semantic replay fields.`
+            },
+            {
+                type: 'image_url',
+                image_url: {
+                    url: highlightedScreenshot,
+                    detail: 'high'
+                }
+            }
+        ];
+        if (afterScreenshot) userContent.push({
+            type: 'text',
+            text: 'Screenshot after the recorded action, for context only:'
+        }, {
+            type: 'image_url',
+            image_url: {
+                url: afterScreenshot,
+                detail: 'high'
+            }
+        });
+        const response = await callAIWithObjectResponse([
+            {
+                role: 'system',
+                content: RECORDER_UI_DESCRIBER_SYSTEM_PROMPT
+            },
+            {
+                role: 'user',
+                content: userContent
+            }
+        ], modelConfig);
+        const content = response.content;
+        if (content.error) throw new Error(content.error);
+        if (isWeakDescription(content.elementDescription)) throw new Error("AI returned a weak recorder event description.");
+        if (isInputValueUsedAsFieldDescription(event, content.elementDescription)) throw new Error("AI used the recorded input value as the field description.");
+        const elementDescription = content.elementDescription.trim();
+        const scrollDestinationDescription = 'scroll' === event.type ? content.scrollDestinationDescription?.trim() : void 0;
+        if ('scroll' === event.type && !hasScrollDestination('', scrollDestinationDescription)) throw new Error("AI returned a scroll description without a destination.");
+        const aiReplayInstruction = content.replayInstruction?.trim();
+        if (aiReplayInstruction && isWeakReplayInstruction(aiReplayInstruction)) throw new Error('AI returned a weak recorder replay instruction.');
+        const semanticAction = buildSemanticAction(event, scrollDestinationDescription);
+        const replayInstruction = buildMidsceneRecorderReplayInstruction(semanticAction, elementDescription);
+        if (isWeakReplayInstruction(replayInstruction)) throw new Error('AI returned a weak recorder replay instruction.');
+        const actionSummary = buildMidsceneRecorderActionSummary(semanticAction, elementDescription);
+        return {
+            source: 'recorderAI',
+            status: 'ready',
+            elementDescription,
+            replayInstruction,
+            actionSummary,
+            confidence: content.confidence || 'medium'
+        };
+    } catch (error) {
+        lastError = error;
+        if (attempt < options.maxRetries) await delay(options.retryDelayMs);
+    }
+    throw lastError;
+}
+async function createScreenshotWithBox(event, rect) {
+    if (event.screenshotWithBox) return event.screenshotWithBox;
+    const screenshot = getRecorderEventScreenshot(event);
+    if (!screenshot) return;
+    const { compositeElementInfoImg } = await import(/* @vite-ignore */ "@midscene/shared/img");
+    return compositeElementInfoImg({
+        inputImgBase64: screenshot,
+        size: event.pageInfo,
+        elementsPositionInfo: [
+            {
+                rect
+            }
+        ],
+        borderThickness: 3,
+        annotationPadding: 2
+    });
+}
+function createFallbackEvent(event, error, screenshotWithBox, target) {
+    const semantic = getMidsceneRecorderSemantic(event);
+    const elementDescription = semantic?.elementDescription && !isWeakDescription(semantic.elementDescription) ? semantic.elementDescription : getFallbackDescription(event, target);
+    return {
+        ...event,
+        semantic: {
+            source: 'heuristic',
+            status: 'ready',
+            elementDescription,
+            replayInstruction: getFallbackReplayInstruction(event, elementDescription),
+            actionSummary: getFallbackActionSummary(event, elementDescription),
+            confidence: 'low',
+            error
+        },
+        screenshotWithBox: screenshotWithBox || event.screenshotWithBox
+    };
+}
+async function describeRecorderUIEvent(input, modelConfig, options = {}) {
+    const event = input.event;
+    const rect = getRecorderUIEventTargetRect(event);
+    const screenshot = getRecorderEventScreenshot(event);
+    if (!rect || !screenshot) {
+        const error = rect ? 'Recorder event has no screenshot.' : 'Recorder event has no target rectangle.';
+        return {
+            usedFallback: true,
+            event: createFallbackEvent(event, error, void 0, input.target)
+        };
+    }
+    let screenshotWithBox;
+    try {
+        const resolvedModelConfig = resolveRecorderModelConfig(modelConfig);
+        screenshotWithBox = await createScreenshotWithBox(event, rect);
+        const semanticFields = await describeWithRetry(event, input.target, screenshotWithBox || screenshot, resolvedModelConfig, {
+            maxRetries: options.maxRetries ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRIES,
+            retryDelayMs: options.retryDelayMs ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS
+        });
+        return {
+            usedFallback: false,
+            event: {
+                ...event,
+                semantic: semanticFields,
+                screenshotWithBox: screenshotWithBox || event.screenshotWithBox
+            }
+        };
+    } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+            usedFallback: true,
+            error: message,
+            event: createFallbackEvent(event, message, screenshotWithBox, input.target)
+        };
+    }
+}
+async function describeRecorderUIEvents(inputs, modelConfig, options = {}) {
+    const concurrency = Math.max(1, options.concurrency ?? RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY);
+    const results = new Array(inputs.length);
+    let cursor = 0;
+    async function worker() {
+        while(cursor < inputs.length){
+            const index = cursor;
+            cursor += 1;
+            results[index] = await describeRecorderUIEvent(inputs[index], modelConfig, options);
+        }
+    }
+    await Promise.all(Array.from({
+        length: Math.min(concurrency, inputs.length)
+    }, ()=>worker()));
+    return results;
+}
+export { describeRecorderUIEvent, describeRecorderUIEvents, getRecorderUIEventTargetRect };
+
+//# sourceMappingURL=recorder-ui-describer.mjs.map