@midscene/playground 1.9.8 → 1.10.0

package/dist/lib/index.js

@@ -26,14 +26,18 @@ __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
     PlaygroundSDK: ()=>index_js_namespaceObject.PlaygroundSDK,
     LocalExecutionAdapter: ()=>local_execution_js_namespaceObject.LocalExecutionAdapter,
+    buildPlaygroundBrowserUrl: ()=>external_server_js_namespaceObject.buildPlaygroundBrowserUrl,
+    getRecorderUIEventTargetRect: ()=>external_recorder_ui_describer_js_namespaceObject.getRecorderUIEventTargetRect,
+    playgroundForAgentFactory: ()=>external_launcher_js_namespaceObject.playgroundForAgentFactory,
     playgroundForSessionManager: ()=>external_launcher_js_namespaceObject.playgroundForSessionManager,
+    noReplayAPIs: ()=>external_common_js_namespaceObject.noReplayAPIs,
     prepareMultiPlatformPlayground: ()=>external_multi_platform_js_namespaceObject.prepareMultiPlatformPlayground,
     resolvePreparedLaunchOptions: ()=>external_platform_js_namespaceObject.resolvePreparedLaunchOptions,
-    validateStructuredParams: ()=>external_common_js_namespaceObject.validateStructuredParams,
-    noReplayAPIs: ()=>external_common_js_namespaceObject.noReplayAPIs,
-    validationAPIs: ()=>external_common_js_namespaceObject.validationAPIs,
     BasePlaygroundAdapter: ()=>base_js_namespaceObject.BasePlaygroundAdapter,
     playgroundForPlatforms: ()=>external_multi_platform_js_namespaceObject.playgroundForPlatforms,
+    validateStructuredParams: ()=>external_common_js_namespaceObject.validateStructuredParams,
+    validationAPIs: ()=>external_common_js_namespaceObject.validationAPIs,
+    describeRecorderUIEvent: ()=>external_recorder_ui_describer_js_namespaceObject.describeRecorderUIEvent,
     dataExtractionAPIs: ()=>external_common_js_namespaceObject.dataExtractionAPIs,
     PlaygroundServer: ()=>external_server_js_namespaceObject.PlaygroundServer,
     createMjpegPreviewDescriptor: ()=>external_platform_js_namespaceObject.createMjpegPreviewDescriptor,
@@ -42,13 +46,14 @@ __webpack_require__.d(__webpack_exports__, {
     formatErrorMessage: ()=>external_common_js_namespaceObject.formatErrorMessage,
     definePlaygroundPlatform: ()=>external_platform_js_namespaceObject.definePlaygroundPlatform,
     createScrcpyPreviewDescriptor: ()=>external_platform_js_namespaceObject.createScrcpyPreviewDescriptor,
-    launchPreparedPlaygroundPlatform: ()=>external_platform_launcher_js_namespaceObject.launchPreparedPlaygroundPlatform,
+    describeRecorderUIEvents: ()=>external_recorder_ui_describer_js_namespaceObject.describeRecorderUIEvents,
     executeAction: ()=>external_common_js_namespaceObject.executeAction,
-    playgroundForAgent: ()=>external_launcher_js_namespaceObject.playgroundForAgent,
-    playgroundForAgentFactory: ()=>external_launcher_js_namespaceObject.playgroundForAgentFactory
+    launchPreparedPlaygroundPlatform: ()=>external_platform_launcher_js_namespaceObject.launchPreparedPlaygroundPlatform,
+    playgroundForAgent: ()=>external_launcher_js_namespaceObject.playgroundForAgent
 });
 const external_common_js_namespaceObject = require("./common.js");
 const external_server_js_namespaceObject = require("./server.js");
+const external_recorder_ui_describer_js_namespaceObject = require("./recorder-ui-describer.js");
 const external_launcher_js_namespaceObject = require("./launcher.js");
 const external_platform_js_namespaceObject = require("./platform.js");
 const external_platform_launcher_js_namespaceObject = require("./platform-launcher.js");
@@ -62,13 +67,17 @@ exports.LocalExecutionAdapter = __webpack_exports__.LocalExecutionAdapter;
 exports.PlaygroundSDK = __webpack_exports__.PlaygroundSDK;
 exports.PlaygroundServer = __webpack_exports__.PlaygroundServer;
 exports.RemoteExecutionAdapter = __webpack_exports__.RemoteExecutionAdapter;
+exports.buildPlaygroundBrowserUrl = __webpack_exports__.buildPlaygroundBrowserUrl;
 exports.createMjpegPreviewDescriptor = __webpack_exports__.createMjpegPreviewDescriptor;
 exports.createScrcpyPreviewDescriptor = __webpack_exports__.createScrcpyPreviewDescriptor;
 exports.createScreenshotPreviewDescriptor = __webpack_exports__.createScreenshotPreviewDescriptor;
 exports.dataExtractionAPIs = __webpack_exports__.dataExtractionAPIs;
 exports.definePlaygroundPlatform = __webpack_exports__.definePlaygroundPlatform;
+exports.describeRecorderUIEvent = __webpack_exports__.describeRecorderUIEvent;
+exports.describeRecorderUIEvents = __webpack_exports__.describeRecorderUIEvents;
 exports.executeAction = __webpack_exports__.executeAction;
 exports.formatErrorMessage = __webpack_exports__.formatErrorMessage;
+exports.getRecorderUIEventTargetRect = __webpack_exports__.getRecorderUIEventTargetRect;
 exports.launchPreparedPlaygroundPlatform = __webpack_exports__.launchPreparedPlaygroundPlatform;
 exports.noReplayAPIs = __webpack_exports__.noReplayAPIs;
 exports.playgroundForAgent = __webpack_exports__.playgroundForAgent;
@@ -85,13 +94,17 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "PlaygroundSDK",
     "PlaygroundServer",
     "RemoteExecutionAdapter",
+    "buildPlaygroundBrowserUrl",
     "createMjpegPreviewDescriptor",
     "createScrcpyPreviewDescriptor",
     "createScreenshotPreviewDescriptor",
     "dataExtractionAPIs",
     "definePlaygroundPlatform",
+    "describeRecorderUIEvent",
+    "describeRecorderUIEvents",
     "executeAction",
     "formatErrorMessage",
+    "getRecorderUIEventTargetRect",
     "launchPreparedPlaygroundPlatform",
     "noReplayAPIs",
     "playgroundForAgent",

package/dist/lib/launcher.js

@@ -88,12 +88,13 @@ function createPlaygroundLauncher(agentOrFactory) {
             if (configureServer) await configureServer(server);
             const launchedServer = await server.launch(port);
             if (verbose) console.log(`✅ Playground server started on port ${port}`);
-            const url = `http://127.0.0.1:${port}`;
+            const host = (0, external_server_js_namespaceObject.resolvePlaygroundBrowserHost)();
+            const url = (0, external_server_js_namespaceObject.buildPlaygroundBrowserUrl)(host, port);
             if (openBrowser) await openInBrowser(url, browserCommand, verbose);
             return {
                 server: launchedServer,
                 port,
-                host: '127.0.0.1',
+                host,
                 close: async ()=>{
                     if (verbose) console.log('🛑 Shutting down Midscene Playground...');
                     try {

package/dist/lib/launcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"launcher.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/launcher.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { spawn } from 'node:child_process';\nimport type { Agent } from '@midscene/core/agent';\nimport { PLAYGROUND_SERVER_PORT } from '@midscene/shared/constants';\nimport cors, { type CorsOptions } from 'cors';\nimport PlaygroundServer from './server';\nimport type { AgentFactory } from './types';\n\nexport interface LaunchPlaygroundOptions {\n  /**\n   * Port to start the playground server on\n   * @default 5800\n   */\n  port?: number;\n\n  /**\n   * Whether to automatically open the playground in browser\n   * @default true\n   */\n  openBrowser?: boolean;\n\n  /**\n   * Custom browser command to open playground\n   * @default 'open' on macOS, 'start' on Windows, 'xdg-open' on Linux\n   */\n  browserCommand?: string;\n\n  /**\n   * Whether to show server logs\n   * @default true\n   */\n  verbose?: boolean;\n\n  /**\n   * Fixed ID for the playground server instance\n   * If provided, the same ID will be used across restarts,\n   * allowing chat history to persist\n   * @default undefined (generates random UUID)\n   */\n  id?: string;\n\n  /**\n   * Whether to enable CORS (Cross-Origin Resource Sharing)\n   * @default false\n   */\n  enableCors?: boolean;\n\n  /**\n   * Custom static assets directory for the playground frontend\n   * @default bundled static assets from @midscene/playground\n   */\n  staticPath?: string;\n\n  /**\n   * Hook for configuring the PlaygroundServer before launch\n   * Useful for adding custom middleware beyond the built-in CORS option\n   */\n  configureServer?: (server: PlaygroundServer) => void | Promise<void>;\n\n  /**\n   * CORS configuration options\n   * @default only allows loopback browser origins when enableCors is true\n   */\n  corsOptions?: CorsOptions;\n}\n\nexport interface LaunchPlaygroundResult {\n  /**\n   * The playground server instance\n   */\n  server: PlaygroundServer;\n\n  /**\n   * The server port\n   */\n  port: number;\n\n  /**\n   * The server host\n   */\n  host: string;\n\n  /**\n   * Function to gracefully shutdown the playground\n   */\n  close: () => Promise<void>;\n}\n\ntype LaunchableAgentSource = Agent | AgentFactory | undefined;\n\nconst LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]']);\n\nfunction isLoopbackOrigin(origin?: string) {\n  if (!origin) {\n    return true;\n  }\n\n  try {\n    const url = new URL(origin);\n    return LOOPBACK_HOSTS.has(url.hostname);\n  } catch {\n    return false;\n  }\n}\n\nfunction createDefaultCorsOptions(): CorsOptions {\n  return {\n    origin(origin, callback) {\n      callback(null, isLoopbackOrigin(origin));\n    },\n    credentials: true,\n  };\n}\n\n/**\n * Create a playground launcher for a specific agent\n *\n * @example\n * ```typescript\n * import { playgroundForAgent } from '@midscene/playground';\n * import { SampleDevice, Agent } from '@midscene/core';\n *\n * const device = new SampleDevice();\n * const agent = new Agent(device);\n *\n * // Launch playground for the agent\n * const server = await playgroundForAgent(agent).launch();\n *\n * // Launch with CORS enabled\n * const serverWithCors = await playgroundForAgent(agent).launch({\n *   enableCors: true,\n *   corsOptions: {\n *     origin: ['http://localhost:3000', 'http://localhost:8080'],\n *     credentials: true\n *   }\n * });\n *\n * // Later, when you want to shutdown:\n * server.close();\n * ```\n */\nfunction createPlaygroundLauncher(agentOrFactory: LaunchableAgentSource) {\n  return {\n    /**\n     * Launch the playground server with optional configuration\n     */\n    async launch(\n      options: LaunchPlaygroundOptions = {},\n    ): Promise<LaunchPlaygroundResult> {\n      const {\n        port = PLAYGROUND_SERVER_PORT,\n        openBrowser = true,\n        browserCommand,\n        verbose = true,\n        id,\n        enableCors = false,\n        staticPath,\n        configureServer,\n        corsOptions = createDefaultCorsOptions(),\n      } = options;\n\n      if (\n        agentOrFactory &&\n        typeof agentOrFactory !== 'function' &&\n        !agentOrFactory.interface\n      ) {\n        throw new Error('Agent must have an interface property');\n      }\n\n      if (verbose) {\n        console.log('🚀 Starting Midscene Playground...');\n        if (!agentOrFactory) {\n          console.log('📱 Agent: session-managed');\n        } else if (typeof agentOrFactory === 'function') {\n          console.log('📱 Agent: factory');\n        } else {\n          console.log(`📱 Agent: ${agentOrFactory.constructor.name}`);\n          console.log(`🖥️ Page: ${agentOrFactory.interface.constructor.name}`);\n        }\n        console.log(`🌐 Port: ${port}`);\n        if (staticPath) {\n          console.log(`📁 Static path: ${staticPath}`);\n        }\n        if (enableCors) {\n          console.log('🔓 CORS enabled');\n        }\n      }\n\n      const server = new PlaygroundServer(agentOrFactory, staticPath, id);\n\n      if (enableCors) {\n        server.app.use(cors(corsOptions));\n      }\n\n      if (configureServer) {\n        await configureServer(server);\n      }\n\n      const launchedServer = await server.launch(port);\n\n      if (verbose) {\n        console.log(`✅ Playground server started on port ${port}`);\n      }\n\n      const url = `http://127.0.0.1:${port}`;\n\n      // Open browser if requested\n      if (openBrowser) {\n        await openInBrowser(url, browserCommand, verbose);\n      }\n\n      return {\n        server: launchedServer,\n        port,\n        host: '127.0.0.1',\n        close: async () => {\n          if (verbose) {\n            console.log('🛑 Shutting down Midscene Playground...');\n          }\n\n          try {\n            await launchedServer.close();\n            if (verbose) {\n              console.log('✅ Playground shutdown complete');\n            }\n          } catch (error) {\n            if (verbose) {\n              console.error('❌ Error during playground shutdown:', error);\n            }\n            throw error;\n          }\n        },\n      };\n    },\n  };\n}\n\n/**\n * Create a playground launcher from an already initialized agent instance\n */\nexport function playgroundForAgent(agent: Agent) {\n  return createPlaygroundLauncher(agent);\n}\n\n/**\n * Create a playground launcher from an agent factory\n * Useful for device-backed agents that need to be recreated after cancellation\n */\nexport function playgroundForAgentFactory(agentFactory: AgentFactory) {\n  return createPlaygroundLauncher(agentFactory);\n}\n\nexport function playgroundForSessionManager() {\n  return createPlaygroundLauncher(undefined);\n}\n\n/**\n * Open URL in browser using platform-appropriate command\n */\nasync function openInBrowser(\n  url: string,\n  customCommand?: string,\n  verbose = true,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    let command: string;\n    let args: string[];\n\n    if (customCommand) {\n      command = customCommand;\n      args = [url];\n    } else {\n      // Detect platform and use appropriate command\n      switch (process.platform) {\n        case 'darwin':\n          command = 'open';\n          args = [url];\n          break;\n        case 'win32':\n          command = 'start';\n          args = ['', url]; // Empty string for title\n          break;\n        default:\n          command = 'xdg-open';\n          args = [url];\n          break;\n      }\n    }\n\n    if (verbose) {\n      console.log(`🌐 Opening browser: ${command} ${args.join(' ')}`);\n    }\n\n    const child = spawn(command, args, {\n      detached: true,\n      stdio: 'ignore',\n    });\n\n    child.on('error', (error) => {\n      if (verbose) {\n        console.warn('⚠️  Failed to open browser automatically:', error.message);\n        console.log(`🌐 Please open manually: ${url}`);\n      }\n      // Don't reject, just continue - browser opening is optional\n      resolve();\n    });\n\n    child.on('close', () => {\n      resolve();\n    });\n\n    // Don't wait for the browser process\n    child.unref();\n  });\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","LOOPBACK_HOSTS","Set","isLoopbackOrigin","origin","url","URL","createDefaultCorsOptions","callback","createPlaygroundLauncher","agentOrFactory","options","port","PLAYGROUND_SERVER_PORT","openBrowser","browserCommand","verbose","id","enableCors","staticPath","configureServer","corsOptions","Error","console","server","PlaygroundServer","cors","launchedServer","openInBrowser","error","playgroundForAgent","agent","playgroundForAgentFactory","agentFactory","playgroundForSessionManager","undefined","customCommand","Promise","resolve","reject","command","args","process","child","spawn"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACmFA,MAAMI,iBAAiB,IAAIC,IAAI;IAAC;IAAa;IAAa;IAAO;CAAQ;AAEzE,SAASC,iBAAiBC,MAAe;IACvC,IAAI,CAACA,QACH,OAAO;IAGT,IAAI;QACF,MAAMC,MAAM,IAAIC,IAAIF;QACpB,OAAOH,eAAe,GAAG,CAACI,IAAI,QAAQ;IACxC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,SAASE;IACP,OAAO;QACL,QAAOH,MAAM,EAAEI,QAAQ;YACrBA,SAAS,MAAML,iBAAiBC;QAClC;QACA,aAAa;IACf;AACF;AA6BA,SAASK,yBAAyBC,cAAqC;IACrE,OAAO;QAIL,MAAM,QACJC,UAAmC,CAAC,CAAC;YAErC,MAAM,EACJC,OAAOC,0BAAAA,sBAAsB,EAC7BC,cAAc,IAAI,EAClBC,cAAc,EACdC,UAAU,IAAI,EACdC,EAAE,EACFC,aAAa,KAAK,EAClBC,UAAU,EACVC,eAAe,EACfC,cAAcd,0BAA0B,EACzC,GAAGI;YAEJ,IACED,kBACA,AAA0B,cAA1B,OAAOA,kBACP,CAACA,eAAe,SAAS,EAEzB,MAAM,IAAIY,MAAM;YAGlB,IAAIN,SAAS;gBACXO,QAAQ,GAAG,CAAC;gBACZ,IAAKb,gBAEE,IAAI,AAA0B,cAA1B,OAAOA,gBAChBa,QAAQ,GAAG,CAAC;qBACP;oBACLA,QAAQ,GAAG,CAAC,CAAC,UAAU,EAAEb,eAAe,WAAW,CAAC,IAAI,EAAE;oBAC1Da,QAAQ,GAAG,CAAC,CAAC,UAAU,EAAEb,eAAe,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE;gBACtE;qBANEa,QAAQ,GAAG,CAAC;gBAOdA,QAAQ,GAAG,CAAC,CAAC,SAAS,EAAEX,MAAM;gBAC9B,IAAIO,YACFI,QAAQ,GAAG,CAAC,CAAC,gBAAgB,EAAEJ,YAAY;gBAE7C,IAAID,YACFK,QAAQ,GAAG,CAAC;YAEhB;YAEA,MAAMC,SAAS,IAAIC,CAAAA,4BAAAA,EAAiBf,gBAAgBS,YAAYF;YAEhE,IAAIC,YACFM,OAAO,GAAG,CAAC,GAAG,CAACE,wBAAKL;YAGtB,IAAID,iBACF,MAAMA,gBAAgBI;YAGxB,MAAMG,iBAAiB,MAAMH,OAAO,MAAM,CAACZ;YAE3C,IAAII,SACFO,QAAQ,GAAG,CAAC,CAAC,oCAAoC,EAAEX,MAAM;YAG3D,MAAMP,MAAM,CAAC,iBAAiB,EAAEO,MAAM;YAGtC,IAAIE,aACF,MAAMc,cAAcvB,KAAKU,gBAAgBC;YAG3C,OAAO;gBACL,QAAQW;gBACRf;gBACA,MAAM;gBACN,OAAO;oBACL,IAAII,SACFO,QAAQ,GAAG,CAAC;oBAGd,IAAI;wBACF,MAAMI,eAAe,KAAK;wBAC1B,IAAIX,SACFO,QAAQ,GAAG,CAAC;oBAEhB,EAAE,OAAOM,OAAO;wBACd,IAAIb,SACFO,QAAQ,KAAK,CAAC,uCAAuCM;wBAEvD,MAAMA;oBACR;gBACF;YACF;QACF;IACF;AACF;AAKO,SAASC,mBAAmBC,KAAY;IAC7C,OAAOtB,yBAAyBsB;AAClC;AAMO,SAASC,0BAA0BC,YAA0B;IAClE,OAAOxB,yBAAyBwB;AAClC;AAEO,SAASC;IACd,OAAOzB,yBAAyB0B;AAClC;AAKA,eAAeP,cACbvB,GAAW,EACX+B,aAAsB,EACtBpB,UAAU,IAAI;IAEd,OAAO,IAAIqB,QAAQ,CAACC,SAASC;QAC3B,IAAIC;QACJ,IAAIC;QAEJ,IAAIL,eAAe;YACjBI,UAAUJ;YACVK,OAAO;gBAACpC;aAAI;QACd,OAEE,OAAQqC,QAAQ,QAAQ;YACtB,KAAK;gBACHF,UAAU;gBACVC,OAAO;oBAACpC;iBAAI;gBACZ;YACF,KAAK;gBACHmC,UAAU;gBACVC,OAAO;oBAAC;oBAAIpC;iBAAI;gBAChB;YACF;gBACEmC,UAAU;gBACVC,OAAO;oBAACpC;iBAAI;gBACZ;QACJ;QAGF,IAAIW,SACFO,QAAQ,GAAG,CAAC,CAAC,oBAAoB,EAAEiB,QAAQ,CAAC,EAAEC,KAAK,IAAI,CAAC,MAAM;QAGhE,MAAME,QAAQC,AAAAA,IAAAA,4CAAAA,KAAAA,AAAAA,EAAMJ,SAASC,MAAM;YACjC,UAAU;YACV,OAAO;QACT;QAEAE,MAAM,EAAE,CAAC,SAAS,CAACd;YACjB,IAAIb,SAAS;gBACXO,QAAQ,IAAI,CAAC,6CAA6CM,MAAM,OAAO;gBACvEN,QAAQ,GAAG,CAAC,CAAC,yBAAyB,EAAElB,KAAK;YAC/C;YAEAiC;QACF;QAEAK,MAAM,EAAE,CAAC,SAAS;YAChBL;QACF;QAGAK,MAAM,KAAK;IACb;AACF"}
+{"version":3,"file":"launcher.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/launcher.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { spawn } from 'node:child_process';\nimport type { Agent } from '@midscene/core/agent';\nimport { PLAYGROUND_SERVER_PORT } from '@midscene/shared/constants';\nimport cors, { type CorsOptions } from 'cors';\nimport PlaygroundServer, {\n  buildPlaygroundBrowserUrl,\n  resolvePlaygroundBrowserHost,\n} from './server';\nimport type { AgentFactory } from './types';\n\nexport interface LaunchPlaygroundOptions {\n  /**\n   * Port to start the playground server on\n   * @default 5800\n   */\n  port?: number;\n\n  /**\n   * Whether to automatically open the playground in browser\n   * @default true\n   */\n  openBrowser?: boolean;\n\n  /**\n   * Custom browser command to open playground\n   * @default 'open' on macOS, 'start' on Windows, 'xdg-open' on Linux\n   */\n  browserCommand?: string;\n\n  /**\n   * Whether to show server logs\n   * @default true\n   */\n  verbose?: boolean;\n\n  /**\n   * Fixed ID for the playground server instance\n   * If provided, the same ID will be used across restarts,\n   * allowing chat history to persist\n   * @default undefined (generates random UUID)\n   */\n  id?: string;\n\n  /**\n   * Whether to enable CORS (Cross-Origin Resource Sharing)\n   * @default false\n   */\n  enableCors?: boolean;\n\n  /**\n   * Custom static assets directory for the playground frontend\n   * @default bundled static assets from @midscene/playground\n   */\n  staticPath?: string;\n\n  /**\n   * Hook for configuring the PlaygroundServer before launch\n   * Useful for adding custom middleware beyond the built-in CORS option\n   */\n  configureServer?: (server: PlaygroundServer) => void | Promise<void>;\n\n  /**\n   * CORS configuration options\n   * @default only allows loopback browser origins when enableCors is true\n   */\n  corsOptions?: CorsOptions;\n}\n\nexport interface LaunchPlaygroundResult {\n  /**\n   * The playground server instance\n   */\n  server: PlaygroundServer;\n\n  /**\n   * The server port\n   */\n  port: number;\n\n  /**\n   * The server host\n   */\n  host: string;\n\n  /**\n   * Function to gracefully shutdown the playground\n   */\n  close: () => Promise<void>;\n}\n\ntype LaunchableAgentSource = Agent | AgentFactory | undefined;\n\nconst LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]']);\n\nfunction isLoopbackOrigin(origin?: string) {\n  if (!origin) {\n    return true;\n  }\n\n  try {\n    const url = new URL(origin);\n    return LOOPBACK_HOSTS.has(url.hostname);\n  } catch {\n    return false;\n  }\n}\n\nfunction createDefaultCorsOptions(): CorsOptions {\n  return {\n    origin(origin, callback) {\n      callback(null, isLoopbackOrigin(origin));\n    },\n    credentials: true,\n  };\n}\n\n/**\n * Create a playground launcher for a specific agent\n *\n * @example\n * ```typescript\n * import { playgroundForAgent } from '@midscene/playground';\n * import { SampleDevice, Agent } from '@midscene/core';\n *\n * const device = new SampleDevice();\n * const agent = new Agent(device);\n *\n * // Launch playground for the agent\n * const server = await playgroundForAgent(agent).launch();\n *\n * // Launch with CORS enabled\n * const serverWithCors = await playgroundForAgent(agent).launch({\n *   enableCors: true,\n *   corsOptions: {\n *     origin: ['http://localhost:3000', 'http://localhost:8080'],\n *     credentials: true\n *   }\n * });\n *\n * // Later, when you want to shutdown:\n * server.close();\n * ```\n */\nfunction createPlaygroundLauncher(agentOrFactory: LaunchableAgentSource) {\n  return {\n    /**\n     * Launch the playground server with optional configuration\n     */\n    async launch(\n      options: LaunchPlaygroundOptions = {},\n    ): Promise<LaunchPlaygroundResult> {\n      const {\n        port = PLAYGROUND_SERVER_PORT,\n        openBrowser = true,\n        browserCommand,\n        verbose = true,\n        id,\n        enableCors = false,\n        staticPath,\n        configureServer,\n        corsOptions = createDefaultCorsOptions(),\n      } = options;\n\n      if (\n        agentOrFactory &&\n        typeof agentOrFactory !== 'function' &&\n        !agentOrFactory.interface\n      ) {\n        throw new Error('Agent must have an interface property');\n      }\n\n      if (verbose) {\n        console.log('🚀 Starting Midscene Playground...');\n        if (!agentOrFactory) {\n          console.log('📱 Agent: session-managed');\n        } else if (typeof agentOrFactory === 'function') {\n          console.log('📱 Agent: factory');\n        } else {\n          console.log(`📱 Agent: ${agentOrFactory.constructor.name}`);\n          console.log(`🖥️ Page: ${agentOrFactory.interface.constructor.name}`);\n        }\n        console.log(`🌐 Port: ${port}`);\n        if (staticPath) {\n          console.log(`📁 Static path: ${staticPath}`);\n        }\n        if (enableCors) {\n          console.log('🔓 CORS enabled');\n        }\n      }\n\n      const server = new PlaygroundServer(agentOrFactory, staticPath, id);\n\n      if (enableCors) {\n        server.app.use(cors(corsOptions));\n      }\n\n      if (configureServer) {\n        await configureServer(server);\n      }\n\n      const launchedServer = await server.launch(port);\n\n      if (verbose) {\n        console.log(`✅ Playground server started on port ${port}`);\n      }\n\n      const host = resolvePlaygroundBrowserHost();\n      const url = buildPlaygroundBrowserUrl(host, port);\n\n      // Open browser if requested\n      if (openBrowser) {\n        await openInBrowser(url, browserCommand, verbose);\n      }\n\n      return {\n        server: launchedServer,\n        port,\n        host,\n        close: async () => {\n          if (verbose) {\n            console.log('🛑 Shutting down Midscene Playground...');\n          }\n\n          try {\n            await launchedServer.close();\n            if (verbose) {\n              console.log('✅ Playground shutdown complete');\n            }\n          } catch (error) {\n            if (verbose) {\n              console.error('❌ Error during playground shutdown:', error);\n            }\n            throw error;\n          }\n        },\n      };\n    },\n  };\n}\n\n/**\n * Create a playground launcher from an already initialized agent instance\n */\nexport function playgroundForAgent(agent: Agent) {\n  return createPlaygroundLauncher(agent);\n}\n\n/**\n * Create a playground launcher from an agent factory\n * Useful for device-backed agents that need to be recreated after cancellation\n */\nexport function playgroundForAgentFactory(agentFactory: AgentFactory) {\n  return createPlaygroundLauncher(agentFactory);\n}\n\nexport function playgroundForSessionManager() {\n  return createPlaygroundLauncher(undefined);\n}\n\n/**\n * Open URL in browser using platform-appropriate command\n */\nasync function openInBrowser(\n  url: string,\n  customCommand?: string,\n  verbose = true,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    let command: string;\n    let args: string[];\n\n    if (customCommand) {\n      command = customCommand;\n      args = [url];\n    } else {\n      // Detect platform and use appropriate command\n      switch (process.platform) {\n        case 'darwin':\n          command = 'open';\n          args = [url];\n          break;\n        case 'win32':\n          command = 'start';\n          args = ['', url]; // Empty string for title\n          break;\n        default:\n          command = 'xdg-open';\n          args = [url];\n          break;\n      }\n    }\n\n    if (verbose) {\n      console.log(`🌐 Opening browser: ${command} ${args.join(' ')}`);\n    }\n\n    const child = spawn(command, args, {\n      detached: true,\n      stdio: 'ignore',\n    });\n\n    child.on('error', (error) => {\n      if (verbose) {\n        console.warn('⚠️  Failed to open browser automatically:', error.message);\n        console.log(`🌐 Please open manually: ${url}`);\n      }\n      // Don't reject, just continue - browser opening is optional\n      resolve();\n    });\n\n    child.on('close', () => {\n      resolve();\n    });\n\n    // Don't wait for the browser process\n    child.unref();\n  });\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","LOOPBACK_HOSTS","Set","isLoopbackOrigin","origin","url","URL","createDefaultCorsOptions","callback","createPlaygroundLauncher","agentOrFactory","options","port","PLAYGROUND_SERVER_PORT","openBrowser","browserCommand","verbose","id","enableCors","staticPath","configureServer","corsOptions","Error","console","server","PlaygroundServer","cors","launchedServer","host","resolvePlaygroundBrowserHost","buildPlaygroundBrowserUrl","openInBrowser","error","playgroundForAgent","agent","playgroundForAgentFactory","agentFactory","playgroundForSessionManager","undefined","customCommand","Promise","resolve","reject","command","args","process","child","spawn"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACsFA,MAAMI,iBAAiB,IAAIC,IAAI;IAAC;IAAa;IAAa;IAAO;CAAQ;AAEzE,SAASC,iBAAiBC,MAAe;IACvC,IAAI,CAACA,QACH,OAAO;IAGT,IAAI;QACF,MAAMC,MAAM,IAAIC,IAAIF;QACpB,OAAOH,eAAe,GAAG,CAACI,IAAI,QAAQ;IACxC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,SAASE;IACP,OAAO;QACL,QAAOH,MAAM,EAAEI,QAAQ;YACrBA,SAAS,MAAML,iBAAiBC;QAClC;QACA,aAAa;IACf;AACF;AA6BA,SAASK,yBAAyBC,cAAqC;IACrE,OAAO;QAIL,MAAM,QACJC,UAAmC,CAAC,CAAC;YAErC,MAAM,EACJC,OAAOC,0BAAAA,sBAAsB,EAC7BC,cAAc,IAAI,EAClBC,cAAc,EACdC,UAAU,IAAI,EACdC,EAAE,EACFC,aAAa,KAAK,EAClBC,UAAU,EACVC,eAAe,EACfC,cAAcd,0BAA0B,EACzC,GAAGI;YAEJ,IACED,kBACA,AAA0B,cAA1B,OAAOA,kBACP,CAACA,eAAe,SAAS,EAEzB,MAAM,IAAIY,MAAM;YAGlB,IAAIN,SAAS;gBACXO,QAAQ,GAAG,CAAC;gBACZ,IAAKb,gBAEE,IAAI,AAA0B,cAA1B,OAAOA,gBAChBa,QAAQ,GAAG,CAAC;qBACP;oBACLA,QAAQ,GAAG,CAAC,CAAC,UAAU,EAAEb,eAAe,WAAW,CAAC,IAAI,EAAE;oBAC1Da,QAAQ,GAAG,CAAC,CAAC,UAAU,EAAEb,eAAe,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE;gBACtE;qBANEa,QAAQ,GAAG,CAAC;gBAOdA,QAAQ,GAAG,CAAC,CAAC,SAAS,EAAEX,MAAM;gBAC9B,IAAIO,YACFI,QAAQ,GAAG,CAAC,CAAC,gBAAgB,EAAEJ,YAAY;gBAE7C,IAAID,YACFK,QAAQ,GAAG,CAAC;YAEhB;YAEA,MAAMC,SAAS,IAAIC,CAAAA,4BAAAA,EAAiBf,gBAAgBS,YAAYF;YAEhE,IAAIC,YACFM,OAAO,GAAG,CAAC,GAAG,CAACE,wBAAKL;YAGtB,IAAID,iBACF,MAAMA,gBAAgBI;YAGxB,MAAMG,iBAAiB,MAAMH,OAAO,MAAM,CAACZ;YAE3C,IAAII,SACFO,QAAQ,GAAG,CAAC,CAAC,oCAAoC,EAAEX,MAAM;YAG3D,MAAMgB,OAAOC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA;YACb,MAAMxB,MAAMyB,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,EAA0BF,MAAMhB;YAG5C,IAAIE,aACF,MAAMiB,cAAc1B,KAAKU,gBAAgBC;YAG3C,OAAO;gBACL,QAAQW;gBACRf;gBACAgB;gBACA,OAAO;oBACL,IAAIZ,SACFO,QAAQ,GAAG,CAAC;oBAGd,IAAI;wBACF,MAAMI,eAAe,KAAK;wBAC1B,IAAIX,SACFO,QAAQ,GAAG,CAAC;oBAEhB,EAAE,OAAOS,OAAO;wBACd,IAAIhB,SACFO,QAAQ,KAAK,CAAC,uCAAuCS;wBAEvD,MAAMA;oBACR;gBACF;YACF;QACF;IACF;AACF;AAKO,SAASC,mBAAmBC,KAAY;IAC7C,OAAOzB,yBAAyByB;AAClC;AAMO,SAASC,0BAA0BC,YAA0B;IAClE,OAAO3B,yBAAyB2B;AAClC;AAEO,SAASC;IACd,OAAO5B,yBAAyB6B;AAClC;AAKA,eAAeP,cACb1B,GAAW,EACXkC,aAAsB,EACtBvB,UAAU,IAAI;IAEd,OAAO,IAAIwB,QAAQ,CAACC,SAASC;QAC3B,IAAIC;QACJ,IAAIC;QAEJ,IAAIL,eAAe;YACjBI,UAAUJ;YACVK,OAAO;gBAACvC;aAAI;QACd,OAEE,OAAQwC,QAAQ,QAAQ;YACtB,KAAK;gBACHF,UAAU;gBACVC,OAAO;oBAACvC;iBAAI;gBACZ;YACF,KAAK;gBACHsC,UAAU;gBACVC,OAAO;oBAAC;oBAAIvC;iBAAI;gBAChB;YACF;gBACEsC,UAAU;gBACVC,OAAO;oBAACvC;iBAAI;gBACZ;QACJ;QAGF,IAAIW,SACFO,QAAQ,GAAG,CAAC,CAAC,oBAAoB,EAAEoB,QAAQ,CAAC,EAAEC,KAAK,IAAI,CAAC,MAAM;QAGhE,MAAME,QAAQC,AAAAA,IAAAA,4CAAAA,KAAAA,AAAAA,EAAMJ,SAASC,MAAM;YACjC,UAAU;YACV,OAAO;QACT;QAEAE,MAAM,EAAE,CAAC,SAAS,CAACd;YACjB,IAAIhB,SAAS;gBACXO,QAAQ,IAAI,CAAC,6CAA6CS,MAAM,OAAO;gBACvET,QAAQ,GAAG,CAAC,CAAC,yBAAyB,EAAElB,KAAK;YAC/C;YAEAoC;QACF;QAEAK,MAAM,EAAE,CAAC,SAAS;YAChBL;QACF;QAGAK,MAAM,KAAK;IACb;AACF"}

package/dist/lib/recorder-ai-service.js

@@ -0,0 +1,38 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    callAIWithObjectResponse: ()=>ai_model_namespaceObject.callAIWithObjectResponse
+});
+const ai_model_namespaceObject = require("@midscene/core/ai-model");
+exports.callAIWithObjectResponse = __webpack_exports__.callAIWithObjectResponse;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "callAIWithObjectResponse"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=recorder-ai-service.js.map

package/dist/lib/recorder-ai-service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recorder-ai-service.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/recorder-ui-describer-prompt.js

@@ -0,0 +1,90 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    RECORDER_UI_DESCRIBER_SYSTEM_PROMPT: ()=>RECORDER_UI_DESCRIBER_SYSTEM_PROMPT
+});
+const RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = `You convert Studio preview recorder UI events into semantic replay instructions.
+
+The recorder works from screenshots and mapped real-device coordinates only. Infer stable UI intent from the highlighted BEFORE screenshot. The AFTER screenshot is contextual evidence for state changes and scroll destinations.
+
+Output JSON only:
+{
+  "elementDescription": "short stable target/region description",
+  "replayInstruction": "one executable natural-language replay step",
+  "actionSummary": "short timeline summary",
+  "scrollDestinationDescription": "for scroll only: concrete newly visible destination content or goal",
+  "confidence": "high" | "medium" | "low",
+  "error"?: "only if no useful visual description can be inferred"
+}
+
+Rules:
+- Do NOT output coordinates as the main description.
+- Do NOT mention "near coordinates", "nearby element", "near point", "red marker", highlighted box, highlighted element, or screenshot.
+- Prefer stable target descriptions in this order: exact stable control text > stable label > role + stable section/context > icon purpose > visual position.
+- Treat placeholder or hint text that can change by user, time, data, or context as dynamic. Do not use dynamic hint values as the primary target description; prefer role + stable region + intent.
+- For repeated collections, treat item identity text as dynamic unless the user is clearly verifying that exact item. This includes any list/grid/table/feed/menu or repeated record surface. Do not output descriptions like "<role> titled/named '<content>'"; prefer stable role + region + selection intent.
+- Keep quoted UI text in the original UI language when it is a stable control label.
+- Apply the platform guidance from the user event:
+  - Web: button, input, link, menu item, tab, dialog, aria-label, placeholder, form section.
+  - Mobile: tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, screen section.
+  - Desktop/computer: menu item, toolbar button, dialog field, sidebar item, window control, file row, application region.
+- Pointer action rules:
+  - Preserve event.actionType semantics. Tap, DoubleClick, LongPress, and RightClick must not all become Click.
+  - Tap replayInstruction format: Tap on the element described as "<elementDescription>".
+  - DoubleClick replayInstruction format: Double click on the element described as "<elementDescription>".
+  - LongPress replayInstruction format: Long press the element described as "<elementDescription>".
+  - RightClick replayInstruction format: Right click on the element described as "<elementDescription>".
+  - Click replayInstruction format: Click on the element described as "<elementDescription>".
+- Input-specific rules:
+  - The highlighted BEFORE screenshot marks the field that receives the text.
+  - The screenshot after the action may show the typed value; use it only to confirm the field, never as the field description.
+  - elementDescription must identify the field itself by stable field role, nearby label, region, section, or sequence intent.
+  - Never use "AI is analyzing element", the typed value, the page title alone, or a generic "input field" as elementDescription.
+  - For consecutive input events, distinguish fields by stable role, section, order, current focus, filled/empty state, and neighboring actions instead of reusing the same generic field description.
+  - Input replayInstruction format: Input "<value>" into the element described as "<elementDescription>".
+- Scroll target quality bar:
+  - elementDescription describes the scrollable page, panel, list, table, or section at the highlighted scroll point.
+  - When multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu. Do not generalize a panel/list scroll into the whole page.
+  - scrollDestinationDescription is required and describes what the scroll is trying to reveal or reach, using newly visible headings, section titles, list items, or stable content from the AFTER screenshot.
+  - Prefer descriptions like "Playwright integration documentation page, scrolling toward the API reference section" or "Android API documentation page, scrolling to the installation steps section".
+  - Do NOT write generic phrases like "more content", "the page", "current screen", or "main scrollable area".
+- Scroll replayInstruction format: Scroll the page/region with description "<elementDescription>" by value "<recorded value>" until "<scrollDestinationDescription>" is visible.
+- Scroll actionSummary format: Scroll <elementDescription> toward <scrollDestinationDescription>.
+- Drag/Swipe rules:
+  - Drag replayInstruction format: Drag through the area described as "<elementDescription>".
+  - Swipe replayInstruction format: Swipe through the area described as "<elementDescription>".
+  - Describe start/end regions or the dragged UI control; do not describe only the gesture path.
+- KeyboardPress replayInstruction format: Press "<value>" on the element described as "<elementDescription>".
+- If uncertain, provide the best concrete visible text/role/context description. Set confidence to "low"; do not fall back to coordinates.`;
+exports.RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = __webpack_exports__.RECORDER_UI_DESCRIBER_SYSTEM_PROMPT;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "RECORDER_UI_DESCRIBER_SYSTEM_PROMPT"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=recorder-ui-describer-prompt.js.map

package/dist/lib/recorder-ui-describer-prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recorder-ui-describer-prompt.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/recorder-ui-describer-prompt.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","export const RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = `You convert Studio preview recorder UI events into semantic replay instructions.\n\nThe recorder works from screenshots and mapped real-device coordinates only. Infer stable UI intent from the highlighted BEFORE screenshot. The AFTER screenshot is contextual evidence for state changes and scroll destinations.\n\nOutput JSON only:\n{\n  \"elementDescription\": \"short stable target/region description\",\n  \"replayInstruction\": \"one executable natural-language replay step\",\n  \"actionSummary\": \"short timeline summary\",\n  \"scrollDestinationDescription\": \"for scroll only: concrete newly visible destination content or goal\",\n  \"confidence\": \"high\" | \"medium\" | \"low\",\n  \"error\"?: \"only if no useful visual description can be inferred\"\n}\n\nRules:\n- Do NOT output coordinates as the main description.\n- Do NOT mention \"near coordinates\", \"nearby element\", \"near point\", \"red marker\", highlighted box, highlighted element, or screenshot.\n- Prefer stable target descriptions in this order: exact stable control text > stable label > role + stable section/context > icon purpose > visual position.\n- Treat placeholder or hint text that can change by user, time, data, or context as dynamic. Do not use dynamic hint values as the primary target description; prefer role + stable region + intent.\n- For repeated collections, treat item identity text as dynamic unless the user is clearly verifying that exact item. This includes any list/grid/table/feed/menu or repeated record surface. Do not output descriptions like \"<role> titled/named '<content>'\"; prefer stable role + region + selection intent.\n- Keep quoted UI text in the original UI language when it is a stable control label.\n- Apply the platform guidance from the user event:\n  - Web: button, input, link, menu item, tab, dialog, aria-label, placeholder, form section.\n  - Mobile: tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, screen section.\n  - Desktop/computer: menu item, toolbar button, dialog field, sidebar item, window control, file row, application region.\n- Pointer action rules:\n  - Preserve event.actionType semantics. Tap, DoubleClick, LongPress, and RightClick must not all become Click.\n  - Tap replayInstruction format: Tap on the element described as \"<elementDescription>\".\n  - DoubleClick replayInstruction format: Double click on the element described as \"<elementDescription>\".\n  - LongPress replayInstruction format: Long press the element described as \"<elementDescription>\".\n  - RightClick replayInstruction format: Right click on the element described as \"<elementDescription>\".\n  - Click replayInstruction format: Click on the element described as \"<elementDescription>\".\n- Input-specific rules:\n  - The highlighted BEFORE screenshot marks the field that receives the text.\n  - The screenshot after the action may show the typed value; use it only to confirm the field, never as the field description.\n  - elementDescription must identify the field itself by stable field role, nearby label, region, section, or sequence intent.\n  - Never use \"AI is analyzing element\", the typed value, the page title alone, or a generic \"input field\" as elementDescription.\n  - For consecutive input events, distinguish fields by stable role, section, order, current focus, filled/empty state, and neighboring actions instead of reusing the same generic field description.\n  - Input replayInstruction format: Input \"<value>\" into the element described as \"<elementDescription>\".\n- Scroll target quality bar:\n  - elementDescription describes the scrollable page, panel, list, table, or section at the highlighted scroll point.\n  - When multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu. Do not generalize a panel/list scroll into the whole page.\n  - scrollDestinationDescription is required and describes what the scroll is trying to reveal or reach, using newly visible headings, section titles, list items, or stable content from the AFTER screenshot.\n  - Prefer descriptions like \"Playwright integration documentation page, scrolling toward the API reference section\" or \"Android API documentation page, scrolling to the installation steps section\".\n  - Do NOT write generic phrases like \"more content\", \"the page\", \"current screen\", or \"main scrollable area\".\n- Scroll replayInstruction format: Scroll the page/region with description \"<elementDescription>\" by value \"<recorded value>\" until \"<scrollDestinationDescription>\" is visible.\n- Scroll actionSummary format: Scroll <elementDescription> toward <scrollDestinationDescription>.\n- Drag/Swipe rules:\n  - Drag replayInstruction format: Drag through the area described as \"<elementDescription>\".\n  - Swipe replayInstruction format: Swipe through the area described as \"<elementDescription>\".\n  - Describe start/end regions or the dragged UI control; do not describe only the gesture path.\n- KeyboardPress replayInstruction format: Press \"<value>\" on the element described as \"<elementDescription>\".\n- If uncertain, provide the best concrete visible text/role/context description. Set confidence to \"low\"; do not fall back to coordinates.`;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","RECORDER_UI_DESCRIBER_SYSTEM_PROMPT"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;ACNO,MAAMI,sCAAsC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;0IAoDsF,CAAC"}