@midscene/playground 1.12.5-beta-20260907092148.0 → 1.12.5

package/dist/es/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type { DeviceAction } from '@midscene/core';\nimport { findAllMidsceneLocatorField } from '@midscene/core/ai-model';\nimport { buildDetailedLocateParam } from '@midscene/core/yaml';\nimport type {\n  ExecutionOptions,\n  FormValue,\n  PlaygroundAgent,\n  ValidationResult,\n} from './types';\n\n// APIs that should not generate replay scripts\nexport const dataExtractionAPIs = [\n  'aiQuery',\n  'aiBoolean',\n  'aiNumber',\n  'aiString',\n  'aiAsk',\n];\n\nexport const validationAPIs = ['aiAssert', 'aiWaitFor'];\n\nexport const noReplayAPIs = [...dataExtractionAPIs, ...validationAPIs];\n\nconst agentPromptAPIs = [\n  'aiAct',\n  'runMarkdown',\n  'aiQuery',\n  'aiBoolean',\n  'aiNumber',\n  'aiString',\n  'aiAsk',\n  'aiWaitFor',\n] as const;\n\ntype AgentPromptAPI = (typeof agentPromptAPIs)[number];\n\nconst agentPromptAPISet = new Set<string>(agentPromptAPIs);\n\nfunction getAgentPromptAPI(\n  activeAgent: PlaygroundAgent,\n  actionType: string,\n):\n  | ((\n      prompt: string | undefined,\n      options: ExecutionOptions,\n    ) => Promise<unknown>)\n  | undefined {\n  if (!agentPromptAPISet.has(actionType)) {\n    return undefined;\n  }\n\n  const methodName = actionType as AgentPromptAPI;\n  const method = activeAgent[methodName] as unknown;\n  if (typeof method !== 'function') {\n    return undefined;\n  }\n\n  return async (prompt, options) =>\n    (\n      method as (prompt: string, options: ExecutionOptions) => Promise<unknown>\n    ).call(activeAgent, prompt || '', options);\n}\n\nexport const formatErrorMessage = (e: any): string => {\n  const errorMessage = e?.message || '';\n\n  if (errorMessage.includes('of different extension')) {\n    return 'Conflicting extension detected. Please disable the suspicious plugins and refresh the page. Guide: https://midscenejs.com/quick-start.html#chrome-extension-faq';\n  }\n\n  if (errorMessage.includes('NOT_IMPLEMENTED_AS_DESIGNED')) {\n    return 'Further actions cannot be performed in the current environment';\n  }\n\n  return errorMessage || 'Unknown error';\n};\n\n// Parse structured parameters for callActionInActionSpace\nexport async function parseStructuredParams(\n  action: DeviceAction<unknown>,\n  params: Record<string, unknown>,\n  options: ExecutionOptions = {},\n): Promise<unknown[]> {\n  if (!action?.paramSchema || !('shape' in action.paramSchema)) {\n    return [params.prompt || '', options];\n  }\n\n  const schema = action.paramSchema;\n  const keys =\n    schema && 'shape' in schema\n      ? Object.keys((schema as { shape: Record<string, unknown> }).shape)\n      : [];\n\n  // Start with options and merge deviceOptions into the same level\n  // Destructure to exclude deviceOptions from the final object\n  const { deviceOptions: _, ...optionsWithoutDeviceOptions } = options;\n  const paramObj: Record<string, unknown> = {\n    ...optionsWithoutDeviceOptions,\n    ...(options.deviceOptions || {}),\n  };\n\n  keys.forEach((key) => {\n    if (\n      params[key] !== undefined &&\n      params[key] !== null &&\n      params[key] !== ''\n    ) {\n      paramObj[key] = params[key];\n    }\n  });\n\n  // Check if there's a locate field that needs detailed locate param processing\n  if (schema) {\n    const locatorFieldKeys = findAllMidsceneLocatorField(schema);\n    locatorFieldKeys.forEach((locateKey: string) => {\n      const locatePrompt = params[locateKey];\n      if (locatePrompt && typeof locatePrompt === 'string') {\n        // Build detailed locate param using the locate prompt and options\n        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, {\n          deepLocate: options.deepLocate,\n          cacheable: true, // Default to true for playground\n        });\n        if (detailedLocateParam) {\n          paramObj[locateKey] = detailedLocateParam;\n        }\n      }\n    });\n  }\n\n  return [paramObj];\n}\n\nexport function validateStructuredParams(\n  value: FormValue,\n  action: DeviceAction<unknown> | undefined,\n): ValidationResult {\n  if (!value.params) {\n    return { valid: false, errorMessage: 'Parameters are required' };\n  }\n\n  if (!action?.paramSchema) {\n    return { valid: true };\n  }\n\n  try {\n    const paramsForValidation = { ...value.params };\n\n    const schema = action.paramSchema;\n    if (schema) {\n      const locatorFieldKeys = findAllMidsceneLocatorField(schema);\n      locatorFieldKeys.forEach((key: string) => {\n        if (typeof paramsForValidation[key] === 'string') {\n          paramsForValidation[key] = {\n            prompt: paramsForValidation[key],\n            center: [0, 0],\n            rect: { left: 0, top: 0, width: 0, height: 0 },\n          };\n        }\n      });\n    }\n\n    action.paramSchema?.parse(paramsForValidation);\n    return { valid: true };\n  } catch (error: unknown) {\n    const zodError = error as {\n      errors?: Array<{ path: string[]; message: string }>;\n    };\n    if (zodError.errors && zodError.errors.length > 0) {\n      const errorMessages = zodError.errors\n        .filter((err) => {\n          const path = err.path.join('.');\n          return !path.includes('center') && !path.includes('rect');\n        })\n        .map((err) => {\n          const field = err.path.join('.');\n          return `${field}: ${err.message}`;\n        });\n\n      if (errorMessages.length > 0) {\n        return {\n          valid: false,\n          errorMessage: `Validation error: ${errorMessages.join(', ')}`,\n        };\n      }\n    } else {\n      const errorMsg =\n        error instanceof Error ? error.message : 'Unknown validation error';\n      return {\n        valid: false,\n        errorMessage: `Parameter validation failed: ${errorMsg}`,\n      };\n    }\n  }\n\n  return { valid: true };\n}\n\nexport async function executeAction(\n  activeAgent: PlaygroundAgent,\n  actionType: string,\n  actionSpace: DeviceAction<unknown>[],\n  value: FormValue,\n  options: ExecutionOptions,\n): Promise<unknown> {\n  const canForwardDeepThink =\n    actionType === 'aiAct' || actionType === 'runMarkdown';\n  if (!canForwardDeepThink && options.deepThink !== undefined) {\n    console.warn(\n      '[Playground] Received deepThink in non-aiAct action options. deepThink is expected to be used with aiAct/runMarkdown during migration.',\n      {\n        actionType,\n        requestId: options.requestId,\n        options,\n      },\n    );\n  }\n\n  const action = actionSpace?.find(\n    (a: DeviceAction<unknown>) =>\n      a.interfaceAlias === actionType || a.name === actionType,\n  );\n\n  if (action && typeof activeAgent.callActionInActionSpace === 'function') {\n    if (value.params) {\n      const parsedParams = await parseStructuredParams(\n        action,\n        value.params,\n        options,\n      );\n      return await activeAgent.callActionInActionSpace(\n        action.name,\n        parsedParams[0],\n      );\n    } else {\n      // For prompt-based actions, we need to build the detailed locate param\n      const detailedLocateParam = value.prompt\n        ? buildDetailedLocateParam(value.prompt, {\n            deepLocate: options.deepLocate,\n            cacheable: true,\n          })\n        : undefined;\n\n      // Flatten deviceOptions into the params\n      // Destructure to exclude deviceOptions from the final object\n      const {\n        deviceOptions: _,\n        reportDisplay: __,\n        ...optionsWithoutDeviceOptions\n      } = options;\n      const actionParams = {\n        locate: detailedLocateParam,\n        ...optionsWithoutDeviceOptions,\n        ...(options.deviceOptions || {}),\n      };\n\n      return await activeAgent.callActionInActionSpace(\n        action.name,\n        actionParams,\n      );\n    }\n  } else {\n    const prompt = value.prompt;\n\n    if (actionType === 'aiAssert') {\n      const { reportDisplay: _reportDisplay, ...optionsWithoutReportDisplay } =\n        options;\n      const { pass, thought } =\n        (await activeAgent?.aiAssert?.(prompt || '', undefined, {\n          keepRawResponse: true,\n          ...optionsWithoutReportDisplay,\n        })) || {};\n      return { pass: pass || false, thought: thought || '' };\n    }\n\n    const agentPromptAPI = getAgentPromptAPI(activeAgent, actionType);\n    if (agentPromptAPI) {\n      const { reportDisplay, ...agentOptions } = options;\n      const optionsForAgent = reportDisplay\n        ? {\n            ...agentOptions,\n            _internalReportDisplay: reportDisplay,\n          }\n        : agentOptions;\n      return await agentPromptAPI(prompt, optionsForAgent);\n    }\n\n    throw new Error(`Unknown action type: ${actionType}`);\n  }\n}\n"],"names":["dataExtractionAPIs","validationAPIs","noReplayAPIs","agentPromptAPIs","agentPromptAPISet","Set","getAgentPromptAPI","activeAgent","actionType","methodName","method","prompt","options","formatErrorMessage","e","errorMessage","parseStructuredParams","action","params","schema","keys","Object","_","optionsWithoutDeviceOptions","paramObj","key","undefined","locatorFieldKeys","findAllMidsceneLocatorField","locateKey","locatePrompt","detailedLocateParam","buildDetailedLocateParam","validateStructuredParams","value","paramsForValidation","error","zodError","errorMessages","err","path","field","errorMsg","Error","executeAction","actionSpace","canForwardDeepThink","console","a","parsedParams","__","actionParams","_reportDisplay","optionsWithoutReportDisplay","pass","thought","agentPromptAPI","reportDisplay","agentOptions","optionsForAgent"],"mappings":";;AAWO,MAAMA,qBAAqB;IAChC;IACA;IACA;IACA;IACA;CACD;AAEM,MAAMC,iBAAiB;IAAC;IAAY;CAAY;AAEhD,MAAMC,eAAe;OAAIF;OAAuBC;CAAe;AAEtE,MAAME,kBAAkB;IACtB;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAID,MAAMC,oBAAoB,IAAIC,IAAYF;AAE1C,SAASG,kBACPC,WAA4B,EAC5BC,UAAkB;IAOlB,IAAI,CAACJ,kBAAkB,GAAG,CAACI,aACzB;IAGF,MAAMC,aAAaD;IACnB,MAAME,SAASH,WAAW,CAACE,WAAW;IACtC,IAAI,AAAkB,cAAlB,OAAOC,QACT;IAGF,OAAO,OAAOC,QAAQC,UAElBF,OACA,IAAI,CAACH,aAAaI,UAAU,IAAIC;AACtC;AAEO,MAAMC,qBAAqB,CAACC;IACjC,MAAMC,eAAeD,GAAG,WAAW;IAEnC,IAAIC,aAAa,QAAQ,CAAC,2BACxB,OAAO;IAGT,IAAIA,aAAa,QAAQ,CAAC,gCACxB,OAAO;IAGT,OAAOA,gBAAgB;AACzB;AAGO,eAAeC,sBACpBC,MAA6B,EAC7BC,MAA+B,EAC/BN,UAA4B,CAAC,CAAC;IAE9B,IAAI,CAACK,QAAQ,eAAe,CAAE,YAAWA,OAAO,WAAU,GACxD,OAAO;QAACC,OAAO,MAAM,IAAI;QAAIN;KAAQ;IAGvC,MAAMO,SAASF,OAAO,WAAW;IACjC,MAAMG,OACJD,UAAU,WAAWA,SACjBE,OAAO,IAAI,CAAEF,OAA8C,KAAK,IAChE,EAAE;IAIR,MAAM,EAAE,eAAeG,CAAC,EAAE,GAAGC,6BAA6B,GAAGX;IAC7D,MAAMY,WAAoC;QACxC,GAAGD,2BAA2B;QAC9B,GAAIX,QAAQ,aAAa,IAAI,CAAC,CAAC;IACjC;IAEAQ,KAAK,OAAO,CAAC,CAACK;QACZ,IACEP,AAAgBQ,WAAhBR,MAAM,CAACO,IAAI,IACXP,AAAgB,SAAhBA,MAAM,CAACO,IAAI,IACXP,AAAgB,OAAhBA,MAAM,CAACO,IAAI,EAEXD,QAAQ,CAACC,IAAI,GAAGP,MAAM,CAACO,IAAI;IAE/B;IAGA,IAAIN,QAAQ;QACV,MAAMQ,mBAAmBC,4BAA4BT;QACrDQ,iBAAiB,OAAO,CAAC,CAACE;YACxB,MAAMC,eAAeZ,MAAM,CAACW,UAAU;YACtC,IAAIC,gBAAgB,AAAwB,YAAxB,OAAOA,cAA2B;gBAEpD,MAAMC,sBAAsBC,yBAAyBF,cAAc;oBACjE,YAAYlB,QAAQ,UAAU;oBAC9B,WAAW;gBACb;gBACA,IAAImB,qBACFP,QAAQ,CAACK,UAAU,GAAGE;YAE1B;QACF;IACF;IAEA,OAAO;QAACP;KAAS;AACnB;AAEO,SAASS,yBACdC,KAAgB,EAChBjB,MAAyC;IAEzC,IAAI,CAACiB,MAAM,MAAM,EACf,OAAO;QAAE,OAAO;QAAO,cAAc;IAA0B;IAGjE,IAAI,CAACjB,QAAQ,aACX,OAAO;QAAE,OAAO;IAAK;IAGvB,IAAI;QACF,MAAMkB,sBAAsB;YAAE,GAAGD,MAAM,MAAM;QAAC;QAE9C,MAAMf,SAASF,OAAO,WAAW;QACjC,IAAIE,QAAQ;YACV,MAAMQ,mBAAmBC,4BAA4BT;YACrDQ,iBAAiB,OAAO,CAAC,CAACF;gBACxB,IAAI,AAAoC,YAApC,OAAOU,mBAAmB,CAACV,IAAI,EACjCU,mBAAmB,CAACV,IAAI,GAAG;oBACzB,QAAQU,mBAAmB,CAACV,IAAI;oBAChC,QAAQ;wBAAC;wBAAG;qBAAE;oBACd,MAAM;wBAAE,MAAM;wBAAG,KAAK;wBAAG,OAAO;wBAAG,QAAQ;oBAAE;gBAC/C;YAEJ;QACF;QAEAR,OAAO,WAAW,EAAE,MAAMkB;IAE5B,EAAE,OAAOC,OAAgB;QACvB,MAAMC,WAAWD;QAGjB,IAAIC,SAAS,MAAM,IAAIA,SAAS,MAAM,CAAC,MAAM,GAAG,GAAG;YACjD,MAAMC,gBAAgBD,SAAS,MAAM,CAClC,MAAM,CAAC,CAACE;gBACP,MAAMC,OAAOD,IAAI,IAAI,CAAC,IAAI,CAAC;gBAC3B,OAAO,CAACC,KAAK,QAAQ,CAAC,aAAa,CAACA,KAAK,QAAQ,CAAC;YACpD,GACC,GAAG,CAAC,CAACD;gBACJ,MAAME,QAAQF,IAAI,IAAI,CAAC,IAAI,CAAC;gBAC5B,OAAO,GAAGE,MAAM,EAAE,EAAEF,IAAI,OAAO,EAAE;YACnC;YAEF,IAAID,cAAc,MAAM,GAAG,GACzB,OAAO;gBACL,OAAO;gBACP,cAAc,CAAC,kBAAkB,EAAEA,cAAc,IAAI,CAAC,OAAO;YAC/D;QAEJ,OAAO;YACL,MAAMI,WACJN,iBAAiBO,QAAQP,MAAM,OAAO,GAAG;YAC3C,OAAO;gBACL,OAAO;gBACP,cAAc,CAAC,6BAA6B,EAAEM,UAAU;YAC1D;QACF;IACF;IAEA,OAAO;QAAE,OAAO;IAAK;AACvB;AAEO,eAAeE,cACpBrC,WAA4B,EAC5BC,UAAkB,EAClBqC,WAAoC,EACpCX,KAAgB,EAChBtB,OAAyB;IAEzB,MAAMkC,sBACJtC,AAAe,YAAfA,cAA0BA,AAAe,kBAAfA;IAC5B,IAAI,CAACsC,uBAAuBlC,AAAsBc,WAAtBd,QAAQ,SAAS,EAC3CmC,QAAQ,IAAI,CACV,0IACA;QACEvC;QACA,WAAWI,QAAQ,SAAS;QAC5BA;IACF;IAIJ,MAAMK,SAAS4B,aAAa,KAC1B,CAACG,IACCA,EAAE,cAAc,KAAKxC,cAAcwC,EAAE,IAAI,KAAKxC;IAGlD,IAAIS,UAAU,AAA+C,cAA/C,OAAOV,YAAY,uBAAuB,EACtD,IAAI2B,MAAM,MAAM,EAAE;QAChB,MAAMe,eAAe,MAAMjC,sBACzBC,QACAiB,MAAM,MAAM,EACZtB;QAEF,OAAO,MAAML,YAAY,uBAAuB,CAC9CU,OAAO,IAAI,EACXgC,YAAY,CAAC,EAAE;IAEnB,OAAO;QAEL,MAAMlB,sBAAsBG,MAAM,MAAM,GACpCF,yBAAyBE,MAAM,MAAM,EAAE;YACrC,YAAYtB,QAAQ,UAAU;YAC9B,WAAW;QACb,KACAc;QAIJ,MAAM,EACJ,eAAeJ,CAAC,EAChB,eAAe4B,EAAE,EACjB,GAAG3B,6BACJ,GAAGX;QACJ,MAAMuC,eAAe;YACnB,QAAQpB;YACR,GAAGR,2BAA2B;YAC9B,GAAIX,QAAQ,aAAa,IAAI,CAAC,CAAC;QACjC;QAEA,OAAO,MAAML,YAAY,uBAAuB,CAC9CU,OAAO,IAAI,EACXkC;IAEJ;IACK;QACL,MAAMxC,SAASuB,MAAM,MAAM;QAE3B,IAAI1B,AAAe,eAAfA,YAA2B;YAC7B,MAAM,EAAE,eAAe4C,cAAc,EAAE,GAAGC,6BAA6B,GACrEzC;YACF,MAAM,EAAE0C,IAAI,EAAEC,OAAO,EAAE,GACpB,MAAMhD,aAAa,WAAWI,UAAU,IAAIe,QAAW;gBACtD,iBAAiB;gBACjB,GAAG2B,2BAA2B;YAChC,MAAO,CAAC;YACV,OAAO;gBAAE,MAAMC,QAAQ;gBAAO,SAASC,WAAW;YAAG;QACvD;QAEA,MAAMC,iBAAiBlD,kBAAkBC,aAAaC;QACtD,IAAIgD,gBAAgB;YAClB,MAAM,EAAEC,aAAa,EAAE,GAAGC,cAAc,GAAG9C;YAC3C,MAAM+C,kBAAkBF,gBACpB;gBACE,GAAGC,YAAY;gBACf,wBAAwBD;YAC1B,IACAC;YACJ,OAAO,MAAMF,eAAe7C,QAAQgD;QACtC;QAEA,MAAM,IAAIhB,MAAM,CAAC,qBAAqB,EAAEnC,YAAY;IACtD;AACF"}
+{"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type { DeviceAction } from '@midscene/core';\nimport { findAllMidsceneLocatorField } from '@midscene/core/ai-model';\nimport { buildDetailedLocateParam } from '@midscene/core/yaml';\nimport type {\n  ExecutionOptions,\n  FormValue,\n  PlaygroundAgent,\n  ValidationResult,\n} from './types';\n\n// APIs that return extracted data from the current interface.\nexport const dataExtractionAPIs = [\n  'aiQuery',\n  'aiBoolean',\n  'aiNumber',\n  'aiString',\n  'aiAsk',\n];\n\nexport const validationAPIs = ['aiAssert', 'aiWaitFor'];\n\n// APIs whose executions should not be rendered as replays by StandardPlayground.\nexport const noReplayAPIs = [...dataExtractionAPIs, ...validationAPIs];\n\nconst agentPromptAPIs = [\n  'aiAct',\n  'runMarkdown',\n  'aiQuery',\n  'aiBoolean',\n  'aiNumber',\n  'aiString',\n  'aiAsk',\n  'aiWaitFor',\n] as const;\n\ntype AgentPromptAPI = (typeof agentPromptAPIs)[number];\n\nconst agentPromptAPISet = new Set<string>(agentPromptAPIs);\n\nfunction getAgentPromptAPI(\n  activeAgent: PlaygroundAgent,\n  actionType: string,\n):\n  | ((\n      prompt: string | undefined,\n      options: ExecutionOptions,\n    ) => Promise<unknown>)\n  | undefined {\n  if (!agentPromptAPISet.has(actionType)) {\n    return undefined;\n  }\n\n  const methodName = actionType as AgentPromptAPI;\n  const method = activeAgent[methodName] as unknown;\n  if (typeof method !== 'function') {\n    return undefined;\n  }\n\n  return async (prompt, options) =>\n    (\n      method as (prompt: string, options: ExecutionOptions) => Promise<unknown>\n    ).call(activeAgent, prompt || '', options);\n}\n\nexport const formatErrorMessage = (e: any): string => {\n  const errorMessage = e?.message || '';\n\n  if (errorMessage.includes('of different extension')) {\n    return 'Conflicting extension detected. Please disable the suspicious plugins and refresh the page. Guide: https://midscenejs.com/quick-start.html#chrome-extension-faq';\n  }\n\n  if (errorMessage.includes('NOT_IMPLEMENTED_AS_DESIGNED')) {\n    return 'Further actions cannot be performed in the current environment';\n  }\n\n  return errorMessage || 'Unknown error';\n};\n\n// Parse structured parameters for callActionInActionSpace\nexport async function parseStructuredParams(\n  action: DeviceAction<unknown>,\n  params: Record<string, unknown>,\n  options: ExecutionOptions = {},\n): Promise<unknown[]> {\n  if (!action?.paramSchema || !('shape' in action.paramSchema)) {\n    return [params.prompt || '', options];\n  }\n\n  const schema = action.paramSchema;\n  const keys =\n    schema && 'shape' in schema\n      ? Object.keys((schema as { shape: Record<string, unknown> }).shape)\n      : [];\n\n  // Start with options and merge deviceOptions into the same level\n  // Destructure to exclude deviceOptions from the final object\n  const { deviceOptions: _, ...optionsWithoutDeviceOptions } = options;\n  const paramObj: Record<string, unknown> = {\n    ...optionsWithoutDeviceOptions,\n    ...(options.deviceOptions || {}),\n  };\n\n  keys.forEach((key) => {\n    if (\n      params[key] !== undefined &&\n      params[key] !== null &&\n      params[key] !== ''\n    ) {\n      paramObj[key] = params[key];\n    }\n  });\n\n  // Check if there's a locate field that needs detailed locate param processing\n  if (schema) {\n    const locatorFieldKeys = findAllMidsceneLocatorField(schema);\n    locatorFieldKeys.forEach((locateKey: string) => {\n      const locatePrompt = params[locateKey];\n      if (locatePrompt && typeof locatePrompt === 'string') {\n        // Build detailed locate param using the locate prompt and options\n        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, {\n          deepLocate: options.deepLocate,\n          cacheable: true, // Default to true for playground\n        });\n        if (detailedLocateParam) {\n          paramObj[locateKey] = detailedLocateParam;\n        }\n      }\n    });\n  }\n\n  return [paramObj];\n}\n\nexport function validateStructuredParams(\n  value: FormValue,\n  action: DeviceAction<unknown> | undefined,\n): ValidationResult {\n  if (!value.params) {\n    return { valid: false, errorMessage: 'Parameters are required' };\n  }\n\n  if (!action?.paramSchema) {\n    return { valid: true };\n  }\n\n  try {\n    const paramsForValidation = { ...value.params };\n\n    const schema = action.paramSchema;\n    if (schema) {\n      const locatorFieldKeys = findAllMidsceneLocatorField(schema);\n      locatorFieldKeys.forEach((key: string) => {\n        if (typeof paramsForValidation[key] === 'string') {\n          paramsForValidation[key] = {\n            prompt: paramsForValidation[key],\n            center: [0, 0],\n            rect: { left: 0, top: 0, width: 0, height: 0 },\n          };\n        }\n      });\n    }\n\n    action.paramSchema?.parse(paramsForValidation);\n    return { valid: true };\n  } catch (error: unknown) {\n    const zodError = error as {\n      errors?: Array<{ path: string[]; message: string }>;\n    };\n    if (zodError.errors && zodError.errors.length > 0) {\n      const errorMessages = zodError.errors\n        .filter((err) => {\n          const path = err.path.join('.');\n          return !path.includes('center') && !path.includes('rect');\n        })\n        .map((err) => {\n          const field = err.path.join('.');\n          return `${field}: ${err.message}`;\n        });\n\n      if (errorMessages.length > 0) {\n        return {\n          valid: false,\n          errorMessage: `Validation error: ${errorMessages.join(', ')}`,\n        };\n      }\n    } else {\n      const errorMsg =\n        error instanceof Error ? error.message : 'Unknown validation error';\n      return {\n        valid: false,\n        errorMessage: `Parameter validation failed: ${errorMsg}`,\n      };\n    }\n  }\n\n  return { valid: true };\n}\n\nexport async function executeAction(\n  activeAgent: PlaygroundAgent,\n  actionType: string,\n  actionSpace: DeviceAction<unknown>[],\n  value: FormValue,\n  options: ExecutionOptions,\n): Promise<unknown> {\n  const canForwardDeepThink =\n    actionType === 'aiAct' || actionType === 'runMarkdown';\n  if (!canForwardDeepThink && options.deepThink !== undefined) {\n    console.warn(\n      '[Playground] Received deepThink in non-aiAct action options. deepThink is expected to be used with aiAct/runMarkdown during migration.',\n      {\n        actionType,\n        requestId: options.requestId,\n        options,\n      },\n    );\n  }\n\n  const action = actionSpace?.find(\n    (a: DeviceAction<unknown>) =>\n      a.interfaceAlias === actionType || a.name === actionType,\n  );\n\n  if (action && typeof activeAgent.callActionInActionSpace === 'function') {\n    if (value.params) {\n      const parsedParams = await parseStructuredParams(\n        action,\n        value.params,\n        options,\n      );\n      return await activeAgent.callActionInActionSpace(\n        action.name,\n        parsedParams[0],\n      );\n    } else {\n      // For prompt-based actions, we need to build the detailed locate param\n      const detailedLocateParam = value.prompt\n        ? buildDetailedLocateParam(value.prompt, {\n            deepLocate: options.deepLocate,\n            cacheable: true,\n          })\n        : undefined;\n\n      // Flatten deviceOptions into the params\n      // Destructure to exclude deviceOptions from the final object\n      const {\n        deviceOptions: _,\n        reportDisplay: __,\n        ...optionsWithoutDeviceOptions\n      } = options;\n      const actionParams = {\n        locate: detailedLocateParam,\n        ...optionsWithoutDeviceOptions,\n        ...(options.deviceOptions || {}),\n      };\n\n      return await activeAgent.callActionInActionSpace(\n        action.name,\n        actionParams,\n      );\n    }\n  } else {\n    const prompt = value.prompt;\n\n    if (actionType === 'aiAssert') {\n      const { reportDisplay: _reportDisplay, ...optionsWithoutReportDisplay } =\n        options;\n      const { pass, thought } =\n        (await activeAgent?.aiAssert?.(prompt || '', undefined, {\n          keepRawResponse: true,\n          ...optionsWithoutReportDisplay,\n        })) || {};\n      return { pass: pass || false, thought: thought || '' };\n    }\n\n    const agentPromptAPI = getAgentPromptAPI(activeAgent, actionType);\n    if (agentPromptAPI) {\n      const { reportDisplay, ...agentOptions } = options;\n      const optionsForAgent = reportDisplay\n        ? {\n            ...agentOptions,\n            _internalReportDisplay: reportDisplay,\n          }\n        : agentOptions;\n      return await agentPromptAPI(prompt, optionsForAgent);\n    }\n\n    throw new Error(`Unknown action type: ${actionType}`);\n  }\n}\n"],"names":["dataExtractionAPIs","validationAPIs","noReplayAPIs","agentPromptAPIs","agentPromptAPISet","Set","getAgentPromptAPI","activeAgent","actionType","methodName","method","prompt","options","formatErrorMessage","e","errorMessage","parseStructuredParams","action","params","schema","keys","Object","_","optionsWithoutDeviceOptions","paramObj","key","undefined","locatorFieldKeys","findAllMidsceneLocatorField","locateKey","locatePrompt","detailedLocateParam","buildDetailedLocateParam","validateStructuredParams","value","paramsForValidation","error","zodError","errorMessages","err","path","field","errorMsg","Error","executeAction","actionSpace","canForwardDeepThink","console","a","parsedParams","__","actionParams","_reportDisplay","optionsWithoutReportDisplay","pass","thought","agentPromptAPI","reportDisplay","agentOptions","optionsForAgent"],"mappings":";;AAWO,MAAMA,qBAAqB;IAChC;IACA;IACA;IACA;IACA;CACD;AAEM,MAAMC,iBAAiB;IAAC;IAAY;CAAY;AAGhD,MAAMC,eAAe;OAAIF;OAAuBC;CAAe;AAEtE,MAAME,kBAAkB;IACtB;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAID,MAAMC,oBAAoB,IAAIC,IAAYF;AAE1C,SAASG,kBACPC,WAA4B,EAC5BC,UAAkB;IAOlB,IAAI,CAACJ,kBAAkB,GAAG,CAACI,aACzB;IAGF,MAAMC,aAAaD;IACnB,MAAME,SAASH,WAAW,CAACE,WAAW;IACtC,IAAI,AAAkB,cAAlB,OAAOC,QACT;IAGF,OAAO,OAAOC,QAAQC,UAElBF,OACA,IAAI,CAACH,aAAaI,UAAU,IAAIC;AACtC;AAEO,MAAMC,qBAAqB,CAACC;IACjC,MAAMC,eAAeD,GAAG,WAAW;IAEnC,IAAIC,aAAa,QAAQ,CAAC,2BACxB,OAAO;IAGT,IAAIA,aAAa,QAAQ,CAAC,gCACxB,OAAO;IAGT,OAAOA,gBAAgB;AACzB;AAGO,eAAeC,sBACpBC,MAA6B,EAC7BC,MAA+B,EAC/BN,UAA4B,CAAC,CAAC;IAE9B,IAAI,CAACK,QAAQ,eAAe,CAAE,YAAWA,OAAO,WAAU,GACxD,OAAO;QAACC,OAAO,MAAM,IAAI;QAAIN;KAAQ;IAGvC,MAAMO,SAASF,OAAO,WAAW;IACjC,MAAMG,OACJD,UAAU,WAAWA,SACjBE,OAAO,IAAI,CAAEF,OAA8C,KAAK,IAChE,EAAE;IAIR,MAAM,EAAE,eAAeG,CAAC,EAAE,GAAGC,6BAA6B,GAAGX;IAC7D,MAAMY,WAAoC;QACxC,GAAGD,2BAA2B;QAC9B,GAAIX,QAAQ,aAAa,IAAI,CAAC,CAAC;IACjC;IAEAQ,KAAK,OAAO,CAAC,CAACK;QACZ,IACEP,AAAgBQ,WAAhBR,MAAM,CAACO,IAAI,IACXP,AAAgB,SAAhBA,MAAM,CAACO,IAAI,IACXP,AAAgB,OAAhBA,MAAM,CAACO,IAAI,EAEXD,QAAQ,CAACC,IAAI,GAAGP,MAAM,CAACO,IAAI;IAE/B;IAGA,IAAIN,QAAQ;QACV,MAAMQ,mBAAmBC,4BAA4BT;QACrDQ,iBAAiB,OAAO,CAAC,CAACE;YACxB,MAAMC,eAAeZ,MAAM,CAACW,UAAU;YACtC,IAAIC,gBAAgB,AAAwB,YAAxB,OAAOA,cAA2B;gBAEpD,MAAMC,sBAAsBC,yBAAyBF,cAAc;oBACjE,YAAYlB,QAAQ,UAAU;oBAC9B,WAAW;gBACb;gBACA,IAAImB,qBACFP,QAAQ,CAACK,UAAU,GAAGE;YAE1B;QACF;IACF;IAEA,OAAO;QAACP;KAAS;AACnB;AAEO,SAASS,yBACdC,KAAgB,EAChBjB,MAAyC;IAEzC,IAAI,CAACiB,MAAM,MAAM,EACf,OAAO;QAAE,OAAO;QAAO,cAAc;IAA0B;IAGjE,IAAI,CAACjB,QAAQ,aACX,OAAO;QAAE,OAAO;IAAK;IAGvB,IAAI;QACF,MAAMkB,sBAAsB;YAAE,GAAGD,MAAM,MAAM;QAAC;QAE9C,MAAMf,SAASF,OAAO,WAAW;QACjC,IAAIE,QAAQ;YACV,MAAMQ,mBAAmBC,4BAA4BT;YACrDQ,iBAAiB,OAAO,CAAC,CAACF;gBACxB,IAAI,AAAoC,YAApC,OAAOU,mBAAmB,CAACV,IAAI,EACjCU,mBAAmB,CAACV,IAAI,GAAG;oBACzB,QAAQU,mBAAmB,CAACV,IAAI;oBAChC,QAAQ;wBAAC;wBAAG;qBAAE;oBACd,MAAM;wBAAE,MAAM;wBAAG,KAAK;wBAAG,OAAO;wBAAG,QAAQ;oBAAE;gBAC/C;YAEJ;QACF;QAEAR,OAAO,WAAW,EAAE,MAAMkB;IAE5B,EAAE,OAAOC,OAAgB;QACvB,MAAMC,WAAWD;QAGjB,IAAIC,SAAS,MAAM,IAAIA,SAAS,MAAM,CAAC,MAAM,GAAG,GAAG;YACjD,MAAMC,gBAAgBD,SAAS,MAAM,CAClC,MAAM,CAAC,CAACE;gBACP,MAAMC,OAAOD,IAAI,IAAI,CAAC,IAAI,CAAC;gBAC3B,OAAO,CAACC,KAAK,QAAQ,CAAC,aAAa,CAACA,KAAK,QAAQ,CAAC;YACpD,GACC,GAAG,CAAC,CAACD;gBACJ,MAAME,QAAQF,IAAI,IAAI,CAAC,IAAI,CAAC;gBAC5B,OAAO,GAAGE,MAAM,EAAE,EAAEF,IAAI,OAAO,EAAE;YACnC;YAEF,IAAID,cAAc,MAAM,GAAG,GACzB,OAAO;gBACL,OAAO;gBACP,cAAc,CAAC,kBAAkB,EAAEA,cAAc,IAAI,CAAC,OAAO;YAC/D;QAEJ,OAAO;YACL,MAAMI,WACJN,iBAAiBO,QAAQP,MAAM,OAAO,GAAG;YAC3C,OAAO;gBACL,OAAO;gBACP,cAAc,CAAC,6BAA6B,EAAEM,UAAU;YAC1D;QACF;IACF;IAEA,OAAO;QAAE,OAAO;IAAK;AACvB;AAEO,eAAeE,cACpBrC,WAA4B,EAC5BC,UAAkB,EAClBqC,WAAoC,EACpCX,KAAgB,EAChBtB,OAAyB;IAEzB,MAAMkC,sBACJtC,AAAe,YAAfA,cAA0BA,AAAe,kBAAfA;IAC5B,IAAI,CAACsC,uBAAuBlC,AAAsBc,WAAtBd,QAAQ,SAAS,EAC3CmC,QAAQ,IAAI,CACV,0IACA;QACEvC;QACA,WAAWI,QAAQ,SAAS;QAC5BA;IACF;IAIJ,MAAMK,SAAS4B,aAAa,KAC1B,CAACG,IACCA,EAAE,cAAc,KAAKxC,cAAcwC,EAAE,IAAI,KAAKxC;IAGlD,IAAIS,UAAU,AAA+C,cAA/C,OAAOV,YAAY,uBAAuB,EACtD,IAAI2B,MAAM,MAAM,EAAE;QAChB,MAAMe,eAAe,MAAMjC,sBACzBC,QACAiB,MAAM,MAAM,EACZtB;QAEF,OAAO,MAAML,YAAY,uBAAuB,CAC9CU,OAAO,IAAI,EACXgC,YAAY,CAAC,EAAE;IAEnB,OAAO;QAEL,MAAMlB,sBAAsBG,MAAM,MAAM,GACpCF,yBAAyBE,MAAM,MAAM,EAAE;YACrC,YAAYtB,QAAQ,UAAU;YAC9B,WAAW;QACb,KACAc;QAIJ,MAAM,EACJ,eAAeJ,CAAC,EAChB,eAAe4B,EAAE,EACjB,GAAG3B,6BACJ,GAAGX;QACJ,MAAMuC,eAAe;YACnB,QAAQpB;YACR,GAAGR,2BAA2B;YAC9B,GAAIX,QAAQ,aAAa,IAAI,CAAC,CAAC;QACjC;QAEA,OAAO,MAAML,YAAY,uBAAuB,CAC9CU,OAAO,IAAI,EACXkC;IAEJ;IACK;QACL,MAAMxC,SAASuB,MAAM,MAAM;QAE3B,IAAI1B,AAAe,eAAfA,YAA2B;YAC7B,MAAM,EAAE,eAAe4C,cAAc,EAAE,GAAGC,6BAA6B,GACrEzC;YACF,MAAM,EAAE0C,IAAI,EAAEC,OAAO,EAAE,GACpB,MAAMhD,aAAa,WAAWI,UAAU,IAAIe,QAAW;gBACtD,iBAAiB;gBACjB,GAAG2B,2BAA2B;YAChC,MAAO,CAAC;YACV,OAAO;gBAAE,MAAMC,QAAQ;gBAAO,SAASC,WAAW;YAAG;QACvD;QAEA,MAAMC,iBAAiBlD,kBAAkBC,aAAaC;QACtD,IAAIgD,gBAAgB;YAClB,MAAM,EAAEC,aAAa,EAAE,GAAGC,cAAc,GAAG9C;YAC3C,MAAM+C,kBAAkBF,gBACpB;gBACE,GAAGC,YAAY;gBACf,wBAAwBD;YAC1B,IACAC;YACJ,OAAO,MAAMF,eAAe7C,QAAQgD;QACtC;QAEA,MAAM,IAAIhB,MAAM,CAAC,qBAAqB,EAAEnC,YAAY;IACtD;AACF"}

package/dist/es/pointer-dispatch.mjs

@@ -139,12 +139,6 @@ async function dispatchPointer(input, body, getScreenSize) {
                             x,
                             y
                         ],
-                        rect: {
-                            left: x,
-                            top: y,
-                            width: 1,
-                            height: 1
-                        },
                         description: 'manual scroll target'
                     } : void 0
                 });
@@ -164,12 +158,6 @@ async function dispatchPointer(input, body, getScreenSize) {
                             center.x,
                             center.y
                         ],
-                        rect: {
-                            left: center.x,
-                            top: center.y,
-                            width: 1,
-                            height: 1
-                        },
                         description: 'manual pinch target'
                     },
                     direction,

package/dist/es/pointer-dispatch.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"pointer-dispatch.mjs","sources":["../../src/pointer-dispatch.ts"],"sourcesContent":["import type {\n  InputPrimitives,\n  KeyboardInputPrimitives,\n  PointerInputPrimitives,\n  PointerPoint,\n  TouchInputPrimitives,\n} from '@midscene/core/device';\nimport { normalizePinchParam } from '@midscene/core/device';\n\n/**\n * Thrown when an /interact request is malformed (missing field, wrong type)\n * or names an action / capability that the connected device does not support.\n * The route handler maps {400, 404} onto the HTTP response.\n */\nexport class PointerInputError extends Error {\n  constructor(\n    message: string,\n    public statusCode: 400 | 404,\n  ) {\n    super(message);\n    this.name = 'PointerInputError';\n  }\n}\n\nfunction requireNumber(value: unknown, field: string): number {\n  if (typeof value !== 'number' || Number.isNaN(value)) {\n    throw new PointerInputError(`${field} must be a number`, 400);\n  }\n  return value;\n}\n\nfunction optionalNumber(value: unknown, field: string): number | undefined {\n  if (value === undefined) return undefined;\n  return requireNumber(value, field);\n}\n\nfunction requireString(value: unknown, field: string): string {\n  if (typeof value !== 'string') {\n    throw new PointerInputError(`${field} must be a string`, 400);\n  }\n  return value;\n}\n\nfunction requirePoint(\n  body: Record<string, unknown>,\n  xField: 'x' | 'endX' = 'x',\n  yField: 'y' | 'endY' = 'y',\n): PointerPoint {\n  return {\n    x: requireNumber(body[xField], xField),\n    y: requireNumber(body[yField], yField),\n  };\n}\n\nfunction ensureCapability<T>(\n  fn: T | undefined,\n  actionType: string,\n): NonNullable<T> {\n  if (typeof fn !== 'function') {\n    throw new PointerInputError(\n      `${actionType} is not supported on this device`,\n      404,\n    );\n  }\n  return fn as NonNullable<T>;\n}\n\nfunction getPointerInput(input: InputPrimitives): PointerInputPrimitives {\n  if (!input.pointer) {\n    throw new PointerInputError(\n      'Pointer input is not supported on this device',\n      404,\n    );\n  }\n  return input.pointer;\n}\n\nfunction getKeyboardInput(input: InputPrimitives): KeyboardInputPrimitives {\n  if (!input.keyboard) {\n    throw new PointerInputError(\n      'Keyboard input is not supported on this device',\n      404,\n    );\n  }\n  return input.keyboard;\n}\n\nfunction getTouchInput(input: InputPrimitives): TouchInputPrimitives {\n  if (!input.touch) {\n    throw new PointerInputError(\n      'Touch input is not supported on this device',\n      404,\n    );\n  }\n  return input.touch;\n}\n\nfunction getScrollInput(input: InputPrimitives) {\n  if (!input.scroll) {\n    throw new PointerInputError(\n      'Scroll input is not supported on this device',\n      404,\n    );\n  }\n  return input.scroll;\n}\n\n/**\n * Translate an `/interact` request body into device input primitive calls.\n *\n * The dispatcher is deliberately a flat switch: each case is HTTP-protocol\n * adaptation (parse + range-check fields, hand them to the typed primitive),\n * not platform business logic.\n */\nexport async function dispatchPointer(\n  input: InputPrimitives,\n  body: Record<string, unknown>,\n  getScreenSize: () => Promise<{ width: number; height: number }>,\n): Promise<void> {\n  const { actionType } = body;\n  if (typeof actionType !== 'string' || !actionType) {\n    throw new PointerInputError('actionType is required', 400);\n  }\n\n  switch (actionType) {\n    case 'Tap': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.tap, 'Tap')(requirePoint(body), {\n        duration: optionalNumber(body.duration, 'duration'),\n      });\n    }\n\n    case 'DoubleClick': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(\n        pointer.doubleClick,\n        'DoubleClick',\n      )(requirePoint(body));\n    }\n\n    case 'LongPress': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.longPress, 'LongPress')(\n        requirePoint(body),\n        {\n          duration: optionalNumber(body.duration, 'duration'),\n        },\n      );\n    }\n\n    case 'Swipe': {\n      const touch = getTouchInput(input);\n      return ensureCapability(touch.swipe, 'Swipe')(\n        requirePoint(body),\n        requirePoint(body, 'endX', 'endY'),\n        {\n          duration: optionalNumber(body.duration, 'duration'),\n          repeat: optionalNumber(body.repeat, 'repeat'),\n        },\n      );\n    }\n\n    case 'DragAndDrop': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.dragAndDrop, 'DragAndDrop')(\n        requirePoint(body),\n        requirePoint(body, 'endX', 'endY'),\n      );\n    }\n\n    case 'KeyboardPress': {\n      const keyboard = getKeyboardInput(input);\n      return ensureCapability(\n        keyboard.keyboardPress,\n        'KeyboardPress',\n      )(requireString(body.keyName, 'keyName'));\n    }\n\n    case 'Input': {\n      const keyboard = getKeyboardInput(input);\n      const value = requireString(body.value, 'value');\n      const at =\n        typeof body.x === 'number' && typeof body.y === 'number'\n          ? requirePoint(body)\n          : undefined;\n      const mode =\n        typeof body.mode === 'string'\n          ? (body.mode as 'replace' | 'clear' | 'typeOnly')\n          : undefined;\n      const autoDismissKeyboard =\n        typeof body.autoDismissKeyboard === 'boolean'\n          ? body.autoDismissKeyboard\n          : undefined;\n      const target = at\n        ? {\n            center: [at.x, at.y] as [number, number],\n            rect: { left: at.x, top: at.y, width: 1, height: 1 },\n            description: 'manual input target',\n          }\n        : undefined;\n      if (mode === 'clear') {\n        await ensureCapability(keyboard.clearInput, 'ClearInput')(target);\n        return;\n      }\n      if (!value) return;\n      return ensureCapability(keyboard.typeText, 'Input')(value, {\n        autoDismissKeyboard,\n        target,\n        replace: mode !== 'typeOnly',\n      });\n    }\n\n    case 'Scroll': {\n      const scroll = getScrollInput(input);\n      const x =\n        typeof body.x === 'number' ? requireNumber(body.x, 'x') : undefined;\n      const y =\n        typeof body.y === 'number' ? requireNumber(body.y, 'y') : undefined;\n      const direction =\n        body.direction === 'up' ||\n        body.direction === 'down' ||\n        body.direction === 'left' ||\n        body.direction === 'right'\n          ? body.direction\n          : 'down';\n      const scrollType =\n        body.scrollType === 'scrollToBottom' ||\n        body.scrollType === 'scrollToTop' ||\n        body.scrollType === 'scrollToLeft' ||\n        body.scrollType === 'scrollToRight' ||\n        body.scrollType === 'singleAction'\n          ? body.scrollType\n          : 'singleAction';\n      const distance =\n        typeof body.distance === 'number'\n          ? requireNumber(body.distance, 'distance')\n          : undefined;\n      return ensureCapability(\n        scroll.scroll,\n        'Scroll',\n      )({\n        direction,\n        scrollType,\n        distance,\n        locate:\n          x !== undefined && y !== undefined\n            ? {\n                center: [x, y],\n                rect: { left: x, top: y, width: 1, height: 1 },\n                description: 'manual scroll target',\n              }\n            : undefined,\n      });\n    }\n\n    case 'Pinch': {\n      const center = requirePoint(body);\n      const direction = ((): 'in' | 'out' => {\n        const d = body.direction;\n        if (d !== 'in' && d !== 'out') {\n          throw new PointerInputError('direction must be \"in\" or \"out\"', 400);\n        }\n        return d;\n      })();\n      const touch = getTouchInput(input);\n      const { startDistance, endDistance, duration } = normalizePinchParam(\n        {\n          locate: {\n            center: [center.x, center.y],\n            rect: { left: center.x, top: center.y, width: 1, height: 1 },\n            description: 'manual pinch target',\n          },\n          direction,\n          distance: optionalNumber(body.distance, 'distance'),\n          duration: optionalNumber(body.duration, 'duration'),\n        },\n        await getScreenSize(),\n      );\n      return ensureCapability(touch.pinch, 'Pinch')(center, {\n        startDistance,\n        endDistance,\n        duration,\n      });\n    }\n\n    default:\n      throw new PointerInputError(`Unknown actionType \"${actionType}\"`, 404);\n  }\n}\n"],"names":["PointerInputError","Error","message","statusCode","requireNumber","value","field","Number","optionalNumber","undefined","requireString","requirePoint","body","xField","yField","ensureCapability","fn","actionType","getPointerInput","input","getKeyboardInput","getTouchInput","getScrollInput","dispatchPointer","getScreenSize","pointer","touch","keyboard","at","mode","autoDismissKeyboard","target","scroll","x","y","direction","scrollType","distance","center","d","startDistance","endDistance","duration","normalizePinchParam"],"mappings":";;;;;;;;;;;AAcO,MAAMA,0BAA0BC;IACrC,YACEC,OAAe,EACRC,UAAqB,CAC5B;QACA,KAAK,CAACD,UAAAA,iBAAAA,IAAAA,EAAAA,cAAAA,KAAAA,IAAAA,IAAAA,CAFCC,UAAU,GAAVA;QAGP,IAAI,CAAC,IAAI,GAAG;IACd;AACF;AAEA,SAASC,cAAcC,KAAc,EAAEC,KAAa;IAClD,IAAI,AAAiB,YAAjB,OAAOD,SAAsBE,OAAO,KAAK,CAACF,QAC5C,MAAM,IAAIL,kBAAkB,GAAGM,MAAM,iBAAiB,CAAC,EAAE;IAE3D,OAAOD;AACT;AAEA,SAASG,eAAeH,KAAc,EAAEC,KAAa;IACnD,IAAID,AAAUI,WAAVJ,OAAqB;IACzB,OAAOD,cAAcC,OAAOC;AAC9B;AAEA,SAASI,cAAcL,KAAc,EAAEC,KAAa;IAClD,IAAI,AAAiB,YAAjB,OAAOD,OACT,MAAM,IAAIL,kBAAkB,GAAGM,MAAM,iBAAiB,CAAC,EAAE;IAE3D,OAAOD;AACT;AAEA,SAASM,aACPC,IAA6B,EAC7BC,SAAuB,GAAG,EAC1BC,SAAuB,GAAG;IAE1B,OAAO;QACL,GAAGV,cAAcQ,IAAI,CAACC,OAAO,EAAEA;QAC/B,GAAGT,cAAcQ,IAAI,CAACE,OAAO,EAAEA;IACjC;AACF;AAEA,SAASC,iBACPC,EAAiB,EACjBC,UAAkB;IAElB,IAAI,AAAc,cAAd,OAAOD,IACT,MAAM,IAAIhB,kBACR,GAAGiB,WAAW,gCAAgC,CAAC,EAC/C;IAGJ,OAAOD;AACT;AAEA,SAASE,gBAAgBC,KAAsB;IAC7C,IAAI,CAACA,MAAM,OAAO,EAChB,MAAM,IAAInB,kBACR,iDACA;IAGJ,OAAOmB,MAAM,OAAO;AACtB;AAEA,SAASC,iBAAiBD,KAAsB;IAC9C,IAAI,CAACA,MAAM,QAAQ,EACjB,MAAM,IAAInB,kBACR,kDACA;IAGJ,OAAOmB,MAAM,QAAQ;AACvB;AAEA,SAASE,cAAcF,KAAsB;IAC3C,IAAI,CAACA,MAAM,KAAK,EACd,MAAM,IAAInB,kBACR,+CACA;IAGJ,OAAOmB,MAAM,KAAK;AACpB;AAEA,SAASG,eAAeH,KAAsB;IAC5C,IAAI,CAACA,MAAM,MAAM,EACf,MAAM,IAAInB,kBACR,gDACA;IAGJ,OAAOmB,MAAM,MAAM;AACrB;AASO,eAAeI,gBACpBJ,KAAsB,EACtBP,IAA6B,EAC7BY,aAA+D;IAE/D,MAAM,EAAEP,UAAU,EAAE,GAAGL;IACvB,IAAI,AAAsB,YAAtB,OAAOK,cAA2B,CAACA,YACrC,MAAM,IAAIjB,kBAAkB,0BAA0B;IAGxD,OAAQiB;QACN,KAAK;YAAO;gBACV,MAAMQ,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,GAAG,EAAE,OAAOd,aAAaC,OAAO;oBAC9D,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C;YACF;QAEA,KAAK;YAAe;gBAClB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBACLU,QAAQ,WAAW,EACnB,eACAd,aAAaC;YACjB;QAEA,KAAK;YAAa;gBAChB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,SAAS,EAAE,aACzCd,aAAaC,OACb;oBACE,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C;YAEJ;QAEA,KAAK;YAAS;gBACZ,MAAMc,QAAQL,cAAcF;gBAC5B,OAAOJ,iBAAiBW,MAAM,KAAK,EAAE,SACnCf,aAAaC,OACbD,aAAaC,MAAM,QAAQ,SAC3B;oBACE,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;oBACxC,QAAQJ,eAAeI,KAAK,MAAM,EAAE;gBACtC;YAEJ;QAEA,KAAK;YAAe;gBAClB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,WAAW,EAAE,eAC3Cd,aAAaC,OACbD,aAAaC,MAAM,QAAQ;YAE/B;QAEA,KAAK;YAAiB;gBACpB,MAAMe,WAAWP,iBAAiBD;gBAClC,OAAOJ,iBACLY,SAAS,aAAa,EACtB,iBACAjB,cAAcE,KAAK,OAAO,EAAE;YAChC;QAEA,KAAK;YAAS;gBACZ,MAAMe,WAAWP,iBAAiBD;gBAClC,MAAMd,QAAQK,cAAcE,KAAK,KAAK,EAAE;gBACxC,MAAMgB,KACJ,AAAkB,YAAlB,OAAOhB,KAAK,CAAC,IAAiB,AAAkB,YAAlB,OAAOA,KAAK,CAAC,GACvCD,aAAaC,QACbH;gBACN,MAAMoB,OACJ,AAAqB,YAArB,OAAOjB,KAAK,IAAI,GACXA,KAAK,IAAI,GACVH;gBACN,MAAMqB,sBACJ,AAAoC,aAApC,OAAOlB,KAAK,mBAAmB,GAC3BA,KAAK,mBAAmB,GACxBH;gBACN,MAAMsB,SAASH,KACX;oBACE,QAAQ;wBAACA,GAAG,CAAC;wBAAEA,GAAG,CAAC;qBAAC;oBACpB,MAAM;wBAAE,MAAMA,GAAG,CAAC;wBAAE,KAAKA,GAAG,CAAC;wBAAE,OAAO;wBAAG,QAAQ;oBAAE;oBACnD,aAAa;gBACf,IACAnB;gBACJ,IAAIoB,AAAS,YAATA,MAAkB,YACpB,MAAMd,iBAAiBY,SAAS,UAAU,EAAE,cAAcI;gBAG5D,IAAI,CAAC1B,OAAO;gBACZ,OAAOU,iBAAiBY,SAAS,QAAQ,EAAE,SAAStB,OAAO;oBACzDyB;oBACAC;oBACA,SAASF,AAAS,eAATA;gBACX;YACF;QAEA,KAAK;YAAU;gBACb,MAAMG,SAASV,eAAeH;gBAC9B,MAAMc,IACJ,AAAkB,YAAlB,OAAOrB,KAAK,CAAC,GAAgBR,cAAcQ,KAAK,CAAC,EAAE,OAAOH;gBAC5D,MAAMyB,IACJ,AAAkB,YAAlB,OAAOtB,KAAK,CAAC,GAAgBR,cAAcQ,KAAK,CAAC,EAAE,OAAOH;gBAC5D,MAAM0B,YACJvB,AAAmB,SAAnBA,KAAK,SAAS,IACdA,AAAmB,WAAnBA,KAAK,SAAS,IACdA,AAAmB,WAAnBA,KAAK,SAAS,IACdA,AAAmB,YAAnBA,KAAK,SAAS,GACVA,KAAK,SAAS,GACd;gBACN,MAAMwB,aACJxB,AAAoB,qBAApBA,KAAK,UAAU,IACfA,AAAoB,kBAApBA,KAAK,UAAU,IACfA,AAAoB,mBAApBA,KAAK,UAAU,IACfA,AAAoB,oBAApBA,KAAK,UAAU,IACfA,AAAoB,mBAApBA,KAAK,UAAU,GACXA,KAAK,UAAU,GACf;gBACN,MAAMyB,WACJ,AAAyB,YAAzB,OAAOzB,KAAK,QAAQ,GAChBR,cAAcQ,KAAK,QAAQ,EAAE,cAC7BH;gBACN,OAAOM,iBACLiB,OAAO,MAAM,EACb,UACA;oBACAG;oBACAC;oBACAC;oBACA,QACEJ,AAAMxB,WAANwB,KAAmBC,AAAMzB,WAANyB,IACf;wBACE,QAAQ;4BAACD;4BAAGC;yBAAE;wBACd,MAAM;4BAAE,MAAMD;4BAAG,KAAKC;4BAAG,OAAO;4BAAG,QAAQ;wBAAE;wBAC7C,aAAa;oBACf,IACAzB;gBACR;YACF;QAEA,KAAK;YAAS;gBACZ,MAAM6B,SAAS3B,aAAaC;gBAC5B,MAAMuB,YAAa,AAAC;oBAClB,MAAMI,IAAI3B,KAAK,SAAS;oBACxB,IAAI2B,AAAM,SAANA,KAAcA,AAAM,UAANA,GAChB,MAAM,IAAIvC,kBAAkB,mCAAmC;oBAEjE,OAAOuC;gBACT;gBACA,MAAMb,QAAQL,cAAcF;gBAC5B,MAAM,EAAEqB,aAAa,EAAEC,WAAW,EAAEC,QAAQ,EAAE,GAAGC,oBAC/C;oBACE,QAAQ;wBACN,QAAQ;4BAACL,OAAO,CAAC;4BAAEA,OAAO,CAAC;yBAAC;wBAC5B,MAAM;4BAAE,MAAMA,OAAO,CAAC;4BAAE,KAAKA,OAAO,CAAC;4BAAE,OAAO;4BAAG,QAAQ;wBAAE;wBAC3D,aAAa;oBACf;oBACAH;oBACA,UAAU3B,eAAeI,KAAK,QAAQ,EAAE;oBACxC,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C,GACA,MAAMY;gBAER,OAAOT,iBAAiBW,MAAM,KAAK,EAAE,SAASY,QAAQ;oBACpDE;oBACAC;oBACAC;gBACF;YACF;QAEA;YACE,MAAM,IAAI1C,kBAAkB,CAAC,oBAAoB,EAAEiB,WAAW,CAAC,CAAC,EAAE;IACtE;AACF"}
+{"version":3,"file":"pointer-dispatch.mjs","sources":["../../src/pointer-dispatch.ts"],"sourcesContent":["import type {\n  InputPrimitives,\n  KeyboardInputPrimitives,\n  PointerInputPrimitives,\n  PointerPoint,\n  TouchInputPrimitives,\n} from '@midscene/core/device';\nimport { normalizePinchParam } from '@midscene/core/device';\n\n/**\n * Thrown when an /interact request is malformed (missing field, wrong type)\n * or names an action / capability that the connected device does not support.\n * The route handler maps {400, 404} onto the HTTP response.\n */\nexport class PointerInputError extends Error {\n  constructor(\n    message: string,\n    public statusCode: 400 | 404,\n  ) {\n    super(message);\n    this.name = 'PointerInputError';\n  }\n}\n\nfunction requireNumber(value: unknown, field: string): number {\n  if (typeof value !== 'number' || Number.isNaN(value)) {\n    throw new PointerInputError(`${field} must be a number`, 400);\n  }\n  return value;\n}\n\nfunction optionalNumber(value: unknown, field: string): number | undefined {\n  if (value === undefined) return undefined;\n  return requireNumber(value, field);\n}\n\nfunction requireString(value: unknown, field: string): string {\n  if (typeof value !== 'string') {\n    throw new PointerInputError(`${field} must be a string`, 400);\n  }\n  return value;\n}\n\nfunction requirePoint(\n  body: Record<string, unknown>,\n  xField: 'x' | 'endX' = 'x',\n  yField: 'y' | 'endY' = 'y',\n): PointerPoint {\n  return {\n    x: requireNumber(body[xField], xField),\n    y: requireNumber(body[yField], yField),\n  };\n}\n\nfunction ensureCapability<T>(\n  fn: T | undefined,\n  actionType: string,\n): NonNullable<T> {\n  if (typeof fn !== 'function') {\n    throw new PointerInputError(\n      `${actionType} is not supported on this device`,\n      404,\n    );\n  }\n  return fn as NonNullable<T>;\n}\n\nfunction getPointerInput(input: InputPrimitives): PointerInputPrimitives {\n  if (!input.pointer) {\n    throw new PointerInputError(\n      'Pointer input is not supported on this device',\n      404,\n    );\n  }\n  return input.pointer;\n}\n\nfunction getKeyboardInput(input: InputPrimitives): KeyboardInputPrimitives {\n  if (!input.keyboard) {\n    throw new PointerInputError(\n      'Keyboard input is not supported on this device',\n      404,\n    );\n  }\n  return input.keyboard;\n}\n\nfunction getTouchInput(input: InputPrimitives): TouchInputPrimitives {\n  if (!input.touch) {\n    throw new PointerInputError(\n      'Touch input is not supported on this device',\n      404,\n    );\n  }\n  return input.touch;\n}\n\nfunction getScrollInput(input: InputPrimitives) {\n  if (!input.scroll) {\n    throw new PointerInputError(\n      'Scroll input is not supported on this device',\n      404,\n    );\n  }\n  return input.scroll;\n}\n\n/**\n * Translate an `/interact` request body into device input primitive calls.\n *\n * The dispatcher is deliberately a flat switch: each case is HTTP-protocol\n * adaptation (parse + range-check fields, hand them to the typed primitive),\n * not platform business logic.\n */\nexport async function dispatchPointer(\n  input: InputPrimitives,\n  body: Record<string, unknown>,\n  getScreenSize: () => Promise<{ width: number; height: number }>,\n): Promise<void> {\n  const { actionType } = body;\n  if (typeof actionType !== 'string' || !actionType) {\n    throw new PointerInputError('actionType is required', 400);\n  }\n\n  switch (actionType) {\n    case 'Tap': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.tap, 'Tap')(requirePoint(body), {\n        duration: optionalNumber(body.duration, 'duration'),\n      });\n    }\n\n    case 'DoubleClick': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(\n        pointer.doubleClick,\n        'DoubleClick',\n      )(requirePoint(body));\n    }\n\n    case 'LongPress': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.longPress, 'LongPress')(\n        requirePoint(body),\n        {\n          duration: optionalNumber(body.duration, 'duration'),\n        },\n      );\n    }\n\n    case 'Swipe': {\n      const touch = getTouchInput(input);\n      return ensureCapability(touch.swipe, 'Swipe')(\n        requirePoint(body),\n        requirePoint(body, 'endX', 'endY'),\n        {\n          duration: optionalNumber(body.duration, 'duration'),\n          repeat: optionalNumber(body.repeat, 'repeat'),\n        },\n      );\n    }\n\n    case 'DragAndDrop': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.dragAndDrop, 'DragAndDrop')(\n        requirePoint(body),\n        requirePoint(body, 'endX', 'endY'),\n      );\n    }\n\n    case 'KeyboardPress': {\n      const keyboard = getKeyboardInput(input);\n      return ensureCapability(\n        keyboard.keyboardPress,\n        'KeyboardPress',\n      )(requireString(body.keyName, 'keyName'));\n    }\n\n    case 'Input': {\n      const keyboard = getKeyboardInput(input);\n      const value = requireString(body.value, 'value');\n      const at =\n        typeof body.x === 'number' && typeof body.y === 'number'\n          ? requirePoint(body)\n          : undefined;\n      const mode =\n        typeof body.mode === 'string'\n          ? (body.mode as 'replace' | 'clear' | 'typeOnly')\n          : undefined;\n      const autoDismissKeyboard =\n        typeof body.autoDismissKeyboard === 'boolean'\n          ? body.autoDismissKeyboard\n          : undefined;\n      const target = at\n        ? {\n            center: [at.x, at.y] as [number, number],\n            rect: { left: at.x, top: at.y, width: 1, height: 1 },\n            description: 'manual input target',\n          }\n        : undefined;\n      if (mode === 'clear') {\n        await ensureCapability(keyboard.clearInput, 'ClearInput')(target);\n        return;\n      }\n      if (!value) return;\n      return ensureCapability(keyboard.typeText, 'Input')(value, {\n        autoDismissKeyboard,\n        target,\n        replace: mode !== 'typeOnly',\n      });\n    }\n\n    case 'Scroll': {\n      const scroll = getScrollInput(input);\n      const x =\n        typeof body.x === 'number' ? requireNumber(body.x, 'x') : undefined;\n      const y =\n        typeof body.y === 'number' ? requireNumber(body.y, 'y') : undefined;\n      const direction =\n        body.direction === 'up' ||\n        body.direction === 'down' ||\n        body.direction === 'left' ||\n        body.direction === 'right'\n          ? body.direction\n          : 'down';\n      const scrollType =\n        body.scrollType === 'scrollToBottom' ||\n        body.scrollType === 'scrollToTop' ||\n        body.scrollType === 'scrollToLeft' ||\n        body.scrollType === 'scrollToRight' ||\n        body.scrollType === 'singleAction'\n          ? body.scrollType\n          : 'singleAction';\n      const distance =\n        typeof body.distance === 'number'\n          ? requireNumber(body.distance, 'distance')\n          : undefined;\n      return ensureCapability(\n        scroll.scroll,\n        'Scroll',\n      )({\n        direction,\n        scrollType,\n        distance,\n        locate:\n          x !== undefined && y !== undefined\n            ? {\n                center: [x, y],\n\n                description: 'manual scroll target',\n              }\n            : undefined,\n      });\n    }\n\n    case 'Pinch': {\n      const center = requirePoint(body);\n      const direction = ((): 'in' | 'out' => {\n        const d = body.direction;\n        if (d !== 'in' && d !== 'out') {\n          throw new PointerInputError('direction must be \"in\" or \"out\"', 400);\n        }\n        return d;\n      })();\n      const touch = getTouchInput(input);\n      const { startDistance, endDistance, duration } = normalizePinchParam(\n        {\n          locate: {\n            center: [center.x, center.y],\n            description: 'manual pinch target',\n          },\n          direction,\n          distance: optionalNumber(body.distance, 'distance'),\n          duration: optionalNumber(body.duration, 'duration'),\n        },\n        await getScreenSize(),\n      );\n      return ensureCapability(touch.pinch, 'Pinch')(center, {\n        startDistance,\n        endDistance,\n        duration,\n      });\n    }\n\n    default:\n      throw new PointerInputError(`Unknown actionType \"${actionType}\"`, 404);\n  }\n}\n"],"names":["PointerInputError","Error","message","statusCode","requireNumber","value","field","Number","optionalNumber","undefined","requireString","requirePoint","body","xField","yField","ensureCapability","fn","actionType","getPointerInput","input","getKeyboardInput","getTouchInput","getScrollInput","dispatchPointer","getScreenSize","pointer","touch","keyboard","at","mode","autoDismissKeyboard","target","scroll","x","y","direction","scrollType","distance","center","d","startDistance","endDistance","duration","normalizePinchParam"],"mappings":";;;;;;;;;;;AAcO,MAAMA,0BAA0BC;IACrC,YACEC,OAAe,EACRC,UAAqB,CAC5B;QACA,KAAK,CAACD,UAAAA,iBAAAA,IAAAA,EAAAA,cAAAA,KAAAA,IAAAA,IAAAA,CAFCC,UAAU,GAAVA;QAGP,IAAI,CAAC,IAAI,GAAG;IACd;AACF;AAEA,SAASC,cAAcC,KAAc,EAAEC,KAAa;IAClD,IAAI,AAAiB,YAAjB,OAAOD,SAAsBE,OAAO,KAAK,CAACF,QAC5C,MAAM,IAAIL,kBAAkB,GAAGM,MAAM,iBAAiB,CAAC,EAAE;IAE3D,OAAOD;AACT;AAEA,SAASG,eAAeH,KAAc,EAAEC,KAAa;IACnD,IAAID,AAAUI,WAAVJ,OAAqB;IACzB,OAAOD,cAAcC,OAAOC;AAC9B;AAEA,SAASI,cAAcL,KAAc,EAAEC,KAAa;IAClD,IAAI,AAAiB,YAAjB,OAAOD,OACT,MAAM,IAAIL,kBAAkB,GAAGM,MAAM,iBAAiB,CAAC,EAAE;IAE3D,OAAOD;AACT;AAEA,SAASM,aACPC,IAA6B,EAC7BC,SAAuB,GAAG,EAC1BC,SAAuB,GAAG;IAE1B,OAAO;QACL,GAAGV,cAAcQ,IAAI,CAACC,OAAO,EAAEA;QAC/B,GAAGT,cAAcQ,IAAI,CAACE,OAAO,EAAEA;IACjC;AACF;AAEA,SAASC,iBACPC,EAAiB,EACjBC,UAAkB;IAElB,IAAI,AAAc,cAAd,OAAOD,IACT,MAAM,IAAIhB,kBACR,GAAGiB,WAAW,gCAAgC,CAAC,EAC/C;IAGJ,OAAOD;AACT;AAEA,SAASE,gBAAgBC,KAAsB;IAC7C,IAAI,CAACA,MAAM,OAAO,EAChB,MAAM,IAAInB,kBACR,iDACA;IAGJ,OAAOmB,MAAM,OAAO;AACtB;AAEA,SAASC,iBAAiBD,KAAsB;IAC9C,IAAI,CAACA,MAAM,QAAQ,EACjB,MAAM,IAAInB,kBACR,kDACA;IAGJ,OAAOmB,MAAM,QAAQ;AACvB;AAEA,SAASE,cAAcF,KAAsB;IAC3C,IAAI,CAACA,MAAM,KAAK,EACd,MAAM,IAAInB,kBACR,+CACA;IAGJ,OAAOmB,MAAM,KAAK;AACpB;AAEA,SAASG,eAAeH,KAAsB;IAC5C,IAAI,CAACA,MAAM,MAAM,EACf,MAAM,IAAInB,kBACR,gDACA;IAGJ,OAAOmB,MAAM,MAAM;AACrB;AASO,eAAeI,gBACpBJ,KAAsB,EACtBP,IAA6B,EAC7BY,aAA+D;IAE/D,MAAM,EAAEP,UAAU,EAAE,GAAGL;IACvB,IAAI,AAAsB,YAAtB,OAAOK,cAA2B,CAACA,YACrC,MAAM,IAAIjB,kBAAkB,0BAA0B;IAGxD,OAAQiB;QACN,KAAK;YAAO;gBACV,MAAMQ,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,GAAG,EAAE,OAAOd,aAAaC,OAAO;oBAC9D,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C;YACF;QAEA,KAAK;YAAe;gBAClB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBACLU,QAAQ,WAAW,EACnB,eACAd,aAAaC;YACjB;QAEA,KAAK;YAAa;gBAChB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,SAAS,EAAE,aACzCd,aAAaC,OACb;oBACE,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C;YAEJ;QAEA,KAAK;YAAS;gBACZ,MAAMc,QAAQL,cAAcF;gBAC5B,OAAOJ,iBAAiBW,MAAM,KAAK,EAAE,SACnCf,aAAaC,OACbD,aAAaC,MAAM,QAAQ,SAC3B;oBACE,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;oBACxC,QAAQJ,eAAeI,KAAK,MAAM,EAAE;gBACtC;YAEJ;QAEA,KAAK;YAAe;gBAClB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,WAAW,EAAE,eAC3Cd,aAAaC,OACbD,aAAaC,MAAM,QAAQ;YAE/B;QAEA,KAAK;YAAiB;gBACpB,MAAMe,WAAWP,iBAAiBD;gBAClC,OAAOJ,iBACLY,SAAS,aAAa,EACtB,iBACAjB,cAAcE,KAAK,OAAO,EAAE;YAChC;QAEA,KAAK;YAAS;gBACZ,MAAMe,WAAWP,iBAAiBD;gBAClC,MAAMd,QAAQK,cAAcE,KAAK,KAAK,EAAE;gBACxC,MAAMgB,KACJ,AAAkB,YAAlB,OAAOhB,KAAK,CAAC,IAAiB,AAAkB,YAAlB,OAAOA,KAAK,CAAC,GACvCD,aAAaC,QACbH;gBACN,MAAMoB,OACJ,AAAqB,YAArB,OAAOjB,KAAK,IAAI,GACXA,KAAK,IAAI,GACVH;gBACN,MAAMqB,sBACJ,AAAoC,aAApC,OAAOlB,KAAK,mBAAmB,GAC3BA,KAAK,mBAAmB,GACxBH;gBACN,MAAMsB,SAASH,KACX;oBACE,QAAQ;wBAACA,GAAG,CAAC;wBAAEA,GAAG,CAAC;qBAAC;oBACpB,MAAM;wBAAE,MAAMA,GAAG,CAAC;wBAAE,KAAKA,GAAG,CAAC;wBAAE,OAAO;wBAAG,QAAQ;oBAAE;oBACnD,aAAa;gBACf,IACAnB;gBACJ,IAAIoB,AAAS,YAATA,MAAkB,YACpB,MAAMd,iBAAiBY,SAAS,UAAU,EAAE,cAAcI;gBAG5D,IAAI,CAAC1B,OAAO;gBACZ,OAAOU,iBAAiBY,SAAS,QAAQ,EAAE,SAAStB,OAAO;oBACzDyB;oBACAC;oBACA,SAASF,AAAS,eAATA;gBACX;YACF;QAEA,KAAK;YAAU;gBACb,MAAMG,SAASV,eAAeH;gBAC9B,MAAMc,IACJ,AAAkB,YAAlB,OAAOrB,KAAK,CAAC,GAAgBR,cAAcQ,KAAK,CAAC,EAAE,OAAOH;gBAC5D,MAAMyB,IACJ,AAAkB,YAAlB,OAAOtB,KAAK,CAAC,GAAgBR,cAAcQ,KAAK,CAAC,EAAE,OAAOH;gBAC5D,MAAM0B,YACJvB,AAAmB,SAAnBA,KAAK,SAAS,IACdA,AAAmB,WAAnBA,KAAK,SAAS,IACdA,AAAmB,WAAnBA,KAAK,SAAS,IACdA,AAAmB,YAAnBA,KAAK,SAAS,GACVA,KAAK,SAAS,GACd;gBACN,MAAMwB,aACJxB,AAAoB,qBAApBA,KAAK,UAAU,IACfA,AAAoB,kBAApBA,KAAK,UAAU,IACfA,AAAoB,mBAApBA,KAAK,UAAU,IACfA,AAAoB,oBAApBA,KAAK,UAAU,IACfA,AAAoB,mBAApBA,KAAK,UAAU,GACXA,KAAK,UAAU,GACf;gBACN,MAAMyB,WACJ,AAAyB,YAAzB,OAAOzB,KAAK,QAAQ,GAChBR,cAAcQ,KAAK,QAAQ,EAAE,cAC7BH;gBACN,OAAOM,iBACLiB,OAAO,MAAM,EACb,UACA;oBACAG;oBACAC;oBACAC;oBACA,QACEJ,AAAMxB,WAANwB,KAAmBC,AAAMzB,WAANyB,IACf;wBACE,QAAQ;4BAACD;4BAAGC;yBAAE;wBAEd,aAAa;oBACf,IACAzB;gBACR;YACF;QAEA,KAAK;YAAS;gBACZ,MAAM6B,SAAS3B,aAAaC;gBAC5B,MAAMuB,YAAa,AAAC;oBAClB,MAAMI,IAAI3B,KAAK,SAAS;oBACxB,IAAI2B,AAAM,SAANA,KAAcA,AAAM,UAANA,GAChB,MAAM,IAAIvC,kBAAkB,mCAAmC;oBAEjE,OAAOuC;gBACT;gBACA,MAAMb,QAAQL,cAAcF;gBAC5B,MAAM,EAAEqB,aAAa,EAAEC,WAAW,EAAEC,QAAQ,EAAE,GAAGC,oBAC/C;oBACE,QAAQ;wBACN,QAAQ;4BAACL,OAAO,CAAC;4BAAEA,OAAO,CAAC;yBAAC;wBAC5B,aAAa;oBACf;oBACAH;oBACA,UAAU3B,eAAeI,KAAK,QAAQ,EAAE;oBACxC,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C,GACA,MAAMY;gBAER,OAAOT,iBAAiBW,MAAM,KAAK,EAAE,SAASY,QAAQ;oBACpDE;oBACAC;oBACAC;gBACF;YACF;QAEA;YACE,MAAM,IAAI1C,kBAAkB,CAAC,oBAAoB,EAAEiB,WAAW,CAAC,CAAC,EAAE;IACtE;AACF"}

package/dist/es/server.mjs

@@ -10,7 +10,6 @@ import { getTmpDir, sleep } from "@midscene/core/utils";
 import { getMidsceneRunSubDir } from "@midscene/shared/common";
 import { PLAYGROUND_SERVER_PORT } from "@midscene/shared/constants";
 import { ModelConfigManager, globalModelConfigManager, overrideAIConfig } from "@midscene/shared/env";
-import { generateElementByPoint } from "@midscene/shared/extractor";
 import { annotateRects, imageInfoOfBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { buildMidsceneRecorderActionSummary, buildMidsceneRecorderReplayInstruction } from "@midscene/shared/recorder";
@@ -558,10 +557,13 @@ function requireNumber(value, field) {
     return value;
 }
 function locateFromPoint(x, y, fieldX, fieldY, description) {
-    return generateElementByPoint([
-        Math.round(requireNumber(x, fieldX)),
-        Math.round(requireNumber(y, fieldY))
-    ], description);
+    return {
+        center: [
+            Math.round(requireNumber(x, fieldX)),
+            Math.round(requireNumber(y, fieldY))
+        ],
+        description: description || ''
+    };
 }
 const POINTER_INTERACT_ACTIONS = new Set([
     'Tap',