@midscene/playground 1.12.5-beta-20260908034641.0 → 1.12.5

package/dist/es/pointer-dispatch.mjs

@@ -139,12 +139,6 @@ async function dispatchPointer(input, body, getScreenSize) {
                             x,
                             y
                         ],
-                        rect: {
-                            left: x,
-                            top: y,
-                            width: 1,
-                            height: 1
-                        },
                         description: 'manual scroll target'
                     } : void 0
                 });
@@ -164,12 +158,6 @@ async function dispatchPointer(input, body, getScreenSize) {
                             center.x,
                             center.y
                         ],
-                        rect: {
-                            left: center.x,
-                            top: center.y,
-                            width: 1,
-                            height: 1
-                        },
                         description: 'manual pinch target'
                     },
                     direction,

package/dist/es/pointer-dispatch.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"pointer-dispatch.mjs","sources":["../../src/pointer-dispatch.ts"],"sourcesContent":["import type {\n  InputPrimitives,\n  KeyboardInputPrimitives,\n  PointerInputPrimitives,\n  PointerPoint,\n  TouchInputPrimitives,\n} from '@midscene/core/device';\nimport { normalizePinchParam } from '@midscene/core/device';\n\n/**\n * Thrown when an /interact request is malformed (missing field, wrong type)\n * or names an action / capability that the connected device does not support.\n * The route handler maps {400, 404} onto the HTTP response.\n */\nexport class PointerInputError extends Error {\n  constructor(\n    message: string,\n    public statusCode: 400 | 404,\n  ) {\n    super(message);\n    this.name = 'PointerInputError';\n  }\n}\n\nfunction requireNumber(value: unknown, field: string): number {\n  if (typeof value !== 'number' || Number.isNaN(value)) {\n    throw new PointerInputError(`${field} must be a number`, 400);\n  }\n  return value;\n}\n\nfunction optionalNumber(value: unknown, field: string): number | undefined {\n  if (value === undefined) return undefined;\n  return requireNumber(value, field);\n}\n\nfunction requireString(value: unknown, field: string): string {\n  if (typeof value !== 'string') {\n    throw new PointerInputError(`${field} must be a string`, 400);\n  }\n  return value;\n}\n\nfunction requirePoint(\n  body: Record<string, unknown>,\n  xField: 'x' | 'endX' = 'x',\n  yField: 'y' | 'endY' = 'y',\n): PointerPoint {\n  return {\n    x: requireNumber(body[xField], xField),\n    y: requireNumber(body[yField], yField),\n  };\n}\n\nfunction ensureCapability<T>(\n  fn: T | undefined,\n  actionType: string,\n): NonNullable<T> {\n  if (typeof fn !== 'function') {\n    throw new PointerInputError(\n      `${actionType} is not supported on this device`,\n      404,\n    );\n  }\n  return fn as NonNullable<T>;\n}\n\nfunction getPointerInput(input: InputPrimitives): PointerInputPrimitives {\n  if (!input.pointer) {\n    throw new PointerInputError(\n      'Pointer input is not supported on this device',\n      404,\n    );\n  }\n  return input.pointer;\n}\n\nfunction getKeyboardInput(input: InputPrimitives): KeyboardInputPrimitives {\n  if (!input.keyboard) {\n    throw new PointerInputError(\n      'Keyboard input is not supported on this device',\n      404,\n    );\n  }\n  return input.keyboard;\n}\n\nfunction getTouchInput(input: InputPrimitives): TouchInputPrimitives {\n  if (!input.touch) {\n    throw new PointerInputError(\n      'Touch input is not supported on this device',\n      404,\n    );\n  }\n  return input.touch;\n}\n\nfunction getScrollInput(input: InputPrimitives) {\n  if (!input.scroll) {\n    throw new PointerInputError(\n      'Scroll input is not supported on this device',\n      404,\n    );\n  }\n  return input.scroll;\n}\n\n/**\n * Translate an `/interact` request body into device input primitive calls.\n *\n * The dispatcher is deliberately a flat switch: each case is HTTP-protocol\n * adaptation (parse + range-check fields, hand them to the typed primitive),\n * not platform business logic.\n */\nexport async function dispatchPointer(\n  input: InputPrimitives,\n  body: Record<string, unknown>,\n  getScreenSize: () => Promise<{ width: number; height: number }>,\n): Promise<void> {\n  const { actionType } = body;\n  if (typeof actionType !== 'string' || !actionType) {\n    throw new PointerInputError('actionType is required', 400);\n  }\n\n  switch (actionType) {\n    case 'Tap': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.tap, 'Tap')(requirePoint(body), {\n        duration: optionalNumber(body.duration, 'duration'),\n      });\n    }\n\n    case 'DoubleClick': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(\n        pointer.doubleClick,\n        'DoubleClick',\n      )(requirePoint(body));\n    }\n\n    case 'LongPress': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.longPress, 'LongPress')(\n        requirePoint(body),\n        {\n          duration: optionalNumber(body.duration, 'duration'),\n        },\n      );\n    }\n\n    case 'Swipe': {\n      const touch = getTouchInput(input);\n      return ensureCapability(touch.swipe, 'Swipe')(\n        requirePoint(body),\n        requirePoint(body, 'endX', 'endY'),\n        {\n          duration: optionalNumber(body.duration, 'duration'),\n          repeat: optionalNumber(body.repeat, 'repeat'),\n        },\n      );\n    }\n\n    case 'DragAndDrop': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.dragAndDrop, 'DragAndDrop')(\n        requirePoint(body),\n        requirePoint(body, 'endX', 'endY'),\n      );\n    }\n\n    case 'KeyboardPress': {\n      const keyboard = getKeyboardInput(input);\n      return ensureCapability(\n        keyboard.keyboardPress,\n        'KeyboardPress',\n      )(requireString(body.keyName, 'keyName'));\n    }\n\n    case 'Input': {\n      const keyboard = getKeyboardInput(input);\n      const value = requireString(body.value, 'value');\n      const at =\n        typeof body.x === 'number' && typeof body.y === 'number'\n          ? requirePoint(body)\n          : undefined;\n      const mode =\n        typeof body.mode === 'string'\n          ? (body.mode as 'replace' | 'clear' | 'typeOnly')\n          : undefined;\n      const autoDismissKeyboard =\n        typeof body.autoDismissKeyboard === 'boolean'\n          ? body.autoDismissKeyboard\n          : undefined;\n      const target = at\n        ? {\n            center: [at.x, at.y] as [number, number],\n            rect: { left: at.x, top: at.y, width: 1, height: 1 },\n            description: 'manual input target',\n          }\n        : undefined;\n      if (mode === 'clear') {\n        await ensureCapability(keyboard.clearInput, 'ClearInput')(target);\n        return;\n      }\n      if (!value) return;\n      return ensureCapability(keyboard.typeText, 'Input')(value, {\n        autoDismissKeyboard,\n        target,\n        replace: mode !== 'typeOnly',\n      });\n    }\n\n    case 'Scroll': {\n      const scroll = getScrollInput(input);\n      const x =\n        typeof body.x === 'number' ? requireNumber(body.x, 'x') : undefined;\n      const y =\n        typeof body.y === 'number' ? requireNumber(body.y, 'y') : undefined;\n      const direction =\n        body.direction === 'up' ||\n        body.direction === 'down' ||\n        body.direction === 'left' ||\n        body.direction === 'right'\n          ? body.direction\n          : 'down';\n      const scrollType =\n        body.scrollType === 'scrollToBottom' ||\n        body.scrollType === 'scrollToTop' ||\n        body.scrollType === 'scrollToLeft' ||\n        body.scrollType === 'scrollToRight' ||\n        body.scrollType === 'singleAction'\n          ? body.scrollType\n          : 'singleAction';\n      const distance =\n        typeof body.distance === 'number'\n          ? requireNumber(body.distance, 'distance')\n          : undefined;\n      return ensureCapability(\n        scroll.scroll,\n        'Scroll',\n      )({\n        direction,\n        scrollType,\n        distance,\n        locate:\n          x !== undefined && y !== undefined\n            ? {\n                center: [x, y],\n                rect: { left: x, top: y, width: 1, height: 1 },\n                description: 'manual scroll target',\n              }\n            : undefined,\n      });\n    }\n\n    case 'Pinch': {\n      const center = requirePoint(body);\n      const direction = ((): 'in' | 'out' => {\n        const d = body.direction;\n        if (d !== 'in' && d !== 'out') {\n          throw new PointerInputError('direction must be \"in\" or \"out\"', 400);\n        }\n        return d;\n      })();\n      const touch = getTouchInput(input);\n      const { startDistance, endDistance, duration } = normalizePinchParam(\n        {\n          locate: {\n            center: [center.x, center.y],\n            rect: { left: center.x, top: center.y, width: 1, height: 1 },\n            description: 'manual pinch target',\n          },\n          direction,\n          distance: optionalNumber(body.distance, 'distance'),\n          duration: optionalNumber(body.duration, 'duration'),\n        },\n        await getScreenSize(),\n      );\n      return ensureCapability(touch.pinch, 'Pinch')(center, {\n        startDistance,\n        endDistance,\n        duration,\n      });\n    }\n\n    default:\n      throw new PointerInputError(`Unknown actionType \"${actionType}\"`, 404);\n  }\n}\n"],"names":["PointerInputError","Error","message","statusCode","requireNumber","value","field","Number","optionalNumber","undefined","requireString","requirePoint","body","xField","yField","ensureCapability","fn","actionType","getPointerInput","input","getKeyboardInput","getTouchInput","getScrollInput","dispatchPointer","getScreenSize","pointer","touch","keyboard","at","mode","autoDismissKeyboard","target","scroll","x","y","direction","scrollType","distance","center","d","startDistance","endDistance","duration","normalizePinchParam"],"mappings":";;;;;;;;;;;AAcO,MAAMA,0BAA0BC;IACrC,YACEC,OAAe,EACRC,UAAqB,CAC5B;QACA,KAAK,CAACD,UAAAA,iBAAAA,IAAAA,EAAAA,cAAAA,KAAAA,IAAAA,IAAAA,CAFCC,UAAU,GAAVA;QAGP,IAAI,CAAC,IAAI,GAAG;IACd;AACF;AAEA,SAASC,cAAcC,KAAc,EAAEC,KAAa;IAClD,IAAI,AAAiB,YAAjB,OAAOD,SAAsBE,OAAO,KAAK,CAACF,QAC5C,MAAM,IAAIL,kBAAkB,GAAGM,MAAM,iBAAiB,CAAC,EAAE;IAE3D,OAAOD;AACT;AAEA,SAASG,eAAeH,KAAc,EAAEC,KAAa;IACnD,IAAID,AAAUI,WAAVJ,OAAqB;IACzB,OAAOD,cAAcC,OAAOC;AAC9B;AAEA,SAASI,cAAcL,KAAc,EAAEC,KAAa;IAClD,IAAI,AAAiB,YAAjB,OAAOD,OACT,MAAM,IAAIL,kBAAkB,GAAGM,MAAM,iBAAiB,CAAC,EAAE;IAE3D,OAAOD;AACT;AAEA,SAASM,aACPC,IAA6B,EAC7BC,SAAuB,GAAG,EAC1BC,SAAuB,GAAG;IAE1B,OAAO;QACL,GAAGV,cAAcQ,IAAI,CAACC,OAAO,EAAEA;QAC/B,GAAGT,cAAcQ,IAAI,CAACE,OAAO,EAAEA;IACjC;AACF;AAEA,SAASC,iBACPC,EAAiB,EACjBC,UAAkB;IAElB,IAAI,AAAc,cAAd,OAAOD,IACT,MAAM,IAAIhB,kBACR,GAAGiB,WAAW,gCAAgC,CAAC,EAC/C;IAGJ,OAAOD;AACT;AAEA,SAASE,gBAAgBC,KAAsB;IAC7C,IAAI,CAACA,MAAM,OAAO,EAChB,MAAM,IAAInB,kBACR,iDACA;IAGJ,OAAOmB,MAAM,OAAO;AACtB;AAEA,SAASC,iBAAiBD,KAAsB;IAC9C,IAAI,CAACA,MAAM,QAAQ,EACjB,MAAM,IAAInB,kBACR,kDACA;IAGJ,OAAOmB,MAAM,QAAQ;AACvB;AAEA,SAASE,cAAcF,KAAsB;IAC3C,IAAI,CAACA,MAAM,KAAK,EACd,MAAM,IAAInB,kBACR,+CACA;IAGJ,OAAOmB,MAAM,KAAK;AACpB;AAEA,SAASG,eAAeH,KAAsB;IAC5C,IAAI,CAACA,MAAM,MAAM,EACf,MAAM,IAAInB,kBACR,gDACA;IAGJ,OAAOmB,MAAM,MAAM;AACrB;AASO,eAAeI,gBACpBJ,KAAsB,EACtBP,IAA6B,EAC7BY,aAA+D;IAE/D,MAAM,EAAEP,UAAU,EAAE,GAAGL;IACvB,IAAI,AAAsB,YAAtB,OAAOK,cAA2B,CAACA,YACrC,MAAM,IAAIjB,kBAAkB,0BAA0B;IAGxD,OAAQiB;QACN,KAAK;YAAO;gBACV,MAAMQ,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,GAAG,EAAE,OAAOd,aAAaC,OAAO;oBAC9D,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C;YACF;QAEA,KAAK;YAAe;gBAClB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBACLU,QAAQ,WAAW,EACnB,eACAd,aAAaC;YACjB;QAEA,KAAK;YAAa;gBAChB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,SAAS,EAAE,aACzCd,aAAaC,OACb;oBACE,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C;YAEJ;QAEA,KAAK;YAAS;gBACZ,MAAMc,QAAQL,cAAcF;gBAC5B,OAAOJ,iBAAiBW,MAAM,KAAK,EAAE,SACnCf,aAAaC,OACbD,aAAaC,MAAM,QAAQ,SAC3B;oBACE,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;oBACxC,QAAQJ,eAAeI,KAAK,MAAM,EAAE;gBACtC;YAEJ;QAEA,KAAK;YAAe;gBAClB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,WAAW,EAAE,eAC3Cd,aAAaC,OACbD,aAAaC,MAAM,QAAQ;YAE/B;QAEA,KAAK;YAAiB;gBACpB,MAAMe,WAAWP,iBAAiBD;gBAClC,OAAOJ,iBACLY,SAAS,aAAa,EACtB,iBACAjB,cAAcE,KAAK,OAAO,EAAE;YAChC;QAEA,KAAK;YAAS;gBACZ,MAAMe,WAAWP,iBAAiBD;gBAClC,MAAMd,QAAQK,cAAcE,KAAK,KAAK,EAAE;gBACxC,MAAMgB,KACJ,AAAkB,YAAlB,OAAOhB,KAAK,CAAC,IAAiB,AAAkB,YAAlB,OAAOA,KAAK,CAAC,GACvCD,aAAaC,QACbH;gBACN,MAAMoB,OACJ,AAAqB,YAArB,OAAOjB,KAAK,IAAI,GACXA,KAAK,IAAI,GACVH;gBACN,MAAMqB,sBACJ,AAAoC,aAApC,OAAOlB,KAAK,mBAAmB,GAC3BA,KAAK,mBAAmB,GACxBH;gBACN,MAAMsB,SAASH,KACX;oBACE,QAAQ;wBAACA,GAAG,CAAC;wBAAEA,GAAG,CAAC;qBAAC;oBACpB,MAAM;wBAAE,MAAMA,GAAG,CAAC;wBAAE,KAAKA,GAAG,CAAC;wBAAE,OAAO;wBAAG,QAAQ;oBAAE;oBACnD,aAAa;gBACf,IACAnB;gBACJ,IAAIoB,AAAS,YAATA,MAAkB,YACpB,MAAMd,iBAAiBY,SAAS,UAAU,EAAE,cAAcI;gBAG5D,IAAI,CAAC1B,OAAO;gBACZ,OAAOU,iBAAiBY,SAAS,QAAQ,EAAE,SAAStB,OAAO;oBACzDyB;oBACAC;oBACA,SAASF,AAAS,eAATA;gBACX;YACF;QAEA,KAAK;YAAU;gBACb,MAAMG,SAASV,eAAeH;gBAC9B,MAAMc,IACJ,AAAkB,YAAlB,OAAOrB,KAAK,CAAC,GAAgBR,cAAcQ,KAAK,CAAC,EAAE,OAAOH;gBAC5D,MAAMyB,IACJ,AAAkB,YAAlB,OAAOtB,KAAK,CAAC,GAAgBR,cAAcQ,KAAK,CAAC,EAAE,OAAOH;gBAC5D,MAAM0B,YACJvB,AAAmB,SAAnBA,KAAK,SAAS,IACdA,AAAmB,WAAnBA,KAAK,SAAS,IACdA,AAAmB,WAAnBA,KAAK,SAAS,IACdA,AAAmB,YAAnBA,KAAK,SAAS,GACVA,KAAK,SAAS,GACd;gBACN,MAAMwB,aACJxB,AAAoB,qBAApBA,KAAK,UAAU,IACfA,AAAoB,kBAApBA,KAAK,UAAU,IACfA,AAAoB,mBAApBA,KAAK,UAAU,IACfA,AAAoB,oBAApBA,KAAK,UAAU,IACfA,AAAoB,mBAApBA,KAAK,UAAU,GACXA,KAAK,UAAU,GACf;gBACN,MAAMyB,WACJ,AAAyB,YAAzB,OAAOzB,KAAK,QAAQ,GAChBR,cAAcQ,KAAK,QAAQ,EAAE,cAC7BH;gBACN,OAAOM,iBACLiB,OAAO,MAAM,EACb,UACA;oBACAG;oBACAC;oBACAC;oBACA,QACEJ,AAAMxB,WAANwB,KAAmBC,AAAMzB,WAANyB,IACf;wBACE,QAAQ;4BAACD;4BAAGC;yBAAE;wBACd,MAAM;4BAAE,MAAMD;4BAAG,KAAKC;4BAAG,OAAO;4BAAG,QAAQ;wBAAE;wBAC7C,aAAa;oBACf,IACAzB;gBACR;YACF;QAEA,KAAK;YAAS;gBACZ,MAAM6B,SAAS3B,aAAaC;gBAC5B,MAAMuB,YAAa,AAAC;oBAClB,MAAMI,IAAI3B,KAAK,SAAS;oBACxB,IAAI2B,AAAM,SAANA,KAAcA,AAAM,UAANA,GAChB,MAAM,IAAIvC,kBAAkB,mCAAmC;oBAEjE,OAAOuC;gBACT;gBACA,MAAMb,QAAQL,cAAcF;gBAC5B,MAAM,EAAEqB,aAAa,EAAEC,WAAW,EAAEC,QAAQ,EAAE,GAAGC,oBAC/C;oBACE,QAAQ;wBACN,QAAQ;4BAACL,OAAO,CAAC;4BAAEA,OAAO,CAAC;yBAAC;wBAC5B,MAAM;4BAAE,MAAMA,OAAO,CAAC;4BAAE,KAAKA,OAAO,CAAC;4BAAE,OAAO;4BAAG,QAAQ;wBAAE;wBAC3D,aAAa;oBACf;oBACAH;oBACA,UAAU3B,eAAeI,KAAK,QAAQ,EAAE;oBACxC,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C,GACA,MAAMY;gBAER,OAAOT,iBAAiBW,MAAM,KAAK,EAAE,SAASY,QAAQ;oBACpDE;oBACAC;oBACAC;gBACF;YACF;QAEA;YACE,MAAM,IAAI1C,kBAAkB,CAAC,oBAAoB,EAAEiB,WAAW,CAAC,CAAC,EAAE;IACtE;AACF"}
+{"version":3,"file":"pointer-dispatch.mjs","sources":["../../src/pointer-dispatch.ts"],"sourcesContent":["import type {\n  InputPrimitives,\n  KeyboardInputPrimitives,\n  PointerInputPrimitives,\n  PointerPoint,\n  TouchInputPrimitives,\n} from '@midscene/core/device';\nimport { normalizePinchParam } from '@midscene/core/device';\n\n/**\n * Thrown when an /interact request is malformed (missing field, wrong type)\n * or names an action / capability that the connected device does not support.\n * The route handler maps {400, 404} onto the HTTP response.\n */\nexport class PointerInputError extends Error {\n  constructor(\n    message: string,\n    public statusCode: 400 | 404,\n  ) {\n    super(message);\n    this.name = 'PointerInputError';\n  }\n}\n\nfunction requireNumber(value: unknown, field: string): number {\n  if (typeof value !== 'number' || Number.isNaN(value)) {\n    throw new PointerInputError(`${field} must be a number`, 400);\n  }\n  return value;\n}\n\nfunction optionalNumber(value: unknown, field: string): number | undefined {\n  if (value === undefined) return undefined;\n  return requireNumber(value, field);\n}\n\nfunction requireString(value: unknown, field: string): string {\n  if (typeof value !== 'string') {\n    throw new PointerInputError(`${field} must be a string`, 400);\n  }\n  return value;\n}\n\nfunction requirePoint(\n  body: Record<string, unknown>,\n  xField: 'x' | 'endX' = 'x',\n  yField: 'y' | 'endY' = 'y',\n): PointerPoint {\n  return {\n    x: requireNumber(body[xField], xField),\n    y: requireNumber(body[yField], yField),\n  };\n}\n\nfunction ensureCapability<T>(\n  fn: T | undefined,\n  actionType: string,\n): NonNullable<T> {\n  if (typeof fn !== 'function') {\n    throw new PointerInputError(\n      `${actionType} is not supported on this device`,\n      404,\n    );\n  }\n  return fn as NonNullable<T>;\n}\n\nfunction getPointerInput(input: InputPrimitives): PointerInputPrimitives {\n  if (!input.pointer) {\n    throw new PointerInputError(\n      'Pointer input is not supported on this device',\n      404,\n    );\n  }\n  return input.pointer;\n}\n\nfunction getKeyboardInput(input: InputPrimitives): KeyboardInputPrimitives {\n  if (!input.keyboard) {\n    throw new PointerInputError(\n      'Keyboard input is not supported on this device',\n      404,\n    );\n  }\n  return input.keyboard;\n}\n\nfunction getTouchInput(input: InputPrimitives): TouchInputPrimitives {\n  if (!input.touch) {\n    throw new PointerInputError(\n      'Touch input is not supported on this device',\n      404,\n    );\n  }\n  return input.touch;\n}\n\nfunction getScrollInput(input: InputPrimitives) {\n  if (!input.scroll) {\n    throw new PointerInputError(\n      'Scroll input is not supported on this device',\n      404,\n    );\n  }\n  return input.scroll;\n}\n\n/**\n * Translate an `/interact` request body into device input primitive calls.\n *\n * The dispatcher is deliberately a flat switch: each case is HTTP-protocol\n * adaptation (parse + range-check fields, hand them to the typed primitive),\n * not platform business logic.\n */\nexport async function dispatchPointer(\n  input: InputPrimitives,\n  body: Record<string, unknown>,\n  getScreenSize: () => Promise<{ width: number; height: number }>,\n): Promise<void> {\n  const { actionType } = body;\n  if (typeof actionType !== 'string' || !actionType) {\n    throw new PointerInputError('actionType is required', 400);\n  }\n\n  switch (actionType) {\n    case 'Tap': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.tap, 'Tap')(requirePoint(body), {\n        duration: optionalNumber(body.duration, 'duration'),\n      });\n    }\n\n    case 'DoubleClick': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(\n        pointer.doubleClick,\n        'DoubleClick',\n      )(requirePoint(body));\n    }\n\n    case 'LongPress': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.longPress, 'LongPress')(\n        requirePoint(body),\n        {\n          duration: optionalNumber(body.duration, 'duration'),\n        },\n      );\n    }\n\n    case 'Swipe': {\n      const touch = getTouchInput(input);\n      return ensureCapability(touch.swipe, 'Swipe')(\n        requirePoint(body),\n        requirePoint(body, 'endX', 'endY'),\n        {\n          duration: optionalNumber(body.duration, 'duration'),\n          repeat: optionalNumber(body.repeat, 'repeat'),\n        },\n      );\n    }\n\n    case 'DragAndDrop': {\n      const pointer = getPointerInput(input);\n      return ensureCapability(pointer.dragAndDrop, 'DragAndDrop')(\n        requirePoint(body),\n        requirePoint(body, 'endX', 'endY'),\n      );\n    }\n\n    case 'KeyboardPress': {\n      const keyboard = getKeyboardInput(input);\n      return ensureCapability(\n        keyboard.keyboardPress,\n        'KeyboardPress',\n      )(requireString(body.keyName, 'keyName'));\n    }\n\n    case 'Input': {\n      const keyboard = getKeyboardInput(input);\n      const value = requireString(body.value, 'value');\n      const at =\n        typeof body.x === 'number' && typeof body.y === 'number'\n          ? requirePoint(body)\n          : undefined;\n      const mode =\n        typeof body.mode === 'string'\n          ? (body.mode as 'replace' | 'clear' | 'typeOnly')\n          : undefined;\n      const autoDismissKeyboard =\n        typeof body.autoDismissKeyboard === 'boolean'\n          ? body.autoDismissKeyboard\n          : undefined;\n      const target = at\n        ? {\n            center: [at.x, at.y] as [number, number],\n            rect: { left: at.x, top: at.y, width: 1, height: 1 },\n            description: 'manual input target',\n          }\n        : undefined;\n      if (mode === 'clear') {\n        await ensureCapability(keyboard.clearInput, 'ClearInput')(target);\n        return;\n      }\n      if (!value) return;\n      return ensureCapability(keyboard.typeText, 'Input')(value, {\n        autoDismissKeyboard,\n        target,\n        replace: mode !== 'typeOnly',\n      });\n    }\n\n    case 'Scroll': {\n      const scroll = getScrollInput(input);\n      const x =\n        typeof body.x === 'number' ? requireNumber(body.x, 'x') : undefined;\n      const y =\n        typeof body.y === 'number' ? requireNumber(body.y, 'y') : undefined;\n      const direction =\n        body.direction === 'up' ||\n        body.direction === 'down' ||\n        body.direction === 'left' ||\n        body.direction === 'right'\n          ? body.direction\n          : 'down';\n      const scrollType =\n        body.scrollType === 'scrollToBottom' ||\n        body.scrollType === 'scrollToTop' ||\n        body.scrollType === 'scrollToLeft' ||\n        body.scrollType === 'scrollToRight' ||\n        body.scrollType === 'singleAction'\n          ? body.scrollType\n          : 'singleAction';\n      const distance =\n        typeof body.distance === 'number'\n          ? requireNumber(body.distance, 'distance')\n          : undefined;\n      return ensureCapability(\n        scroll.scroll,\n        'Scroll',\n      )({\n        direction,\n        scrollType,\n        distance,\n        locate:\n          x !== undefined && y !== undefined\n            ? {\n                center: [x, y],\n\n                description: 'manual scroll target',\n              }\n            : undefined,\n      });\n    }\n\n    case 'Pinch': {\n      const center = requirePoint(body);\n      const direction = ((): 'in' | 'out' => {\n        const d = body.direction;\n        if (d !== 'in' && d !== 'out') {\n          throw new PointerInputError('direction must be \"in\" or \"out\"', 400);\n        }\n        return d;\n      })();\n      const touch = getTouchInput(input);\n      const { startDistance, endDistance, duration } = normalizePinchParam(\n        {\n          locate: {\n            center: [center.x, center.y],\n            description: 'manual pinch target',\n          },\n          direction,\n          distance: optionalNumber(body.distance, 'distance'),\n          duration: optionalNumber(body.duration, 'duration'),\n        },\n        await getScreenSize(),\n      );\n      return ensureCapability(touch.pinch, 'Pinch')(center, {\n        startDistance,\n        endDistance,\n        duration,\n      });\n    }\n\n    default:\n      throw new PointerInputError(`Unknown actionType \"${actionType}\"`, 404);\n  }\n}\n"],"names":["PointerInputError","Error","message","statusCode","requireNumber","value","field","Number","optionalNumber","undefined","requireString","requirePoint","body","xField","yField","ensureCapability","fn","actionType","getPointerInput","input","getKeyboardInput","getTouchInput","getScrollInput","dispatchPointer","getScreenSize","pointer","touch","keyboard","at","mode","autoDismissKeyboard","target","scroll","x","y","direction","scrollType","distance","center","d","startDistance","endDistance","duration","normalizePinchParam"],"mappings":";;;;;;;;;;;AAcO,MAAMA,0BAA0BC;IACrC,YACEC,OAAe,EACRC,UAAqB,CAC5B;QACA,KAAK,CAACD,UAAAA,iBAAAA,IAAAA,EAAAA,cAAAA,KAAAA,IAAAA,IAAAA,CAFCC,UAAU,GAAVA;QAGP,IAAI,CAAC,IAAI,GAAG;IACd;AACF;AAEA,SAASC,cAAcC,KAAc,EAAEC,KAAa;IAClD,IAAI,AAAiB,YAAjB,OAAOD,SAAsBE,OAAO,KAAK,CAACF,QAC5C,MAAM,IAAIL,kBAAkB,GAAGM,MAAM,iBAAiB,CAAC,EAAE;IAE3D,OAAOD;AACT;AAEA,SAASG,eAAeH,KAAc,EAAEC,KAAa;IACnD,IAAID,AAAUI,WAAVJ,OAAqB;IACzB,OAAOD,cAAcC,OAAOC;AAC9B;AAEA,SAASI,cAAcL,KAAc,EAAEC,KAAa;IAClD,IAAI,AAAiB,YAAjB,OAAOD,OACT,MAAM,IAAIL,kBAAkB,GAAGM,MAAM,iBAAiB,CAAC,EAAE;IAE3D,OAAOD;AACT;AAEA,SAASM,aACPC,IAA6B,EAC7BC,SAAuB,GAAG,EAC1BC,SAAuB,GAAG;IAE1B,OAAO;QACL,GAAGV,cAAcQ,IAAI,CAACC,OAAO,EAAEA;QAC/B,GAAGT,cAAcQ,IAAI,CAACE,OAAO,EAAEA;IACjC;AACF;AAEA,SAASC,iBACPC,EAAiB,EACjBC,UAAkB;IAElB,IAAI,AAAc,cAAd,OAAOD,IACT,MAAM,IAAIhB,kBACR,GAAGiB,WAAW,gCAAgC,CAAC,EAC/C;IAGJ,OAAOD;AACT;AAEA,SAASE,gBAAgBC,KAAsB;IAC7C,IAAI,CAACA,MAAM,OAAO,EAChB,MAAM,IAAInB,kBACR,iDACA;IAGJ,OAAOmB,MAAM,OAAO;AACtB;AAEA,SAASC,iBAAiBD,KAAsB;IAC9C,IAAI,CAACA,MAAM,QAAQ,EACjB,MAAM,IAAInB,kBACR,kDACA;IAGJ,OAAOmB,MAAM,QAAQ;AACvB;AAEA,SAASE,cAAcF,KAAsB;IAC3C,IAAI,CAACA,MAAM,KAAK,EACd,MAAM,IAAInB,kBACR,+CACA;IAGJ,OAAOmB,MAAM,KAAK;AACpB;AAEA,SAASG,eAAeH,KAAsB;IAC5C,IAAI,CAACA,MAAM,MAAM,EACf,MAAM,IAAInB,kBACR,gDACA;IAGJ,OAAOmB,MAAM,MAAM;AACrB;AASO,eAAeI,gBACpBJ,KAAsB,EACtBP,IAA6B,EAC7BY,aAA+D;IAE/D,MAAM,EAAEP,UAAU,EAAE,GAAGL;IACvB,IAAI,AAAsB,YAAtB,OAAOK,cAA2B,CAACA,YACrC,MAAM,IAAIjB,kBAAkB,0BAA0B;IAGxD,OAAQiB;QACN,KAAK;YAAO;gBACV,MAAMQ,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,GAAG,EAAE,OAAOd,aAAaC,OAAO;oBAC9D,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C;YACF;QAEA,KAAK;YAAe;gBAClB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBACLU,QAAQ,WAAW,EACnB,eACAd,aAAaC;YACjB;QAEA,KAAK;YAAa;gBAChB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,SAAS,EAAE,aACzCd,aAAaC,OACb;oBACE,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C;YAEJ;QAEA,KAAK;YAAS;gBACZ,MAAMc,QAAQL,cAAcF;gBAC5B,OAAOJ,iBAAiBW,MAAM,KAAK,EAAE,SACnCf,aAAaC,OACbD,aAAaC,MAAM,QAAQ,SAC3B;oBACE,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;oBACxC,QAAQJ,eAAeI,KAAK,MAAM,EAAE;gBACtC;YAEJ;QAEA,KAAK;YAAe;gBAClB,MAAMa,UAAUP,gBAAgBC;gBAChC,OAAOJ,iBAAiBU,QAAQ,WAAW,EAAE,eAC3Cd,aAAaC,OACbD,aAAaC,MAAM,QAAQ;YAE/B;QAEA,KAAK;YAAiB;gBACpB,MAAMe,WAAWP,iBAAiBD;gBAClC,OAAOJ,iBACLY,SAAS,aAAa,EACtB,iBACAjB,cAAcE,KAAK,OAAO,EAAE;YAChC;QAEA,KAAK;YAAS;gBACZ,MAAMe,WAAWP,iBAAiBD;gBAClC,MAAMd,QAAQK,cAAcE,KAAK,KAAK,EAAE;gBACxC,MAAMgB,KACJ,AAAkB,YAAlB,OAAOhB,KAAK,CAAC,IAAiB,AAAkB,YAAlB,OAAOA,KAAK,CAAC,GACvCD,aAAaC,QACbH;gBACN,MAAMoB,OACJ,AAAqB,YAArB,OAAOjB,KAAK,IAAI,GACXA,KAAK,IAAI,GACVH;gBACN,MAAMqB,sBACJ,AAAoC,aAApC,OAAOlB,KAAK,mBAAmB,GAC3BA,KAAK,mBAAmB,GACxBH;gBACN,MAAMsB,SAASH,KACX;oBACE,QAAQ;wBAACA,GAAG,CAAC;wBAAEA,GAAG,CAAC;qBAAC;oBACpB,MAAM;wBAAE,MAAMA,GAAG,CAAC;wBAAE,KAAKA,GAAG,CAAC;wBAAE,OAAO;wBAAG,QAAQ;oBAAE;oBACnD,aAAa;gBACf,IACAnB;gBACJ,IAAIoB,AAAS,YAATA,MAAkB,YACpB,MAAMd,iBAAiBY,SAAS,UAAU,EAAE,cAAcI;gBAG5D,IAAI,CAAC1B,OAAO;gBACZ,OAAOU,iBAAiBY,SAAS,QAAQ,EAAE,SAAStB,OAAO;oBACzDyB;oBACAC;oBACA,SAASF,AAAS,eAATA;gBACX;YACF;QAEA,KAAK;YAAU;gBACb,MAAMG,SAASV,eAAeH;gBAC9B,MAAMc,IACJ,AAAkB,YAAlB,OAAOrB,KAAK,CAAC,GAAgBR,cAAcQ,KAAK,CAAC,EAAE,OAAOH;gBAC5D,MAAMyB,IACJ,AAAkB,YAAlB,OAAOtB,KAAK,CAAC,GAAgBR,cAAcQ,KAAK,CAAC,EAAE,OAAOH;gBAC5D,MAAM0B,YACJvB,AAAmB,SAAnBA,KAAK,SAAS,IACdA,AAAmB,WAAnBA,KAAK,SAAS,IACdA,AAAmB,WAAnBA,KAAK,SAAS,IACdA,AAAmB,YAAnBA,KAAK,SAAS,GACVA,KAAK,SAAS,GACd;gBACN,MAAMwB,aACJxB,AAAoB,qBAApBA,KAAK,UAAU,IACfA,AAAoB,kBAApBA,KAAK,UAAU,IACfA,AAAoB,mBAApBA,KAAK,UAAU,IACfA,AAAoB,oBAApBA,KAAK,UAAU,IACfA,AAAoB,mBAApBA,KAAK,UAAU,GACXA,KAAK,UAAU,GACf;gBACN,MAAMyB,WACJ,AAAyB,YAAzB,OAAOzB,KAAK,QAAQ,GAChBR,cAAcQ,KAAK,QAAQ,EAAE,cAC7BH;gBACN,OAAOM,iBACLiB,OAAO,MAAM,EACb,UACA;oBACAG;oBACAC;oBACAC;oBACA,QACEJ,AAAMxB,WAANwB,KAAmBC,AAAMzB,WAANyB,IACf;wBACE,QAAQ;4BAACD;4BAAGC;yBAAE;wBAEd,aAAa;oBACf,IACAzB;gBACR;YACF;QAEA,KAAK;YAAS;gBACZ,MAAM6B,SAAS3B,aAAaC;gBAC5B,MAAMuB,YAAa,AAAC;oBAClB,MAAMI,IAAI3B,KAAK,SAAS;oBACxB,IAAI2B,AAAM,SAANA,KAAcA,AAAM,UAANA,GAChB,MAAM,IAAIvC,kBAAkB,mCAAmC;oBAEjE,OAAOuC;gBACT;gBACA,MAAMb,QAAQL,cAAcF;gBAC5B,MAAM,EAAEqB,aAAa,EAAEC,WAAW,EAAEC,QAAQ,EAAE,GAAGC,oBAC/C;oBACE,QAAQ;wBACN,QAAQ;4BAACL,OAAO,CAAC;4BAAEA,OAAO,CAAC;yBAAC;wBAC5B,aAAa;oBACf;oBACAH;oBACA,UAAU3B,eAAeI,KAAK,QAAQ,EAAE;oBACxC,UAAUJ,eAAeI,KAAK,QAAQ,EAAE;gBAC1C,GACA,MAAMY;gBAER,OAAOT,iBAAiBW,MAAM,KAAK,EAAE,SAASY,QAAQ;oBACpDE;oBACAC;oBACAC;gBACF;YACF;QAEA;YACE,MAAM,IAAI1C,kBAAkB,CAAC,oBAAoB,EAAEiB,WAAW,CAAC,CAAC,EAAE;IACtE;AACF"}

package/dist/es/server.mjs

@@ -10,7 +10,6 @@ import { getTmpDir, sleep } from "@midscene/core/utils";
 import { getMidsceneRunSubDir } from "@midscene/shared/common";
 import { PLAYGROUND_SERVER_PORT } from "@midscene/shared/constants";
 import { ModelConfigManager, globalModelConfigManager, overrideAIConfig } from "@midscene/shared/env";
-import { generateElementByPoint } from "@midscene/shared/extractor";
 import { annotateRects, imageInfoOfBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { buildMidsceneRecorderActionSummary, buildMidsceneRecorderReplayInstruction } from "@midscene/shared/recorder";
@@ -558,10 +557,13 @@ function requireNumber(value, field) {
     return value;
 }
 function locateFromPoint(x, y, fieldX, fieldY, description) {
-    return generateElementByPoint([
-        Math.round(requireNumber(x, fieldX)),
-        Math.round(requireNumber(y, fieldY))
-    ], description);
+    return {
+        center: [
+            Math.round(requireNumber(x, fieldX)),
+            Math.round(requireNumber(y, fieldY))
+        ],
+        description: description || ''
+    };
 }
 const POINTER_INTERACT_ACTIONS = new Set([
     'Tap',