@midscene/core 1.12.8-beta-20260916030003.0 → 1.12.8

package/dist/es/ai-model/workflows/planning/parse-planning-actions.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/planning/parse-planning-actions.mjs","sources":["../../../../../src/ai-model/workflows/planning/parse-planning-actions.ts"],"sourcesContent":["import {\n  TUserPromptSchema,\n  findActionInActionSpaceOrThrow,\n  findAllMidsceneLocatorField,\n  parseActionParam,\n  validateRequiredLocateFields,\n} from '@/common';\nimport type { DeviceAction, PlanningAction } from '@/types';\nimport { isPlainObject } from '@midscene/shared/utils';\nimport { z } from 'zod';\n\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport {\n  type PlanningLocateNormalizationOptions,\n  normalizePlanningLocateParameter,\n} from './locate-normalization';\n\n// Mutates locator fields in place; ordinary parameters retain their model values.\nexport function parsePlanningActions(\n  actions: PlanningAction[],\n  {\n    actionSpace,\n    parseRawLocateParameter,\n    ...normalizationOptions\n  }: PlanningLocateNormalizationOptions & {\n    actionSpace: DeviceAction[];\n    parseRawLocateParameter: (value: unknown) => ParsedPlanningLocateParameter;\n  },\n): void {\n  for (const action of actions) {\n    const definition = findActionInActionSpaceOrThrow(action.type, actionSpace);\n    if (!definition.paramSchema) {\n      continue;\n    }\n\n    const locateFields = findAllMidsceneLocatorField(definition.paramSchema);\n    try {\n      for (const field of locateFields) {\n        const rawLocate = action.param?.[field];\n        if (rawLocate !== undefined && rawLocate !== null) {\n          action.param[field] = parseRawLocateParameter(rawLocate);\n        }\n      }\n      validateRequiredLocateFields(action.param, definition.paramSchema);\n      // Only preflight validation: keep ordinary parameters unchanged. Parsing can\n      // evaluate defaults/transforms, but its output is consumed only at execution.\n      parseActionParam(action.param, definition.paramSchema);\n\n      for (const field of locateFields) {\n        const locate = action.param?.[field];\n        if (locate === undefined) {\n          continue;\n        }\n        // Validate only the model-facing prompt; coordinates are normalized later.\n        if (!isPlainObject(locate)) {\n          throw new Error(`${field}: Expected an object with a prompt field`);\n        }\n        const prompt = locate.prompt;\n        if (prompt === undefined) {\n          throw new Error(`${field}.prompt: Required`);\n        }\n        const result = TUserPromptSchema.safeParse(prompt);\n        if (!result.success) {\n          throw new z.ZodError(\n            result.error.issues.map((issue) => ({\n              ...issue,\n              path: [field, 'prompt', ...issue.path],\n            })),\n          );\n        }\n        action.param[field] = normalizePlanningLocateParameter(\n          locate,\n          normalizationOptions,\n        );\n      }\n    } catch (error) {\n      const details =\n        error instanceof z.ZodError\n          ? error.issues\n              .map(\n                (issue) =>\n                  `${issue.path.join('.') || 'param'}: ${issue.message}`,\n              )\n              .join('; ')\n          : error instanceof Error\n            ? error.message\n            : String(error);\n      throw new Error(\n        `Invalid parameters for action ${action.type}: ${details}`,\n        { cause: error },\n      );\n    }\n  }\n}\n"],"names":["parsePlanningActions","actions","actionSpace","parseRawLocateParameter","normalizationOptions","action","definition","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","field","rawLocate","validateRequiredLocateFields","parseActionParam","locate","undefined","isPlainObject","Error","prompt","result","TUserPromptSchema","z","issue","normalizePlanningLocateParameter","error","details","String"],"mappings":";;;;AAkBO,SAASA,qBACdC,OAAyB,EACzB,EACEC,WAAW,EACXC,uBAAuB,EACvB,GAAGC,sBAIJ;IAED,KAAK,MAAMC,UAAUJ,QAAS;QAC5B,MAAMK,aAAaC,+BAA+BF,OAAO,IAAI,EAAEH;QAC/D,IAAI,CAACI,WAAW,WAAW,EACzB;QAGF,MAAME,eAAeC,4BAA4BH,WAAW,WAAW;QACvE,IAAI;YACF,KAAK,MAAMI,SAASF,aAAc;gBAChC,MAAMG,YAAYN,OAAO,KAAK,EAAE,CAACK,MAAM;gBACvC,IAAIC,QAAAA,WACFN,OAAO,KAAK,CAACK,MAAM,GAAGP,wBAAwBQ;YAElD;YACAC,6BAA6BP,OAAO,KAAK,EAAEC,WAAW,WAAW;YAGjEO,iBAAiBR,OAAO,KAAK,EAAEC,WAAW,WAAW;YAErD,KAAK,MAAMI,SAASF,aAAc;gBAChC,MAAMM,SAAST,OAAO,KAAK,EAAE,CAACK,MAAM;gBACpC,IAAII,AAAWC,WAAXD,QACF;gBAGF,IAAI,CAACE,cAAcF,SACjB,MAAM,IAAIG,MAAM,GAAGP,MAAM,wCAAwC,CAAC;gBAEpE,MAAMQ,SAASJ,OAAO,MAAM;gBAC5B,IAAII,AAAWH,WAAXG,QACF,MAAM,IAAID,MAAM,GAAGP,MAAM,iBAAiB,CAAC;gBAE7C,MAAMS,SAASC,kBAAkB,SAAS,CAACF;gBAC3C,IAAI,CAACC,OAAO,OAAO,EACjB,MAAM,IAAIE,EAAE,QAAQ,CAClBF,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAACG,QAAW;wBAClC,GAAGA,KAAK;wBACR,MAAM;4BAACZ;4BAAO;+BAAaY,MAAM,IAAI;yBAAC;oBACxC;gBAGJjB,OAAO,KAAK,CAACK,MAAM,GAAGa,iCACpBT,QACAV;YAEJ;QACF,EAAE,OAAOoB,OAAO;YACd,MAAMC,UACJD,iBAAiBH,EAAE,QAAQ,GACvBG,MAAM,MAAM,CACT,GAAG,CACF,CAACF,QACC,GAAGA,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,QAAQ,EAAE,EAAEA,MAAM,OAAO,EAAE,EAEzD,IAAI,CAAC,QACRE,iBAAiBP,QACfO,MAAM,OAAO,GACbE,OAAOF;YACf,MAAM,IAAIP,MACR,CAAC,8BAA8B,EAAEZ,OAAO,IAAI,CAAC,EAAE,EAAEoB,SAAS,EAC1D;gBAAE,OAAOD;YAAM;QAEnB;IACF;AACF"}

package/dist/es/ai-model/workflows/planning/standard-planning.mjs

@@ -7,7 +7,7 @@ import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
 import { planningModelFamilyRequiredForLocateMessage } from "../../shared/model-locate-result/errors.mjs";
 import { preparedReferenceImagesToChatMessages } from "../../shared/multimodal-prompt.mjs";
-import { normalizePlanningActionLocateFields } from "./locate-normalization.mjs";
+import { parsePlanningActions } from "./parse-planning-actions.mjs";
 import { parseStandardPlanningResponse } from "./standard-planning-parser.mjs";
 const debug = getDebug('planning');
 const warnLog = getDebug('planning', {
@@ -31,6 +31,7 @@ async function callAndParsePlanningResponse(options) {
                 actionSpace,
                 logSource: includeLog ? 'model' : 'action'
             });
+            if (!planFromAI.action && void 0 === planFromAI.finalizeSuccess && !planFromAI.error) throw new Error('Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.');
             if (planFromAI.action && void 0 !== planFromAI.finalizeSuccess) {
                 warnLog('Planning response included both an action and <complete>; ignoring <complete> output.');
                 planFromAI.finalizeMessage = void 0;
@@ -39,20 +40,18 @@ async function callAndParsePlanningResponse(options) {
             const actions = planFromAI.action ? [
                 planFromAI.action
             ] : [];
-            normalizePlanningActionLocateFields(actions, {
+            parsePlanningActions(actions, {
+                parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,
                 actionSpace,
                 includeLocateInPlanning,
                 locateResultCodec,
                 locateResultContext,
-                acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,
-                parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter
+                acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias
             });
-            const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);
             return {
                 response,
                 planFromAI,
-                actions,
-                yamlFlow
+                actions
             };
         },
         toParseError: (parseError, response)=>{
@@ -165,7 +164,7 @@ async function standardPlan(userInstruction, opts) {
         ...instruction,
         ...historyLog
     ];
-    const { response: { content: rawResponse, usage, reasoning_content, rawChoiceMessage }, planFromAI, actions, yamlFlow } = await callAndParsePlanningResponse({
+    const { response: { content: rawResponse, usage, reasoning_content, rawChoiceMessage }, planFromAI, actions } = await callAndParsePlanningResponse({
         messages: msgs,
         modelRuntime,
         abortSignal: opts.abortSignal,
@@ -179,6 +178,7 @@ async function standardPlan(userInstruction, opts) {
         includeThought,
         includeLog
     });
+    const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);
     let shouldContinuePlanning = true;
     if (void 0 !== planFromAI.finalizeSuccess) {
         debug('task completed via <complete> tag, stop planning');

package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n        acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n      });\n      // dumpActionParam keeps only the locator prompt, so runtime-only\n      // locatedPixelResult fields added during normalization are not serialized.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","actions","normalizePlanningActionLocateFields","yamlFlow","buildYamlFlowFromPlans","parseError","errorMessage","Error","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IAAIS,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAMC,UAAUH,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DI,oCAAoCD,SAAS;gBAC3ChB;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;gBACzD,yBAAyBS,qBAAqB,uBAAuB;YACvE;YAGA,MAAMY,WAAWC,uBAAuBH,SAAShB;YACjD,OAAO;gBAAEY;gBAAUC;gBAAYG;gBAASE;YAAS;QACnD;QACA,cAAc,CAACE,YAAYR;YACzB,MAAMS,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACsB;YACb9B,MACE,mDACA8B,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;QAE9D;IACF;AACF;AAEO,eAAeK,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM9B,eAAe8B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGjC;IACpB,MAAM,EAAEkC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDvB,OACEyB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC9B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIyB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrDiC,QAAQ,QAAQ,CAAC,iBAAiB,GAClCf;IAGJ,MAAMoB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMxB,iBAAiBwB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMvB,aAAauB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAAC1B,mBACnC,MAAM,IAAIqB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMuC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BxB;QACAC;QACA+B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAI1B,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMqC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpC,uBAAuB;IAGpC,MAAMuD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACD1C,UAAU,EACVG,OAAO,EACPE,QAAQ,EACT,GAAG,MAAMxB,6BAA6B;QACrC,UAAUyD;QACVtD;QACA,aAAa8B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7B1B;QACA,qBAAqB;YACnB,cAAcqC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAnC;QACAC;IACF;IAEA,IAAIoD,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIrB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAG5C,UAAU;QACbG;QACAoC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIsB,iBAAiB;QACnB,IAAItB,WAAW,cAAc,EAAE,QAC7BgB,oBAAoB,aAAa,CAAChB,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDgB,oBAAoB,mBAAmB,CAAC6B;QAI5C,IAAI7C,WAAW,GAAG,EAChBgB,oBAAoB,gBAAgB,CAAChB,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBgB,oBAAoB,mBAAmB,CAAChB,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBgB,oBAAoB,YAAY,CAAChB,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7D0D,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOK;AACT"}
+{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (\n        !planFromAI.action &&\n        planFromAI.finalizeSuccess === undefined &&\n        !planFromAI.error\n      ) {\n        throw new Error(\n          'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n        );\n      }\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      parsePlanningActions(actions, {\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n        acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n      });\n      return { response, planFromAI, actions };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  // YAML is derived from validated actions outside the model-response retry scope.\n  // dumpActionParam omits runtime-only locatedPixelResult fields.\n  const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,qBAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;YAC3D;YACA,OAAO;gBAAEe;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,OACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,uBAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7DwD,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOO;AACT"}

package/dist/es/common.mjs

@@ -102,6 +102,9 @@ const findAllMidsceneLocatorField = (zodType, requiredOnly)=>{
     }
     return [];
 };
+function validateRequiredLocateFields(param, schema) {
+    for (const field of findAllMidsceneLocatorField(schema, true))if (!param?.[field]) throw new Error(`${field}: Required locate field is not provided`);
+}
 const dumpActionParam = (jsonObject, zodSchema)=>{
     if (!isPlainObject(jsonObject)) return {};
     const locatorFields = findAllMidsceneLocatorField(zodSchema);
@@ -161,6 +164,6 @@ const getReadableTimeString = (format = 'YYYY-MM-DD HH:mm:ss', timestamp)=>{
     const timeString = format.replace('YYYY', String(year)).replace('MM', month).replace('DD', day).replace('HH', hours).replace('mm', minutes).replace('ss', seconds);
     return `${timeString} (${format})`;
 };
-export { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, buildYamlFlowFromPlans, dumpActionParam, finalizeActionName, findActionInActionSpaceOrThrow, findAllMidsceneLocatorField, getMidsceneLocationSchema, getReadableTimeString, ifMidsceneLocatorField, parseActionParam, userPromptToMultimodalPrompt, userPromptToString };
+export { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, buildYamlFlowFromPlans, dumpActionParam, finalizeActionName, findActionInActionSpaceOrThrow, findAllMidsceneLocatorField, getMidsceneLocationSchema, getReadableTimeString, ifMidsceneLocatorField, parseActionParam, userPromptToMultimodalPrompt, userPromptToString, validateRequiredLocateFields };
 
 //# sourceMappingURL=common.mjs.map

package/dist/es/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n} from '@/types';\nimport { isPlainObject } from '@midscene/shared/utils';\nimport { z } from 'zod';\n\nexport function findActionInActionSpaceOrThrow(\n  planType: string,\n  actionSpace: DeviceAction<any>[],\n): DeviceAction<any> {\n  const action = actionSpace.find((item) => item.name === planType);\n  if (!action) {\n    const available = actionSpace.map((item) => item.name).join(', ');\n    throw new Error(\n      `Action type '${planType}' is not in the current action space. Available actions: ${available || '(none)'}`,\n    );\n  }\n  return action;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = findActionInActionSpaceOrThrow(verb, actionSpace);\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    // For actions whose param is a single string field (e.g. Launch/Terminate's\n    // `uri`, RunAdbShell's `command`), inline the value on the flowKey. Writing\n    // `{ terminate: '', uri: '...' }` makes the YAML player treat the empty\n    // string as the param and drop the sibling `uri`, so cache replay would\n    // call the action with an empty argument.\n    const shortcutField =\n      action.name === 'Launch' || action.interfaceAlias === 'launch'\n        ? 'uri'\n        : action.name === 'Terminate' || action.interfaceAlias === 'terminate'\n          ? 'uri'\n          : action.name === 'RunAdbShell' ||\n              action.interfaceAlias === 'runAdbShell' ||\n              action.name === 'RunHdcShell' ||\n              action.interfaceAlias === 'runHdcShell'\n            ? 'command'\n            : undefined;\n    const shortcutKeys = shortcutField ? Object.keys(flowParam) : [];\n    const canInlineShortcut =\n      shortcutField &&\n      shortcutKeys.length === 1 &&\n      shortcutKeys[0] === shortcutField &&\n      typeof flowParam[shortcutField] === 'string';\n\n    const flowItem: MidsceneYamlFlowItem = canInlineShortcut\n      ? { [flowKey]: flowParam[shortcutField as string] }\n      : { [flowKey]: '', ...flowParam };\n\n    flow.push(flowItem);\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nexport const userPromptToString = (prompt: TUserPrompt): string => {\n  return typeof prompt === 'string' ? prompt : prompt.prompt;\n};\n\nexport const userPromptToMultimodalPrompt = (\n  prompt: TUserPrompt,\n): TMultimodalPrompt | undefined => {\n  if (typeof prompt === 'string' || !prompt.images) {\n    return undefined;\n  }\n  return {\n    images: prompt.images,\n    convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n  };\n};\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepLocate: z.boolean().optional(),\n    deepThink: z\n      .boolean()\n      .optional()\n      .describe('@deprecated Use `deepLocate` instead.'),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nconst formatPromptWithImages = (\n  promptObj: Exclude<TUserPrompt, string>,\n): string => {\n  let promptString = promptObj.prompt;\n  if (Array.isArray(promptObj.images) && promptObj.images.length > 0) {\n    const imageCount = promptObj.images.length;\n    promptString += ` (with ${imageCount} image${imageCount > 1 ? 's' : ''})`;\n  }\n  return promptString;\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      // TODO: A similar locator-field check exists as isMidsceneLocatorField in\n      // @midscene/shared/zod-schema-utils.\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n  if (!isPlainObject(jsonObject)) {\n    return {};\n  }\n\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = formatPromptWithImages(fieldValue.prompt);\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n *\n * When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields\n * are transformed from screenshot space to logical space.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any> | undefined,\n  zodSchema?: z.ZodType<any>,\n  options?: { shrunkShotToLogicalRatio?: number },\n): Record<string, any> | undefined => {\n  // If no schema is provided, return undefined (action takes no parameters)\n  if (!zodSchema) {\n    return undefined;\n  }\n\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement),\n  // and transform coordinates from screenshot space to logical space if needed\n  const ratio = options?.shrunkShotToLogicalRatio;\n  for (const fieldName in locateFieldValues) {\n    let value = locateFieldValues[fieldName];\n    if (\n      ratio !== undefined &&\n      value &&\n      typeof value === 'object' &&\n      value.center\n    ) {\n      value = {\n        ...value,\n        center: [\n          Math.round(value.center[0] / ratio),\n          Math.round(value.center[1] / ratio),\n        ],\n      };\n    }\n    validated[fieldName] = value;\n  }\n\n  return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/**\n * Get a readable time string for a given timestamp or the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n  format = 'YYYY-MM-DD HH:mm:ss',\n  timestamp?: number,\n): string => {\n  const now = timestamp !== undefined ? new Date(timestamp) : new Date();\n  const year = now.getFullYear();\n  const month = String(now.getMonth() + 1).padStart(2, '0');\n  const day = String(now.getDate()).padStart(2, '0');\n  const hours = String(now.getHours()).padStart(2, '0');\n  const minutes = String(now.getMinutes()).padStart(2, '0');\n  const seconds = String(now.getSeconds()).padStart(2, '0');\n\n  const timeString = format\n    .replace('YYYY', String(year))\n    .replace('MM', month)\n    .replace('DD', day)\n    .replace('HH', hours)\n    .replace('mm', minutes)\n    .replace('ss', seconds);\n\n  return `${timeString} (${format})`;\n};\n"],"names":["findActionInActionSpaceOrThrow","planType","actionSpace","action","item","available","Error","buildYamlFlowFromPlans","plans","flow","plan","verb","flowKey","flowParam","dumpActionParam","shortcutField","undefined","shortcutKeys","Object","canInlineShortcut","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","userPromptToString","prompt","userPromptToMultimodalPrompt","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","formatPromptWithImages","promptObj","promptString","Array","imageCount","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","key","jsonObject","zodSchema","isPlainObject","locatorFields","result","fieldName","fieldValue","parseActionParam","rawParam","options","param","locateFields","locateFieldValues","paramsForValidation","validated","ratio","value","Math","finalizeActionName","getReadableTimeString","format","timestamp","now","Date","year","month","String","day","hours","minutes","seconds","timeString"],"mappings":";;AAQO,SAASA,+BACdC,QAAgB,EAChBC,WAAgC;IAEhC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;IACxD,IAAI,CAACE,QAAQ;QACX,MAAME,YAAYH,YAAY,GAAG,CAAC,CAACE,OAASA,KAAK,IAAI,EAAE,IAAI,CAAC;QAC5D,MAAM,IAAIE,MACR,CAAC,aAAa,EAAEL,SAAS,yDAAyD,EAAEI,aAAa,UAAU;IAE/G;IACA,OAAOF;AACT;AAEO,SAASI,uBACdC,KAAuB,EACvBN,WAAgC;IAEhC,MAAMO,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQF,MAAO;QACxB,MAAMG,OAAOD,KAAK,IAAI;QAEtB,MAAMP,SAASH,+BAA+BW,MAAMT;QAEpD,MAAMU,UAAUT,OAAO,cAAc,IAAIQ;QACzC,MAAME,YAAYV,OAAO,WAAW,GAChCW,gBAAgBJ,KAAK,KAAK,IAAI,CAAC,GAAGP,OAAO,WAAW,IACpD,CAAC;QAOL,MAAMY,gBACJZ,AAAgB,aAAhBA,OAAO,IAAI,IAAiBA,AAA0B,aAA1BA,OAAO,cAAc,GAC7C,QACAA,AAAgB,gBAAhBA,OAAO,IAAI,IAAoBA,AAA0B,gBAA1BA,OAAO,cAAc,GAClD,QACAA,AAAgB,kBAAhBA,OAAO,IAAI,IACTA,AAA0B,kBAA1BA,OAAO,cAAc,IACrBA,AAAgB,kBAAhBA,OAAO,IAAI,IACXA,AAA0B,kBAA1BA,OAAO,cAAc,GACrB,YACAa;QACV,MAAMC,eAAeF,gBAAgBG,OAAO,IAAI,CAACL,aAAa,EAAE;QAChE,MAAMM,oBACJJ,iBACAE,AAAwB,MAAxBA,aAAa,MAAM,IACnBA,YAAY,CAAC,EAAE,KAAKF,iBACpB,AAAoC,YAApC,OAAOF,SAAS,CAACE,cAAc;QAEjC,MAAMK,WAAiCD,oBACnC;YAAE,CAACP,QAAQ,EAAEC,SAAS,CAACE,cAAwB;QAAC,IAChD;YAAE,CAACH,QAAQ,EAAE;YAAI,GAAGC,SAAS;QAAC;QAElCJ,KAAK,IAAI,CAACW;IACZ;IAEA,OAAOX;AACT;AAGO,MAAMY,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;AAClB;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMM,MAAME,qBAAqB,CAACC,SAC1B,AAAkB,YAAlB,OAAOA,SAAsBA,SAASA,OAAO,MAAM;AAGrD,MAAMC,+BAA+B,CAC1CD;IAEA,IAAI,AAAkB,YAAlB,OAAOA,UAAuB,CAACA,OAAO,MAAM,EAC9C;IAEF,OAAO;QACL,QAAQA,OAAO,MAAM;QACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;IAC3D;AACF;AAGA,MAAME,wBAAwBR,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,YAAYJ,EAAE,OAAO,GAAG,QAAQ;IAChC,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAMP,MAAMS,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAI,YAAYC,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEA,MAAMC,yBAAyB,CAC7BC;IAEA,IAAIC,eAAeD,UAAU,MAAM;IACnC,IAAIE,MAAM,OAAO,CAACF,UAAU,MAAM,KAAKA,UAAU,MAAM,CAAC,MAAM,GAAG,GAAG;QAClE,MAAMG,aAAaH,UAAU,MAAM,CAAC,MAAM;QAC1CC,gBAAgB,CAAC,OAAO,EAAEE,WAAW,MAAM,EAAEA,aAAa,IAAI,MAAM,GAAG,CAAC,CAAC;IAC3E;IACA,OAAOF;AACT;AAEO,MAAMG,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAO3B,OAAO,IAAI,CAAC0B,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACC;YAClB,MAAMb,QAAQW,UAAU,KAAK,CAACE,IAAI;YAGlC,IAAI,CAACd,uBAAuBC,QAC1B,OAAO;YAIT,IAAIU,cACF,OAAOV,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMnB,kBAAkB,CAC7BiC,YACAC;IAGA,IAAI,CAACC,cAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBT,4BAA4BO;IAClD,MAAMG,SAAS;QAAE,GAAGJ,UAAU;IAAC;IAE/B,KAAK,MAAMK,aAAaF,cAAe;QACrC,MAAMG,aAAaF,MAAM,CAACC,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTF,MAAM,CAACC,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BF,MAAM,CAACC,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBF,MAAM,CAACC,UAAU,GAAGhB,uBAAuBiB,WAAW,MAAM;gBAC9D;YACF;QACF;IAEJ;IAEA,OAAOF;AACT;AAaO,MAAMG,mBAAmB,CAC9BC,UACAP,WACAQ;IAGA,IAAI,CAACR,WACH;IAIF,MAAMS,QAAQF,YAAY,CAAC;IAG3B,MAAMG,eAAejB,4BAA4BO;IAGjD,IAAIU,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOV,UAAU,KAAK,CAACS;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMd,OAAOW,MAChB,IAAIC,aAAa,QAAQ,CAACZ,MAExBc,mBAAmB,CAACd,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/Cc,mBAAmB,CAACd,IAAI,GAAGW,KAAK,CAACX,IAAI;IAKzC,MAAMe,YAAYb,UAAU,KAAK,CAACY;IAIlC,MAAME,QAAQN,SAAS;IACvB,IAAK,MAAMJ,aAAaO,kBAAmB;QACzC,IAAII,QAAQJ,iBAAiB,CAACP,UAAU;QACxC,IACEU,AAAU9C,WAAV8C,SACAC,SACA,AAAiB,YAAjB,OAAOA,SACPA,MAAM,MAAM,EAEZA,QAAQ;YACN,GAAGA,KAAK;YACR,QAAQ;gBACNC,KAAK,KAAK,CAACD,MAAM,MAAM,CAAC,EAAE,GAAGD;gBAC7BE,KAAK,KAAK,CAACD,MAAM,MAAM,CAAC,EAAE,GAAGD;aAC9B;QACH;QAEFD,SAAS,CAACT,UAAU,GAAGW;IACzB;IAEA,OAAOF;AACT;AAEO,MAAMI,qBAAqB;AAQ3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB,EAC9BC;IAEA,MAAMC,MAAMD,AAAcpD,WAAdoD,YAA0B,IAAIE,KAAKF,aAAa,IAAIE;IAChE,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQC,OAAOJ,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMK,MAAMD,OAAOJ,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMM,QAAQF,OAAOJ,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMO,UAAUH,OAAOJ,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMQ,UAAUJ,OAAOJ,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMS,aAAaX,OAChB,OAAO,CAAC,QAAQM,OAAOF,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAME,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAEX,OAAO,CAAC,CAAC;AACpC"}
+{"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n} from '@/types';\nimport { isPlainObject } from '@midscene/shared/utils';\nimport { z } from 'zod';\n\nexport function findActionInActionSpaceOrThrow(\n  planType: string,\n  actionSpace: DeviceAction<any>[],\n): DeviceAction<any> {\n  const action = actionSpace.find((item) => item.name === planType);\n  if (!action) {\n    const available = actionSpace.map((item) => item.name).join(', ');\n    throw new Error(\n      `Action type '${planType}' is not in the current action space. Available actions: ${available || '(none)'}`,\n    );\n  }\n  return action;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = findActionInActionSpaceOrThrow(verb, actionSpace);\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    // For actions whose param is a single string field (e.g. Launch/Terminate's\n    // `uri`, RunAdbShell's `command`), inline the value on the flowKey. Writing\n    // `{ terminate: '', uri: '...' }` makes the YAML player treat the empty\n    // string as the param and drop the sibling `uri`, so cache replay would\n    // call the action with an empty argument.\n    const shortcutField =\n      action.name === 'Launch' || action.interfaceAlias === 'launch'\n        ? 'uri'\n        : action.name === 'Terminate' || action.interfaceAlias === 'terminate'\n          ? 'uri'\n          : action.name === 'RunAdbShell' ||\n              action.interfaceAlias === 'runAdbShell' ||\n              action.name === 'RunHdcShell' ||\n              action.interfaceAlias === 'runHdcShell'\n            ? 'command'\n            : undefined;\n    const shortcutKeys = shortcutField ? Object.keys(flowParam) : [];\n    const canInlineShortcut =\n      shortcutField &&\n      shortcutKeys.length === 1 &&\n      shortcutKeys[0] === shortcutField &&\n      typeof flowParam[shortcutField] === 'string';\n\n    const flowItem: MidsceneYamlFlowItem = canInlineShortcut\n      ? { [flowKey]: flowParam[shortcutField as string] }\n      : { [flowKey]: '', ...flowParam };\n\n    flow.push(flowItem);\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nexport const userPromptToString = (prompt: TUserPrompt): string => {\n  return typeof prompt === 'string' ? prompt : prompt.prompt;\n};\n\nexport const userPromptToMultimodalPrompt = (\n  prompt: TUserPrompt,\n): TMultimodalPrompt | undefined => {\n  if (typeof prompt === 'string' || !prompt.images) {\n    return undefined;\n  }\n  return {\n    images: prompt.images,\n    convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n  };\n};\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepLocate: z.boolean().optional(),\n    deepThink: z\n      .boolean()\n      .optional()\n      .describe('@deprecated Use `deepLocate` instead.'),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nconst formatPromptWithImages = (\n  promptObj: Exclude<TUserPrompt, string>,\n): string => {\n  let promptString = promptObj.prompt;\n  if (Array.isArray(promptObj.images) && promptObj.images.length > 0) {\n    const imageCount = promptObj.images.length;\n    promptString += ` (with ${imageCount} image${imageCount > 1 ? 's' : ''})`;\n  }\n  return promptString;\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      // TODO: A similar locator-field check exists as isMidsceneLocatorField in\n      // @midscene/shared/zod-schema-utils.\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport function validateRequiredLocateFields(\n  param: Record<string, any> | undefined,\n  schema: z.ZodType<any> | undefined,\n): void {\n  for (const field of findAllMidsceneLocatorField(schema, true)) {\n    if (!param?.[field]) {\n      throw new Error(`${field}: Required locate field is not provided`);\n    }\n  }\n}\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n  if (!isPlainObject(jsonObject)) {\n    return {};\n  }\n\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = formatPromptWithImages(fieldValue.prompt);\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n *\n * When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields\n * are transformed from screenshot space to logical space.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any> | undefined,\n  zodSchema?: z.ZodType<any>,\n  options?: { shrunkShotToLogicalRatio?: number },\n): Record<string, any> | undefined => {\n  // If no schema is provided, return undefined (action takes no parameters)\n  if (!zodSchema) {\n    return undefined;\n  }\n\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement),\n  // and transform coordinates from screenshot space to logical space if needed\n  const ratio = options?.shrunkShotToLogicalRatio;\n  for (const fieldName in locateFieldValues) {\n    let value = locateFieldValues[fieldName];\n    if (\n      ratio !== undefined &&\n      value &&\n      typeof value === 'object' &&\n      value.center\n    ) {\n      value = {\n        ...value,\n        center: [\n          Math.round(value.center[0] / ratio),\n          Math.round(value.center[1] / ratio),\n        ],\n      };\n    }\n    validated[fieldName] = value;\n  }\n\n  return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/**\n * Get a readable time string for a given timestamp or the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n  format = 'YYYY-MM-DD HH:mm:ss',\n  timestamp?: number,\n): string => {\n  const now = timestamp !== undefined ? new Date(timestamp) : new Date();\n  const year = now.getFullYear();\n  const month = String(now.getMonth() + 1).padStart(2, '0');\n  const day = String(now.getDate()).padStart(2, '0');\n  const hours = String(now.getHours()).padStart(2, '0');\n  const minutes = String(now.getMinutes()).padStart(2, '0');\n  const seconds = String(now.getSeconds()).padStart(2, '0');\n\n  const timeString = format\n    .replace('YYYY', String(year))\n    .replace('MM', month)\n    .replace('DD', day)\n    .replace('HH', hours)\n    .replace('mm', minutes)\n    .replace('ss', seconds);\n\n  return `${timeString} (${format})`;\n};\n"],"names":["findActionInActionSpaceOrThrow","planType","actionSpace","action","item","available","Error","buildYamlFlowFromPlans","plans","flow","plan","verb","flowKey","flowParam","dumpActionParam","shortcutField","undefined","shortcutKeys","Object","canInlineShortcut","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","userPromptToString","prompt","userPromptToMultimodalPrompt","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","formatPromptWithImages","promptObj","promptString","Array","imageCount","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","key","validateRequiredLocateFields","param","schema","jsonObject","zodSchema","isPlainObject","locatorFields","result","fieldName","fieldValue","parseActionParam","rawParam","options","locateFields","locateFieldValues","paramsForValidation","validated","ratio","value","Math","finalizeActionName","getReadableTimeString","format","timestamp","now","Date","year","month","String","day","hours","minutes","seconds","timeString"],"mappings":";;AAQO,SAASA,+BACdC,QAAgB,EAChBC,WAAgC;IAEhC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;IACxD,IAAI,CAACE,QAAQ;QACX,MAAME,YAAYH,YAAY,GAAG,CAAC,CAACE,OAASA,KAAK,IAAI,EAAE,IAAI,CAAC;QAC5D,MAAM,IAAIE,MACR,CAAC,aAAa,EAAEL,SAAS,yDAAyD,EAAEI,aAAa,UAAU;IAE/G;IACA,OAAOF;AACT;AAEO,SAASI,uBACdC,KAAuB,EACvBN,WAAgC;IAEhC,MAAMO,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQF,MAAO;QACxB,MAAMG,OAAOD,KAAK,IAAI;QAEtB,MAAMP,SAASH,+BAA+BW,MAAMT;QAEpD,MAAMU,UAAUT,OAAO,cAAc,IAAIQ;QACzC,MAAME,YAAYV,OAAO,WAAW,GAChCW,gBAAgBJ,KAAK,KAAK,IAAI,CAAC,GAAGP,OAAO,WAAW,IACpD,CAAC;QAOL,MAAMY,gBACJZ,AAAgB,aAAhBA,OAAO,IAAI,IAAiBA,AAA0B,aAA1BA,OAAO,cAAc,GAC7C,QACAA,AAAgB,gBAAhBA,OAAO,IAAI,IAAoBA,AAA0B,gBAA1BA,OAAO,cAAc,GAClD,QACAA,AAAgB,kBAAhBA,OAAO,IAAI,IACTA,AAA0B,kBAA1BA,OAAO,cAAc,IACrBA,AAAgB,kBAAhBA,OAAO,IAAI,IACXA,AAA0B,kBAA1BA,OAAO,cAAc,GACrB,YACAa;QACV,MAAMC,eAAeF,gBAAgBG,OAAO,IAAI,CAACL,aAAa,EAAE;QAChE,MAAMM,oBACJJ,iBACAE,AAAwB,MAAxBA,aAAa,MAAM,IACnBA,YAAY,CAAC,EAAE,KAAKF,iBACpB,AAAoC,YAApC,OAAOF,SAAS,CAACE,cAAc;QAEjC,MAAMK,WAAiCD,oBACnC;YAAE,CAACP,QAAQ,EAAEC,SAAS,CAACE,cAAwB;QAAC,IAChD;YAAE,CAACH,QAAQ,EAAE;YAAI,GAAGC,SAAS;QAAC;QAElCJ,KAAK,IAAI,CAACW;IACZ;IAEA,OAAOX;AACT;AAGO,MAAMY,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;AAClB;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMM,MAAME,qBAAqB,CAACC,SAC1B,AAAkB,YAAlB,OAAOA,SAAsBA,SAASA,OAAO,MAAM;AAGrD,MAAMC,+BAA+B,CAC1CD;IAEA,IAAI,AAAkB,YAAlB,OAAOA,UAAuB,CAACA,OAAO,MAAM,EAC9C;IAEF,OAAO;QACL,QAAQA,OAAO,MAAM;QACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;IAC3D;AACF;AAGA,MAAME,wBAAwBR,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,YAAYJ,EAAE,OAAO,GAAG,QAAQ;IAChC,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAMP,MAAMS,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAI,YAAYC,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEA,MAAMC,yBAAyB,CAC7BC;IAEA,IAAIC,eAAeD,UAAU,MAAM;IACnC,IAAIE,MAAM,OAAO,CAACF,UAAU,MAAM,KAAKA,UAAU,MAAM,CAAC,MAAM,GAAG,GAAG;QAClE,MAAMG,aAAaH,UAAU,MAAM,CAAC,MAAM;QAC1CC,gBAAgB,CAAC,OAAO,EAAEE,WAAW,MAAM,EAAEA,aAAa,IAAI,MAAM,GAAG,CAAC,CAAC;IAC3E;IACA,OAAOF;AACT;AAEO,MAAMG,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAO3B,OAAO,IAAI,CAAC0B,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACC;YAClB,MAAMb,QAAQW,UAAU,KAAK,CAACE,IAAI;YAGlC,IAAI,CAACd,uBAAuBC,QAC1B,OAAO;YAIT,IAAIU,cACF,OAAOV,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,SAASc,6BACdC,KAAsC,EACtCC,MAAkC;IAElC,KAAK,MAAMhB,SAASQ,4BAA4BQ,QAAQ,MACtD,IAAI,CAACD,OAAO,CAACf,MAAM,EACjB,MAAM,IAAI3B,MAAM,GAAG2B,MAAM,uCAAuC,CAAC;AAGvE;AAEO,MAAMnB,kBAAkB,CAC7BoC,YACAC;IAGA,IAAI,CAACC,cAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBZ,4BAA4BU;IAClD,MAAMG,SAAS;QAAE,GAAGJ,UAAU;IAAC;IAE/B,KAAK,MAAMK,aAAaF,cAAe;QACrC,MAAMG,aAAaF,MAAM,CAACC,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTF,MAAM,CAACC,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BF,MAAM,CAACC,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBF,MAAM,CAACC,UAAU,GAAGnB,uBAAuBoB,WAAW,MAAM;gBAC9D;YACF;QACF;IAEJ;IAEA,OAAOF;AACT;AAaO,MAAMG,mBAAmB,CAC9BC,UACAP,WACAQ;IAGA,IAAI,CAACR,WACH;IAIF,MAAMH,QAAQU,YAAY,CAAC;IAG3B,MAAME,eAAenB,4BAA4BU;IAGjD,IAAIS,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOT,UAAU,KAAK,CAACH;IAIzB,MAAMa,oBAAyC,CAAC;IAChD,KAAK,MAAMN,aAAaK,aACtB,IAAIL,aAAaP,OACfa,iBAAiB,CAACN,UAAU,GAAGP,KAAK,CAACO,UAAU;IAKnD,MAAMO,sBAA2C,CAAC;IAClD,IAAK,MAAMhB,OAAOE,MAChB,IAAIY,aAAa,QAAQ,CAACd,MAExBgB,mBAAmB,CAAChB,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/CgB,mBAAmB,CAAChB,IAAI,GAAGE,KAAK,CAACF,IAAI;IAKzC,MAAMiB,YAAYZ,UAAU,KAAK,CAACW;IAIlC,MAAME,QAAQL,SAAS;IACvB,IAAK,MAAMJ,aAAaM,kBAAmB;QACzC,IAAII,QAAQJ,iBAAiB,CAACN,UAAU;QACxC,IACES,AAAUhD,WAAVgD,SACAC,SACA,AAAiB,YAAjB,OAAOA,SACPA,MAAM,MAAM,EAEZA,QAAQ;YACN,GAAGA,KAAK;YACR,QAAQ;gBACNC,KAAK,KAAK,CAACD,MAAM,MAAM,CAAC,EAAE,GAAGD;gBAC7BE,KAAK,KAAK,CAACD,MAAM,MAAM,CAAC,EAAE,GAAGD;aAC9B;QACH;QAEFD,SAAS,CAACR,UAAU,GAAGU;IACzB;IAEA,OAAOF;AACT;AAEO,MAAMI,qBAAqB;AAQ3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB,EAC9BC;IAEA,MAAMC,MAAMD,AAActD,WAAdsD,YAA0B,IAAIE,KAAKF,aAAa,IAAIE;IAChE,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQC,OAAOJ,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMK,MAAMD,OAAOJ,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMM,QAAQF,OAAOJ,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMO,UAAUH,OAAOJ,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMQ,UAAUJ,OAAOJ,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMS,aAAaX,OAChB,OAAO,CAAC,QAAQM,OAAOF,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAME,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAEX,OAAO,CAAC,CAAC;AACpC"}

package/dist/es/device/index.mjs

@@ -12,16 +12,6 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-function resolveSwipeInputPrimitive(input) {
-    if (input.touch?.swipe) return {
-        swipe: input.touch.swipe,
-        inputMode: 'touch'
-    };
-    if (input.pointer?.swipe) return {
-        swipe: input.pointer.swipe,
-        inputMode: 'mouse'
-    };
-}
 class AbstractInterface {
     constructor(){
         _define_property(this, "mjpegStreamUrl", void 0);
@@ -217,7 +207,7 @@ const actionScrollParamSchema = z.object({
 });
 const defineActionScroll = (scroll)=>defineAction({
         name: 'Scroll',
-        description: 'Scroll a page or scrollable region to reveal off-screen content. Use Scroll when the goal is to browse content outside the current viewport. For direct gesture interactions, such as adjusting a slider or wheel picker, switching between paged cards or images, following an on-screen swipe gesture to continue or dismiss, or swiping an item to delete it, use Swipe instead. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',
+        description: 'Scroll a page or scrollable region to reveal off-screen content. Use Scroll when the goal is to browse content outside the current viewport. For direct gesture interactions, such as adjusting a slider or wheel picker, switching between paged cards or images, following an on-screen swipe gesture to continue or dismiss, or swiping an item to delete it, use Swipe instead if available in the current Action Space. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',
         interfaceAlias: 'aiScroll',
         paramSchema: actionScrollParamSchema,
         sample: {
@@ -283,24 +273,20 @@ const defineActionLongPress = (longPress)=>defineLocatedPointAction({
             });
         }
     });
-function createActionSwipeParamSchema(inputMode) {
-    const movementSource = 'touch' === inputMode ? 'finger' : 'pointer';
-    return z.object({
-        start: getMidsceneLocationSchema().optional().describe(`Optional starting point of the ${movementSource} movement. Available in both relative and endpoint forms. If omitted, the center of the page is used.`),
-        direction: z["enum"]([
-            'up',
-            'down',
-            'left',
-            'right'
-        ]).optional().describe(`${'finger' === movementSource ? 'Finger' : 'Pointer'} movement direction. Required together with a positive distance for a relative swipe. Omit when using end.`),
-        distance: z.number().positive().optional().describe(`Positive length of the ${movementSource} movement in pixels. Required together with direction for a relative swipe. Omit when using end.`),
-        end: getMidsceneLocationSchema().optional().describe(`Endpoint of the ${movementSource} movement. Use for an endpoint swipe, optionally with start. Do not provide direction or distance when using end.`),
-        duration: z.number().default(300).describe('Duration of the swipe gesture in milliseconds'),
-        repeat: z.number().optional().describe('The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)')
-    });
-}
-const ActionSwipeParamSchema = createActionSwipeParamSchema('touch');
-function normalizeSwipeParam(param, screenSize) {
+const ActionSwipeParamSchema = z.object({
+    start: getMidsceneLocationSchema().optional().describe('Optional starting point of the finger movement. Available in both relative and endpoint forms. If omitted, the center of the page is used.'),
+    direction: z["enum"]([
+        'up',
+        'down',
+        'left',
+        'right'
+    ]).optional().describe('Finger movement direction. Required together with a positive distance for a relative swipe. Omit when using end.'),
+    distance: z.number().positive().optional().describe('Positive length of the finger movement in pixels. Required together with direction for a relative swipe. Omit when using end.'),
+    end: getMidsceneLocationSchema().optional().describe('Endpoint of the finger movement. Use for an endpoint swipe, optionally with start. Do not provide direction or distance when using end.'),
+    duration: z.number().default(300).describe('Duration of the swipe gesture in milliseconds'),
+    repeat: z.number().optional().describe('The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)')
+});
+function normalizeMobileSwipeParam(param, screenSize) {
     const { width, height } = screenSize;
     const { start, end } = param;
     const startPoint = start ? {
@@ -337,14 +323,10 @@ function normalizeSwipeParam(param, screenSize) {
         repeatCount
     };
 }
-const normalizeMobileSwipeParam = normalizeSwipeParam;
-const defineActionSwipe = (config)=>{
-    const inputMode = config.inputMode ?? 'touch';
-    return defineAction({
+const defineActionSwipe = (config)=>defineAction({
         name: 'Swipe',
-        description: 'touch' === inputMode ? 'Perform a touch gesture that directly manipulates the UI (e.g., adjust a continuous control such as a slider or wheel picker, switch between paged cards or images, follow an on-screen swipe gesture to continue or dismiss, or swipe an item to delete it). For browsing off-screen content in a page or scrollable region, use Scroll instead. Choose exactly one movement form: (1) relative swipe — provide "direction" and a positive "distance"; or (2) endpoint swipe — provide "end". "start" is optional for both forms and defaults to the center of the page. Do not combine "end" with "direction" or "distance".' : 'Perform a primary-mouse-button gesture that continuously presses and moves the pointer across the desktop UI. Use it to adjust a continuous control such as a slider or wheel picker, switch between paged cards or images, follow an on-screen swipe gesture to continue or dismiss, or swipe an item to delete it. For browsing off-screen content in a page or scrollable region, use Scroll instead. Choose exactly one movement form: (1) relative swipe — provide "direction" and a positive "distance"; or (2) endpoint swipe — provide "end". "start" is optional for both forms and defaults to the center of the page. Do not combine "end" with "direction" or "distance".',
-        interfaceAlias: 'aiSwipe',
-        paramSchema: 'touch' === inputMode ? ActionSwipeParamSchema : createActionSwipeParamSchema(inputMode),
+        description: 'Perform a touch gesture that directly manipulates the UI (e.g., adjust a continuous control such as a slider or wheel picker, switch between paged cards or images, follow an on-screen swipe gesture to continue or dismiss, or swipe an item to delete it). For browsing off-screen content in a page or scrollable region, use Scroll instead. Choose exactly one movement form: (1) relative swipe — provide "direction" and a positive "distance"; or (2) endpoint swipe — provide "end". "start" is optional for both forms and defaults to the center of the page. Do not combine "end" with "direction" or "distance".',
+        paramSchema: ActionSwipeParamSchema,
         sample: {
             start: {
                 prompt: 'center of the notification'
@@ -354,13 +336,12 @@ const defineActionSwipe = (config)=>{
             }
         },
         call: async (param)=>{
-            const { startPoint, endPoint, duration, repeatCount } = normalizeSwipeParam(param, await config.size());
+            const { startPoint, endPoint, duration, repeatCount } = normalizeMobileSwipeParam(param, await config.size());
             for(let i = 0; i < repeatCount; i++)await config.swipe(startPoint, endPoint, {
                 duration
             });
         }
     });
-};
 const actionClearInputParamSchema = z.object({
     locate: getMidsceneLocationSchema().describe('The input field to be cleared').optional()
 });
@@ -484,12 +465,8 @@ function defineActionsFromInputPrimitives(input, options = {}) {
         sleep: options.sleep
     }));
     if (scroll) actions.push(defineActionScroll(scroll.scroll));
-    const swipeConfig = resolveSwipeInputPrimitive({
-        pointer,
-        touch
-    });
-    if (swipeConfig && options.size && false !== options.includeSwipe) actions.push(defineActionSwipe({
-        ...swipeConfig,
+    if (touch?.swipe && options.size && false !== options.includeSwipe) actions.push(defineActionSwipe({
+        swipe: touch.swipe,
         size: options.size
     }));
     if (touch?.pinch && options.size && false !== options.includePinch) actions.push(defineActionPinch({
@@ -527,6 +504,6 @@ const defineActionSleep = ()=>defineAction({
             await new Promise((resolve)=>setTimeout(resolve, duration));
         }
     });
-export { AbstractInterface, ActionLongPressParamSchema, ActionPinchParamSchema, ActionSleepParamSchema, ActionSwipeParamSchema, actionClearInputParamSchema, actionCursorMoveParamSchema, actionDoubleClickParamSchema, actionDragAndDropParamSchema, actionHoverParamSchema, actionInputParamSchema, actionKeyboardPressParamSchema, actionRightClickParamSchema, actionScrollParamSchema, actionTapParamSchema, createDefaultMobileActions, defineAction, defineActionClearInput, defineActionCursorMove, defineActionDoubleClick, defineActionDragAndDrop, defineActionHover, defineActionInput, defineActionKeyboardPress, defineActionLongPress, defineActionPinch, defineActionRegisterFileChooserAccept, defineActionRightClick, defineActionScroll, defineActionSleep, defineActionSwipe, defineActionTap, defineActionsFromInputPrimitives, inputStrategies, normalizeMobileSwipeParam, normalizePinchParam, normalizeSwipeParam, registerFileChooserAcceptParamSchema, resolveInputStrategy, resolveSwipeInputPrimitive, resolveTextInputOptions, sendTextSequentially, shouldInputSequentially };
+export { AbstractInterface, ActionLongPressParamSchema, ActionPinchParamSchema, ActionSleepParamSchema, ActionSwipeParamSchema, actionClearInputParamSchema, actionCursorMoveParamSchema, actionDoubleClickParamSchema, actionDragAndDropParamSchema, actionHoverParamSchema, actionInputParamSchema, actionKeyboardPressParamSchema, actionRightClickParamSchema, actionScrollParamSchema, actionTapParamSchema, createDefaultMobileActions, defineAction, defineActionClearInput, defineActionCursorMove, defineActionDoubleClick, defineActionDragAndDrop, defineActionHover, defineActionInput, defineActionKeyboardPress, defineActionLongPress, defineActionPinch, defineActionRegisterFileChooserAccept, defineActionRightClick, defineActionScroll, defineActionSleep, defineActionSwipe, defineActionTap, defineActionsFromInputPrimitives, inputStrategies, normalizeMobileSwipeParam, normalizePinchParam, registerFileChooserAcceptParamSchema, resolveInputStrategy, resolveTextInputOptions, sendTextSequentially, shouldInputSequentially };
 
 //# sourceMappingURL=index.mjs.map