@midscene/core 1.17.4-beta-20261030030502.0 → 1.17.4

package/dist/es/agent/test-runner-nodes.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/test-runner-nodes.mjs","sources":["../../../src/agent/test-runner-nodes.ts"],"sourcesContent":["import { AGENT_AI_CONTEXT_KEYS } from '@midscene/shared/agent-tools/agent-context';\nimport { z } from 'zod/v4';\nimport type { TUserPrompt } from '../common';\nimport { inputStrategies } from '../device/input-strategy';\nimport { NodeExecutionError } from '../test-runner/errors';\nimport type { LocateOption, ScrollParam } from '../yaml';\nimport { buildDetailedLocateParam } from '../yaml/utils';\nimport type { Agent } from './agent';\n\nexport interface AgentTestRunnerNodeResult<TData = unknown> {\n  summary?: string;\n  data?: TData;\n}\n\nexport interface AgentTestRunnerNodeExecutionContext {\n  signal: AbortSignal;\n}\n\n/** Agent-owned Node description consumed by Test Runner adapters. */\nexport interface AgentTestRunnerNodeDefinition<\n  TInput extends Record<string, unknown> = Record<string, unknown>,\n  TData = unknown,\n> {\n  name: string;\n  title?: string;\n  description?: string;\n  /** Node input field populated by YAML string shorthand; omit to disable. */\n  stringInputKey?: string | false;\n  inputSchema: z.ZodObject;\n  execute(\n    agent: unknown,\n    input: TInput,\n    context: AgentTestRunnerNodeExecutionContext,\n  ):\n    | Promise<AgentTestRunnerNodeResult<TData> | undefined>\n    | AgentTestRunnerNodeResult<TData>\n    | undefined;\n}\n\n/** Static capability implemented by Agent classes that expose Runner Nodes. */\nexport interface AgentTestRunnerNodeProvider {\n  getTestRunnerNodeDefinitions(): readonly AgentTestRunnerNodeDefinition[];\n}\n\ntype AnyAgentMethod = (...args: any[]) => unknown;\ntype AgentMethodName<TAgent extends object> = {\n  [TKey in keyof TAgent]-?: TAgent[TKey] extends AnyAgentMethod ? TKey : never;\n}[keyof TAgent] &\n  string;\ntype AgentMethod<\n  TAgent extends object,\n  TMethod extends AgentMethodName<TAgent>,\n> = Extract<TAgent[TMethod], AnyAgentMethod>;\n\nexport interface DefineAgentTestRunnerNodeOptions<\n  TAgent extends object,\n  TMethod extends AgentMethodName<TAgent>,\n  TSchema extends z.ZodObject,\n  TData = unknown,\n> {\n  method: TMethod;\n  name?: string;\n  title?: string;\n  description?: string;\n  stringInputKey?: (keyof z.output<TSchema> & string) | false;\n  inputSchema: TSchema;\n  toArgs(\n    input: z.output<TSchema>,\n    context: AgentTestRunnerNodeExecutionContext,\n  ): Parameters<AgentMethod<TAgent, TMethod>>;\n  toResult?(\n    result: Awaited<ReturnType<AgentMethod<TAgent, TMethod>>>,\n    input: z.output<TSchema>,\n  ): AgentTestRunnerNodeResult<TData> | undefined;\n}\n\n/** Create a Node factory whose adapters are checked against an Agent API. */\nexport const createAgentTestRunnerNodeDefinition =\n  <TAgent extends object>(agentLabel = 'an Agent') =>\n  <\n    TMethod extends AgentMethodName<TAgent>,\n    TSchema extends z.ZodObject,\n    TData = unknown,\n  >(\n    definition: DefineAgentTestRunnerNodeOptions<\n      TAgent,\n      TMethod,\n      TSchema,\n      TData\n    >,\n  ): AgentTestRunnerNodeDefinition => ({\n    name: definition.name ?? definition.method,\n    ...(definition.title === undefined ? {} : { title: definition.title }),\n    ...(definition.description === undefined\n      ? {}\n      : { description: definition.description }),\n    ...(definition.stringInputKey === undefined\n      ? {}\n      : { stringInputKey: definition.stringInputKey }),\n    inputSchema: definition.inputSchema,\n    async execute(agent, input, context) {\n      if (\n        typeof agent !== 'object' ||\n        agent === null ||\n        typeof (agent as Record<string, unknown>)[definition.method] !==\n          'function'\n      ) {\n        throw new TypeError(\n          `getAgent() must return ${agentLabel} with ${definition.method}().`,\n        );\n      }\n\n      const typedAgent = agent as TAgent;\n      const method = typedAgent[definition.method] as AgentMethod<\n        TAgent,\n        TMethod\n      >;\n      const typedInput = input as z.output<TSchema>;\n      const result = (await method.apply(\n        typedAgent,\n        definition.toArgs(typedInput, context),\n      )) as Awaited<ReturnType<AgentMethod<TAgent, TMethod>>>;\n      return definition.toResult?.(result, typedInput);\n    },\n  });\n\nconst nonBlankText = (description: string) =>\n  z\n    .string()\n    .regex(/\\S/, 'value must contain a non-whitespace character')\n    .describe(description);\n\nexport const promptImageInputSchema = z.strictObject({\n  name: nonBlankText('The name used to identify this reference image.'),\n  url: nonBlankText('The URL, data URL, or file path of this reference image.'),\n});\n\nexport const structuredUserPromptInputSchema = z.strictObject({\n  prompt: nonBlankText('The natural-language prompt.'),\n  images: z.array(promptImageInputSchema).min(1).optional(),\n  convertHttpImage2Base64: z\n    .boolean()\n    .optional()\n    .describe('Whether HTTP reference images are converted to base64 first.'),\n});\n\nexport const userPromptInputSchema = z.union([\n  nonBlankText('The natural-language prompt.'),\n  structuredUserPromptInputSchema,\n]);\n\nexport const aiActOptionsInputSchema = z.strictObject({\n  effort: z\n    .enum(['fast', 'balance', 'deepThink'])\n    .optional()\n    .describe('Action planning effort: fast, balanced, or deeper reasoning.'),\n  cacheable: z\n    .boolean()\n    .optional()\n    .describe('Whether this action may use the Midscene cache.'),\n  fileChooserAccept: z\n    .union([z.string(), z.array(z.string())])\n    .optional()\n    .describe('Accepted file types for a file chooser.'),\n  fileChooserAllowedDir: z\n    .string()\n    .optional()\n    .describe('Directory allowed for prompt-driven file uploads.'),\n  deepThink: z\n    .union([z.literal('unset'), z.boolean()])\n    .optional()\n    .describe('Whether to enable deep thinking for this action.'),\n  deepLocate: z\n    .boolean()\n    .optional()\n    .describe('Whether to use deep element location.'),\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.',\n    ),\n});\n\nexport const setAIContextInputSchema = z.strictObject({\n  target: z\n    .enum(AGENT_AI_CONTEXT_KEYS)\n    .describe(\n      'The AI API to configure, e.g. aiAct or aiAssert. Use default as the shared fallback for all AI APIs when neither the call nor the API has its own context.',\n    ),\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Agent-level guidance. Omit to remove the selected context; an empty string disables inherited context.',\n    ),\n});\n\nexport const aiActInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: aiActOptionsInputSchema.optional(),\n});\n\nexport const insightOptionsInputSchema = z.strictObject({\n  domIncluded: z\n    .union([z.boolean(), z.literal('visible-only')])\n    .optional()\n    .describe('How DOM information is included.'),\n  screenshotIncluded: z\n    .boolean()\n    .optional()\n    .describe('Whether the request includes a screenshot.'),\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.',\n    ),\n});\n\nexport const aiAssertOptionsInputSchema = insightOptionsInputSchema.extend({\n  keepRawResponse: z\n    .boolean()\n    .optional()\n    .describe('Whether the Agent returns the structured assertion result.'),\n});\n\nexport const aiAssertInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  message: z.string().optional().describe('The assertion failure message.'),\n  options: aiAssertOptionsInputSchema.optional(),\n});\n\nexport const locateOptionsInputSchema = z.strictObject({\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.',\n    ),\n  deepLocate: z\n    .boolean()\n    .optional()\n    .describe('Whether to use deep element location.'),\n  deepThink: z\n    .boolean()\n    .optional()\n    .describe('Deprecated alias for deepLocate.'),\n  cacheable: z\n    .boolean()\n    .optional()\n    .describe('Whether this location may use the Midscene cache.'),\n  xpath: z.string().optional().describe('A web XPath location hint.'),\n  fileChooserAccept: z\n    .union([z.string(), z.array(z.string())])\n    .optional()\n    .describe('Files accepted when tapping opens a file chooser.'),\n});\n\nexport const aiTapInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: locateOptionsInputSchema.optional(),\n});\n\nexport const aiInputInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  value: z.union([z.string(), z.number()]),\n  options: locateOptionsInputSchema\n    .extend({\n      mode: z.enum(['replace', 'clear', 'typeOnly', 'append']).optional(),\n      autoDismissKeyboard: z.boolean().optional(),\n      keyboardTypeDelay: z.number().nonnegative().optional(),\n      inputStrategy: z.enum(inputStrategies).optional(),\n    })\n    .optional(),\n});\n\nexport const aiKeyboardPressInputSchema = z.strictObject({\n  prompt: userPromptInputSchema.optional(),\n  keyName: nonBlankText('The key or key combination to press.'),\n  options: locateOptionsInputSchema.optional(),\n});\n\nexport const aiScrollInputSchema = z.strictObject({\n  prompt: userPromptInputSchema.optional(),\n  options: locateOptionsInputSchema\n    .extend({\n      direction: z.enum(['down', 'up', 'right', 'left']).optional(),\n      scrollType: z\n        .enum([\n          'singleAction',\n          'scrollToBottom',\n          'scrollToTop',\n          'scrollToRight',\n          'scrollToLeft',\n          'once',\n          'untilBottom',\n          'untilTop',\n          'untilRight',\n          'untilLeft',\n        ])\n        .optional(),\n      distance: z.number().nullable().optional(),\n    })\n    .optional(),\n});\n\nexport const aiPinchInputSchema = z.strictObject({\n  prompt: userPromptInputSchema.optional(),\n  direction: z.enum(['in', 'out']),\n  options: locateOptionsInputSchema\n    .extend({\n      distance: z.number().optional(),\n      duration: z.number().nonnegative().optional(),\n    })\n    .optional(),\n});\n\nexport const aiLongPressInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: locateOptionsInputSchema\n    .extend({ duration: z.number().nonnegative().optional() })\n    .optional(),\n});\n\nexport const aiDragAndDropInputSchema = z.strictObject({\n  from: userPromptInputSchema,\n  to: userPromptInputSchema,\n  options: locateOptionsInputSchema.optional(),\n});\n\nexport const aiQueryInputSchema = z.strictObject({\n  prompt: z.union([\n    nonBlankText('The extraction request.'),\n    z.record(z.string(), z.string()),\n  ]),\n  options: insightOptionsInputSchema.optional(),\n});\n\nexport const aiWaitForOptionsInputSchema = insightOptionsInputSchema\n  .extend({\n    timeoutMs: z.number().positive().optional(),\n    checkIntervalMs: z.number().positive().optional(),\n  })\n  .superRefine((input, context) => {\n    if ((input.checkIntervalMs ?? 3000) > (input.timeoutMs ?? 15000)) {\n      context.addIssue({\n        code: 'custom',\n        path: ['checkIntervalMs'],\n        message: 'checkIntervalMs must not exceed timeoutMs',\n      });\n    }\n  });\n\nexport const aiWaitForInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: aiWaitForOptionsInputSchema.optional(),\n});\n\nexport type AiWaitForNodeOptions = z.infer<typeof aiWaitForOptionsInputSchema>;\n\nexport const javascriptInputSchema = z.strictObject({\n  script: nonBlankText('JavaScript to evaluate through the current interface.'),\n});\n\nexport const runGherkinScenarioInputSchema = z.strictObject({\n  scenario: nonBlankText('The Gherkin scenario to execute.'),\n  options: aiActOptionsInputSchema.optional(),\n});\n\nexport const actionInputSchema = z.strictObject({\n  name: nonBlankText('The ActionSpace action name, for example DragAndDrop.'),\n  params: z.unknown().optional(),\n});\n\nexport const insightInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: insightOptionsInputSchema.optional(),\n});\n\nexport const reportScreenshotInputSchema = z.strictObject({\n  base64: z.string().min(1).describe('A base64-encoded screenshot.'),\n  description: z.string().optional().describe('What the screenshot shows.'),\n});\n\nexport const recordToReportOptionsInputSchema = z\n  .strictObject({\n    content: z.string().optional().describe('The report text content.'),\n    screenshotBase64: z\n      .string()\n      .optional()\n      .describe('One base64-encoded screenshot.'),\n    screenshots: z\n      .array(reportScreenshotInputSchema)\n      .min(1)\n      .optional()\n      .describe('Screenshots attached to the report section.'),\n  })\n  .superRefine((input, context) => {\n    if (\n      input.screenshotBase64 !== undefined &&\n      input.screenshots !== undefined\n    ) {\n      context.addIssue({\n        code: 'custom',\n        message: 'screenshotBase64 and screenshots are mutually exclusive',\n      });\n    }\n  });\n\nexport const recordToReportInputSchema = z.strictObject({\n  title: z.string().optional().describe('The report section title.'),\n  options: recordToReportOptionsInputSchema.optional(),\n});\n\nexport type UserPromptNodeInput = z.infer<typeof userPromptInputSchema>;\nexport type SetAIContextNodeInput = z.infer<typeof setAIContextInputSchema>;\nexport type AiActNodeOptions = z.infer<typeof aiActOptionsInputSchema>;\nexport type AiActNodeInput = z.infer<typeof aiActInputSchema>;\nexport type AiAssertNodeOptions = z.infer<typeof aiAssertOptionsInputSchema>;\nexport type AiAssertNodeInput = z.infer<typeof aiAssertInputSchema>;\nexport type AiTapNodeOptions = z.infer<typeof locateOptionsInputSchema>;\nexport type AiTapNodeInput = z.infer<typeof aiTapInputSchema>;\nexport type AiInputNodeInput = z.infer<typeof aiInputInputSchema>;\nexport type AiKeyboardPressNodeInput = z.infer<\n  typeof aiKeyboardPressInputSchema\n>;\nexport type AiScrollNodeInput = z.infer<typeof aiScrollInputSchema>;\nexport type AiPinchNodeInput = z.infer<typeof aiPinchInputSchema>;\nexport type AiLongPressNodeInput = z.infer<typeof aiLongPressInputSchema>;\nexport type AiDragAndDropNodeInput = z.infer<typeof aiDragAndDropInputSchema>;\nexport type AiQueryNodeInput = z.infer<typeof aiQueryInputSchema>;\nexport type AiWaitForNodeInput = z.infer<typeof aiWaitForInputSchema>;\nexport type JavascriptNodeInput = z.infer<typeof javascriptInputSchema>;\nexport type RunGherkinScenarioNodeInput = z.infer<\n  typeof runGherkinScenarioInputSchema\n>;\nexport type ActionNodeInput = z.infer<typeof actionInputSchema>;\nexport type InsightNodeOptions = z.infer<typeof insightOptionsInputSchema>;\nexport type InsightNodeInput = z.infer<typeof insightInputSchema>;\nexport type RecordToReportNodeOptions = z.infer<\n  typeof recordToReportOptionsInputSchema\n>;\nexport type RecordToReportNodeInput = z.infer<typeof recordToReportInputSchema>;\n\nexport type CommonAgentTestRunnerApi = Pick<\n  Agent,\n  | 'aiAct'\n  | 'aiTap'\n  | 'aiAssert'\n  | 'aiBoolean'\n  | 'aiNumber'\n  | 'aiString'\n  | 'aiAsk'\n  | 'recordToReport'\n>;\n\nconst defineCommonAgentNode =\n  createAgentTestRunnerNodeDefinition<CommonAgentTestRunnerApi>('an Agent');\n\n// Select the current overloads without expanding the minimum API required by\n// existing getAgent() providers. Each Node checks its own method at execution.\ntype AgentActionNodeApi = Pick<\n  Agent,\n  | 'sleep'\n  | 'setAIContext'\n  | 'aiHover'\n  | 'aiDoubleClick'\n  | 'aiRightClick'\n  | 'aiLongPress'\n  | 'aiClearInput'\n  | 'aiPinch'\n  | 'aiLocate'\n  | 'aiQuery'\n  | 'aiWaitFor'\n  | 'evaluateJavaScript'\n  | 'runGherkinScenario'\n  | 'callActionInActionSpace'\n> & {\n  aiInput(\n    prompt: TUserPrompt,\n    options: NonNullable<Parameters<Agent['aiInput']>[2]> & {\n      value: string | number;\n    },\n  ): Promise<void>;\n  aiKeyboardPress(\n    prompt: TUserPrompt | undefined,\n    options: LocateOption & { keyName: string },\n  ): Promise<void>;\n  aiScroll(\n    prompt: TUserPrompt | undefined,\n    options: LocateOption & ScrollParam,\n  ): Promise<void>;\n};\n\nconst defineAgentActionNode =\n  createAgentTestRunnerNodeDefinition<AgentActionNodeApi>('an Agent');\n\nconst promptText = (prompt: UserPromptNodeInput): string =>\n  typeof prompt === 'string' ? prompt : prompt.prompt;\n\nconst valueResult = (\n  value: unknown,\n): AgentTestRunnerNodeResult | undefined => ({ data: value });\n\nconst aiActNode = defineCommonAgentNode({\n  method: 'aiAct',\n  description: 'Perform a natural-language task with a Midscene UI Agent.',\n  stringInputKey: 'prompt',\n  inputSchema: aiActInputSchema,\n  toArgs(input, context) {\n    return [\n      input.prompt,\n      {\n        ...input.options,\n        abortSignal: context.signal,\n      },\n    ];\n  },\n  toResult(output) {\n    return output === undefined ? undefined : { summary: output };\n  },\n});\n\nconst aiTapNode = defineCommonAgentNode({\n  method: 'aiTap',\n  description: 'Locate and tap an element with a Midscene UI Agent.',\n  stringInputKey: 'prompt',\n  inputSchema: aiTapInputSchema,\n  toArgs(input) {\n    return [input.prompt, input.options];\n  },\n  toResult(_output, input) {\n    return { summary: `Tapped: ${promptText(input.prompt)}` };\n  },\n});\n\nconst aiAssertNode = defineCommonAgentNode({\n  method: 'aiAssert',\n  description: 'Assert a natural-language condition with a Midscene UI Agent.',\n  stringInputKey: 'prompt',\n  inputSchema: aiAssertInputSchema,\n  toArgs(input, context) {\n    return [\n      input.prompt,\n      input.message,\n      {\n        keepRawResponse: true,\n        ...input.options,\n        abortSignal: context.signal,\n      },\n    ];\n  },\n  toResult(output, input) {\n    const result = {\n      summary: `Assertion ${output?.pass === false ? 'failed' : 'passed'}: ${promptText(input.prompt)}`,\n      ...(output === undefined\n        ? {}\n        : {\n            data: {\n              pass: output.pass,\n              ...(output.thought === undefined\n                ? {}\n                : { thought: output.thought }),\n              ...(output.message === undefined\n                ? {}\n                : { message: output.message }),\n            },\n          }),\n    };\n    if (output?.pass === false)\n      throw new NodeExecutionError(\n        'aiAssert',\n        new Error(\n          output.message ||\n            input.message ||\n            output.thought ||\n            `Assertion failed: ${promptText(input.prompt)}`,\n        ),\n        result,\n      );\n    return result;\n  },\n});\n\nconst locateActionNode = (\n  method: 'aiHover' | 'aiDoubleClick' | 'aiRightClick' | 'aiClearInput',\n) =>\n  defineAgentActionNode({\n    method,\n    description: {\n      aiHover: 'Locate and hover over an element with a Midscene UI Agent.',\n      aiDoubleClick:\n        'Locate and double-click an element with a Midscene UI Agent.',\n      aiRightClick:\n        'Locate and right-click an element with a Midscene UI Agent.',\n      aiClearInput:\n        'Locate an input element and clear its value with a Midscene UI Agent.',\n    }[method],\n    stringInputKey: 'prompt',\n    inputSchema: aiTapInputSchema,\n    toArgs: (input) => [input.prompt, input.options],\n    toResult: (_output, input) => ({\n      summary: `${method}: ${promptText(input.prompt)}`,\n    }),\n  });\n\nconst additionalAgentNodes: readonly AgentTestRunnerNodeDefinition[] = [\n  defineAgentActionNode({\n    method: 'setAIContext',\n    description:\n      'Set or remove Agent-level AI guidance for subsequent calls on the same Agent.',\n    inputSchema: setAIContextInputSchema,\n    toArgs: (input) => [input.target, input.context],\n  }),\n  locateActionNode('aiHover'),\n  locateActionNode('aiDoubleClick'),\n  locateActionNode('aiRightClick'),\n  locateActionNode('aiClearInput'),\n  defineAgentActionNode({\n    method: 'aiInput',\n    description:\n      'Locate an input element and enter a value with a Midscene UI Agent.',\n    inputSchema: aiInputInputSchema,\n    toArgs: (input) => [input.prompt, { ...input.options, value: input.value }],\n  }),\n  defineAgentActionNode({\n    method: 'aiKeyboardPress',\n    description:\n      'Press a key or key combination, optionally targeting a located element.',\n    stringInputKey: 'keyName',\n    inputSchema: aiKeyboardPressInputSchema,\n    toArgs: (input) => [\n      input.prompt,\n      { ...input.options, keyName: input.keyName },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'aiScroll',\n    description:\n      'Scroll the page or a located region with a Midscene UI Agent.',\n    stringInputKey: 'prompt',\n    inputSchema: aiScrollInputSchema,\n    toArgs: (input) => [input.prompt, { ...input.options }],\n  }),\n  defineAgentActionNode({\n    method: 'aiPinch',\n    description:\n      'Perform a pinch-in or pinch-out gesture with a Midscene UI Agent.',\n    inputSchema: aiPinchInputSchema,\n    toArgs: (input) => [\n      input.prompt,\n      { ...input.options, direction: input.direction },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'aiLongPress',\n    description: 'Locate and long-press an element with a Midscene UI Agent.',\n    stringInputKey: 'prompt',\n    inputSchema: aiLongPressInputSchema,\n    toArgs: (input) => [input.prompt, input.options],\n  }),\n  defineAgentActionNode({\n    method: 'callActionInActionSpace',\n    name: 'aiDragAndDrop',\n    description:\n      'Locate source and destination elements, then drag the source to the destination.',\n    inputSchema: aiDragAndDropInputSchema,\n    toArgs: (input) => [\n      'DragAndDrop',\n      {\n        from: buildDetailedLocateParam(input.from, input.options),\n        to: buildDetailedLocateParam(input.to, input.options),\n      },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'aiLocate',\n    description:\n      'Locate an element from a natural-language description and store the result.',\n    stringInputKey: 'prompt',\n    inputSchema: aiTapInputSchema,\n    toArgs: (input, context) => [\n      input.prompt,\n      { ...input.options, abortSignal: context.signal },\n    ],\n    toResult: valueResult,\n  }),\n  defineAgentActionNode({\n    method: 'aiQuery',\n    description:\n      'Extract structured data from the current interface and store the result.',\n    stringInputKey: 'prompt',\n    inputSchema: aiQueryInputSchema,\n    toArgs: (input, context) => [\n      input.prompt,\n      { ...input.options, abortSignal: context.signal },\n    ],\n    toResult: valueResult,\n  }),\n  defineAgentActionNode({\n    method: 'aiWaitFor',\n    description: 'Wait until a natural-language condition is satisfied.',\n    stringInputKey: 'prompt',\n    inputSchema: aiWaitForInputSchema,\n    toArgs: (input, context) => [\n      input.prompt,\n      { ...input.options, abortSignal: context.signal },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'evaluateJavaScript',\n    name: 'javascript',\n    description:\n      'Evaluate JavaScript in the current interface and store the result.',\n    stringInputKey: 'script',\n    inputSchema: javascriptInputSchema,\n    toArgs: (input) => [input.script],\n    toResult: valueResult,\n  }),\n  defineAgentActionNode({\n    method: 'runGherkinScenario',\n    description:\n      'Execute a Gherkin scenario with the current Midscene UI Agent.',\n    stringInputKey: 'scenario',\n    inputSchema: runGherkinScenarioInputSchema,\n    toArgs: (input, context) => [\n      input.scenario,\n      { ...input.options, abortSignal: context.signal },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'callActionInActionSpace',\n    name: 'action',\n    description:\n      'Call a platform or custom action. Its ActionSpace schema validates params when the action executes.',\n    inputSchema: actionInputSchema,\n    toArgs: (input) => [input.name, input.params],\n    toResult: (value) => (value === undefined ? undefined : valueResult(value)),\n  }),\n];\n\nconst insightNode = (method: 'aiBoolean' | 'aiNumber' | 'aiString' | 'aiAsk') =>\n  defineCommonAgentNode({\n    method,\n    description: `Run ${method} with a Midscene UI Agent and store its value.`,\n    stringInputKey: 'prompt',\n    inputSchema: insightInputSchema,\n    toArgs: (input, context) => [\n      input.prompt,\n      { ...input.options, abortSignal: context.signal },\n    ],\n    toResult: (value) => ({\n      summary: `${method} returned ${\n        method === 'aiString' || method === 'aiAsk'\n          ? JSON.stringify(value)\n          : value\n      }`,\n      data: { value },\n    }),\n  });\n\nconst recordToReportNode = defineCommonAgentNode({\n  method: 'recordToReport',\n  description: 'Add text or screenshots to the current Midscene report.',\n  stringInputKey: 'title',\n  inputSchema: recordToReportInputSchema,\n  toArgs(input) {\n    return [input.title, input.options];\n  },\n  toResult(_output, input) {\n    return { summary: `Recorded to report: ${input.title ?? 'untitled'}` };\n  },\n});\n\nexport const commonAgentTestRunnerNodeDefinitions: readonly AgentTestRunnerNodeDefinition[] =\n  [\n    defineAgentActionNode({\n      method: 'sleep',\n      description:\n        'Wait for a fixed number of milliseconds, recording standard UI snapshots and honoring cancellation.',\n      inputSchema: z.strictObject({ ms: z.number().positive() }),\n      toArgs(input, { signal }) {\n        signal.throwIfAborted();\n        return [input.ms, { abortSignal: signal }];\n      },\n    }),\n    aiActNode,\n    aiTapNode,\n    aiAssertNode,\n    insightNode('aiBoolean'),\n    insightNode('aiNumber'),\n    insightNode('aiString'),\n    insightNode('aiAsk'),\n    recordToReportNode,\n    ...additionalAgentNodes,\n  ];\n"],"names":["createAgentTestRunnerNodeDefinition","agentLabel","definition","undefined","agent","input","context","TypeError","typedAgent","method","typedInput","result","nonBlankText","description","z","promptImageInputSchema","structuredUserPromptInputSchema","userPromptInputSchema","aiActOptionsInputSchema","setAIContextInputSchema","AGENT_AI_CONTEXT_KEYS","aiActInputSchema","insightOptionsInputSchema","aiAssertOptionsInputSchema","aiAssertInputSchema","locateOptionsInputSchema","aiTapInputSchema","aiInputInputSchema","inputStrategies","aiKeyboardPressInputSchema","aiScrollInputSchema","aiPinchInputSchema","aiLongPressInputSchema","aiDragAndDropInputSchema","aiQueryInputSchema","aiWaitForOptionsInputSchema","aiWaitForInputSchema","javascriptInputSchema","runGherkinScenarioInputSchema","actionInputSchema","insightInputSchema","reportScreenshotInputSchema","recordToReportOptionsInputSchema","recordToReportInputSchema","defineCommonAgentNode","defineAgentActionNode","promptText","prompt","valueResult","value","aiActNode","output","aiTapNode","_output","aiAssertNode","NodeExecutionError","Error","locateActionNode","additionalAgentNodes","buildDetailedLocateParam","insightNode","JSON","recordToReportNode","commonAgentTestRunnerNodeDefinitions","signal"],"mappings":";;;;;AA6EO,MAAMA,sCACX,CAAwBC,aAAa,UAAU,GAC/C,CAKEC,aAMmC;YACnC,MAAMA,WAAW,IAAI,IAAIA,WAAW,MAAM;YAC1C,GAAIA,AAAqBC,WAArBD,WAAW,KAAK,GAAiB,CAAC,IAAI;gBAAE,OAAOA,WAAW,KAAK;YAAC,CAAC;YACrE,GAAIA,AAA2BC,WAA3BD,WAAW,WAAW,GACtB,CAAC,IACD;gBAAE,aAAaA,WAAW,WAAW;YAAC,CAAC;YAC3C,GAAIA,AAA8BC,WAA9BD,WAAW,cAAc,GACzB,CAAC,IACD;gBAAE,gBAAgBA,WAAW,cAAc;YAAC,CAAC;YACjD,aAAaA,WAAW,WAAW;YACnC,MAAM,SAAQE,KAAK,EAAEC,KAAK,EAAEC,OAAO;gBACjC,IACE,AAAiB,YAAjB,OAAOF,SACPA,AAAU,SAAVA,SACA,AACE,cADF,OAAQA,KAAiC,CAACF,WAAW,MAAM,CAAC,EAG5D,MAAM,IAAIK,UACR,CAAC,uBAAuB,EAAEN,WAAW,MAAM,EAAEC,WAAW,MAAM,CAAC,GAAG,CAAC;gBAIvE,MAAMM,aAAaJ;gBACnB,MAAMK,SAASD,UAAU,CAACN,WAAW,MAAM,CAAC;gBAI5C,MAAMQ,aAAaL;gBACnB,MAAMM,SAAU,MAAMF,OAAO,KAAK,CAChCD,YACAN,WAAW,MAAM,CAACQ,YAAYJ;gBAEhC,OAAOJ,WAAW,QAAQ,GAAGS,QAAQD;YACvC;QACF;AAEF,MAAME,eAAe,CAACC,cACpBC,EAAAA,MACS,GACN,KAAK,CAAC,MAAM,iDACZ,QAAQ,CAACD;AAEP,MAAME,yBAAyBD,EAAE,YAAY,CAAC;IACnD,MAAMF,aAAa;IACnB,KAAKA,aAAa;AACpB;AAEO,MAAMI,kCAAkCF,EAAE,YAAY,CAAC;IAC5D,QAAQF,aAAa;IACrB,QAAQE,EAAE,KAAK,CAACC,wBAAwB,GAAG,CAAC,GAAG,QAAQ;IACvD,yBAAyBD,EAAAA,OACf,GACP,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMG,wBAAwBH,EAAE,KAAK,CAAC;IAC3CF,aAAa;IACbI;CACD;AAEM,MAAME,0BAA0BJ,EAAE,YAAY,CAAC;IACpD,QAAQA,CAAC,CAADA,OACD,CAAC;QAAC;QAAQ;QAAW;KAAY,EACrC,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,mBAAmBA,EAAAA,KACX,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,KAAK,CAACA,EAAE,MAAM;KAAI,EACvC,QAAQ,GACR,QAAQ,CAAC;IACZ,uBAAuBA,EAAAA,MACd,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAAA,KACH,CAAC;QAACA,EAAE,OAAO,CAAC;QAAUA,EAAE,OAAO;KAAG,EACvC,QAAQ,GACR,QAAQ,CAAC;IACZ,YAAYA,EAAAA,OACF,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,SAASA,EAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMK,0BAA0BL,EAAE,YAAY,CAAC;IACpD,QAAQA,CAAC,CAADA,OACD,CAACM,uBACL,QAAQ,CACP;IAEJ,SAASN,EAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMO,mBAAmBP,EAAE,YAAY,CAAC;IAC7C,QAAQG;IACR,SAASC,wBAAwB,QAAQ;AAC3C;AAEO,MAAMI,4BAA4BR,EAAE,YAAY,CAAC;IACtD,aAAaA,EAAAA,KACL,CAAC;QAACA,EAAE,OAAO;QAAIA,EAAE,OAAO,CAAC;KAAgB,EAC9C,QAAQ,GACR,QAAQ,CAAC;IACZ,oBAAoBA,EAAAA,OACV,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,SAASA,EAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMS,6BAA6BD,0BAA0B,MAAM,CAAC;IACzE,iBAAiBR,EAAAA,OACP,GACP,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMU,sBAAsBV,EAAE,YAAY,CAAC;IAChD,QAAQG;IACR,SAASH,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACxC,SAASS,2BAA2B,QAAQ;AAC9C;AAEO,MAAME,2BAA2BX,EAAE,YAAY,CAAC;IACrD,SAASA,EAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;IAEJ,YAAYA,EAAAA,OACF,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,OAAOA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACtC,mBAAmBA,EAAAA,KACX,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,KAAK,CAACA,EAAE,MAAM;KAAI,EACvC,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMY,mBAAmBZ,EAAE,YAAY,CAAC;IAC7C,QAAQG;IACR,SAASQ,yBAAyB,QAAQ;AAC5C;AAEO,MAAME,qBAAqBb,EAAE,YAAY,CAAC;IAC/C,QAAQG;IACR,OAAOH,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACvC,SAASW,yBACN,MAAM,CAAC;QACN,MAAMX,CAAC,CAADA,OAAM,CAAC;YAAC;YAAW;YAAS;YAAY;SAAS,EAAE,QAAQ;QACjE,qBAAqBA,EAAE,OAAO,GAAG,QAAQ;QACzC,mBAAmBA,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ;QACpD,eAAeA,CAAC,CAADA,OAAM,CAACc,iBAAiB,QAAQ;IACjD,GACC,QAAQ;AACb;AAEO,MAAMC,6BAA6Bf,EAAE,YAAY,CAAC;IACvD,QAAQG,sBAAsB,QAAQ;IACtC,SAASL,aAAa;IACtB,SAASa,yBAAyB,QAAQ;AAC5C;AAEO,MAAMK,sBAAsBhB,EAAE,YAAY,CAAC;IAChD,QAAQG,sBAAsB,QAAQ;IACtC,SAASQ,yBACN,MAAM,CAAC;QACN,WAAWX,CAAC,CAADA,OAAM,CAAC;YAAC;YAAQ;YAAM;YAAS;SAAO,EAAE,QAAQ;QAC3D,YAAYA,CAAC,CAADA,OACL,CAAC;YACJ;YACA;YACA;YACA;YACA;YACA;YACA;YACA;YACA;YACA;SACD,EACA,QAAQ;QACX,UAAUA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ;IAC1C,GACC,QAAQ;AACb;AAEO,MAAMiB,qBAAqBjB,EAAE,YAAY,CAAC;IAC/C,QAAQG,sBAAsB,QAAQ;IACtC,WAAWH,CAAC,CAADA,OAAM,CAAC;QAAC;QAAM;KAAM;IAC/B,SAASW,yBACN,MAAM,CAAC;QACN,UAAUX,EAAE,MAAM,GAAG,QAAQ;QAC7B,UAAUA,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ;IAC7C,GACC,QAAQ;AACb;AAEO,MAAMkB,yBAAyBlB,EAAE,YAAY,CAAC;IACnD,QAAQG;IACR,SAASQ,yBACN,MAAM,CAAC;QAAE,UAAUX,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ;IAAG,GACvD,QAAQ;AACb;AAEO,MAAMmB,2BAA2BnB,EAAE,YAAY,CAAC;IACrD,MAAMG;IACN,IAAIA;IACJ,SAASQ,yBAAyB,QAAQ;AAC5C;AAEO,MAAMS,qBAAqBpB,EAAE,YAAY,CAAC;IAC/C,QAAQA,EAAE,KAAK,CAAC;QACdF,aAAa;QACbE,EAAE,MAAM,CAACA,EAAE,MAAM,IAAIA,EAAE,MAAM;KAC9B;IACD,SAASQ,0BAA0B,QAAQ;AAC7C;AAEO,MAAMa,8BAA8Bb,0BACxC,MAAM,CAAC;IACN,WAAWR,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ;IACzC,iBAAiBA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ;AACjD,GACC,WAAW,CAAC,CAACT,OAAOC;IACnB,IAAKD,AAAAA,CAAAA,MAAM,eAAe,IAAI,IAAG,IAAMA,CAAAA,MAAM,SAAS,IAAI,KAAI,GAC5DC,QAAQ,QAAQ,CAAC;QACf,MAAM;QACN,MAAM;YAAC;SAAkB;QACzB,SAAS;IACX;AAEJ;AAEK,MAAM8B,uBAAuBtB,EAAE,YAAY,CAAC;IACjD,QAAQG;IACR,SAASkB,4BAA4B,QAAQ;AAC/C;AAIO,MAAME,wBAAwBvB,EAAE,YAAY,CAAC;IAClD,QAAQF,aAAa;AACvB;AAEO,MAAM0B,gCAAgCxB,EAAE,YAAY,CAAC;IAC1D,UAAUF,aAAa;IACvB,SAASM,wBAAwB,QAAQ;AAC3C;AAEO,MAAMqB,oBAAoBzB,EAAE,YAAY,CAAC;IAC9C,MAAMF,aAAa;IACnB,QAAQE,EAAE,OAAO,GAAG,QAAQ;AAC9B;AAEO,MAAM0B,qBAAqB1B,EAAE,YAAY,CAAC;IAC/C,QAAQG;IACR,SAASK,0BAA0B,QAAQ;AAC7C;AAEO,MAAMmB,8BAA8B3B,EAAE,YAAY,CAAC;IACxD,QAAQA,EAAE,MAAM,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC;IACnC,aAAaA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAC9C;AAEO,MAAM4B,mCAAmC5B,EAAAA,YACjC,CAAC;IACZ,SAASA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACxC,kBAAkBA,EAAAA,MACT,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,aAAaA,EAAAA,KACL,CAAC2B,6BACN,GAAG,CAAC,GACJ,QAAQ,GACR,QAAQ,CAAC;AACd,GACC,WAAW,CAAC,CAACpC,OAAOC;IACnB,IACED,AAA2BF,WAA3BE,MAAM,gBAAgB,IACtBA,AAAsBF,WAAtBE,MAAM,WAAW,EAEjBC,QAAQ,QAAQ,CAAC;QACf,MAAM;QACN,SAAS;IACX;AAEJ;AAEK,MAAMqC,4BAA4B7B,EAAE,YAAY,CAAC;IACtD,OAAOA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACtC,SAAS4B,iCAAiC,QAAQ;AACpD;AA4CA,MAAME,wBACJ5C,oCAA8D;AAqChE,MAAM6C,wBACJ7C,oCAAwD;AAE1D,MAAM8C,aAAa,CAACC,SAClB,AAAkB,YAAlB,OAAOA,SAAsBA,SAASA,OAAO,MAAM;AAErD,MAAMC,cAAc,CAClBC,QAC2C;QAAE,MAAMA;IAAM;AAE3D,MAAMC,YAAYN,sBAAsB;IACtC,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAavB;IACb,QAAOhB,KAAK,EAAEC,OAAO;QACnB,OAAO;YACLD,MAAM,MAAM;YACZ;gBACE,GAAGA,MAAM,OAAO;gBAChB,aAAaC,QAAQ,MAAM;YAC7B;SACD;IACH;IACA,UAAS6C,MAAM;QACb,OAAOA,AAAWhD,WAAXgD,SAAuBhD,SAAY;YAAE,SAASgD;QAAO;IAC9D;AACF;AAEA,MAAMC,YAAYR,sBAAsB;IACtC,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAalB;IACb,QAAOrB,KAAK;QACV,OAAO;YAACA,MAAM,MAAM;YAAEA,MAAM,OAAO;SAAC;IACtC;IACA,UAASgD,OAAO,EAAEhD,KAAK;QACrB,OAAO;YAAE,SAAS,CAAC,QAAQ,EAAEyC,WAAWzC,MAAM,MAAM,GAAG;QAAC;IAC1D;AACF;AAEA,MAAMiD,eAAeV,sBAAsB;IACzC,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAapB;IACb,QAAOnB,KAAK,EAAEC,OAAO;QACnB,OAAO;YACLD,MAAM,MAAM;YACZA,MAAM,OAAO;YACb;gBACE,iBAAiB;gBACjB,GAAGA,MAAM,OAAO;gBAChB,aAAaC,QAAQ,MAAM;YAC7B;SACD;IACH;IACA,UAAS6C,MAAM,EAAE9C,KAAK;QACpB,MAAMM,SAAS;YACb,SAAS,CAAC,UAAU,EAAEwC,QAAQ,SAAS,QAAQ,WAAW,SAAS,EAAE,EAAEL,WAAWzC,MAAM,MAAM,GAAG;YACjG,GAAI8C,AAAWhD,WAAXgD,SACA,CAAC,IACD;gBACE,MAAM;oBACJ,MAAMA,OAAO,IAAI;oBACjB,GAAIA,AAAmBhD,WAAnBgD,OAAO,OAAO,GACd,CAAC,IACD;wBAAE,SAASA,OAAO,OAAO;oBAAC,CAAC;oBAC/B,GAAIA,AAAmBhD,WAAnBgD,OAAO,OAAO,GACd,CAAC,IACD;wBAAE,SAASA,OAAO,OAAO;oBAAC,CAAC;gBACjC;YACF,CAAC;QACP;QACA,IAAIA,QAAQ,SAAS,OACnB,MAAM,IAAII,mBACR,YACA,IAAIC,MACFL,OAAO,OAAO,IACZ9C,MAAM,OAAO,IACb8C,OAAO,OAAO,IACd,CAAC,kBAAkB,EAAEL,WAAWzC,MAAM,MAAM,GAAG,GAEnDM;QAEJ,OAAOA;IACT;AACF;AAEA,MAAM8C,mBAAmB,CACvBhD,SAEAoC,sBAAsB;QACpBpC;QACA,aAAa;YACX,SAAS;YACT,eACE;YACF,cACE;YACF,cACE;QACJ,CAAC,CAACA,OAAO;QACT,gBAAgB;QAChB,aAAaiB;QACb,QAAQ,CAACrB,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;QAChD,UAAU,CAACgD,SAAShD,QAAW;gBAC7B,SAAS,GAAGI,OAAO,EAAE,EAAEqC,WAAWzC,MAAM,MAAM,GAAG;YACnD;IACF;AAEF,MAAMqD,uBAAiE;IACrEb,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,aAAa1B;QACb,QAAQ,CAACd,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;IAClD;IACAoD,iBAAiB;IACjBA,iBAAiB;IACjBA,iBAAiB;IACjBA,iBAAiB;IACjBZ,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,aAAalB;QACb,QAAQ,CAACtB,QAAU;gBAACA,MAAM,MAAM;gBAAE;oBAAE,GAAGA,MAAM,OAAO;oBAAE,OAAOA,MAAM,KAAK;gBAAC;aAAE;IAC7E;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAahB;QACb,QAAQ,CAACxB,QAAU;gBACjBA,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,SAASA,MAAM,OAAO;gBAAC;aAC5C;IACH;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAaf;QACb,QAAQ,CAACzB,QAAU;gBAACA,MAAM,MAAM;gBAAE;oBAAE,GAAGA,MAAM,OAAO;gBAAC;aAAE;IACzD;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,aAAad;QACb,QAAQ,CAAC1B,QAAU;gBACjBA,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,WAAWA,MAAM,SAAS;gBAAC;aAChD;IACH;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aAAa;QACb,gBAAgB;QAChB,aAAab;QACb,QAAQ,CAAC3B,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;IAClD;IACAwC,sBAAsB;QACpB,QAAQ;QACR,MAAM;QACN,aACE;QACF,aAAaZ;QACb,QAAQ,CAAC5B,QAAU;gBACjB;gBACA;oBACE,MAAMsD,yBAAyBtD,MAAM,IAAI,EAAEA,MAAM,OAAO;oBACxD,IAAIsD,yBAAyBtD,MAAM,EAAE,EAAEA,MAAM,OAAO;gBACtD;aACD;IACH;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAanB;QACb,QAAQ,CAACrB,OAAOC,UAAY;gBAC1BD,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;QACD,UAAU0C;IACZ;IACAH,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAaX;QACb,QAAQ,CAAC7B,OAAOC,UAAY;gBAC1BD,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;QACD,UAAU0C;IACZ;IACAH,sBAAsB;QACpB,QAAQ;QACR,aAAa;QACb,gBAAgB;QAChB,aAAaT;QACb,QAAQ,CAAC/B,OAAOC,UAAY;gBAC1BD,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;IACH;IACAuC,sBAAsB;QACpB,QAAQ;QACR,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaR;QACb,QAAQ,CAAChC,QAAU;gBAACA,MAAM,MAAM;aAAC;QACjC,UAAU2C;IACZ;IACAH,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAaP;QACb,QAAQ,CAACjC,OAAOC,UAAY;gBAC1BD,MAAM,QAAQ;gBACd;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;IACH;IACAuC,sBAAsB;QACpB,QAAQ;QACR,MAAM;QACN,aACE;QACF,aAAaN;QACb,QAAQ,CAAClC,QAAU;gBAACA,MAAM,IAAI;gBAAEA,MAAM,MAAM;aAAC;QAC7C,UAAU,CAAC4C,QAAWA,AAAU9C,WAAV8C,QAAsB9C,SAAY6C,YAAYC;IACtE;CACD;AAED,MAAMW,cAAc,CAACnD,SACnBmC,sBAAsB;QACpBnC;QACA,aAAa,CAAC,IAAI,EAAEA,OAAO,8CAA8C,CAAC;QAC1E,gBAAgB;QAChB,aAAa+B;QACb,QAAQ,CAACnC,OAAOC,UAAY;gBAC1BD,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;QACD,UAAU,CAAC2C,QAAW;gBACpB,SAAS,GAAGxC,OAAO,UAAU,EAC3BA,AAAW,eAAXA,UAAyBA,AAAW,YAAXA,SACrBoD,KAAK,SAAS,CAACZ,SACfA,OACJ;gBACF,MAAM;oBAAEA;gBAAM;YAChB;IACF;AAEF,MAAMa,qBAAqBlB,sBAAsB;IAC/C,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAaD;IACb,QAAOtC,KAAK;QACV,OAAO;YAACA,MAAM,KAAK;YAAEA,MAAM,OAAO;SAAC;IACrC;IACA,UAASgD,OAAO,EAAEhD,KAAK;QACrB,OAAO;YAAE,SAAS,CAAC,oBAAoB,EAAEA,MAAM,KAAK,IAAI,YAAY;QAAC;IACvE;AACF;AAEO,MAAM0D,uCACX;IACElB,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,aAAa/B,EAAE,YAAY,CAAC;YAAE,IAAIA,EAAE,MAAM,GAAG,QAAQ;QAAG;QACxD,QAAOT,KAAK,EAAE,EAAE2D,MAAM,EAAE;YACtBA,OAAO,cAAc;YACrB,OAAO;gBAAC3D,MAAM,EAAE;gBAAE;oBAAE,aAAa2D;gBAAO;aAAE;QAC5C;IACF;IACAd;IACAE;IACAE;IACAM,YAAY;IACZA,YAAY;IACZA,YAAY;IACZA,YAAY;IACZE;OACGJ;CACJ"}
+{"version":3,"file":"agent/test-runner-nodes.mjs","sources":["../../../src/agent/test-runner-nodes.ts"],"sourcesContent":["import { AGENT_AI_CONTEXT_KEYS } from '@midscene/shared/agent-tools/agent-context';\nimport { z } from 'zod/v4';\nimport type { TUserPrompt } from '../common';\nimport { inputStrategies } from '../device/input-strategy';\nimport { NodeExecutionError } from '../test-runner/errors';\nimport type { LocateOption, ScrollParam } from '../yaml';\nimport { buildDetailedLocateParam } from '../yaml/utils';\nimport type { Agent } from './agent';\n\nexport interface AgentTestRunnerNodeResult<TData = unknown> {\n  summary?: string;\n  data?: TData;\n}\n\nexport interface AgentTestRunnerNodeExecutionContext {\n  signal: AbortSignal;\n}\n\n/** Agent-owned Node description consumed by Test Runner adapters. */\nexport interface AgentTestRunnerNodeDefinition<\n  TInput extends Record<string, unknown> = Record<string, unknown>,\n  TData = unknown,\n> {\n  name: string;\n  title?: string;\n  description?: string;\n  /** Node input field populated by YAML string shorthand; omit to disable. */\n  stringInputKey?: string | false;\n  inputSchema: z.ZodObject;\n  execute(\n    agent: unknown,\n    input: TInput,\n    context: AgentTestRunnerNodeExecutionContext,\n  ):\n    | Promise<AgentTestRunnerNodeResult<TData> | undefined>\n    | AgentTestRunnerNodeResult<TData>\n    | undefined;\n}\n\n/** Static capability implemented by Agent classes that expose Runner Nodes. */\nexport interface AgentTestRunnerNodeProvider {\n  getTestRunnerNodeDefinitions(): readonly AgentTestRunnerNodeDefinition[];\n}\n\ntype AnyAgentMethod = (...args: any[]) => unknown;\ntype AgentMethodName<TAgent extends object> = {\n  [TKey in keyof TAgent]-?: TAgent[TKey] extends AnyAgentMethod ? TKey : never;\n}[keyof TAgent] &\n  string;\ntype AgentMethod<\n  TAgent extends object,\n  TMethod extends AgentMethodName<TAgent>,\n> = Extract<TAgent[TMethod], AnyAgentMethod>;\n\nexport interface DefineAgentTestRunnerNodeOptions<\n  TAgent extends object,\n  TMethod extends AgentMethodName<TAgent>,\n  TSchema extends z.ZodObject,\n  TData = unknown,\n> {\n  method: TMethod;\n  name?: string;\n  title?: string;\n  description?: string;\n  stringInputKey?: (keyof z.output<TSchema> & string) | false;\n  inputSchema: TSchema;\n  toArgs(\n    input: z.output<TSchema>,\n    context: AgentTestRunnerNodeExecutionContext,\n  ): Parameters<AgentMethod<TAgent, TMethod>>;\n  toResult?(\n    result: Awaited<ReturnType<AgentMethod<TAgent, TMethod>>>,\n    input: z.output<TSchema>,\n  ): AgentTestRunnerNodeResult<TData> | undefined;\n}\n\n/** Create a Node factory whose adapters are checked against an Agent API. */\nexport const createAgentTestRunnerNodeDefinition =\n  <TAgent extends object>(agentLabel = 'an Agent') =>\n  <\n    TMethod extends AgentMethodName<TAgent>,\n    TSchema extends z.ZodObject,\n    TData = unknown,\n  >(\n    definition: DefineAgentTestRunnerNodeOptions<\n      TAgent,\n      TMethod,\n      TSchema,\n      TData\n    >,\n  ): AgentTestRunnerNodeDefinition => ({\n    name: definition.name ?? definition.method,\n    ...(definition.title === undefined ? {} : { title: definition.title }),\n    ...(definition.description === undefined\n      ? {}\n      : { description: definition.description }),\n    ...(definition.stringInputKey === undefined\n      ? {}\n      : { stringInputKey: definition.stringInputKey }),\n    inputSchema: definition.inputSchema,\n    async execute(agent, input, context) {\n      if (\n        typeof agent !== 'object' ||\n        agent === null ||\n        typeof (agent as Record<string, unknown>)[definition.method] !==\n          'function'\n      ) {\n        throw new TypeError(\n          `getAgent() must return ${agentLabel} with ${definition.method}().`,\n        );\n      }\n\n      const typedAgent = agent as TAgent;\n      const method = typedAgent[definition.method] as AgentMethod<\n        TAgent,\n        TMethod\n      >;\n      const typedInput = input as z.output<TSchema>;\n      const result = (await method.apply(\n        typedAgent,\n        definition.toArgs(typedInput, context),\n      )) as Awaited<ReturnType<AgentMethod<TAgent, TMethod>>>;\n      return definition.toResult?.(result, typedInput);\n    },\n  });\n\nconst nonBlankText = (description: string) =>\n  z\n    .string()\n    .regex(/\\S/, 'value must contain a non-whitespace character')\n    .describe(description);\n\nexport const promptImageInputSchema = z.strictObject({\n  name: nonBlankText('The name used to identify this reference image.'),\n  url: nonBlankText('The URL, data URL, or file path of this reference image.'),\n});\n\nexport const structuredUserPromptInputSchema = z.strictObject({\n  prompt: nonBlankText('The natural-language prompt.'),\n  images: z.array(promptImageInputSchema).min(1).optional(),\n  convertHttpImage2Base64: z\n    .boolean()\n    .optional()\n    .describe('Whether HTTP reference images are converted to base64 first.'),\n});\n\nexport const userPromptInputSchema = z.union([\n  nonBlankText('The natural-language prompt.'),\n  structuredUserPromptInputSchema,\n]);\n\nexport const aiActOptionsInputSchema = z.strictObject({\n  effort: z\n    .enum(['fast', 'balance', 'deepThink'])\n    .optional()\n    .describe('Action planning effort: fast, balanced, or deeper reasoning.'),\n  cacheable: z\n    .boolean()\n    .optional()\n    .describe('Whether this action may use the Midscene cache.'),\n  fileChooserAccept: z\n    .union([z.string(), z.array(z.string())])\n    .optional()\n    .describe('Accepted file types for a file chooser.'),\n  fileChooserAllowedDir: z\n    .string()\n    .optional()\n    .describe('Directory allowed for prompt-driven file uploads.'),\n  deepThink: z\n    .union([z.literal('unset'), z.boolean()])\n    .optional()\n    .describe('Whether to enable deep thinking for this action.'),\n  deepLocate: z\n    .boolean()\n    .optional()\n    .describe('Whether to use deep element location.'),\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.',\n    ),\n});\n\nexport const setAIContextInputSchema = z.strictObject({\n  target: z\n    .enum(AGENT_AI_CONTEXT_KEYS)\n    .describe(\n      'The AI API to configure, e.g. aiAct or aiAssert. Use default as the shared fallback for all AI APIs when neither the call nor the API has its own context.',\n    ),\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Agent-level guidance. Omit to remove the selected context; an empty string disables inherited context.',\n    ),\n});\n\nexport const aiActInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: aiActOptionsInputSchema.optional(),\n});\n\nexport const insightOptionsInputSchema = z.strictObject({\n  domIncluded: z\n    .union([z.boolean(), z.literal('visible-only')])\n    .optional()\n    .describe('How DOM information is included.'),\n  screenshotIncluded: z\n    .boolean()\n    .optional()\n    .describe('Whether the request includes a screenshot.'),\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.',\n    ),\n});\n\nexport const aiAssertOptionsInputSchema = insightOptionsInputSchema.extend({\n  keepRawResponse: z\n    .boolean()\n    .optional()\n    .describe('Whether the Agent returns the structured assertion result.'),\n});\n\nexport const aiAssertInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  message: z.string().optional().describe('The assertion failure message.'),\n  options: aiAssertOptionsInputSchema.optional(),\n});\n\nexport const locateOptionsInputSchema = z.strictObject({\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.',\n    ),\n  deepLocate: z\n    .boolean()\n    .optional()\n    .describe('Whether to use deep element location.'),\n  deepThink: z\n    .boolean()\n    .optional()\n    .describe('Deprecated alias for deepLocate.'),\n  cacheable: z\n    .boolean()\n    .optional()\n    .describe('Whether this location may use the Midscene cache.'),\n  xpath: z.string().optional().describe('A web XPath location hint.'),\n  fileChooserAccept: z\n    .union([z.string(), z.array(z.string())])\n    .optional()\n    .describe('Files accepted when tapping opens a file chooser.'),\n});\n\nexport const aiTapInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: locateOptionsInputSchema.optional(),\n});\n\nexport const aiInputInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  value: z.union([z.string(), z.number()]),\n  options: locateOptionsInputSchema\n    .extend({\n      mode: z.enum(['replace', 'clear', 'typeOnly', 'append']).optional(),\n      autoDismissKeyboard: z.boolean().optional(),\n      keyboardTypeDelay: z.number().nonnegative().optional(),\n      inputStrategy: z.enum(inputStrategies).optional(),\n    })\n    .optional(),\n});\n\nexport const aiKeyboardPressInputSchema = z.strictObject({\n  prompt: userPromptInputSchema.optional(),\n  keyName: nonBlankText('The key or key combination to press.'),\n  options: locateOptionsInputSchema.optional(),\n});\n\nexport const aiScrollInputSchema = z.strictObject({\n  prompt: userPromptInputSchema.optional(),\n  options: locateOptionsInputSchema\n    .extend({\n      direction: z.enum(['down', 'up', 'right', 'left']).optional(),\n      scrollType: z\n        .enum([\n          'singleAction',\n          'scrollToBottom',\n          'scrollToTop',\n          'scrollToRight',\n          'scrollToLeft',\n          'once',\n          'untilBottom',\n          'untilTop',\n          'untilRight',\n          'untilLeft',\n        ])\n        .optional(),\n      distance: z.number().nullable().optional(),\n    })\n    .optional(),\n});\n\nexport const aiPinchInputSchema = z.strictObject({\n  prompt: userPromptInputSchema.optional(),\n  direction: z.enum(['in', 'out']),\n  options: locateOptionsInputSchema\n    .extend({\n      distance: z.number().optional(),\n      duration: z.number().nonnegative().optional(),\n    })\n    .optional(),\n});\n\nexport const aiLongPressInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: locateOptionsInputSchema\n    .extend({ duration: z.number().nonnegative().optional() })\n    .optional(),\n});\n\nexport const aiDragAndDropInputSchema = z.strictObject({\n  from: userPromptInputSchema,\n  to: userPromptInputSchema,\n  options: locateOptionsInputSchema.optional(),\n});\n\nexport const aiQueryInputSchema = z.strictObject({\n  prompt: z.union([\n    nonBlankText('The extraction request.'),\n    z.record(z.string(), z.string()),\n  ]),\n  options: insightOptionsInputSchema.optional(),\n});\n\nexport const aiWaitForOptionsInputSchema = insightOptionsInputSchema\n  .extend({\n    timeoutMs: z.number().positive().optional(),\n    checkIntervalMs: z.number().positive().optional(),\n  })\n  .superRefine((input, context) => {\n    if ((input.checkIntervalMs ?? 3000) > (input.timeoutMs ?? 15000)) {\n      context.addIssue({\n        code: 'custom',\n        path: ['checkIntervalMs'],\n        message: 'checkIntervalMs must not exceed timeoutMs',\n      });\n    }\n  });\n\nexport const aiWaitForInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: aiWaitForOptionsInputSchema.optional(),\n});\n\nexport type AiWaitForNodeOptions = z.infer<typeof aiWaitForOptionsInputSchema>;\n\nexport const javascriptInputSchema = z.strictObject({\n  script: nonBlankText('JavaScript to evaluate through the current interface.'),\n});\n\nexport const runGherkinScenarioInputSchema = z.strictObject({\n  scenario: nonBlankText('The Gherkin scenario to execute.'),\n  options: aiActOptionsInputSchema.optional(),\n});\n\nexport const actionInputSchema = z.strictObject({\n  name: nonBlankText('The ActionSpace action name, for example DragAndDrop.'),\n  params: z.unknown().optional(),\n});\n\nexport const insightInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: insightOptionsInputSchema.optional(),\n});\n\nexport const reportScreenshotInputSchema = z.strictObject({\n  base64: z.string().min(1).describe('A base64-encoded screenshot.'),\n  description: z.string().optional().describe('What the screenshot shows.'),\n});\n\nexport const recordToReportOptionsInputSchema = z\n  .strictObject({\n    content: z.string().optional().describe('The report text content.'),\n    screenshotBase64: z\n      .string()\n      .optional()\n      .describe('One base64-encoded screenshot.'),\n    screenshots: z\n      .array(reportScreenshotInputSchema)\n      .min(1)\n      .optional()\n      .describe('Screenshots attached to the report section.'),\n  })\n  .superRefine((input, context) => {\n    if (\n      input.screenshotBase64 !== undefined &&\n      input.screenshots !== undefined\n    ) {\n      context.addIssue({\n        code: 'custom',\n        message: 'screenshotBase64 and screenshots are mutually exclusive',\n      });\n    }\n  });\n\nexport const recordToReportInputSchema = z.strictObject({\n  title: z.string().optional().describe('The report section title.'),\n  options: recordToReportOptionsInputSchema.optional(),\n});\n\nexport type UserPromptNodeInput = z.infer<typeof userPromptInputSchema>;\nexport type SetAIContextNodeInput = z.infer<typeof setAIContextInputSchema>;\nexport type AiActNodeOptions = z.infer<typeof aiActOptionsInputSchema>;\nexport type AiActNodeInput = z.infer<typeof aiActInputSchema>;\nexport type AiAssertNodeOptions = z.infer<typeof aiAssertOptionsInputSchema>;\nexport type AiAssertNodeInput = z.infer<typeof aiAssertInputSchema>;\nexport type AiTapNodeOptions = z.infer<typeof locateOptionsInputSchema>;\nexport type AiTapNodeInput = z.infer<typeof aiTapInputSchema>;\nexport type AiInputNodeInput = z.infer<typeof aiInputInputSchema>;\nexport type AiKeyboardPressNodeInput = z.infer<\n  typeof aiKeyboardPressInputSchema\n>;\nexport type AiScrollNodeInput = z.infer<typeof aiScrollInputSchema>;\nexport type AiPinchNodeInput = z.infer<typeof aiPinchInputSchema>;\nexport type AiLongPressNodeInput = z.infer<typeof aiLongPressInputSchema>;\nexport type AiDragAndDropNodeInput = z.infer<typeof aiDragAndDropInputSchema>;\nexport type AiQueryNodeInput = z.infer<typeof aiQueryInputSchema>;\nexport type AiWaitForNodeInput = z.infer<typeof aiWaitForInputSchema>;\nexport type JavascriptNodeInput = z.infer<typeof javascriptInputSchema>;\nexport type RunGherkinScenarioNodeInput = z.infer<\n  typeof runGherkinScenarioInputSchema\n>;\nexport type ActionNodeInput = z.infer<typeof actionInputSchema>;\nexport type InsightNodeOptions = z.infer<typeof insightOptionsInputSchema>;\nexport type InsightNodeInput = z.infer<typeof insightInputSchema>;\nexport type RecordToReportNodeOptions = z.infer<\n  typeof recordToReportOptionsInputSchema\n>;\nexport type RecordToReportNodeInput = z.infer<typeof recordToReportInputSchema>;\n\nexport type CommonAgentTestRunnerApi = Pick<\n  Agent,\n  | 'aiAct'\n  | 'aiTap'\n  | 'aiAssert'\n  | 'aiBoolean'\n  | 'aiNumber'\n  | 'aiString'\n  | 'aiAsk'\n  | 'recordToReport'\n>;\n\nconst defineCommonAgentNode =\n  createAgentTestRunnerNodeDefinition<CommonAgentTestRunnerApi>('an Agent');\n\n// Select the current overloads without expanding the minimum API required by\n// existing getAgent() providers. Each Node checks its own method at execution.\ntype AgentActionNodeApi = Pick<\n  Agent,\n  | 'sleep'\n  | 'setAIContext'\n  | 'aiHover'\n  | 'aiDoubleClick'\n  | 'aiRightClick'\n  | 'aiMiddleClick'\n  | 'aiLongPress'\n  | 'aiClearInput'\n  | 'aiPinch'\n  | 'aiLocate'\n  | 'aiQuery'\n  | 'aiWaitFor'\n  | 'evaluateJavaScript'\n  | 'runGherkinScenario'\n  | 'callActionInActionSpace'\n> & {\n  aiInput(\n    prompt: TUserPrompt,\n    options: NonNullable<Parameters<Agent['aiInput']>[2]> & {\n      value: string | number;\n    },\n  ): Promise<void>;\n  aiKeyboardPress(\n    prompt: TUserPrompt | undefined,\n    options: LocateOption & { keyName: string },\n  ): Promise<void>;\n  aiScroll(\n    prompt: TUserPrompt | undefined,\n    options: LocateOption & ScrollParam,\n  ): Promise<void>;\n};\n\nconst defineAgentActionNode =\n  createAgentTestRunnerNodeDefinition<AgentActionNodeApi>('an Agent');\n\nconst promptText = (prompt: UserPromptNodeInput): string =>\n  typeof prompt === 'string' ? prompt : prompt.prompt;\n\nconst valueResult = (\n  value: unknown,\n): AgentTestRunnerNodeResult | undefined => ({ data: value });\n\nconst aiActNode = defineCommonAgentNode({\n  method: 'aiAct',\n  description: 'Perform a natural-language task with a Midscene UI Agent.',\n  stringInputKey: 'prompt',\n  inputSchema: aiActInputSchema,\n  toArgs(input, context) {\n    return [\n      input.prompt,\n      {\n        ...input.options,\n        abortSignal: context.signal,\n      },\n    ];\n  },\n  toResult(output) {\n    return output === undefined ? undefined : { summary: output };\n  },\n});\n\nconst aiTapNode = defineCommonAgentNode({\n  method: 'aiTap',\n  description: 'Locate and tap an element with a Midscene UI Agent.',\n  stringInputKey: 'prompt',\n  inputSchema: aiTapInputSchema,\n  toArgs(input) {\n    return [input.prompt, input.options];\n  },\n  toResult(_output, input) {\n    return { summary: `Tapped: ${promptText(input.prompt)}` };\n  },\n});\n\nconst aiAssertNode = defineCommonAgentNode({\n  method: 'aiAssert',\n  description: 'Assert a natural-language condition with a Midscene UI Agent.',\n  stringInputKey: 'prompt',\n  inputSchema: aiAssertInputSchema,\n  toArgs(input, context) {\n    return [\n      input.prompt,\n      input.message,\n      {\n        keepRawResponse: true,\n        ...input.options,\n        abortSignal: context.signal,\n      },\n    ];\n  },\n  toResult(output, input) {\n    const result = {\n      summary: `Assertion ${output?.pass === false ? 'failed' : 'passed'}: ${promptText(input.prompt)}`,\n      ...(output === undefined\n        ? {}\n        : {\n            data: {\n              pass: output.pass,\n              ...(output.thought === undefined\n                ? {}\n                : { thought: output.thought }),\n              ...(output.message === undefined\n                ? {}\n                : { message: output.message }),\n            },\n          }),\n    };\n    if (output?.pass === false)\n      throw new NodeExecutionError(\n        'aiAssert',\n        new Error(\n          output.message ||\n            input.message ||\n            output.thought ||\n            `Assertion failed: ${promptText(input.prompt)}`,\n        ),\n        result,\n      );\n    return result;\n  },\n});\n\nconst locateActionNode = (\n  method:\n    | 'aiHover'\n    | 'aiDoubleClick'\n    | 'aiRightClick'\n    | 'aiMiddleClick'\n    | 'aiClearInput',\n) =>\n  defineAgentActionNode({\n    method,\n    description: {\n      aiHover: 'Locate and hover over an element with a Midscene UI Agent.',\n      aiDoubleClick:\n        'Locate and double-click an element with a Midscene UI Agent.',\n      aiMiddleClick:\n        'Locate and middle-click an element with a Midscene UI Agent.',\n      aiRightClick:\n        'Locate and right-click an element with a Midscene UI Agent.',\n      aiClearInput:\n        'Locate an input element and clear its value with a Midscene UI Agent.',\n    }[method],\n    stringInputKey: 'prompt',\n    inputSchema: aiTapInputSchema,\n    toArgs: (input) => [input.prompt, input.options],\n    toResult: (_output, input) => ({\n      summary: `${method}: ${promptText(input.prompt)}`,\n    }),\n  });\n\nconst additionalAgentNodes: readonly AgentTestRunnerNodeDefinition[] = [\n  defineAgentActionNode({\n    method: 'setAIContext',\n    description:\n      'Set or remove Agent-level AI guidance for subsequent calls on the same Agent.',\n    inputSchema: setAIContextInputSchema,\n    toArgs: (input) => [input.target, input.context],\n  }),\n  locateActionNode('aiHover'),\n  locateActionNode('aiDoubleClick'),\n  locateActionNode('aiRightClick'),\n  locateActionNode('aiMiddleClick'),\n  locateActionNode('aiClearInput'),\n  defineAgentActionNode({\n    method: 'aiInput',\n    description:\n      'Locate an input element and enter a value with a Midscene UI Agent.',\n    inputSchema: aiInputInputSchema,\n    toArgs: (input) => [input.prompt, { ...input.options, value: input.value }],\n  }),\n  defineAgentActionNode({\n    method: 'aiKeyboardPress',\n    description:\n      'Press a key or key combination, optionally targeting a located element.',\n    stringInputKey: 'keyName',\n    inputSchema: aiKeyboardPressInputSchema,\n    toArgs: (input) => [\n      input.prompt,\n      { ...input.options, keyName: input.keyName },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'aiScroll',\n    description:\n      'Scroll the page or a located region with a Midscene UI Agent.',\n    stringInputKey: 'prompt',\n    inputSchema: aiScrollInputSchema,\n    toArgs: (input) => [input.prompt, { ...input.options }],\n  }),\n  defineAgentActionNode({\n    method: 'aiPinch',\n    description:\n      'Perform a pinch-in or pinch-out gesture with a Midscene UI Agent.',\n    inputSchema: aiPinchInputSchema,\n    toArgs: (input) => [\n      input.prompt,\n      { ...input.options, direction: input.direction },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'aiLongPress',\n    description: 'Locate and long-press an element with a Midscene UI Agent.',\n    stringInputKey: 'prompt',\n    inputSchema: aiLongPressInputSchema,\n    toArgs: (input) => [input.prompt, input.options],\n  }),\n  defineAgentActionNode({\n    method: 'callActionInActionSpace',\n    name: 'aiDragAndDrop',\n    description:\n      'Locate source and destination elements, then drag the source to the destination.',\n    inputSchema: aiDragAndDropInputSchema,\n    toArgs: (input) => [\n      'DragAndDrop',\n      {\n        from: buildDetailedLocateParam(input.from, input.options),\n        to: buildDetailedLocateParam(input.to, input.options),\n      },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'aiLocate',\n    description:\n      'Locate an element from a natural-language description and store the result.',\n    stringInputKey: 'prompt',\n    inputSchema: aiTapInputSchema,\n    toArgs: (input, context) => [\n      input.prompt,\n      { ...input.options, abortSignal: context.signal },\n    ],\n    toResult: valueResult,\n  }),\n  defineAgentActionNode({\n    method: 'aiQuery',\n    description:\n      'Extract structured data from the current interface and store the result.',\n    stringInputKey: 'prompt',\n    inputSchema: aiQueryInputSchema,\n    toArgs: (input, context) => [\n      input.prompt,\n      { ...input.options, abortSignal: context.signal },\n    ],\n    toResult: valueResult,\n  }),\n  defineAgentActionNode({\n    method: 'aiWaitFor',\n    description: 'Wait until a natural-language condition is satisfied.',\n    stringInputKey: 'prompt',\n    inputSchema: aiWaitForInputSchema,\n    toArgs: (input, context) => [\n      input.prompt,\n      { ...input.options, abortSignal: context.signal },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'evaluateJavaScript',\n    name: 'javascript',\n    description:\n      'Evaluate JavaScript in the current interface and store the result.',\n    stringInputKey: 'script',\n    inputSchema: javascriptInputSchema,\n    toArgs: (input) => [input.script],\n    toResult: valueResult,\n  }),\n  defineAgentActionNode({\n    method: 'runGherkinScenario',\n    description:\n      'Execute a Gherkin scenario with the current Midscene UI Agent.',\n    stringInputKey: 'scenario',\n    inputSchema: runGherkinScenarioInputSchema,\n    toArgs: (input, context) => [\n      input.scenario,\n      { ...input.options, abortSignal: context.signal },\n    ],\n  }),\n  defineAgentActionNode({\n    method: 'callActionInActionSpace',\n    name: 'action',\n    description:\n      'Call a platform or custom action. Its ActionSpace schema validates params when the action executes.',\n    inputSchema: actionInputSchema,\n    toArgs: (input) => [input.name, input.params],\n    toResult: (value) => (value === undefined ? undefined : valueResult(value)),\n  }),\n];\n\nconst insightNode = (method: 'aiBoolean' | 'aiNumber' | 'aiString' | 'aiAsk') =>\n  defineCommonAgentNode({\n    method,\n    description: `Run ${method} with a Midscene UI Agent and store its value.`,\n    stringInputKey: 'prompt',\n    inputSchema: insightInputSchema,\n    toArgs: (input, context) => [\n      input.prompt,\n      { ...input.options, abortSignal: context.signal },\n    ],\n    toResult: (value) => ({\n      summary: `${method} returned ${\n        method === 'aiString' || method === 'aiAsk'\n          ? JSON.stringify(value)\n          : value\n      }`,\n      data: { value },\n    }),\n  });\n\nconst recordToReportNode = defineCommonAgentNode({\n  method: 'recordToReport',\n  description: 'Add text or screenshots to the current Midscene report.',\n  stringInputKey: 'title',\n  inputSchema: recordToReportInputSchema,\n  toArgs(input) {\n    return [input.title, input.options];\n  },\n  toResult(_output, input) {\n    return { summary: `Recorded to report: ${input.title ?? 'untitled'}` };\n  },\n});\n\nexport const commonAgentTestRunnerNodeDefinitions: readonly AgentTestRunnerNodeDefinition[] =\n  [\n    defineAgentActionNode({\n      method: 'sleep',\n      description:\n        'Wait for a fixed number of milliseconds, recording standard UI snapshots and honoring cancellation.',\n      inputSchema: z.strictObject({ ms: z.number().positive() }),\n      toArgs(input, { signal }) {\n        signal.throwIfAborted();\n        return [input.ms, { abortSignal: signal }];\n      },\n    }),\n    aiActNode,\n    aiTapNode,\n    aiAssertNode,\n    insightNode('aiBoolean'),\n    insightNode('aiNumber'),\n    insightNode('aiString'),\n    insightNode('aiAsk'),\n    recordToReportNode,\n    ...additionalAgentNodes,\n  ];\n"],"names":["createAgentTestRunnerNodeDefinition","agentLabel","definition","undefined","agent","input","context","TypeError","typedAgent","method","typedInput","result","nonBlankText","description","z","promptImageInputSchema","structuredUserPromptInputSchema","userPromptInputSchema","aiActOptionsInputSchema","setAIContextInputSchema","AGENT_AI_CONTEXT_KEYS","aiActInputSchema","insightOptionsInputSchema","aiAssertOptionsInputSchema","aiAssertInputSchema","locateOptionsInputSchema","aiTapInputSchema","aiInputInputSchema","inputStrategies","aiKeyboardPressInputSchema","aiScrollInputSchema","aiPinchInputSchema","aiLongPressInputSchema","aiDragAndDropInputSchema","aiQueryInputSchema","aiWaitForOptionsInputSchema","aiWaitForInputSchema","javascriptInputSchema","runGherkinScenarioInputSchema","actionInputSchema","insightInputSchema","reportScreenshotInputSchema","recordToReportOptionsInputSchema","recordToReportInputSchema","defineCommonAgentNode","defineAgentActionNode","promptText","prompt","valueResult","value","aiActNode","output","aiTapNode","_output","aiAssertNode","NodeExecutionError","Error","locateActionNode","additionalAgentNodes","buildDetailedLocateParam","insightNode","JSON","recordToReportNode","commonAgentTestRunnerNodeDefinitions","signal"],"mappings":";;;;;AA6EO,MAAMA,sCACX,CAAwBC,aAAa,UAAU,GAC/C,CAKEC,aAMmC;YACnC,MAAMA,WAAW,IAAI,IAAIA,WAAW,MAAM;YAC1C,GAAIA,AAAqBC,WAArBD,WAAW,KAAK,GAAiB,CAAC,IAAI;gBAAE,OAAOA,WAAW,KAAK;YAAC,CAAC;YACrE,GAAIA,AAA2BC,WAA3BD,WAAW,WAAW,GACtB,CAAC,IACD;gBAAE,aAAaA,WAAW,WAAW;YAAC,CAAC;YAC3C,GAAIA,AAA8BC,WAA9BD,WAAW,cAAc,GACzB,CAAC,IACD;gBAAE,gBAAgBA,WAAW,cAAc;YAAC,CAAC;YACjD,aAAaA,WAAW,WAAW;YACnC,MAAM,SAAQE,KAAK,EAAEC,KAAK,EAAEC,OAAO;gBACjC,IACE,AAAiB,YAAjB,OAAOF,SACPA,AAAU,SAAVA,SACA,AACE,cADF,OAAQA,KAAiC,CAACF,WAAW,MAAM,CAAC,EAG5D,MAAM,IAAIK,UACR,CAAC,uBAAuB,EAAEN,WAAW,MAAM,EAAEC,WAAW,MAAM,CAAC,GAAG,CAAC;gBAIvE,MAAMM,aAAaJ;gBACnB,MAAMK,SAASD,UAAU,CAACN,WAAW,MAAM,CAAC;gBAI5C,MAAMQ,aAAaL;gBACnB,MAAMM,SAAU,MAAMF,OAAO,KAAK,CAChCD,YACAN,WAAW,MAAM,CAACQ,YAAYJ;gBAEhC,OAAOJ,WAAW,QAAQ,GAAGS,QAAQD;YACvC;QACF;AAEF,MAAME,eAAe,CAACC,cACpBC,EAAAA,MACS,GACN,KAAK,CAAC,MAAM,iDACZ,QAAQ,CAACD;AAEP,MAAME,yBAAyBD,EAAE,YAAY,CAAC;IACnD,MAAMF,aAAa;IACnB,KAAKA,aAAa;AACpB;AAEO,MAAMI,kCAAkCF,EAAE,YAAY,CAAC;IAC5D,QAAQF,aAAa;IACrB,QAAQE,EAAE,KAAK,CAACC,wBAAwB,GAAG,CAAC,GAAG,QAAQ;IACvD,yBAAyBD,EAAAA,OACf,GACP,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMG,wBAAwBH,EAAE,KAAK,CAAC;IAC3CF,aAAa;IACbI;CACD;AAEM,MAAME,0BAA0BJ,EAAE,YAAY,CAAC;IACpD,QAAQA,CAAC,CAADA,OACD,CAAC;QAAC;QAAQ;QAAW;KAAY,EACrC,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,mBAAmBA,EAAAA,KACX,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,KAAK,CAACA,EAAE,MAAM;KAAI,EACvC,QAAQ,GACR,QAAQ,CAAC;IACZ,uBAAuBA,EAAAA,MACd,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAAA,KACH,CAAC;QAACA,EAAE,OAAO,CAAC;QAAUA,EAAE,OAAO;KAAG,EACvC,QAAQ,GACR,QAAQ,CAAC;IACZ,YAAYA,EAAAA,OACF,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,SAASA,EAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMK,0BAA0BL,EAAE,YAAY,CAAC;IACpD,QAAQA,CAAC,CAADA,OACD,CAACM,uBACL,QAAQ,CACP;IAEJ,SAASN,EAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMO,mBAAmBP,EAAE,YAAY,CAAC;IAC7C,QAAQG;IACR,SAASC,wBAAwB,QAAQ;AAC3C;AAEO,MAAMI,4BAA4BR,EAAE,YAAY,CAAC;IACtD,aAAaA,EAAAA,KACL,CAAC;QAACA,EAAE,OAAO;QAAIA,EAAE,OAAO,CAAC;KAAgB,EAC9C,QAAQ,GACR,QAAQ,CAAC;IACZ,oBAAoBA,EAAAA,OACV,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,SAASA,EAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMS,6BAA6BD,0BAA0B,MAAM,CAAC;IACzE,iBAAiBR,EAAAA,OACP,GACP,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMU,sBAAsBV,EAAE,YAAY,CAAC;IAChD,QAAQG;IACR,SAASH,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACxC,SAASS,2BAA2B,QAAQ;AAC9C;AAEO,MAAME,2BAA2BX,EAAE,YAAY,CAAC;IACrD,SAASA,EAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;IAEJ,YAAYA,EAAAA,OACF,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,OAAOA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACtC,mBAAmBA,EAAAA,KACX,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,KAAK,CAACA,EAAE,MAAM;KAAI,EACvC,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMY,mBAAmBZ,EAAE,YAAY,CAAC;IAC7C,QAAQG;IACR,SAASQ,yBAAyB,QAAQ;AAC5C;AAEO,MAAME,qBAAqBb,EAAE,YAAY,CAAC;IAC/C,QAAQG;IACR,OAAOH,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACvC,SAASW,yBACN,MAAM,CAAC;QACN,MAAMX,CAAC,CAADA,OAAM,CAAC;YAAC;YAAW;YAAS;YAAY;SAAS,EAAE,QAAQ;QACjE,qBAAqBA,EAAE,OAAO,GAAG,QAAQ;QACzC,mBAAmBA,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ;QACpD,eAAeA,CAAC,CAADA,OAAM,CAACc,iBAAiB,QAAQ;IACjD,GACC,QAAQ;AACb;AAEO,MAAMC,6BAA6Bf,EAAE,YAAY,CAAC;IACvD,QAAQG,sBAAsB,QAAQ;IACtC,SAASL,aAAa;IACtB,SAASa,yBAAyB,QAAQ;AAC5C;AAEO,MAAMK,sBAAsBhB,EAAE,YAAY,CAAC;IAChD,QAAQG,sBAAsB,QAAQ;IACtC,SAASQ,yBACN,MAAM,CAAC;QACN,WAAWX,CAAC,CAADA,OAAM,CAAC;YAAC;YAAQ;YAAM;YAAS;SAAO,EAAE,QAAQ;QAC3D,YAAYA,CAAC,CAADA,OACL,CAAC;YACJ;YACA;YACA;YACA;YACA;YACA;YACA;YACA;YACA;YACA;SACD,EACA,QAAQ;QACX,UAAUA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ;IAC1C,GACC,QAAQ;AACb;AAEO,MAAMiB,qBAAqBjB,EAAE,YAAY,CAAC;IAC/C,QAAQG,sBAAsB,QAAQ;IACtC,WAAWH,CAAC,CAADA,OAAM,CAAC;QAAC;QAAM;KAAM;IAC/B,SAASW,yBACN,MAAM,CAAC;QACN,UAAUX,EAAE,MAAM,GAAG,QAAQ;QAC7B,UAAUA,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ;IAC7C,GACC,QAAQ;AACb;AAEO,MAAMkB,yBAAyBlB,EAAE,YAAY,CAAC;IACnD,QAAQG;IACR,SAASQ,yBACN,MAAM,CAAC;QAAE,UAAUX,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ;IAAG,GACvD,QAAQ;AACb;AAEO,MAAMmB,2BAA2BnB,EAAE,YAAY,CAAC;IACrD,MAAMG;IACN,IAAIA;IACJ,SAASQ,yBAAyB,QAAQ;AAC5C;AAEO,MAAMS,qBAAqBpB,EAAE,YAAY,CAAC;IAC/C,QAAQA,EAAE,KAAK,CAAC;QACdF,aAAa;QACbE,EAAE,MAAM,CAACA,EAAE,MAAM,IAAIA,EAAE,MAAM;KAC9B;IACD,SAASQ,0BAA0B,QAAQ;AAC7C;AAEO,MAAMa,8BAA8Bb,0BACxC,MAAM,CAAC;IACN,WAAWR,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ;IACzC,iBAAiBA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ;AACjD,GACC,WAAW,CAAC,CAACT,OAAOC;IACnB,IAAKD,AAAAA,CAAAA,MAAM,eAAe,IAAI,IAAG,IAAMA,CAAAA,MAAM,SAAS,IAAI,KAAI,GAC5DC,QAAQ,QAAQ,CAAC;QACf,MAAM;QACN,MAAM;YAAC;SAAkB;QACzB,SAAS;IACX;AAEJ;AAEK,MAAM8B,uBAAuBtB,EAAE,YAAY,CAAC;IACjD,QAAQG;IACR,SAASkB,4BAA4B,QAAQ;AAC/C;AAIO,MAAME,wBAAwBvB,EAAE,YAAY,CAAC;IAClD,QAAQF,aAAa;AACvB;AAEO,MAAM0B,gCAAgCxB,EAAE,YAAY,CAAC;IAC1D,UAAUF,aAAa;IACvB,SAASM,wBAAwB,QAAQ;AAC3C;AAEO,MAAMqB,oBAAoBzB,EAAE,YAAY,CAAC;IAC9C,MAAMF,aAAa;IACnB,QAAQE,EAAE,OAAO,GAAG,QAAQ;AAC9B;AAEO,MAAM0B,qBAAqB1B,EAAE,YAAY,CAAC;IAC/C,QAAQG;IACR,SAASK,0BAA0B,QAAQ;AAC7C;AAEO,MAAMmB,8BAA8B3B,EAAE,YAAY,CAAC;IACxD,QAAQA,EAAE,MAAM,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC;IACnC,aAAaA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAC9C;AAEO,MAAM4B,mCAAmC5B,EAAAA,YACjC,CAAC;IACZ,SAASA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACxC,kBAAkBA,EAAAA,MACT,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,aAAaA,EAAAA,KACL,CAAC2B,6BACN,GAAG,CAAC,GACJ,QAAQ,GACR,QAAQ,CAAC;AACd,GACC,WAAW,CAAC,CAACpC,OAAOC;IACnB,IACED,AAA2BF,WAA3BE,MAAM,gBAAgB,IACtBA,AAAsBF,WAAtBE,MAAM,WAAW,EAEjBC,QAAQ,QAAQ,CAAC;QACf,MAAM;QACN,SAAS;IACX;AAEJ;AAEK,MAAMqC,4BAA4B7B,EAAE,YAAY,CAAC;IACtD,OAAOA,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACtC,SAAS4B,iCAAiC,QAAQ;AACpD;AA4CA,MAAME,wBACJ5C,oCAA8D;AAsChE,MAAM6C,wBACJ7C,oCAAwD;AAE1D,MAAM8C,aAAa,CAACC,SAClB,AAAkB,YAAlB,OAAOA,SAAsBA,SAASA,OAAO,MAAM;AAErD,MAAMC,cAAc,CAClBC,QAC2C;QAAE,MAAMA;IAAM;AAE3D,MAAMC,YAAYN,sBAAsB;IACtC,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAavB;IACb,QAAOhB,KAAK,EAAEC,OAAO;QACnB,OAAO;YACLD,MAAM,MAAM;YACZ;gBACE,GAAGA,MAAM,OAAO;gBAChB,aAAaC,QAAQ,MAAM;YAC7B;SACD;IACH;IACA,UAAS6C,MAAM;QACb,OAAOA,AAAWhD,WAAXgD,SAAuBhD,SAAY;YAAE,SAASgD;QAAO;IAC9D;AACF;AAEA,MAAMC,YAAYR,sBAAsB;IACtC,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAalB;IACb,QAAOrB,KAAK;QACV,OAAO;YAACA,MAAM,MAAM;YAAEA,MAAM,OAAO;SAAC;IACtC;IACA,UAASgD,OAAO,EAAEhD,KAAK;QACrB,OAAO;YAAE,SAAS,CAAC,QAAQ,EAAEyC,WAAWzC,MAAM,MAAM,GAAG;QAAC;IAC1D;AACF;AAEA,MAAMiD,eAAeV,sBAAsB;IACzC,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAapB;IACb,QAAOnB,KAAK,EAAEC,OAAO;QACnB,OAAO;YACLD,MAAM,MAAM;YACZA,MAAM,OAAO;YACb;gBACE,iBAAiB;gBACjB,GAAGA,MAAM,OAAO;gBAChB,aAAaC,QAAQ,MAAM;YAC7B;SACD;IACH;IACA,UAAS6C,MAAM,EAAE9C,KAAK;QACpB,MAAMM,SAAS;YACb,SAAS,CAAC,UAAU,EAAEwC,QAAQ,SAAS,QAAQ,WAAW,SAAS,EAAE,EAAEL,WAAWzC,MAAM,MAAM,GAAG;YACjG,GAAI8C,AAAWhD,WAAXgD,SACA,CAAC,IACD;gBACE,MAAM;oBACJ,MAAMA,OAAO,IAAI;oBACjB,GAAIA,AAAmBhD,WAAnBgD,OAAO,OAAO,GACd,CAAC,IACD;wBAAE,SAASA,OAAO,OAAO;oBAAC,CAAC;oBAC/B,GAAIA,AAAmBhD,WAAnBgD,OAAO,OAAO,GACd,CAAC,IACD;wBAAE,SAASA,OAAO,OAAO;oBAAC,CAAC;gBACjC;YACF,CAAC;QACP;QACA,IAAIA,QAAQ,SAAS,OACnB,MAAM,IAAII,mBACR,YACA,IAAIC,MACFL,OAAO,OAAO,IACZ9C,MAAM,OAAO,IACb8C,OAAO,OAAO,IACd,CAAC,kBAAkB,EAAEL,WAAWzC,MAAM,MAAM,GAAG,GAEnDM;QAEJ,OAAOA;IACT;AACF;AAEA,MAAM8C,mBAAmB,CACvBhD,SAOAoC,sBAAsB;QACpBpC;QACA,aAAa;YACX,SAAS;YACT,eACE;YACF,eACE;YACF,cACE;YACF,cACE;QACJ,CAAC,CAACA,OAAO;QACT,gBAAgB;QAChB,aAAaiB;QACb,QAAQ,CAACrB,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;QAChD,UAAU,CAACgD,SAAShD,QAAW;gBAC7B,SAAS,GAAGI,OAAO,EAAE,EAAEqC,WAAWzC,MAAM,MAAM,GAAG;YACnD;IACF;AAEF,MAAMqD,uBAAiE;IACrEb,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,aAAa1B;QACb,QAAQ,CAACd,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;IAClD;IACAoD,iBAAiB;IACjBA,iBAAiB;IACjBA,iBAAiB;IACjBA,iBAAiB;IACjBA,iBAAiB;IACjBZ,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,aAAalB;QACb,QAAQ,CAACtB,QAAU;gBAACA,MAAM,MAAM;gBAAE;oBAAE,GAAGA,MAAM,OAAO;oBAAE,OAAOA,MAAM,KAAK;gBAAC;aAAE;IAC7E;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAahB;QACb,QAAQ,CAACxB,QAAU;gBACjBA,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,SAASA,MAAM,OAAO;gBAAC;aAC5C;IACH;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAaf;QACb,QAAQ,CAACzB,QAAU;gBAACA,MAAM,MAAM;gBAAE;oBAAE,GAAGA,MAAM,OAAO;gBAAC;aAAE;IACzD;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,aAAad;QACb,QAAQ,CAAC1B,QAAU;gBACjBA,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,WAAWA,MAAM,SAAS;gBAAC;aAChD;IACH;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aAAa;QACb,gBAAgB;QAChB,aAAab;QACb,QAAQ,CAAC3B,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;IAClD;IACAwC,sBAAsB;QACpB,QAAQ;QACR,MAAM;QACN,aACE;QACF,aAAaZ;QACb,QAAQ,CAAC5B,QAAU;gBACjB;gBACA;oBACE,MAAMsD,yBAAyBtD,MAAM,IAAI,EAAEA,MAAM,OAAO;oBACxD,IAAIsD,yBAAyBtD,MAAM,EAAE,EAAEA,MAAM,OAAO;gBACtD;aACD;IACH;IACAwC,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAanB;QACb,QAAQ,CAACrB,OAAOC,UAAY;gBAC1BD,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;QACD,UAAU0C;IACZ;IACAH,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAaX;QACb,QAAQ,CAAC7B,OAAOC,UAAY;gBAC1BD,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;QACD,UAAU0C;IACZ;IACAH,sBAAsB;QACpB,QAAQ;QACR,aAAa;QACb,gBAAgB;QAChB,aAAaT;QACb,QAAQ,CAAC/B,OAAOC,UAAY;gBAC1BD,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;IACH;IACAuC,sBAAsB;QACpB,QAAQ;QACR,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaR;QACb,QAAQ,CAAChC,QAAU;gBAACA,MAAM,MAAM;aAAC;QACjC,UAAU2C;IACZ;IACAH,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,gBAAgB;QAChB,aAAaP;QACb,QAAQ,CAACjC,OAAOC,UAAY;gBAC1BD,MAAM,QAAQ;gBACd;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;IACH;IACAuC,sBAAsB;QACpB,QAAQ;QACR,MAAM;QACN,aACE;QACF,aAAaN;QACb,QAAQ,CAAClC,QAAU;gBAACA,MAAM,IAAI;gBAAEA,MAAM,MAAM;aAAC;QAC7C,UAAU,CAAC4C,QAAWA,AAAU9C,WAAV8C,QAAsB9C,SAAY6C,YAAYC;IACtE;CACD;AAED,MAAMW,cAAc,CAACnD,SACnBmC,sBAAsB;QACpBnC;QACA,aAAa,CAAC,IAAI,EAAEA,OAAO,8CAA8C,CAAC;QAC1E,gBAAgB;QAChB,aAAa+B;QACb,QAAQ,CAACnC,OAAOC,UAAY;gBAC1BD,MAAM,MAAM;gBACZ;oBAAE,GAAGA,MAAM,OAAO;oBAAE,aAAaC,QAAQ,MAAM;gBAAC;aACjD;QACD,UAAU,CAAC2C,QAAW;gBACpB,SAAS,GAAGxC,OAAO,UAAU,EAC3BA,AAAW,eAAXA,UAAyBA,AAAW,YAAXA,SACrBoD,KAAK,SAAS,CAACZ,SACfA,OACJ;gBACF,MAAM;oBAAEA;gBAAM;YAChB;IACF;AAEF,MAAMa,qBAAqBlB,sBAAsB;IAC/C,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAaD;IACb,QAAOtC,KAAK;QACV,OAAO;YAACA,MAAM,KAAK;YAAEA,MAAM,OAAO;SAAC;IACrC;IACA,UAASgD,OAAO,EAAEhD,KAAK;QACrB,OAAO;YAAE,SAAS,CAAC,oBAAoB,EAAEA,MAAM,KAAK,IAAI,YAAY;QAAC;IACvE;AACF;AAEO,MAAM0D,uCACX;IACElB,sBAAsB;QACpB,QAAQ;QACR,aACE;QACF,aAAa/B,EAAE,YAAY,CAAC;YAAE,IAAIA,EAAE,MAAM,GAAG,QAAQ;QAAG;QACxD,QAAOT,KAAK,EAAE,EAAE2D,MAAM,EAAE;YACtBA,OAAO,cAAc;YACrB,OAAO;gBAAC3D,MAAM,EAAE;gBAAE;oBAAE,aAAa2D;gBAAO;aAAE;QAC5C;IACF;IACAd;IACAE;IACAE;IACAM,YAAY;IACZA,YAAY;IACZA,YAAY;IACZA,YAAY;IACZE;OACGJ;CACJ"}

package/dist/es/agent/utils.mjs

@@ -153,7 +153,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.17.4-beta-20261030030502.0";
+const getMidsceneVersion = ()=>"1.17.4";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/model-adapter/default-planning-protocol.mjs

@@ -1,36 +1,51 @@
-import { getZodDescription, getZodTypeName, isMidsceneLocatorField } from "@midscene/shared/zod-schema-utils";
+import { findAllMidsceneLocatorField } from "../../common.mjs";
+import { getZodDescription, getZodTypeName, unwrapZodField } from "@midscene/shared/zod-schema-utils";
 import { getZodDefaultValue } from "../shared/action-schema.mjs";
 import { assertJsonObject } from "../shared/json.mjs";
 import { formatLocateExampleValue } from "../shared/model-locate-result/index.mjs";
 import { extractXMLTag } from "../shared/xml.mjs";
+const buildActionParamDescription = (field, locateFieldDescription)=>{
+    const isLocator = void 0 !== locateFieldDescription;
+    const result = {
+        type: isLocator ? locateFieldDescription : getZodTypeName(field)
+    };
+    if (field.isOptional()) result.optional = true;
+    const description = getZodDescription(field);
+    if (description) result.description = description;
+    const defaultValue = getZodDefaultValue(field);
+    if (void 0 !== defaultValue) result.default = defaultValue;
+    if (isLocator) return result;
+    const actualField = unwrapZodField(field);
+    const definition = actualField._def;
+    if ('ZodObject' === definition.typeName) result.properties = Object.fromEntries(Object.entries(definition.shape()).map(([name, nested])=>[
+            name,
+            buildActionParamDescription(nested)
+        ]));
+    else if ('ZodArray' === definition.typeName) result.items = buildActionParamDescription(definition.type);
+    else if ('ZodUnion' === definition.typeName || 'ZodDiscriminatedUnion' === definition.typeName) {
+        const variants = definition.options.map((option)=>buildActionParamDescription(option));
+        result.type = variants.map((variant)=>variant.type).join(' | ');
+        if (variants.some((variant)=>variant.properties || variant.items || variant.anyOf)) result.anyOf = variants;
+    } else if ('ZodLiteral' === definition.typeName) result.type = 'string' == typeof definition.value ? JSON.stringify(definition.value) : String(definition.value);
+    else if ('ZodNull' === definition.typeName) result.type = 'null';
+    return result;
+};
 const buildActionDescription = ({ action, locateFieldDescription, actionOutputExample })=>{
     const actionDescription = {
         type: action.name,
         description: action.description || "No description provided"
     };
     if (action.paramSchema) {
-        const schema = action.paramSchema;
+        const schema = unwrapZodField(action.paramSchema);
         const isZodObject = schema._def?.typeName === 'ZodObject';
         if (isZodObject && schema.shape) {
             const param = {};
-            for (const [key, field] of Object.entries(schema.shape))if (field && 'object' == typeof field) {
-                const isOptional = 'function' == typeof field.isOptional && field.isOptional();
-                const typeDescription = isMidsceneLocatorField(field) ? locateFieldDescription : getZodTypeName(field);
-                const description = getZodDescription(field);
-                const defaultValue = getZodDefaultValue(field);
-                const hasDefault = void 0 !== defaultValue;
-                const paramDescription = {
-                    type: typeDescription
-                };
-                if (isOptional) paramDescription.optional = true;
-                if (description) paramDescription.description = description;
-                if (hasDefault) paramDescription.default = defaultValue;
-                param[key] = paramDescription;
-            }
+            const locateFields = findAllMidsceneLocatorField(action.paramSchema);
+            for (const [key, field] of Object.entries(schema.shape))if (field && 'object' == typeof field) param[key] = buildActionParamDescription(field, locateFields.includes(key) ? locateFieldDescription : void 0);
             if (Object.keys(param).length > 0) actionDescription.param = param;
         } else {
             const typeName = getZodTypeName(schema);
-            const description = getZodDescription(schema);
+            const description = getZodDescription(action.paramSchema);
             const paramDescription = {
                 type: typeName,
                 instruction: 'Pass the value directly, not as an object.'

package/dist/es/ai-model/model-adapter/default-planning-protocol.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/default-planning-protocol.mjs","sources":["../../../../src/ai-model/model-adapter/default-planning-protocol.ts"],"sourcesContent":["import {\n  getZodDescription,\n  getZodTypeName,\n  isMidsceneLocatorField,\n} from '@midscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { getZodDefaultValue } from '../shared/action-schema';\nimport { type JsonParser, assertJsonObject } from '../shared/json';\nimport {\n  type LocateResultPromptSpec,\n  formatLocateExampleValue,\n} from '../shared/model-locate-result';\nimport { extractXMLTag } from '../shared/xml';\nimport type {\n  ParsedPlanningLocateParameter,\n  PlanningActionDescriptionBuildInput,\n  PlanningActionOutputBuildInput,\n  StandardPlanningProtocolFactory,\n} from './planning-protocol';\n\ntype ActionParamDescription = {\n  type: string;\n  optional?: true;\n  description?: string;\n  default?: unknown;\n  instruction?: string;\n};\n\ntype ActionDescription = {\n  type: string;\n  description: string;\n  param?: Record<string, ActionParamDescription> | ActionParamDescription;\n  sample?: string;\n};\n\nexport const buildActionDescription = ({\n  action,\n  locateFieldDescription,\n  actionOutputExample,\n}: PlanningActionDescriptionBuildInput) => {\n  const actionDescription: ActionDescription = {\n    type: action.name,\n    description: action.description || 'No description provided',\n  };\n\n  if (action.paramSchema) {\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      const param: Record<string, ActionParamDescription> = {};\n\n      for (const [key, field] of Object.entries(schema.shape)) {\n        if (field && typeof field === 'object') {\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          // Deliberately do not resolve locators nested in ZodUnion here:\n          // findAllMidsceneLocatorField does not recognize them either, so\n          // describing them as locators would diverge from the execution path.\n          const typeDescription = isMidsceneLocatorField(field)\n            ? locateFieldDescription\n            : getZodTypeName(field);\n          const description = getZodDescription(field as z.ZodTypeAny);\n          const defaultValue = getZodDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          const paramDescription: ActionParamDescription = {\n            type: typeDescription,\n          };\n          if (isOptional) {\n            paramDescription.optional = true;\n          }\n          if (description) {\n            paramDescription.description = description;\n          }\n          if (hasDefault) {\n            paramDescription.default = defaultValue;\n          }\n\n          param[key] = paramDescription;\n        }\n      }\n\n      if (Object.keys(param).length > 0) {\n        actionDescription.param = param;\n      }\n    } else {\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n      const paramDescription: ActionParamDescription = {\n        type: typeName,\n        instruction: 'Pass the value directly, not as an object.',\n      };\n      if (description) {\n        paramDescription.description = description;\n      }\n      actionDescription.param = paramDescription;\n    }\n  }\n\n  if (actionOutputExample) {\n    actionDescription.sample = actionOutputExample;\n  }\n\n  return actionDescription;\n};\n\nexport const buildLocateFieldDescription = (\n  promptSpec?: LocateResultPromptSpec,\n) => {\n  if (promptSpec) {\n    return `{ prompt: string, ${promptSpec.resultKey}: ${promptSpec.resultValueSchema} /* ${promptSpec.resultValueDescription} */ }`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\nconst serializeActionParam = (\n  param: Record<string, unknown>,\n  locateFields: string[],\n  locateResultKey: string,\n) => {\n  const locatorObjects = new WeakSet<object>();\n  for (const field of locateFields) {\n    const value = param[field];\n    if (value && typeof value === 'object') {\n      locatorObjects.add(value);\n    }\n  }\n\n  const originalJson = JSON.stringify(param);\n  // Ensure generated markers cannot collide with strings already present in the sample.\n  let markerPrefix = '__MIDSCENE_LOCATE_RESULT_EXAMPLE_';\n  while (originalJson.includes(markerPrefix)) {\n    markerPrefix = `_${markerPrefix}`;\n  }\n\n  const replacements = new Map<string, string>();\n  const serializedParam = JSON.stringify(\n    param,\n    function (this: object, key, value) {\n      if (locatorObjects.has(this) && key === locateResultKey) {\n        const marker = `${markerPrefix}${replacements.size}__`;\n        replacements.set(marker, formatLocateExampleValue(value));\n        return marker;\n      }\n      return value;\n    },\n    2,\n  );\n\n  return [...replacements].reduce(\n    (result, [marker, locateResultValue]) =>\n      result.replace(JSON.stringify(marker), locateResultValue),\n    serializedParam,\n  );\n};\n\nexport const buildPlanningActionOutput = ({\n  actionName,\n  param,\n  locateFields,\n  locateResultKey,\n}: PlanningActionOutputBuildInput) => `<action-type>${actionName}</action-type>\n<action-param-json>\n${\n  // Keep locate result arrays on one line when serializing action examples.\n  locateFields && locateResultKey\n    ? serializeActionParam(param, locateFields, locateResultKey)\n    : JSON.stringify(param, null, 2)\n}\n</action-param-json>`;\n\nexport const createMidscenePlanningActionOutputParser =\n  (jsonParser: JsonParser) => (content: string) => {\n    const actionType = extractXMLTag(content, 'action-type');\n    const actionParamStr = extractXMLTag(content, 'action-param-json');\n\n    if (!actionType || actionType.toLowerCase() === 'null') {\n      return null;\n    }\n\n    // Strip any trailing XML tags that leaked into the action type.\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n        assertJsonObject(param);\n      } catch (error) {\n        throw new Error(`Failed to parse action-param-json: ${error}`);\n      }\n    }\n\n    return {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  };\n\nexport const createDefaultMidscenePlanningProtocol: StandardPlanningProtocolFactory =\n  ({ jsonParser }) => {\n    const parseActionOutput =\n      createMidscenePlanningActionOutputParser(jsonParser);\n\n    return {\n      actionSpaceProtocol: {\n        title: 'Supporting actions list',\n        format: 'yaml',\n        includeActionOutputExample: true,\n        buildLocateFieldDescription,\n        buildActionDescription,\n      },\n      actionOutputProtocol: {\n        actionOutputTagNames: ['action-type', 'action-param-json'],\n        actionOutputRules: [\n          '- Use the <action-type> and <action-param-json> tags to output the action to be executed.',\n          \"- The value inside <action-type> MUST exactly match the 'type' field of one action in the Supporting actions list. 'complete' is NOT a valid action-type.\",\n          '- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If the selected action provides a \"sample\" field, use the XML structure shown in that sample as the exact format for the action output.',\n        ].join('\\n'),\n        actionOutputPlaceholder: [\n          '<action-type>...</action-type>',\n          '<action-param-json>...</action-param-json>',\n        ].join('\\n'),\n        buildActionOutput: buildPlanningActionOutput,\n        parseActionOutput,\n        parseRawLocateParameter: (value) =>\n          value as ParsedPlanningLocateParameter,\n      },\n    };\n  };\n"],"names":["buildActionDescription","action","locateFieldDescription","actionOutputExample","actionDescription","schema","isZodObject","param","key","field","Object","isOptional","typeDescription","isMidsceneLocatorField","getZodTypeName","description","getZodDescription","defaultValue","getZodDefaultValue","hasDefault","undefined","paramDescription","typeName","buildLocateFieldDescription","promptSpec","serializeActionParam","locateFields","locateResultKey","locatorObjects","WeakSet","value","originalJson","JSON","markerPrefix","replacements","Map","serializedParam","marker","formatLocateExampleValue","result","locateResultValue","buildPlanningActionOutput","actionName","createMidscenePlanningActionOutputParser","jsonParser","content","actionType","extractXMLTag","actionParamStr","type","assertJsonObject","error","Error","createDefaultMidscenePlanningProtocol","parseActionOutput"],"mappings":";;;;;AAmCO,MAAMA,yBAAyB,CAAC,EACrCC,MAAM,EACNC,sBAAsB,EACtBC,mBAAmB,EACiB;IACpC,MAAMC,oBAAuC;QAC3C,MAAMH,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW,IAAI;IACrC;IAEA,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,SAASJ,OAAO,WAAW;QAIjC,MAAMK,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAC/B,MAAME,QAAgD,CAAC;YAEvD,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACL,OAAO,KAAK,EACpD,IAAII,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBACtC,MAAME,aACJ,AACE,cADF,OAAQF,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBAIrD,MAAMG,kBAAkBC,uBAAuBJ,SAC3CP,yBACAY,eAAeL;gBACnB,MAAMM,cAAcC,kBAAkBP;gBACtC,MAAMQ,eAAeC,mBAAmBT;gBACxC,MAAMU,aAAaF,AAAiBG,WAAjBH;gBAEnB,MAAMI,mBAA2C;oBAC/C,MAAMT;gBACR;gBACA,IAAID,YACFU,iBAAiB,QAAQ,GAAG;gBAE9B,IAAIN,aACFM,iBAAiB,WAAW,GAAGN;gBAEjC,IAAII,YACFE,iBAAiB,OAAO,GAAGJ;gBAG7BV,KAAK,CAACC,IAAI,GAAGa;YACf;YAGF,IAAIX,OAAO,IAAI,CAACH,OAAO,MAAM,GAAG,GAC9BH,kBAAkB,KAAK,GAAGG;QAE9B,OAAO;YACL,MAAMe,WAAWR,eAAeT;YAChC,MAAMU,cAAcC,kBAAkBX;YACtC,MAAMgB,mBAA2C;gBAC/C,MAAMC;gBACN,aAAa;YACf;YACA,IAAIP,aACFM,iBAAiB,WAAW,GAAGN;YAEjCX,kBAAkB,KAAK,GAAGiB;QAC5B;IACF;IAEA,IAAIlB,qBACFC,kBAAkB,MAAM,GAAGD;IAG7B,OAAOC;AACT;AAEO,MAAMmB,8BAA8B,CACzCC;IAEA,IAAIA,YACF,OAAO,CAAC,kBAAkB,EAAEA,WAAW,SAAS,CAAC,EAAE,EAAEA,WAAW,iBAAiB,CAAC,IAAI,EAAEA,WAAW,sBAAsB,CAAC,KAAK,CAAC;IAElI,OAAO;AACT;AAEA,MAAMC,uBAAuB,CAC3BlB,OACAmB,cACAC;IAEA,MAAMC,iBAAiB,IAAIC;IAC3B,KAAK,MAAMpB,SAASiB,aAAc;QAChC,MAAMI,QAAQvB,KAAK,CAACE,MAAM;QAC1B,IAAIqB,SAAS,AAAiB,YAAjB,OAAOA,OAClBF,eAAe,GAAG,CAACE;IAEvB;IAEA,MAAMC,eAAeC,KAAK,SAAS,CAACzB;IAEpC,IAAI0B,eAAe;IACnB,MAAOF,aAAa,QAAQ,CAACE,cAC3BA,eAAe,CAAC,CAAC,EAAEA,cAAc;IAGnC,MAAMC,eAAe,IAAIC;IACzB,MAAMC,kBAAkBJ,KAAK,SAAS,CACpCzB,OACA,SAAwBC,GAAG,EAAEsB,KAAK;QAChC,IAAIF,eAAe,GAAG,CAAC,IAAI,KAAKpB,QAAQmB,iBAAiB;YACvD,MAAMU,SAAS,GAAGJ,eAAeC,aAAa,IAAI,CAAC,EAAE,CAAC;YACtDA,aAAa,GAAG,CAACG,QAAQC,yBAAyBR;YAClD,OAAOO;QACT;QACA,OAAOP;IACT,GACA;IAGF,OAAO;WAAII;KAAa,CAAC,MAAM,CAC7B,CAACK,QAAQ,CAACF,QAAQG,kBAAkB,GAClCD,OAAO,OAAO,CAACP,KAAK,SAAS,CAACK,SAASG,oBACzCJ;AAEJ;AAEO,MAAMK,4BAA4B,CAAC,EACxCC,UAAU,EACVnC,KAAK,EACLmB,YAAY,EACZC,eAAe,EACgB,GAAK,CAAC,aAAa,EAAEe,WAAW;;AAEjE,EAEEhB,gBAAgBC,kBACZF,qBAAqBlB,OAAOmB,cAAcC,mBAC1CK,KAAK,SAAS,CAACzB,OAAO,MAAM,GACjC;oBACmB,CAAC;AAEd,MAAMoC,2CACX,CAACC,aAA2B,CAACC;QAC3B,MAAMC,aAAaC,cAAcF,SAAS;QAC1C,MAAMG,iBAAiBD,cAAcF,SAAS;QAE9C,IAAI,CAACC,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IACvC,OAAO;QAIT,MAAMG,OAAOH,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIvC;QAEJ,IAAIyC,gBACF,IAAI;YACFzC,QAAQqC,WAAWI,gBAAgB;gBACjC,QAAQ;gBACR,yBACEC,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAG7B;YACjD;YACA8B,iBAAiB3C;QACnB,EAAE,OAAO4C,OAAO;YACd,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,OAAO;QAC/D;QAGF,OAAO;YACLF;YACA,GAAI1C,AAAUa,WAAVb,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;AAEK,MAAM8C,wCACX,CAAC,EAAET,UAAU,EAAE;IACb,MAAMU,oBACJX,yCAAyCC;IAE3C,OAAO;QACL,qBAAqB;YACnB,OAAO;YACP,QAAQ;YACR,4BAA4B;YAC5BrB;YACAvB;QACF;QACA,sBAAsB;YACpB,sBAAsB;gBAAC;gBAAe;aAAoB;YAC1D,mBAAmB;YAKnB,yBAAyB;YAIzB,mBAAmByC;YACnBa;YACA,yBAAyB,CAACxB,QACxBA;QACJ;IACF;AACF"}
+{"version":3,"file":"ai-model/model-adapter/default-planning-protocol.mjs","sources":["../../../../src/ai-model/model-adapter/default-planning-protocol.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport {\n  getZodDescription,\n  getZodTypeName,\n  unwrapZodField,\n} from '@midscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { getZodDefaultValue } from '../shared/action-schema';\nimport { type JsonParser, assertJsonObject } from '../shared/json';\nimport {\n  type LocateResultPromptSpec,\n  formatLocateExampleValue,\n} from '../shared/model-locate-result';\nimport { extractXMLTag } from '../shared/xml';\nimport type {\n  ParsedPlanningLocateParameter,\n  PlanningActionDescriptionBuildInput,\n  PlanningActionOutputBuildInput,\n  StandardPlanningProtocolFactory,\n} from './planning-protocol';\n\ntype ActionParamDescription = {\n  type: string;\n  optional?: true;\n  description?: string;\n  default?: unknown;\n  instruction?: string;\n  properties?: Record<string, ActionParamDescription>;\n  items?: ActionParamDescription;\n  anyOf?: ActionParamDescription[];\n};\n\ntype ActionDescription = {\n  type: string;\n  description: string;\n  param?: Record<string, ActionParamDescription> | ActionParamDescription;\n  sample?: string;\n};\n\nconst buildActionParamDescription = (\n  field: z.ZodTypeAny,\n  locateFieldDescription?: string,\n): ActionParamDescription => {\n  const isLocator = locateFieldDescription !== undefined;\n  const result: ActionParamDescription = {\n    type: isLocator ? locateFieldDescription : getZodTypeName(field),\n  };\n  if (field.isOptional()) result.optional = true;\n  const description = getZodDescription(field);\n  if (description) result.description = description;\n  const defaultValue = getZodDefaultValue(field);\n  if (defaultValue !== undefined) result.default = defaultValue;\n\n  // Only top-level locator fields are normalized by the execution path.\n  // Ordinary nested objects must retain their own fields, not visual coordinates.\n  if (isLocator) return result;\n  const actualField = unwrapZodField(field) as z.ZodFirstPartySchemaTypes;\n  const definition = actualField._def;\n  if (definition.typeName === 'ZodObject') {\n    result.properties = Object.fromEntries(\n      Object.entries(definition.shape() as z.ZodRawShape).map(\n        ([name, nested]) => [name, buildActionParamDescription(nested)],\n      ),\n    );\n  } else if (definition.typeName === 'ZodArray') {\n    result.items = buildActionParamDescription(definition.type);\n  } else if (\n    definition.typeName === 'ZodUnion' ||\n    definition.typeName === 'ZodDiscriminatedUnion'\n  ) {\n    const variants: ActionParamDescription[] = definition.options.map(\n      (option: z.ZodTypeAny) => buildActionParamDescription(option),\n    );\n    result.type = variants.map((variant) => variant.type).join(' | ');\n    // Keep the compact representation for unions of primitive types.\n    if (\n      variants.some(\n        (variant) => variant.properties || variant.items || variant.anyOf,\n      )\n    ) {\n      result.anyOf = variants;\n    }\n  } else if (definition.typeName === 'ZodLiteral') {\n    result.type =\n      typeof definition.value === 'string'\n        ? JSON.stringify(definition.value)\n        : String(definition.value);\n  } else if (definition.typeName === 'ZodNull') {\n    result.type = 'null';\n  }\n  return result;\n};\n\nexport const buildActionDescription = ({\n  action,\n  locateFieldDescription,\n  actionOutputExample,\n}: PlanningActionDescriptionBuildInput) => {\n  const actionDescription: ActionDescription = {\n    type: action.name,\n    description: action.description || 'No description provided',\n  };\n\n  if (action.paramSchema) {\n    const schema = unwrapZodField(action.paramSchema) as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      const param: Record<string, ActionParamDescription> = {};\n      const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n      for (const [key, field] of Object.entries(schema.shape)) {\n        if (field && typeof field === 'object') {\n          param[key] = buildActionParamDescription(\n            field as z.ZodTypeAny,\n            locateFields.includes(key) ? locateFieldDescription : undefined,\n          );\n        }\n      }\n\n      if (Object.keys(param).length > 0) {\n        actionDescription.param = param;\n      }\n    } else {\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(action.paramSchema as z.ZodTypeAny);\n      const paramDescription: ActionParamDescription = {\n        type: typeName,\n        instruction: 'Pass the value directly, not as an object.',\n      };\n      if (description) {\n        paramDescription.description = description;\n      }\n      actionDescription.param = paramDescription;\n    }\n  }\n\n  if (actionOutputExample) {\n    actionDescription.sample = actionOutputExample;\n  }\n\n  return actionDescription;\n};\n\nexport const buildLocateFieldDescription = (\n  promptSpec?: LocateResultPromptSpec,\n) => {\n  if (promptSpec) {\n    return `{ prompt: string, ${promptSpec.resultKey}: ${promptSpec.resultValueSchema} /* ${promptSpec.resultValueDescription} */ }`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\nconst serializeActionParam = (\n  param: Record<string, unknown>,\n  locateFields: string[],\n  locateResultKey: string,\n) => {\n  const locatorObjects = new WeakSet<object>();\n  for (const field of locateFields) {\n    const value = param[field];\n    if (value && typeof value === 'object') {\n      locatorObjects.add(value);\n    }\n  }\n\n  const originalJson = JSON.stringify(param);\n  // Ensure generated markers cannot collide with strings already present in the sample.\n  let markerPrefix = '__MIDSCENE_LOCATE_RESULT_EXAMPLE_';\n  while (originalJson.includes(markerPrefix)) {\n    markerPrefix = `_${markerPrefix}`;\n  }\n\n  const replacements = new Map<string, string>();\n  const serializedParam = JSON.stringify(\n    param,\n    function (this: object, key, value) {\n      if (locatorObjects.has(this) && key === locateResultKey) {\n        const marker = `${markerPrefix}${replacements.size}__`;\n        replacements.set(marker, formatLocateExampleValue(value));\n        return marker;\n      }\n      return value;\n    },\n    2,\n  );\n\n  return [...replacements].reduce(\n    (result, [marker, locateResultValue]) =>\n      result.replace(JSON.stringify(marker), locateResultValue),\n    serializedParam,\n  );\n};\n\nexport const buildPlanningActionOutput = ({\n  actionName,\n  param,\n  locateFields,\n  locateResultKey,\n}: PlanningActionOutputBuildInput) => `<action-type>${actionName}</action-type>\n<action-param-json>\n${\n  // Keep locate result arrays on one line when serializing action examples.\n  locateFields && locateResultKey\n    ? serializeActionParam(param, locateFields, locateResultKey)\n    : JSON.stringify(param, null, 2)\n}\n</action-param-json>`;\n\nexport const createMidscenePlanningActionOutputParser =\n  (jsonParser: JsonParser) => (content: string) => {\n    const actionType = extractXMLTag(content, 'action-type');\n    const actionParamStr = extractXMLTag(content, 'action-param-json');\n\n    if (!actionType || actionType.toLowerCase() === 'null') {\n      return null;\n    }\n\n    // Strip any trailing XML tags that leaked into the action type.\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n        assertJsonObject(param);\n      } catch (error) {\n        throw new Error(`Failed to parse action-param-json: ${error}`);\n      }\n    }\n\n    return {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  };\n\nexport const createDefaultMidscenePlanningProtocol: StandardPlanningProtocolFactory =\n  ({ jsonParser }) => {\n    const parseActionOutput =\n      createMidscenePlanningActionOutputParser(jsonParser);\n\n    return {\n      actionSpaceProtocol: {\n        title: 'Supporting actions list',\n        format: 'yaml',\n        includeActionOutputExample: true,\n        buildLocateFieldDescription,\n        buildActionDescription,\n      },\n      actionOutputProtocol: {\n        actionOutputTagNames: ['action-type', 'action-param-json'],\n        actionOutputRules: [\n          '- Use the <action-type> and <action-param-json> tags to output the action to be executed.',\n          \"- The value inside <action-type> MUST exactly match the 'type' field of one action in the Supporting actions list. 'complete' is NOT a valid action-type.\",\n          '- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If the selected action provides a \"sample\" field, use the XML structure shown in that sample as the exact format for the action output.',\n        ].join('\\n'),\n        actionOutputPlaceholder: [\n          '<action-type>...</action-type>',\n          '<action-param-json>...</action-param-json>',\n        ].join('\\n'),\n        buildActionOutput: buildPlanningActionOutput,\n        parseActionOutput,\n        parseRawLocateParameter: (value) =>\n          value as ParsedPlanningLocateParameter,\n      },\n    };\n  };\n"],"names":["buildActionParamDescription","field","locateFieldDescription","isLocator","undefined","result","getZodTypeName","description","getZodDescription","defaultValue","getZodDefaultValue","actualField","unwrapZodField","definition","Object","name","nested","variants","option","variant","JSON","String","buildActionDescription","action","actionOutputExample","actionDescription","schema","isZodObject","param","locateFields","findAllMidsceneLocatorField","key","typeName","paramDescription","buildLocateFieldDescription","promptSpec","serializeActionParam","locateResultKey","locatorObjects","WeakSet","value","originalJson","markerPrefix","replacements","Map","serializedParam","marker","formatLocateExampleValue","locateResultValue","buildPlanningActionOutput","actionName","createMidscenePlanningActionOutputParser","jsonParser","content","actionType","extractXMLTag","actionParamStr","type","assertJsonObject","error","Error","createDefaultMidscenePlanningProtocol","parseActionOutput"],"mappings":";;;;;;AAuCA,MAAMA,8BAA8B,CAClCC,OACAC;IAEA,MAAMC,YAAYD,AAA2BE,WAA3BF;IAClB,MAAMG,SAAiC;QACrC,MAAMF,YAAYD,yBAAyBI,eAAeL;IAC5D;IACA,IAAIA,MAAM,UAAU,IAAII,OAAO,QAAQ,GAAG;IAC1C,MAAME,cAAcC,kBAAkBP;IACtC,IAAIM,aAAaF,OAAO,WAAW,GAAGE;IACtC,MAAME,eAAeC,mBAAmBT;IACxC,IAAIQ,AAAiBL,WAAjBK,cAA4BJ,OAAO,OAAO,GAAGI;IAIjD,IAAIN,WAAW,OAAOE;IACtB,MAAMM,cAAcC,eAAeX;IACnC,MAAMY,aAAaF,YAAY,IAAI;IACnC,IAAIE,AAAwB,gBAAxBA,WAAW,QAAQ,EACrBR,OAAO,UAAU,GAAGS,OAAO,WAAW,CACpCA,OAAO,OAAO,CAACD,WAAW,KAAK,IAAqB,GAAG,CACrD,CAAC,CAACE,MAAMC,OAAO,GAAK;YAACD;YAAMf,4BAA4BgB;SAAQ;SAG9D,IAAIH,AAAwB,eAAxBA,WAAW,QAAQ,EAC5BR,OAAO,KAAK,GAAGL,4BAA4Ba,WAAW,IAAI;SACrD,IACLA,AAAwB,eAAxBA,WAAW,QAAQ,IACnBA,AAAwB,4BAAxBA,WAAW,QAAQ,EACnB;QACA,MAAMI,WAAqCJ,WAAW,OAAO,CAAC,GAAG,CAC/D,CAACK,SAAyBlB,4BAA4BkB;QAExDb,OAAO,IAAI,GAAGY,SAAS,GAAG,CAAC,CAACE,UAAYA,QAAQ,IAAI,EAAE,IAAI,CAAC;QAE3D,IACEF,SAAS,IAAI,CACX,CAACE,UAAYA,QAAQ,UAAU,IAAIA,QAAQ,KAAK,IAAIA,QAAQ,KAAK,GAGnEd,OAAO,KAAK,GAAGY;IAEnB,OAAO,IAAIJ,AAAwB,iBAAxBA,WAAW,QAAQ,EAC5BR,OAAO,IAAI,GACT,AAA4B,YAA5B,OAAOQ,WAAW,KAAK,GACnBO,KAAK,SAAS,CAACP,WAAW,KAAK,IAC/BQ,OAAOR,WAAW,KAAK;SACxB,IAAIA,AAAwB,cAAxBA,WAAW,QAAQ,EAC5BR,OAAO,IAAI,GAAG;IAEhB,OAAOA;AACT;AAEO,MAAMiB,yBAAyB,CAAC,EACrCC,MAAM,EACNrB,sBAAsB,EACtBsB,mBAAmB,EACiB;IACpC,MAAMC,oBAAuC;QAC3C,MAAMF,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW,IAAI;IACrC;IAEA,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMG,SAASd,eAAeW,OAAO,WAAW;QAIhD,MAAMI,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAC/B,MAAME,QAAgD,CAAC;YACvD,MAAMC,eAAeC,4BAA4BP,OAAO,WAAW;YAEnE,KAAK,MAAM,CAACQ,KAAK9B,MAAM,IAAIa,OAAO,OAAO,CAACY,OAAO,KAAK,EACpD,IAAIzB,SAAS,AAAiB,YAAjB,OAAOA,OAClB2B,KAAK,CAACG,IAAI,GAAG/B,4BACXC,OACA4B,aAAa,QAAQ,CAACE,OAAO7B,yBAAyBE;YAK5D,IAAIU,OAAO,IAAI,CAACc,OAAO,MAAM,GAAG,GAC9BH,kBAAkB,KAAK,GAAGG;QAE9B,OAAO;YACL,MAAMI,WAAW1B,eAAeoB;YAChC,MAAMnB,cAAcC,kBAAkBe,OAAO,WAAW;YACxD,MAAMU,mBAA2C;gBAC/C,MAAMD;gBACN,aAAa;YACf;YACA,IAAIzB,aACF0B,iBAAiB,WAAW,GAAG1B;YAEjCkB,kBAAkB,KAAK,GAAGQ;QAC5B;IACF;IAEA,IAAIT,qBACFC,kBAAkB,MAAM,GAAGD;IAG7B,OAAOC;AACT;AAEO,MAAMS,8BAA8B,CACzCC;IAEA,IAAIA,YACF,OAAO,CAAC,kBAAkB,EAAEA,WAAW,SAAS,CAAC,EAAE,EAAEA,WAAW,iBAAiB,CAAC,IAAI,EAAEA,WAAW,sBAAsB,CAAC,KAAK,CAAC;IAElI,OAAO;AACT;AAEA,MAAMC,uBAAuB,CAC3BR,OACAC,cACAQ;IAEA,MAAMC,iBAAiB,IAAIC;IAC3B,KAAK,MAAMtC,SAAS4B,aAAc;QAChC,MAAMW,QAAQZ,KAAK,CAAC3B,MAAM;QAC1B,IAAIuC,SAAS,AAAiB,YAAjB,OAAOA,OAClBF,eAAe,GAAG,CAACE;IAEvB;IAEA,MAAMC,eAAerB,KAAK,SAAS,CAACQ;IAEpC,IAAIc,eAAe;IACnB,MAAOD,aAAa,QAAQ,CAACC,cAC3BA,eAAe,CAAC,CAAC,EAAEA,cAAc;IAGnC,MAAMC,eAAe,IAAIC;IACzB,MAAMC,kBAAkBzB,KAAK,SAAS,CACpCQ,OACA,SAAwBG,GAAG,EAAES,KAAK;QAChC,IAAIF,eAAe,GAAG,CAAC,IAAI,KAAKP,QAAQM,iBAAiB;YACvD,MAAMS,SAAS,GAAGJ,eAAeC,aAAa,IAAI,CAAC,EAAE,CAAC;YACtDA,aAAa,GAAG,CAACG,QAAQC,yBAAyBP;YAClD,OAAOM;QACT;QACA,OAAON;IACT,GACA;IAGF,OAAO;WAAIG;KAAa,CAAC,MAAM,CAC7B,CAACtC,QAAQ,CAACyC,QAAQE,kBAAkB,GAClC3C,OAAO,OAAO,CAACe,KAAK,SAAS,CAAC0B,SAASE,oBACzCH;AAEJ;AAEO,MAAMI,4BAA4B,CAAC,EACxCC,UAAU,EACVtB,KAAK,EACLC,YAAY,EACZQ,eAAe,EACgB,GAAK,CAAC,aAAa,EAAEa,WAAW;;AAEjE,EAEErB,gBAAgBQ,kBACZD,qBAAqBR,OAAOC,cAAcQ,mBAC1CjB,KAAK,SAAS,CAACQ,OAAO,MAAM,GACjC;oBACmB,CAAC;AAEd,MAAMuB,2CACX,CAACC,aAA2B,CAACC;QAC3B,MAAMC,aAAaC,cAAcF,SAAS;QAC1C,MAAMG,iBAAiBD,cAAcF,SAAS;QAE9C,IAAI,CAACC,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IACvC,OAAO;QAIT,MAAMG,OAAOH,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAI1B;QAEJ,IAAI4B,gBACF,IAAI;YACF5B,QAAQwB,WAAWI,gBAAgB;gBACjC,QAAQ;gBACR,yBACEC,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGrD;YACjD;YACAsD,iBAAiB9B;QACnB,EAAE,OAAO+B,OAAO;YACd,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,OAAO;QAC/D;QAGF,OAAO;YACLF;YACA,GAAI7B,AAAUxB,WAAVwB,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;AAEK,MAAMiC,wCACX,CAAC,EAAET,UAAU,EAAE;IACb,MAAMU,oBACJX,yCAAyCC;IAE3C,OAAO;QACL,qBAAqB;YACnB,OAAO;YACP,QAAQ;YACR,4BAA4B;YAC5BlB;YACAZ;QACF;QACA,sBAAsB;YACpB,sBAAsB;gBAAC;gBAAe;aAAoB;YAC1D,mBAAmB;YAKnB,yBAAyB;YAIzB,mBAAmB2B;YACnBa;YACA,yBAAyB,CAACtB,QACxBA;QACJ;IACF;AACF"}

package/dist/es/ai-model/prompt/describe.mjs

@@ -28,14 +28,14 @@ const describeElementReturnJsonSchema = ()=>`{
 const elementDescriberInstruction = ()=>{
     const preferredLanguage = getPreferredLanguage();
     return `
-Describe the real page element indicated by the temporary callout.
-The callout is an annotation overlay. It is not part of the page or target.
+Describe the real page element indicated by the temporary target marker (a ring with a crosshair).
+The marker is an annotation overlay. It is not part of the page or target.
 The description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption.
 
 IMPORTANT: You MUST write the description in ${preferredLanguage}.
 
 OBSERVE IN THIS ORDER:
-1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.
+1. Target first: identify the smallest real UI part at the marker center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.
 2. Primitive: name what that smallest part is before adding surrounding context.
 3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text.
 4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state.
@@ -43,20 +43,20 @@ OBSERVE IN THIS ORDER:
 RULES:
 - Keep description under 35 words.
 - Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it.
-- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target.
-- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center.
+- Ignore every annotation overlay, including ring, crosshair, line, color, marker, border, dot, or selection box. Never describe the annotation as the target.
+- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the marker center.
 - For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group.
-- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself.
-- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.
-- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.
-- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries.
-- Only use primitive "icon" or "arrow" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.
-- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible.
-- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line.
+- If the marker center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the marker center is on that icon itself.
+- If the marker center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.
+- If the marker center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly at the marker center. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.
+- If the marker center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries.
+- Only use primitive "icon" or "arrow" when the marker center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.
+- For compound controls or stacked glyphs, describe only the sub-part containing the marker center, using upper/lower or left/right only when visible.
+- For inline text, links, or substrings, describe only the exact substring/link at the marker center, not the whole line.
 - For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label.
 - For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible.
-- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state.
-- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.
+- Use selected, highlighted, hovered, focused, or active state only if the marker center is inside that state.
+- If the marker center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.
 - Use actual visible text from the current screenshot when available; do not copy labels from the examples.
 - **Write the description in ${preferredLanguage}**
 

package/dist/es/ai-model/prompt/describe.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/describe.mjs","sources":["../../../../src/ai-model/prompt/describe.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\n\nconst examplesMap: Record<string, string[]> = {\n  Chinese: [\n    '\"登录表单中的\"登录\"按钮\"',\n    '\"搜索输入框，placeholder 为\"请输入关键词\"\"',\n    '\"顶部导航栏中文字为\"首页\"的链接\"',\n    '\"联系表单中的提交按钮\"',\n    '\"aria-label 为\"打开菜单\"的菜单图标\"',\n    '\"左侧导航栏中当前分组标题右侧的折叠图标\"',\n  ],\n  English: [\n    '\"Login button with text \\'Sign In\\'\"',\n    '\"Search input with placeholder \\'Enter keywords\\'\"',\n    '\"Navigation link with text \\'Home\\' in header\"',\n    '\"Submit button in contact form\"',\n    '\"Menu icon with aria-label \\'Open menu\\'\"',\n    '\"Collapse icon to the right of the current section title in the left sidebar\"',\n  ],\n};\n\nconst getExamples = (language: string) => {\n  const examples = examplesMap[language] || examplesMap.English;\n  return examples.map((e) => `- ${e}`).join('\\n');\n};\n\nconst describeElementReturnJsonSchema = () => `{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n\nexport const elementDescriberInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nDescribe the real page element indicated by the temporary callout.\nThe callout is an annotation overlay. It is not part of the page or target.\nThe description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption.\n\nIMPORTANT: You MUST write the description in ${preferredLanguage}.\n\nOBSERVE IN THIS ORDER:\n1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.\n2. Primitive: name what that smallest part is before adding surrounding context.\n3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text.\n4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state.\n\nRULES:\n- Keep description under 35 words.\n- Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it.\n- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target.\n- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center.\n- For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group.\n- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself.\n- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive \"dropdown\" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.\n- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.\n- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive \"option\" for selectable list options or \"menuitem\" for command menu entries.\n- Only use primitive \"icon\" or \"arrow\" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.\n- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible.\n- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line.\n- For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label.\n- For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible.\n- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state.\n- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.\n- Use actual visible text from the current screenshot when available; do not copy labels from the examples.\n- **Write the description in ${preferredLanguage}**\n\nEXAMPLES:\n${getExamples(preferredLanguage)}\n\nReturn JSON:\n${describeElementReturnJsonSchema()}`;\n};\n"],"names":["examplesMap","getExamples","language","examples","e","describeElementReturnJsonSchema","elementDescriberInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";AAEA,MAAMA,cAAwC;IAC5C,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;IACD,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,MAAMC,cAAc,CAACC;IACnB,MAAMC,WAAWH,WAAW,CAACE,SAAS,IAAIF,YAAY,OAAO;IAC7D,OAAOG,SAAS,GAAG,CAAC,CAACC,IAAM,CAAC,EAAE,EAAEA,GAAG,EAAE,IAAI,CAAC;AAC5C;AAEA,MAAMC,kCAAkC,IAAM,CAAC;;;CAG9C,CAAC;AAEK,MAAMC,8BAA8B;IACzC,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;;;6CAKmC,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;6BA0BpC,EAAEA,kBAAkB;;;AAGjD,EAAEN,YAAYM,mBAAmB;;;AAGjC,EAAEF,mCAAmC;AACrC"}
+{"version":3,"file":"ai-model/prompt/describe.mjs","sources":["../../../../src/ai-model/prompt/describe.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\n\nconst examplesMap: Record<string, string[]> = {\n  Chinese: [\n    '\"登录表单中的\"登录\"按钮\"',\n    '\"搜索输入框，placeholder 为\"请输入关键词\"\"',\n    '\"顶部导航栏中文字为\"首页\"的链接\"',\n    '\"联系表单中的提交按钮\"',\n    '\"aria-label 为\"打开菜单\"的菜单图标\"',\n    '\"左侧导航栏中当前分组标题右侧的折叠图标\"',\n  ],\n  English: [\n    '\"Login button with text \\'Sign In\\'\"',\n    '\"Search input with placeholder \\'Enter keywords\\'\"',\n    '\"Navigation link with text \\'Home\\' in header\"',\n    '\"Submit button in contact form\"',\n    '\"Menu icon with aria-label \\'Open menu\\'\"',\n    '\"Collapse icon to the right of the current section title in the left sidebar\"',\n  ],\n};\n\nconst getExamples = (language: string) => {\n  const examples = examplesMap[language] || examplesMap.English;\n  return examples.map((e) => `- ${e}`).join('\\n');\n};\n\nconst describeElementReturnJsonSchema = () => `{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n\nexport const elementDescriberInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nDescribe the real page element indicated by the temporary target marker (a ring with a crosshair).\nThe marker is an annotation overlay. It is not part of the page or target.\nThe description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption.\n\nIMPORTANT: You MUST write the description in ${preferredLanguage}.\n\nOBSERVE IN THIS ORDER:\n1. Target first: identify the smallest real UI part at the marker center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.\n2. Primitive: name what that smallest part is before adding surrounding context.\n3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text.\n4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state.\n\nRULES:\n- Keep description under 35 words.\n- Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it.\n- Ignore every annotation overlay, including ring, crosshair, line, color, marker, border, dot, or selection box. Never describe the annotation as the target.\n- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the marker center.\n- For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group.\n- If the marker center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the marker center is on that icon itself.\n- If the marker center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive \"dropdown\" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.\n- If the marker center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly at the marker center. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.\n- If the marker center is on an expanded dropdown/select/menu list item, use primitive \"option\" for selectable list options or \"menuitem\" for command menu entries.\n- Only use primitive \"icon\" or \"arrow\" when the marker center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.\n- For compound controls or stacked glyphs, describe only the sub-part containing the marker center, using upper/lower or left/right only when visible.\n- For inline text, links, or substrings, describe only the exact substring/link at the marker center, not the whole line.\n- For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label.\n- For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible.\n- Use selected, highlighted, hovered, focused, or active state only if the marker center is inside that state.\n- If the marker center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.\n- Use actual visible text from the current screenshot when available; do not copy labels from the examples.\n- **Write the description in ${preferredLanguage}**\n\nEXAMPLES:\n${getExamples(preferredLanguage)}\n\nReturn JSON:\n${describeElementReturnJsonSchema()}`;\n};\n"],"names":["examplesMap","getExamples","language","examples","e","describeElementReturnJsonSchema","elementDescriberInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";AAEA,MAAMA,cAAwC;IAC5C,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;IACD,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,MAAMC,cAAc,CAACC;IACnB,MAAMC,WAAWH,WAAW,CAACE,SAAS,IAAIF,YAAY,OAAO;IAC7D,OAAOG,SAAS,GAAG,CAAC,CAACC,IAAM,CAAC,EAAE,EAAEA,GAAG,EAAE,IAAI,CAAC;AAC5C;AAEA,MAAMC,kCAAkC,IAAM,CAAC;;;CAG9C,CAAC;AAEK,MAAMC,8BAA8B;IACzC,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;;;6CAKmC,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;6BA0BpC,EAAEA,kBAAkB;;;AAGjD,EAAEN,YAAYM,mBAAmB;;;AAGjC,EAAEF,mCAAmC;AACrC"}

package/dist/es/device/index.mjs

@@ -109,6 +109,24 @@ const defineActionRightClick = (rightClick)=>defineLocatedPointAction({
             await rightClick(point);
         }
     });
+const actionMiddleClickParamSchema = z.object({
+    locate: getMidsceneLocationSchema().describe('The element to be middle clicked')
+});
+const defineActionMiddleClick = (middleClick)=>defineLocatedPointAction({
+        name: 'MiddleClick',
+        description: 'Middle click the element',
+        interfaceAlias: 'aiMiddleClick',
+        paramSchema: actionMiddleClickParamSchema,
+        sample: {
+            locate: {
+                prompt: 'the link to open in a new tab'
+            }
+        },
+        missingLocateMessage: 'Element not found, cannot middle click',
+        call: async (point)=>{
+            await middleClick(point);
+        }
+    });
 const actionDoubleClickParamSchema = z.object({
     locate: getMidsceneLocationSchema().describe('The element to be double clicked')
 });
@@ -491,6 +509,7 @@ function defineActionsFromInputPrimitives(input, options = {}) {
         actions.push(defineActionTap(pointer.tap));
         if (pointer.doubleClick) actions.push(defineActionDoubleClick(pointer.doubleClick));
         if (pointer.rightClick) actions.push(defineActionRightClick(pointer.rightClick));
+        if (pointer.middleClick) actions.push(defineActionMiddleClick(pointer.middleClick));
         if (pointer.hover) actions.push(defineActionHover(pointer.hover));
         if (pointer.dragAndDrop) actions.push(defineActionDragAndDrop(pointer.dragAndDrop));
         if (pointer.longPress) actions.push(defineActionLongPress(pointer.longPress));
@@ -557,6 +576,6 @@ const defineActionSleep = (abortSignal)=>defineAction({
             });
         }
     });
-export { AbstractInterface, ActionLongPressParamSchema, ActionPinchParamSchema, ActionSleepParamSchema, ActionSwipeParamSchema, actionClearInputParamSchema, actionCursorMoveParamSchema, actionDoubleClickParamSchema, actionDragAndDropParamSchema, actionHoverParamSchema, actionInputParamSchema, actionKeyboardPressParamSchema, actionRightClickParamSchema, actionScrollParamSchema, actionTapParamSchema, createDefaultMobileActions, defineAction, defineActionClearInput, defineActionCursorMove, defineActionDoubleClick, defineActionDragAndDrop, defineActionHover, defineActionInput, defineActionKeyboardPress, defineActionLongPress, defineActionPinch, defineActionRegisterFileChooserAccept, defineActionRightClick, defineActionScroll, defineActionSleep, defineActionSwipe, defineActionTap, defineActionsFromInputPrimitives, getMidsceneLocationSchema, inputStrategies, normalizeMobileSwipeParam, normalizePinchParam, normalizeSwipeParam, registerFileChooserAcceptParamSchema, resolveDefaultMobileScroll, resolveInputStrategy, resolveSwipeInputPrimitive, resolveTextInputOptions, sendTextSequentially, shouldInputSequentially, z };
+export { AbstractInterface, ActionLongPressParamSchema, ActionPinchParamSchema, ActionSleepParamSchema, ActionSwipeParamSchema, actionClearInputParamSchema, actionCursorMoveParamSchema, actionDoubleClickParamSchema, actionDragAndDropParamSchema, actionHoverParamSchema, actionInputParamSchema, actionKeyboardPressParamSchema, actionMiddleClickParamSchema, actionRightClickParamSchema, actionScrollParamSchema, actionTapParamSchema, createDefaultMobileActions, defineAction, defineActionClearInput, defineActionCursorMove, defineActionDoubleClick, defineActionDragAndDrop, defineActionHover, defineActionInput, defineActionKeyboardPress, defineActionLongPress, defineActionMiddleClick, defineActionPinch, defineActionRegisterFileChooserAccept, defineActionRightClick, defineActionScroll, defineActionSleep, defineActionSwipe, defineActionTap, defineActionsFromInputPrimitives, getMidsceneLocationSchema, inputStrategies, normalizeMobileSwipeParam, normalizePinchParam, normalizeSwipeParam, registerFileChooserAcceptParamSchema, resolveDefaultMobileScroll, resolveInputStrategy, resolveSwipeInputPrimitive, resolveTextInputOptions, sendTextSequentially, shouldInputSequentially, z };
 
 //# sourceMappingURL=index.mjs.map