@midscene/core 1.12.4-beta-20260904065253.0 → 1.12.4

package/dist/lib/agent/test-runner-nodes.js

@@ -0,0 +1,353 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    aiActOptionsInputSchema: ()=>aiActOptionsInputSchema,
+    aiTapInputSchema: ()=>aiTapInputSchema,
+    reportScreenshotInputSchema: ()=>reportScreenshotInputSchema,
+    createAgentTestRunnerNodeDefinition: ()=>createAgentTestRunnerNodeDefinition,
+    insightInputSchema: ()=>insightInputSchema,
+    locateOptionsInputSchema: ()=>locateOptionsInputSchema,
+    structuredUserPromptInputSchema: ()=>structuredUserPromptInputSchema,
+    promptImageInputSchema: ()=>promptImageInputSchema,
+    aiAssertInputSchema: ()=>aiAssertInputSchema,
+    commonAgentTestRunnerNodeDefinitions: ()=>commonAgentTestRunnerNodeDefinitions,
+    recordToReportInputSchema: ()=>recordToReportInputSchema,
+    recordToReportOptionsInputSchema: ()=>recordToReportOptionsInputSchema,
+    userPromptInputSchema: ()=>userPromptInputSchema,
+    aiAssertOptionsInputSchema: ()=>aiAssertOptionsInputSchema,
+    aiActInputSchema: ()=>aiActInputSchema,
+    insightOptionsInputSchema: ()=>insightOptionsInputSchema
+});
+const v4_namespaceObject = require("zod/v4");
+const createAgentTestRunnerNodeDefinition = (agentLabel = 'an Agent')=>(definition)=>({
+            name: definition.name ?? definition.method,
+            ...void 0 === definition.title ? {} : {
+                title: definition.title
+            },
+            ...void 0 === definition.description ? {} : {
+                description: definition.description
+            },
+            ...void 0 === definition.stringInputKey ? {} : {
+                stringInputKey: definition.stringInputKey
+            },
+            inputSchema: definition.inputSchema,
+            async execute (agent, input, context) {
+                if ('object' != typeof agent || null === agent || 'function' != typeof agent[definition.method]) throw new TypeError(`getAgent() must return ${agentLabel} with ${definition.method}().`);
+                const typedAgent = agent;
+                const method = typedAgent[definition.method];
+                const typedInput = input;
+                const result = await method.apply(typedAgent, definition.toArgs(typedInput, context));
+                return definition.toResult?.(result, typedInput);
+            }
+        });
+const nonBlankText = (description)=>v4_namespaceObject.z.string().regex(/\S/, 'value must contain a non-whitespace character').describe(description);
+const promptImageInputSchema = v4_namespaceObject.z.strictObject({
+    name: nonBlankText('The name used to identify this reference image.'),
+    url: nonBlankText('The URL, data URL, or file path of this reference image.')
+});
+const structuredUserPromptInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: nonBlankText('The natural-language prompt.'),
+    images: v4_namespaceObject.z.array(promptImageInputSchema).min(1).optional(),
+    convertHttpImage2Base64: v4_namespaceObject.z.boolean().optional().describe('Whether HTTP reference images are converted to base64 first.')
+});
+const userPromptInputSchema = v4_namespaceObject.z.union([
+    nonBlankText('The natural-language prompt.'),
+    structuredUserPromptInputSchema
+]);
+const aiActOptionsInputSchema = v4_namespaceObject.z.strictObject({
+    cacheable: v4_namespaceObject.z.boolean().optional().describe('Whether this action may use the Midscene cache.'),
+    fileChooserAccept: v4_namespaceObject.z.union([
+        v4_namespaceObject.z.string(),
+        v4_namespaceObject.z.array(v4_namespaceObject.z.string())
+    ]).optional().describe('Accepted file types for a file chooser.'),
+    fileChooserAllowedDir: v4_namespaceObject.z.string().optional().describe('Directory allowed for prompt-driven file uploads.'),
+    deepThink: v4_namespaceObject.z.union([
+        v4_namespaceObject.z.literal('unset'),
+        v4_namespaceObject.z.boolean()
+    ]).optional().describe('Whether to enable deep thinking for this action.'),
+    deepLocate: v4_namespaceObject.z.boolean().optional().describe('Whether to use deep element location.'),
+    context: v4_namespaceObject.z.string().optional().describe('Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.')
+});
+const aiActInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema,
+    options: aiActOptionsInputSchema.optional()
+});
+const insightOptionsInputSchema = v4_namespaceObject.z.strictObject({
+    domIncluded: v4_namespaceObject.z.union([
+        v4_namespaceObject.z.boolean(),
+        v4_namespaceObject.z.literal('visible-only')
+    ]).optional().describe('How DOM information is included.'),
+    screenshotIncluded: v4_namespaceObject.z.boolean().optional().describe('Whether the request includes a screenshot.'),
+    context: v4_namespaceObject.z.string().optional().describe('Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.')
+});
+const aiAssertOptionsInputSchema = insightOptionsInputSchema.extend({
+    keepRawResponse: v4_namespaceObject.z.boolean().optional().describe('Whether the Agent returns the structured assertion result.')
+});
+const aiAssertInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema,
+    message: v4_namespaceObject.z.string().optional().describe('The assertion failure message.'),
+    options: aiAssertOptionsInputSchema.optional()
+});
+const locateOptionsInputSchema = v4_namespaceObject.z.strictObject({
+    context: v4_namespaceObject.z.string().optional().describe('Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.'),
+    deepLocate: v4_namespaceObject.z.boolean().optional().describe('Whether to use deep element location.'),
+    deepThink: v4_namespaceObject.z.boolean().optional().describe('Deprecated alias for deepLocate.'),
+    cacheable: v4_namespaceObject.z.boolean().optional().describe('Whether this location may use the Midscene cache.'),
+    xpath: v4_namespaceObject.z.string().optional().describe('A web XPath location hint.'),
+    fileChooserAccept: v4_namespaceObject.z.union([
+        v4_namespaceObject.z.string(),
+        v4_namespaceObject.z.array(v4_namespaceObject.z.string())
+    ]).optional().describe('Files accepted when tapping opens a file chooser.')
+});
+const aiTapInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema,
+    options: locateOptionsInputSchema.optional()
+});
+const insightInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema,
+    options: insightOptionsInputSchema.optional()
+});
+const reportScreenshotInputSchema = v4_namespaceObject.z.strictObject({
+    base64: v4_namespaceObject.z.string().min(1).describe('A base64-encoded screenshot.'),
+    description: v4_namespaceObject.z.string().optional().describe('What the screenshot shows.')
+});
+const recordToReportOptionsInputSchema = v4_namespaceObject.z.strictObject({
+    content: v4_namespaceObject.z.string().optional().describe('The report text content.'),
+    screenshotBase64: v4_namespaceObject.z.string().optional().describe('One base64-encoded screenshot.'),
+    screenshots: v4_namespaceObject.z.array(reportScreenshotInputSchema).min(1).optional().describe('Screenshots attached to the report section.')
+}).superRefine((input, context)=>{
+    if (void 0 !== input.screenshotBase64 && void 0 !== input.screenshots) context.addIssue({
+        code: 'custom',
+        message: 'screenshotBase64 and screenshots are mutually exclusive'
+    });
+});
+const recordToReportInputSchema = v4_namespaceObject.z.strictObject({
+    title: v4_namespaceObject.z.string().optional().describe('The report section title.'),
+    options: recordToReportOptionsInputSchema.optional()
+});
+const defineCommonAgentNode = createAgentTestRunnerNodeDefinition('an Agent');
+const promptText = (prompt)=>'string' == typeof prompt ? prompt : prompt.prompt;
+const aiActNode = defineCommonAgentNode({
+    method: 'aiAct',
+    description: 'Perform a natural-language task with a Midscene UI Agent.',
+    stringInputKey: 'prompt',
+    inputSchema: aiActInputSchema,
+    toArgs (input, context) {
+        return [
+            input.prompt,
+            {
+                ...input.options,
+                abortSignal: context.signal
+            }
+        ];
+    },
+    toResult (output) {
+        return void 0 === output ? void 0 : {
+            summary: output
+        };
+    }
+});
+const aiTapNode = defineCommonAgentNode({
+    method: 'aiTap',
+    description: 'Locate and tap an element with a Midscene UI Agent.',
+    stringInputKey: 'prompt',
+    inputSchema: aiTapInputSchema,
+    toArgs (input) {
+        return [
+            input.prompt,
+            input.options
+        ];
+    },
+    toResult (_output, input) {
+        return {
+            summary: `Tapped: ${promptText(input.prompt)}`
+        };
+    }
+});
+const aiAssertNode = defineCommonAgentNode({
+    method: 'aiAssert',
+    description: 'Assert a natural-language condition with a Midscene UI Agent.',
+    stringInputKey: 'prompt',
+    inputSchema: aiAssertInputSchema,
+    toArgs (input, context) {
+        return [
+            input.prompt,
+            input.message,
+            {
+                ...input.options,
+                abortSignal: context.signal
+            }
+        ];
+    },
+    toResult (output, input) {
+        return {
+            summary: `Assertion passed: ${promptText(input.prompt)}`,
+            ...void 0 === output ? {} : {
+                data: {
+                    pass: output.pass,
+                    ...void 0 === output.thought ? {} : {
+                        thought: output.thought
+                    },
+                    ...void 0 === output.message ? {} : {
+                        message: output.message
+                    }
+                }
+            }
+        };
+    }
+});
+const insightNode = (method)=>{
+    if ('aiBoolean' === method) return defineCommonAgentNode({
+        method,
+        description: `Run ${method} with a Midscene UI Agent and store its value.`,
+        stringInputKey: 'prompt',
+        inputSchema: insightInputSchema,
+        toArgs: (input)=>[
+                input.prompt,
+                input.options
+            ],
+        toResult: (value)=>({
+                summary: `${method} returned ${value}`,
+                data: {
+                    value
+                }
+            })
+    });
+    if ('aiNumber' === method) return defineCommonAgentNode({
+        method,
+        description: `Run ${method} with a Midscene UI Agent and store its value.`,
+        stringInputKey: 'prompt',
+        inputSchema: insightInputSchema,
+        toArgs: (input)=>[
+                input.prompt,
+                input.options
+            ],
+        toResult: (value)=>({
+                summary: `${method} returned ${value}`,
+                data: {
+                    value
+                }
+            })
+    });
+    if ('aiString' === method) return defineCommonAgentNode({
+        method,
+        description: `Run ${method} with a Midscene UI Agent and store its value.`,
+        stringInputKey: 'prompt',
+        inputSchema: insightInputSchema,
+        toArgs: (input)=>[
+                input.prompt,
+                input.options
+            ],
+        toResult: (value)=>({
+                summary: `${method} returned ${JSON.stringify(value)}`,
+                data: {
+                    value
+                }
+            })
+    });
+    return defineCommonAgentNode({
+        method,
+        description: `Run ${method} with a Midscene UI Agent and store its value.`,
+        stringInputKey: 'prompt',
+        inputSchema: insightInputSchema,
+        toArgs: (input)=>[
+                input.prompt,
+                input.options
+            ],
+        toResult: (value)=>({
+                summary: `${method} returned ${JSON.stringify(value)}`,
+                data: {
+                    value
+                }
+            })
+    });
+};
+const recordToReportNode = defineCommonAgentNode({
+    method: 'recordToReport',
+    description: 'Add text or screenshots to the current Midscene report.',
+    stringInputKey: 'title',
+    inputSchema: recordToReportInputSchema,
+    toArgs (input) {
+        return [
+            input.title,
+            input.options
+        ];
+    },
+    toResult (_output, input) {
+        return {
+            summary: `Recorded to report: ${input.title ?? 'untitled'}`
+        };
+    }
+});
+const commonAgentTestRunnerNodeDefinitions = [
+    aiActNode,
+    aiTapNode,
+    aiAssertNode,
+    insightNode('aiBoolean'),
+    insightNode('aiNumber'),
+    insightNode('aiString'),
+    insightNode('aiAsk'),
+    recordToReportNode
+];
+exports.aiActInputSchema = __webpack_exports__.aiActInputSchema;
+exports.aiActOptionsInputSchema = __webpack_exports__.aiActOptionsInputSchema;
+exports.aiAssertInputSchema = __webpack_exports__.aiAssertInputSchema;
+exports.aiAssertOptionsInputSchema = __webpack_exports__.aiAssertOptionsInputSchema;
+exports.aiTapInputSchema = __webpack_exports__.aiTapInputSchema;
+exports.commonAgentTestRunnerNodeDefinitions = __webpack_exports__.commonAgentTestRunnerNodeDefinitions;
+exports.createAgentTestRunnerNodeDefinition = __webpack_exports__.createAgentTestRunnerNodeDefinition;
+exports.insightInputSchema = __webpack_exports__.insightInputSchema;
+exports.insightOptionsInputSchema = __webpack_exports__.insightOptionsInputSchema;
+exports.locateOptionsInputSchema = __webpack_exports__.locateOptionsInputSchema;
+exports.promptImageInputSchema = __webpack_exports__.promptImageInputSchema;
+exports.recordToReportInputSchema = __webpack_exports__.recordToReportInputSchema;
+exports.recordToReportOptionsInputSchema = __webpack_exports__.recordToReportOptionsInputSchema;
+exports.reportScreenshotInputSchema = __webpack_exports__.reportScreenshotInputSchema;
+exports.structuredUserPromptInputSchema = __webpack_exports__.structuredUserPromptInputSchema;
+exports.userPromptInputSchema = __webpack_exports__.userPromptInputSchema;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "aiActInputSchema",
+    "aiActOptionsInputSchema",
+    "aiAssertInputSchema",
+    "aiAssertOptionsInputSchema",
+    "aiTapInputSchema",
+    "commonAgentTestRunnerNodeDefinitions",
+    "createAgentTestRunnerNodeDefinition",
+    "insightInputSchema",
+    "insightOptionsInputSchema",
+    "locateOptionsInputSchema",
+    "promptImageInputSchema",
+    "recordToReportInputSchema",
+    "recordToReportOptionsInputSchema",
+    "reportScreenshotInputSchema",
+    "structuredUserPromptInputSchema",
+    "userPromptInputSchema"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=test-runner-nodes.js.map

package/dist/lib/agent/test-runner-nodes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/test-runner-nodes.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/test-runner-nodes.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { z } from 'zod/v4';\nimport type { Agent } from './agent';\n\nexport interface AgentTestRunnerNodeResult<TData = unknown> {\n  summary?: string;\n  data?: TData;\n}\n\nexport interface AgentTestRunnerNodeExecutionContext {\n  signal: AbortSignal;\n}\n\n/** Agent-owned Node description consumed by Test Runner adapters. */\nexport interface AgentTestRunnerNodeDefinition<\n  TInput extends Record<string, unknown> = Record<string, unknown>,\n  TData = unknown,\n> {\n  name: string;\n  title?: string;\n  description?: string;\n  /** Node input field populated by YAML string shorthand; omit to disable. */\n  stringInputKey?: string | false;\n  inputSchema: z.ZodObject;\n  execute(\n    agent: unknown,\n    input: TInput,\n    context: AgentTestRunnerNodeExecutionContext,\n  ):\n    | Promise<AgentTestRunnerNodeResult<TData> | undefined>\n    | AgentTestRunnerNodeResult<TData>\n    | undefined;\n}\n\n/** Static capability implemented by Agent classes that expose Runner Nodes. */\nexport interface AgentTestRunnerNodeProvider {\n  getTestRunnerNodeDefinitions(): readonly AgentTestRunnerNodeDefinition[];\n}\n\ntype AnyAgentMethod = (...args: any[]) => unknown;\ntype AgentMethodName<TAgent extends object> = {\n  [TKey in keyof TAgent]-?: TAgent[TKey] extends AnyAgentMethod ? TKey : never;\n}[keyof TAgent] &\n  string;\ntype AgentMethod<\n  TAgent extends object,\n  TMethod extends AgentMethodName<TAgent>,\n> = Extract<TAgent[TMethod], AnyAgentMethod>;\n\nexport interface DefineAgentTestRunnerNodeOptions<\n  TAgent extends object,\n  TMethod extends AgentMethodName<TAgent>,\n  TSchema extends z.ZodObject,\n  TData = unknown,\n> {\n  method: TMethod;\n  name?: string;\n  title?: string;\n  description?: string;\n  stringInputKey?: (keyof z.output<TSchema> & string) | false;\n  inputSchema: TSchema;\n  toArgs(\n    input: z.output<TSchema>,\n    context: AgentTestRunnerNodeExecutionContext,\n  ): Parameters<AgentMethod<TAgent, TMethod>>;\n  toResult?(\n    result: Awaited<ReturnType<AgentMethod<TAgent, TMethod>>>,\n    input: z.output<TSchema>,\n  ): AgentTestRunnerNodeResult<TData> | undefined;\n}\n\n/** Create a Node factory whose adapters are checked against an Agent API. */\nexport const createAgentTestRunnerNodeDefinition =\n  <TAgent extends object>(agentLabel = 'an Agent') =>\n  <\n    TMethod extends AgentMethodName<TAgent>,\n    TSchema extends z.ZodObject,\n    TData = unknown,\n  >(\n    definition: DefineAgentTestRunnerNodeOptions<\n      TAgent,\n      TMethod,\n      TSchema,\n      TData\n    >,\n  ): AgentTestRunnerNodeDefinition => ({\n    name: definition.name ?? definition.method,\n    ...(definition.title === undefined ? {} : { title: definition.title }),\n    ...(definition.description === undefined\n      ? {}\n      : { description: definition.description }),\n    ...(definition.stringInputKey === undefined\n      ? {}\n      : { stringInputKey: definition.stringInputKey }),\n    inputSchema: definition.inputSchema,\n    async execute(agent, input, context) {\n      if (\n        typeof agent !== 'object' ||\n        agent === null ||\n        typeof (agent as Record<string, unknown>)[definition.method] !==\n          'function'\n      ) {\n        throw new TypeError(\n          `getAgent() must return ${agentLabel} with ${definition.method}().`,\n        );\n      }\n\n      const typedAgent = agent as TAgent;\n      const method = typedAgent[definition.method] as AgentMethod<\n        TAgent,\n        TMethod\n      >;\n      const typedInput = input as z.output<TSchema>;\n      const result = (await method.apply(\n        typedAgent,\n        definition.toArgs(typedInput, context),\n      )) as Awaited<ReturnType<AgentMethod<TAgent, TMethod>>>;\n      return definition.toResult?.(result, typedInput);\n    },\n  });\n\nconst nonBlankText = (description: string) =>\n  z\n    .string()\n    .regex(/\\S/, 'value must contain a non-whitespace character')\n    .describe(description);\n\nexport const promptImageInputSchema = z.strictObject({\n  name: nonBlankText('The name used to identify this reference image.'),\n  url: nonBlankText('The URL, data URL, or file path of this reference image.'),\n});\n\nexport const structuredUserPromptInputSchema = z.strictObject({\n  prompt: nonBlankText('The natural-language prompt.'),\n  images: z.array(promptImageInputSchema).min(1).optional(),\n  convertHttpImage2Base64: z\n    .boolean()\n    .optional()\n    .describe('Whether HTTP reference images are converted to base64 first.'),\n});\n\nexport const userPromptInputSchema = z.union([\n  nonBlankText('The natural-language prompt.'),\n  structuredUserPromptInputSchema,\n]);\n\nexport const aiActOptionsInputSchema = z.strictObject({\n  cacheable: z\n    .boolean()\n    .optional()\n    .describe('Whether this action may use the Midscene cache.'),\n  fileChooserAccept: z\n    .union([z.string(), z.array(z.string())])\n    .optional()\n    .describe('Accepted file types for a file chooser.'),\n  fileChooserAllowedDir: z\n    .string()\n    .optional()\n    .describe('Directory allowed for prompt-driven file uploads.'),\n  deepThink: z\n    .union([z.literal('unset'), z.boolean()])\n    .optional()\n    .describe('Whether to enable deep thinking for this action.'),\n  deepLocate: z\n    .boolean()\n    .optional()\n    .describe('Whether to use deep element location.'),\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.',\n    ),\n});\n\nexport const aiActInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: aiActOptionsInputSchema.optional(),\n});\n\nexport const insightOptionsInputSchema = z.strictObject({\n  domIncluded: z\n    .union([z.boolean(), z.literal('visible-only')])\n    .optional()\n    .describe('How DOM information is included.'),\n  screenshotIncluded: z\n    .boolean()\n    .optional()\n    .describe('Whether the request includes a screenshot.'),\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.',\n    ),\n});\n\nexport const aiAssertOptionsInputSchema = insightOptionsInputSchema.extend({\n  keepRawResponse: z\n    .boolean()\n    .optional()\n    .describe('Whether the Agent returns the structured assertion result.'),\n});\n\nexport const aiAssertInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  message: z.string().optional().describe('The assertion failure message.'),\n  options: aiAssertOptionsInputSchema.optional(),\n});\n\nexport const locateOptionsInputSchema = z.strictObject({\n  context: z\n    .string()\n    .optional()\n    .describe(\n      'Additional facts, rules, constraints, or output requirements for this AI call. Overrides inherited aiContexts; an empty string disables inherited user context.',\n    ),\n  deepLocate: z\n    .boolean()\n    .optional()\n    .describe('Whether to use deep element location.'),\n  deepThink: z\n    .boolean()\n    .optional()\n    .describe('Deprecated alias for deepLocate.'),\n  cacheable: z\n    .boolean()\n    .optional()\n    .describe('Whether this location may use the Midscene cache.'),\n  xpath: z.string().optional().describe('A web XPath location hint.'),\n  fileChooserAccept: z\n    .union([z.string(), z.array(z.string())])\n    .optional()\n    .describe('Files accepted when tapping opens a file chooser.'),\n});\n\nexport const aiTapInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: locateOptionsInputSchema.optional(),\n});\n\nexport const insightInputSchema = z.strictObject({\n  prompt: userPromptInputSchema,\n  options: insightOptionsInputSchema.optional(),\n});\n\nexport const reportScreenshotInputSchema = z.strictObject({\n  base64: z.string().min(1).describe('A base64-encoded screenshot.'),\n  description: z.string().optional().describe('What the screenshot shows.'),\n});\n\nexport const recordToReportOptionsInputSchema = z\n  .strictObject({\n    content: z.string().optional().describe('The report text content.'),\n    screenshotBase64: z\n      .string()\n      .optional()\n      .describe('One base64-encoded screenshot.'),\n    screenshots: z\n      .array(reportScreenshotInputSchema)\n      .min(1)\n      .optional()\n      .describe('Screenshots attached to the report section.'),\n  })\n  .superRefine((input, context) => {\n    if (\n      input.screenshotBase64 !== undefined &&\n      input.screenshots !== undefined\n    ) {\n      context.addIssue({\n        code: 'custom',\n        message: 'screenshotBase64 and screenshots are mutually exclusive',\n      });\n    }\n  });\n\nexport const recordToReportInputSchema = z.strictObject({\n  title: z.string().optional().describe('The report section title.'),\n  options: recordToReportOptionsInputSchema.optional(),\n});\n\nexport type UserPromptNodeInput = z.infer<typeof userPromptInputSchema>;\nexport type AiActNodeOptions = z.infer<typeof aiActOptionsInputSchema>;\nexport type AiActNodeInput = z.infer<typeof aiActInputSchema>;\nexport type AiAssertNodeOptions = z.infer<typeof aiAssertOptionsInputSchema>;\nexport type AiAssertNodeInput = z.infer<typeof aiAssertInputSchema>;\nexport type AiTapNodeOptions = z.infer<typeof locateOptionsInputSchema>;\nexport type AiTapNodeInput = z.infer<typeof aiTapInputSchema>;\nexport type InsightNodeOptions = z.infer<typeof insightOptionsInputSchema>;\nexport type InsightNodeInput = z.infer<typeof insightInputSchema>;\nexport type RecordToReportNodeOptions = z.infer<\n  typeof recordToReportOptionsInputSchema\n>;\nexport type RecordToReportNodeInput = z.infer<typeof recordToReportInputSchema>;\n\nexport type CommonAgentTestRunnerApi = Pick<\n  Agent,\n  | 'aiAct'\n  | 'aiTap'\n  | 'aiAssert'\n  | 'aiBoolean'\n  | 'aiNumber'\n  | 'aiString'\n  | 'aiAsk'\n  | 'recordToReport'\n>;\n\nconst defineCommonAgentNode =\n  createAgentTestRunnerNodeDefinition<CommonAgentTestRunnerApi>('an Agent');\n\nconst promptText = (prompt: UserPromptNodeInput): string =>\n  typeof prompt === 'string' ? prompt : prompt.prompt;\n\nconst aiActNode = defineCommonAgentNode({\n  method: 'aiAct',\n  description: 'Perform a natural-language task with a Midscene UI Agent.',\n  stringInputKey: 'prompt',\n  inputSchema: aiActInputSchema,\n  toArgs(input, context) {\n    return [\n      input.prompt,\n      {\n        ...input.options,\n        abortSignal: context.signal,\n      },\n    ];\n  },\n  toResult(output) {\n    return output === undefined ? undefined : { summary: output };\n  },\n});\n\nconst aiTapNode = defineCommonAgentNode({\n  method: 'aiTap',\n  description: 'Locate and tap an element with a Midscene UI Agent.',\n  stringInputKey: 'prompt',\n  inputSchema: aiTapInputSchema,\n  toArgs(input) {\n    return [input.prompt, input.options];\n  },\n  toResult(_output, input) {\n    return { summary: `Tapped: ${promptText(input.prompt)}` };\n  },\n});\n\nconst aiAssertNode = defineCommonAgentNode({\n  method: 'aiAssert',\n  description: 'Assert a natural-language condition with a Midscene UI Agent.',\n  stringInputKey: 'prompt',\n  inputSchema: aiAssertInputSchema,\n  toArgs(input, context) {\n    return [\n      input.prompt,\n      input.message,\n      {\n        ...input.options,\n        abortSignal: context.signal,\n      },\n    ];\n  },\n  toResult(output, input) {\n    return {\n      summary: `Assertion passed: ${promptText(input.prompt)}`,\n      ...(output === undefined\n        ? {}\n        : {\n            data: {\n              pass: output.pass,\n              ...(output.thought === undefined\n                ? {}\n                : { thought: output.thought }),\n              ...(output.message === undefined\n                ? {}\n                : { message: output.message }),\n            },\n          }),\n    };\n  },\n});\n\nconst insightNode = (\n  method: 'aiBoolean' | 'aiNumber' | 'aiString' | 'aiAsk',\n) => {\n  if (method === 'aiBoolean') {\n    return defineCommonAgentNode({\n      method,\n      description: `Run ${method} with a Midscene UI Agent and store its value.`,\n      stringInputKey: 'prompt',\n      inputSchema: insightInputSchema,\n      toArgs: (input) => [input.prompt, input.options],\n      toResult: (value) => ({\n        summary: `${method} returned ${value}`,\n        data: { value },\n      }),\n    });\n  }\n  if (method === 'aiNumber') {\n    return defineCommonAgentNode({\n      method,\n      description: `Run ${method} with a Midscene UI Agent and store its value.`,\n      stringInputKey: 'prompt',\n      inputSchema: insightInputSchema,\n      toArgs: (input) => [input.prompt, input.options],\n      toResult: (value) => ({\n        summary: `${method} returned ${value}`,\n        data: { value },\n      }),\n    });\n  }\n  if (method === 'aiString') {\n    return defineCommonAgentNode({\n      method,\n      description: `Run ${method} with a Midscene UI Agent and store its value.`,\n      stringInputKey: 'prompt',\n      inputSchema: insightInputSchema,\n      toArgs: (input) => [input.prompt, input.options],\n      toResult: (value) => ({\n        summary: `${method} returned ${JSON.stringify(value)}`,\n        data: { value },\n      }),\n    });\n  }\n  return defineCommonAgentNode({\n    method,\n    description: `Run ${method} with a Midscene UI Agent and store its value.`,\n    stringInputKey: 'prompt',\n    inputSchema: insightInputSchema,\n    toArgs: (input) => [input.prompt, input.options],\n    toResult: (value) => ({\n      summary: `${method} returned ${JSON.stringify(value)}`,\n      data: { value },\n    }),\n  });\n};\n\nconst recordToReportNode = defineCommonAgentNode({\n  method: 'recordToReport',\n  description: 'Add text or screenshots to the current Midscene report.',\n  stringInputKey: 'title',\n  inputSchema: recordToReportInputSchema,\n  toArgs(input) {\n    return [input.title, input.options];\n  },\n  toResult(_output, input) {\n    return { summary: `Recorded to report: ${input.title ?? 'untitled'}` };\n  },\n});\n\nexport const commonAgentTestRunnerNodeDefinitions: readonly AgentTestRunnerNodeDefinition[] =\n  [\n    aiActNode,\n    aiTapNode,\n    aiAssertNode,\n    insightNode('aiBoolean'),\n    insightNode('aiNumber'),\n    insightNode('aiString'),\n    insightNode('aiAsk'),\n    recordToReportNode,\n  ];\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createAgentTestRunnerNodeDefinition","agentLabel","undefined","agent","input","context","TypeError","typedAgent","method","typedInput","result","nonBlankText","description","z","promptImageInputSchema","structuredUserPromptInputSchema","userPromptInputSchema","aiActOptionsInputSchema","aiActInputSchema","insightOptionsInputSchema","aiAssertOptionsInputSchema","aiAssertInputSchema","locateOptionsInputSchema","aiTapInputSchema","insightInputSchema","reportScreenshotInputSchema","recordToReportOptionsInputSchema","recordToReportInputSchema","defineCommonAgentNode","promptText","prompt","aiActNode","output","aiTapNode","_output","aiAssertNode","insightNode","value","JSON","recordToReportNode","commonAgentTestRunnerNodeDefinitions"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;ACiEO,MAAMI,sCACX,CAAwBC,aAAa,UAAU,GAC/C,CAKEP,aAMmC;YACnC,MAAMA,WAAW,IAAI,IAAIA,WAAW,MAAM;YAC1C,GAAIA,AAAqBQ,WAArBR,WAAW,KAAK,GAAiB,CAAC,IAAI;gBAAE,OAAOA,WAAW,KAAK;YAAC,CAAC;YACrE,GAAIA,AAA2BQ,WAA3BR,WAAW,WAAW,GACtB,CAAC,IACD;gBAAE,aAAaA,WAAW,WAAW;YAAC,CAAC;YAC3C,GAAIA,AAA8BQ,WAA9BR,WAAW,cAAc,GACzB,CAAC,IACD;gBAAE,gBAAgBA,WAAW,cAAc;YAAC,CAAC;YACjD,aAAaA,WAAW,WAAW;YACnC,MAAM,SAAQS,KAAK,EAAEC,KAAK,EAAEC,OAAO;gBACjC,IACE,AAAiB,YAAjB,OAAOF,SACPA,AAAU,SAAVA,SACA,AACE,cADF,OAAQA,KAAiC,CAACT,WAAW,MAAM,CAAC,EAG5D,MAAM,IAAIY,UACR,CAAC,uBAAuB,EAAEL,WAAW,MAAM,EAAEP,WAAW,MAAM,CAAC,GAAG,CAAC;gBAIvE,MAAMa,aAAaJ;gBACnB,MAAMK,SAASD,UAAU,CAACb,WAAW,MAAM,CAAC;gBAI5C,MAAMe,aAAaL;gBACnB,MAAMM,SAAU,MAAMF,OAAO,KAAK,CAChCD,YACAb,WAAW,MAAM,CAACe,YAAYJ;gBAEhC,OAAOX,WAAW,QAAQ,GAAGgB,QAAQD;YACvC;QACF;AAEF,MAAME,eAAe,CAACC,cACpBC,mBAAAA,CAAAA,CAAAA,MACS,GACN,KAAK,CAAC,MAAM,iDACZ,QAAQ,CAACD;AAEP,MAAME,yBAAyBD,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IACnD,MAAMF,aAAa;IACnB,KAAKA,aAAa;AACpB;AAEO,MAAMI,kCAAkCF,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IAC5D,QAAQF,aAAa;IACrB,QAAQE,mBAAAA,CAAAA,CAAAA,KAAO,CAACC,wBAAwB,GAAG,CAAC,GAAG,QAAQ;IACvD,yBAAyBD,mBAAAA,CAAAA,CAAAA,OACf,GACP,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMG,wBAAwBH,mBAAAA,CAAAA,CAAAA,KAAO,CAAC;IAC3CF,aAAa;IACbI;CACD;AAEM,MAAME,0BAA0BJ,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IACpD,WAAWA,mBAAAA,CAAAA,CAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,mBAAmBA,mBAAAA,CAAAA,CAAAA,KACX,CAAC;QAACA,mBAAAA,CAAAA,CAAAA,MAAQ;QAAIA,mBAAAA,CAAAA,CAAAA,KAAO,CAACA,mBAAAA,CAAAA,CAAAA,MAAQ;KAAI,EACvC,QAAQ,GACR,QAAQ,CAAC;IACZ,uBAAuBA,mBAAAA,CAAAA,CAAAA,MACd,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,mBAAAA,CAAAA,CAAAA,KACH,CAAC;QAACA,mBAAAA,CAAAA,CAAAA,OAAS,CAAC;QAAUA,mBAAAA,CAAAA,CAAAA,OAAS;KAAG,EACvC,QAAQ,GACR,QAAQ,CAAC;IACZ,YAAYA,mBAAAA,CAAAA,CAAAA,OACF,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,SAASA,mBAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMK,mBAAmBL,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IAC7C,QAAQG;IACR,SAASC,wBAAwB,QAAQ;AAC3C;AAEO,MAAME,4BAA4BN,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IACtD,aAAaA,mBAAAA,CAAAA,CAAAA,KACL,CAAC;QAACA,mBAAAA,CAAAA,CAAAA,OAAS;QAAIA,mBAAAA,CAAAA,CAAAA,OAAS,CAAC;KAAgB,EAC9C,QAAQ,GACR,QAAQ,CAAC;IACZ,oBAAoBA,mBAAAA,CAAAA,CAAAA,OACV,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,SAASA,mBAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMO,6BAA6BD,0BAA0B,MAAM,CAAC;IACzE,iBAAiBN,mBAAAA,CAAAA,CAAAA,OACP,GACP,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMQ,sBAAsBR,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IAChD,QAAQG;IACR,SAASH,mBAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACxC,SAASO,2BAA2B,QAAQ;AAC9C;AAEO,MAAME,2BAA2BT,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IACrD,SAASA,mBAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,GACR,QAAQ,CACP;IAEJ,YAAYA,mBAAAA,CAAAA,CAAAA,OACF,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,mBAAAA,CAAAA,CAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,mBAAAA,CAAAA,CAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,OAAOA,mBAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACtC,mBAAmBA,mBAAAA,CAAAA,CAAAA,KACX,CAAC;QAACA,mBAAAA,CAAAA,CAAAA,MAAQ;QAAIA,mBAAAA,CAAAA,CAAAA,KAAO,CAACA,mBAAAA,CAAAA,CAAAA,MAAQ;KAAI,EACvC,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMU,mBAAmBV,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IAC7C,QAAQG;IACR,SAASM,yBAAyB,QAAQ;AAC5C;AAEO,MAAME,qBAAqBX,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IAC/C,QAAQG;IACR,SAASG,0BAA0B,QAAQ;AAC7C;AAEO,MAAMM,8BAA8BZ,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IACxD,QAAQA,mBAAAA,CAAAA,CAAAA,MAAQ,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC;IACnC,aAAaA,mBAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAC9C;AAEO,MAAMa,mCAAmCb,mBAAAA,CAAAA,CAAAA,YACjC,CAAC;IACZ,SAASA,mBAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACxC,kBAAkBA,mBAAAA,CAAAA,CAAAA,MACT,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,aAAaA,mBAAAA,CAAAA,CAAAA,KACL,CAACY,6BACN,GAAG,CAAC,GACJ,QAAQ,GACR,QAAQ,CAAC;AACd,GACC,WAAW,CAAC,CAACrB,OAAOC;IACnB,IACED,AAA2BF,WAA3BE,MAAM,gBAAgB,IACtBA,AAAsBF,WAAtBE,MAAM,WAAW,EAEjBC,QAAQ,QAAQ,CAAC;QACf,MAAM;QACN,SAAS;IACX;AAEJ;AAEK,MAAMsB,4BAA4Bd,mBAAAA,CAAAA,CAAAA,YAAc,CAAC;IACtD,OAAOA,mBAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACtC,SAASa,iCAAiC,QAAQ;AACpD;AA4BA,MAAME,wBACJ5B,oCAA8D;AAEhE,MAAM6B,aAAa,CAACC,SAClB,AAAkB,YAAlB,OAAOA,SAAsBA,SAASA,OAAO,MAAM;AAErD,MAAMC,YAAYH,sBAAsB;IACtC,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAaV;IACb,QAAOd,KAAK,EAAEC,OAAO;QACnB,OAAO;YACLD,MAAM,MAAM;YACZ;gBACE,GAAGA,MAAM,OAAO;gBAChB,aAAaC,QAAQ,MAAM;YAC7B;SACD;IACH;IACA,UAAS2B,MAAM;QACb,OAAOA,AAAW9B,WAAX8B,SAAuB9B,SAAY;YAAE,SAAS8B;QAAO;IAC9D;AACF;AAEA,MAAMC,YAAYL,sBAAsB;IACtC,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAaL;IACb,QAAOnB,KAAK;QACV,OAAO;YAACA,MAAM,MAAM;YAAEA,MAAM,OAAO;SAAC;IACtC;IACA,UAAS8B,OAAO,EAAE9B,KAAK;QACrB,OAAO;YAAE,SAAS,CAAC,QAAQ,EAAEyB,WAAWzB,MAAM,MAAM,GAAG;QAAC;IAC1D;AACF;AAEA,MAAM+B,eAAeP,sBAAsB;IACzC,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAaP;IACb,QAAOjB,KAAK,EAAEC,OAAO;QACnB,OAAO;YACLD,MAAM,MAAM;YACZA,MAAM,OAAO;YACb;gBACE,GAAGA,MAAM,OAAO;gBAChB,aAAaC,QAAQ,MAAM;YAC7B;SACD;IACH;IACA,UAAS2B,MAAM,EAAE5B,KAAK;QACpB,OAAO;YACL,SAAS,CAAC,kBAAkB,EAAEyB,WAAWzB,MAAM,MAAM,GAAG;YACxD,GAAI4B,AAAW9B,WAAX8B,SACA,CAAC,IACD;gBACE,MAAM;oBACJ,MAAMA,OAAO,IAAI;oBACjB,GAAIA,AAAmB9B,WAAnB8B,OAAO,OAAO,GACd,CAAC,IACD;wBAAE,SAASA,OAAO,OAAO;oBAAC,CAAC;oBAC/B,GAAIA,AAAmB9B,WAAnB8B,OAAO,OAAO,GACd,CAAC,IACD;wBAAE,SAASA,OAAO,OAAO;oBAAC,CAAC;gBACjC;YACF,CAAC;QACP;IACF;AACF;AAEA,MAAMI,cAAc,CAClB5B;IAEA,IAAIA,AAAW,gBAAXA,QACF,OAAOoB,sBAAsB;QAC3BpB;QACA,aAAa,CAAC,IAAI,EAAEA,OAAO,8CAA8C,CAAC;QAC1E,gBAAgB;QAChB,aAAagB;QACb,QAAQ,CAACpB,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;QAChD,UAAU,CAACiC,QAAW;gBACpB,SAAS,GAAG7B,OAAO,UAAU,EAAE6B,OAAO;gBACtC,MAAM;oBAAEA;gBAAM;YAChB;IACF;IAEF,IAAI7B,AAAW,eAAXA,QACF,OAAOoB,sBAAsB;QAC3BpB;QACA,aAAa,CAAC,IAAI,EAAEA,OAAO,8CAA8C,CAAC;QAC1E,gBAAgB;QAChB,aAAagB;QACb,QAAQ,CAACpB,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;QAChD,UAAU,CAACiC,QAAW;gBACpB,SAAS,GAAG7B,OAAO,UAAU,EAAE6B,OAAO;gBACtC,MAAM;oBAAEA;gBAAM;YAChB;IACF;IAEF,IAAI7B,AAAW,eAAXA,QACF,OAAOoB,sBAAsB;QAC3BpB;QACA,aAAa,CAAC,IAAI,EAAEA,OAAO,8CAA8C,CAAC;QAC1E,gBAAgB;QAChB,aAAagB;QACb,QAAQ,CAACpB,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;QAChD,UAAU,CAACiC,QAAW;gBACpB,SAAS,GAAG7B,OAAO,UAAU,EAAE8B,KAAK,SAAS,CAACD,QAAQ;gBACtD,MAAM;oBAAEA;gBAAM;YAChB;IACF;IAEF,OAAOT,sBAAsB;QAC3BpB;QACA,aAAa,CAAC,IAAI,EAAEA,OAAO,8CAA8C,CAAC;QAC1E,gBAAgB;QAChB,aAAagB;QACb,QAAQ,CAACpB,QAAU;gBAACA,MAAM,MAAM;gBAAEA,MAAM,OAAO;aAAC;QAChD,UAAU,CAACiC,QAAW;gBACpB,SAAS,GAAG7B,OAAO,UAAU,EAAE8B,KAAK,SAAS,CAACD,QAAQ;gBACtD,MAAM;oBAAEA;gBAAM;YAChB;IACF;AACF;AAEA,MAAME,qBAAqBX,sBAAsB;IAC/C,QAAQ;IACR,aAAa;IACb,gBAAgB;IAChB,aAAaD;IACb,QAAOvB,KAAK;QACV,OAAO;YAACA,MAAM,KAAK;YAAEA,MAAM,OAAO;SAAC;IACrC;IACA,UAAS8B,OAAO,EAAE9B,KAAK;QACrB,OAAO;YAAE,SAAS,CAAC,oBAAoB,EAAEA,MAAM,KAAK,IAAI,YAAY;QAAC;IACvE;AACF;AAEO,MAAMoC,uCACX;IACET;IACAE;IACAE;IACAC,YAAY;IACZA,YAAY;IACZA,YAAY;IACZA,YAAY;IACZG;CACD"}

package/dist/lib/agent/utils.js

@@ -205,7 +205,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.12.4-beta-20260904065253.0";
+const getMidsceneVersion = ()=>"1.12.4";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/ai-model/prompt/insight/user-prompt.js

@@ -36,7 +36,7 @@ function buildTypeQueryDemandValue(type, demand) {
 const extractDataQueryPrompt = (pageDescription, dataQuery, context)=>{
     const dataQueryText = 'string' == typeof dataQuery ? dataQuery : JSON.stringify(dataQuery, null, 2);
     const trimmedContext = context?.trim();
-    const contextSection = trimmedContext ? `\n<CONTEXT>\n${trimmedContext}\n</CONTEXT>\n` : '';
+    const contextSection = trimmedContext ? `\n${trimmedContext}\n` : '';
     return `
 <PageDescription>
 ${pageDescription}

package/dist/lib/ai-model/prompt/insight/user-prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/insight/user-prompt.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/prompt/insight/user-prompt.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ServiceExtractParam } from '@/types';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n  context?: string,\n) => {\n  const dataQueryText =\n    typeof dataQuery === 'string'\n      ? dataQuery\n      : JSON.stringify(dataQuery, null, 2);\n\n  const trimmedContext = context?.trim();\n  const contextSection = trimmedContext\n    ? `\\n<CONTEXT>\\n${trimmedContext}\\n</CONTEXT>\\n`\n    : '';\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n${contextSection}\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","extractDataQueryPrompt","pageDescription","dataQuery","context","dataQueryText","JSON","trimmedContext","contextSection"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACJO,SAASI,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAEO,MAAME,yBAAyB,CACpCC,iBACAC,WACAC;IAEA,MAAMC,gBACJ,AAAqB,YAArB,OAAOF,YACHA,YACAG,KAAK,SAAS,CAACH,WAAW,MAAM;IAEtC,MAAMI,iBAAiBH,SAAS;IAChC,MAAMI,iBAAiBD,iBACnB,CAAC,aAAa,EAAEA,eAAe,cAAc,CAAC,GAC9C;IAEJ,OAAO,CAAC;;AAEV,EAAEL,gBAAgB;;AAElB,EAAEM,eAAe;;AAEjB,EAAEH,cAAc;;EAEd,CAAC;AACH"}
+{"version":3,"file":"ai-model/prompt/insight/user-prompt.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/prompt/insight/user-prompt.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ServiceExtractParam } from '@/types';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n  context?: string,\n) => {\n  const dataQueryText =\n    typeof dataQuery === 'string'\n      ? dataQuery\n      : JSON.stringify(dataQuery, null, 2);\n\n  const trimmedContext = context?.trim();\n  const contextSection = trimmedContext ? `\\n${trimmedContext}\\n` : '';\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n${contextSection}\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","extractDataQueryPrompt","pageDescription","dataQuery","context","dataQueryText","JSON","trimmedContext","contextSection"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACJO,SAASI,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAEO,MAAME,yBAAyB,CACpCC,iBACAC,WACAC;IAEA,MAAMC,gBACJ,AAAqB,YAArB,OAAOF,YACHA,YACAG,KAAK,SAAS,CAACH,WAAW,MAAM;IAEtC,MAAMI,iBAAiBH,SAAS;IAChC,MAAMI,iBAAiBD,iBAAiB,CAAC,EAAE,EAAEA,eAAe,EAAE,CAAC,GAAG;IAElE,OAAO,CAAC;;AAEV,EAAEL,gBAAgB;;AAElB,EAAEM,eAAe;;AAEjB,EAAEH,cAAc;;EAEd,CAAC;AACH"}

package/dist/lib/ai-model/workflows/insight/extraction.js

@@ -27,6 +27,7 @@ __webpack_require__.d(__webpack_exports__, {
     AiExtractElementInfo: ()=>AiExtractElementInfo
 });
 const logger_namespaceObject = require("@midscene/shared/logger");
+const prompt_context_js_namespaceObject = require("../../../agent/prompt-context.js");
 const index_js_namespaceObject = require("../../prompt/insight/index.js");
 const external_service_caller_index_js_namespaceObject = require("../../service-caller/index.js");
 const semantic_retry_js_namespaceObject = require("../../service-caller/semantic-retry.js");
@@ -42,7 +43,8 @@ async function AiExtractElementInfo(options) {
         insightProtocol
     });
     const screenshotBase64 = context.screenshot.base64;
-    const extractDataPromptText = (0, index_js_namespaceObject.extractDataQueryPrompt)(options.pageDescription || '', dataQuery, extractOption?.context);
+    const renderedContext = (0, prompt_context_js_namespaceObject.renderAIContext)(extractOption?.context);
+    const extractDataPromptText = (0, index_js_namespaceObject.extractDataQueryPrompt)(options.pageDescription || '', dataQuery, renderedContext);
     const userContent = [];
     if (extractOption?.screenshotIncluded !== false) {
         const screenshotSequence = context.screenshotSequence;

package/dist/lib/ai-model/workflows/insight/extraction.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToChatMessages } from '../../shared/multimodal-prompt';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    extractOption?.context,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInsight","getDebug","AiExtractElementInfo","options","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToChatMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawChoiceMessage","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACmBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEvB,eAAeC,qBAAwBC,OAQ7C;IACC,MAAM,EAAEC,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEL;IACF,MAAMM,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAMQ,wBAAwBC,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAC5BX,QAAQ,eAAe,IAAI,IAC3BC,WACAE,eAAe;IAGjB,MAAMS,cAAyD,EAAE;IAEjE,IAAIT,eAAe,uBAAuB,OAAO;QAC/C,MAAMU,qBAAqBX,QAAQ,kBAAkB;QACrD,IAAIW,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKH;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAG,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAAST;QAAa;QACxC;YACE,MAAM;YACN,SAASK;QACX;KACD;IAED,IAAIR,kBAAkB;QACpB,MAAMa,SAAS,MAAMC,AAAAA,IAAAA,qCAAAA,8BAAAA,AAAAA,EAA+Bd;QACpDY,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,MAAMK,qBAChChB,cACA;gBACE,aAAaL,QAAQ,WAAW;gBAChC,sBAAsBoB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAGJ;YACJ,MAAMK,cAAcC,AAAAA,IAAAA,oDAAAA,oBAAAA,AAAAA,EAClBL,aACAnB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACLwB;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBnB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaL,QAAQ,WAAW;QAChC,cAAc,CAACoC;YACbvC,aACE,iDACAuC,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToChatMessages } from '../../shared/multimodal-prompt';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInsight","getDebug","AiExtractElementInfo","options","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToChatMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawChoiceMessage","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACoBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEvB,eAAeC,qBAAwBC,OAQ7C;IACC,MAAM,EAAEC,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEL;IACF,MAAMM,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAC5Bb,QAAQ,eAAe,IAAI,IAC3BC,WACAS;IAGF,MAAMI,cAAyD,EAAE;IAEjE,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKL;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,AAAAA,IAAAA,qCAAAA,8BAAAA,AAAAA,EAA+BhB;QACpDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaL,QAAQ,WAAW;gBAChC,sBAAsBsB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAGJ;YACJ,MAAMK,cAAcC,AAAAA,IAAAA,oDAAAA,oBAAAA,AAAAA,EAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaL,QAAQ,WAAW;QAChC,cAAc,CAACsC;YACbzC,aACE,iDACAyC,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}

package/dist/lib/ai-model/workflows/planning/custom-planning.js

@@ -32,13 +32,13 @@ const image_preprocess_js_namespaceObject = require("../../model-adapter/image-p
 const index_js_namespaceObject = require("../../service-caller/index.js");
 const multimodal_prompt_js_namespaceObject = require("../../shared/multimodal-prompt.js");
 const external_locate_normalization_js_namespaceObject = require("./locate-normalization.js");
-function appendHighPriorityKnowledge(systemPrompt, actionContext) {
-    return systemPrompt + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n` : '');
+function appendActionContext(systemPrompt, actionContext) {
+    return systemPrompt + (actionContext ? `${actionContext}\n` : '');
 }
 function buildCustomPlanningMessages(input, config) {
     const { options } = input;
     const { conversationHistory, context, actionContext } = options;
-    const systemPrompt = appendHighPriorityKnowledge(config.buildSystemPrompt(), actionContext);
+    const systemPrompt = appendActionContext(config.buildSystemPrompt(), actionContext);
     const userInstructionText = input.userInstruction.text;
     const userInstruction = config.buildUserInstruction ? config.buildUserInstruction(userInstructionText) : userInstructionText;
     const referenceImageMessages = (0, multimodal_prompt_js_namespaceObject.preparedReferenceImagesToChatMessages)(input.userInstruction.referenceImages);

package/dist/lib/ai-model/workflows/planning/custom-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/custom-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendHighPriorityKnowledge(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return (\n    systemPrompt +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\\n`\n      : '')\n  );\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ChatCompletionMessageParam[] {\n  const { options } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendHighPriorityKnowledge(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstructionText = input.userInstruction.text;\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    input.userInstruction.referenceImages,\n  );\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.append({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: { url: context.screenshot.base64 },\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...referenceImageMessages,\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...referenceImageMessages,\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: PreparedUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultCodec: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n      acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,\n      parseRawLocateParameter: (value) =>\n        value as ParsedPlanningLocateParameter,\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  // UI-TARS and Auto-GLM define their own assistant history format.\n  if (assistantContent) {\n    options.conversationHistory.append({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawChoiceMessage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","appendHighPriorityKnowledge","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToChatMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawChoiceMessage","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","value","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACeA,SAASI,4BACPC,YAAoB,EACpBC,aAAsB;IAEtB,OACED,eACCC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,4BAA4B,CAAC,GACvE,EAAC;AAET;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,4BACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,AAAAA,IAAAA,qCAAAA,qCAAAA,AAAAA,EAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBAAE,KAAKC,QAAQ,UAAU,CAAC,MAAM;gBAAC;YAC9C;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,4CAAAA,cAAAA,CAAAA,MAAqB,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,gBAAgB,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACjDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,IAAAA,iDAAAA,mCAAAA,AAAAA,EAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;YACA,mBAAmBE,gBAAgB,YAAY,CAAC,OAAO,CAAC,iBAAiB;YACzE,yBAAyB,CAACW,QACxBA;QACJ;QACAF,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOI,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,yBAAAA,oBAAoBA,CAC5BH,cACAI,KAAK,SAAS,CAACd,SAASe,QAAW,IACnCd,OACAC;IAEJ;IAEA,MAAMc,mBAAmB9B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IAAI+B,kBACF7B,QAAQ,mBAAmB,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,SAAS6B;IACX;IAGF,OAAO;QACLX;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaQ,KAAK,SAAS,CAACd,SAASe,QAAW;QAChDb;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/planning/custom-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendActionContext(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return systemPrompt + (actionContext ? `${actionContext}\\n` : '');\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ChatCompletionMessageParam[] {\n  const { options } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendActionContext(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstructionText = input.userInstruction.text;\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    input.userInstruction.referenceImages,\n  );\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.append({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: { url: context.screenshot.base64 },\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...referenceImageMessages,\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...referenceImageMessages,\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: PreparedUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultCodec: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n      acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,\n      parseRawLocateParameter: (value) =>\n        value as ParsedPlanningLocateParameter,\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  // UI-TARS and Auto-GLM define their own assistant history format.\n  if (assistantContent) {\n    options.conversationHistory.append({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawChoiceMessage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","appendActionContext","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToChatMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawChoiceMessage","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","value","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACeA,SAASI,oBACPC,YAAoB,EACpBC,aAAsB;IAEtB,OAAOD,eAAgBC,CAAAA,gBAAgB,GAAGA,cAAc,EAAE,CAAC,GAAG,EAAC;AACjE;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,oBACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,AAAAA,IAAAA,qCAAAA,qCAAAA,AAAAA,EAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBAAE,KAAKC,QAAQ,UAAU,CAAC,MAAM;gBAAC;YAC9C;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,4CAAAA,cAAAA,CAAAA,MAAqB,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,gBAAgB,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACjDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,IAAAA,iDAAAA,mCAAAA,AAAAA,EAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;YACA,mBAAmBE,gBAAgB,YAAY,CAAC,OAAO,CAAC,iBAAiB;YACzE,yBAAyB,CAACW,QACxBA;QACJ;QACAF,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOI,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,yBAAAA,oBAAoBA,CAC5BH,cACAI,KAAK,SAAS,CAACd,SAASe,QAAW,IACnCd,OACAC;IAEJ;IAEA,MAAMc,mBAAmB9B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IAAI+B,kBACF7B,QAAQ,mBAAmB,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,SAAS6B;IACX;IAGF,OAAO;QACLX;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaQ,KAAK,SAAS,CAACd,SAASe,QAAW;QAChDb;IACF;AACF"}

package/dist/lib/ai-model/workflows/planning/standard-planning.js

@@ -129,7 +129,7 @@ async function standardPlan(userInstruction, opts) {
         policy: adapter.imagePreprocess
     });
     const imagePayload = preparedImage.imageBase64;
-    const actionContext = opts.actionContext ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\n` : '';
+    const actionContext = opts.actionContext ? `${opts.actionContext}\n` : '';
     const referenceImageMessages = (0, multimodal_prompt_js_namespaceObject.preparedReferenceImagesToChatMessages)(userInstruction.referenceImages);
     const instruction = [
         {