@aiscene/shared 1.6.0 → 1.7.2

package/dist/es/cli/cli-runner.mjs

@@ -89,7 +89,7 @@ function printHelp(scriptName, commands, version) {
     }
     console.log(`\nUsage: ${scriptName} <command> [options]\n`);
     console.log('Commands:');
-    for (const { name, def } of commands)console.log(`  ${name.padEnd(30)} ${def.description}`);
+    for (const { name, def } of commands.filter((command)=>!command.hidden))console.log(`  ${name.padEnd(30)} ${def.description}`);
     console.log(`  ${'version'.padEnd(30)} Show CLI version`);
     console.log(`\nRun "${scriptName} <command> --help" for more info.`);
 }
@@ -105,6 +105,18 @@ async function runToolsCLI(tools, scriptName, options) {
             name: removePrefix(def.name, options?.stripPrefix).toLowerCase(),
             def
         }));
+    if (options?.extraCommands?.length) commands.push(...options.extraCommands.flatMap((cmd)=>[
+            {
+                name: cmd.name.toLowerCase(),
+                def: cmd.def,
+                hidden: cmd.hidden
+            },
+            ...(cmd.aliases ?? []).map((alias)=>({
+                    name: alias.toLowerCase(),
+                    def: cmd.def,
+                    hidden: true
+                }))
+        ]));
     const cliVersion = options?.version;
     const [commandName, ...restArgs] = rawArgs;
     if (!commandName || '--help' === commandName || '-h' === commandName) {

package/dist/es/constants/example-code.mjs

@@ -80,6 +80,23 @@ test('ai shop', async ({
 });
 `;
 const YAML_EXAMPLE_CODE = `
+CRITICAL - YAML Indentation Rules:
+For actions with additional parameters (aiScroll, aiInput, aiKeyboardPress), the parameters must be SIBLING keys at the SAME indentation level as the action key, NOT nested children indented further.
+CORRECT (parameters align with the action key):
+      - aiScroll:
+        direction: 'down'
+        scrollType: 'singleAction'
+        distance: 500
+        locate: "main content area"
+      - aiInput: 'text value'
+        locate: 'input field description'
+WRONG (parameters are indented further than the action key, DO NOT do this):
+      - aiScroll:
+          direction: 'down'
+          scrollType: 'singleAction'
+      - aiInput: 'text value'
+          locate: 'input field description'
+
 1. Format:
 
 web:
@@ -91,13 +108,18 @@ tasks:
   - name: "descriptive task name"
     flow:
       - aiTap: "element description"
+        xpath: '/html/body/div[1]/button[1]'
       - aiInput: 'text value'
         locate: 'input field description'
+        xpath: '/html/body/div[1]/input[1]'
       - aiScroll:
-        direction: down/up
-        scrollType: scrollToBottom/scrollToTop/singleAction
+        direction: 'down'
+        scrollType: 'singleAction'
+        distance: 500
+        locate: "scrollable area description"
+        xpath: '/html/body/div[1]/main[1]'
       - aiAssert: "expected state"
-      - sleep: milliseconds
+      - sleep: 1000
 
 2. Action Types:
 - aiTap: for clicks (natural language targeting)
@@ -106,24 +128,6 @@ tasks:
 - aiAssert: for validations
 - sleep: for delays (milliseconds)
 
-3. Best Practices:
-- Group related actions into logical tasks
-- Use natural language descriptions
-- Add deepLocate: true for complex interactions
-- Keep task names concise but descriptive
-
-4. CRITICAL - YAML Indentation Rules:
-- For actions with additional parameters (aiScroll, aiInput, aiKeyboardPress), the parameters must be SIBLING keys, NOT nested children
-- Parameters like direction, scrollType, locate must align with the action key, not indented further
-- CORRECT indentation example:
-      - aiScroll:
-        direction: down
-        scrollType: singleAction
-- WRONG indentation (DO NOT do this):
-      - aiScroll:
-          direction: down
-          scrollType: singleAction
-
 
 
 YAML type

package/dist/es/env/parse-model-config.mjs

@@ -43,6 +43,16 @@ const getModelDescription = (modelFamily, uiTarsModelVersion)=>{
     if (modelFamily) return `${modelFamily} mode`;
     return '';
 };
+const normalizeOpenaiExtraConfig = (config)=>{
+    if (!config || 'object' != typeof config || Array.isArray(config)) return;
+    const { defaultHeaders, extra_headers, extraHeaders, ...rest } = config;
+    const headers = defaultHeaders ?? extra_headers ?? extraHeaders;
+    if (void 0 !== headers) return {
+        ...rest,
+        defaultHeaders: headers
+    };
+    return rest;
+};
 const parseOpenaiSdkConfig = ({ keys, provider, useLegacyLogic = false })=>{
     initDebugConfig();
     const debugLog = getDebug('ai:config');
@@ -73,7 +83,7 @@ const parseOpenaiSdkConfig = ({ keys, provider, useLegacyLogic = false })=>{
         httpProxy,
         openaiBaseURL,
         openaiApiKey,
-        openaiExtraConfig,
+        openaiExtraConfig: normalizeOpenaiExtraConfig(openaiExtraConfig),
         extraBody,
         modelFamily,
         uiTarsModelVersion,

package/dist/es/env/types.mjs

@@ -193,6 +193,7 @@ const MODEL_FAMILY_VALUES = [
     'qwen2.5-vl',
     'qwen3-vl',
     'qwen3.5',
+    'qwen3.6',
     'vlm-ui-tars',
     'vlm-ui-tars-doubao',
     'vlm-ui-tars-doubao-1.5',

package/dist/es/extractor/dom-util.mjs

@@ -81,12 +81,16 @@ function generateElementByPoint(center, description, edgeSize = 8) {
         description: description || ''
     };
 }
-function generateElementByRect(sourceRect, description, edgeSize = 8) {
+function generateElementByRect(sourceRect, description, _edgeSize = 8) {
     const centerX = sourceRect.left + Math.floor((sourceRect.width - 1) / 2);
     const centerY = sourceRect.top + Math.floor((sourceRect.height - 1) / 2);
-    return generateElementByPoint([
-        centerX,
-        centerY
-    ], description, edgeSize);
+    return {
+        rect: sourceRect,
+        center: [
+            centerX,
+            centerY
+        ],
+        description: description || ''
+    };
 }
 export { generateElementByPoint, generateElementByRect, isAElement, isButtonElement, isContainerElement, isFormElement, isImgElement, isNotContainerElement, isSvgElement, isTextElement };

package/dist/es/extractor/locator.mjs

@@ -143,8 +143,8 @@ function getXpathsById(id) {
     ];
 }
 function getXpathsByPoint(point, isOrderSensitive) {
-    let currentWindow = 'undefined' != typeof window ? window : void 0;
-    let currentDocument = 'undefined' != typeof document ? document : void 0;
+    let currentWindow = "u" > typeof window ? window : void 0;
+    let currentDocument = "u" > typeof document ? document : void 0;
     let { left, top } = point;
     let depth = 0;
     const MAX_DEPTH = 10;
@@ -199,7 +199,7 @@ function getXpathsByPoint(point, isOrderSensitive) {
 function getNodeInfoByXpath(xpath) {
     const parts = xpath.split(SUB_XPATH_SEPARATOR).map((p)=>p.trim()).filter(Boolean);
     if (0 === parts.length) return null;
-    let currentDocument = 'undefined' != typeof document ? document : void 0;
+    let currentDocument = "u" > typeof document ? document : void 0;
     let node = null;
     for(let i = 0; i < parts.length; i++){
         const currentXpath = parts[i];
@@ -227,13 +227,13 @@ function getNodeInfoByXpath(xpath) {
 function getElementInfoByXpath(xpath) {
     const node = getNodeInfoByXpath(xpath);
     if (!node) return null;
-    let targetWindow = 'undefined' != typeof window ? window : void 0;
-    let targetDocument = 'undefined' != typeof document ? document : void 0;
+    let targetWindow = "u" > typeof window ? window : void 0;
+    let targetDocument = "u" > typeof document ? document : void 0;
     if (node.ownerDocument?.defaultView) {
         targetWindow = node.ownerDocument.defaultView;
         targetDocument = node.ownerDocument;
     }
-    const rootDoc = 'undefined' != typeof document ? document : null;
+    const rootDoc = "u" > typeof document ? document : null;
     const iframeOffset = calculateIframeOffset(node.ownerDocument ?? null, rootDoc);
     const targetWin = targetWindow;
     const targetDoc = targetDocument;

package/dist/es/extractor/util.mjs

@@ -199,10 +199,10 @@ function getNodeAttributes(node, currentWindow) {
 }
 const NODE_CACHE_MAX_SIZE = 2000;
 function setNodeHashCacheListOnWindow() {
-    if ('undefined' != typeof window) window.midsceneNodeHashCache = new Map();
+    if ("u" > typeof window) window.midsceneNodeHashCache = new Map();
 }
 function getNodeCacheMap() {
-    if ('undefined' == typeof window) return;
+    if ("u" < typeof window) return;
     return window.midsceneNodeHashCache;
 }
 function setNodeToCacheList(node, id) {
@@ -221,7 +221,7 @@ function getNodeFromCacheList(id) {
 function midsceneGenerateHash(node, content, rect) {
     const slicedHash = generateHashId(rect, content);
     if (node) {
-        if ('undefined' != typeof window && !getNodeCacheMap()) setNodeHashCacheListOnWindow();
+        if ("u" > typeof window && !getNodeCacheMap()) setNodeHashCacheListOnWindow();
         setNodeToCacheList(node, slicedHash);
     }
     return slicedHash;
@@ -230,13 +230,13 @@ function generateId(numberId) {
     return `${numberId}`;
 }
 function setGenerateHashOnWindow() {
-    if ('undefined' != typeof window) window.midsceneGenerateHash = midsceneGenerateHash;
+    if ("u" > typeof window) window.midsceneGenerateHash = midsceneGenerateHash;
 }
 function setMidsceneVisibleRectOnWindow() {
-    if ('undefined' != typeof window) window.midsceneVisibleRect = elementRect;
+    if ("u" > typeof window) window.midsceneVisibleRect = elementRect;
 }
 function setExtractTextWithPositionOnWindow() {
-    if ('undefined' != typeof window) window.extractTextWithPosition = extractTextWithPosition;
+    if ("u" > typeof window) window.extractTextWithPosition = extractTextWithPosition;
 }
 function getTopDocument() {
     const container = document.body || document;

package/dist/es/img/canvas-fallback.mjs

@@ -58,7 +58,7 @@ class CanvasImage {
     static async new_from_byteslice(bytes) {
         return new Promise((resolve, reject)=>{
             const blob = new Blob([
-                bytes
+                bytes.buffer
             ], {
                 type: 'image/png'
             });

package/dist/es/index.mjs

@@ -1,4 +1,2 @@
-import { createCopyStaticPlugin, createPlaygroundCopyPlugin } from "./build/copy-static.mjs";
-import { commonIgnoreWarnings } from "./build/rspack-config.mjs";
 const src = {};
-export { commonIgnoreWarnings, createCopyStaticPlugin, createPlaygroundCopyPlugin, src as default };
+export { src as default };

package/dist/es/logger.mjs

@@ -61,10 +61,4 @@ function enableDebug(topic) {
     if (ifInNode) return;
     debug.enable(`${topicPrefix}:${topic}`);
 }
-function cleanupLogStreams() {
-    if (!ifInNode) return;
-    for (const stream of logStreams.values())stream.end();
-    logStreams.clear();
-    debugInstances.clear();
-}
-export { cleanupLogStreams, enableDebug, getDebug };
+export { enableDebug, getDebug };

package/dist/es/mcp/base-server.mjs

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { setIsMcp } from "../utils.mjs";
+import { setIsMcp } from "@aiscene/shared/utils";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";

package/dist/es/mcp/base-tools.mjs

@@ -1,5 +1,5 @@
-import { parseBase64 } from "../img/index.mjs";
-import { getDebug } from "../logger.mjs";
+import { parseBase64 } from "@aiscene/shared/img";
+import { getDebug } from "@aiscene/shared/logger";
 import { generateCommonTools, generateToolsFromActionSpace } from "./tool-generator.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {

package/dist/es/mcp/tool-generator.mjs

@@ -1,22 +1,21 @@
-import { parseBase64 } from "../img/index.mjs";
+import { parseBase64 } from "@aiscene/shared/img";
 import { z } from "zod";
-import { getZodDescription, getZodTypeName } from "../zod-schema-utils.mjs";
+import { getZodDescription, getZodTypeName, isMidsceneLocatorField, unwrapZodField } from "../zod-schema-utils.mjs";
 function getErrorMessage(error) {
     return error instanceof Error ? error.message : String(error);
 }
 function describeActionForMCP(action) {
     const actionDesc = action.description || `Execute ${action.name} action`;
     if (!action.paramSchema) return `${action.name} action, ${actionDesc}`;
-    const schema = action.paramSchema;
-    const isZodObjectType = schema._def?.typeName === 'ZodObject';
-    if (!isZodObjectType || !schema.shape) {
-        const typeName = getZodTypeName(schema);
-        const description = getZodDescription(schema);
+    const shape = getZodObjectShape(action.paramSchema);
+    if (!shape) {
+        const typeName = getZodTypeName(action.paramSchema);
+        const description = getZodDescription(action.paramSchema);
         const paramDesc = description ? `${typeName} - ${description}` : typeName;
         return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
     }
     const paramDescriptions = [];
-    for (const [key, field] of Object.entries(schema.shape))if (field && 'object' == typeof field) {
+    for (const [key, field] of Object.entries(shape))if (field && 'object' == typeof field) {
         const isFieldOptional = 'function' == typeof field.isOptional && field.isOptional();
         const typeName = getZodTypeName(field);
         const description = getZodDescription(field);
@@ -30,9 +29,6 @@ function describeActionForMCP(action) {
 function isZodOptional(value) {
     return '_def' in value && value._def?.typeName === 'ZodOptional';
 }
-function isZodObject(value) {
-    return '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value;
-}
 function unwrapOptional(value) {
     if (isZodOptional(value)) return {
         innerValue: value._def.innerType,
@@ -43,24 +39,31 @@ function unwrapOptional(value) {
         isOptional: false
     };
 }
-function isLocateField(value) {
-    if (!isZodObject(value)) return false;
-    return 'prompt' in value.shape;
+function getZodObjectShape(value) {
+    if (!value) return;
+    const actualValue = unwrapZodField(value);
+    if (actualValue._def?.typeName !== 'ZodObject') return;
+    if ('function' == typeof actualValue._def.shape) return actualValue._def.shape();
+    return actualValue.shape;
+}
+function isRecord(value) {
+    return 'object' == typeof value && null !== value && !Array.isArray(value);
 }
-function makePromptOptional(value, wrapInOptional) {
+function makePromptOptional(shape, wrapInOptional) {
     const newShape = {
-        ...value.shape
+        ...shape
     };
-    newShape.prompt = value.shape.prompt.optional();
+    newShape.prompt = shape.prompt.optional();
     let newSchema = z.object(newShape).passthrough();
     if (wrapInOptional) newSchema = newSchema.optional();
     return newSchema;
 }
 function transformSchemaField(key, value) {
     const { innerValue, isOptional } = unwrapOptional(value);
-    if (isZodObject(innerValue) && isLocateField(innerValue)) return [
+    const shape = getZodObjectShape(innerValue);
+    if (shape && isMidsceneLocatorField(innerValue)) return [
         key,
-        makePromptOptional(innerValue, isOptional)
+        makePromptOptional(shape, isOptional)
     ];
     return [
         key,
@@ -69,9 +72,66 @@ function transformSchemaField(key, value) {
 }
 function extractActionSchema(paramSchema) {
     if (!paramSchema) return {};
-    const schema = paramSchema;
-    if (!isZodObject(schema)) return schema;
-    return Object.fromEntries(Object.entries(schema.shape).map(([key, value])=>transformSchemaField(key, value)));
+    const shape = getZodObjectShape(paramSchema);
+    if (!shape) return paramSchema;
+    return Object.fromEntries(Object.entries(shape).map(([key, value])=>transformSchemaField(key, value)));
+}
+function getPromptText(prompt) {
+    if ('string' == typeof prompt) return prompt;
+    if (isRecord(prompt) && 'string' == typeof prompt.prompt) return prompt.prompt;
+}
+function moveLocateExtrasIntoPrompt(value, locateFieldKeys) {
+    const promptText = getPromptText(value.prompt);
+    if (!promptText) return value;
+    const normalizedPrompt = isRecord(value.prompt) ? {
+        ...value.prompt
+    } : {
+        prompt: promptText
+    };
+    const normalizedLocate = {};
+    let movedExtraField = false;
+    for (const [key, fieldValue] of Object.entries(value))if ('prompt' !== key) {
+        if (locateFieldKeys.has(key)) {
+            normalizedLocate[key] = fieldValue;
+            continue;
+        }
+        movedExtraField = true;
+        if (!(key in normalizedPrompt)) normalizedPrompt[key] = fieldValue;
+    }
+    if (!movedExtraField) return value;
+    return {
+        ...normalizedLocate,
+        prompt: normalizedPrompt
+    };
+}
+function normalizeLocateLikeArg(value, fieldSchema) {
+    if ('string' == typeof value) return {
+        prompt: value
+    };
+    if (!isRecord(value)) return value;
+    const shape = getZodObjectShape(fieldSchema);
+    if (!shape) return value;
+    return moveLocateExtrasIntoPrompt(value, new Set(Object.keys(shape)));
+}
+function normalizeActionArgs(args, paramSchema) {
+    if (!paramSchema) return args;
+    const shape = getZodObjectShape(paramSchema);
+    if (!shape) return args;
+    return Object.fromEntries(Object.entries(args).map(([key, value])=>{
+        const fieldSchema = shape[key];
+        if (!fieldSchema) return [
+            key,
+            value
+        ];
+        if (isMidsceneLocatorField(fieldSchema)) return [
+            key,
+            normalizeLocateLikeArg(value, fieldSchema)
+        ];
+        return [
+            key,
+            value
+        ];
+    }));
 }
 function serializeArgsToDescription(args) {
     try {
@@ -90,7 +150,7 @@ function serializeArgsToDescription(args) {
     }
 }
 function buildActionInstruction(actionName, args) {
-    const locatePrompt = args.locate && 'object' == typeof args.locate ? args.locate.prompt : void 0;
+    const locatePrompt = isRecord(args.locate) ? getPromptText(args.locate.prompt) : void 0;
     switch(actionName){
         case 'Tap':
             return locatePrompt ? `Tap on "${locatePrompt}"` : 'Tap';
@@ -118,44 +178,59 @@ function buildActionInstruction(actionName, args) {
             }
     }
 }
-async function captureScreenshotResult(agent, actionName) {
+async function executeAction(agent, actionName, args) {
+    if (agent.callActionInActionSpace) return agent.callActionInActionSpace(actionName, args);
+    if (agent.aiAction) {
+        const instruction = buildActionInstruction(actionName, args);
+        return agent.aiAction(instruction);
+    }
+    throw new Error(`Action "${actionName}" is not supported by this agent`);
+}
+async function captureScreenshotResult(agent, actionName, actionResult) {
+    const content = [
+        {
+            type: 'text',
+            text: `Action "${actionName}" completed.`
+        }
+    ];
+    if (void 0 !== actionResult) content.push({
+        type: 'text',
+        text: `Result: ${serializeActionResult(actionResult)}`
+    });
     try {
         const screenshot = await agent.page?.screenshotBase64();
         if (!screenshot) return {
-            content: [
-                {
-                    type: 'text',
-                    text: `Action "${actionName}" completed.`
-                }
-            ]
+            content
         };
         const { mimeType, body } = parseBase64(screenshot);
+        content.push({
+            type: 'image',
+            data: body,
+            mimeType
+        });
         return {
-            content: [
-                {
-                    type: 'text',
-                    text: `Action "${actionName}" completed.`
-                },
-                {
-                    type: 'image',
-                    data: body,
-                    mimeType
-                }
-            ]
+            content
         };
     } catch (error) {
         const errorMessage = getErrorMessage(error);
         console.error('Error capturing screenshot:', errorMessage);
+        content[0] = {
+            type: 'text',
+            text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
+        };
         return {
-            content: [
-                {
-                    type: 'text',
-                    text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
-                }
-            ]
+            content
         };
     }
 }
+function serializeActionResult(actionResult) {
+    if ('string' == typeof actionResult) return actionResult;
+    try {
+        return JSON.stringify(actionResult);
+    } catch  {
+        return String(actionResult);
+    }
+}
 function createErrorResult(message) {
     return {
         content: [
@@ -214,17 +289,16 @@ function generateToolsFromActionSpace(actionSpace, getAgent) {
             handler: async (args)=>{
                 try {
                     const agent = await getAgent();
-                    if (agent.aiAction) {
-                        const instruction = buildActionInstruction(action.name, args);
-                        try {
-                            await agent.aiAction(instruction);
-                        } catch (error) {
-                            const errorMessage = getErrorMessage(error);
-                            console.error(`Error executing action "${action.name}":`, errorMessage);
-                            return await captureFailureResult(agent, action.name, errorMessage);
-                        }
+                    const normalizedArgs = normalizeActionArgs(args, action.paramSchema);
+                    let actionResult;
+                    try {
+                        actionResult = await executeAction(agent, action.name, normalizedArgs);
+                    } catch (error) {
+                        const errorMessage = getErrorMessage(error);
+                        console.error(`Error executing action "${action.name}":`, errorMessage);
+                        return await captureFailureResult(agent, action.name, errorMessage);
                     }
-                    return await captureScreenshotResult(agent, action.name);
+                    return await captureScreenshotResult(agent, action.name, actionResult);
                 } catch (error) {
                     const errorMessage = getErrorMessage(error);
                     console.error(`Error in handler for "${action.name}":`, errorMessage);