@aiscene/shared 1.6.0 → 1.8.0

package/src/cli/cli-runner.ts

@@ -15,12 +15,21 @@ const debug = getDebug('cli-runner');
 interface CLICommand {
   name: string;
   def: ToolDefinition;
+  hidden?: boolean;
+}
+
+export interface CLIExtraCommand {
+  name: string;
+  def: ToolDefinition;
+  aliases?: string[];
+  hidden?: boolean;
 }
 
 export interface CLIRunnerOptions {
   stripPrefix?: string;
   argv?: string[];
   version?: string;
+  extraCommands?: CLIExtraCommand[];
 }
 
 export class CLIError extends Error {
@@ -142,7 +151,7 @@ function printHelp(
   }
   console.log(`\nUsage: ${scriptName} <command> [options]\n`);
   console.log('Commands:');
-  for (const { name, def } of commands) {
+  for (const { name, def } of commands.filter((command) => !command.hidden)) {
     console.log(`  ${name.padEnd(30)} ${def.description}`);
   }
   console.log(`  ${'version'.padEnd(30)} Show CLI version`);
@@ -169,6 +178,22 @@ export async function runToolsCLI(
     name: removePrefix(def.name, options?.stripPrefix).toLowerCase(),
     def,
   }));
+  if (options?.extraCommands?.length) {
+    commands.push(
+      ...options.extraCommands.flatMap((cmd) => [
+        {
+          name: cmd.name.toLowerCase(),
+          def: cmd.def,
+          hidden: cmd.hidden,
+        },
+        ...(cmd.aliases ?? []).map((alias) => ({
+          name: alias.toLowerCase(),
+          def: cmd.def,
+          hidden: true,
+        })),
+      ]),
+    );
+  }
   const cliVersion = options?.version;
 
   const [commandName, ...restArgs] = rawArgs;

package/src/cli/index.ts

@@ -5,4 +5,4 @@ export {
   parseCliArgs,
   removePrefix,
 } from './cli-runner';
-export type { CLIRunnerOptions } from './cli-runner';
+export type { CLIRunnerOptions, CLIExtraCommand } from './cli-runner';

package/src/constants/example-code.ts

@@ -81,6 +81,23 @@ test('ai shop', async ({
 `;
 
 export const YAML_EXAMPLE_CODE = `
+CRITICAL - YAML Indentation Rules:
+For actions with additional parameters (aiScroll, aiInput, aiKeyboardPress), the parameters must be SIBLING keys at the SAME indentation level as the action key, NOT nested children indented further.
+CORRECT (parameters align with the action key):
+      - aiScroll:
+        direction: 'down'
+        scrollType: 'singleAction'
+        distance: 500
+        locate: "main content area"
+      - aiInput: 'text value'
+        locate: 'input field description'
+WRONG (parameters are indented further than the action key, DO NOT do this):
+      - aiScroll:
+          direction: 'down'
+          scrollType: 'singleAction'
+      - aiInput: 'text value'
+          locate: 'input field description'
+
 1. Format:
 
 web:
@@ -92,13 +109,18 @@ tasks:
   - name: "descriptive task name"
     flow:
       - aiTap: "element description"
+        xpath: '/html/body/div[1]/button[1]'
       - aiInput: 'text value'
         locate: 'input field description'
+        xpath: '/html/body/div[1]/input[1]'
       - aiScroll:
-        direction: down/up
-        scrollType: scrollToBottom/scrollToTop/singleAction
+        direction: 'down'
+        scrollType: 'singleAction'
+        distance: 500
+        locate: "scrollable area description"
+        xpath: '/html/body/div[1]/main[1]'
       - aiAssert: "expected state"
-      - sleep: milliseconds
+      - sleep: 1000
 
 2. Action Types:
 - aiTap: for clicks (natural language targeting)
@@ -107,24 +129,6 @@ tasks:
 - aiAssert: for validations
 - sleep: for delays (milliseconds)
 
-3. Best Practices:
-- Group related actions into logical tasks
-- Use natural language descriptions
-- Add deepLocate: true for complex interactions
-- Keep task names concise but descriptive
-
-4. CRITICAL - YAML Indentation Rules:
-- For actions with additional parameters (aiScroll, aiInput, aiKeyboardPress), the parameters must be SIBLING keys, NOT nested children
-- Parameters like direction, scrollType, locate must align with the action key, not indented further
-- CORRECT indentation example:
-      - aiScroll:
-        direction: down
-        scrollType: singleAction
-- WRONG indentation (DO NOT do this):
-      - aiScroll:
-          direction: down
-          scrollType: singleAction
-
 
 
 YAML type

package/src/env/parse-model-config.ts

@@ -136,6 +136,26 @@ const getModelDescription = (
   return '';
 };
 
+const normalizeOpenaiExtraConfig = (
+  config: unknown,
+): Record<string, unknown> | undefined => {
+  if (!config || typeof config !== 'object' || Array.isArray(config)) {
+    return undefined;
+  }
+
+  const { defaultHeaders, extra_headers, extraHeaders, ...rest } =
+    config as Record<string, unknown>;
+
+  // Priority: defaultHeaders > extra_headers > extraHeaders
+  const headers = defaultHeaders ?? extra_headers ?? extraHeaders;
+
+  if (headers !== undefined) {
+    return { ...rest, defaultHeaders: headers };
+  }
+
+  return rest;
+};
+
 /**
  * Parse OpenAI SDK config
  */
@@ -201,7 +221,7 @@ export const parseOpenaiSdkConfig = ({
     httpProxy,
     openaiBaseURL,
     openaiApiKey,
-    openaiExtraConfig,
+    openaiExtraConfig: normalizeOpenaiExtraConfig(openaiExtraConfig),
     extraBody,
     modelFamily,
     uiTarsModelVersion,

package/src/env/types.ts

@@ -293,6 +293,7 @@ export type TModelFamily =
   | 'qwen2.5-vl'
   | 'qwen3-vl'
   | 'qwen3.5'
+  | 'qwen3.6'
   | 'doubao-vision'
   | 'doubao-seed'
   | 'gemini'
@@ -311,6 +312,7 @@ export const MODEL_FAMILY_VALUES: TModelFamily[] = [
   'qwen2.5-vl',
   'qwen3-vl',
   'qwen3.5',
+  'qwen3.6',
   'vlm-ui-tars',
   'vlm-ui-tars-doubao',
   'vlm-ui-tars-doubao-1.5',

package/src/extractor/dom-util.ts

@@ -181,7 +181,8 @@ export function generateElementByPoint(
 
 /**
  * Generate a LocateResultElement from a rect.
- * This function calculates the center point from the rect and expands the rect by edgeSize.
+ * This function calculates the center point from the rect and preserves the
+ * original rect as the returned element boundary.
  *
  * Note: The rect uses inclusive coordinates where:
  * - A rect from [left=10, top=10] with [width=1, height=1] covers exactly 1 pixel
@@ -189,13 +190,13 @@ export function generateElementByPoint(
  *
  * @param sourceRect - The source rect to generate element from (typically contains integer values)
  * @param description - Description of the element
- * @param edgeSize - Size to expand around the center point (default: 8)
- * @returns A LocateResultElement with rect, center (always integers), and description
+ * @param edgeSize - Deprecated, retained for backward compatibility
+ * @returns A LocateResultElement with the original rect, center (always integers), and description
  */
 export function generateElementByRect(
   sourceRect: { left: number; top: number; width: number; height: number },
   description: string,
-  edgeSize = 8,
+  _edgeSize = 8,
 ): LocateResultElement {
   /**
    * Calculate center point from rect
@@ -222,5 +223,9 @@ export function generateElementByRect(
   const centerX = sourceRect.left + Math.floor((sourceRect.width - 1) / 2);
   const centerY = sourceRect.top + Math.floor((sourceRect.height - 1) / 2);
 
-  return generateElementByPoint([centerX, centerY], description, edgeSize);
+  return {
+    rect: sourceRect,
+    center: [centerX, centerY],
+    description: description || '',
+  };
 }

package/src/index.ts

@@ -1,8 +1 @@
-export {
-  createCopyStaticPlugin,
-  createPlaygroundCopyPlugin,
-} from './build/copy-static';
-
-export { commonIgnoreWarnings } from './build/rspack-config';
-
 export default {};

package/src/logger.ts

@@ -94,14 +94,3 @@ export function enableDebug(topic: string): void {
   }
   debug.enable(`${topicPrefix}:${topic}`);
 }
-
-// Cleanup function to close all log streams
-export function cleanupLogStreams(): void {
-  if (!ifInNode) return;
-
-  for (const stream of logStreams.values()) {
-    stream.end();
-  }
-  logStreams.clear();
-  debugInstances.clear();
-}

package/src/mcp/tool-generator.ts

@@ -1,6 +1,11 @@
 import { parseBase64 } from '../img';
 import { z } from 'zod';
-import { getZodDescription, getZodTypeName } from '../zod-schema-utils';
+import {
+  getZodDescription,
+  getZodTypeName,
+  isMidsceneLocatorField,
+  unwrapZodField,
+} from '../zod-schema-utils';
 import type {
   ActionSpaceItem,
   BaseAgent,
@@ -26,23 +31,18 @@ function describeActionForMCP(action: ActionSpaceItem): string {
     return `${action.name} action, ${actionDesc}`;
   }
 
-  const schema = action.paramSchema as {
-    _def?: { typeName?: string };
-    shape?: Record<string, unknown>;
-  };
-  const isZodObjectType = schema._def?.typeName === 'ZodObject';
-
-  if (!isZodObjectType || !schema.shape) {
+  const shape = getZodObjectShape(action.paramSchema);
+  if (!shape) {
     // Simple type schema
-    const typeName = getZodTypeName(schema);
-    const description = getZodDescription(schema as z.ZodTypeAny);
+    const typeName = getZodTypeName(action.paramSchema);
+    const description = getZodDescription(action.paramSchema as z.ZodTypeAny);
     const paramDesc = description ? `${typeName} - ${description}` : typeName;
     return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
   }
 
   // Object schema with multiple fields
   const paramDescriptions: string[] = [];
-  for (const [key, field] of Object.entries(schema.shape)) {
+  for (const [key, field] of Object.entries(shape)) {
     if (field && typeof field === 'object') {
       const isFieldOptional =
         typeof (field as { isOptional?: () => boolean }).isOptional ===
@@ -96,25 +96,42 @@ function unwrapOptional(value: z.ZodTypeAny): {
   return { innerValue: value, isOptional: false };
 }
 
-/**
- * Check if a Zod object schema contains a 'prompt' field (locate field pattern)
- */
-function isLocateField(value: z.ZodTypeAny): boolean {
-  if (!isZodObject(value)) {
-    return false;
+function getZodObjectShape(
+  value: z.ZodTypeAny | undefined,
+): Record<string, z.ZodTypeAny> | undefined {
+  if (!value) {
+    return undefined;
   }
-  return 'prompt' in value.shape;
+
+  const actualValue = unwrapZodField(value) as {
+    _def?: { typeName?: string; shape?: () => Record<string, z.ZodTypeAny> };
+    shape?: Record<string, z.ZodTypeAny>;
+  };
+
+  if (actualValue._def?.typeName !== 'ZodObject') {
+    return undefined;
+  }
+
+  if (typeof actualValue._def.shape === 'function') {
+    return actualValue._def.shape();
+  }
+
+  return actualValue.shape;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
 
 /**
  * Transform a locate field schema to make its 'prompt' field optional
  */
 function makePromptOptional(
-  value: z.ZodObject<z.ZodRawShape>,
+  shape: Record<string, z.ZodTypeAny>,
   wrapInOptional: boolean,
 ): z.ZodTypeAny {
-  const newShape = { ...value.shape };
-  newShape.prompt = value.shape.prompt.optional();
+  const newShape = { ...shape };
+  newShape.prompt = shape.prompt.optional();
 
   let newSchema: z.ZodTypeAny = z.object(newShape).passthrough();
   if (wrapInOptional) {
@@ -131,9 +148,10 @@ function transformSchemaField(
   value: z.ZodTypeAny,
 ): [string, z.ZodTypeAny] {
   const { innerValue, isOptional } = unwrapOptional(value);
+  const shape = getZodObjectShape(innerValue);
 
-  if (isZodObject(innerValue) && isLocateField(innerValue)) {
-    return [key, makePromptOptional(innerValue, isOptional)];
+  if (shape && isMidsceneLocatorField(innerValue)) {
+    return [key, makePromptOptional(shape, isOptional)];
   }
   return [key, value];
 }
@@ -148,18 +166,117 @@ function extractActionSchema(
     return {};
   }
 
-  const schema = paramSchema as z.ZodTypeAny;
-  if (!isZodObject(schema)) {
-    return schema as unknown as Record<string, z.ZodTypeAny>;
+  const shape = getZodObjectShape(paramSchema);
+  if (!shape) {
+    return paramSchema as unknown as Record<string, z.ZodTypeAny>;
   }
 
   return Object.fromEntries(
-    Object.entries(schema.shape).map(([key, value]) =>
+    Object.entries(shape).map(([key, value]) =>
       transformSchemaField(key, value as z.ZodTypeAny),
     ),
   );
 }
 
+function getPromptText(prompt: unknown): string | undefined {
+  if (typeof prompt === 'string') {
+    return prompt;
+  }
+
+  if (isRecord(prompt) && typeof prompt.prompt === 'string') {
+    return prompt.prompt;
+  }
+
+  return undefined;
+}
+
+function moveLocateExtrasIntoPrompt(
+  value: Record<string, unknown>,
+  locateFieldKeys: Set<string>,
+): Record<string, unknown> {
+  const promptText = getPromptText(value.prompt);
+  if (!promptText) {
+    return value;
+  }
+
+  const normalizedPrompt: Record<string, unknown> = isRecord(value.prompt)
+    ? { ...value.prompt }
+    : { prompt: promptText };
+  const normalizedLocate: Record<string, unknown> = {};
+  let movedExtraField = false;
+
+  for (const [key, fieldValue] of Object.entries(value)) {
+    if (key === 'prompt') {
+      continue;
+    }
+
+    if (locateFieldKeys.has(key)) {
+      normalizedLocate[key] = fieldValue;
+      continue;
+    }
+
+    movedExtraField = true;
+    if (!(key in normalizedPrompt)) {
+      normalizedPrompt[key] = fieldValue;
+    }
+  }
+
+  if (!movedExtraField) {
+    return value;
+  }
+
+  return { ...normalizedLocate, prompt: normalizedPrompt };
+}
+
+function normalizeLocateLikeArg(
+  value: unknown,
+  fieldSchema: z.ZodTypeAny,
+): unknown {
+  if (typeof value === 'string') {
+    return { prompt: value };
+  }
+
+  if (!isRecord(value)) {
+    return value;
+  }
+
+  const shape = getZodObjectShape(fieldSchema);
+  if (!shape) {
+    return value;
+  }
+
+  return moveLocateExtrasIntoPrompt(value, new Set(Object.keys(shape)));
+}
+
+function normalizeActionArgs(
+  args: Record<string, unknown>,
+  paramSchema?: z.ZodTypeAny,
+): Record<string, unknown> {
+  if (!paramSchema) {
+    return args;
+  }
+
+  const shape = getZodObjectShape(paramSchema);
+  if (!shape) {
+    return args;
+  }
+
+  return Object.fromEntries(
+    Object.entries(args).map(([key, value]) => {
+      const fieldSchema = shape[key] as z.ZodTypeAny | undefined;
+      if (!fieldSchema) {
+        return [key, value];
+      }
+
+      if (isMidsceneLocatorField(fieldSchema)) {
+        return [key, normalizeLocateLikeArg(value, fieldSchema)];
+      }
+
+      return [key, value];
+    }),
+  );
+}
+
 /**
  * Serialize args to human-readable description for AI action
  */
@@ -194,10 +311,9 @@ function buildActionInstruction(
   actionName: string,
   args: Record<string, unknown>,
 ): string {
-  const locatePrompt =
-    args.locate && typeof args.locate === 'object'
-      ? (args.locate as { prompt?: string }).prompt
-      : undefined;
+  const locatePrompt = isRecord(args.locate)
+    ? getPromptText(args.locate.prompt)
+    : undefined;
 
   switch (actionName) {
     case 'Tap':
@@ -227,39 +343,71 @@ function buildActionInstruction(
   }
 }
 
+async function executeAction(
+  agent: BaseAgent,
+  actionName: string,
+  args: Record<string, unknown>,
+): Promise<unknown> {
+  if (agent.callActionInActionSpace) {
+    return agent.callActionInActionSpace(actionName, args);
+  }
+
+  if (agent.aiAction) {
+    const instruction = buildActionInstruction(actionName, args);
+    return agent.aiAction(instruction);
+  }
+
+  throw new Error(`Action "${actionName}" is not supported by this agent`);
+}
+
 /**
  * Capture screenshot and return as tool result
  */
 async function captureScreenshotResult(
   agent: BaseAgent,
   actionName: string,
+  actionResult?: unknown,
 ): Promise<ToolResult> {
+  const content: ToolResult['content'] = [
+    { type: 'text', text: `Action "${actionName}" completed.` },
+  ];
+
+  if (actionResult !== undefined) {
+    content.push({
+      type: 'text',
+      text: `Result: ${serializeActionResult(actionResult)}`,
+    });
+  }
+
   try {
     const screenshot = await agent.page?.screenshotBase64();
     if (!screenshot) {
-      return {
-        content: [{ type: 'text', text: `Action "${actionName}" completed.` }],
-      };
+      return { content };
     }
 
     const { mimeType, body } = parseBase64(screenshot);
-    return {
-      content: [
-        { type: 'text', text: `Action "${actionName}" completed.` },
-        { type: 'image', data: body, mimeType },
-      ],
-    };
+    content.push({ type: 'image', data: body, mimeType });
+    return { content };
   } catch (error: unknown) {
     const errorMessage = getErrorMessage(error);
     console.error('Error capturing screenshot:', errorMessage);
-    return {
-      content: [
-        {
-          type: 'text',
-          text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`,
-        },
-      ],
+    content[0] = {
+      type: 'text',
+      text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`,
     };
+    return { content };
+  }
+}
+
+function serializeActionResult(actionResult: unknown): string {
+  if (typeof actionResult === 'string') {
+    return actionResult;
+  }
+
+  try {
+    return JSON.stringify(actionResult);
+  } catch {
+    return String(actionResult);
   }
 }
 
@@ -323,28 +471,31 @@ export function generateToolsFromActionSpace(
       handler: async (args: Record<string, unknown>) => {
         try {
           const agent = await getAgent();
+          const normalizedArgs = normalizeActionArgs(args, action.paramSchema);
+          let actionResult: unknown;
 
-          if (agent.aiAction) {
-            const instruction = buildActionInstruction(action.name, args);
-            try {
-              await agent.aiAction(instruction);
-            } catch (error: unknown) {
-              const errorMessage = getErrorMessage(error);
-              console.error(
-                `Error executing action "${action.name}":`,
-                errorMessage,
-              );
-              // Return screenshot + warning instead of hard error,
-              // so the AI agent can see current state and decide to retry or adjust strategy
-              return await captureFailureResult(
-                agent,
-                action.name,
-                errorMessage,
-              );
-            }
+          try {
+            actionResult = await executeAction(
+              agent,
+              action.name,
+              normalizedArgs,
+            );
+          } catch (error: unknown) {
+            const errorMessage = getErrorMessage(error);
+            console.error(
+              `Error executing action "${action.name}":`,
+              errorMessage,
+            );
+            // Return screenshot + warning instead of hard error,
+            // so the AI agent can see current state and decide to retry or adjust strategy
+            return await captureFailureResult(agent, action.name, errorMessage);
           }
 
-          return await captureScreenshotResult(agent, action.name);
+          return await captureScreenshotResult(
+            agent,
+            action.name,
+            actionResult,
+          );
         } catch (error: unknown) {
           // Connection/agent errors are still hard errors
           const errorMessage = getErrorMessage(error);

package/src/mcp/types.ts

@@ -84,6 +84,10 @@ export interface BaseAgent {
   page?: {
     screenshotBase64(): Promise<string>;
   };
+  callActionInActionSpace?: (
+    actionName: string,
+    params?: unknown,
+  ) => Promise<unknown>;
   aiAction?: (
     description: string,
     params?: Record<string, unknown>,

package/src/utils.ts

@@ -64,23 +64,6 @@ export function assert(condition: any, message?: string): asserts condition {
   }
 }
 
-type GlobalScope = typeof window | typeof globalThis | typeof self | undefined;
-
-export function getGlobalScope(): GlobalScope {
-  if (typeof window !== 'undefined') {
-    return window;
-  }
-
-  if (typeof globalThis !== 'undefined') {
-    return globalThis;
-  }
-
-  if (typeof self !== 'undefined') {
-    return self;
-  }
-  return undefined;
-}
-
 let isMcp = false;
 
 export function setIsMcp(value: boolean) {