@aiscene/shared 1.6.0 → 1.7.2

package/dist/types/env/types.d.ts

@@ -128,7 +128,7 @@ export type TGlobalConfig = Record<TEnvKeys, string | undefined>;
 /**
  * valid Model family types
  */
-export type TModelFamily = 'qwen2.5-vl' | 'qwen3-vl' | 'qwen3.5' | 'doubao-vision' | 'doubao-seed' | 'gemini' | 'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5' | 'glm-v' | 'auto-glm' | 'auto-glm-multilingual' | 'gpt-5';
+export type TModelFamily = 'qwen2.5-vl' | 'qwen3-vl' | 'qwen3.5' | 'qwen3.6' | 'doubao-vision' | 'doubao-seed' | 'gemini' | 'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5' | 'glm-v' | 'auto-glm' | 'auto-glm-multilingual' | 'gpt-5';
 export declare const MODEL_FAMILY_VALUES: TModelFamily[];
 export interface IModelConfigForInsight {
     [MIDSCENE_INSIGHT_MODEL_NAME]: string;

package/dist/types/extractor/dom-util.d.ts

@@ -37,7 +37,8 @@ export declare function isContainerElement(node: globalThis.Node): node is globa
 export declare function generateElementByPoint(center: [number, number], description: string, edgeSize?: number): LocateResultElement;
 /**
  * Generate a LocateResultElement from a rect.
- * This function calculates the center point from the rect and expands the rect by edgeSize.
+ * This function calculates the center point from the rect and preserves the
+ * original rect as the returned element boundary.
  *
  * Note: The rect uses inclusive coordinates where:
  * - A rect from [left=10, top=10] with [width=1, height=1] covers exactly 1 pixel
@@ -45,12 +46,12 @@ export declare function generateElementByPoint(center: [number, number], descrip
  *
  * @param sourceRect - The source rect to generate element from (typically contains integer values)
  * @param description - Description of the element
- * @param edgeSize - Size to expand around the center point (default: 8)
- * @returns A LocateResultElement with rect, center (always integers), and description
+ * @param edgeSize - Deprecated, retained for backward compatibility
+ * @returns A LocateResultElement with the original rect, center (always integers), and description
  */
 export declare function generateElementByRect(sourceRect: {
     left: number;
     top: number;
     width: number;
     height: number;
-}, description: string, edgeSize?: number): LocateResultElement;
+}, description: string, _edgeSize?: number): LocateResultElement;

package/dist/types/index.d.ts

@@ -1,4 +1,2 @@
-export { createCopyStaticPlugin, createPlaygroundCopyPlugin, } from './build/copy-static';
-export { commonIgnoreWarnings } from './build/rspack-config';
 declare const _default: {};
 export default _default;

package/dist/types/logger.d.ts

@@ -3,4 +3,3 @@ export declare function getDebug(topic: string, options?: {
     console?: boolean;
 }): DebugFunction;
 export declare function enableDebug(topic: string): void;
-export declare function cleanupLogStreams(): void;

package/dist/types/mcp/types.d.ts

@@ -83,6 +83,7 @@ export interface BaseAgent {
     page?: {
         screenshotBase64(): Promise<string>;
     };
+    callActionInActionSpace?: (actionName: string, params?: unknown) => Promise<unknown>;
     aiAction?: (description: string, params?: Record<string, unknown>) => Promise<unknown>;
     aiWaitFor?: (assertion: string, options: Record<string, unknown>) => Promise<unknown>;
 }

package/dist/types/utils.d.ts

@@ -11,8 +11,6 @@ export declare function generateHashId(rect: any, content?: string): string;
  * @throws Error with the provided message if the condition is false
  */
 export declare function assert(condition: any, message?: string): asserts condition;
-type GlobalScope = typeof window | typeof globalThis | typeof self | undefined;
-export declare function getGlobalScope(): GlobalScope;
 export declare function setIsMcp(value: boolean): void;
 export declare function logMsg(...message: Parameters<typeof console.log>): void;
 export declare function repeat(times: number, fn: (index: number) => Promise<void>): Promise<void>;
@@ -34,4 +32,3 @@ export declare function normalizeForComparison(str: string): string;
  * User-provided mapping takes precedence over default mapping.
  */
 export declare function mergeAndNormalizeAppNameMapping(defaultMapping: Record<string, string>, userMapping?: Record<string, string>): Record<string, string>;
-export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiscene/shared",
-  "version": "1.6.0",
+  "version": "1.7.2",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",
@@ -99,7 +99,7 @@
     "sharp": "^0.34.3",
     "dotenv": "^16.4.5",
     "uuid": "11.1.0",
-    "zod": "3.24.3"
+    "zod": "^3.25.1"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",

package/src/cli/cli-runner.ts

@@ -15,12 +15,21 @@ const debug = getDebug('cli-runner');
 interface CLICommand {
   name: string;
   def: ToolDefinition;
+  hidden?: boolean;
+}
+
+export interface CLIExtraCommand {
+  name: string;
+  def: ToolDefinition;
+  aliases?: string[];
+  hidden?: boolean;
 }
 
 export interface CLIRunnerOptions {
   stripPrefix?: string;
   argv?: string[];
   version?: string;
+  extraCommands?: CLIExtraCommand[];
 }
 
 export class CLIError extends Error {
@@ -142,7 +151,7 @@ function printHelp(
   }
   console.log(`\nUsage: ${scriptName} <command> [options]\n`);
   console.log('Commands:');
-  for (const { name, def } of commands) {
+  for (const { name, def } of commands.filter((command) => !command.hidden)) {
     console.log(`  ${name.padEnd(30)} ${def.description}`);
   }
   console.log(`  ${'version'.padEnd(30)} Show CLI version`);
@@ -169,6 +178,22 @@ export async function runToolsCLI(
     name: removePrefix(def.name, options?.stripPrefix).toLowerCase(),
     def,
   }));
+  if (options?.extraCommands?.length) {
+    commands.push(
+      ...options.extraCommands.flatMap((cmd) => [
+        {
+          name: cmd.name.toLowerCase(),
+          def: cmd.def,
+          hidden: cmd.hidden,
+        },
+        ...(cmd.aliases ?? []).map((alias) => ({
+          name: alias.toLowerCase(),
+          def: cmd.def,
+          hidden: true,
+        })),
+      ]),
+    );
+  }
   const cliVersion = options?.version;
 
   const [commandName, ...restArgs] = rawArgs;

package/src/cli/index.ts

@@ -5,4 +5,4 @@ export {
   parseCliArgs,
   removePrefix,
 } from './cli-runner';
-export type { CLIRunnerOptions } from './cli-runner';
+export type { CLIRunnerOptions, CLIExtraCommand } from './cli-runner';

package/src/constants/example-code.ts

@@ -81,6 +81,23 @@ test('ai shop', async ({
 `;
 
 export const YAML_EXAMPLE_CODE = `
+CRITICAL - YAML Indentation Rules:
+For actions with additional parameters (aiScroll, aiInput, aiKeyboardPress), the parameters must be SIBLING keys at the SAME indentation level as the action key, NOT nested children indented further.
+CORRECT (parameters align with the action key):
+      - aiScroll:
+        direction: 'down'
+        scrollType: 'singleAction'
+        distance: 500
+        locate: "main content area"
+      - aiInput: 'text value'
+        locate: 'input field description'
+WRONG (parameters are indented further than the action key, DO NOT do this):
+      - aiScroll:
+          direction: 'down'
+          scrollType: 'singleAction'
+      - aiInput: 'text value'
+          locate: 'input field description'
+
 1. Format:
 
 web:
@@ -92,13 +109,18 @@ tasks:
   - name: "descriptive task name"
     flow:
       - aiTap: "element description"
+        xpath: '/html/body/div[1]/button[1]'
       - aiInput: 'text value'
         locate: 'input field description'
+        xpath: '/html/body/div[1]/input[1]'
       - aiScroll:
-        direction: down/up
-        scrollType: scrollToBottom/scrollToTop/singleAction
+        direction: 'down'
+        scrollType: 'singleAction'
+        distance: 500
+        locate: "scrollable area description"
+        xpath: '/html/body/div[1]/main[1]'
       - aiAssert: "expected state"
-      - sleep: milliseconds
+      - sleep: 1000
 
 2. Action Types:
 - aiTap: for clicks (natural language targeting)
@@ -107,24 +129,6 @@ tasks:
 - aiAssert: for validations
 - sleep: for delays (milliseconds)
 
-3. Best Practices:
-- Group related actions into logical tasks
-- Use natural language descriptions
-- Add deepLocate: true for complex interactions
-- Keep task names concise but descriptive
-
-4. CRITICAL - YAML Indentation Rules:
-- For actions with additional parameters (aiScroll, aiInput, aiKeyboardPress), the parameters must be SIBLING keys, NOT nested children
-- Parameters like direction, scrollType, locate must align with the action key, not indented further
-- CORRECT indentation example:
-      - aiScroll:
-        direction: down
-        scrollType: singleAction
-- WRONG indentation (DO NOT do this):
-      - aiScroll:
-          direction: down
-          scrollType: singleAction
-
 
 
 YAML type

package/src/env/parse-model-config.ts

@@ -136,6 +136,26 @@ const getModelDescription = (
   return '';
 };
 
+const normalizeOpenaiExtraConfig = (
+  config: unknown,
+): Record<string, unknown> | undefined => {
+  if (!config || typeof config !== 'object' || Array.isArray(config)) {
+    return undefined;
+  }
+
+  const { defaultHeaders, extra_headers, extraHeaders, ...rest } =
+    config as Record<string, unknown>;
+
+  // Priority: defaultHeaders > extra_headers > extraHeaders
+  const headers = defaultHeaders ?? extra_headers ?? extraHeaders;
+
+  if (headers !== undefined) {
+    return { ...rest, defaultHeaders: headers };
+  }
+
+  return rest;
+};
+
 /**
  * Parse OpenAI SDK config
  */
@@ -201,7 +221,7 @@ export const parseOpenaiSdkConfig = ({
     httpProxy,
     openaiBaseURL,
     openaiApiKey,
-    openaiExtraConfig,
+    openaiExtraConfig: normalizeOpenaiExtraConfig(openaiExtraConfig),
     extraBody,
     modelFamily,
     uiTarsModelVersion,

package/src/env/types.ts

@@ -293,6 +293,7 @@ export type TModelFamily =
   | 'qwen2.5-vl'
   | 'qwen3-vl'
   | 'qwen3.5'
+  | 'qwen3.6'
   | 'doubao-vision'
   | 'doubao-seed'
   | 'gemini'
@@ -311,6 +312,7 @@ export const MODEL_FAMILY_VALUES: TModelFamily[] = [
   'qwen2.5-vl',
   'qwen3-vl',
   'qwen3.5',
+  'qwen3.6',
   'vlm-ui-tars',
   'vlm-ui-tars-doubao',
   'vlm-ui-tars-doubao-1.5',

package/src/extractor/dom-util.ts

@@ -181,7 +181,8 @@ export function generateElementByPoint(
 
 /**
  * Generate a LocateResultElement from a rect.
- * This function calculates the center point from the rect and expands the rect by edgeSize.
+ * This function calculates the center point from the rect and preserves the
+ * original rect as the returned element boundary.
  *
  * Note: The rect uses inclusive coordinates where:
  * - A rect from [left=10, top=10] with [width=1, height=1] covers exactly 1 pixel
@@ -189,13 +190,13 @@ export function generateElementByPoint(
  *
  * @param sourceRect - The source rect to generate element from (typically contains integer values)
  * @param description - Description of the element
- * @param edgeSize - Size to expand around the center point (default: 8)
- * @returns A LocateResultElement with rect, center (always integers), and description
+ * @param edgeSize - Deprecated, retained for backward compatibility
+ * @returns A LocateResultElement with the original rect, center (always integers), and description
  */
 export function generateElementByRect(
   sourceRect: { left: number; top: number; width: number; height: number },
   description: string,
-  edgeSize = 8,
+  _edgeSize = 8,
 ): LocateResultElement {
   /**
    * Calculate center point from rect
@@ -222,5 +223,9 @@ export function generateElementByRect(
   const centerX = sourceRect.left + Math.floor((sourceRect.width - 1) / 2);
   const centerY = sourceRect.top + Math.floor((sourceRect.height - 1) / 2);
 
-  return generateElementByPoint([centerX, centerY], description, edgeSize);
+  return {
+    rect: sourceRect,
+    center: [centerX, centerY],
+    description: description || '',
+  };
 }

package/src/img/canvas-fallback.ts

@@ -99,7 +99,7 @@ export class CanvasImage {
    */
   static async new_from_byteslice(bytes: Uint8Array): Promise<CanvasImage> {
     return new Promise((resolve, reject) => {
-      const blob = new Blob([bytes], { type: 'image/png' });
+      const blob = new Blob([bytes.buffer as ArrayBuffer], { type: 'image/png' });
       const url = URL.createObjectURL(blob);
       const img = new Image();
 

package/src/index.ts

@@ -1,8 +1 @@
-export {
-  createCopyStaticPlugin,
-  createPlaygroundCopyPlugin,
-} from './build/copy-static';
-
-export { commonIgnoreWarnings } from './build/rspack-config';
-
 export default {};

package/src/logger.ts

@@ -94,14 +94,3 @@ export function enableDebug(topic: string): void {
   }
   debug.enable(`${topicPrefix}:${topic}`);
 }
-
-// Cleanup function to close all log streams
-export function cleanupLogStreams(): void {
-  if (!ifInNode) return;
-
-  for (const stream of logStreams.values()) {
-    stream.end();
-  }
-  logStreams.clear();
-  debugInstances.clear();
-}

package/src/mcp/base-server.ts

@@ -1,6 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import type { ParseArgsConfig } from 'node:util';
-import { setIsMcp } from '../utils';
+import { setIsMcp } from '@aiscene/shared/utils';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';

package/src/mcp/base-tools.ts

@@ -1,5 +1,5 @@
-import { parseBase64 } from '../img';
-import { getDebug } from '../logger';
+import { parseBase64 } from '@aiscene/shared/img';
+import { getDebug } from '@aiscene/shared/logger';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import {
   generateCommonTools,