@aiscene/shared 1.6.0 → 1.7.2

package/src/mcp/tool-generator.ts

@@ -1,6 +1,11 @@
-import { parseBase64 } from '../img';
+import { parseBase64 } from '@aiscene/shared/img';
 import { z } from 'zod';
-import { getZodDescription, getZodTypeName } from '../zod-schema-utils';
+import {
+  getZodDescription,
+  getZodTypeName,
+  isMidsceneLocatorField,
+  unwrapZodField,
+} from '../zod-schema-utils';
 import type {
   ActionSpaceItem,
   BaseAgent,
@@ -26,23 +31,18 @@ function describeActionForMCP(action: ActionSpaceItem): string {
     return `${action.name} action, ${actionDesc}`;
   }
 
-  const schema = action.paramSchema as {
-    _def?: { typeName?: string };
-    shape?: Record<string, unknown>;
-  };
-  const isZodObjectType = schema._def?.typeName === 'ZodObject';
-
-  if (!isZodObjectType || !schema.shape) {
+  const shape = getZodObjectShape(action.paramSchema);
+  if (!shape) {
     // Simple type schema
-    const typeName = getZodTypeName(schema);
-    const description = getZodDescription(schema as z.ZodTypeAny);
+    const typeName = getZodTypeName(action.paramSchema);
+    const description = getZodDescription(action.paramSchema as z.ZodTypeAny);
     const paramDesc = description ? `${typeName} - ${description}` : typeName;
     return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
   }
 
   // Object schema with multiple fields
   const paramDescriptions: string[] = [];
-  for (const [key, field] of Object.entries(schema.shape)) {
+  for (const [key, field] of Object.entries(shape)) {
     if (field && typeof field === 'object') {
       const isFieldOptional =
         typeof (field as { isOptional?: () => boolean }).isOptional ===
@@ -96,25 +96,42 @@ function unwrapOptional(value: z.ZodTypeAny): {
   return { innerValue: value, isOptional: false };
 }
 
-/**
- * Check if a Zod object schema contains a 'prompt' field (locate field pattern)
- */
-function isLocateField(value: z.ZodTypeAny): boolean {
-  if (!isZodObject(value)) {
-    return false;
+function getZodObjectShape(
+  value: z.ZodTypeAny | undefined,
+): Record<string, z.ZodTypeAny> | undefined {
+  if (!value) {
+    return undefined;
   }
-  return 'prompt' in value.shape;
+
+  const actualValue = unwrapZodField(value) as {
+    _def?: { typeName?: string; shape?: () => Record<string, z.ZodTypeAny> };
+    shape?: Record<string, z.ZodTypeAny>;
+  };
+
+  if (actualValue._def?.typeName !== 'ZodObject') {
+    return undefined;
+  }
+
+  if (typeof actualValue._def.shape === 'function') {
+    return actualValue._def.shape();
+  }
+
+  return actualValue.shape;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
 
 /**
  * Transform a locate field schema to make its 'prompt' field optional
  */
 function makePromptOptional(
-  value: z.ZodObject<z.ZodRawShape>,
+  shape: Record<string, z.ZodTypeAny>,
   wrapInOptional: boolean,
 ): z.ZodTypeAny {
-  const newShape = { ...value.shape };
-  newShape.prompt = value.shape.prompt.optional();
+  const newShape = { ...shape };
+  newShape.prompt = shape.prompt.optional();
 
   let newSchema: z.ZodTypeAny = z.object(newShape).passthrough();
   if (wrapInOptional) {
@@ -131,9 +148,10 @@ function transformSchemaField(
   value: z.ZodTypeAny,
 ): [string, z.ZodTypeAny] {
   const { innerValue, isOptional } = unwrapOptional(value);
+  const shape = getZodObjectShape(innerValue);
 
-  if (isZodObject(innerValue) && isLocateField(innerValue)) {
-    return [key, makePromptOptional(innerValue, isOptional)];
+  if (shape && isMidsceneLocatorField(innerValue)) {
+    return [key, makePromptOptional(shape, isOptional)];
   }
   return [key, value];
 }
@@ -148,18 +166,117 @@ function extractActionSchema(
     return {};
   }
 
-  const schema = paramSchema as z.ZodTypeAny;
-  if (!isZodObject(schema)) {
-    return schema as unknown as Record<string, z.ZodTypeAny>;
+  const shape = getZodObjectShape(paramSchema);
+  if (!shape) {
+    return paramSchema as unknown as Record<string, z.ZodTypeAny>;
   }
 
   return Object.fromEntries(
-    Object.entries(schema.shape).map(([key, value]) =>
+    Object.entries(shape).map(([key, value]) =>
       transformSchemaField(key, value as z.ZodTypeAny),
     ),
   );
 }
 
+function getPromptText(prompt: unknown): string | undefined {
+  if (typeof prompt === 'string') {
+    return prompt;
+  }
+
+  if (isRecord(prompt) && typeof prompt.prompt === 'string') {
+    return prompt.prompt;
+  }
+
+  return undefined;
+}
+
+function moveLocateExtrasIntoPrompt(
+  value: Record<string, unknown>,
+  locateFieldKeys: Set<string>,
+): Record<string, unknown> {
+  const promptText = getPromptText(value.prompt);
+  if (!promptText) {
+    return value;
+  }
+
+  const normalizedPrompt: Record<string, unknown> = isRecord(value.prompt)
+    ? { ...value.prompt }
+    : { prompt: promptText };
+  const normalizedLocate: Record<string, unknown> = {};
+  let movedExtraField = false;
+
+  for (const [key, fieldValue] of Object.entries(value)) {
+    if (key === 'prompt') {
+      continue;
+    }
+
+    if (locateFieldKeys.has(key)) {
+      normalizedLocate[key] = fieldValue;
+      continue;
+    }
+
+    movedExtraField = true;
+    if (!(key in normalizedPrompt)) {
+      normalizedPrompt[key] = fieldValue;
+    }
+  }
+
+  if (!movedExtraField) {
+    return value;
+  }
+
+  return { ...normalizedLocate, prompt: normalizedPrompt };
+}
+
+function normalizeLocateLikeArg(
+  value: unknown,
+  fieldSchema: z.ZodTypeAny,
+): unknown {
+  if (typeof value === 'string') {
+    return { prompt: value };
+  }
+
+  if (!isRecord(value)) {
+    return value;
+  }
+
+  const shape = getZodObjectShape(fieldSchema);
+  if (!shape) {
+    return value;
+  }
+
+  return moveLocateExtrasIntoPrompt(value, new Set(Object.keys(shape)));
+}
+
+function normalizeActionArgs(
+  args: Record<string, unknown>,
+  paramSchema?: z.ZodTypeAny,
+): Record<string, unknown> {
+  if (!paramSchema) {
+    return args;
+  }
+
+  const shape = getZodObjectShape(paramSchema);
+  if (!shape) {
+    return args;
+  }
+
+  return Object.fromEntries(
+    Object.entries(args).map(([key, value]) => {
+      const fieldSchema = shape[key] as z.ZodTypeAny | undefined;
+      if (!fieldSchema) {
+        return [key, value];
+      }
+
+      if (isMidsceneLocatorField(fieldSchema)) {
+        return [key, normalizeLocateLikeArg(value, fieldSchema)];
+      }
+
+      return [key, value];
+    }),
+  );
+}
+
 /**
  * Serialize args to human-readable description for AI action
  */
@@ -194,10 +311,9 @@ function buildActionInstruction(
   actionName: string,
   args: Record<string, unknown>,
 ): string {
-  const locatePrompt =
-    args.locate && typeof args.locate === 'object'
-      ? (args.locate as { prompt?: string }).prompt
-      : undefined;
+  const locatePrompt = isRecord(args.locate)
+    ? getPromptText(args.locate.prompt)
+    : undefined;
 
   switch (actionName) {
     case 'Tap':
@@ -227,39 +343,71 @@ function buildActionInstruction(
   }
 }
 
+async function executeAction(
+  agent: BaseAgent,
+  actionName: string,
+  args: Record<string, unknown>,
+): Promise<unknown> {
+  if (agent.callActionInActionSpace) {
+    return agent.callActionInActionSpace(actionName, args);
+  }
+
+  if (agent.aiAction) {
+    const instruction = buildActionInstruction(actionName, args);
+    return agent.aiAction(instruction);
+  }
+
+  throw new Error(`Action "${actionName}" is not supported by this agent`);
+}
+
 /**
  * Capture screenshot and return as tool result
  */
 async function captureScreenshotResult(
   agent: BaseAgent,
   actionName: string,
+  actionResult?: unknown,
 ): Promise<ToolResult> {
+  const content: ToolResult['content'] = [
+    { type: 'text', text: `Action "${actionName}" completed.` },
+  ];
+
+  if (actionResult !== undefined) {
+    content.push({
+      type: 'text',
+      text: `Result: ${serializeActionResult(actionResult)}`,
+    });
+  }
+
   try {
     const screenshot = await agent.page?.screenshotBase64();
     if (!screenshot) {
-      return {
-        content: [{ type: 'text', text: `Action "${actionName}" completed.` }],
-      };
+      return { content };
     }
 
     const { mimeType, body } = parseBase64(screenshot);
-    return {
-      content: [
-        { type: 'text', text: `Action "${actionName}" completed.` },
-        { type: 'image', data: body, mimeType },
-      ],
-    };
+    content.push({ type: 'image', data: body, mimeType });
+    return { content };
   } catch (error: unknown) {
     const errorMessage = getErrorMessage(error);
     console.error('Error capturing screenshot:', errorMessage);
-    return {
-      content: [
-        {
-          type: 'text',
-          text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`,
-        },
-      ],
+    content[0] = {
+      type: 'text',
+      text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`,
     };
+    return { content };
+  }
+}
+
+function serializeActionResult(actionResult: unknown): string {
+  if (typeof actionResult === 'string') {
+    return actionResult;
+  }
+
+  try {
+    return JSON.stringify(actionResult);
+  } catch {
+    return String(actionResult);
   }
 }
 
@@ -323,28 +471,31 @@ export function generateToolsFromActionSpace(
       handler: async (args: Record<string, unknown>) => {
         try {
           const agent = await getAgent();
+          const normalizedArgs = normalizeActionArgs(args, action.paramSchema);
+          let actionResult: unknown;
 
-          if (agent.aiAction) {
-            const instruction = buildActionInstruction(action.name, args);
-            try {
-              await agent.aiAction(instruction);
-            } catch (error: unknown) {
-              const errorMessage = getErrorMessage(error);
-              console.error(
-                `Error executing action "${action.name}":`,
-                errorMessage,
-              );
-              // Return screenshot + warning instead of hard error,
-              // so the AI agent can see current state and decide to retry or adjust strategy
-              return await captureFailureResult(
-                agent,
-                action.name,
-                errorMessage,
-              );
-            }
+          try {
+            actionResult = await executeAction(
+              agent,
+              action.name,
+              normalizedArgs,
+            );
+          } catch (error: unknown) {
+            const errorMessage = getErrorMessage(error);
+            console.error(
+              `Error executing action "${action.name}":`,
+              errorMessage,
+            );
+            // Return screenshot + warning instead of hard error,
+            // so the AI agent can see current state and decide to retry or adjust strategy
+            return await captureFailureResult(agent, action.name, errorMessage);
           }
 
-          return await captureScreenshotResult(agent, action.name);
+          return await captureScreenshotResult(
+            agent,
+            action.name,
+            actionResult,
+          );
         } catch (error: unknown) {
           // Connection/agent errors are still hard errors
           const errorMessage = getErrorMessage(error);

package/src/mcp/types.ts

@@ -84,6 +84,10 @@ export interface BaseAgent {
   page?: {
     screenshotBase64(): Promise<string>;
   };
+  callActionInActionSpace?: (
+    actionName: string,
+    params?: unknown,
+  ) => Promise<unknown>;
   aiAction?: (
     description: string,
     params?: Record<string, unknown>,

package/src/utils.ts

@@ -64,23 +64,6 @@ export function assert(condition: any, message?: string): asserts condition {
   }
 }
 
-type GlobalScope = typeof window | typeof globalThis | typeof self | undefined;
-
-export function getGlobalScope(): GlobalScope {
-  if (typeof window !== 'undefined') {
-    return window;
-  }
-
-  if (typeof globalThis !== 'undefined') {
-    return globalThis;
-  }
-
-  if (typeof self !== 'undefined') {
-    return self;
-  }
-  return undefined;
-}
-
 let isMcp = false;
 
 export function setIsMcp(value: boolean) {

package/dist/es/build/copy-static.mjs

@@ -1,31 +0,0 @@
-import node_fs from "node:fs";
-import node_path from "node:path";
-const createCopyStaticPlugin = (options)=>({
-        name: options.pluginName || 'copy-static',
-        setup (api) {
-            api.onAfterBuild(async ()=>{
-                const { srcDir, destDir, faviconPath } = options;
-                const stat = await node_fs.promises.lstat(destDir).catch(()=>null);
-                if (stat?.isSymbolicLink()) await node_fs.promises.unlink(destDir);
-                await node_fs.promises.mkdir(destDir, {
-                    recursive: true
-                });
-                await node_fs.promises.cp(srcDir, destDir, {
-                    recursive: true
-                });
-                console.log(`Copied build artifacts from ${srcDir} to ${destDir}`);
-                if (faviconPath) {
-                    const faviconDest = node_path.join(destDir, 'favicon.ico');
-                    await node_fs.promises.copyFile(faviconPath, faviconDest);
-                    console.log(`Copied favicon from ${faviconPath} to ${faviconDest}`);
-                }
-            });
-        }
-    });
-const createPlaygroundCopyPlugin = (srcDir, destDir, pluginName, faviconSrc)=>createCopyStaticPlugin({
-        srcDir,
-        destDir,
-        faviconPath: faviconSrc,
-        pluginName
-    });
-export { createCopyStaticPlugin, createPlaygroundCopyPlugin };

package/dist/es/build/rspack-config.mjs

@@ -1,4 +0,0 @@
-const commonIgnoreWarnings = [
-    /Critical dependency: the request of a dependency is an expression/
-];
-export { commonIgnoreWarnings };

package/dist/lib/build/copy-static.js

@@ -1,79 +0,0 @@
-"use strict";
-var __webpack_require__ = {};
-(()=>{
-    __webpack_require__.n = (module)=>{
-        var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
-        __webpack_require__.d(getter, {
-            a: getter
-        });
-        return getter;
-    };
-})();
-(()=>{
-    __webpack_require__.d = (exports1, definition)=>{
-        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
-            enumerable: true,
-            get: definition[key]
-        });
-    };
-})();
-(()=>{
-    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
-})();
-(()=>{
-    __webpack_require__.r = (exports1)=>{
-        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
-            value: 'Module'
-        });
-        Object.defineProperty(exports1, '__esModule', {
-            value: true
-        });
-    };
-})();
-var __webpack_exports__ = {};
-__webpack_require__.r(__webpack_exports__);
-__webpack_require__.d(__webpack_exports__, {
-    createCopyStaticPlugin: ()=>createCopyStaticPlugin,
-    createPlaygroundCopyPlugin: ()=>createPlaygroundCopyPlugin
-});
-const external_node_fs_namespaceObject = require("node:fs");
-var external_node_fs_default = /*#__PURE__*/ __webpack_require__.n(external_node_fs_namespaceObject);
-const external_node_path_namespaceObject = require("node:path");
-var external_node_path_default = /*#__PURE__*/ __webpack_require__.n(external_node_path_namespaceObject);
-const createCopyStaticPlugin = (options)=>({
-        name: options.pluginName || 'copy-static',
-        setup (api) {
-            api.onAfterBuild(async ()=>{
-                const { srcDir, destDir, faviconPath } = options;
-                const stat = await external_node_fs_default().promises.lstat(destDir).catch(()=>null);
-                if (stat?.isSymbolicLink()) await external_node_fs_default().promises.unlink(destDir);
-                await external_node_fs_default().promises.mkdir(destDir, {
-                    recursive: true
-                });
-                await external_node_fs_default().promises.cp(srcDir, destDir, {
-                    recursive: true
-                });
-                console.log(`Copied build artifacts from ${srcDir} to ${destDir}`);
-                if (faviconPath) {
-                    const faviconDest = external_node_path_default().join(destDir, 'favicon.ico');
-                    await external_node_fs_default().promises.copyFile(faviconPath, faviconDest);
-                    console.log(`Copied favicon from ${faviconPath} to ${faviconDest}`);
-                }
-            });
-        }
-    });
-const createPlaygroundCopyPlugin = (srcDir, destDir, pluginName, faviconSrc)=>createCopyStaticPlugin({
-        srcDir,
-        destDir,
-        faviconPath: faviconSrc,
-        pluginName
-    });
-exports.createCopyStaticPlugin = __webpack_exports__.createCopyStaticPlugin;
-exports.createPlaygroundCopyPlugin = __webpack_exports__.createPlaygroundCopyPlugin;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
-    "createCopyStaticPlugin",
-    "createPlaygroundCopyPlugin"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
-Object.defineProperty(exports, '__esModule', {
-    value: true
-});

package/dist/lib/build/rspack-config.js

@@ -1,38 +0,0 @@
-"use strict";
-var __webpack_require__ = {};
-(()=>{
-    __webpack_require__.d = (exports1, definition)=>{
-        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
-            enumerable: true,
-            get: definition[key]
-        });
-    };
-})();
-(()=>{
-    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
-})();
-(()=>{
-    __webpack_require__.r = (exports1)=>{
-        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
-            value: 'Module'
-        });
-        Object.defineProperty(exports1, '__esModule', {
-            value: true
-        });
-    };
-})();
-var __webpack_exports__ = {};
-__webpack_require__.r(__webpack_exports__);
-__webpack_require__.d(__webpack_exports__, {
-    commonIgnoreWarnings: ()=>commonIgnoreWarnings
-});
-const commonIgnoreWarnings = [
-    /Critical dependency: the request of a dependency is an expression/
-];
-exports.commonIgnoreWarnings = __webpack_exports__.commonIgnoreWarnings;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
-    "commonIgnoreWarnings"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
-Object.defineProperty(exports, '__esModule', {
-    value: true
-});

package/dist/types/build/copy-static.d.ts

@@ -1,31 +0,0 @@
-export interface CopyStaticOptions {
-    /** Source directory to copy from */
-    srcDir: string;
-    /** Destination directory to copy to */
-    destDir: string;
-    /** Optional favicon source path (relative to directory containing srcDir) */
-    faviconPath?: string;
-    /** Name for the rsbuild plugin */
-    pluginName?: string;
-}
-/**
- * Creates an rsbuild plugin that copies static files after build
- * @param options Configuration options for copying static files
- * @returns Rsbuild plugin object
- */
-export declare const createCopyStaticPlugin: (options: CopyStaticOptions) => {
-    name: string;
-    setup(api: any): void;
-};
-/**
- * Helper function to create a copy static plugin for playground builds
- * @param srcDir Source directory (usually dist directory)
- * @param destDir Destination directory
- * @param pluginName Optional plugin name
- * @param faviconSrc Optional favicon source path
- * @returns Rsbuild plugin
- */
-export declare const createPlaygroundCopyPlugin: (srcDir: string, destDir: string, pluginName?: string, faviconSrc?: string) => {
-    name: string;
-    setup(api: any): void;
-};

package/dist/types/build/rspack-config.d.ts

@@ -1,8 +0,0 @@
-/**
- * Common Rspack configuration helpers for rsbuild projects
- */
-/**
- * Common warning patterns to ignore in Rspack builds.
- * These warnings are typically from optional dependencies or known non-critical issues.
- */
-export declare const commonIgnoreWarnings: RegExp[];

package/src/build/copy-static.ts

@@ -1,68 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-
-export interface CopyStaticOptions {
-  /** Source directory to copy from */
-  srcDir: string;
-  /** Destination directory to copy to */
-  destDir: string;
-  /** Optional favicon source path (relative to directory containing srcDir) */
-  faviconPath?: string;
-  /** Name for the rsbuild plugin */
-  pluginName?: string;
-}
-
-/**
- * Creates an rsbuild plugin that copies static files after build
- * @param options Configuration options for copying static files
- * @returns Rsbuild plugin object
- */
-export const createCopyStaticPlugin = (options: CopyStaticOptions) => ({
-  name: options.pluginName || 'copy-static',
-  setup(api: any) {
-    api.onAfterBuild(async () => {
-      const { srcDir, destDir, faviconPath } = options;
-
-      // Remove symlink left by dev mode before copying
-      const stat = await fs.promises.lstat(destDir).catch(() => null);
-      if (stat?.isSymbolicLink()) {
-        await fs.promises.unlink(destDir);
-      }
-
-      await fs.promises.mkdir(destDir, { recursive: true });
-
-      // Copy directory contents recursively
-      await fs.promises.cp(srcDir, destDir, { recursive: true });
-      console.log(`Copied build artifacts from ${srcDir} to ${destDir}`);
-
-      // Copy favicon if specified
-      if (faviconPath) {
-        const faviconDest = path.join(destDir, 'favicon.ico');
-        await fs.promises.copyFile(faviconPath, faviconDest);
-        console.log(`Copied favicon from ${faviconPath} to ${faviconDest}`);
-      }
-    });
-  },
-});
-
-/**
- * Helper function to create a copy static plugin for playground builds
- * @param srcDir Source directory (usually dist directory)
- * @param destDir Destination directory
- * @param pluginName Optional plugin name
- * @param faviconSrc Optional favicon source path
- * @returns Rsbuild plugin
- */
-export const createPlaygroundCopyPlugin = (
-  srcDir: string,
-  destDir: string,
-  pluginName?: string,
-  faviconSrc?: string,
-) => {
-  return createCopyStaticPlugin({
-    srcDir,
-    destDir,
-    faviconPath: faviconSrc,
-    pluginName,
-  });
-};