@aiscene/shared 8.0.2 → 8.0.4

package/src/agent-tools/base-tools.ts

@@ -0,0 +1,399 @@
+import { parseBase64 } from '@aiscene/shared/img';
+import { getDebug } from '@aiscene/shared/logger';
+import type { z } from 'zod';
+import { createRecordCliCommand } from '../cli/record-command';
+import { camelToKebab, getKeyAliases } from '../key-alias-utils';
+import {
+  type CliReportSession,
+  generateCliReportSession,
+  readCliReportSession,
+  writeCliReportSession,
+} from './cli-report-session';
+import {
+  createNamespacedInitArgSchema,
+  extractNamespacedArgs,
+  sanitizeNamespacedArgs,
+} from './init-arg-utils';
+import { type ToolDefaults, mergeToolDefaults } from './tool-defaults';
+import {
+  generateCommonTools,
+  generateToolsFromActionSpace,
+} from './tool-generator';
+import type {
+  ActionSpaceItem,
+  BaseAgent,
+  BaseDevice,
+  IMidsceneTools,
+  ToolCliMetadata,
+  ToolDefinition,
+  ToolSchema,
+} from './types';
+
+const debug = getDebug('agent-tools:base-tools');
+
+/**
+ * Declarative description of a platform's agent init args.
+ * Collapses the `extractAgentInitParam` / `sanitizeToolArgs` /
+ * `getAgentInitArgSchema` trio into a single data declaration.
+ */
+export interface InitArgSpec<TInitParam> {
+  /** Arg namespace, e.g. `android`, `ios`. */
+  namespace: string;
+  /** Zod shape describing the init args. Field names drive the tool schema. */
+  shape: Record<string, z.ZodTypeAny>;
+  /**
+   * Optional CLI presentation hints. These affect `--help` output for
+   * single-platform CLIs but do not alter YAML protocol keys.
+   */
+  cli?: {
+    /** Prefer bare `--device-id`-style options in platform CLI help output. */
+    preferBareKeys?: boolean;
+    /** Override the displayed option name for specific init arg fields. */
+    preferredNames?: Record<string, string>;
+  };
+  /**
+   * Adapt extracted namespaced args into the concrete `TInitParam` passed to
+   * `ensureAgent`. Defaults to returning the raw extracted record.
+   */
+  adapt?: (
+    extracted: Record<string, unknown> | undefined,
+  ) => TInitParam | undefined;
+}
+
+/**
+ * Base class for platform-specific Midscene tools.
+ * @typeParam TAgent - Platform-specific agent type.
+ * @typeParam TInitParam - Platform-specific init parameter consumed by
+ *   `ensureAgent`. Defaults to `undefined` for platforms that take no args.
+ */
+export abstract class BaseMidsceneTools<
+  TAgent extends BaseAgent = BaseAgent,
+  TInitParam = unknown,
+> implements IMidsceneTools
+{
+  protected agent?: TAgent;
+  protected toolDefinitions: ToolDefinition[] = [];
+
+  /**
+   * Default options injected into every generated tool call (e.g. forced deep
+   * locate / deep think). Set from startup/CLI behavior flags before
+   * `initTools()` so they are baked into the generated tool handlers.
+   * See https://github.com/web-infra-dev/midscene/issues/2446.
+   */
+  protected toolDefaults: ToolDefaults = {};
+
+  /**
+   * Declarative init-arg spec. Subclasses that accept CLI init args should
+   * set this once and get `extractAgentInitParam` / `sanitizeToolArgs` /
+   * `getAgentInitArgSchema` auto-implemented.
+   *
+   * Declared with `declare` so that TS doesn't emit an `Object.defineProperty`
+   * for this field on the base constructor, which would otherwise overwrite
+   * a subclass field initializer under `useDefineForClassFields`.
+   */
+  protected declare readonly initArgSpec?: InitArgSpec<TInitParam>;
+
+  /**
+   * Ensure agent is initialized and ready for use.
+   * Must be implemented by subclasses to create platform-specific agent.
+   * @param initParam Optional initialization parameter (platform-specific, e.g., URL, device ID)
+   * @returns Promise resolving to initialized agent instance
+   * @throws Error if agent initialization fails
+   */
+  protected abstract ensureAgent(initParam?: TInitParam): Promise<TAgent>;
+
+  private getInitArgKeys(): readonly string[] {
+    return this.initArgSpec ? Object.keys(this.initArgSpec.shape) : [];
+  }
+
+  /**
+   * Extract a platform-specific agent init parameter from CLI tool args.
+   */
+  protected extractAgentInitParam(
+    args: Record<string, unknown>,
+  ): TInitParam | undefined {
+    if (!this.initArgSpec) {
+      return undefined;
+    }
+    const extracted = extractNamespacedArgs(
+      args,
+      this.initArgSpec.namespace,
+      this.getInitArgKeys(),
+    );
+    if (this.initArgSpec.adapt) {
+      return this.initArgSpec.adapt(extracted);
+    }
+    return extracted as TInitParam | undefined;
+  }
+
+  /**
+   * Remove platform-specific init args before dispatching a tool payload to the action itself.
+   */
+  protected sanitizeToolArgs(
+    args: Record<string, unknown>,
+  ): Record<string, unknown> {
+    if (!this.initArgSpec) {
+      return args;
+    }
+    return sanitizeNamespacedArgs(
+      args,
+      this.initArgSpec.namespace,
+      this.getInitArgKeys(),
+    );
+  }
+
+  /**
+   * Expose platform-specific init args on action/common tool schemas.
+   */
+  protected getAgentInitArgSchema(): ToolSchema {
+    if (!this.initArgSpec) {
+      return {};
+    }
+    return createNamespacedInitArgSchema(
+      this.initArgSpec.namespace,
+      this.initArgSpec.shape,
+    );
+  }
+
+  /**
+   * Expose CLI-only metadata for platform init args so single-platform help can
+   * show ergonomic bare flags while the underlying schema stays namespaced.
+   * When `preferBareKeys` is enabled, single-platform CLIs only accept the
+   * bare spellings; namespaced dotted spellings remain available through the
+   * YAML schema instead of the platform CLI surface.
+   */
+  protected getAgentInitArgCliMetadata(): ToolCliMetadata | undefined {
+    if (!this.initArgSpec?.cli) {
+      return undefined;
+    }
+
+    const options = Object.fromEntries(
+      this.getInitArgKeys().map((key) => {
+        const canonicalKey = `${this.initArgSpec!.namespace}.${key}`;
+        const preferredName =
+          this.initArgSpec!.cli?.preferredNames?.[key] ??
+          (this.initArgSpec!.cli?.preferBareKeys
+            ? camelToKebab(key)
+            : canonicalKey);
+
+        const acceptedNames = new Set<string>([
+          preferredName,
+          ...(this.initArgSpec!.cli?.preferBareKeys
+            ? getKeyAliases(key)
+            : getKeyAliases(canonicalKey)),
+        ]);
+        acceptedNames.delete(preferredName);
+
+        return [
+          canonicalKey,
+          {
+            preferredName,
+            aliases: [...acceptedNames],
+          },
+        ];
+      }),
+    );
+
+    return { options };
+  }
+
+  /**
+   * Optional: prepare platform-specific tools (e.g., device connection)
+   */
+  protected preparePlatformTools(): ToolDefinition[] {
+    return [];
+  }
+
+  protected getCliReportSessionName(): string | undefined {
+    return undefined;
+  }
+
+  protected createNewCliReportSession(
+    targetIdentity?: string,
+  ): CliReportSession | undefined {
+    const sessionName = this.getCliReportSessionName();
+    if (!sessionName) {
+      return undefined;
+    }
+    return generateCliReportSession(sessionName, targetIdentity);
+  }
+
+  protected commitCliReportSession(session?: CliReportSession): void {
+    if (session) {
+      writeCliReportSession(session);
+    }
+  }
+
+  protected readCliReportFileName(): string | undefined {
+    const sessionName = this.getCliReportSessionName();
+    if (!sessionName) {
+      return undefined;
+    }
+    return readCliReportSession(sessionName)?.reportFileName;
+  }
+
+  protected readCliReportAgentOptions():
+    | {
+        reportFileName: string;
+        reportAttributes: Record<string, string>;
+      }
+    | undefined {
+    const reportFileName = this.readCliReportFileName();
+    if (!reportFileName) {
+      return undefined;
+    }
+    return {
+      reportFileName,
+      reportAttributes: {
+        'data-group-id': reportFileName,
+      },
+    };
+  }
+
+  /**
+   * Must be implemented by subclasses to create a temporary device instance
+   * This allows getting real actionSpace without connecting to device
+   */
+  protected abstract createTemporaryDevice(): BaseDevice;
+
+  /**
+   * Initialize all tools by querying actionSpace
+   * Uses two-layer fallback strategy:
+   * 1. Try to get actionSpace from connected agent (if available)
+   * 2. Create temporary device instance to read actionSpace (always succeeds)
+   */
+  public async initTools(): Promise<void> {
+    this.toolDefinitions = [];
+
+    // 1. Add platform-specific tools first (device connection, etc.)
+    // These don't require an agent and should always be available
+    const platformTools = this.preparePlatformTools();
+    this.toolDefinitions.push(...platformTools);
+
+    // 2. Get action space: use pre-set agent if available, otherwise temp device.
+    //    For CLI usage, agent is deferred to the first real command.
+    let actionSpace: ActionSpaceItem[];
+    if (this.agent) {
+      actionSpace = await this.agent.getActionSpace();
+      debug(
+        'Action space from agent:',
+        actionSpace.map((a) => a.name).join(', '),
+      );
+    } else {
+      const tempDevice = this.createTemporaryDevice();
+      actionSpace = tempDevice.actionSpace();
+      await tempDevice.destroy?.();
+      debug(
+        'Action space from temporary device:',
+        actionSpace.map((a) => a.name).join(', '),
+      );
+    }
+
+    // 3. Generate tools from action space (core innovation)
+    const actionTools = generateToolsFromActionSpace(
+      actionSpace,
+      (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
+      (args = {}) => this.sanitizeToolArgs(args),
+      this.getAgentInitArgSchema(),
+      this.getAgentInitArgCliMetadata(),
+      this.toolDefaults,
+    );
+
+    // 4. Add common tools (screenshot, act, assert)
+    const commonTools = generateCommonTools(
+      (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
+      this.getAgentInitArgSchema(),
+      this.getAgentInitArgCliMetadata(),
+      this.toolDefaults,
+    );
+    this.toolDefinitions.push(...actionTools, ...commonTools);
+
+    debug('Total tools prepared:', this.toolDefinitions.length);
+  }
+
+  /**
+   * Cleanup method - destroy agent and release resources
+   */
+  public async destroy(): Promise<void> {
+    await this.agent?.destroy?.();
+  }
+
+  /**
+   * Get tool definitions
+   */
+  public getToolDefinitions(): ToolDefinition[] {
+    return this.toolDefinitions;
+  }
+
+  /** Commands that exist only on the foreground CLI surface. */
+  public getCliToolDefinitions(): ToolDefinition[] {
+    return [
+      createRecordCliCommand(
+        (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
+        this.getAgentInitArgSchema(),
+        this.getAgentInitArgCliMetadata(),
+      ),
+    ];
+  }
+
+  /**
+   * Set agent for the tools manager
+   */
+  public setAgent(agent: TAgent): void {
+    this.agent = agent;
+  }
+
+  /**
+   * Set the default options injected into generated tool calls. Must be called
+   * before `initTools()` because the values are captured into the generated
+   * tool handlers. Merges with any previously set defaults.
+   */
+  public setToolDefaults(toolDefaults: ToolDefaults): void {
+    this.toolDefaults = mergeToolDefaults(this.toolDefaults, toolDefaults);
+  }
+
+  /**
+   * Helper: Convert base64 screenshot to image content array
+   */
+  protected buildScreenshotContent(screenshot: string) {
+    const { mimeType, body } = parseBase64(screenshot);
+    return [
+      {
+        type: 'image' as const,
+        data: body,
+        mimeType,
+      },
+    ];
+  }
+
+  /**
+   * Helper: Build a simple text result for tool responses
+   */
+  protected buildTextResult(text: string) {
+    return {
+      content: [{ type: 'text' as const, text }],
+    };
+  }
+
+  /**
+   * Create a disconnect handler for releasing platform resources
+   * @param platformName Human-readable platform name for the response message
+   * @returns Handler function that destroys the agent and returns appropriate response
+   */
+  protected createDisconnectHandler(platformName: string) {
+    return async () => {
+      if (!this.agent) {
+        return this.buildTextResult('No active connection to disconnect');
+      }
+
+      try {
+        await this.agent.destroy?.();
+      } catch (error) {
+        debug('Failed to destroy agent during disconnect:', error);
+      }
+      this.agent = undefined;
+
+      return this.buildTextResult(`Disconnected from ${platformName}`);
+    };
+  }
+}

package/src/agent-tools/chrome-path.ts

@@ -0,0 +1,74 @@
+import { existsSync } from 'node:fs';
+import {
+  MIDSCENE_CHROME_PATH,
+  MIDSCENE_MCP_CHROME_PATH,
+  globalConfigManager,
+} from '../env';
+import { getDebug } from '../logger';
+
+const warnChromePath = getDebug('agent-tools:chrome-path', { console: true });
+let hasWarnedLegacyChromePath = false;
+let cachedSystemChromePath: string | undefined;
+
+export function getSystemChromePath(): string | undefined {
+  if (cachedSystemChromePath !== undefined) {
+    return cachedSystemChromePath;
+  }
+
+  const platform = process.platform;
+
+  const chromePaths: Record<string, string[]> = {
+    darwin: [
+      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+      '/Applications/Chromium.app/Contents/MacOS/Chromium',
+    ],
+    win32: [
+      'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
+      'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
+      `C:\\Users\\${process.env.USERNAME ?? process.env.USER}\\AppData\\Local\\Google\\Chrome\\Application\\chrome.exe`,
+    ],
+    linux: [
+      // Prefer actual binaries over wrapper scripts.
+      // Wrappers in /usr/bin may strip --user-data-dir, causing
+      // "DevTools remote debugging requires a non-default data directory" errors.
+      '/opt/google/chrome/chrome',
+      '/opt/google/chrome/google-chrome',
+      '/usr/bin/google-chrome-stable',
+      '/usr/bin/google-chrome',
+      '/usr/bin/chromium-browser',
+      '/usr/bin/chromium',
+      '/snap/bin/chromium',
+    ],
+  };
+
+  const paths = chromePaths[platform] ?? [];
+  const foundPath = paths.find((p) => existsSync(p));
+  if (foundPath) {
+    cachedSystemChromePath = foundPath;
+  }
+  return foundPath;
+}
+
+export function resolveChromePath(): string {
+  const primaryEnvPath =
+    globalConfigManager.getEnvConfigValue(MIDSCENE_CHROME_PATH);
+  const legacyEnvPath = globalConfigManager.getEnvConfigValue(
+    MIDSCENE_MCP_CHROME_PATH,
+  );
+  const envPath = primaryEnvPath || legacyEnvPath;
+  if (!primaryEnvPath && legacyEnvPath && !hasWarnedLegacyChromePath) {
+    warnChromePath(
+      'MIDSCENE_MCP_CHROME_PATH is deprecated. Use MIDSCENE_CHROME_PATH instead.',
+    );
+    hasWarnedLegacyChromePath = true;
+  }
+  if (envPath && envPath !== 'auto' && existsSync(envPath)) {
+    return envPath;
+  }
+  const systemPath = getSystemChromePath();
+  if (systemPath) return systemPath;
+
+  throw new Error(
+    'Chrome not found. Install Google Chrome or set MIDSCENE_CHROME_PATH environment variable.',
+  );
+}

package/src/agent-tools/cli-report-session.ts

@@ -0,0 +1,130 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { getMidsceneRunBaseDir, getMidsceneRunSubDir } from '../common';
+
+export interface CliReportSession {
+  version: 1;
+  sessionName: string;
+  targetIdentity?: string;
+  reportFileName: string;
+  reportPath: string;
+  createdAt: number;
+}
+
+const sessionDirName = 'cli-report-session';
+
+function sanitizeSessionName(sessionName: string): string {
+  return sessionName.replace(/[^a-zA-Z0-9._-]/g, '_') || 'default';
+}
+
+function sanitizeFileSegment(segment: string): string {
+  const sanitized = segment.replace(/[^a-zA-Z0-9._-]/g, '_') || 'unknown';
+  return sanitized.slice(0, 80);
+}
+
+function ensureHtmlFileName(reportFileName: string): string {
+  return reportFileName.endsWith('.html')
+    ? reportFileName
+    : `${reportFileName}.html`;
+}
+
+function formatDateForFileName(date: Date): string {
+  const pad = (value: number) => String(value).padStart(2, '0');
+  const day = [
+    date.getFullYear(),
+    pad(date.getMonth() + 1),
+    pad(date.getDate()),
+  ].join('-');
+  const time = [
+    pad(date.getHours()),
+    pad(date.getMinutes()),
+    pad(date.getSeconds()),
+  ].join('-');
+  return `${day}_${time}`;
+}
+
+function randomId(): string {
+  return Math.random().toString(36).slice(2, 10);
+}
+
+function getCliReportSessionDir(): string {
+  const dir = join(getMidsceneRunBaseDir(), sessionDirName);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  return dir;
+}
+
+function getCliReportSessionPath(sessionName: string): string {
+  return join(
+    getCliReportSessionDir(),
+    `${sanitizeSessionName(sessionName)}.json`,
+  );
+}
+
+export function generateCliReportSession(
+  sessionName: string,
+  targetIdentity?: string,
+): CliReportSession {
+  const identitySegment = targetIdentity
+    ? `-${sanitizeFileSegment(targetIdentity)}`
+    : '';
+  const reportFileName = `${sanitizeSessionName(sessionName)}${identitySegment}-${formatDateForFileName(new Date())}-${randomId()}`;
+  const reportPath = join(
+    getMidsceneRunSubDir('report'),
+    ensureHtmlFileName(reportFileName),
+  );
+  const session: CliReportSession = {
+    version: 1,
+    sessionName,
+    ...(targetIdentity ? { targetIdentity } : {}),
+    reportFileName,
+    reportPath,
+    createdAt: Date.now(),
+  };
+  return session;
+}
+
+export function writeCliReportSession(session: CliReportSession): void {
+  writeFileSync(
+    getCliReportSessionPath(session.sessionName),
+    JSON.stringify(session, null, 2),
+    'utf-8',
+  );
+}
+
+export function createCliReportSession(
+  sessionName: string,
+  targetIdentity?: string,
+): CliReportSession {
+  const session = generateCliReportSession(sessionName, targetIdentity);
+  writeCliReportSession(session);
+  return session;
+}
+
+export function readCliReportSession(
+  sessionName: string,
+): CliReportSession | undefined {
+  const sessionPath = getCliReportSessionPath(sessionName);
+  if (!existsSync(sessionPath)) {
+    return undefined;
+  }
+
+  try {
+    const raw = readFileSync(sessionPath, 'utf-8');
+    const parsed = JSON.parse(raw) as Partial<CliReportSession>;
+    if (
+      parsed.version !== 1 ||
+      parsed.sessionName !== sessionName ||
+      typeof parsed.reportFileName !== 'string' ||
+      !parsed.reportFileName.trim() ||
+      /[\\/]/.test(parsed.reportFileName)
+    ) {
+      return undefined;
+    }
+
+    return parsed as CliReportSession;
+  } catch {
+    return undefined;
+  }
+}