@aiscene/shared 8.0.2 → 8.0.4

package/src/agent-tools/types.ts

@@ -0,0 +1,250 @@
+import type { z } from 'zod';
+import type { ToolDefaults } from './tool-defaults';
+
+// Avoid circular dependency: don't import from @aiscene/core
+// Instead, use generic types that will be provided by implementation
+
+/**
+ * Default timeout constants for app loading verification
+ */
+export const defaultAppLoadingTimeoutMs = 10000;
+export const defaultAppLoadingCheckIntervalMs = 2000;
+
+/**
+ * Content item types for tool results.
+ */
+export type ToolResultContent =
+  | { type: 'text'; text: string }
+  | { type: 'image'; data: string; mimeType: string }
+  | { type: 'audio'; data: string; mimeType: string }
+  | {
+      type: 'resource';
+      resource:
+        | { text: string; uri: string; mimeType?: string }
+        | { uri: string; blob: string; mimeType?: string };
+    };
+
+/**
+ * Result type for tool execution.
+ */
+export interface ToolResult {
+  [x: string]: unknown;
+  content: ToolResultContent[];
+  isError?: boolean;
+  _meta?: Record<string, unknown>;
+}
+
+/**
+ * Tool handler function type
+ * Takes parsed arguments and returns a tool result
+ */
+export type ToolHandler<T = Record<string, unknown>> = (
+  args: T,
+) => Promise<ToolResult>;
+
+/**
+ * Tool schema type using Zod
+ */
+export type ToolSchema = Record<string, z.ZodTypeAny>;
+
+export interface ToolCliOption {
+  preferredName?: string;
+  aliases?: string[];
+  hidden?: boolean;
+}
+
+export interface ToolCliMetadata {
+  options?: Record<string, ToolCliOption>;
+  /** Schema keys populated from leading positional CLI arguments. */
+  positionals?: string[];
+}
+
+/**
+ * Tool definition for Midscene CLI and Skill surfaces.
+ */
+export interface ToolDefinition<T = Record<string, unknown>> {
+  name: string;
+  description: string;
+  schema: ToolSchema;
+  handler: ToolHandler<T>;
+  cli?: ToolCliMetadata;
+}
+
+export type Tool = ToolDefinition;
+
+/**
+ * Action space item definition
+ * Note: Intentionally no index signature to maintain compatibility with DeviceAction
+ */
+export interface ActionSpaceItem {
+  name: string;
+  description?: string;
+  args?: Record<string, unknown>;
+  paramSchema?: z.ZodTypeAny;
+}
+
+/**
+ * Structural shape compatible with @aiscene/core `TUserPrompt`.
+ * Declared locally to avoid a circular dep on `@aiscene/core` from `@aiscene/shared`.
+ *
+ * Consumed by the `assert` and `act` tools in `generateCommonTools`, both of
+ * which forward reference images to core (`aiAssert` / `aiAct`). `aiWaitFor`
+ * stays string-only at the CLI surface because the tools generator does not
+ * yet expose a multimodal entry point for it.
+ */
+export type UserPromptLike =
+  | string
+  | {
+      prompt: string;
+      images?: Array<{ name: string; url: string }>;
+      convertHttpImage2Base64?: boolean;
+    };
+
+export interface RecordToReportScreenshot {
+  /**
+   * PNG/JPEG data URI, or raw PNG base64 body.
+   */
+  base64: string;
+  description?: string;
+}
+
+export interface RecordToReportOptions {
+  content?: string;
+  /**
+   * @deprecated Use `screenshots: [{ base64 }]` instead.
+   */
+  screenshotBase64?: string;
+  screenshots?: RecordToReportScreenshot[];
+}
+
+/** Generic progress-bus envelope as seen by tool consumers (all untrusted). */
+export interface BaseAgentProgressEvent {
+  scope?: unknown;
+  phase?: unknown;
+  sequence?: unknown;
+  data?: unknown;
+}
+
+/** A single frame in a portable UI observation record. */
+export interface UIObservationFrame {
+  /** Path to the captured image. Serialized manifests use relative paths. */
+  path: string;
+  /** MIME type of the captured image file. */
+  mimeType: 'image/png' | 'image/jpeg';
+  /** Capture timestamp in milliseconds. */
+  capturedAt: number;
+}
+
+/**
+ * Serializable observation window that can be persisted and loaded later.
+ * Frames are ordered from earliest to latest; the final frame represents the
+ * UI state at the end of the observation window.
+ */
+export interface UIObservationRecord {
+  type: 'midscene_ui_observation';
+  version: 1;
+  /** Time when observation sampling started. */
+  startedAt: number;
+  /** Time when the final representative frame was captured. */
+  endedAt: number;
+  frames: UIObservationFrame[];
+  shotSize: {
+    width: number;
+    height: number;
+  };
+  shrunkShotToLogicalRatio: number;
+}
+
+/** Fixed observation window consumed by shared CLI surfaces. */
+export interface BaseUIObservation {
+  readonly frameCount: number;
+  readonly startedAt: number;
+  readonly endedAt: number;
+  aiAssert(
+    assertion: UserPromptLike,
+    message?: string,
+    options?: Record<string, unknown>,
+  ): Promise<unknown>;
+  /** Release any resources owned by this observation. */
+  dispose?(): Promise<void>;
+}
+
+/** Minimal UI observation lifecycle required by shared tool surfaces. */
+export interface BaseUIObserver {
+  readonly bufferedFrameCount: number;
+  /** Stop sampling and return the fixed observed window. */
+  stop(): Promise<BaseUIObservation>;
+  /** Release temporary backing files if observation does not complete. */
+  dispose?(): Promise<void>;
+}
+
+/** Options for {@link BaseAgent.startObserving}. */
+export interface BaseUIObserverOptions {
+  /** Sampling interval in milliseconds. Defaults to 1000; minimum 200. */
+  intervalMs?: number;
+  /** Maximum number of buffered frames. Defaults to 30; minimum 2. */
+  maxFrames?: number;
+  /** Auto-stop timeout in milliseconds. Defaults to 300000; 0 disables it. */
+  watchdogMs?: number;
+}
+
+/**
+ * Base agent interface
+ * Represents a platform-specific agent (Android, iOS, Web)
+ * Note: Return types use `unknown` for compatibility with platform-specific implementations
+ */
+export interface BaseAgent {
+  getActionSpace(): Promise<ActionSpaceItem[]>;
+  destroy?(): Promise<void>;
+  reportFile?: string | null;
+  page?: {
+    screenshotBase64(): Promise<string>;
+  };
+  addDumpUpdateListener?(
+    listener: (dump: string, executionDump?: unknown) => void,
+  ): () => void;
+  addProgressListener?(
+    listener: (event: BaseAgentProgressEvent) => void,
+  ): () => void;
+  recordToReport?(
+    title?: string,
+    opt?: RecordToReportOptions,
+  ): Promise<void>;
+  callActionInActionSpace?(
+    actionName: string,
+    params?: unknown,
+  ): Promise<unknown>;
+  aiAction?(
+    description: UserPromptLike,
+    params?: Record<string, unknown>,
+  ): Promise<unknown>;
+  aiWaitFor?(
+    assertion: string,
+    options: Record<string, unknown>,
+  ): Promise<unknown>;
+  aiAssert?(
+    assertion: UserPromptLike,
+    msg?: string,
+    options?: Record<string, unknown>,
+  ): Promise<unknown>;
+  /** Start a UI observation window and capture its baseline frame. */
+  startObserving?(options?: BaseUIObserverOptions): Promise<BaseUIObserver>;
+}
+
+/**
+ * Base device interface for temporary device instances
+ */
+export interface BaseDevice {
+  actionSpace(): ActionSpaceItem[];
+  destroy?(): Promise<void>;
+}
+
+/**
+ * Interface for platform-specific tools manager.
+ */
+export interface IMidsceneTools {
+  initTools(): Promise<void>;
+  destroy?(): Promise<void>;
+  getCliToolDefinitions?(): ToolDefinition[];
+  setToolDefaults?(toolDefaults: ToolDefaults): void;
+}

package/src/agent-tools/user-prompt.ts

@@ -0,0 +1,102 @@
+import { z } from 'zod';
+import type { UserPromptLike } from './types';
+
+type PromptReferenceImage = { name: string; url: string };
+
+function normalizeStringList(raw: unknown, fieldName: string): string[] {
+  if (raw === undefined || raw === null) return [];
+  if (typeof raw === 'string') {
+    const trimmed = raw.trim();
+    return trimmed ? [trimmed] : [];
+  }
+  if (Array.isArray(raw)) {
+    return raw.map((item, index) => {
+      if (typeof item !== 'string') {
+        throw new Error(`${fieldName}[${index}]: expected a string.`);
+      }
+      return item.trim();
+    });
+  }
+  throw new Error(
+    `${fieldName}: expected a string or string array, got ${typeof raw}.`,
+  );
+}
+
+function composeImages(input: {
+  image?: unknown;
+  imageName?: unknown;
+}): PromptReferenceImage[] {
+  const urls = normalizeStringList(input.image, 'image');
+  const names = normalizeStringList(input.imageName, 'imageName');
+
+  if (urls.length !== names.length) {
+    throw new Error(
+      `image/imageName: expected the same number of --image and --image-name values, got ${urls.length} image(s) and ${names.length} image name(s).`,
+    );
+  }
+
+  return urls.map((url, index) => ({ name: names[index], url }));
+}
+
+function coerceBoolean(value: unknown): boolean | undefined {
+  if (value === undefined || value === null) return undefined;
+  if (typeof value === 'boolean') return value;
+  if (typeof value === 'string') {
+    const trimmed = value.trim();
+    if (!trimmed) return undefined;
+    const v = trimmed.toLowerCase();
+    if (v === 'true' || v === '1') return true;
+    if (v === 'false' || v === '0') return false;
+    throw new Error(
+      `convertHttpImage2Base64: expected "true", "false", "1", or "0"; got ${JSON.stringify(value)}.`,
+    );
+  }
+  throw new Error(
+    `convertHttpImage2Base64: expected a boolean, got ${typeof value}.`,
+  );
+}
+
+export function composeUserPrompt(input: {
+  prompt: string;
+  image?: unknown;
+  imageName?: unknown;
+  convertHttpImage2Base64?: unknown;
+}): UserPromptLike {
+  const images = composeImages({
+    image: input.image,
+    imageName: input.imageName,
+  });
+  const convertFlag = coerceBoolean(input.convertHttpImage2Base64);
+
+  if (images.length === 0 && convertFlag === undefined) {
+    return input.prompt;
+  }
+
+  const payload: Exclude<UserPromptLike, string> = { prompt: input.prompt };
+  if (images.length > 0) {
+    payload.images = images;
+  }
+  if (convertFlag !== undefined) {
+    payload.convertHttpImage2Base64 = convertFlag;
+  }
+  return payload;
+}
+
+export const promptInputExtraSchema = {
+  image: z
+    .union([z.string(), z.array(z.string())])
+    .optional()
+    .describe('Reference image URL/path. Repeat --image for multiple images.'),
+  imageName: z
+    .union([z.string(), z.array(z.string())])
+    .optional()
+    .describe(
+      'Reference image name. Repeat --image-name; must align with --image order.',
+    ),
+  convertHttpImage2Base64: z
+    .union([z.boolean(), z.string()])
+    .optional()
+    .describe(
+      'If true, convert http(s) image URLs to base64 before sending to the model.',
+    ),
+};

package/src/cli/cli-args.ts

@@ -0,0 +1,173 @@
+import { z } from 'zod';
+import { getKeyAliases } from '../key-alias-utils';
+import type { ToolCliOption, ToolDefinition } from '../mcp/types';
+
+export function parseValue(raw: string): unknown {
+  if (raw.startsWith('{') || raw.startsWith('[')) {
+    try {
+      return JSON.parse(raw);
+    } catch {
+      // Not valid JSON, treat as string below
+    }
+  }
+
+  if (/^-?\d+(\.\d+)?$/.test(raw)) {
+    return Number(raw);
+  }
+
+  return raw;
+}
+
+function walkCliArgs(
+  args: string[],
+  setArgValue: (key: string, value: unknown) => void,
+): void {
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (!arg.startsWith('--')) continue;
+
+    const body = arg.slice(2);
+    const eqIdx = body.indexOf('=');
+
+    if (eqIdx >= 0) {
+      setArgValue(body.slice(0, eqIdx), parseValue(body.slice(eqIdx + 1)));
+    } else if (args[i + 1] && !args[i + 1].startsWith('--')) {
+      i++;
+      setArgValue(body, parseValue(args[i]));
+    } else {
+      setArgValue(body, true);
+    }
+  }
+}
+
+export function parseCliArgs(args: string[]): Record<string, unknown> {
+  const result: Record<string, unknown> = {};
+
+  walkCliArgs(args, (key, value) => {
+    result[key] = value;
+  });
+
+  return result;
+}
+
+function formatCliOptionName(name: string): string {
+  return `--${name}`;
+}
+
+export function getCliOptionDisplay(
+  key: string,
+  cliOption?: ToolCliOption,
+): { label: string; aliases: string[] } {
+  const label = formatCliOptionName(cliOption?.preferredName ?? key);
+  const aliases = [...new Set(cliOption?.aliases ?? [])]
+    .map((alias) => formatCliOptionName(alias))
+    .filter((alias) => alias !== label);
+
+  return { label, aliases };
+}
+
+function getAcceptedCliOptionNames(
+  key: string,
+  cliOption?: ToolCliOption,
+): string[] {
+  return [
+    ...new Set(
+      cliOption
+        ? [cliOption.preferredName ?? key, ...(cliOption.aliases ?? [])]
+        : [key, ...getKeyAliases(key)],
+    ),
+  ];
+}
+
+function toOptionalCliSchemaField(field: unknown): z.ZodTypeAny {
+  if (
+    typeof field === 'object' &&
+    field !== null &&
+    typeof (field as z.ZodTypeAny).optional === 'function'
+  ) {
+    return (field as z.ZodTypeAny).optional();
+  }
+
+  const description =
+    typeof field === 'object' &&
+    field !== null &&
+    'description' in field &&
+    typeof (field as { description?: unknown }).description === 'string'
+      ? (field as { description: string }).description
+      : undefined;
+  return description ? z.any().describe(description) : z.any();
+}
+
+function buildCliArgSchema(def: ToolDefinition): Record<string, z.ZodTypeAny> {
+  return Object.fromEntries(
+    Object.entries(def.schema).flatMap(([key, zodType]) =>
+      getAcceptedCliOptionNames(key, def.cli?.options?.[key]).map((cliKey) => [
+        cliKey,
+        toOptionalCliSchemaField(zodType),
+      ]),
+    ),
+  );
+}
+
+function buildDisallowedCliSpellings(def: ToolDefinition): Set<string> {
+  const disallowedSpellings = new Set<string>();
+
+  for (const [key] of Object.entries(def.schema)) {
+    const cliOption = def.cli?.options?.[key];
+    const acceptedNames = new Set(getAcceptedCliOptionNames(key, cliOption));
+    const knownSpellings = new Set<string>([
+      key,
+      ...getKeyAliases(key),
+      ...(cliOption?.preferredName
+        ? getKeyAliases(cliOption.preferredName)
+        : []),
+      ...(cliOption?.aliases ?? []),
+    ]);
+
+    for (const spelling of knownSpellings) {
+      if (!acceptedNames.has(spelling)) {
+        disallowedSpellings.add(spelling);
+      }
+    }
+  }
+
+  return disallowedSpellings;
+}
+
+export function formatCliValidationError(
+  scriptName: string,
+  commandName: string,
+  def: ToolDefinition,
+  rawArgs: Record<string, unknown>,
+): string | undefined {
+  if (Object.keys(def.schema).length === 0) {
+    return undefined;
+  }
+
+  const cliSchema = z.object(buildCliArgSchema(def)).strict();
+  const parsed = cliSchema.safeParse(rawArgs);
+  if (parsed.success) {
+    return undefined;
+  }
+
+  const disallowedSpellings = buildDisallowedCliSpellings(def);
+  const unknownKeys = parsed.error.issues.flatMap((issue) =>
+    issue.code === 'unrecognized_keys' ? issue.keys : [],
+  );
+
+  if (unknownKeys.length > 0) {
+    return unknownKeys
+      .map((key) => {
+        if (disallowedSpellings.has(key)) {
+          return `Unsupported option "--${key}" for ${scriptName} ${commandName}.`;
+        }
+        return `Unknown option "--${key}" for ${scriptName} ${commandName}.`;
+      })
+      .join('\n');
+  }
+
+  const [issue] = parsed.error.issues;
+  const optionName =
+    typeof issue?.path[0] === 'string' ? `--${issue.path[0]}` : 'CLI arguments';
+  return `Invalid value for "${optionName}" in ${scriptName} ${commandName}: ${issue?.message ?? parsed.error.message}`;
+}

package/src/cli/cli-error.ts

@@ -0,0 +1,24 @@
+export class CLIError extends Error {
+  constructor(
+    message: string,
+    public exitCode = 1,
+  ) {
+    super(message);
+  }
+}
+
+export function reportCLIError(
+  error: unknown,
+  log: (
+    message?: unknown,
+    ...optionalParams: unknown[]
+  ) => void = console.error,
+): number {
+  if (error instanceof CLIError) {
+    log(error.message);
+    return error.exitCode;
+  }
+
+  log(error);
+  return 1;
+}

package/src/cli/cli-runner.ts

@@ -9,6 +9,12 @@ import type {
   ToolResult,
   ToolResultContent,
 } from '../mcp/types';
+import {
+  formatCliValidationError,
+  getCliOptionDisplay,
+  parseCliArgs,
+} from './cli-args';
+import { CLIError } from './cli-error';
 
 const debug = getDebug('cli-runner');
 
@@ -32,58 +38,8 @@ export interface CLIRunnerOptions {
   extraCommands?: CLIExtraCommand[];
 }
 
-export class CLIError extends Error {
-  constructor(
-    message: string,
-    public exitCode = 1,
-  ) {
-    super(message);
-  }
-}
-
-export function parseValue(raw: string): unknown {
-  // JSON objects/arrays
-  if (raw.startsWith('{') || raw.startsWith('[')) {
-    try {
-      return JSON.parse(raw);
-    } catch {
-      // Not valid JSON, treat as string below
-    }
-  }
-
-  // Numbers
-  if (/^-?\d+(\.\d+)?$/.test(raw)) {
-    return Number(raw);
-  }
-
-  return raw;
-}
-
-export function parseCliArgs(args: string[]): Record<string, unknown> {
-  const result: Record<string, unknown> = {};
-
-  for (let i = 0; i < args.length; i++) {
-    const arg = args[i];
-    if (!arg.startsWith('--')) continue;
-
-    const body = arg.slice(2);
-    const eqIdx = body.indexOf('=');
-
-    if (eqIdx >= 0) {
-      // --key=value
-      result[body.slice(0, eqIdx)] = parseValue(body.slice(eqIdx + 1));
-    } else if (args[i + 1] && !args[i + 1].startsWith('--')) {
-      // --key value
-      i++;
-      result[body] = parseValue(args[i]);
-    } else {
-      // --flag (boolean)
-      result[body] = true;
-    }
-  }
-
-  return result;
-}
+export { parseCliArgs, parseValue } from './cli-args';
+export { CLIError, reportCLIError } from './cli-error';
 
 function outputContentItem(item: ToolResultContent, isError: boolean): void {
   switch (item.type) {
@@ -128,10 +84,23 @@ function printCommandHelp(scriptName: string, cmd: CLICommand): void {
 
   const schemaEntries = Object.entries(def.schema);
   if (schemaEntries.length > 0) {
+    const optionWidth = Math.max(
+      22,
+      ...schemaEntries.map(
+        ([key]) =>
+          getCliOptionDisplay(key, def.cli?.options?.[key]).label.length,
+      ),
+    );
     console.log('\nOptions:');
     for (const [key, zodType] of schemaEntries) {
+      const { label, aliases } = getCliOptionDisplay(
+        key,
+        def.cli?.options?.[key],
+      );
       const desc = zodType.description ?? '';
-      console.log(`  --${key.padEnd(20)} ${desc}`);
+      const aliasText =
+        aliases.length > 0 ? ` (aliases: ${aliases.join(', ')})` : '';
+      console.log(`  ${label.padEnd(optionWidth)} ${desc}${aliasText}`);
     }
   }
 }
@@ -158,8 +127,10 @@ function printHelp(
   console.log(`\nRun "${scriptName} <command> --help" for more info.`);
 }
 
+type AnyMidsceneTools = BaseMidsceneTools<any, any>;
+
 export async function runToolsCLI(
-  tools: BaseMidsceneTools,
+  tools: AnyMidsceneTools,
   scriptName: string,
   options?: CLIRunnerOptions,
 ): Promise<void> {
@@ -227,14 +198,24 @@ export async function runToolsCLI(
   }
 
   const parsedArgs = parseCliArgs(restArgs);
-  debug('command: %s, args: %s', match.name, JSON.stringify(parsedArgs));
-
   if (parsedArgs.help === true) {
     debug('showing command help for: %s', match.name);
     printCommandHelp(scriptName, match);
     return;
   }
 
+  const cliValidationError = formatCliValidationError(
+    scriptName,
+    match.name,
+    match.def,
+    parsedArgs,
+  );
+  if (cliValidationError) {
+    throw new CLIError(cliValidationError);
+  }
+
+  debug('command: %s, args: %s', match.name, JSON.stringify(parsedArgs));
+
   const result = await match.def.handler(parsedArgs);
   debug(
     'command %s completed, isError: %s',

package/src/cli/index.ts

@@ -1,8 +1,4 @@
-export {
-  runToolsCLI,
-  CLIError,
-  parseValue,
-  parseCliArgs,
-  removePrefix,
-} from './cli-runner';
+export { CLIError, reportCLIError } from './cli-error';
+export { parseCliArgs, parseValue } from './cli-args';
+export { runToolsCLI, removePrefix } from './cli-runner';
 export type { CLIRunnerOptions, CLIExtraCommand } from './cli-runner';