@aiscene/shared 8.0.3 → 8.0.4

package/src/agent-tools/types.ts

@@ -0,0 +1,250 @@
+import type { z } from 'zod';
+import type { ToolDefaults } from './tool-defaults';
+
+// Avoid circular dependency: don't import from @aiscene/core
+// Instead, use generic types that will be provided by implementation
+
+/**
+ * Default timeout constants for app loading verification
+ */
+export const defaultAppLoadingTimeoutMs = 10000;
+export const defaultAppLoadingCheckIntervalMs = 2000;
+
+/**
+ * Content item types for tool results.
+ */
+export type ToolResultContent =
+  | { type: 'text'; text: string }
+  | { type: 'image'; data: string; mimeType: string }
+  | { type: 'audio'; data: string; mimeType: string }
+  | {
+      type: 'resource';
+      resource:
+        | { text: string; uri: string; mimeType?: string }
+        | { uri: string; blob: string; mimeType?: string };
+    };
+
+/**
+ * Result type for tool execution.
+ */
+export interface ToolResult {
+  [x: string]: unknown;
+  content: ToolResultContent[];
+  isError?: boolean;
+  _meta?: Record<string, unknown>;
+}
+
+/**
+ * Tool handler function type
+ * Takes parsed arguments and returns a tool result
+ */
+export type ToolHandler<T = Record<string, unknown>> = (
+  args: T,
+) => Promise<ToolResult>;
+
+/**
+ * Tool schema type using Zod
+ */
+export type ToolSchema = Record<string, z.ZodTypeAny>;
+
+export interface ToolCliOption {
+  preferredName?: string;
+  aliases?: string[];
+  hidden?: boolean;
+}
+
+export interface ToolCliMetadata {
+  options?: Record<string, ToolCliOption>;
+  /** Schema keys populated from leading positional CLI arguments. */
+  positionals?: string[];
+}
+
+/**
+ * Tool definition for Midscene CLI and Skill surfaces.
+ */
+export interface ToolDefinition<T = Record<string, unknown>> {
+  name: string;
+  description: string;
+  schema: ToolSchema;
+  handler: ToolHandler<T>;
+  cli?: ToolCliMetadata;
+}
+
+export type Tool = ToolDefinition;
+
+/**
+ * Action space item definition
+ * Note: Intentionally no index signature to maintain compatibility with DeviceAction
+ */
+export interface ActionSpaceItem {
+  name: string;
+  description?: string;
+  args?: Record<string, unknown>;
+  paramSchema?: z.ZodTypeAny;
+}
+
+/**
+ * Structural shape compatible with @aiscene/core `TUserPrompt`.
+ * Declared locally to avoid a circular dep on `@aiscene/core` from `@aiscene/shared`.
+ *
+ * Consumed by the `assert` and `act` tools in `generateCommonTools`, both of
+ * which forward reference images to core (`aiAssert` / `aiAct`). `aiWaitFor`
+ * stays string-only at the CLI surface because the tools generator does not
+ * yet expose a multimodal entry point for it.
+ */
+export type UserPromptLike =
+  | string
+  | {
+      prompt: string;
+      images?: Array<{ name: string; url: string }>;
+      convertHttpImage2Base64?: boolean;
+    };
+
+export interface RecordToReportScreenshot {
+  /**
+   * PNG/JPEG data URI, or raw PNG base64 body.
+   */
+  base64: string;
+  description?: string;
+}
+
+export interface RecordToReportOptions {
+  content?: string;
+  /**
+   * @deprecated Use `screenshots: [{ base64 }]` instead.
+   */
+  screenshotBase64?: string;
+  screenshots?: RecordToReportScreenshot[];
+}
+
+/** Generic progress-bus envelope as seen by tool consumers (all untrusted). */
+export interface BaseAgentProgressEvent {
+  scope?: unknown;
+  phase?: unknown;
+  sequence?: unknown;
+  data?: unknown;
+}
+
+/** A single frame in a portable UI observation record. */
+export interface UIObservationFrame {
+  /** Path to the captured image. Serialized manifests use relative paths. */
+  path: string;
+  /** MIME type of the captured image file. */
+  mimeType: 'image/png' | 'image/jpeg';
+  /** Capture timestamp in milliseconds. */
+  capturedAt: number;
+}
+
+/**
+ * Serializable observation window that can be persisted and loaded later.
+ * Frames are ordered from earliest to latest; the final frame represents the
+ * UI state at the end of the observation window.
+ */
+export interface UIObservationRecord {
+  type: 'midscene_ui_observation';
+  version: 1;
+  /** Time when observation sampling started. */
+  startedAt: number;
+  /** Time when the final representative frame was captured. */
+  endedAt: number;
+  frames: UIObservationFrame[];
+  shotSize: {
+    width: number;
+    height: number;
+  };
+  shrunkShotToLogicalRatio: number;
+}
+
+/** Fixed observation window consumed by shared CLI surfaces. */
+export interface BaseUIObservation {
+  readonly frameCount: number;
+  readonly startedAt: number;
+  readonly endedAt: number;
+  aiAssert(
+    assertion: UserPromptLike,
+    message?: string,
+    options?: Record<string, unknown>,
+  ): Promise<unknown>;
+  /** Release any resources owned by this observation. */
+  dispose?(): Promise<void>;
+}
+
+/** Minimal UI observation lifecycle required by shared tool surfaces. */
+export interface BaseUIObserver {
+  readonly bufferedFrameCount: number;
+  /** Stop sampling and return the fixed observed window. */
+  stop(): Promise<BaseUIObservation>;
+  /** Release temporary backing files if observation does not complete. */
+  dispose?(): Promise<void>;
+}
+
+/** Options for {@link BaseAgent.startObserving}. */
+export interface BaseUIObserverOptions {
+  /** Sampling interval in milliseconds. Defaults to 1000; minimum 200. */
+  intervalMs?: number;
+  /** Maximum number of buffered frames. Defaults to 30; minimum 2. */
+  maxFrames?: number;
+  /** Auto-stop timeout in milliseconds. Defaults to 300000; 0 disables it. */
+  watchdogMs?: number;
+}
+
+/**
+ * Base agent interface
+ * Represents a platform-specific agent (Android, iOS, Web)
+ * Note: Return types use `unknown` for compatibility with platform-specific implementations
+ */
+export interface BaseAgent {
+  getActionSpace(): Promise<ActionSpaceItem[]>;
+  destroy?(): Promise<void>;
+  reportFile?: string | null;
+  page?: {
+    screenshotBase64(): Promise<string>;
+  };
+  addDumpUpdateListener?(
+    listener: (dump: string, executionDump?: unknown) => void,
+  ): () => void;
+  addProgressListener?(
+    listener: (event: BaseAgentProgressEvent) => void,
+  ): () => void;
+  recordToReport?(
+    title?: string,
+    opt?: RecordToReportOptions,
+  ): Promise<void>;
+  callActionInActionSpace?(
+    actionName: string,
+    params?: unknown,
+  ): Promise<unknown>;
+  aiAction?(
+    description: UserPromptLike,
+    params?: Record<string, unknown>,
+  ): Promise<unknown>;
+  aiWaitFor?(
+    assertion: string,
+    options: Record<string, unknown>,
+  ): Promise<unknown>;
+  aiAssert?(
+    assertion: UserPromptLike,
+    msg?: string,
+    options?: Record<string, unknown>,
+  ): Promise<unknown>;
+  /** Start a UI observation window and capture its baseline frame. */
+  startObserving?(options?: BaseUIObserverOptions): Promise<BaseUIObserver>;
+}
+
+/**
+ * Base device interface for temporary device instances
+ */
+export interface BaseDevice {
+  actionSpace(): ActionSpaceItem[];
+  destroy?(): Promise<void>;
+}
+
+/**
+ * Interface for platform-specific tools manager.
+ */
+export interface IMidsceneTools {
+  initTools(): Promise<void>;
+  destroy?(): Promise<void>;
+  getCliToolDefinitions?(): ToolDefinition[];
+  setToolDefaults?(toolDefaults: ToolDefaults): void;
+}

package/src/agent-tools/user-prompt.ts

@@ -0,0 +1,102 @@
+import { z } from 'zod';
+import type { UserPromptLike } from './types';
+
+type PromptReferenceImage = { name: string; url: string };
+
+function normalizeStringList(raw: unknown, fieldName: string): string[] {
+  if (raw === undefined || raw === null) return [];
+  if (typeof raw === 'string') {
+    const trimmed = raw.trim();
+    return trimmed ? [trimmed] : [];
+  }
+  if (Array.isArray(raw)) {
+    return raw.map((item, index) => {
+      if (typeof item !== 'string') {
+        throw new Error(`${fieldName}[${index}]: expected a string.`);
+      }
+      return item.trim();
+    });
+  }
+  throw new Error(
+    `${fieldName}: expected a string or string array, got ${typeof raw}.`,
+  );
+}
+
+function composeImages(input: {
+  image?: unknown;
+  imageName?: unknown;
+}): PromptReferenceImage[] {
+  const urls = normalizeStringList(input.image, 'image');
+  const names = normalizeStringList(input.imageName, 'imageName');
+
+  if (urls.length !== names.length) {
+    throw new Error(
+      `image/imageName: expected the same number of --image and --image-name values, got ${urls.length} image(s) and ${names.length} image name(s).`,
+    );
+  }
+
+  return urls.map((url, index) => ({ name: names[index], url }));
+}
+
+function coerceBoolean(value: unknown): boolean | undefined {
+  if (value === undefined || value === null) return undefined;
+  if (typeof value === 'boolean') return value;
+  if (typeof value === 'string') {
+    const trimmed = value.trim();
+    if (!trimmed) return undefined;
+    const v = trimmed.toLowerCase();
+    if (v === 'true' || v === '1') return true;
+    if (v === 'false' || v === '0') return false;
+    throw new Error(
+      `convertHttpImage2Base64: expected "true", "false", "1", or "0"; got ${JSON.stringify(value)}.`,
+    );
+  }
+  throw new Error(
+    `convertHttpImage2Base64: expected a boolean, got ${typeof value}.`,
+  );
+}
+
+export function composeUserPrompt(input: {
+  prompt: string;
+  image?: unknown;
+  imageName?: unknown;
+  convertHttpImage2Base64?: unknown;
+}): UserPromptLike {
+  const images = composeImages({
+    image: input.image,
+    imageName: input.imageName,
+  });
+  const convertFlag = coerceBoolean(input.convertHttpImage2Base64);
+
+  if (images.length === 0 && convertFlag === undefined) {
+    return input.prompt;
+  }
+
+  const payload: Exclude<UserPromptLike, string> = { prompt: input.prompt };
+  if (images.length > 0) {
+    payload.images = images;
+  }
+  if (convertFlag !== undefined) {
+    payload.convertHttpImage2Base64 = convertFlag;
+  }
+  return payload;
+}
+
+export const promptInputExtraSchema = {
+  image: z
+    .union([z.string(), z.array(z.string())])
+    .optional()
+    .describe('Reference image URL/path. Repeat --image for multiple images.'),
+  imageName: z
+    .union([z.string(), z.array(z.string())])
+    .optional()
+    .describe(
+      'Reference image name. Repeat --image-name; must align with --image order.',
+    ),
+  convertHttpImage2Base64: z
+    .union([z.boolean(), z.string()])
+    .optional()
+    .describe(
+      'If true, convert http(s) image URLs to base64 before sending to the model.',
+    ),
+};

package/src/cli/interrupt.ts

@@ -0,0 +1,207 @@
+export type CliInterruptReason = 'sigint' | 'sigterm' | 'sighup' | 'watchdog';
+
+type CliInterruptSignal = 'SIGINT' | 'SIGTERM' | 'SIGHUP';
+
+export interface CliInterruptSource {
+  on(event: CliInterruptSignal, listener: () => void): unknown;
+  removeListener(event: CliInterruptSignal, listener: () => void): unknown;
+}
+
+export interface CliInterruptInputSource {
+  readonly isTTY?: boolean;
+  readonly isRaw?: boolean;
+  readonly readableFlowing?: boolean | null;
+  on(event: 'data', listener: (chunk: unknown) => void): unknown;
+  removeListener(event: 'data', listener: (chunk: unknown) => void): unknown;
+  setRawMode?(enabled: boolean): unknown;
+  pause?(): unknown;
+}
+
+export interface CliInterruptWaiter {
+  /** Resolves on the first stop signal or watchdog timeout. */
+  readonly result: Promise<CliInterruptReason>;
+  /** Release signal handlers after asynchronous finalization has completed. */
+  dispose(): void;
+}
+
+export interface CliInterruptWaiterOptions {
+  source?: CliInterruptSource;
+  input?: CliInterruptInputSource;
+  /** Called after restoring the terminal when Ctrl+C is pressed again. */
+  forceExit?: (exitCode: number) => void;
+}
+
+const activeInterruptWaiters = new WeakMap<object, number>();
+const noop = () => {};
+const sigintExitCode = 130;
+
+function guardTerminalCtrlC(
+  input: CliInterruptInputSource | undefined,
+  onInterrupt: () => void,
+): () => void {
+  if (!input?.isTTY || !input.setRawMode) return noop;
+
+  const wasRaw = input.isRaw === true;
+  const wasFlowing = input.readableFlowing === true;
+  const onData = (chunk: unknown) => {
+    const includesCtrlC =
+      (typeof chunk === 'string' && chunk.includes('\u0003')) ||
+      (chunk instanceof Uint8Array && chunk.includes(3));
+    if (includesCtrlC) onInterrupt();
+  };
+  const dispose = () => {
+    input.removeListener('data', onData);
+    try {
+      if (!wasFlowing) input.pause?.();
+    } finally {
+      if (!wasRaw) input.setRawMode?.(false);
+    }
+  };
+
+  input.setRawMode(true);
+  try {
+    input.on('data', onData);
+  } catch (error) {
+    dispose();
+    throw error;
+  }
+  return dispose;
+}
+
+function registerCliInterruptWaiter(source: CliInterruptSource): () => void {
+  const key = source as object;
+  activeInterruptWaiters.set(key, (activeInterruptWaiters.get(key) ?? 0) + 1);
+
+  let active = true;
+  return () => {
+    if (!active) return;
+    active = false;
+
+    const remaining = (activeInterruptWaiters.get(key) ?? 1) - 1;
+    if (remaining > 0) {
+      activeInterruptWaiters.set(key, remaining);
+    } else {
+      activeInterruptWaiters.delete(key);
+    }
+  };
+}
+
+/** Whether a foreground CLI command is currently waiting for this source. */
+export function hasActiveCliInterruptWaiter(
+  source: CliInterruptSource = process,
+): boolean {
+  return (activeInterruptWaiters.get(source as object) ?? 0) > 0;
+}
+
+/**
+ * Keep graceful-stop handlers installed until the caller has finished saving.
+ *
+ * Package runners such as pnpm can deliver SIGINT to the foreground child,
+ * immediately follow it with SIGTERM, then cause SIGHUP when the runner exits
+ * and its pseudo-terminal closes. Resolving on the first signal is not enough:
+ * removing any handler at that point lets a subsequent signal kill the child
+ * during asynchronous artifact finalization.
+ *
+ * On a TTY, Ctrl+C is captured as raw input so the package runner itself stays
+ * alive until the child has saved and restored the terminal. Signal handlers
+ * remain as the graceful-stop path for externally delivered termination.
+ */
+export function createCliInterruptWaiter(
+  watchdogMs: number,
+  options: CliInterruptWaiterOptions = {},
+): CliInterruptWaiter {
+  const source = options.source ?? process;
+  const input =
+    options.input ?? (source === process ? process.stdin : undefined);
+  const forceExit =
+    options.forceExit ??
+    (source === process
+      ? (exitCode: number) => {
+          process.exit(exitCode);
+        }
+      : undefined);
+  const unregisterWaiter = registerCliInterruptWaiter(source);
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let finished = false;
+  let disposed = false;
+  let disposeInput = noop;
+  let resolveResult!: (reason: CliInterruptReason) => void;
+  let rejectResult!: (error: unknown) => void;
+
+  const result = new Promise<CliInterruptReason>((resolve, reject) => {
+    resolveResult = resolve;
+    rejectResult = reject;
+  });
+
+  const finish = (reason: CliInterruptReason) => {
+    if (finished) return;
+    finished = true;
+    if (timer) {
+      clearTimeout(timer);
+      timer = undefined;
+    }
+    resolveResult(reason);
+  };
+  const onSigint = () => finish('sigint');
+  const onSigterm = () => finish('sigterm');
+  const onSighup = () => finish('sighup');
+
+  function dispose() {
+    if (disposed) return;
+    disposed = true;
+    source.removeListener('SIGINT', onSigint);
+    source.removeListener('SIGTERM', onSigterm);
+    source.removeListener('SIGHUP', onSighup);
+    try {
+      disposeInput();
+    } finally {
+      if (timer) {
+        clearTimeout(timer);
+        timer = undefined;
+      }
+      unregisterWaiter();
+    }
+  }
+
+  const onTerminalCtrlC = () => {
+    if (!finished) {
+      finish('sigint');
+      return;
+    }
+
+    // The first Ctrl+C protects asynchronous artifact finalization. A second
+    // explicit Ctrl+C is the user's escape hatch if device or file I/O hangs.
+    dispose();
+    forceExit?.(sigintExitCode);
+  };
+
+  try {
+    source.on('SIGINT', onSigint);
+    source.on('SIGTERM', onSigterm);
+    source.on('SIGHUP', onSighup);
+    disposeInput = guardTerminalCtrlC(input, onTerminalCtrlC);
+    if (watchdogMs > 0) {
+      timer = setTimeout(() => finish('watchdog'), watchdogMs);
+    }
+  } catch (error) {
+    dispose();
+    rejectResult(error);
+  }
+
+  return { result, dispose };
+}
+
+/**
+ * Wait for one stop request and release the handlers immediately afterwards.
+ * Long-running finalizers should use {@link createCliInterruptWaiter} instead.
+ */
+export function waitForCliInterrupt(
+  watchdogMs: number,
+  source: CliInterruptSource = process,
+  input: CliInterruptInputSource | undefined = source === process
+    ? process.stdin
+    : undefined,
+): Promise<CliInterruptReason> {
+  const waiter = createCliInterruptWaiter(watchdogMs, { source, input });
+  return waiter.result.finally(waiter.dispose);
+}