@epsilon-asi/actors 0.0.21 → 0.0.32

package/src/native/NativeCoordinateMapper.ts

@@ -0,0 +1,258 @@
+import type { PageLike } from '../browser/PuppeteerLike.js';
+import type { Logger } from '../logging/Logger.js';
+import type {
+  BrowserPoint,
+  BrowserViewportSnapshot,
+  CalibrationOptions,
+  CoordinateMapOptions,
+  CoordinateTransform,
+  NativeCoordinateMapper,
+  NativeImageFinder,
+  NativeTemplateRef,
+  NativeWindowService,
+  ScreenPoint
+} from './types.js';
+import { CoordinateCalibrationError } from './errors.js';
+import { centerOf } from './utils/geometry.js';
+
+interface MarkerSpec {
+  id: string;
+  point: BrowserPoint;
+  size: number;
+  color: string;
+}
+
+interface MarkerInstallPayload {
+  markers: MarkerSpec[];
+}
+
+export interface ImageBasedCoordinateMapperOptions {
+  images: NativeImageFinder;
+  windows?: NativeWindowService;
+  logger?: Logger;
+  defaultTimeoutMs?: number;
+  defaultCacheTtlMs?: number;
+}
+
+const DEFAULT_MARKER_A: BrowserPoint = { x: 120, y: 120 };
+const DEFAULT_MARKER_B: BrowserPoint = { x: 420, y: 320 };
+const DEFAULT_MARKER_SIZE = 22;
+
+function assertFinite(value: number, name: string): void {
+  if (!Number.isFinite(value)) {
+    throw new CoordinateCalibrationError(`Coordinate calibration produced a non-finite ${name}.`, { [name]: value });
+  }
+}
+
+async function getViewportSnapshot(page: PageLike): Promise<BrowserViewportSnapshot> {
+  return page.evaluate<BrowserViewportSnapshot>(() => {
+    const view = globalThis.window;
+    return {
+      screenX: view.screenX,
+      screenY: view.screenY,
+      outerWidth: view.outerWidth,
+      outerHeight: view.outerHeight,
+      innerWidth: view.innerWidth,
+      innerHeight: view.innerHeight,
+      devicePixelRatio: view.devicePixelRatio
+    };
+  });
+}
+
+async function installMarkers(page: PageLike, markers: MarkerSpec[]): Promise<void> {
+  const payload: MarkerInstallPayload = { markers };
+  await page.evaluate<void>((rawPayload: unknown) => {
+    const { markers: markerSpecs } = rawPayload as MarkerInstallPayload;
+    const existing = document.getElementById('__paf_native_coordinate_markers__');
+    existing?.remove();
+
+    const root = document.createElement('div');
+    root.id = '__paf_native_coordinate_markers__';
+    root.style.position = 'fixed';
+    root.style.left = '0';
+    root.style.top = '0';
+    root.style.width = '0';
+    root.style.height = '0';
+    root.style.zIndex = '2147483647';
+    root.style.pointerEvents = 'none';
+
+    for (const marker of markerSpecs) {
+      const element = document.createElement('div');
+      element.id = marker.id;
+      element.setAttribute('data-paf-native-calibration-marker', marker.id);
+      element.style.position = 'fixed';
+      element.style.left = `${marker.point.x}px`;
+      element.style.top = `${marker.point.y}px`;
+      element.style.width = `${marker.size}px`;
+      element.style.height = `${marker.size}px`;
+      element.style.background = marker.color;
+      element.style.border = '3px solid #000';
+      element.style.boxSizing = 'border-box';
+      element.style.boxShadow = '0 0 0 2px #fff';
+      root.appendChild(element);
+    }
+
+    document.documentElement.appendChild(root);
+  }, payload);
+}
+
+async function removeMarkers(page: PageLike): Promise<void> {
+  await page.evaluate<void>(() => {
+    document.getElementById('__paf_native_coordinate_markers__')?.remove();
+  });
+}
+
+async function captureMarkerTemplate(page: PageLike, marker: MarkerSpec): Promise<NativeTemplateRef> {
+  const margin = 8;
+  const screenshot = await page.screenshot({
+    type: 'png',
+    clip: {
+      x: marker.point.x - margin,
+      y: marker.point.y - margin,
+      width: marker.size + margin * 2,
+      height: marker.size + margin * 2
+    }
+  });
+
+  if (typeof screenshot === 'string') {
+    return { name: `${marker.id}.png`, buffer: Buffer.from(screenshot, 'base64') };
+  }
+
+  return { name: `${marker.id}.png`, buffer: screenshot };
+}
+
+export class ImageBasedCoordinateMapper implements NativeCoordinateMapper {
+  private cached: CoordinateTransform | null = null;
+  private cacheExpiresAt = 0;
+
+  constructor(private readonly options: ImageBasedCoordinateMapperOptions) {}
+
+  async calibrate(page: PageLike, options: CalibrationOptions = {}): Promise<CoordinateTransform> {
+    const markerSize = options.markerSizePx ?? DEFAULT_MARKER_SIZE;
+    const markerA: MarkerSpec = {
+      id: 'paf-native-calibration-a',
+      point: options.markerA ?? DEFAULT_MARKER_A,
+      size: markerSize,
+      color: '#ff00ff'
+    };
+    const markerB: MarkerSpec = {
+      id: 'paf-native-calibration-b',
+      point: options.markerB ?? DEFAULT_MARKER_B,
+      size: markerSize,
+      color: '#00ffff'
+    };
+
+    await installMarkers(page, [markerA, markerB]);
+
+    try {
+      const viewport = await getViewportSnapshot(page);
+      const [templateA, templateB] = await Promise.all([
+        captureMarkerTemplate(page, markerA),
+        captureMarkerTemplate(page, markerB)
+      ]);
+
+      const imageOptions: { timeoutMs?: number; intervalMs?: number } = {};
+      const timeoutMs = options.timeoutMs ?? this.options.defaultTimeoutMs;
+      if (timeoutMs !== undefined) imageOptions.timeoutMs = timeoutMs;
+      if (options.intervalMs !== undefined) imageOptions.intervalMs = options.intervalMs;
+
+      const [matchA, matchB] = await Promise.all([
+        this.options.images.waitForTemplate(templateA, imageOptions),
+        this.options.images.waitForTemplate(templateB, imageOptions)
+      ]);
+
+      const screenA = centerOf(matchA.region);
+      const screenB = centerOf(matchB.region);
+      const browserA = {
+        x: markerA.point.x + markerA.size / 2,
+        y: markerA.point.y + markerA.size / 2
+      };
+      const browserB = {
+        x: markerB.point.x + markerB.size / 2,
+        y: markerB.point.y + markerB.size / 2
+      };
+
+      const scaleX = (screenB.x - screenA.x) / (browserB.x - browserA.x);
+      const scaleY = (screenB.y - screenA.y) / (browserB.y - browserA.y);
+      const offsetX = screenA.x - scaleX * browserA.x;
+      const offsetY = screenA.y - scaleY * browserA.y;
+
+      this.validateTransform({ offsetX, offsetY, scaleX, scaleY }, viewport);
+
+      const transform: CoordinateTransform = {
+        offsetX,
+        offsetY,
+        scaleX,
+        scaleY,
+        calibratedAt: Date.now(),
+        viewport: {
+          innerWidth: viewport.innerWidth,
+          innerHeight: viewport.innerHeight,
+          devicePixelRatio: viewport.devicePixelRatio
+        }
+      };
+
+      if (this.options.windows !== undefined && options.windowMatcher !== undefined) {
+        const window = await this.options.windows.find(options.windowMatcher);
+        if (window !== null) transform.window = window;
+      }
+
+      this.cached = transform;
+      this.cacheExpiresAt = transform.calibratedAt + (options.cacheTtlMs ?? this.options.defaultCacheTtlMs ?? 60_000);
+      return transform;
+    } finally {
+      await removeMarkers(page).catch(error => {
+        this.options.logger?.warn('Failed to remove native coordinate calibration markers.', { error });
+      });
+    }
+  }
+
+  async browserToScreen(point: BrowserPoint, options: CoordinateMapOptions = {}): Promise<ScreenPoint> {
+    const transform = await this.getTransform(options);
+    return {
+      x: transform.offsetX + transform.scaleX * point.x,
+      y: transform.offsetY + transform.scaleY * point.y
+    };
+  }
+
+  async screenToBrowser(point: ScreenPoint, options: CoordinateMapOptions = {}): Promise<BrowserPoint> {
+    const transform = await this.getTransform(options);
+    return {
+      x: (point.x - transform.offsetX) / transform.scaleX,
+      y: (point.y - transform.offsetY) / transform.scaleY
+    };
+  }
+
+  invalidate(): void {
+    this.cached = null;
+    this.cacheExpiresAt = 0;
+  }
+
+  private async getTransform(options: CoordinateMapOptions): Promise<CoordinateTransform> {
+    const useCache = options.useCache ?? true;
+    if (useCache && this.cached !== null && Date.now() < this.cacheExpiresAt) {
+      return this.cached;
+    }
+
+    if (options.page === undefined) {
+      throw new CoordinateCalibrationError('Coordinate mapping requires a calibrated transform or a page for calibration.');
+    }
+
+    return this.calibrate(options.page);
+  }
+
+  private validateTransform(transform: Pick<CoordinateTransform, 'offsetX' | 'offsetY' | 'scaleX' | 'scaleY'>, viewport: BrowserViewportSnapshot): void {
+    assertFinite(transform.offsetX, 'offsetX');
+    assertFinite(transform.offsetY, 'offsetY');
+    assertFinite(transform.scaleX, 'scaleX');
+    assertFinite(transform.scaleY, 'scaleY');
+
+    if (transform.scaleX < 0.5 || transform.scaleX > 3.5 || transform.scaleY < 0.5 || transform.scaleY > 3.5) {
+      throw new CoordinateCalibrationError('Coordinate calibration produced an implausible scale.', {
+        scaleX: transform.scaleX,
+        scaleY: transform.scaleY,
+        viewport
+      });
+    }
+  }
+}

package/src/native/NativeFileDialogService.ts

@@ -0,0 +1,138 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import type { Logger } from '../logging/Logger.js';
+import { NativeFileDialogError } from './errors.js';
+import type { NativeImageFinder, NativeKeyboard, NativeMouse, SelectFileOptions } from './types.js';
+import { centerOf } from './utils/geometry.js';
+import { redactPath } from './utils/redactNative.js';
+
+export interface FileDialogAccessibilityStrategy {
+  selectFile(filePath: string, options: SelectFileOptions): Promise<void>;
+}
+
+export interface NativeFileDialogServiceDependencies {
+  keyboard: NativeKeyboard;
+  mouse?: NativeMouse;
+  images?: NativeImageFinder;
+  accessibility?: FileDialogAccessibilityStrategy;
+  logger?: Logger;
+  defaultTimeoutMs?: number;
+  revealFilePathsInLogs?: boolean;
+}
+
+export class DefaultNativeFileDialogService {
+  constructor(private readonly deps: NativeFileDialogServiceDependencies) {}
+
+  async selectFile(filePath: string, options: SelectFileOptions = {}): Promise<void> {
+    const absolutePath = path.resolve(filePath);
+    await this.assertFileExists(absolutePath);
+
+    const strategy = options.strategy ?? 'auto';
+    const failures: unknown[] = [];
+
+    if (strategy === 'applescript' || strategy === 'auto') {
+      if (this.deps.accessibility !== undefined) {
+        try {
+          await this.deps.accessibility.selectFile(absolutePath, options);
+          this.logSuccess('applescript', absolutePath);
+          return;
+        } catch (error) {
+          failures.push(error);
+          if (strategy === 'applescript') throw this.wrapFailure('applescript', absolutePath, failures);
+        }
+      } else if (strategy === 'applescript') {
+        throw new NativeFileDialogError('AppleScript file-dialog strategy is not available.', { strategy });
+      }
+    }
+
+    if (strategy === 'visual' || strategy === 'auto') {
+      if (this.deps.images !== undefined && this.deps.mouse !== undefined) {
+        try {
+          await this.selectFileVisually(absolutePath, options);
+          this.logSuccess('visual', absolutePath);
+          return;
+        } catch (error) {
+          failures.push(error);
+          if (strategy === 'visual') throw this.wrapFailure('visual', absolutePath, failures);
+        }
+      } else if (strategy === 'visual') {
+        throw new NativeFileDialogError('Visual file-dialog strategy requires image finder and native mouse dependencies.', { strategy });
+      }
+    }
+
+    if (strategy === 'keyboard' || strategy === 'auto') {
+      try {
+        await this.selectFileWithKeyboard(absolutePath, options);
+        this.logSuccess('keyboard', absolutePath);
+        return;
+      } catch (error) {
+        failures.push(error);
+      }
+    }
+
+    throw this.wrapFailure(strategy, absolutePath, failures);
+  }
+
+  private async assertFileExists(filePath: string): Promise<void> {
+    const stat = await fs.stat(filePath).catch(() => null);
+    if (stat === null || !stat.isFile()) {
+      throw new NativeFileDialogError('Cannot select file because the path does not exist or is not a file.', {
+        filePath: redactPath(filePath, this.deps.revealFilePathsInLogs)
+      });
+    }
+  }
+
+  private async selectFileVisually(filePath: string, options: SelectFileOptions): Promise<void> {
+    if (this.deps.images === undefined || this.deps.mouse === undefined) {
+      throw new NativeFileDialogError('Visual file-dialog strategy requires image finder and native mouse dependencies.');
+    }
+
+    if (options.inputFieldTemplate !== undefined) {
+      const input = await this.deps.images.waitForTemplate(options.inputFieldTemplate, this.imageSearchOptions(options));
+      await this.deps.mouse.click(centerOf(input.region));
+    } else {
+      await this.deps.keyboard.pressShortcut(['Meta', 'Shift', 'G']);
+    }
+
+    await this.deps.keyboard.type(filePath);
+    await this.deps.keyboard.pressKey('Enter');
+
+    if ((options.clickOpen ?? true) && options.openButtonTemplate !== undefined) {
+      const openButton = await this.deps.images.waitForTemplate(options.openButtonTemplate, this.imageSearchOptions(options));
+      await this.deps.mouse.click(centerOf(openButton.region));
+    } else if (options.clickOpen ?? true) {
+      await this.deps.keyboard.pressKey('Enter');
+    }
+  }
+
+  private async selectFileWithKeyboard(filePath: string, options: SelectFileOptions): Promise<void> {
+    await this.deps.keyboard.pressShortcut(['Meta', 'Shift', 'G']);
+    await this.deps.keyboard.type(filePath);
+    await this.deps.keyboard.pressKey('Enter');
+    if (options.clickOpen ?? true) {
+      await this.deps.keyboard.pressKey('Enter');
+    }
+  }
+
+  private imageSearchOptions(options: SelectFileOptions) {
+    const searchOptions: { timeoutMs?: number; intervalMs?: number } = {};
+    if (options.timeoutMs !== undefined) searchOptions.timeoutMs = options.timeoutMs;
+    if (options.intervalMs !== undefined) searchOptions.intervalMs = options.intervalMs;
+    return searchOptions;
+  }
+
+  private wrapFailure(strategy: string, filePath: string, failures: unknown[]): NativeFileDialogError {
+    return new NativeFileDialogError('Failed to select a file in the native file dialog.', {
+      strategy,
+      filePath: redactPath(filePath, this.deps.revealFilePathsInLogs),
+      failures
+    });
+  }
+
+  private logSuccess(strategy: string, filePath: string): void {
+    this.deps.logger?.debug('Selected file in native dialog.', {
+      strategy,
+      filePath: redactPath(filePath, this.deps.revealFilePathsInLogs)
+    });
+  }
+}

package/src/native/NativeImageFinder.ts

@@ -0,0 +1,33 @@
+import type { NativeImageFinder, NativeImageMatch, NativeImageSearchOptions, NativeTemplateRef } from './types.js';
+import { NativeImageNotFoundError } from './errors.js';
+import { waitFor } from './utils/waitFor.js';
+
+export interface NativeImageFinderDriver {
+  findTemplate(template: string | NativeTemplateRef, options?: NativeImageSearchOptions): Promise<NativeImageMatch | null>;
+}
+
+export class DefaultNativeImageFinder implements NativeImageFinder {
+  constructor(private readonly driver: NativeImageFinderDriver, private readonly defaultTimeoutMs = 15_000) {}
+
+  async findTemplate(template: string | NativeTemplateRef, options?: NativeImageSearchOptions): Promise<NativeImageMatch | null> {
+    return this.driver.findTemplate(template, options);
+  }
+
+  async waitForTemplate(template: string | NativeTemplateRef, options: NativeImageSearchOptions = {}): Promise<NativeImageMatch> {
+    try {
+      return await waitFor(
+        () => this.findTemplate(template, options),
+        {
+          timeoutMs: options.timeoutMs ?? this.defaultTimeoutMs,
+          intervalMs: options.intervalMs ?? 100,
+          description: `native image template ${typeof template === 'string' ? template : template.name}`
+        }
+      );
+    } catch (error) {
+      throw new NativeImageNotFoundError({
+        template: typeof template === 'string' ? template : template.name,
+        cause: error
+      });
+    }
+  }
+}

package/src/native/NativeKeyboard.ts

@@ -0,0 +1,18 @@
+import type { NativeKeyboardDriver } from './drivers.js';
+import type { NativeKeyboard, NativeKeyboardTypeOptions } from './types.js';
+
+export class DefaultNativeKeyboard implements NativeKeyboard {
+  constructor(private readonly driver: NativeKeyboardDriver) {}
+
+  async type(text: string, options: NativeKeyboardTypeOptions = {}): Promise<void> {
+    await this.driver.type(text, options.delayMs === undefined ? undefined : { delayMs: options.delayMs });
+  }
+
+  async pressKey(key: string): Promise<void> {
+    await this.driver.pressKey(key);
+  }
+
+  async pressShortcut(keys: readonly string[]): Promise<void> {
+    await this.driver.pressShortcut(keys);
+  }
+}

package/src/native/NativeMouse.ts

@@ -0,0 +1,116 @@
+import { path as ghostCursorPath } from 'ghost-cursor';
+import type { PageLike } from '../browser/PuppeteerLike.js';
+import type { NativeMouseDriver } from './drivers.js';
+import type {
+  BrowserPoint,
+  NativeMouse,
+  NativeMouseClickOptions,
+  NativeMouseMoveOptions,
+  NativePoint,
+  NativeRegion,
+  ScreenPoint
+} from './types.js';
+import { centerOf, clampPointToRegion, randomPointInRegion, roundPoint } from './utils/geometry.js';
+
+export type NativeRandomFunction = () => number;
+export type NativeSleepFunction = (ms: number) => Promise<void>;
+export type PathGenerator = (from: NativePoint, to: NativePoint, options?: { moveSpeed?: number }) => NativePoint[];
+
+export interface HumanizedNativeMouseOptions {
+  jitterPixels?: number;
+  minJitterPixels?: number;
+  moveSpeed?: number;
+  random?: NativeRandomFunction;
+  sleep?: NativeSleepFunction;
+  pathGenerator?: PathGenerator;
+  coordinateMapper: {
+    browserToScreen(point: BrowserPoint, options?: { page?: PageLike }): Promise<ScreenPoint>;
+  };
+}
+
+const defaultSleep: NativeSleepFunction = async ms => {
+  if (ms <= 0) return;
+  await new Promise<void>(resolve => setTimeout(resolve, ms));
+};
+
+function defaultPathGenerator(from: NativePoint, to: NativePoint, options?: { moveSpeed?: number }): NativePoint[] {
+  const pathOptions = options?.moveSpeed === undefined ? undefined : { moveSpeed: options.moveSpeed };
+  return ghostCursorPath(from, to, pathOptions) as NativePoint[];
+}
+
+function normalizeJitter(minJitterPixels: number | undefined, jitterPixels: number | undefined): { min: number; max: number } {
+  const max = Math.max(0, jitterPixels ?? 3);
+  const min = Math.min(max, Math.max(0, minJitterPixels ?? Math.min(2, max)));
+  return { min, max };
+}
+
+export class HumanizedNativeMouse implements NativeMouse {
+  private readonly random: NativeRandomFunction;
+  private readonly sleep: NativeSleepFunction;
+  private readonly pathGenerator: PathGenerator;
+  private readonly jitterPixels: number;
+  private readonly minJitterPixels: number;
+  private readonly moveSpeed: number | undefined;
+
+  constructor(private readonly driver: NativeMouseDriver, private readonly options: HumanizedNativeMouseOptions) {
+    this.random = options.random ?? Math.random;
+    this.sleep = options.sleep ?? defaultSleep;
+    this.pathGenerator = options.pathGenerator ?? defaultPathGenerator;
+    this.jitterPixels = options.jitterPixels ?? 3;
+    this.minJitterPixels = options.minJitterPixels ?? 2;
+    this.moveSpeed = options.moveSpeed;
+  }
+
+  async moveTo(point: ScreenPoint, options: NativeMouseMoveOptions = {}): Promise<void> {
+    const current = await this.driver.getPosition();
+    const target = roundPoint(this.applyJitter(point, options));
+
+    if (options.disableHumanPath === true) {
+      await this.driver.move(target);
+      return;
+    }
+
+    const moveSpeed = options.moveSpeed ?? this.moveSpeed;
+    const route = this.pathGenerator(current, target, moveSpeed === undefined ? undefined : { moveSpeed });
+    const points = route.length === 0 ? [target] : route;
+
+    for (const [index, routePoint] of points.entries()) {
+      const isLast = index === points.length - 1;
+      const nextPoint = isLast ? target : roundPoint(this.applyJitter(routePoint, options));
+      await this.driver.move(nextPoint);
+    }
+  }
+
+  async click(point: ScreenPoint, options: NativeMouseClickOptions = {}): Promise<void> {
+    await this.moveTo(point, options);
+    await this.sleep(options.delayBeforeClickMs ?? 0);
+    await this.driver.click(options.button ?? 'left');
+    await this.sleep(options.delayAfterMoveMs ?? 0);
+  }
+
+  async clickRegion(region: NativeRegion, options: NativeMouseClickOptions = {}): Promise<void> {
+    const padding = options.regionPaddingPx ?? 4;
+    const point = randomPointInRegion(region, this.random, padding);
+    const target = options.precise === true ? point : clampPointToRegion(this.applyJitter(point, options), region);
+    await this.click(roundPoint(target), { ...options, precise: true });
+  }
+
+  async clickBrowserPoint(page: PageLike, point: BrowserPoint, options: NativeMouseClickOptions = {}): Promise<void> {
+    const screenPoint = await this.options.coordinateMapper.browserToScreen(point, { page });
+    await this.click(screenPoint, options);
+  }
+
+  private applyJitter(point: NativePoint, options: NativeMouseMoveOptions): NativePoint {
+    if (options.precise === true) return point;
+
+    const { min, max } = normalizeJitter(options.minJitterPixels ?? this.minJitterPixels, options.jitterPixels ?? this.jitterPixels);
+    if (max <= 0) return point;
+
+    const distance = min + this.random() * (max - min);
+    const angle = this.random() * Math.PI * 2;
+    return {
+      x: point.x + Math.cos(angle) * distance,
+      y: point.y + Math.sin(angle) * distance
+    };
+  }
+}