@epsilon-asi/actors 0.0.21 → 0.0.32

package/src/native/types.ts

@@ -0,0 +1,224 @@
+import type { BrowserSession } from '../browser/BrowserSession.js';
+import type { PageLike } from '../browser/PuppeteerLike.js';
+import type { RuntimeConfig } from '../browser/RuntimeConfig.js';
+import type { Logger } from '../logging/Logger.js';
+
+export type NativePlatform = 'macos';
+export type NativeMouseButton = 'left' | 'right' | 'middle';
+export type NativeFileDialogStrategy = 'auto' | 'applescript' | 'visual' | 'keyboard';
+export type NativeWindowControlStrategy = 'auto' | 'accessibility' | 'visual';
+
+export interface NativePoint {
+  x: number;
+  y: number;
+}
+
+export interface ScreenPoint extends NativePoint {}
+export interface BrowserPoint extends NativePoint {}
+
+export interface NativeSize {
+  width: number;
+  height: number;
+}
+
+export interface NativeRegion extends NativePoint, NativeSize {}
+
+export interface WaitOptions {
+  timeoutMs?: number;
+  intervalMs?: number;
+}
+
+export interface WindowMatcher {
+  appName?: string;
+  title?: string;
+  titleIncludes?: string;
+  titleRegex?: RegExp | string;
+  bundleId?: string;
+  role?: string;
+}
+
+export interface NativeWindowInfo {
+  id?: string | number;
+  title: string;
+  appName?: string;
+  bundleId?: string;
+  role?: string;
+  region: NativeRegion;
+  isFocused?: boolean;
+  isMinimized?: boolean;
+  isFullscreen?: boolean;
+}
+
+export interface EnsureWindowInViewOptions extends WaitOptions {
+  restore?: boolean;
+  focus?: boolean;
+  moveIntoPrimaryDisplay?: boolean;
+  minimumVisibleWidth?: number;
+  minimumVisibleHeight?: number;
+}
+
+export interface WindowButtonOptions extends WaitOptions {
+  strategy?: NativeWindowControlStrategy;
+  fullscreenButtonTemplate?: string;
+  searchRegionPaddingPx?: number;
+}
+
+export interface SelectFileOptions extends WaitOptions {
+  appName?: string;
+  dialogTitle?: string;
+  openButtonTemplate?: string;
+  inputFieldTemplate?: string;
+  strategy?: NativeFileDialogStrategy;
+  clickOpen?: boolean;
+}
+
+export interface NativeMouseMoveOptions {
+  precise?: boolean;
+  jitterPixels?: number;
+  minJitterPixels?: number;
+  moveSpeed?: number;
+  disableHumanPath?: boolean;
+}
+
+export interface NativeMouseClickOptions extends NativeMouseMoveOptions {
+  button?: NativeMouseButton;
+  delayBeforeClickMs?: number;
+  delayAfterMoveMs?: number;
+  regionPaddingPx?: number;
+}
+
+export interface NativeKeyboardTypeOptions {
+  delayMs?: number;
+}
+
+export interface NativeKeyboard {
+  type(text: string, options?: NativeKeyboardTypeOptions): Promise<void>;
+  pressKey(key: string): Promise<void>;
+  pressShortcut(keys: readonly string[]): Promise<void>;
+}
+
+export interface NativeMouse {
+  moveTo(point: ScreenPoint, options?: NativeMouseMoveOptions): Promise<void>;
+  click(point: ScreenPoint, options?: NativeMouseClickOptions): Promise<void>;
+  clickRegion(region: NativeRegion, options?: NativeMouseClickOptions): Promise<void>;
+  clickBrowserPoint(page: PageLike, point: BrowserPoint, options?: NativeMouseClickOptions): Promise<void>;
+}
+
+export interface NativeTemplateRef {
+  name: string;
+  buffer?: Uint8Array;
+}
+
+export interface NativeImageSearchOptions extends WaitOptions {
+  searchRegion?: NativeRegion;
+  confidence?: number;
+}
+
+export interface NativeImageMatch {
+  region: NativeRegion;
+  confidence?: number;
+}
+
+export interface NativeImageFinder {
+  findTemplate(template: string | NativeTemplateRef, options?: NativeImageSearchOptions): Promise<NativeImageMatch | null>;
+  waitForTemplate(template: string | NativeTemplateRef, options?: NativeImageSearchOptions): Promise<NativeImageMatch>;
+}
+
+export interface NativeWindowService {
+  list(): Promise<NativeWindowInfo[]>;
+  find(matcher: WindowMatcher, options?: WaitOptions): Promise<NativeWindowInfo | null>;
+  waitFor(matcher: WindowMatcher, options?: WaitOptions): Promise<NativeWindowInfo>;
+  isOpen(matcher: WindowMatcher): Promise<boolean>;
+  focus(matcher: WindowMatcher): Promise<NativeWindowInfo>;
+  ensureInView(matcher: WindowMatcher, options?: EnsureWindowInViewOptions): Promise<NativeWindowInfo>;
+  setFullscreen(matcher: WindowMatcher, fullscreen: boolean, options?: WindowButtonOptions): Promise<void>;
+  toggleFullscreen(matcher: WindowMatcher, options?: WindowButtonOptions): Promise<void>;
+  minimize(matcher: WindowMatcher): Promise<void>;
+  restore(matcher: WindowMatcher): Promise<void>;
+}
+
+export interface NativeFileDialogService {
+  selectFile(filePath: string, options?: SelectFileOptions): Promise<void>;
+}
+
+export interface BrowserViewportSnapshot {
+  screenX: number;
+  screenY: number;
+  outerWidth: number;
+  outerHeight: number;
+  innerWidth: number;
+  innerHeight: number;
+  devicePixelRatio: number;
+}
+
+export interface CoordinateTransform {
+  offsetX: number;
+  offsetY: number;
+  scaleX: number;
+  scaleY: number;
+  calibratedAt: number;
+  viewport: Pick<BrowserViewportSnapshot, 'innerWidth' | 'innerHeight' | 'devicePixelRatio'>;
+  window?: NativeWindowInfo;
+}
+
+export interface CalibrationOptions extends WaitOptions {
+  markerSizePx?: number;
+  markerA?: BrowserPoint;
+  markerB?: BrowserPoint;
+  cacheTtlMs?: number;
+  windowMatcher?: WindowMatcher;
+}
+
+export interface CoordinateMapOptions {
+  page?: PageLike;
+  useCache?: boolean;
+}
+
+export interface NativeCoordinateMapper {
+  calibrate(page: PageLike, options?: CalibrationOptions): Promise<CoordinateTransform>;
+  browserToScreen(point: BrowserPoint, options?: CoordinateMapOptions): Promise<ScreenPoint>;
+  screenToBrowser(point: ScreenPoint, options?: CoordinateMapOptions): Promise<BrowserPoint>;
+  invalidate(): void;
+}
+
+export interface NativeActionContext {
+  automation: NativeAutomation;
+  logger: Logger;
+}
+
+export interface NativeAction<TInput = unknown, TOutput = unknown> {
+  name: string;
+  description?: string;
+  run(context: NativeActionContext, input: TInput): Promise<TOutput>;
+}
+
+export interface NativeActionMetadata {
+  name: string;
+  description?: string;
+}
+
+export interface NativeActionRegistry {
+  register<TInput, TOutput>(action: NativeAction<TInput, TOutput>): this;
+  get<TInput = unknown, TOutput = unknown>(name: string): NativeAction<TInput, TOutput>;
+  run<TInput = unknown, TOutput = unknown>(name: string, input: TInput): Promise<TOutput>;
+  list(): NativeActionMetadata[];
+}
+
+export interface NativeAutomation {
+  windows: NativeWindowService;
+  fileDialogs: NativeFileDialogService;
+  mouse: NativeMouse;
+  keyboard: NativeKeyboard;
+  coordinates: NativeCoordinateMapper;
+  images: NativeImageFinder;
+  actions: NativeActionRegistry;
+}
+
+export interface NativeAutomationFactoryArgs {
+  session: BrowserSession;
+  page: PageLike;
+  config: RuntimeConfig;
+  logger: Logger;
+}
+
+export type NativeAutomationFactory = (args: NativeAutomationFactoryArgs) => NativeAutomation;

package/src/native/utils/appleScriptEscape.ts

@@ -0,0 +1,11 @@
+/**
+ * Escapes arbitrary text for use as an AppleScript string literal.
+ * AppleScript string literals are double-quoted and escape double quotes/backslashes.
+ */
+export function appleScriptString(value: string): string {
+  return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
+}
+
+export function appleScriptRegexLiteral(value: RegExp | string): string {
+  return value instanceof RegExp ? value.source : value;
+}

package/src/native/utils/geometry.ts

@@ -0,0 +1,88 @@
+import type { NativePoint, NativeRegion } from '../types.js';
+
+export function centerOf(region: NativeRegion): NativePoint {
+  return {
+    x: region.x + region.width / 2,
+    y: region.y + region.height / 2
+  };
+}
+
+export function normalizeRegion(region: NativeRegion): NativeRegion {
+  return {
+    x: region.x,
+    y: region.y,
+    width: Math.max(0, region.width),
+    height: Math.max(0, region.height)
+  };
+}
+
+export function isPointInsideRegion(point: NativePoint, region: NativeRegion): boolean {
+  return point.x >= region.x
+    && point.x <= region.x + region.width
+    && point.y >= region.y
+    && point.y <= region.y + region.height;
+}
+
+export function clampPointToRegion(point: NativePoint, region: NativeRegion): NativePoint {
+  return {
+    x: Math.min(Math.max(point.x, region.x), region.x + region.width),
+    y: Math.min(Math.max(point.y, region.y), region.y + region.height)
+  };
+}
+
+export function paddedRegion(region: NativeRegion, paddingPx: number): NativeRegion {
+  const x = region.x + paddingPx;
+  const y = region.y + paddingPx;
+  return normalizeRegion({
+    x,
+    y,
+    width: region.width - paddingPx * 2,
+    height: region.height - paddingPx * 2
+  });
+}
+
+export function intersects(a: NativeRegion, b: NativeRegion): boolean {
+  return a.x < b.x + b.width
+    && a.x + a.width > b.x
+    && a.y < b.y + b.height
+    && a.y + a.height > b.y;
+}
+
+export function intersection(a: NativeRegion, b: NativeRegion): NativeRegion {
+  const x = Math.max(a.x, b.x);
+  const y = Math.max(a.y, b.y);
+  const right = Math.min(a.x + a.width, b.x + b.width);
+  const bottom = Math.min(a.y + a.height, b.y + b.height);
+
+  return normalizeRegion({
+    x,
+    y,
+    width: right - x,
+    height: bottom - y
+  });
+}
+
+export function visibleEnough(region: NativeRegion, viewport: NativeRegion, minimumWidth: number, minimumHeight: number): boolean {
+  if (!intersects(region, viewport)) return false;
+  const visible = intersection(region, viewport);
+  return visible.width >= Math.min(region.width, minimumWidth) && visible.height >= Math.min(region.height, minimumHeight);
+}
+
+export function randomPointInRegion(region: NativeRegion, random: () => number, paddingPx = 0): NativePoint {
+  const safe = paddedRegion(region, paddingPx);
+  if (safe.width <= 0 || safe.height <= 0) {
+    return centerOf(region);
+  }
+
+  return {
+    x: safe.x + random() * safe.width,
+    y: safe.y + random() * safe.height
+  };
+}
+
+export function roundPoint(point: NativePoint): NativePoint {
+  return {
+    x: Math.round(point.x),
+    y: Math.round(point.y)
+  };
+}

package/src/native/utils/redactNative.ts

@@ -0,0 +1,6 @@
+import path from 'node:path';
+
+export function redactPath(filePath: string, reveal = false): string {
+  if (reveal) return filePath;
+  return path.basename(filePath) === '' ? '<path>' : `<path:${path.basename(filePath)}>`;
+}

package/src/native/utils/waitFor.ts

@@ -0,0 +1,25 @@
+import { delay } from '../../utils/delay.js';
+
+export interface WaitForOptions {
+  timeoutMs?: number;
+  intervalMs?: number;
+  description?: string;
+}
+
+export async function waitFor<T>(predicate: () => Promise<T | null | undefined | false>, options: WaitForOptions = {}): Promise<T> {
+  const timeoutMs = options.timeoutMs ?? 15_000;
+  const intervalMs = options.intervalMs ?? 100;
+  const startedAt = Date.now();
+  let lastValue: T | null | undefined | false;
+
+  do {
+    lastValue = await predicate();
+    if (lastValue !== null && lastValue !== undefined && lastValue !== false) {
+      return lastValue;
+    }
+    await delay(intervalMs);
+  } while (Date.now() - startedAt < timeoutMs);
+
+  const description = options.description ?? 'condition';
+  throw new Error(`Timed out after ${timeoutMs}ms waiting for ${description}.`);
+}

package/src/sites/upwork-com/upwork-com.actor.ts

@@ -6,6 +6,11 @@ import {parseRate} from "./upwork-com.util.js";
 import {ScrapeDashboardInput} from "../example/index.js";
 import {parseUpworkSearchResults, UpworkJobListing} from "./util/scrapeJobListing.js";
 import {Page} from "puppeteer-core";
+import {
+    fillUpworkProposalQuestions,
+    parseUpworkProposalQuestions,
+    UpworkProposalQuestions
+} from "./util/parseJobApplicationDetails.js";
 
 export const upworkComActor = defineActor({
     id: 'upwork-com',
@@ -57,34 +62,60 @@ export const upworkComActor = defineActor({
         ]
     }),
     tasks: {
+        test: async (context, input) => {
+          await context.native.mouse.moveTo({x: 100, y: 200})
+        },
         searchJobs: async (context, input: UpworkJobSearchFields = {}): Promise<UpworkJobListing[]> => {
             const path = '/nx/s/universal-search/jobs?category2_uid=531770282580668420,531770282580668419,531770282580668418&client_hires=1-9,10-&payment_verified=1&q=%27rancher%27%20or%20%27terraform%27%20or%20%27gitops%27%20or%20%27azure%27%20or%20%27microsoft%20azure%27%20or%20%27cloud%20architect%27%20or%20%27ai%20architect%27%20or%20%27forward%20deployed%20engineer%27%20or%20%27aws%27%20or%20%27aks%27%20or%20%27eks%27%20or%20%27gke%27%20or%20%27cloud%20engineer%27%20or%20devops%20or%20kuberentes%20or%20%27platform%20engineer%27%20or%20%27infrastructure%20engineer%27%20or%20"google%20cloud%20platform"%20or%20"GCP"%20or%20"langsmith"%20or%20"langgraph"%20or%20"gemini%20enterprise"&sort=recency&user_location_match=1';
 
-            await context.nav.goto(path, {
-
-            });
+            await context.nav.goto(path, {});
 
             return await parseUpworkSearchResults(context.session.page as Page);
         },
         scrapeJobs: async (context, input: ScrapeDashboardInput = {}): Promise<any> => {
             await parseUpworkSearchResults(context.session.page as Page)
         },
-        applyToJob: async (_context, input: UpworkApplyToJobInput): Promise<UpworkApplyToJobResult> => {
-            const coverLetter = input.coverLetter.trim();
-            if (coverLetter.length === 0) {
-                throw new Error('coverLetter must be a non-empty string.');
-            }
+        applyToJobPart1: async (_context, input: UpworkApplyToJobInput): Promise<UpworkProposalQuestions> => {
 
-            const rate = parseRate(input.rate);
-            if (!Number.isFinite(rate) || rate <= 0) {
-                throw new Error('rate must be a positive number.');
-            }
+            await _context.cursor.scrollIntoView(`[data-ev-job-uid="${input.jobId}"]`);
+
+            await _context.cursor.move(`[data-ev-job-uid="${input.jobId}"]`);
+            await _context.cursor.click(`[data-ev-job-uid="${input.jobId}"]`);
+
+            await _context.cursor.move(`[data-test="Apply"]>button`);
+            // await _context.cursor.click(`[data-test="Apply"]>button`);
+
+            await _context.nav.goto(`/nx/proposals/job/~${input.jobId}/apply/`);
+
+            await _context.cursor.move('.up-fe-agency-member-selector > [data-test="dropdown-toggle"]')
+            await _context.cursor.click('.up-fe-agency-member-selector > [data-test="dropdown-toggle"]')
+
+            await _context.cursor.move('li[role="option"][tabindex="0"]')
+            await _context.cursor.click('li[role="option"][tabindex="0"]')
+
+            await _context.cursor.move('.up-fe-contractor-selector > [data-test="dropdown-toggle"]')
+            await _context.cursor.click('.up-fe-contractor-selector > [data-test="dropdown-toggle"]')
+
+            await _context.cursor.move('li[role="option"][tabindex="0"]')
+            await _context.cursor.click('li[role="option"][tabindex="0"]')
+
+            await _context.cursor.scrollIntoView('[data-test="sri-form-card"] > [data-test="dropdown-toggle"] > [aria-label="How often do you want a rate increase?"]')
+
+            await _context.cursor.move('[data-test="sri-form-card"] > [data-test="dropdown-toggle"] > [aria-label="How often do you want a rate increase?"]')
+            await _context.cursor.click('[data-test="sri-form-card"] > [data-test="dropdown-toggle"] > [aria-label="How often do you want a rate increase?"]')
+            await _context.cursor.move('li[role="option"][tabindex="0"]')
+            await _context.cursor.click('li[role="option"][tabindex="0"]')
+
+
+            return await parseUpworkProposalQuestions(_context.session.page as Page);
+        },
+        applyToJobPart2: async (_context, input: UpworkApplyToJobInput): Promise<UpworkApplyToJobResult> => {
+            await fillUpworkProposalQuestions(_context.session.page as Page, input.answers)
 
             return {
-                coverLetter,
-                rate
+                coverLetter: input.answers.coverLetter ?? '',
+                rate: parseRate(input.rate)
             };
         }
     }
 });
-`asd*poq@#aes48d#@jna8sndaskDF#KN$@#a78sdASD1`

package/src/sites/upwork-com/upwork-com.types.ts

@@ -1,3 +1,5 @@
+import {UpworkProposalQuestionAnswers} from "./util/parseJobApplicationDetails.js";
+
 export enum JobSearchSort {
   BestMatch = 'best_match',
 }
@@ -92,8 +94,9 @@ export interface UpworkJobSearchResult {
 }
 
 export interface UpworkApplyToJobInput {
-  coverLetter: string;
+  jobId: string;
   rate: number | string;
+  answers: UpworkProposalQuestionAnswers
 }
 
 export interface UpworkApplyToJobResult {