@blario/react-native 0.0.0-stage → 0.1.0

package/src/actions/commit-watch.ts

@@ -0,0 +1,86 @@
+import { fiberRootOf } from '../fiber/reader.ts';
+import type { Fiber, FiberRoot } from '../fiber/types.ts';
+
+export class CommitWatch {
+  private watchedRoot: FiberRoot | null = null;
+
+  private originalDescriptor: PropertyDescriptor | null = null;
+
+  private watchedSetter: ((next: Fiber) => void) | null = null;
+
+  private commits = 0;
+
+  private baseline = 0;
+
+  constructor(
+    private readonly getAnchorFiber: () => Fiber | null,
+    private readonly onCommit?: () => void,
+  ) {}
+
+  begin(): void {
+    this.install();
+    this.baseline = this.commits;
+  }
+
+  hasCommitted(): boolean {
+    return this.commits > this.baseline;
+  }
+
+  release(): void {
+    const root = this.watchedRoot;
+    if (root === null) {
+      return;
+    }
+    const descriptor = this.originalDescriptor;
+    const setter = this.watchedSetter;
+    this.watchedRoot = null;
+    this.originalDescriptor = null;
+    this.watchedSetter = null;
+    try {
+      if (descriptor !== null && Object.getOwnPropertyDescriptor(root, 'current')?.set === setter) {
+        Object.defineProperty(root, 'current', { ...descriptor, value: root.current });
+      }
+    } catch {
+      return;
+    }
+  }
+
+  private install(): void {
+    const anchor = this.getAnchorFiber();
+    const root = anchor === null ? null : fiberRootOf(anchor);
+    if (root === null || root === this.watchedRoot) {
+      return;
+    }
+    this.release();
+    const descriptor = Object.getOwnPropertyDescriptor(root, 'current');
+    if (descriptor === undefined || !descriptor.configurable || !descriptor.writable || !('value' in descriptor)) {
+      return;
+    }
+    let latest = root.current;
+    const setter = (next: Fiber): void => {
+      const committed = next !== latest;
+      latest = next;
+      if (committed && this.watchedRoot === root) {
+        this.commits += 1;
+        try {
+          this.onCommit?.();
+        } catch {
+          return;
+        }
+      }
+    };
+    try {
+      Object.defineProperty(root, 'current', {
+        configurable: true,
+        enumerable: descriptor.enumerable,
+        get: () => latest,
+        set: setter,
+      });
+    } catch {
+      return;
+    }
+    this.watchedRoot = root;
+    this.originalDescriptor = descriptor;
+    this.watchedSetter = setter;
+  }
+}

package/src/actions/events.ts

@@ -0,0 +1,58 @@
+import { type Rect, centerOf } from '../host/rect.ts';
+
+export interface SyntheticPressEvent {
+  nativeEvent: {
+    pageX: number;
+    pageY: number;
+    locationX: number;
+    locationY: number;
+    timestamp: number;
+    target: number;
+    touches: never[];
+    changedTouches: never[];
+  };
+  persist: () => void;
+  preventDefault: () => void;
+  stopPropagation: () => void;
+  isPropagationStopped: () => boolean;
+  isDefaultPrevented: () => boolean;
+}
+
+export interface SyntheticTextChangeEvent {
+  nativeEvent: { text: string; eventCount: number; target: number };
+}
+
+export interface SyntheticSubmitEvent {
+  nativeEvent: { text: string; eventCount: number; target: number };
+}
+
+function noop(): void {}
+
+export function buildPressEvent(rect: Rect): SyntheticPressEvent {
+  const center = centerOf(rect);
+  return {
+    nativeEvent: {
+      pageX: center.x,
+      pageY: center.y,
+      locationX: rect.width / 2,
+      locationY: rect.height / 2,
+      timestamp: Date.now(),
+      target: 0,
+      touches: [],
+      changedTouches: [],
+    },
+    persist: noop,
+    preventDefault: noop,
+    stopPropagation: noop,
+    isPropagationStopped: () => false,
+    isDefaultPrevented: () => false,
+  };
+}
+
+export function buildTextChangeEvent(text: string): SyntheticTextChangeEvent {
+  return { nativeEvent: { text, eventCount: 1, target: 0 } };
+}
+
+export function buildSubmitEvent(text: string): SyntheticSubmitEvent {
+  return { nativeEvent: { text, eventCount: 1, target: 0 } };
+}

package/src/actions/executor.ts

@@ -0,0 +1,147 @@
+import { SnapshotCapturer } from '../snapshot/capture.ts';
+import { click } from './click.ts';
+import type { CommitWatch } from './commit-watch.ts';
+import { highlight } from './highlight.ts';
+import { tapsTarget, waitForLanding, waitsForLanding } from './landing.ts';
+import { failed, failedFromException } from './result.ts';
+import { scrollContainer, scrollIntoView } from './scrolling.ts';
+import { SYSTEM_CLOCK } from './settle.ts';
+import type { ActionServices } from './services.ts';
+import { waitForSettle } from './settle.ts';
+import { toggleSwitch } from './switch.ts';
+import { measureTarget, type ResolvedTarget } from './target.ts';
+import { submitInput, typeText } from './text-input.ts';
+import type { ActionOutcome, AgentAction, AgentCommandData, FailedResult, HandlerResult } from './types.ts';
+
+const ACTIONS_WITHOUT_TARGET: readonly AgentAction['type'][] = ['navigate', 'press_key'];
+const ACTIONS_NEEDING_VISIBLE_TARGET: readonly AgentAction['type'][] = ['click', 'type', 'check', 'highlight'];
+const REVEAL_SETTLE_MS = 150;
+
+export interface ExecutionOptions {
+  isCancelled: () => boolean;
+}
+
+export type Preparation = { ok: true; target: ResolvedTarget | null } | { ok: false; outcome: ActionOutcome };
+
+const CANCELLED_MESSAGE = 'Cancelled by the user before the action ran';
+
+function outcomeOf(result: FailedResult): ActionOutcome {
+  return { success: false, no_op: false, error: result.error, message: result.message };
+}
+
+export class ActionExecutor {
+  constructor(
+    private readonly capturer: SnapshotCapturer,
+    private readonly services: ActionServices,
+    private readonly commitWatch: CommitWatch,
+  ) {}
+
+  async prepare(command: AgentCommandData): Promise<Preparation> {
+    if (command.action.type === 'navigate') {
+      return { ok: true, target: null };
+    }
+    const target = this.resolveTarget(command);
+    if (target === null) {
+      return { ok: false, outcome: outcomeOf(this.missingTarget(command)) };
+    }
+    try {
+      await this.revealTarget(command.action, target);
+    } catch (error) {
+      return { ok: false, outcome: outcomeOf(failedFromException(error)) };
+    }
+    return { ok: true, target };
+  }
+
+  async execute(command: AgentCommandData, target: ResolvedTarget | null, options: ExecutionOptions): Promise<ActionOutcome> {
+    const before = (await this.capturer.capture('peek')).text;
+    this.commitWatch.begin();
+    const result = await this.perform(command, target, options);
+    if (!result.ok) {
+      return outcomeOf(result);
+    }
+    const after = await waitForSettle(async () => (await this.capturer.capture('peek')).text);
+    const expectsChange = command.action.type !== 'highlight';
+    const unchanged = before === after && !this.commitWatch.hasCommitted();
+    return { success: true, no_op: expectsChange && unchanged, message: result.message };
+  }
+
+  private async perform(command: AgentCommandData, target: ResolvedTarget | null, options: ExecutionOptions): Promise<HandlerResult> {
+    try {
+      const { action } = command;
+      if (action.type === 'navigate') {
+        return options.isCancelled() ? failed('user_cancelled', CANCELLED_MESSAGE) : this.services.navigation.navigateTo(action.route, action.params);
+      }
+      if (target === null) {
+        return this.missingTarget(command);
+      }
+      await this.pointAt(action, target);
+      if (options.isCancelled()) {
+        return failed('user_cancelled', CANCELLED_MESSAGE);
+      }
+      return await this.dispatch(command, target);
+    } catch (error) {
+      return failedFromException(error);
+    }
+  }
+
+  private async revealTarget(action: AgentAction, target: ResolvedTarget): Promise<void> {
+    if (!ACTIONS_NEEDING_VISIBLE_TARGET.includes(action.type)) {
+      return;
+    }
+    const revealed = await scrollIntoView(target, { animated: false });
+    if (revealed.ok && revealed.message === undefined) {
+      await SYSTEM_CLOCK.sleep(REVEAL_SETTLE_MS);
+    }
+  }
+
+  private async pointAt(action: AgentAction, target: ResolvedTarget): Promise<void> {
+    if (!tapsTarget(action)) {
+      return;
+    }
+    this.services.cursor.moveTo(measureTarget(target));
+    if (waitsForLanding(action)) {
+      await waitForLanding(this.services.cursor, SYSTEM_CLOCK);
+    }
+  }
+
+  private resolveTarget(command: AgentCommandData): ResolvedTarget | null {
+    if (command.agent_id !== null) {
+      return this.services.resolver.resolveAgentId(command.agent_id);
+    }
+    const remembered = this.services.inputMemory.last;
+    if (command.action.type === 'press_key' && remembered !== null) {
+      return this.services.resolver.resolveElement(remembered);
+    }
+    return null;
+  }
+
+  private missingTarget(command: AgentCommandData): FailedResult {
+    if (command.agent_id === null && !ACTIONS_WITHOUT_TARGET.includes(command.action.type)) {
+      return failed('element_not_found', 'target.agent_id is required for this action');
+    }
+    const reason = command.agent_id === null ? 'no input has been used yet' : `${command.agent_id} is not in the latest capture or is no longer mounted`;
+    return failed('element_not_found', reason);
+  }
+
+  private dispatch(command: AgentCommandData, target: ResolvedTarget): Promise<HandlerResult> | HandlerResult {
+    const { action } = command;
+    switch (action.type) {
+      case 'click':
+        return click(target, this.services);
+      case 'type':
+        return typeText(target, action, this.services);
+      case 'check':
+        return toggleSwitch(target, action.checked);
+      case 'press_key':
+        return action.key === 'Enter' ? submitInput(target) : failed('unsupported_action', `key ${action.key} is not supported`);
+      case 'scroll':
+        return scrollContainer(target, action);
+      case 'scroll_to':
+        return scrollIntoView(target);
+      case 'highlight':
+        return highlight(target, command.teaching_explanation, this.services);
+      case 'navigate':
+        return failed('unsupported_action', 'navigate does not take a target');
+    }
+  }
+}

package/src/actions/highlight.ts

@@ -0,0 +1,13 @@
+import { failed, succeeded } from './result.ts';
+import type { ActionServices } from './services.ts';
+import { measureTarget, type ResolvedTarget, viewOf } from './target.ts';
+import type { HandlerResult } from './types.ts';
+
+export async function highlight(target: ResolvedTarget, text: string | null, services: ActionServices): Promise<HandlerResult> {
+  const rect = (await measureTarget(target)()) ?? target.element.rect;
+  if (rect.width <= 0 || rect.height <= 0) {
+    return failed('element_not_found', 'the element has no on-screen size');
+  }
+  const completed = await services.reading.read({ ...viewOf(target), text });
+  return completed ? succeeded() : failed('user_cancelled', 'User cancelled during highlight');
+}

package/src/actions/interaction-guards.ts

@@ -0,0 +1,44 @@
+import { isRecord } from '../fiber/narrow.ts';
+import { propsOf } from '../fiber/reader.ts';
+import type { Fiber } from '../fiber/types.ts';
+import { accessibilityFlagsOf } from '../snapshot/properties.ts';
+
+export type GuardVerdict = { allowed: true } | { allowed: false; reason: string };
+
+const ALLOWED: GuardVerdict = { allowed: true };
+
+function pointerEventsOf(fiber: Fiber): string | null {
+  const props = propsOf(fiber);
+  if (typeof props.pointerEvents === 'string') {
+    return props.pointerEvents;
+  }
+  const style = props.style;
+  return isRecord(style) && !Array.isArray(style) && typeof style.pointerEvents === 'string' ? style.pointerEvents : null;
+}
+
+function blocksPointerEvents(fiber: Fiber, isTarget: boolean): boolean {
+  const pointerEvents = pointerEventsOf(fiber);
+  if (pointerEvents === 'none') {
+    return true;
+  }
+  return !isTarget && pointerEvents === 'box-only';
+}
+
+export function checkInteractable(fiber: Fiber, ancestors: Fiber[]): GuardVerdict {
+  if (accessibilityFlagsOf(propsOf(fiber)).disabled) {
+    return { allowed: false, reason: 'the element is disabled' };
+  }
+  if (blocksPointerEvents(fiber, true)) {
+    return { allowed: false, reason: 'the element has pointerEvents none' };
+  }
+  const blocker = ancestors.find((ancestor) => blocksPointerEvents(ancestor, false));
+  return blocker === undefined ? ALLOWED : { allowed: false, reason: 'an ancestor blocks pointer events' };
+}
+
+export function checkEditable(fiber: Fiber, ancestors: Fiber[]): GuardVerdict {
+  const verdict = checkInteractable(fiber, ancestors);
+  if (!verdict.allowed) {
+    return verdict;
+  }
+  return propsOf(fiber).editable === false ? { allowed: false, reason: 'the input is not editable' } : ALLOWED;
+}

package/src/actions/landing.ts

@@ -0,0 +1,26 @@
+import type { Rect } from '../host/rect.ts';
+import type { Clock } from './settle.ts';
+import type { AgentAction } from './types.ts';
+
+export interface CursorPort {
+  moveTo: (measure: () => Promise<Rect | null>) => void;
+  whenLanded: () => Promise<void>;
+}
+
+export const LANDING_MAX_MS = 1800;
+
+const TAP_ACTIONS: readonly AgentAction['type'][] = ['click', 'check', 'type'];
+
+const LANDING_ACTIONS: readonly AgentAction['type'][] = ['click', 'check'];
+
+export function tapsTarget(action: AgentAction): boolean {
+  return TAP_ACTIONS.includes(action.type);
+}
+
+export function waitsForLanding(action: AgentAction): boolean {
+  return LANDING_ACTIONS.includes(action.type);
+}
+
+export async function waitForLanding(cursor: CursorPort, clock: Clock): Promise<void> {
+  await Promise.race([cursor.whenLanded(), clock.sleep(LANDING_MAX_MS)]);
+}

package/src/actions/navigation-driver.ts

@@ -0,0 +1,131 @@
+import { isFunction, isRecord, type UnknownRecord } from '../fiber/narrow.ts';
+import { routeNamesOf, type ScreenRef } from '../snapshot/navigation.ts';
+import { failed, succeeded } from './result.ts';
+import type { HandlerResult } from './types.ts';
+
+export interface NavigationRefLike {
+  isReady: () => boolean;
+  navigate(...args: unknown[]): void;
+  getRootState: () => unknown;
+}
+
+export interface NavigationSources {
+  readFocusedScreens: () => ScreenRef[];
+  getNavigationRef: () => NavigationRefLike | null;
+}
+
+function routeStatesIn(state: unknown): UnknownRecord[] {
+  if (!isRecord(state) || !Array.isArray(state.routes)) {
+    return [];
+  }
+  return state.routes.filter(isRecord);
+}
+
+function pathToRoute(state: unknown, target: string): string[] | null {
+  for (const route of routeStatesIn(state)) {
+    if (route.name === target) {
+      return [target];
+    }
+    const nested = pathToRoute(route.state, target);
+    if (nested !== null && typeof route.name === 'string') {
+      return [route.name, ...nested];
+    }
+  }
+  return null;
+}
+
+function nestedNavigateArguments(path: string[], params: UnknownRecord | null): [string, UnknownRecord | undefined] {
+  const [first, ...rest] = path;
+  if (first === undefined) {
+    return ['', undefined];
+  }
+  if (rest.length === 0) {
+    return [first, params ?? undefined];
+  }
+  const [screen, nestedParams] = nestedNavigateArguments(rest, params);
+  return [first, { screen, params: nestedParams }];
+}
+
+export class NavigationDriver {
+  constructor(private readonly sources: NavigationSources) {}
+
+  listRefRoutePaths(): string[] {
+    const ref = this.sources.getNavigationRef();
+    if (ref === null || !ref.isReady()) {
+      return [];
+    }
+    const paths = new Set<string>();
+    this.collectRoutePaths(ref.getRootState(), [], paths);
+    return [...paths];
+  }
+
+  navigateTo(route: string, params: UnknownRecord | null): HandlerResult {
+    const path = route.split('/').filter((segment) => segment.length > 0);
+    if (path.length > 1) {
+      return this.navigateAlongPath(path, params);
+    }
+    const direct = this.navigateThroughFocusedScreens(route, params);
+    if (direct.ok) {
+      return direct;
+    }
+    return this.navigateThroughRef(route, params) ?? direct;
+  }
+
+  private navigateAlongPath(path: string[], params: UnknownRecord | null): HandlerResult {
+    const [head] = path;
+    const innermostFirst = [...this.sources.readFocusedScreens()].reverse();
+    const owner = innermostFirst.find((screen) => head !== undefined && routeNamesOf(screen.navigation).includes(head));
+    if (owner === undefined) {
+      return failed('element_not_found', `route "${head}" is not reachable from the focused screens`);
+    }
+    try {
+      const [name, nested] = nestedNavigateArguments(path, params);
+      owner.navigation.navigate(...(nested === undefined ? [name] : [name, nested]));
+      return succeeded();
+    } catch (error) {
+      return failed('execution_error', error instanceof Error ? error.message : String(error));
+    }
+  }
+
+  private collectRoutePaths(state: unknown, prefix: string[], paths: Set<string>): void {
+    for (const route of routeStatesIn(state)) {
+      if (typeof route.name === 'string') {
+        const path = [...prefix, route.name];
+        paths.add(path.join('/'));
+        this.collectRoutePaths(route.state, path, paths);
+      }
+    }
+  }
+
+  private navigateThroughFocusedScreens(route: string, params: UnknownRecord | null): HandlerResult {
+    const innermostFirst = [...this.sources.readFocusedScreens()].reverse();
+    const owner = innermostFirst.find((screen) => routeNamesOf(screen.navigation).includes(route));
+    if (owner === undefined) {
+      return failed('element_not_found', `route "${route}" is not reachable from the focused screens`);
+    }
+    try {
+      owner.navigation.navigate(...(params === null ? [route] : [route, params]));
+      return succeeded();
+    } catch (error) {
+      return failed('execution_error', error instanceof Error ? error.message : String(error));
+    }
+  }
+
+  private navigateThroughRef(route: string, params: UnknownRecord | null): HandlerResult | null {
+    const ref = this.sources.getNavigationRef();
+    if (ref === null || !ref.isReady() || !isFunction(ref.navigate)) {
+      return null;
+    }
+    const path = pathToRoute(ref.getRootState(), route);
+    if (path === null) {
+      return null;
+    }
+    try {
+      const [name, nested] = nestedNavigateArguments(path, params);
+      ref.navigate(...(nested === undefined ? [name] : [name, nested]));
+      return succeeded();
+    } catch (error) {
+      return failed('execution_error', error instanceof Error ? error.message : String(error));
+    }
+  }
+}

package/src/actions/parse.ts

@@ -0,0 +1,82 @@
+import { isRecord, nonEmptyString } from '../fiber/narrow.ts';
+import { type AgentAction, type AgentCommandData, type PageContentRequest, type RiskLevel, type ScrollDirectionName } from './types.ts';
+
+const DIRECTIONS: readonly ScrollDirectionName[] = ['up', 'down', 'left', 'right'];
+const RISK_LEVELS: readonly RiskLevel[] = ['safe', 'reversible', 'irreversible', 'destructive'];
+const NO_LIMIT = 0;
+
+function isDirection(value: unknown): value is ScrollDirectionName {
+  return DIRECTIONS.some((direction) => direction === value);
+}
+
+function parseRiskLevel(value: unknown): RiskLevel {
+  return RISK_LEVELS.find((level) => level === value) ?? 'reversible';
+}
+
+export function parsePageContentRequest(options: unknown): PageContentRequest {
+  const record = isRecord(options) ? options : {};
+  return {
+    viewportOnly: record.viewport_only === true,
+    maxChars: typeof record.max_chars === 'number' ? record.max_chars : NO_LIMIT,
+  };
+}
+
+function parseAction(raw: unknown): AgentAction | null {
+  if (!isRecord(raw)) {
+    return null;
+  }
+  switch (raw.type) {
+    case 'click':
+      return { type: 'click' };
+    case 'type':
+      return typeof raw.value === 'string' ? { type: 'type', value: raw.value, clear_first: raw.clear_first !== false } : null;
+    case 'check':
+      return { type: 'check', checked: typeof raw.checked === 'boolean' ? raw.checked : null };
+    case 'press_key':
+      return typeof raw.key === 'string' ? { type: 'press_key', key: raw.key } : null;
+    case 'scroll':
+      return isDirection(raw.direction)
+        ? { type: 'scroll', direction: raw.direction, amount: typeof raw.amount === 'number' ? raw.amount : null }
+        : null;
+    case 'scroll_to':
+      return { type: 'scroll_to' };
+    case 'navigate': {
+      const route = nonEmptyString(raw.route) ?? nonEmptyString(raw.name) ?? nonEmptyString(raw.url);
+      return route === null ? null : { type: 'navigate', route, params: isRecord(raw.params) ? raw.params : null };
+    }
+    case 'highlight':
+      return { type: 'highlight' };
+    default:
+      return null;
+  }
+}
+
+export function parseAgentCommand(data: unknown): AgentCommandData | null {
+  if (!isRecord(data)) {
+    return null;
+  }
+  const commandId = nonEmptyString(data.command_id);
+  const action = parseAction(data.action);
+  if (commandId === null || action === null) {
+    return null;
+  }
+  const agentId = isRecord(data.target) ? nonEmptyString(data.target.agent_id) : null;
+  return {
+    command_id: commandId,
+    action,
+    agent_id: agentId,
+    risk_level: parseRiskLevel(data.risk_level),
+    confirmation_required: data.confirmation_required === true,
+    teaching_explanation: nonEmptyString(data.teaching_explanation),
+    include_page_content: data.include_page_content === true,
+    page_content_options: parsePageContentRequest(data.page_content_options),
+    dom_settle_ms: typeof data.dom_settle_ms === 'number' && data.dom_settle_ms > 0 ? data.dom_settle_ms : 0,
+  };
+}
+
+export function describeUnparsedAction(data: unknown): string {
+  if (!isRecord(data) || !isRecord(data.action)) {
+    return 'agent_command without an action';
+  }
+  return `unsupported or malformed action "${String(data.action.type)}"`;
+}

package/src/actions/result.ts

@@ -0,0 +1,14 @@
+import type { ActionErrorCode, FailedResult, HandlerResult } from './types.ts';
+
+export function succeeded(message?: string): HandlerResult {
+  return message === undefined ? { ok: true } : { ok: true, message };
+}
+
+export function failed(error: ActionErrorCode, message: string): FailedResult {
+  return { ok: false, error, message };
+}
+
+export function failedFromException(error: unknown): FailedResult {
+  const message = error instanceof Error ? error.message : String(error);
+  return failed('execution_error', message);
+}