@blario/react-native 0.1.3 → 0.1.5

package/README.md

@@ -284,6 +284,16 @@ npx expo start --clear                     # Metro; the simulator reaches 127.0.
 xcrun simctl launch "iPhone 17 iOS 26" io.blar.example
 ```
 
+### On an Android emulator
+
+```bash
+npx expo prebuild -p android --no-install   # it rewrites package.json's scripts: revert that
+# newArchEnabled=true in android/gradle.properties, and sdk.dir in android/local.properties
+(cd android && ./gradlew app:assembleDebug)
+adb install -r android/app/build/outputs/apk/debug/app-debug.apk
+adb reverse tcp:8081 tcp:8081 && adb reverse tcp:8790 tcp:8790
+```
+
 `.maestro/` holds [Maestro](https://maestro.dev) flows that tap through the app, chat WebView included
 (`brew install mobile-dev-inc/tap/maestro`, needs Java 17): `maestro test .maestro/chat-keyboard.yaml`.
 
@@ -312,9 +322,10 @@ Wire envelope (mirrors the backend): `agent_command` / `request_dom_state` / `re
 
 | `action.type` | Fields | Target |
 | --- | --- | --- |
-| `click` | | `btn-N` (a switch toggles, an input is focused) |
+| `click` | | `btn-N` (a switch toggles, an input is focused, a wheel picker's row scrolls to the center) |
 | `type` | `value`, `clear_first` (default true) | `inp-N` text input |
 | `check` | `checked` (omit to toggle) | `inp-N` switch |
+| `select` | `value` | `scr-N` or one of its rows: centers the row with that text, in one list at a time |
 | `press_key` | `key` (`Enter` only) | `inp-N`, or the last input used |
 | `scroll` | `direction` up/down/left/right, `amount` px | `scr-N` |
 | `scroll_to` | | any element, uses its nearest scroll container |
@@ -322,7 +333,10 @@ Wire envelope (mirrors the backend): `agent_command` / `request_dom_state` / `re
 | `highlight` | `duration_ms` (default 3000) | any element |
 
 `click`, `type`, `check` and `highlight` scroll the target into view first. Elements that sit beyond their scroll
-viewport are listed with `offscreen=above|below|left|right` (at most 30, nearest first).
+viewport are listed with `offscreen=above|below|left|right` (at most 30, nearest first). A wheel picker is a snapping
+list (`snapToInterval` or `snapToOffsets`) of plain text rows: each row is a `btn-N`, and all of them stay listed. Lists
+carry `list=<name>` from their own or their wrapper's `accessibilityLabel` or `testID`, and so do those rows. `select`
+needs no snap props: it reaches any list, so a wheel that snaps in JS works through it.
 
 ## Develop
 

package/dist/actions/scrolling.d.ts

@@ -1,10 +1,14 @@
 import { type Band } from '../host/rect.ts';
+import { type ScrollGeometry } from '../snapshot/scroll.ts';
 import type { ResolvedTarget } from './target.ts';
 import type { AgentAction, HandlerResult } from './types.ts';
 type ScrollAction = Extract<AgentAction, {
     type: 'scroll';
 }>;
+export declare function applyOffset(target: ResolvedTarget, geometry: ScrollGeometry, offset: number, animated: boolean): HandlerResult;
 export declare function scrollContainer(target: ResolvedTarget, action: ScrollAction): Promise<HandlerResult>;
+export declare function nearestScrollAncestor(target: ResolvedTarget): ResolvedTarget['fiber'] | null;
+export declare function isInWheel(target: ResolvedTarget): boolean;
 export declare function scrollIntoView(target: ResolvedTarget, options?: {
     animated: boolean;
 }): Promise<HandlerResult>;

package/dist/actions/select.d.ts

@@ -0,0 +1,16 @@
+import type { Fiber } from '../fiber/types.ts';
+import type { ActionServices } from './services.ts';
+import type { ResolvedTarget } from './target.ts';
+import type { HandlerResult } from './types.ts';
+interface ListRow {
+    host: Fiber;
+    text: string;
+}
+export declare function centerInList(target: ResolvedTarget, list: Fiber, row: Fiber): Promise<HandlerResult>;
+export declare function scrollListTo(target: ResolvedTarget, list: Fiber, offset: number): Promise<HandlerResult>;
+export declare function selectWheelOption(target: ResolvedTarget): Promise<HandlerResult>;
+export declare function rowsOf(list: Fiber): ListRow[];
+export declare function matchesRow(text: string, value: string): boolean;
+export declare function selectValue(target: ResolvedTarget, value: string, services: ActionServices): Promise<HandlerResult>;
+export declare function walk(from: number, to: number, step: number): number[];
+export {};

package/dist/actions/types.d.ts

@@ -9,6 +9,9 @@ export type AgentAction = {
 } | {
     type: 'check';
     checked: boolean | null;
+} | {
+    type: 'select';
+    value: string;
 } | {
     type: 'press_key';
     key: string;

package/dist/backend/app-info.d.ts

@@ -1,5 +1,6 @@
 export interface AppInfo {
     platform: 'react-native';
+    sdk_version: string;
     os: string;
     os_version: string;
     rn_version: string;

package/dist/fiber/text.d.ts

@@ -1,4 +1,5 @@
 import type { Fiber } from './types.ts';
+export declare const TEXT_HOST_TYPE = "RCTText";
 export declare function normalizeText(value: string): string;
 export declare function collectText(fiber: Fiber, limit: number): string;
 export declare function precedingText(ancestors: Fiber[], fiber: Fiber, limit: number): string;

package/dist/snapshot/collect.d.ts

@@ -8,6 +8,7 @@ export interface Candidate {
     host: Fiber;
     ancestors: Fiber[];
     scrollContainers: Fiber[];
+    wheelOption?: boolean;
 }
 export interface TextBlock {
     host: Fiber;

package/dist/snapshot/scroll.d.ts

@@ -7,9 +7,11 @@ export interface ScrollGeometry {
     horizontal: boolean;
 }
 export declare function isHorizontal(fiber: Fiber): boolean;
+export declare function isWheel(fiber: Fiber): boolean;
 export declare function measureScrollGeometry(scrollFiber: Fiber): Promise<ScrollGeometry | null>;
 export declare function scrollOffsetOf(geometry: ScrollGeometry): number;
 export declare function maxScrollOffsetOf(geometry: ScrollGeometry): number;
 export declare function viewportExtentOf(geometry: ScrollGeometry): number;
+export declare function offsetCentering(rect: Rect, geometry: ScrollGeometry): number;
 export declare function scrollDirectionsOf(geometry: ScrollGeometry): ScrollDirection[];
 export declare function describeScrollDirections(geometry: ScrollGeometry | null): string;

package/dist/snapshot/view.d.ts

@@ -17,8 +17,10 @@ export interface ElementView {
     selected: boolean | null;
     scroll: string | null;
     offscreen: Offscreen | null;
+    list: string | null;
     rect: Rect;
 }
+export declare function listNameOf(list: Fiber, ancestors: Fiber[]): string | null;
 export interface ViewSource {
     kind: ElementKind;
     fiber: Fiber;
@@ -26,5 +28,6 @@ export interface ViewSource {
     rect: Rect;
     scroll: string | null;
     offscreen?: Offscreen | null;
+    wheelOption?: boolean;
 }
 export declare function buildView(source: ViewSource): ElementView;

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const SDK_VERSION = "0.1.3";
+export declare const SDK_VERSION = "0.1.5";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blario/react-native",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Blar agent SDK for React Native",
   "license": "MIT",
   "repository": {

package/src/actions/click.ts

@@ -6,6 +6,8 @@ import { buildPressEvent } from './events.ts';
 import { checkInteractable } from './interaction-guards.ts';
 import { failed, failedFromException, succeeded } from './result.ts';
 import type { ActionServices } from './services.ts';
+import { isInWheel } from './scrolling.ts';
+import { selectWheelOption } from './select.ts';
 import type { ResolvedTarget } from './target.ts';
 import type { HandlerResult } from './types.ts';
 import { toggleSwitch } from './switch.ts';
@@ -49,6 +51,6 @@ export async function click(target: ResolvedTarget, services: ActionServices): P
     case 'text_input':
       return focusInput(target, services);
     default:
-      return pressButton(target);
+      return !isFunction(target.props.onPress) && isInWheel(target) ? selectWheelOption(target) : pressButton(target);
   }
 }

package/src/actions/executor.ts

@@ -5,6 +5,7 @@ import { highlight } from './highlight.ts';
 import { tapsTarget, waitForLanding, waitsForLanding } from './landing.ts';
 import { failed, failedFromException } from './result.ts';
 import { scrollContainer, scrollIntoView } from './scrolling.ts';
+import { selectValue } from './select.ts';
 import { SYSTEM_CLOCK } from './settle.ts';
 import type { ActionServices } from './services.ts';
 import { waitForSettle } from './settle.ts';
@@ -132,6 +133,8 @@ export class ActionExecutor {
         return typeText(target, action, this.services);
       case 'check':
         return toggleSwitch(target, action.checked);
+      case 'select':
+        return selectValue(target, action.value, this.services);
       case 'press_key':
         return action.key === 'Enter' ? submitInput(target) : failed('unsupported_action', `key ${action.key} is not supported`);
       case 'scroll':

package/src/actions/landing.ts

@@ -9,9 +9,9 @@ export interface CursorPort {
 
 export const LANDING_MAX_MS = 1800;
 
-const TAP_ACTIONS: readonly AgentAction['type'][] = ['click', 'check', 'type'];
+const TAP_ACTIONS: readonly AgentAction['type'][] = ['click', 'check', 'select', 'type'];
 
-const LANDING_ACTIONS: readonly AgentAction['type'][] = ['click', 'check'];
+const LANDING_ACTIONS: readonly AgentAction['type'][] = ['click', 'check', 'select'];
 
 export function tapsTarget(action: AgentAction): boolean {
   return TAP_ACTIONS.includes(action.type);

package/src/actions/parse.ts

@@ -32,6 +32,8 @@ function parseAction(raw: unknown): AgentAction | null {
       return typeof raw.value === 'string' ? { type: 'type', value: raw.value, clear_first: raw.clear_first !== false } : null;
     case 'check':
       return { type: 'check', checked: typeof raw.checked === 'boolean' ? raw.checked : null };
+    case 'select':
+      return typeof raw.value === 'string' ? { type: 'select', value: raw.value } : null;
     case 'press_key':
       return typeof raw.key === 'string' ? { type: 'press_key', key: raw.key } : null;
     case 'scroll':

package/src/actions/scrolling.ts

@@ -1,7 +1,7 @@
 import { measureHost } from '../host/measure.ts';
 import { type Band, scrollDeltaIntoBand } from '../host/rect.ts';
 import { scrollInstanceOf } from '../snapshot/classify.ts';
-import { maxScrollOffsetOf, measureScrollGeometry, scrollOffsetOf, type ScrollGeometry, viewportExtentOf } from '../snapshot/scroll.ts';
+import { isWheel, maxScrollOffsetOf, measureScrollGeometry, scrollOffsetOf, type ScrollGeometry, viewportExtentOf } from '../snapshot/scroll.ts';
 import { failed, failedFromException, succeeded } from './result.ts';
 import type { ResolvedTarget } from './target.ts';
 import type { AgentAction, HandlerResult } from './types.ts';
@@ -16,7 +16,7 @@ function clamp(value: number, minimum: number, maximum: number): number {
   return Math.min(Math.max(value, minimum), maximum);
 }
 
-function applyOffset(target: ResolvedTarget, geometry: ScrollGeometry, offset: number, animated: boolean): HandlerResult {
+export function applyOffset(target: ResolvedTarget, geometry: ScrollGeometry, offset: number, animated: boolean): HandlerResult {
   const instance = scrollInstanceOf(target.fiber);
   if (instance === null) {
     return failed('element_not_interactable', 'the scroll container has no scrollTo');
@@ -51,7 +51,7 @@ export async function scrollContainer(target: ResolvedTarget, action: ScrollActi
   return applyOffset(target, geometry, next, true);
 }
 
-function nearestScrollAncestor(target: ResolvedTarget): ResolvedTarget['fiber'] | null {
+export function nearestScrollAncestor(target: ResolvedTarget): ResolvedTarget['fiber'] | null {
   for (let index = target.ancestors.length - 1; index >= 0; index -= 1) {
     const ancestor = target.ancestors[index];
     if (ancestor !== undefined && scrollInstanceOf(ancestor) !== null) {
@@ -61,6 +61,11 @@ function nearestScrollAncestor(target: ResolvedTarget): ResolvedTarget['fiber']
   return null;
 }
 
+export function isInWheel(target: ResolvedTarget): boolean {
+  const container = nearestScrollAncestor(target);
+  return container !== null && isWheel(container);
+}
+
 function offsetDeltaToReveal(start: number, end: number, viewportStart: number, viewportEnd: number): number {
   const startShortfall = start - (viewportStart + VISIBLE_MARGIN);
   if (startShortfall < 0) {

package/src/actions/select.ts

@@ -0,0 +1,162 @@
+import { Platform } from 'react-native';
+
+import { isFunction } from '../fiber/narrow.ts';
+import { childrenOf, isHostComponent, propsOf } from '../fiber/reader.ts';
+import { collectText, TEXT_HOST_TYPE } from '../fiber/text.ts';
+import type { Fiber } from '../fiber/types.ts';
+import { measureHost } from '../host/measure.ts';
+import { maxScrollOffsetOf, measureScrollGeometry, offsetCentering, type ScrollGeometry, scrollOffsetOf, viewportExtentOf } from '../snapshot/scroll.ts';
+import { runBatched } from './batched.ts';
+import { failed, succeeded } from './result.ts';
+import { applyOffset, isInWheel, nearestScrollAncestor } from './scrolling.ts';
+import { SYSTEM_CLOCK } from './settle.ts';
+import type { ActionServices } from './services.ts';
+import type { ResolvedTarget } from './target.ts';
+import type { HandlerResult } from './types.ts';
+
+const ROW_TEXT_LIMIT = 80;
+const MOUNT_WAIT_MS = 600;
+const WALK_STEP_FRACTION = 3;
+const WALK_SETTLE_MS = 120;
+const MOUNT_POLL_MS = 150;
+const LISTED_OPTIONS = 30;
+const MOVEMENT_TOLERANCE = 1;
+const SCROLL_ANIMATION_MS = 400;
+
+interface ListRow {
+  host: Fiber;
+  text: string;
+}
+
+// iOS ends an animated scrollTo in onMomentumScrollEnd, the event a wheel commits its value on, as a user's swipe does.
+// Android only fires it after a fling, so the wheel is told here.
+function commitScrollEnd(list: Fiber, geometry: ScrollGeometry, offset: number): void {
+  const onMomentumScrollEnd = propsOf(list).onMomentumScrollEnd;
+  if (Platform.OS !== 'android' || !isFunction(onMomentumScrollEnd)) {
+    return;
+  }
+  const { viewport, content, horizontal } = geometry;
+  runBatched(() =>
+    onMomentumScrollEnd({
+      nativeEvent: {
+        contentOffset: horizontal ? { x: offset, y: 0 } : { x: 0, y: offset },
+        contentSize: { width: content.width, height: content.height },
+        layoutMeasurement: { width: viewport.width, height: viewport.height },
+      },
+    }),
+  );
+}
+
+export async function centerInList(target: ResolvedTarget, list: Fiber, row: Fiber): Promise<HandlerResult> {
+  const [geometry, rect] = await Promise.all([measureScrollGeometry(list), measureHost(row)]);
+  if (geometry === null || rect === null) {
+    return failed('element_not_found', 'the option or its list could not be measured');
+  }
+  const next = offsetCentering(rect, geometry);
+  if (Math.abs(next - scrollOffsetOf(geometry)) < MOVEMENT_TOLERANCE) {
+    commitScrollEnd(list, geometry, next);
+    return succeeded('the option is already selected');
+  }
+  const moved = applyOffset({ ...target, fiber: list }, geometry, next, true);
+  if (moved.ok) {
+    await SYSTEM_CLOCK.sleep(SCROLL_ANIMATION_MS);
+    commitScrollEnd(list, geometry, next);
+  }
+  return moved;
+}
+
+export async function scrollListTo(target: ResolvedTarget, list: Fiber, offset: number): Promise<HandlerResult> {
+  const geometry = await measureScrollGeometry(list);
+  return geometry === null ? failed('element_not_found', 'the list could not be measured') : applyOffset({ ...target, fiber: list }, geometry, offset, false);
+}
+
+export async function selectWheelOption(target: ResolvedTarget): Promise<HandlerResult> {
+  const container = nearestScrollAncestor(target);
+  if (container === null || target.host === null) {
+    return failed('element_not_interactable', 'the option is not inside a wheel');
+  }
+  return centerInList(target, container, target.host);
+}
+
+export function rowsOf(list: Fiber): ListRow[] {
+  const rows: ListRow[] = [];
+  const visit = (fiber: Fiber): void => {
+    if (isHostComponent(fiber) && fiber.type === TEXT_HOST_TYPE) {
+      const text = collectText(fiber, ROW_TEXT_LIMIT);
+      if (text.length > 0) {
+        rows.push({ host: fiber, text });
+      }
+      return;
+    }
+    childrenOf(fiber).forEach(visit);
+  };
+  visit(list);
+  return rows;
+}
+
+export function matchesRow(text: string, value: string): boolean {
+  const [left, right] = [text.trim().toLowerCase(), value.trim().toLowerCase()];
+  return left === right || (/^\d+$/.test(left) && /^\d+$/.test(right) && Number(left) === Number(right));
+}
+
+// Any list counts when the agent names it, not only a snapping one: this is what reaches a wheel that snaps in JS.
+function listOf(target: ResolvedTarget): Fiber | null {
+  if (target.element.kind === 'scroll') {
+    return target.fiber;
+  }
+  return isInWheel(target) ? nearestScrollAncestor(target) : null;
+}
+
+// A virtualized list mounts its rows shortly after it opens, and a distant row only once scrolling gets near it: wait
+// a moment, then walk the list. A value that isn't there leaves the list where it was.
+export async function selectValue(target: ResolvedTarget, value: string, services: ActionServices): Promise<HandlerResult> {
+  const first = listOf(target);
+  if (first === null) {
+    return failed('element_not_interactable', 'select works on one open list: click this field to open its picker, then select each value on its list (scr-N)');
+  }
+  let current = target;
+  let list = first;
+  const seen = new Set<string>();
+  const findRow = (): ListRow | undefined => {
+    current = services.resolver.resolveElement(target.element) ?? current;
+    list = listOf(current) ?? list;
+    const rows = rowsOf(list);
+    rows.forEach((row) => seen.add(row.text));
+    return rows.find((row) => matchesRow(row.text, value));
+  };
+  let row = findRow();
+  for (let waited = 0; row === undefined && waited < MOUNT_WAIT_MS; waited += MOUNT_POLL_MS) {
+    await SYSTEM_CLOCK.sleep(MOUNT_POLL_MS);
+    row = findRow();
+  }
+  if (row !== undefined) {
+    return centerInList(current, list, row.host);
+  }
+  const geometry = await measureScrollGeometry(list);
+  if (geometry !== null) {
+    const start = scrollOffsetOf(geometry);
+    const step = Math.max(1, viewportExtentOf(geometry) * WALK_STEP_FRACTION);
+    const end = maxScrollOffsetOf(geometry);
+    const stops = [...walk(start, end, step), ...walk(start, 0, step)];
+    for (const stop of stops) {
+      await scrollListTo(current, list, stop);
+      await SYSTEM_CLOCK.sleep(WALK_SETTLE_MS);
+      row = findRow();
+      if (row !== undefined) {
+        return centerInList(current, list, row.host);
+      }
+    }
+    await scrollListTo(current, list, start);
+  }
+  const options = [...seen].slice(0, LISTED_OPTIONS).join(', ');
+  return failed('element_not_found', `"${value}" is not in this list. Options: ${options}`);
+}
+
+export function walk(from: number, to: number, step: number): number[] {
+  const stops: number[] = [];
+  const direction = to >= from ? 1 : -1;
+  for (let offset = from + direction * step; direction > 0 ? offset < to : offset > to; offset += direction * step) {
+    stops.push(offset);
+  }
+  return from === to ? stops : [...stops, to];
+}

package/src/actions/types.ts

@@ -6,6 +6,7 @@ export type AgentAction =
   | { type: 'click' }
   | { type: 'type'; value: string; clear_first: boolean }
   | { type: 'check'; checked: boolean | null }
+  | { type: 'select'; value: string }
   | { type: 'press_key'; key: string }
   | { type: 'scroll'; direction: ScrollDirectionName; amount: number | null }
   | { type: 'scroll_to' }

package/src/backend/app-info.ts

@@ -1,9 +1,11 @@
 import { Platform } from 'react-native';
 
 import { isRecord } from '../fiber/narrow.ts';
+import { SDK_VERSION } from '../version.ts';
 
 export interface AppInfo {
   platform: 'react-native';
+  sdk_version: string;
   os: string;
   os_version: string;
   rn_version: string;
@@ -32,6 +34,7 @@ export function detectNewArchitecture(markers: GlobalMarkers): boolean {
 export function readAppInfo(): AppInfo {
   return {
     platform: 'react-native',
+    sdk_version: SDK_VERSION,
     os: Platform.OS,
     os_version: String(Platform.Version),
     rn_version: readReactNativeVersion(),